纯手工部署k8s高可用集群

hostname IP 组件 备注
master-1 172.17.2.241 ansible,etcd,kube-apiserver,kube-controller-manager,kuber-scheduler
master-2 172.17.2.242 etcd,kube-apiserver,kube-controller-manager,kuber-scheduler
master-3 172.17.2.243 etcd,kube-apiserver,kube-controller-manager,kuber-scheduler
node-1 172.17.2.244 kunelet,kube-proxy,docker,flannel
node-2 172.17.2.245 kunelet,kube-proxy,docker,flannel
node-3 172.17.2.246 kunelet,kube-proxy,docker,flannel

1.1 初始化

centos7_initial_scripts,参考此处对OS进行初始化配置。

1.2 ansible准备

然后参考推送公钥对master-1进行配置,使之能批量管理集群。

yum install ansible -y
sed -i 's/#host_key_checking/host_key_checking/' /etc/ansible/ansible.cfg
cat > /etc/ansible/hosts <<'EOF'
[etcd]
172.17.2.[241:243]
[node]
172.17.2.[244:247]
[all:children]
etcd
node
[all:vars]
ansible_user='root'
ansible_password='123123'
EOF

1.3 ansible批量安装node节点的docker-ce

docker-install.sh
#!/bin/bash
yum remove -y docker \
    docker-client \
    docker-client-latest \
    docker-common \
    docker-latest \
    docker-latest-logrotate \
    docker-logrotate \
    docker-engine
 
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
sed -i 's%download.docker.com%mirrors.ustc.edu.cn/docker-ce%g' /etc/yum.repos.d/docker-ce.repo
yum install -y docker-ce docker-ce-cli containerd.io
systemctl enable docker
systemctl start docker

安装:

ansible node -m script -a '/tmp/docker-install.sh'
# 添加registry镜像
cat >> /tmp/daemon.json <<'EOF'
{
  "registry-mirrors": [
    "https://dockerhub.azk8s.cn",
    "https://reg-mirror.qiniu.com"
  ]
}
EOF
# 分发,查看
ansible node -m copy -a "src=/tmp/daemon.json dest=/etc/docker/daemon.json"
ansible node -m shell -a 'systemctl restart docker'
# 检查
ansible node -m shell -a 'docker info | grep qiniu'
ansible node -m shell -a 'systemctl status docker | grep Active'
ansible node -m shell -a 'docker pull centos:7'

2.1 准备etcd证书

下载cloudflare的证书签名工具:

mkdir /tmp/certs
cd /tmp/certs
 
cat > cfssl.sh <<'EOF'
#!/bin/bash
curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o /usr/local/bin/cfssl
curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o /usr/local/bin/cfssljson
curl -L https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -o /usr/local/bin/cfssl-certinfo
chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson /usr/local/bin/cfssl-certinfo
EOF
 
bash cfssl.sh
which cfssl cfssl-certinfo cfssljson

配置证书:

cat > ca-config.json <<EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "www": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}
EOF
 
cat > ca-csr.json <<EOF
{
    "CN": "etcd CA",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing"
        }
    ]
}
EOF
 
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
 
# 注意修改hosts里的IP
cat > server-csr.json <<EOF
{
    "CN": "etcd",
    "hosts": [
    "172.17.2.241",
    "172.17.2.242",
    "172.17.2.243"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "BeiJing",
            "ST": "BeiJing"
        }
    ]
}
EOF
 
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server

2.2 部署etcd集群

Warning: 请使用etcd 3.3版本部署,不然和flannel组件不兼容!!!

使用etcd官方二进制包部署:

mkdir -p /opt/etcd
cd /opt/etcd
wget https://github.com/etcd-io/etcd/releases/download/v3.3.17/etcd-v3.3.17-linux-amd64.tar.gz
tar zxf etcd-v3.3.17-linux-amd64.tar.gz
mkdir {data,bin,conf,ssl}
mv etcd-v3.3.17-linux-amd64/etcd* bin/
rm -rf etcd-v3.3.17-linux-amd64*
tree .

生成配置文件、服务启动文件:

etcd.sh
#!/bin/bash
 
ETCD_NAME=$1
ETCD_IP=$2
ETCD_CLUSTER=$3
WORK_DIR=/opt/etcd
 
# 生成配置文件
cat > ${WORK_DIR}/conf/etcd.conf <<EOF
#[Member]
ETCD_NAME="${ETCD_NAME}"
ETCD_DATA_DIR="${WORK_DIR}/data"
ETCD_LISTEN_PEER_URLS="https://${ETCD_IP}:2380"
ETCD_LISTEN_CLIENT_URLS="https://${ETCD_IP}:2379"
 
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://${ETCD_IP}:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://${ETCD_IP}:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://${ETCD_IP}:2380,${ETCD_CLUSTER}"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_ENABLE_V2="true"
EOF
 
# 生成systemd文件
cat > /lib/systemd/system/etcd.service <<EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
 
[Service]
Type=notify
EnvironmentFile=${WORK_DIR}/conf/etcd.conf
ExecStart=${WORK_DIR}/bin/etcd \\
--cert-file=${WORK_DIR}/ssl/server.pem \\
--key-file=${WORK_DIR}/ssl/server-key.pem \\
--peer-cert-file=${WORK_DIR}/ssl/server.pem \\
--peer-key-file=${WORK_DIR}/ssl/server-key.pem \\
--trusted-ca-file=${WORK_DIR}/ssl/ca.pem \\
--peer-trusted-ca-file=${WORK_DIR}/ssl/ca.pem
Restart=on-failure
LimitNOFILE=65536
 
[Install]
WantedBy=multi-user.target
EOF
cp /materials/*.pem ssl
sh etcd.sh etcd01 172.17.2.241 etcd02=https://172.17.2.242:2380,etcd03=https://172.17.2.243:2380
cat /opt/etcd/conf/etcd.conf /lib/systemd/system/etcd.service

分发文件到其他两个节点 & 修改/opt/etcd/conf/etcd.conf

scp -r /opt/etcd/ 172.17.2.243:/opt/
scp -r /lib/systemd/system/etcd.service 172.17.2.243:/lib/systemd/system/etcd.service

启动 & 检测集群健康:

systemctl daemon-reload
systemctl start etcd.service
systemctl enable etcd.service
netstat -anltp | grep etcd | grep LISTEN
 
/opt/etcd/bin/etcdctl \
    --ca-file=/opt/etcd/ssl/ca.pem \
    --cert-file=/opt/etcd/ssl/server.pem \
    --key-file=/opt/etcd/ssl/server-key.pem \
    --endpoints="https://172.17.2.241:2379,https://172.17.2.242:2379,https://172.17.2.243:2379" \
    cluster-health
    # 集群成员
    member list

Flannel 也属于是 Overlay Network,覆盖网络的一种,也是将源数据包封装在另一种网络包里面进行路由转发和通信,目前支持 UDP、VXLAN、AWS VPC和GCE 路由等数据转发方式,Flannel 会通过 etcd 存储一个路由表,以实现跨主机通讯。

3.1 写入网段信息到etcd

# 设置
/opt/etcd/bin/etcdctl \
    --ca-file=/opt/etcd/ssl/ca.pem \
    --cert-file=/opt/etcd/ssl/server.pem \
    --key-file=/opt/etcd/ssl/server-key.pem \
    --endpoints="https://172.17.2.241:2379,https://172.17.2.242:2379,https://172.17.2.243:2379" \
    set /kubernetes/network/config '{"Network":"12.13.0.0/16","Backend":{"Type":"vxlan"}}'
    # 查看
    get /kubernetes/network/config

3.2 安装Flannel

# 将下载好的包分发到node节点
wget https://github.com/coreos/flannel/releases/download/v0.11.0/flannel-v0.11.0-linux-amd64.tar.gz
ansible node -m copy -a "src=flannel-v0.11.0-linux-amd64.tar.gz dest=/tmp"
# 创建相关目录
ansible node -m shell -a "mkdir -p /opt/kubernetes/{conf,bin,ssl}"
ansible node -m shell -a "mkdir -p /opt/kubernetes/conf/flanneld"
# 将flannel解压到bin目录
ansible node -m unarchive -a "src=/tmp/flannel-v0.11.0-linux-amd64.tar.gz dest=/opt/kubernetes/bin copy=no"
# 将etcd部署时候生成的证书copy到node节点给k8s使用
ansible node -m copy -a "src=/opt/etcd/ssl dest=/opt/kubernetes/"
ansible node -m shell -a "ls /opt/kubernetes/ssl"

3.3 编写配置文件及启动文件

Note: 在ansible控制端操作

Flanneld配置文件,注意修改etcd集群地址和证书位置:

cat > flanneld.conf <<'EOF'
# etcd url location.  Point this to the server where etcd runs
FLANNEL_ETCD_ENDPOINTS="https://172.17.2.241:2379,https://172.17.2.242:2379,https://172.17.2.243:2379"
# etcd config key.  This is the configuration key that flannel queries for address range assignment
FLANNEL_ETCD_PREFIX="/kubernetes/network"
# Any additional options that you want to pass
FLANNEL_OPTIONS="-etcd-cafile=/opt/kubernetes/ssl/ca.pem -etcd-certfile=/opt/kubernetes/ssl/server.pem -etcd-keyfile=/opt/kubernetes/ssl/server-key.pem"
EOF

Flannel启动文件,注意修改相关文件位置:

cat  > flanneld.service <<'EOF'
[Unit]
Description=Flanneld overlay address etcd agent
After=network-online.target network.target
Before=docker.service
 
[Service]
Type=notify
EnvironmentFile=/opt/kubernetes/conf/flanneld/flanneld.conf
ExecStart=/opt/kubernetes/bin/flanneld --ip-masq \
  -etcd-endpoints=${FLANNEL_ETCD_ENDPOINTS} \
  -etcd-prefix=${FLANNEL_ETCD_PREFIX} \
  $FLANNEL_OPTIONS
ExecStartPost=/opt/kubernetes/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/subnet.env
Restart=on-failure
 
[Install]
WantedBy=multi-user.target
RequiredBy=docker.service
EOF

分发这开两个文件至node节点:

ansible node -m copy -a "src=flanneld.conf dest=/opt/kubernetes/conf/flanneld/"
ansible node -m copy -a "src=flanneld.service dest=/usr/lib/systemd/system/"

启动Flannel:

ansible node -m systemd -a "name=flanneld.service daemon_reload=yes state=started enabled=yes"

3.4 检查

docker host上会生成flannel网络相关的信息:

# 确认一下文件是否生成了
ansible node -m shell -a "cat /run/flannel/subnet.env"
# 还会有一个名为 flannel.1 的网卡
ansible node -m shell -a "ifconfig flannel.1"
# 路由表
ansible node -m shell -a "route"

分别在两个host上创建容器进行测试:

# 创建容器
docker run -dit centos:7 /bin/bash
# 登录容器
docker exec -ti silly_galois /bin/bash
yum install net-tools mtr iproute -y

  • virtualization/k8s/k8s_deploy.txt
  • 最后更改: 2020/08/05 03:45
  • (外部编辑)