No Description

huangguanqin 6a28344734 u 6 days ago
helm 6a28344734 u 6 days ago
upgrade_doc 38c53706f6 first commit 1 week ago
.gitignore 38c53706f6 first commit 1 week ago
LICENSE 38c53706f6 first commit 1 week ago
README.md 38c53706f6 first commit 1 week ago

README.md

Finchat部署说明

部署前期环境准备

安装配置docker,优化内核参数 每台宿主机(Ubuntu16.04)均需执行以下命令安装配置docker,以及优化内核参数,或者下载到本地检查完脚本后安装.

curl -fsSL "https://static.finogeeks.club/deploy/setup_docker.sh" | bash

#!/bin/bash

IMAGES_DOMAIN=https://download.docker.com/linux/static/stable/x86_64
DOCKER_DIR=/var/lib/docker
DOCKER_VERSION=18.09.3

function is_root(){
if [ ${UID} -ne 0 ];then
    echo "Please use root !"
    exit 1
fi
}

function setup_docker(){

    mkdir -p /tmp/docker
    wget ${IMAGES_DOMAIN}/docker-${DOCKER_VERSION}.tgz -O /tmp/docker-${DOCKER_VERSION}.tgz
    tar -zxf /tmp/docker-${DOCKER_VERSION}.tgz -C /tmp/docker/
    chmod +x /tmp/docker/docker/*
    mv -f /tmp/docker/docker/* /usr/bin/
    curl -L https://github.com/docker/compose/releases/download/1.24.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose
    chmod +x /usr/local/bin/docker-compose

}

function setup_docker_service(){

cat > /lib/systemd/system/docker.service << EOF
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target
[Service]
Type=notify
ExecStart=/usr/bin/dockerd  -H unix:///var/run/docker.sock -H tcp://127.0.0.1:2375
ExecReload=/bin/kill -s HUP $MAINPID
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TimeoutStartSec=0
Delegate=yes
KillMode=process
[Install]
WantedBy=multi-user.target
EOF
cat > /lib/systemd/system/docker.socket << EOF
[Unit]
Description=Docker Socket for the API
PartOf=docker.service
[Socket]
ListenStream=/var/run/docker.sock
SocketMode=0660
SocketUser=root
SocketGroup=docker
[Install]
WantedBy=sockets.target
EOF

mkdir -p /etc/docker/

cat > /etc/docker/daemon.json << EOF
{
    "graph": "${DOCKER_DIR}",
    "registry-mirrors": ["https://registry.docker-cn.com/"],
    "insecure-registries": ["docker.finogeeks.club"],
    "log-opts": {
      "max-size": "100m",
      "max-file": "1"
    }
}
EOF


systemctl daemon-reload
systemctl enable docker
systemctl restart docker
}


function sysctl_conf (){
    grep "^net.ipv4.ip_forward" /etc/sysctl.conf && sed -i  's/net.ipv4.ip_forward.*$/net.ipv4.ip_forward = 1/g' /etc/sysctl.conf || echo 'net.ipv4.ip_forward = 1' >> /etc/sysctl.conf
    grep "^net.bridge.bridge-nf-call-iptables" /etc/sysctl.conf && sed -i  's/net.bridge.bridge-nf-call-iptables.*$/net.bridge.bridge-nf-call-iptables = 1/g' /etc/sysctl.conf || echo 'net.bridge.bridge-nf-call-iptables = 1' >> /etc/sysctl.conf
    grep "^vm.max_map_count" /etc/sysctl.conf && sed -i  's/vm.max_map_count.*$/vm.max_map_count = 262144/g' /etc/sysctl.conf || echo 'vm.max_map_count = 262144' >> /etc/sysctl.conf
    grep "^fs.file-max" /etc/sysctl.conf && sed -i  's/fs.file-max.*$/fs.file-max = 8000000/g' /etc/sysctl.conf || echo 'fs.file-max = 8000000' >> /etc/sysctl.conf
    sysctl -p
}

is_root
setup_docker
setup_docker_service
sysctl_conf


在服务对应标签宿主机新建目录并授权 rm -rf /mnt/data/dendrite/ && mkdir -p /mnt/data/dendrite/{kafka,elasticsearch,minio,pg-master,pg-slave,zookeeper,mongo/db,es/{data,logs}} && chmod 777 -R /mnt/data/dendrite/

部署

  • 省去linux初始化,以及rancher安装部署

rancher2.x上部署finochat顺序

  1. global 新增helm库的catalog https://user:password@git.finogeeks.club/helm/swan-poc.git
  2. 新增docker镜像仓库账号 project ->resources -> registries (新增名为registrykey-docker-finogeeks-club的docker.finogeeks.club账号)
  3. 对node打label
regtech-oa-01 
kubectl label nodes --overwrite  regtech-oa-01 minio.env=yes  zookeeper.env=yes kafka.env=yes  mongo.env=yes  

regtech-oa-02 
kubectl label nodes --overwrite  regtech-oa-02 finochat.pg-master=yes   zookeeper.env=yes kafka.env=yes   mongo.env=yes  elasticsearch.env=yes

regtech-oa-03 
kubectl label nodes --overwrite  regtech-oa-03 finochat.pg-slave=yes zookeeper.env=yes kafka.env=yes   mongo.env=yes elasticsearch.env=yes 

regtech-oa-04 
kubectl label nodes --overwrite  regtech-oa-04  prometheus.env=yes  elasticsearch.env=yes publish-center.env=yes pv-static.env=yes
  1. 部署有状态服务,依照以下顺序部署
    • 4.1 在服务对应标签宿主机新建目录并授权
    • rm -rf /mnt/data/dendrite/ && mkdir -p /mnt/data/dendrite/{kafka,minio,pg-master,pg-slave,zookeeper,mongo/db,elasticsearch/{data,logs},es/{data,logs}} && chmod 777 -R /mnt/data/dendrite/
    • 4.2 部署zookeeper
    • 4.3 部署kafka
    • 4.4 部署storage -> mongo(namespace选择storage)
    • 4.5 部署codis(测试环境用主从,生产环境用redis-cluster)
    • 4.6 部署业务服务-chat->platform->auth->adapter->gateway->swan->mobile-web->
    • 4.7 导入账号,规则

注意事项

  • 不同project的resources是不通用的,新project需要新建resources
  • label 用yes , true 容易被判断成bool型 会出错,而且k8s配置文件里变量值加上引号

  • 删除并重建codis后,zookeeper会有历史缓存数据,导致冲突

    # 清除历史数据
    kubectl -n zookeeper exec -it zk-0 -- /opt/zookeeper-3.4.10/bin/zkCli.sh -server zk-0:2181 rmr /codis3/
    
  • 部署完codis,第一次需要初始化

    # 初始化codis
    kubectl -n redis-cluster  exec -it codis-server-0 -- codis-admin  --dashboard=codis-dashboard:18080 --rebalance --confirm
    # 检测codis状态是否正常
    kubectl -n redis-cluster  exec -it codis-dashboard-0 -- redis-cli -h codis-proxy -p 19000 PING
    
find . -name "*.yaml" -o -name "*.yml"|xargs -i  sed -i  's#docker.finogeeks.club/#docker.finogeeks.club:5000/#g' {}
 docker images |grep docker.finogeeks|awk -F'[\/\t ]+' '{print "docker tag " $1"/"$2"/"$3":"$4"   " $1":5000/"$2"/"$3":"$4   }' |bash
docker images |grep "club:5000"|awk -F"[\t ]+"  '{print "docker push  " $1":"$2}' |bash

grep "docker.finogeeks.club:5000" -R *|awk '{print $NF}'|sort -rn|tr "'" " " | tr '"' " "|sort -rn |uniq |awk  -F "[/|:]" '{print "docker save "$0 "|gzip -c > /mnt/data/release_images/"$4"."$5".tar.gz"}' |bash


需要修改linux内核参数

    grep "^net.ipv4.ip_forward" /etc/sysctl.conf && sed -i  's/net.ipv4.ip_forward.*$/net.ipv4.ip_forward = 1/g' /etc/sysctl.conf || echo 'net.ipv4.ip_forward = 1' >> /etc/sysctl.conf
    grep "^net.bridge.bridge-nf-call-iptables" /etc/sysctl.conf && sed -i  's/net.bridge.bridge-nf-call-iptables.*$/net.bridge.bridge-nf-call-iptables = 1/g' /etc/sysctl.conf || echo 'net.bridge.bridge-nf-call-iptables = 1' >> /etc/sysctl.conf
    grep "^vm.max_map_count" /etc/sysctl.conf && sed -i  's/vm.max_map_count.*$/vm.max_map_count = 262144/g' /etc/sysctl.conf || echo 'vm.max_map_count = 262144' >> /etc/sysctl.conf
    grep "^fs.file-max" /etc/sysctl.conf && sed -i  's/fs.file-max.*$/fs.file-max = 8000000/g' /etc/sysctl.conf || echo 'fs.file-max = 8000000' >> /etc/sysctl.conf
    sysctl -p


重复安装rancher/kubernetes需要删除缓存

df -h|grep kubelet |awk -F % '{print $2}'|xargs umount 
rm /var/lib/kubelet/ -rf
rm /etc/kubernetes/ -rf
rm /var/lib/rancher/ -rf
rm /var/lib/etcd/ -rf
rm /var/lib/cni/ -rf
rm /etc/cni/ -rf 
rm /opt/cni/ -rf 
iptables -F && iptables -t nat -F
ip link del flannel.1
ip link del cni0
docker ps -a|awk '{print $1}'|xargs docker rm -f
docker volume ls|awk '{print $2}'|xargs docker volume rm
rm -rf /var/run/calico
rm -rf /var/lib/calico/