01-docker安装

发表于 | 分类于 | 评论数： | 阅读次数：
本文字数： 17k | 阅读时长 ≈ 16 分钟

摘要

本文内容转自网络，个人学习记录使用，请勿传播

docker部署

https://www.cnblogs.com/sparkdev/p/9129334.html

关闭防火墙和selinux

1
2
3
systemctl disable --now firewalld NetworkManager
setenforce 0
sed -ri '/^[^#]*SELINUX=/s#=.+$#=disabled#' /etc/selinux/config

关闭dnsmasq

linux 系统开启了 dnsmasq 后(如 GUI 环境)，将系统 DNS Server 设置为 127.0.0.1，这会导致 docker 容器无法解析域名，需要关闭它

1
systemctl disable --now dnsmasq

关闭swap

1
2
3
swapoff -a && sysctl -w vm.swappiness=0

sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab

升级系统软件包

1
2
3
4
5
yum install epel-release -y
yum install wget git jq psmisc socat pstree tree -y
yum install lvm2 git -y
yum install openssl-devel openssl -y
#yum update -y --exclude=kernel*

开启ipvs以及overlay等

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
lsmod | grep over
# 临时开启
modprobe overlay
# 所有机器安装ipvs(ipvs性能甩iptables几条街并且排错更直观)
yum install ipvsadm ipset sysstat conntrack libseccomp -y
# ubuntu
# sudo apt-get install -y wget git conntrack ipvsadm ipset jq sysstat curl iptables libseccomp

:> /etc/modules-load.d/ipvs.conf
module=(
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
br_netfilter
overlay
 )
for kernel_module in ${module[@]};do
 /sbin/modinfo -F filename $kernel_module |& grep -qv ERROR && echo $kernel_module >> /etc/modules-load.d/ipvs.conf || :
done

#for kernel_module in ${module[@]};do
#  /sbin/modinfo -F filename $kernel_module |& grep -qv ERROR && modprobe $kernel_module || :
#done
systemctl enable --now systemd-modules-load.service

上面如果systemctl enable命令报错可以systemctl status -l systemd-modules-load.service看看哪个内核模块加载不了,在/etc/modules-load.d/ipvs.conf里注释掉它再enable试试

升级内核

docker在低版本内核下（包括4.15内核），会存在以下bug

1
kernel:unregister_netdevice: waiting for lo to become free. Usage count = 1

所以为了安全起见，需要升级内核

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
# 内核依赖perl
[ ! -f /usr/bin/perl ] && yum install perl -y
yum install gcc gcc-c++ -y
yum install bison flex -y
yum install libelf-dev libelf-devel  elfutils-libelf-devel -y

# overlay2需要使用4.0以上版本的内核，如果使用的是RHEL或CentOS，需要3.10.0-514以上版本的内核

# 导入elrepo源
rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm

## 查看可用内核版本
yum --disablerepo="*" --enablerepo="elrepo-kernel" list available --showduplicates

## 安装最新内核

yum --disablerepo="*" --enablerepo="elrepo-kernel" list available --showduplicates | grep -Po '^kernel-ml.x86_64\s+\K\S+(?=.el7)'
yum --disablerepo="*" --enablerepo=elrepo-kernel install -y kernel-ml{,-devel}

# 自选安装内核版本

export Kernel_Version=4.18.9-1
wget http://mirror.rc.usf.edu/compute_lock/elrepo/kernel/el7/x86_64/RPMS/kernel-ml{,-devel}-${Kernel_Version}.el7.elrepo.x86_64.rpm
yum localinstall -y kernel-ml*

## centos6
# export k=4.18.20-1
# wget http://mirror.rc.usf.edu/compute_lock/elrepo/kernel/el6/x86_64/RPMS/kernel-ml{,-devel}-${k}.el6.elrepo.x86_64.rpm
# yum localinstall -y kernel-ml*

# 编译安装内核

export KERNEL_VER="4.19.28"
wget https://mirrors.edge.kernel.org/pub/linux/kernel/v4.x/linux-${KERNEL_VER}.tar.gz
wget https://mirrors.edge.kernel.org/pub/linux/kernel/v4.x/linux-4.19.28.tar.gz

make mrproper
make clean              #这两条用于清除之前编译生成的.ko和.config
make oldconfig          #使用当前的内核配置
make                    #编译，半小时以上是正常的
make modules_install    #把编译出的内核模块复制到/lib/modules/${KERNEL_VERSION}
make install

# 查看内核顺序
awk -F\' '$1=="menuentry " {print $2}' /etc/grub2.cfg

# 修改默认加载内核位置
grub2-set-default 0 && grub2-mkconfig -o /etc/grub2.cfg
## 等同 vim /etc/default/grub GRUB_DEFAULT=saved改为GRUB_0=saved
## grub2-mkconfig -o /boot/grub2/grub.cfg

## 查看确认下是否启动默认内核指向上面安装的内核
grubby --default-kernel

## docker官方的内核检查脚本建议(RHEL7/CentOS7: User namespaces disabled; add ‘user_namespace.enable=1’ to boot command line),使用下面命令开启
grubby --args="user_namespace.enable=1" --update-kernel="$(grubby --default-kernel)"

echo -e 'export PATH="/home/docker/docker_isme/bin:$PATH"' >> /etc/profile
source /etc/profile

创建docker用户

1
2
3
4
5
6
7
8
9
10
11
groupadd -g 1001 docker
usermod -G docker work
useradd -g docker -u 1001 docker

dockerd运行用户： root
docker使用用户：work、docker？

systemd管理dockerd ？

需要优化，固定一些内核参数
需要优化和tcp，iptables有关的参数

内核参数优化

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
cat <<EOF > /etc/sysctl.d/docker.conf
# https://github.com/moby/moby/issues/31208 
# ipvsadm -l --timout
# 修复ipvs模式下长连接timeout问题 小于900即可
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_keepalive_probes = 10

# 禁用ipv6
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
net.ipv6.conf.lo.disable_ipv6 = 1

net.ipv4.ip_forward = 1
net.ipv4.tcp_max_tw_buckets = 5000
net.ipv4.tcp_timestamps                            = 1
net.ipv4.tcp_fin_timeout                           = 10
net.ipv4.tcp_tw_reuse                              = 1
net.ipv4.tcp_tw_recycle                            = 0
net.ipv4.ip_local_port_range                       = 1024 65535
net.netfilter.nf_conntrack_max                     = 1048576
net.netfilter.nf_conntrack_tcp_timeout_close       = 1
net.netfilter.nf_conntrack_tcp_timeout_close_wait  = 10
net.netfilter.nf_conntrack_tcp_timeout_time_wait   = 30
net.ipv4.neigh.default.gc_stale_time = 120
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.default.rp_filter = 0
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_announce = 2
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 1024
net.ipv4.tcp_synack_retries = 2
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.netfilter.nf_conntrack_max = 2310720
fs.inotify.max_user_watches=89100
fs.may_detach_mounts = 1
fs.file-max = 52706963
fs.nr_open = 52706963
net.bridge.bridge-nf-call-arptables = 1
vm.swappiness = 0
vm.overcommit_memory=1
vm.panic_on_oom=0
EOF


sysctl --system

参数备用

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
net.ipv4.ip_forward                                = 1
net.ipv4.tcp_timestamps                            = 1
net.core.somaxconn                                 = 4096
net.core.rmem_default                              = 262144
net.core.wmem_default                              = 262144
net.core.rmem_max                                  = 16777216
net.core.wmem_max                                  = 16777216
net.ipv4.tcp_rmem                                  = 4096 4096 16777216
net.ipv4.tcp_wmem                                  = 4096 4096 16777216
net.ipv4.tcp_mem                                   = 786432 2097152 3145728
net.ipv4.tcp_max_syn_backlog                       = 16384
net.core.netdev_max_backlog                        = 20000
net.ipv4.tcp_fin_timeout                           = 10
net.ipv4.tcp_tw_reuse                              = 1
net.ipv4.tcp_tw_recycle                            = 0
net.ipv4.tcp_max_tw_buckets                        = 180000
net.ipv4.ip_local_port_range                       = 1024 65535
vm.swappiness                                      = 0
vm.dirty_ratio                                     = 20
vm.dirty_background_ratio                          = 10
vm.overcommit_memory                               = 1
net.netfilter.nf_conntrack_max                     = 1048576
net.netfilter.nf_conntrack_tcp_timeout_close       = 1
net.netfilter.nf_conntrack_tcp_timeout_close_wait  = 10
net.netfilter.nf_conntrack_tcp_timeout_time_wait   = 30

cat <<EOF > /etc/sysctl.d/k8s.conf
# https://github.com/moby/moby/issues/31208 
# ipvsadm -l --timout
# 修复ipvs模式下长连接timeout问题 小于900即可
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_keepalive_probes = 10

# 禁用ipv6
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
net.ipv6.conf.lo.disable_ipv6 = 1

net.ipv4.neigh.default.gc_stale_time = 120
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.default.rp_filter = 0
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_announce = 2
net.ipv4.ip_forward = 1
net.ipv4.tcp_max_tw_buckets = 5000
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 1024
net.ipv4.tcp_synack_retries = 2
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.netfilter.nf_conntrack_max = 2310720
fs.inotify.max_user_watches=89100
fs.may_detach_mounts = 1
fs.file-max = 52706963
fs.nr_open = 52706963
net.bridge.bridge-nf-call-arptables = 1
vm.swappiness = 0
vm.overcommit_memory=1
vm.panic_on_oom=0

net.netfilter.nf_conntrack_tcp_timeout_close = 10
net.netfilter.nf_conntrack_tcp_timeout_close_wait = 60
net.netfilter.nf_conntrack_tcp_timeout_established = 432000
net.netfilter.nf_conntrack_tcp_timeout_fin_wait = 120
net.netfilter.nf_conntrack_tcp_timeout_last_ack = 30
net.netfilter.nf_conntrack_tcp_timeout_max_retrans = 300
net.netfilter.nf_conntrack_tcp_timeout_syn_recv = 60
net.netfilter.nf_conntrack_tcp_timeout_syn_sent = 120
net.netfilter.nf_conntrack_tcp_timeout_time_wait = 120
net.netfilter.nf_conntrack_tcp_timeout_unacknowledged = 300
EOF

环境检查

1
2
curl https://raw.githubusercontent.com/docker/docker/master/contrib/check-config.sh > check-config.sh
bash ./check-config.sh

版本

1
2
3
4
5
6
7
8
9
10
11
export VERSION="18.06.3-ce"
export WORKDIR="/home/work"
export DOCKERDIR="${WORKDIR}/docker_isme"
export COMPOSE_VERSION="1.23.2"

mkdir -p ${DOCKERDIR}/{bin,conf,logs,sbin,data}

wget https://download.docker.com/linux/static/stable/x86_64/docker-${VERSION}.tgz -O ${DOCKERDIR}/docker-${VERSION}.tgz

curl -L https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` -o ${DOCKERDIR}/bin/docker-compose
chmod +x ${DOCKERDIR}/bin/docker-compose

升级runc

由于runc近期爆发严重漏洞，需要进行升级，建议下载docker18.09以上升级包，替换当前版本的runc即可。

二进制启动文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
#!/bin/sh
# author: wangping01_isme
# 用于管理dockerd_isme

#debug
#set -x
#check unbound variables
#set -u
# Scripts PATH enviroment
BASEDIR=$( dirname $(cd $(dirname $0); pwd))
export PATH="/usr/bin:/bin:/usr/sbin:/sbin:${BASEDIR}/bin"
# disable core dump
ulimit -c 0
# return value
RETVAL=0

# node_exporter general config
PROG="dockerd"
USER="root"
LOGLEVEL="info"
#BASEDIR=${SYMLINK:-/usr/sbin/node_exporter}
DAEMON="${BASEDIR}/bin/${PROG}"
CONFIG="${BASEDIR}/conf/daemon.json"
LOGFILE="${BASEDIR}/logs/${PROG}.log"
# nginx start timeout milliscond
STARTTIME=10000
# nginx stop timeout milliscond
STOPTIME=10000

color_msg(){
    local COLOR=$1
    local MSG=$2
    OFFSET="\033[80G"
    NORMAL="\033[0m"
    case $COLOR in
        red)
            COLOR="\033[1;40;31m"
            ;;
        green)
            COLOR="\033[1;40;32m"
            ;;
        yellow)
            COLOR="\033[1;40;33m"
            ;;
        *)
            COLOR="\033[0m"
            ;;
    esac
    echo -en "$OFFSET [$COLOR $MSG $NORMAL"
    echo     "]"
}

start() {
    echo -n "Starting $PROG : "
    PROC_PID=$(pgrep -P 1 -u $USER ^$PROG)
    if [ -n "$PROC_PID" ]; then
        echo -n "is already running."
        color_msg yellow WARNING
    else
        nohup $DAEMON  --config-file ${CONFIG} &> ${LOGFILE} &
        if [  $? -eq 0 ]; then
            color_msg green SUCCESS
        else
            color_msg red FAILED && exit 1
        fi
    fi
}

stop() {
    echo -n "Stopping $PROG : "
    PROC_PID=$(pgrep -P 1 -u $USER ^$PROG)
    if [ -z "$PROC_PID" ]; then
        echo -n "is not running."
        color_msg yellow WARNING
    else
        kill -TERM ${PROC_PID} >/dev/null 2>&1
	#/bin/kill -s HUP $MAINPID
        while [ "$STOPTIME" -gt 0 ]; do
            kill -0 ${PROC_PID} >/dev/null 2>&1 || break
            STOPTIME=$(($STOPTIME-1))
            echo -n "." && sleep 0.001s
        done
        if [ "$STOPTIME" -le 0 ]; then
            color_msg red TIMEOUT && exit 1
        else
            color_msg green SUCCESS
        fi
	fi
}

restart() {
        echo -n "Restart $PROG : "
        echo
        echo -en "\t" && stop
        echo -en "\t" && start
}

reload() {
    echo -n "Reloading $PROG : "
	PROC_PID=$(pgrep -P 1 -u $USER ^$PROG)
	if [ -n "$PROC_PID" ]; then
	    kill -HUP ${PROC_PID} >/dev/null 2>&1
        if [  $? -eq 0 ]; then
            color_msg green SUCCESS
        else
            color_msg red FAILED && exit 1
        fi
	else
        echo -n "is not running."
        color_msg yellow WARNING
    fi
}

status() {
    PROC_PID=$(pgrep -P 1 -u $USER ^$PROG)
    if [ -z "$PROC_PID" ];then
        echo "$PROG is stopped"
        exit 3
    else
        echo "$PROG (pid $PROC_PID) is running..."
        exit 0
    fi
}

init_config() {
    if [ -f "${CONFIG}" ];then
        echo "${CONFIG} already exists!"
    else
        data_root="${BASEDIR}/data/container"
        exec_root="${BASEDIR}/data/bootstrap"
        pidfile="${BASEDIR}/data/docker.pid"
        docker_init="${BASEDIR}/bin/docker-init"
cat > ${CONFIG} << E0F
{
    "data-root": "${data_root}",
    "exec-opts": [],
    "exec-root": "${exec_root}",
    "experimental": false,
    "storage-driver": "overlay2",
    "storage-opts": [
        "overlay2.override_kernel_check=true"
    ],
    "labels": [],
    "live-restore": true,
    "log-driver": "json-file",
    "log-opts": {
        "labels": "io.isme.log",
        "max-size": "1g",
        "max-file": "100"
    },
    "pidfile": "${pidfile}",
    "max-concurrent-downloads": 3,
    "max-concurrent-uploads": 3,
    "debug": false,
    "hosts": [
        "unix:///var/run/docker.sock",
        "tcp://127.0.0.1:2376"
    ],
    "log-level": "info",
    "default-ulimits": {
        "nofile": {
            "Name": "nofile",
            "Hard": 10240,
            "Soft": 10240
        }
    },
    "init": true,
    "init-path": "${docker_init}",
    "bip": "192.168.1.1/16",
    "registry-mirrors": [
        "https://registry.docker-cn.com"
    ],
    "insecure-registries": [
        "https://r.isme.pub"
    ]
}
E0F
    fi
}

case C"$1" in
    Cinit)
        init_config
        ;;
    Cstart)
        init_config
        start
        ;;
    Cstop)
        stop
        ;;
    Creload)
        reload
        ;;
    Crestart)
        restart
        ;;
    Cstatus)
        status
        ;;
    *)
        echo $"Usage: $0 {start|stop|restart|reload|status|init_config}"
        exit 3
        ;;
esac

配置文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
cat daemon.json
{
    "data-root": "/home/docker/docker_isme/data/container",
    "exec-opts": [],
    "exec-root": "/home/docker/docker_isme/data/bootstrap",
    "experimental": false,
    "storage-driver": "overlay2",
    "storage-opts": [
        "overlay2.override_kernel_check=true"
    ],
    "labels": [],
    "live-restore": true,
    "log-driver": "json-file",
    "log-opts": {
        "labels": "io.isme.log",
        "max-size": "1g",
        "max-file": "100"
    },
    "pidfile": "/home/docker/docker_isme/data/docker.pid",
    "max-concurrent-downloads": 3,
    "max-concurrent-uploads": 3,
    "debug": false,
    "hosts": [
        "unix:///var/run/docker.sock",
        "tcp://127.0.0.1:2376"
    ],
    "log-level": "info",
    "default-ulimits": {
        "nofile": {
            "Name": "nofile",
            "Hard": 10240,
            "Soft": 10240
        }
    },
    "init": true,
    "init-path": "/home/docker/docker_isme/bin/docker-init",
    "bip": "192.168.1.1/16",
    "registry-mirrors": [
        "https://registry.docker-cn.com"
    ],
    "insecure-registries": [
        "https://r.isme.pub"
    ]
}

systemd启动文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
cat > /usr/lib/systemd/system/docker.service <<"EOF"
[Unit]
Description=Docker Application Container Engine
Documentation=http://docs.docker.io
[Service]
Environment="PATH=/usr/local/bin:/bin:/sbin:/usr/bin:/usr/sbin"
EnvironmentFile=-/run/flannel/docker
ExecStart=/usr/local/bin/dockerd --log-level=error $DOCKER_NETWORK_OPTIONS
ExecReload=/bin/kill -s HUP $MAINPID
Restart=on-failure
RestartSec=5
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
Delegate=yes
KillMode=process

[Install]
WantedBy=multi-user.target
EOF
1
nohup dockerd --config-file conf/daemon.json &> logs/dockerd.log &

开启docker命令补全

1
2
3
# yum安装，二进制安装的命令补全待有空补充
yum install -y  bash-completion && cp /usr/share/bash-completion/completions/docker /etc/bash_completion.d/
systemctl enable --now docker

dockerd 守护进程