摘要

原创

原环境是购买的国内一家docker云解决方案公司的一套私有环境，搭建在公司的阿里云服务器上。

整套环境包括registry、web管理界面、api接口等；主要使用单机的形式运行，新版本功能较为完善，但需要重新购买，因此决定撤掉这套环境。

由于之前是将容器当做虚拟机的形式使用的，因此迁移过程只迁移以下部分：

registry中镜像
dockerfile
所有容器（保留原ip）

环境说明

原环境

系统：ubuntu14.04
一台master（web,api,registry,etcd）
四台node (30.7/24 ~ 30.10/24)

迁移环境

系统：ubuntu14.04
一台master（web,api,registry,etcd）
四台node (30.7/24 ~ 30.10/24)，每台机器配置三个docker网桥
搭建etcd集群

环境搭建

安装docker

172.16.7.1 master
- docker0 172.30.1.1/24
172.16.7.2 node01
- docker0 172.30.2.1/24
- docker1 172.30.7.1/24
- docker2 172.30.9.1/24
172.16.7.3 node02
- docker0 172.30.3.1/24
- docker1 172.30.8.1/24
- docker2 172.30.10.1/24

apt update && apt upgrade
apt install -y curl
curl https://get.docker.com/ | sh

vim /etc/default/docker
DOCKER_OPTS="-g /data/docker --insecure-registry opreg.chinawayltd.com -b=docker0 -H tcp://0.0.0.0:4273 -H unix:///var/run/docker.sock"
	
/etc/init.d/docker restart
docker info

配置etcd集群

参考 etcd-cluster 集群部署

master配置

安装配置registry

参考 docker registry v2 私有仓库搭建

网桥配置

$ cat ip0.sh
ip link set dev docker0 down
brctl delbr docker0
brctl addbr docker0
ip addr add 172.30.1.1/24 dev docker0
ip link set dev docker0 up

sh ip0.sh
/etc/init.d/docker restart

node配置

网桥配置

$ cat ip0.sh
ip link set dev docker0 down
brctl delbr docker0
brctl addbr docker0
ip addr add 172.30.2.1/24 dev docker0
ip link set dev docker0 up

$ cat br.sh
docker network rm bridge1
docker network rm bridge2
docker network create --subnet 172.30.7.1/24 \
--opt com.docker.network.bridge.name=docker1 \
--opt com.docker.network.driver.mtu=1500 \
--opt com.docker.network.bridge.enable_icc=true \
--opt com.docker.network.bridge.enable_ip_masquerade=true \
--opt com.docker.network.bridge.host_binding_ipv4=0.0.0.0 \
bridge1
docker network create --subnet 172.30.9.1/24 \
--opt com.docker.network.bridge.name=docker2 \
--opt com.docker.network.driver.mtu=1500 \
--opt com.docker.network.bridge.enable_icc=true \
--opt com.docker.network.bridge.enable_ip_masquerade=true \
--opt com.docker.network.bridge.host_binding_ipv4=0.0.0.0 \
bridge2

sh ip0.sh
sh br.sh
/etc/init.d/docker restart
ip a

容器ip配置脚本

cat set_ip.sh
#!/usr/bin/env bash
# filename: set_ip.sh

if [ `id -u` -ne 0 ];then
    echo '必须使用root权限'
    exit
fi

container_name=$1
bind_ip=$2
dobr=$3

if [ -z $dobr ]; then
    dobr="docker0"
fi

container_id=`docker inspect -f '{{.Id}}' $container_name 2> /dev/null`
if [ ! $container_id ];then
    echo "容器不存在"
    exit 2
fi
bind_ip=`echo $bind_ip | egrep '^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$'`
if [ ! $bind_ip ];then
    echo "IP地址格式不正确"
    exit 3
fi

if ifconfig $dobr &> /dev/null; then
    echo "网桥不存在"
    exit 4
fi

container_minid=`echo $container_id | cut -c 1-10`
container_netmask=`ip addr show $dobr | grep "inet\b" | awk '{print $2}' | cut -d / -f2`
container_gw=`ip addr show $dobr | grep "inet\b" | awk '{print $2}' | cut -d / -f1`

bridge_name="veth_$container_minid"
container_ip=$bind_ip/$container_netmask
pid=`docker inspect -f '{{.State.Pid}}' $container_name 2> /dev/null`
if [ ! $pid ];then
    echo "获取容器$container_name的id失败"
    exit 5
fi

if [ ! -d /var/run/netns ];then
    mkdir -p /var/run/netns
fi

ln -sf /proc/$pid/ns/net /var/run/netns/$pid

ip link add $bridge_name type veth peer name X
brctl addif $dobr $bridge_name
ip link set $bridge_name up
ip link set X netns $pid
ip netns exec $pid ip link set dev X name eth0
ip netns exec $pid ip link set eth0 up
ip netns exec $pid ip addr add $container_ip dev eth0
ip netns exec $pid ip route add default via $container_gw

容器启动脚本

cat create.sh
docker run -d --name cont_name-01-9.82 --cpuset-cpus 0-3 --memory 8G --net=none img_name

cat sip.sh
sh /data/bin/set_ip.sh cont_name-01-9.82 172.30.9.82 docker2

迁移步骤

将容器打包成镜像
将打包好的镜像导出成tar包
将导出的tar包scp到导入端进行导入
启动容器，配置ip
修改路由

docker commit -p cont_name img_name
docker save -o /data/img_name.tar img_name
docker load -i img_name.tar

sh create.sh
sh sip.sh

修复一个问题

偶然发现同一个宿主机上的两个网段机器不通，发现是docker自己加了iptables，应该是因为资源隔离的问题。

# Generated by iptables-save v1.4.21 on Fri Feb 24 15:32:26 2017
*nat
:PREROUTING ACCEPT [129399744:173285947397]
:INPUT ACCEPT [30953:1451912]
:OUTPUT ACCEPT [98806:7478654]
:POSTROUTING ACCEPT [118180652:172566860675]
:CATTLE_NAT_POSTROUTING - [0:0]
:CATTLE_OUTPUT - [0:0]
:CATTLE_POSTROUTING - [0:0]
:CATTLE_PREROUTING - [0:0]
:DOCKER - [0:0]
-A PREROUTING -m addrtype --dst-type LOCAL -j CATTLE_PREROUTING
-A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER
-A OUTPUT -m addrtype --dst-type LOCAL -j CATTLE_OUTPUT
-A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER
-A POSTROUTING -j CATTLE_NAT_POSTROUTING
-A POSTROUTING -s 172.30.3.0/24 ! -o docker0 -j MASQUERADE
-A POSTROUTING -s 172.30.10.0/24 ! -o docker2 -j MASQUERADE
-A POSTROUTING -s 172.30.8.0/24 ! -o docker1 -j MASQUERADE
-A CATTLE_NAT_POSTROUTING -s 10.42.0.0/16 ! -o docker0 -p tcp -j MASQUERADE --to-ports 1024-65535
-A CATTLE_NAT_POSTROUTING -s 10.42.0.0/16 ! -o docker0 -p udp -j MASQUERADE --to-ports 1024-65535
-A CATTLE_NAT_POSTROUTING -s 10.42.0.0/16 ! -o docker0 -j MASQUERADE
-A CATTLE_NAT_POSTROUTING -o docker0 -m addrtype --src-type LOCAL --dst-type UNICAST -j MASQUERADE
-A DOCKER -i docker0 -j RETURN
-A DOCKER -i docker2 -j RETURN
-A DOCKER -i docker1 -j RETURN
COMMIT
# Completed on Fri Feb 24 15:32:26 2017
# Generated by iptables-save v1.4.21 on Fri Feb 24 15:32:26 2017
*filter
:INPUT ACCEPT [218437:116429305]
:FORWARD ACCEPT [417068536:662927571688]
:OUTPUT ACCEPT [14372495:1274827596]
:CATTLE_FORWARD - [0:0]
:DOCKER - [0:0]
:DOCKER-ISOLATION - [0:0]
-A FORWARD -j CATTLE_FORWARD
-A FORWARD -j DOCKER-ISOLATION
-A FORWARD -o docker0 -j DOCKER
-A FORWARD -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -i docker0 ! -o docker0 -j ACCEPT
-A FORWARD -i docker0 -o docker0 -j ACCEPT
-A FORWARD -o docker2 -j DOCKER
-A FORWARD -o docker2 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -i docker2 ! -o docker2 -j ACCEPT
-A FORWARD -i docker2 -o docker2 -j ACCEPT
-A FORWARD -o docker1 -j DOCKER
-A FORWARD -o docker1 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -i docker1 ! -o docker1 -j ACCEPT
-A FORWARD -i docker1 -o docker1 -j ACCEPT
-A CATTLE_FORWARD -m mark --mark 0x1068 -j ACCEPT

# 删掉下面几行
-A DOCKER-ISOLATION -i docker2 -o docker0 -j DROP
-A DOCKER-ISOLATION -i docker0 -o docker2 -j DROP
-A DOCKER-ISOLATION -i docker1 -o docker0 -j DROP
-A DOCKER-ISOLATION -i docker0 -o docker1 -j DROP
-A DOCKER-ISOLATION -i docker1 -o docker2 -j DROP
-A DOCKER-ISOLATION -i docker2 -o docker1 -j DROP

-A DOCKER-ISOLATION -j RETURN
COMMIT
# Completed on Fri Feb 24 15:32:26 2017