Linux: 容器编排Mesos

#1. 拉取docker registry
docker pull registry:2.5.0
#2. 安装docker-compose
curl -L "https://github.com/docker/compose/releases/download/1.23.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
chmod +x /usr/local/bin/docker-compose
#3. 通过distribution 来安装registry
git clone https://github.com/docker/distribution.git
cd distribution/
git checkout release/2.0
cd contrib/compose/nginx/

NAME=my.registry.cici.me
COUNTRY=CN
PROVINCE=BEIJING
CITY=BEIJING
ORGANIZATION=$NAME
GROUP=$NAME
HOST=$NAME
EMAIL[email protected]
SUBJ="/C=$COUNTRY/ST=$PROVINCE/L=$CITY/O=$ORGANIZATION/OU=$GROUP/CN=$HOST/emailAddress=$EMAIL"

openssl genrsa -out $NAME.pem 2048
openssl req -new -sha256 -key $NAME.pem -out $NAME.csr -subj "$SUBJ"
openssl x509 -req -days 3650 -in $NAME.csr -signkey $NAME.pem -out $NAME.crt

#4. 安装httpd-tools 生成密码
yum install httpd-tools -y
htpasswd -c .htpasswd wowotuanops
registrywowotuan
#5. 修改dockerfile
删除COPY docker-registry.conf /etc/nginx/docker-registry.conf
echo "COPY .htpasswd /etc/nginx/.htpasswd" >>Dockerfile
echo "COPY $NAME.pem /etc/nginx/$NAME.pem" >>Dockerfile
echo "COPY $NAME.crt /etc/nginx/$NAME.crt" >>Dockerfile
#6. 编辑registry.conf文件
#注意：mesos集群加hosts解析
cat >registry.conf<<EOF
# Docker registry proxy for api versions 1 and 2

upstream docker-registry-v2 {
  ip_hash;
  server registryv2:5000;
}

upstream docker-registry-api {
  ip_hash;
  server registryapi:5000;
}

# No client auth or TLS
server {
  listen 5000;
  server_name $NAME.me;

  # disable any limits to avoid HTTP 413 for large image uploads
  client_max_body_size 0;

  # required to avoid HTTP 411: see Issue #1486 (https://github.com/docker/docker/issues/1486)
  chunked_transfer_encoding on;

  # for ssl
  ssl on;
  ssl_certificate /etc/nginx/$NAME.crt;
  ssl_certificate_key /etc/nginx/$NAME.pem;

  location /v2/ {
    # Do not allow connections from docker 1.5 and earlier
    # docker pre-1.6.0 did not properly set the user agent on ping, catch "Go *" user agents
    auth_basic "registry v2 realm";
    auth_basic_user_file /etc/nginx/.htpasswd;
    add_header 'Docker-Distribution-Api-Version' 'registry/2.0' always;

    include               docker-registry-v2.conf;
    }
}
server {
  listen 5001;
  server_name $NAME.me;

  # disable any limits to avoid HTTP 413 for large image uploads
  client_max_body_size 0;

  # required to avoid HTTP 411: see Issue #1486 (https://github.com/docker/docker/issues/1486)
  chunked_transfer_encoding on;

  location /search/imagesinfo {
proxy_set_header  Host              $http_host;   # required for docker client's sake
proxy_set_header  X-Real-IP         $remote_addr; # pass on real client's IP
proxy_set_header  X-Forwarded-For   $proxy_add_x_forwarded_for;
proxy_set_header  X-Forwarded-Proto $scheme;
proxy_read_timeout                  900;
    proxy_pass  http://docker-registry-api;
  }

  location /repositories {
proxy_set_header  Host              $http_host;   # required for docker client's sake
proxy_set_header  X-Real-IP         $remote_addr; # pass on real client's IP
proxy_set_header  X-Forwarded-For   $proxy_add_x_forwarded_for;
proxy_set_header  X-Forwarded-Proto $scheme;
proxy_read_timeout                  900;
    proxy_pass http://docker-registry-api;
  }
}
EOF
#7. 修改docker-compose.yml
cd ..
cat >docker-compose.yml <<\EOF
nginx:
  build: "nginx"
  ports:
    - "5000:5000"
    - "443:5000"
    - "80:5001"
  links:
    - registryv2:registryv2
    - registryapi:registryapi
registryv2:
  image: myegistry.55tuan.me:443/registry:2.5.0
  ports:
    - "5000"
  volumes:
    - /data/registryv2/registry:/etc/docker/registry
    - /data/registryv2/data:/var/lib/registry
registryapi:
  image: myregistry.55tuan.me:443/regisitryweb:v1
  ports:
    - "5001:5000"
  volumes:
    - /data/registryv2/data:/data/registryv2/data
EOF

mkdir /data/registryv2/{registry,data} -pv
cat >/data/registryv2/registry/config.yml<<\EOF
version: 0.1
log:
  level: info
  fields:
    service: registry
#notifications:
##  endpoints:
##    - name: local-python-8000
##      url: http://10.9.120.24:8000/
##      timeout: 1s
##      threshold: 10
##      backoff: 1s
##      disabled: false
storage:
  cache:
    blobdescriptor: inmemory
  filesystem:
    rootdirectory: /var/lib/registry
  delete:
    enabled: true
http:
  addr: :5000
EOF

cat add-mesos.sh 
#!/bin/bash
###time:2018-9-19

ip=`ifconfig  eth0|grep 'inet '|awk '{print $2}'`
echo """
192.168.100.247  mesos-master1
192.168.101.150 mesos-master2
192.168.101.151 mesos-master3
192.168.100.249 mesos-slave1
192.168.100.251 mesos-slave2
192.168.100.250 mesos-slave3
192.168.101.152 mesos-slave4
192.168.101.153 mesos-slave5
192.168.101.166 mesos-slave6
192.168.101.167 mesos-slave7
192.168.101.181 mesos-slave8
192.168.101.182 mesos-slave9
192.168.101.183 mesos-slave10
192.168.101.184 mesos-slave11
192.168.101.192 mesos-slave12
192.168.101.193 mesos-slave13
192.168.101.194 mesos-slave14
192.168.101.195 mesos-slave15

116.213.103.33 msg.cici
192.168.100.44 ldapserver-44.zke.cici
192.168.101.106 ldapserver-106.zke.cici
192.168.101.184 mesos-slave-11
""" >> /etc/hosts
echo """
nameserver 192.168.100.44
nameserver 192.168.101.106
""" >  /etc/resolv.conf 

rpm -Uhv http://repos.mesosphere.com/el/7/noarch/RPMS/mesosphere-el-repo-7-2.noarch.rpm
rpm -ivh https://mirrors.aliyun.com/epel/epel-release-latest-7.noarch.rpm
yum install -y yum-utils device-mapper-persistent-data  xfsprogs  lvm2 gcc*  install zlib-devel curl-devel openssl-devel httpd-devel apr-devel apr-util-devel mysql-devel libffi-devel 

if [ ! -b "/dev/vdb1" ];then
parted -s /dev/vdb mklabel gpt
parted -s /dev/vdb mkpart primary 0 100%
mkfs.xfs /dev/vdb1
else
echo "/dev/vdb1 exist"
fi

if [ ! -b "/dev/vdc1" ];then
parted -s /dev/vdc mklabel gpt
parted -s /dev/vdc mkpart primary 0 100%
mkfs.xfs /dev/vdc1
else
echo "/dev/vdc1 exist"
fi

if [ ! -d "/data" ];then
    mkdir /data
fi
cat /etc/fstab |grep -E "/data|/dev/vdc1"
if [ $? ];then
echo "/dev/vdc1                /data                   xfs     defaults        0 0" >>/etc/fstab
mount -a
fi



###devicemapper
pvcreate /dev/vdb1<<EOF
y

EOF
vgcreate docker /dev/vdb1
lvcreate --wipesignatures y -n thinpool docker -l 95%VG<<EOF
y

EOF
lvcreate --wipesignatures y -n thinpoolmeta docker -l 1%VG<<EOF
y

EOF
lvconvert -y --zero n -c 512K --thinpool docker/thinpool --poolmetadata docker/thinpoolmeta
cat >> /etc/lvm/profile/docker-thinpool.profile << EOF
activation {
    thin_pool_autoextend_threshold=80
    thin_pool_autoextend_percent=20
}
EOF
lvchange --metadataprofile docker-thinpool docker/thinpool

cd
wget http://192.168.101.104/mesos/ruby-2.2.7.tar.gz
tar -zxf ruby-2.2.7.tar.gz
cd ruby-2.2.7
./configure 
make -j 3
make install
cd /root/ruby-2.2.7/ext/zlib
ruby ./extconf.rb
make
make install
cd /root/ruby-2.2.7/ext/openssl
ln -s /root/ruby-2.2.7/include/ /
ruby extconf.rb
make  -j 3
make install
cd /usr/local/bin/ 
./gem sources --remove https://rubygems.org/
gem sources -a https://gems.ruby-china.com/
./gem install certificate_authority
cd 
wget http://192.168.101.104/mesos/certgen1.rb
ruby certgen1.rb  $ip
if [ ! -d "/root/.docker/$ip/" ];then
cd
ruby certgen1.rb  $ip
fi
mkdir  /etc/docker
cp /root/.docker/$ip/* /etc/docker
cd
wget http://192.168.101.104/mesos/docker-engine-1.10.3-1.el7.centos.x86_64.rpm
wget http://192.168.101.104/mesos/docker-engine-selinux-1.10.3-1.el7.centos.noarch.rpm
wget -O /etc/pki/ca-trust/source/anchors/hb2.registry.cici.crt http://192.168.101.104/mesos/hb2.registry.cici.crt
update-ca-trust extract 

yum localinstall -y docker-engine*
wget -O /usr/lib/systemd/system/docker.service  http://192.168.101.104/mesos/docker.service
systemctl daemon-reload
systemctl start docker && systemctl enable docker

docker login -u wowotuanops -p registrywowotuan -e [email protected] hb2.registry.cici:443
if [ $? -eq 0 ];then
cd /root && tar -czvf /etc/docker.tar.gz .docker/
cd
else
echo "docker login err"
fi

yum install mesos -y
mkdir -p /etc/mesos-slave/attributes
echo $ip > /etc/mesos-slave/attributes/host
echo 'main' > /etc/mesos-slave/attributes/hostgroup
echo 'zk://192.168.100.247:2181,192.168.101.150:2181,192.168.101.151:2181/mesos' > /etc/mesos/zk
echo $ip > /etc/mesos-slave/hostname
echo $ip > /etc/mesos-slave/ip
echo 'docker,mesos' > /etc/mesos-slave/containerizers

#systemctl enable mesos-slave && systemctl start  mesos-slave && systemctl disable mesos-master

wget -O /usr/bin/consul  http://192.168.101.104/mesos/consul
wget -O /etc/systemd/system/consul.service http://192.168.101.104/mesos/consul.service
chmod +x /usr/bin/consul 
sed -i "s/IP/$ip/g" /etc/systemd/system/consul.service
systemctl daemon-reload
#systemctl start consul  && systemctl enable consul 

 docker run --restart='always' -d --name=registrator --hostname=${ip} --volume=/var/run/docker.sock:/tmp/docker.sock   hb2.registry.cici:443/registrator:v7 -ip ${ip} consul://${ip}:8500

pip install prometheus_client

cd /home 
wget http://192.168.101.104/mesos/sh.tar.gz
tar zxf sh.tar.gz
cd /data
wget http://192.168.101.104/mesos/check_jvm.tar.gz
tar -zxf check_jvm.tar.gz
echo """
*/1 * * * *  /usr/bin/python /home/sh/altersms.py
0 */6 * * *  cd /data/check_jvm/ && /bin/sh /data/check_jvm/check_jvm_Running.sh
""">>/var/spool/cron/root

# nginx with consul-template
# 1. 查看marathon，并找出对应的应用
访问http://IP:8080/ 找到对应应用的configuration，如container_uuid=cross-bmserver-yzprod-10580
# 2. 编写consul-template模板
如
[root@tc-xy-nginx001 vhosts]# head xr-cicinet.ctmpl
upstream yzbizcenter-api-yzprod-11045 {
    {{range service "yzbizcenter-api-yzprod-11045"}}server {{.Address}}:{{.Port}};{{else}}server 1.1.1.1:8000;{{end}}
}

# 3. 利用脚本生成consul-template服务
[root@tc-xy-nginx001 ~]# cd /home/sh/add-template/
[root@tc-xy-nginx001 add-template]# ll
total 112
-rwxr-xr-x 1 root root  579 Dec 14 14:07 add-nginx-consul-template.sh
-rw-r--r-- 1 root root  213 Jan 14 17:30 consul.cfg
-rw-r--r-- 1 root root  453 Dec 14 14:00 consul-template.service
[root@tc-xy-nginx001 add-template]# cat add-nginx-consul-template.sh
#!/bin/bash
path=../add-template
if [ ! -n "$1" ] ;then
  echo $"usage: $0 nginx-templates-name"
else
  if [ -f /etc/$1.cfg -a -f /etc/systemd/system/consul-template-$1.service ];then
    echo "$1 already exists!!!"
    exit 1
  else
    cp -f consul.cfg $1.cfg
    sed -i s/TEMPLATE/$1/g $1.cfg
    cp -f $1.cfg /etc/
    cp -f consul-template.service consul-template-$1.service
    sed -i s/TEMPLATE/$1/g consul-template-$1.service
    cp -f consul-template-$1.service  /etc/systemd/system/
    #systemctl daemon-reload
    #systemctl enable consul-template-$1.service
  fi
fi

[root@tc-xy-nginx001 add-template]# cat consul.cfg
consul = "192.168.100.9:8500"
template {
  source = "/usr/local/nginx/conf/vhosts/TEMPLATE.ctmpl"
  destination = "/usr/local/nginx/conf/vhosts/TEMPLATE.conf"
  command = "/usr/local/nginx/sbin/nginx -s reload"
}

[root@tc-xy-nginx001 add-template]# cat consul-template.service
# this is consul-template systemd script
# Version: 1.0
# Descripthin: consul
[Unit]
Description=Consul-template
After=network.target
[Service]
ExecStart=/usr/bin/consul-template --config /etc/TEMPLATE.cfg
#ExecStop=/usr/bin/killall -9 /usr/bin/consul-template
ExecStop=/usr/bin/ps aux | /usr/bin/grep "consul-template"| /usr/bin/grep TEMPLATE | /usr/bin/grep -v grep | /usr/bin/awk '{print $2}' | /usr/bin/xargs kill

[Install]
WantedBy=default.target