1 容器化微服务项目一般步奏

具体步骤：
第一步：熟悉Spring Cloud微服务项目
第二步：源代码编译构建
第三步：构建项目镜像并推送到镜像仓库
第四步：K8s服务编排
第五步：在K8s中部署Eureka集群（注册中心）和MySQL数据库
第六步：部署微服务网关服务
第七步：部署微服务业务程序
第八步：部署微服务前端
第九步：微服务对外发布
第十步：微服务升级与扩容

2 部署一套基于springcloud的微服务demo

https://github.com/lizhenliang/simple-microservice

代码分支说明：

• dev1 交付代码
• dev2 增加Dockerfile
• dev3 增加K8S资源编排
• dev4 增加微服务链路监控
• master 最终上线

2.1 源代码编译构建

2.1.1 下载项目代码

mkdir springcloud
cd springcloud/
git clon https://github.com/lizhenliang/simple-microservice.git 

2.1.2 安装java和maven环境

yum install java-1.8.0-openjdk maven

vim /etc/maven/settings.xml
  <mirrors>
     <mirror>
        <id>aliyunmaven</id>
        <mirrorOf>*</mirrorOf>
        <name>阿里云公共仓库</name>
        <url>https://maven.aliyun.com/repository/public</url>
    </mirror>
  </mirrors>

2.1.3 编译

cd /root/springcloud/simple-microservice_dev2

 mvn clean package -Dmaven.test.skip=true    #编译

2.2 配置微服务镜像

2.2.1 配置gateway镜像并推送到镜像仓库

cd /root/springcloud/simple-microservice_dev2/gateway-service
cat Dockerfile
FROM java:8-jdk-alpine
LABEL maintainer www.aliangedu.cn
RUN  sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && \
     apk add -U tzdata && \
     ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
COPY ./target/gateway-service.jar ./
EXPOSE 9999
CMD java -jar /gateway-service.jar

构建gateway镜像

docker build -t gateway .
推送harbor仓库
docker login 192.168.10.20:8081
docker tag gateway:latest 192.168.10.20:8081/springcloud/gateway:latest
docker push 192.168.10.20:8081/springcloud/gateway:latest

2.2.2 配置order镜像并推送到镜像仓库

[root@gitlab order-service-biz]# cat Dockerfile 
FROM java:8-jdk-alpine
LABEL maintainer www.aliangedu.cn
RUN  sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && \
     apk add -U tzdata && \
     ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
COPY ./target/order-service-biz.jar ./
EXPOSE 8020
CMD java -jar /order-service-biz.jar
构建order镜像
docker build -t order .

推送harbor仓库
docker tag order:latest 192.168.10.20:8081/springcloud/order:latest
docker push  192.168.10.20:8081/springcloud/order:latest

2.2.3 配置portal镜像并推送到镜像仓库

cat portal-service/Dockerfile
FROM java:8-jdk-alpine
LABEL maintainer www.aliangedu.cn
RUN  sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && \
     apk add -U tzdata && \
     ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
COPY ./target/portal-service.jar ./
EXPOSE 8080
CMD java -jar /portal-service.jar

构建order镜像
docker build -t portal .

推送harbor仓库
docker tag portal:latest 192.168.10.20:8081/springcloud/portal:latest
docker push  192.168.10.20:8081/springcloud/portal:latest

2.2.4 配置stock镜像并推送到镜像仓库

[root@gitlab simple-microservice_dev3]# cat stock-service/stock-service-biz/Dockerfile 
FROM java:8-jdk-alpine
LABEL maintainer www.aliangedu.cn
RUN  sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && \
     apk add -U tzdata && \
     ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
COPY ./target/stock-service-biz.jar ./
EXPOSE 8030
CMD java -jar /stock-service-biz.jar

构建order镜像
docker build -t stock .

推送harbor仓库
docker tag stock:latest 192.168.10.20:8081/springcloud/stock:latest
docker push  192.168.10.20:8081/springcloud/stock:latest

2.2.5 配置product镜像并推送到镜像仓库

[root@gitlab simple-microservice_dev3]# cat product-service/product-service-biz/Dockerfile 
FROM java:8-jdk-alpine
LABEL maintainer www.aliangedu.cn
RUN  sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && \
     apk add -U tzdata && \
     ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
COPY ./target/product-service-biz.jar ./
EXPOSE 8010
CMD java -jar /product-service-biz.jar

构建order镜像
docker build -t product .

推送harbor仓库
docker tag product:latest 192.168.10.20:8081/springcloud/product:latest
docker push  192.168.10.20:8081/springcloud/product:latest

2.2.6 配置eureka镜像并推送到镜像仓库

[root@gitlab simple-microservice_dev3]# cat eureka-service/Dockerfile 
FROM java:8-jdk-alpine
LABEL maintainer www.aliangedu.cn
RUN  sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && \
     apk add -U tzdata && \
     ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
COPY ./target/eureka-service.jar ./
EXPOSE 8888
CMD java -jar -Deureka.instance.hostname=${MY_POD_NAME}.eureka.ms /eureka-service.jar

构建order镜像
docker build -t eureka .

推送harbor仓库
docker tag eureka:latest 192.168.10.20:8081/springcloud/eureka:latest
docker push  192.168.10.20:8081/springcloud/eureka:latest

2.2.7 脚本构建微服务镜像并推送到镜像仓库

[root@gitlab simple-microservice_dev3]# cat k8s/docker_build.sh 
#!/bin/bash

docker_registry=192.168.10.20:8081
# 存储登录Harbor认证信息
#kubectl create secret docker-registry registry-pull-secret --docker-server=docker_registry --docker-username=admin --docker-password=Harbor12345 --docker-email=admin@ctnrs.com -n ms

service_list="eureka-service gateway-service order-service product-service stock-service portal-service"
service_list={1:-{service_list}}
work_dir=(dirname PWD)
current_dir=PWD

cd work_dir
mvn clean package -Dmaven.test.skip=true

for service inservice_list; do
   cd work_dir/service
   # 业务程序需进入biz目录里构建
   if ls |grep biz &>/dev/null; then
      cd {service}-biz
   fi
   service={service%-*}
   image_name=docker_registry/springcloud/{service}:(date +%F-%H-%M-%S)
   docker build -t{image_name} .
   docker push {image_name}   # 修改yaml中镜像地址为新推送的，并apply
   #sed -i -r "s#(image: )(.*)#\1image_name#" {current_dir}/{service}.yaml
   #kubectl apply -f {current_dir}/{service}.yaml
done

如图 ,镜像已做好.

2.3 在K8s中部署Eureka集群（注册中心）和MySQL数据库

2.3.1 部署mysql,数据导入

数据库建议不部署在k8s中
由于CentOS7的yum源中没有mysql，需要到mysql的官网下载yum repo配置文件。
下载命令：
wget https://dev.mysql.com/get/mysql57-community-release-el7-9.noarch.rpm
然后进行yum源的安装：
rpm -ivh mysql57-community-release-el7-9.noarch.rpm
安装完成后，就可以使用yum命令安装mysql了：

yum -y install mysql-server
启动mysql：

systemctl start mysqld
查看mysql状态：

systemctl status mysqld
获取mysql的临时密码：

grep 'temporary password' /var/log/mysqld.log
2022-02-23T08:43:27.020830Z 1 [Note] A temporary password is generated for root@localhost: -lN.YigwO2e:

mysql -u root -p(此处不用输入密码，按下回车后会专门要你再输入密码的)
登录成功后，做任何操作都会被要求先修改密码

show databases; 

按照系统的默认要求，修改密码：
ALTER USER 'root'@'localhost' IDENTIFIED BY 'password'

创建新用户
grant all privileges on *.* to 'user'@'%' identified by 'user' with grant option;

导入数据库
 mysql -ums -p < order.sql 
 mysql -ums -p < product.sql 
 mysql -ums -p < stock.sql 

2.3.2 部署有状态的eureka集群

[root@k8s-master01 simple-microservice_dev3]# kubectl create namespace ms

cat k8s/eureka.yaml 
---
apiVersion: v1
kind: Service
metadata:
  name: eureka
  namespace: ms
spec:
  clusterIP: None
  ports:
  - port: 8888
    name: eureka
  selector:
    project: ms
    app: eureka

---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: eureka 
  namespace: ms 
spec:
  rules:
    - host: eureka.devopstack.cn 
      http:
        paths:
        - path: /
          pathType: Prefix
          backend:
            serviceName: eureka
            servicePort: 8888
---

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: eureka
  namespace: ms 
spec:
  replicas: 3
  selector:
    matchLabels:
      project: ms
      app: eureka
  serviceName: "eureka"
  template:
    metadata:
      labels:
        project: ms 
        app: eureka
    spec:
      imagePullSecrets:
      - name: dockerpullauth
      containers:
      - name: eureka
        image: 192.168.10.20:8081/springcloud/eureka:2022-02-23-17-06-43
        ports:
          - protocol: TCP
            containerPort: 8888
        env:
          - name: MY_POD_NAME
            valueFrom:
              fieldRef:
                fieldPath: metadata.name
        resources:
          requests:
            cpu: 0.5
            memory: 256Mi
          limits:
            cpu: 1 
            memory: 1Gi
        readinessProbe:
          tcpSocket:
            port: 8888
          initialDelaySeconds: 60
          periodSeconds: 10
        livenessProbe:
          tcpSocket:
            port: 8888
          initialDelaySeconds: 60
          periodSeconds: 10

部署资源

[root@k8s-master01 simple-microservice_dev3]# kubectl apply  -f k8s/eureka.yaml 

查看状态
[root@k8s-master01 ~]# kubectl get pods -n ms
NAME       READY   STATUS    RESTARTS   AGE
eureka-0   1/1     Running   0          103m
eureka-1   1/1     Running   0          105m
eureka-2   1/1     Running   0          107m

查看ingress
 kubectl get ingress -n ms
NAME     CLASS    HOSTS                  ADDRESS   PORTS   AGE
eureka   <none>   eureka.devopstack.cn             80      3h18m

绑定192.168.10.71 eureka.devopstack.cn 到宿主机hosts文件
如图 eureka已部署

2.3 部署微服务业务程序

2.3.1 微服务架构

2.3.2 配置文件修改

[root@k8s-master01 simple-microservice_dev3]# cat stock-service/stock-service-biz/src/main/resources/application-fat.yml 
spring:
  datasource:
    url: jdbc:mysql://192.168.10.20:3306/tb_stock?characterEncoding=utf-8
    username: ms
    password: Devopstack@suixiaofeng_2
    driver-class-name: com.mysql.jdbc.Driver

eureka:
  instance:
    prefer-ip-address: true
  client:
    register-with-eureka: true
    fetch-registry: true
    service-url:
      defaultZone: http://eureka-0.eureka.ms:8888/eureka,http://eureka-1.eureka.ms:8888/eureka,http://eureka-2.eureka.ms:8888/eureka

2.3.2 部署微服务(脚本)

$ cd microservic-code/simple-microservice-dev3
$ cd k8s && ls
docker_build.sh  eureka.yaml  gateway.yaml  order.yaml  portal.yaml  product.yaml  stock.yaml

$ ./docker_build.sh     # 自动构建并上传镜像，同时启动服务
$ kubectl get pod -n ms   # 查看构建之后的pod是否正常
NAME                       READY   STATUS    RESTARTS   AGE
eureka-0                   1/1     Running   0          4m15s
eureka-1                   1/1     Running   0          6m3s
eureka-2                   1/1     Running   0          7m43s
gateway-664f898649-mbkk7   0/1     Running   0          53s
gateway-664f898649-ngc4q   0/1     Running   0          53s
order-5944945bcc-2fvpg     1/1     Running   0          8m21s
order-5944945bcc-5m4cc     1/1     Running   0          8m21s
portal-85dbd7658d-drh9v    1/1     Running   0          4m46s
portal-85dbd7658d-hxsxd    1/1     Running   0          4m46s
product-5884f5bbb7-gcqdb   1/1     Running   0          8m16s
product-5884f5bbb7-xfwwt   1/1     Running   0          8m16s
stock-b8bf4db55-5gwf2      1/1     Running   0          8m11s
stock-b8bf4db55-wjl7f      1/1     Running   0          8m11s

微服务对外发布
root@k8s-master01 k8s]# kubectl get ingress -n ms
NAME      CLASS    HOSTS                   ADDRESS   PORTS   AGE
eureka    <none>   eureka.devopstack.cn              80      3h47m
gateway   <none>   gateway.devopstack.cn             80      97s
portal    <none>   portal.devopstack.cn              80      5m30s
绑定上述域名到192.168.10.71 的hosts 解析文件，即可访问
## springcloud
192.168.10.71 eureka.devopstack.cn
192.168.10.71 portal.devopstack.cn
192.168.10.71 gateway.devopstack.cn

如eureka的信息,相关服务已注册

2.3.3 验证服务部署情况

如下图,微服务已部署成功

2.3.4 验证微服务组件联动

＞先把商品加入购物车，然后去购物车查看

＞查看购物车里是否添加成功

如上所示，微服务之间的联动是没有问题的，微服务demo在k8s中部署成功．

2.3.4 各微服务yaml

[root@k8s-master01 k8s]# cat gateway.yaml 
---
apiVersion: extensions/v1beta1 
kind: Ingress
metadata:
  name: gateway 
  namespace: ms 
spec:
  rules:
    - host: gateway.devopstack.cn 
      http:
        paths:
        - path: /
          pathType: Prefix
          backend:
            serviceName: gateway
            servicePort: 9999
---
apiVersion: v1
kind: Service
metadata:
  name: gateway
  namespace: ms
spec:
  ports:
  - port: 9999 
    name: gateway
  selector:
    project: ms
    app: gateway
---
apiVersion: apps/v1
kind: Deployment 
metadata:
  name: gateway
  namespace: ms 
spec:
  replicas: 2
  selector:
    matchLabels:
      project: ms
      app: gateway
  template:
    metadata:
      labels:
        project: ms 
        app: gateway
    spec:
      imagePullSecrets:
      - name: registry-pull-secret
      containers:
      - name: gateway
        image: 192.168.10.20:8081/springcloud/gateway:2022-02-24-20-30-01
        imagePullPolicy: Always
        ports:
          - protocol: TCP
            containerPort: 9999 
        resources:
          requests:
            cpu: 0.5
            memory: 256Mi
          limits:
            cpu: 1
            memory: 1Gi
        readinessProbe:
          tcpSocket:
            port: 9999
          initialDelaySeconds: 60
          periodSeconds: 10
        livenessProbe:
          tcpSocket:
            port: 9999
          initialDelaySeconds: 60
          periodSeconds: 10

[root@k8s-master01 k8s]# cat order.yaml
apiVersion: apps/v1
kind: Deployment 
metadata:
  name: order
  namespace: ms 
spec:
  replicas: 2
  selector:
    matchLabels:
      project: ms
      app: order
  template:
    metadata:
      labels:
        project: ms 
        app: order
    spec:
      imagePullSecrets:
      - name: registry-pull-secret
      containers:
      - name: order
        image: 192.168.10.20:8081/springcloud/order:2022-02-24-20-30-05
        imagePullPolicy: Always
        ports:
          - protocol: TCP
            containerPort: 8020 
        resources:
          requests:
            cpu: 0.5
            memory: 256Mi
          limits:
            cpu: 1
            memory: 1Gi
        readinessProbe:
          tcpSocket:
            port: 8020
          initialDelaySeconds: 60
          periodSeconds: 10
        livenessProbe:
          tcpSocket:
            port: 8020
          initialDelaySeconds: 60
          periodSeconds: 10
[root@k8s-master01 k8s]# cat portal.yaml 
apiVersion:  extensions/v1beta1 
kind: Ingress
metadata:
  name: portal 
  namespace: ms 
spec:
  rules:
    - host: portal.devopstack.cn 
      http:
        paths:
        - path: /
          pathType: Prefix
          backend:
              serviceName: portal
              servicePort: 8080
---
apiVersion: v1
kind: Service
metadata:
  name: portal
  namespace: ms
spec:
  ports:
  - port: 8080
    name: portal 
  selector:
    project: ms
    app: portal
---
apiVersion: apps/v1
kind: Deployment 
metadata:
  name: portal
  namespace: ms 
spec:
  replicas: 2
  selector:
    matchLabels:
      project: ms
      app: portal
  template:
    metadata:
      labels:
        project: ms 
        app: portal
    spec:
      imagePullSecrets:
      - name: registry-pull-secret
      containers:
      - name: portal
        image: 192.168.10.20:8081/springcloud/portal:2022-02-24-20-30-20
        imagePullPolicy: Always
        ports:
          - protocol: TCP
            containerPort: 8080 
        resources:
          requests:
            cpu: 0.5
            memory: 256Mi
          limits:
            cpu: 1
            memory: 1Gi
        readinessProbe:
          tcpSocket:
            port: 8080
          initialDelaySeconds: 60
          periodSeconds: 10
        livenessProbe:
          tcpSocket:
            port: 8080
          initialDelaySeconds: 60
          periodSeconds: 10
[root@k8s-master01 k8s]# cat stock.yaml 
apiVersion: apps/v1
kind: Deployment 
metadata:
  name: stock
  namespace: ms 
spec:
  replicas: 2
  selector:
    matchLabels:
      project: ms
      app: stock
  template:
    metadata:
      labels:
        project: ms 
        app: stock
    spec:
      imagePullSecrets:
      - name: registry-pull-secret
      containers:
      - name: stock
        image: 192.168.10.20:8081/springcloud/stock:2022-02-24-20-30-15
        imagePullPolicy: Always
        ports:
          - protocol: TCP
            containerPort: 8030
        resources:
          requests:
            cpu: 0.5
            memory: 256Mi
          limits:
            cpu: 1
            memory: 1Gi
        readinessProbe:
          tcpSocket:
            port: 8030
          initialDelaySeconds: 60
          periodSeconds: 10
        livenessProbe:
          tcpSocket:
            port: 8030
          initialDelaySeconds: 60
          periodSeconds: 10

[root@k8s-master01 k8s]# cat product.yaml
apiVersion: apps/v1
kind: Deployment 
metadata:
  name: product
  namespace: ms 
spec:
  replicas: 2
  selector:
    matchLabels:
      project: ms
      app: product
  template:
    metadata:
      labels:
        project: ms 
        app: product
    spec:
      imagePullSecrets:
      - name: registry-pull-secret
      containers:
      - name: product
        image: 192.168.10.20:8081/springcloud/product:2022-02-24-20-30-10
        imagePullPolicy: Always
        ports:
          - protocol: TCP
            containerPort: 8010 
        resources:
          requests:
            cpu: 0.5
            memory: 256Mi
          limits:
            cpu: 1
            memory: 1Gi
        readinessProbe:
          tcpSocket:
            port: 8010
          initialDelaySeconds: 60
          periodSeconds: 10
        livenessProbe:
          tcpSocket:
            port: 8010
          initialDelaySeconds: 60
          periodSeconds: 10

3 可能会遇到的问题

3.1 限制了容器资源，还经常被杀死

java不能自动发现docker设置的堆内存，这将会导致

JVM资源不稳定，超出limits限制，k8s会杀掉该容器！

解决办法：

• 手动指定JVM堆内存大小

• 配置JVM自动识别（1.9版本+才支持）-

XX:+UnlockExperimentalVMOptions -

XX:+UseCGroupMemoryLimitForHeap

3.2 滚动更新之健康检查的重要性

滚动更新是默认发布策略，当配置健康检查时，滚动更新会根据Probe状态来决定是否继续更新以及是否允许接入流量，这样在整个滚动更新过程中可抱歉始终会有可用的Pod存在，达到平滑升级。

3.3 滚动更新之流量丢失

滚动更新触发，Pod在删除过程中，有些节点kube-proxy还没来得及同步iptables规则，从而部分流量请求到Terminating的Pod上，导致请求出错。 解决办法：配置preStop回调，在容器终止前优雅暂停5秒，给kube-proxy多预留一点时间