工作隨筆 - k8s離線安裝存儲/中間件/CICD等組件

前提是k8s已經(jīng)部署好，具體安裝步驟參考http://www.itdecent.cn/p/98b352f16223。存儲不再使用NFC，替代方案longhorn。

安裝Helm，方便快速安裝組件

# helm安裝
tar -xzvf helm-v3.5.4-linux-amd64.tar.gz
mv linux-amd64/helm /usr/local/bin/ && chmod +x /usr/local/bin/helm
helm version

安裝longhorn存儲來作為PV使用

# 安裝依賴
yum -y install iscsi-initiator-utils
systemctl enable iscsid
systemctl start iscsid
yum -y install nfs-utils

# 下載helm模板
helm repo add longhorn https://charts.longhorn.io
helm search repo longhorn/longhorn  -l
helm fetch longhorn/longhorn --version v1.2.4

# 解壓
tar -xzvf longhorn-1.2.4.tgz

# 準(zhǔn)備離線鏡像
harbor.test.lo:5000/longhornio/longhorn-engine:v1.2.4
harbor.test.lo:5000/longhornio/longhorn-manager:v1.2.4
harbor.test.lo:5000/longhornio/longhorn-ui:v1.2.4
harbor.test.lo:5000/longhornio/longhorn-instance-manager:v1_20220303
harbor.test.lo:5000/longhornio/longhorn-share-manager:v1_20211020
harbor.test.lo:5000/longhornio/backing-image-manager:v2_20210820
harbor.test.lo:5000/longhornio/csi-attacher:v3.2.1
harbor.test.lo:5000/longhornio/csi-provisioner:v2.1.2
harbor.test.lo:5000/longhornio/csi-node-driver-registrar:v2.3.0
harbor.test.lo:5000/longhornio/csi-resizer:v1.2.0
harbor.test.lo:5000/longhornio/csi-snapshotter:v3.0.3

# --set defaultSettings.createDefaultDiskLabeledNodes=true 設(shè)置只有添加標(biāo)簽的才會設(shè)置為存儲節(jié)點(diǎn)，默認(rèn)把worker節(jié)點(diǎn)作為longhorn可調(diào)度節(jié)點(diǎn)
kubectl label nodes {worker-node-4,worker-node-5,worker-node-64} node.longhorn.io/create-default-disk=true

# Harbor允許匿名拉取鏡像，不需要配置harbor倉庫的賬號密碼。注意修改本地的path:/home/test/longhorn
helm install longhorn ./longhorn --namespace longhorn --create-namespace --set defaultSettings.createDefaultDiskLabeledNodes=true --set defaultSettings.defaultDataPath="/home/test/longhorn" --set defaultSettings.priority-class=high-priority --set defaultSettings.taintToleration="app=longhorn:NoSchedule" --set defaultSettings.defaultReplicaCount=2 --set service.ui.type=NodePort --set service.ui.nodePort=30001 --set privateRegistry.registryUrl=harbor.test.lo:5000

# 如果Harbor需要賬號密碼訪問
# 創(chuàng)建默認(rèn)拉取harbor鏡像的secret
kubectl create secret docker-registry harbor-secret \
  --docker-server=harbor.test.lo:5000 \
  --docker-username=admin \
  --docker-password=pwd4test \
  -n longhorn

# 安裝,設(shè)置harbor地址，設(shè)置準(zhǔn)備存儲數(shù)據(jù)的目錄（注意不要 掛載到根目錄/longhorn ），
helm install longhorn ./longhorn --namespace longhorn --create-namespace --set defaultSettings.createDefaultDiskLabeledNodes=true --set defaultSettings.defaultDataPath="/home/test/longhorn" --set defaultSettings.priority-class=high-priority --set defaultSettings.taintToleration="app=longhorn:NoSchedule" --set defaultSettings.defaultReplicaCount=2 --set service.ui.type=NodePort --set service.ui.nodePort=30001 --set privateRegistry.registryUrl=harbor.test.lo:5000 --set privateRegistry.registryUser=admin --set privateRegistry.registryPasswd=pwd4test  --set privateRegistry.registrySecret=harbor-secret

# 查看資源是否啟動成功
kubectl get all -n longhorn

# 控制臺(由于nodeport暴露端口30001)
http://192.168.0.1:30001/

Jenkins部署，以及流水線配置

1. 提前準(zhǔn)備離線鏡像

harbor.test.lo:5000/middleware/jenkins:2.365
harbor.test.lo:5000/base/jnlp-slave:4.13.2-1-jdk11
harbor.test.lo:5000/base/maven:3.5.3
harbor.test.lo:5000/base/node:lts
harbor.test.lo:5000/base/alpine:tools
harbor.test.lo:5000/base/helm:v3.7.0-p
harbor.test.lo:5000/base/docker:19.03.9-git

2. 為減少構(gòu)建時間，需將部分?jǐn)?shù)據(jù)持久化，創(chuàng)建pvc：

# 創(chuàng)建namespace
kubectl create ns jenkins
# 創(chuàng)建PVC
cat<<EOF | kubectl apply -f -
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: jenkins-workspace
  namespace: jenkins
spec:
  storageClassName: longhorn
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 50Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: maven-m2
  namespace: jenkins
spec:
  storageClassName: longhorn
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 10Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: jenkins-data
  namespace: jenkins
spec:
  storageClassName: longhorn
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 10Gi
EOF

3. docker方式啟動Jenkins Master節(jié)點(diǎn)（方式一）

# docker方式啟動docker master節(jié)點(diǎn)，50000端口為slave連接master端口
mkdir -p /home/finance/Data/Jenkins && chmod -R 777 /home/finance/Data/Jenkins
docker run -d --name=jenkins --restart=always -e JENKINS_HTTP_PORT_NUMBER=8080 -p 18081:8080 -p 50000:50000 -v /home/finance/Data/Jenkins:/var/jenkins_home harbor.test.lo:5000/middleware/jenkins:2.365
# 驗證Jenkins，創(chuàng)建賬號，修改密碼
http://192.168.0.1:18081
# 安裝插件 (插件提前在有公網(wǎng)環(huán)境的容器中拷貝出來)
tar -xzvf /home/finance/packages/plugins.tgz -C /tmp/
docker cp /tmp/plugins jenkins:/var/jenkins_home/
docker restart jenkins

4. k8s方式啟動Jenkins Master節(jié)點(diǎn)（方式二）

# k8s方式啟動docker master節(jié)點(diǎn)
# 給node打標(biāo)簽，固定jenkins的pod到此幾點(diǎn)
kubectl label nodes worker-node-1 build=true
# 啟動jenkins
cat<<EOF | kubectl apply -f -
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: jenkins-data
  namespace: jenkins
spec:
  storageClassName: longhorn
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 10Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: jenkins-master
  name: jenkins-master
  namespace: jenkins
spec:
  replicas: 1
  selector:
    matchLabels:
      apps: jenkins-master
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      creationTimestamp: null
      labels:
        apps: jenkins-master
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: build
                operator: In
                values:
                - "true"
      containers:
      - env:
        - name: JENKINS_HTTP_PORT_NUMBER
          value: "8080"
        - name: JENKINS_USERNAME
          value: admin
        - name: JENKINS_PASSWORD
          value: pwd4test
        image: harbor.test.lo:5000/middleware/jenkins:2.365
        imagePullPolicy: Always
        name: jenkins-master
        resources:
          limits:
            cpu: "1"
            memory: 2Gi
          requests:
            cpu: 200m
            memory: 1Gi
        volumeMounts:
        - mountPath: /var/jenkins_home
          name: jenkins-data
      restartPolicy: Always
      terminationGracePeriodSeconds: 30
      volumes:
      - name: jenkins-data
        persistentVolumeClaim:
          claimName: jenkins-data
---
apiVersion: v1
kind: Service
metadata:
  name: jenkins-master
  namespace: jenkins
spec:
  ports:
  - name: web
    nodePort: 30007
    port: 8080
    protocol: TCP
    targetPort: 8080
  - name: agent
    nodePort: 30017
    port: 50000
    protocol: TCP
    targetPort: 50000    
  selector:
    apps: jenkins-master
  type: NodePort
EOF

# 獲取初始密碼
kubectl exec -n jenkins -it $(kubectl get pod -n jenkins -l apps=jenkins-master -o jsonpath='{.items[0].metadata.name}') -- cat /var/jenkins_home/secrets/initialAdminPassword

# 驗證Jenkins，創(chuàng)建賬號，修改密碼(由于NodePort設(shè)置30007端口，按需修改)
http://192.168.0.1:30007/
# 安裝plugin （提前在有公網(wǎng)環(huán)境服務(wù)器下載好plugin，打包）
kubectl cp /tmp/plugins `kubectl get pod -n jenkins -l apps=jenkins-master -o jsonpath='{.items[0].metadata.name}'`:/var/jenkins_home/ -n jenkins
# 刪除POD重建，重啟jenkins，登陸驗證plugin是否已安裝

5. 配置Jenkins使用動態(tài)的k8s中Jenkins Slave節(jié)點(diǎn)

登陸Jenkins -> 系統(tǒng)管理 -> Manage Credentials -> ## Stores scoped to Jenkins -> 全局憑據(jù) (unrestricted) -> 添加憑證

  • 添加憑證，Harbor的賬號密碼、 GitLab 的私鑰、 Kubernetes 的證書均使用 Jenkins 的 Credentials 管理。
    示例憑據(jù)信息：
  • 類型: SSH Username with password；默認(rèn)代碼倉庫ID: GITLAB;
  • 類型: SSH Username with password；默認(rèn)HARBOR倉庫ID: HARBOR_ACCOUNT;
  • 類型: secret-file;默認(rèn)K8S證書憑證ID: K8S-STANDARD;文件選擇：上傳~/.kube/config

登陸Jenkins -> 全局安全配置 -> TCP port for inbound agents -> 指定端口（50000）備注：需要和jenkins master的service中暴露的端口對應(yīng)，用于slave節(jié)點(diǎn)連接master節(jié)點(diǎn)

Jenkins配置Slave節(jié)點(diǎn)

Jenkins的slave節(jié)點(diǎn)，可以通過標(biāo)簽控制調(diào)度到某個k8s worker節(jié)點(diǎn)

kubectl label node <node-name> build=true

登陸Jenkins -> 系統(tǒng)管理 -> 節(jié)點(diǎn)管理 -> Configure Clouds -> 配置集群 -> kubernetes Cloud Details

  • name: standard (需要和Jenkinsfile中agent下cloud 'standard'對應(yīng))
  • Kubernetes 地址: http://k8svip.test.lo:6443（apiserver地址）
  • Kubernetes 命名空間: Jenkins （slave節(jié)點(diǎn)調(diào)度的命名空間）
  • 憑據(jù)：config（選擇剛剛創(chuàng)建的k8s證書憑證）
  • Jenkins 地址：http://jenkins-master:8080（master節(jié)點(diǎn)的service地址，如果容器版本master節(jié)點(diǎn)，請適當(dāng)修改）
    點(diǎn)擊連接測試

6. 在代碼倉庫中配置Jenkinsfile和Dockerfile

Jenkinsfile具體內(nèi)容示例如下

pipeline {
  # 保留多少個流水線的構(gòu)建記錄配置
  options {
   buildDiscarder(logRotator(numToKeepStr: '2'))
  }
  # 定義使用 Kubernetes 作為 agent
  agent {
    kubernetes {
      # 選擇的云為之前配置的名字
      cloud 'standard'
      # 將 workspace 改成 PVC 的模式，也可以使用改成 hostPath：workspaceVolume hostPathWorkspaceVolume(hostPath: "/opt/workspace", readOnly: false)
      workspaceVolume persistentVolumeClaimWorkspaceVolume(claimName: 'jenkins-workspace', readOnly: false)
      yaml '''
apiVersion: v1
kind: Pod
spec:
  containers:
    # jnlp容器 ，和Jenkins主節(jié)點(diǎn)通信
    - args: [\'$(JENKINS_SECRET)\', \'$(JENKINS_NAME)\']
      image: 'harbor.test.lo:5000/base/jnlp-slave:4.13.2-1-jdk11'
      name: jnlp
      imagePullPolicy: IfNotPresent
      volumeMounts:
        - mountPath: "/etc/localtime"
          name: "volume-2"
          readOnly: false
        - mountPath: "/etc/hosts"
          name: "volume-hosts"
          readOnly: false
    # build 容器，包含執(zhí)行構(gòu)建的命令， 比如Java的需要 mvn構(gòu)建，就可以用一個maven的鏡像，NodeJS應(yīng)用就使用node鏡像   
    - command:
        - "cat"
      env:
        - name: "LANGUAGE"
          value: "en_US:en"
        - name: "LC_ALL"
          value: "en_US.UTF-8"
        - name: "LANG"
          value: "en_US.UTF-8"
      # 使用Maven鏡像，包含mvn工具，NodeJS 可以用 node 的鏡像
      image: "harbor.test.lo:5000/base/maven:3.5.3"
      imagePullPolicy: "IfNotPresent"
      name: "build"
      tty: true
      volumeMounts:
        - mountPath: "/etc/localtime"
          name: "volume-2"
          readOnly: false
       # Pod 單獨(dú)創(chuàng)建了一個緩存的 volume ，將其掛載到了maven插件的緩存目錄，默認(rèn)是/root/.m2
        - mountPath: "/root/.m2/"
          name: "volume-maven-repo"
          readOnly: false
        - mountPath: "/etc/hosts"
          name: "volume-hosts"
          readOnly: false
        - mountPath: "/root/settings"
          name: settings
    # 變更容器，因為最終是發(fā)版至 Kubernetes 的，所以需要有一個helm命令來實(shí)現(xiàn)變更
    - command:
        - "cat"
      env:
        - name: "LANGUAGE"
          value: "en_US:en"
        - name: "LC_ALL"
          value: "en_US.UTF-8"
        - name: "LANG"
          value: "en_US.UTF-8"
      image: "harbor.test.lo:5000/base/helm:v3.7.0-p"
      imagePullPolicy: "IfNotPresent"
      name: "helm"
      tty: true
      volumeMounts:
        - mountPath: "/etc/localtime"
          name: "volume-2"
          readOnly: false
        - mountPath: "/var/run/docker.sock"
          name: "volume-docker"
          readOnly: false
        - mountPath: "/mnt/.kube/"
          name: "volume-kubeconfig"
          readOnly: false
        - mountPath: "/etc/hosts"
          name: "volume-hosts"
          readOnly: false
    # 用于bulid鏡像的容器，需要包含docker命令
    - command:
        - "cat"
      env:
        - name: "LANGUAGE"
          value: "en_US:en"
        - name: "LC_ALL"
          value: "en_US.UTF-8"
        - name: "LANG"
          value: "en_US.UTF-8"
      image: "harbor.test.lo:5000/base/docker:19.03.9-git"
      imagePullPolicy: "IfNotPresent"
      name: "docker"
      tty: true
      volumeMounts:
        - mountPath: "/etc/localtime"
          name: "volume-2"
          readOnly: false
        # 由于容器沒有啟動 docker 服務(wù)，所以將宿主機(jī)的 docker 經(jīng)常掛載至容器即可
        - mountPath: "/var/run/docker.sock"
          name: "volume-docker"
          readOnly: false
        - mountPath: "/etc/hosts"
          name: "volume-hosts"
          readOnly: false
  restartPolicy: "Never"
  # 固定節(jié)點(diǎn)部署slave，和之前創(chuàng)建的標(biāo)簽需一致
  nodeSelector:
    build: "true"
  securityContext: {}
  volumes:
    - hostPath:
        path: "/var/run/docker.sock"
      name: "volume-docker"
    - hostPath:
        path: "/usr/share/zoneinfo/Asia/Shanghai"
      name: "volume-2"
    - hostPath:
        path: "/etc/hosts"
      name: "volume-hosts"
    - name: "volume-maven-repo"
      persistentVolumeClaim:
        claimName: "maven-m2"
    - name: "volume-kubeconfig"
      secret:
        # 由master節(jié)點(diǎn)下面的~/.kube/config創(chuàng)建的
        secretName: "multi-kube-config"
    - name: "settings"
      secret:
        # 由maven的/root/settings/settings.xml創(chuàng)建，內(nèi)容包含認(rèn)證maven倉庫等信息
        secretName: "zkj-settings"
'''
  }
}
  stages {
    # 拉取指定分支代碼
    stage('Pulling Code') {
      parallel {
        stage('Pulling Code by Jenkins') {
          when {
            expression {
              env.gitlabBranch == null
            }
             
        }
          steps {
            git(branch: "${BRANCH}", credentialsId: 'GITLAB', url: "${GIT_URL}")
            script {
              COMMIT_ID = sh(returnStdout: true, script: "git log -n 1 --pretty=format:'%h'").trim()
              TAG = BUILD_ID+'_'+COMMIT_ID
              println "Env is ${ENV}, App_name is ${APP_NAME}, Current branch is ${BRANCH}, Commit ID is ${COMMIT_ID}, Image TAG is ${TAG}, Giturl is ${GIT_URL}"
            }
          }
        }
        stage('Pulling Code by trigger') {
          when {
            expression {
              env.gitlabBranch != null
           }
         }
         steps {
           git(branch: "${BRANCH}", credentialsId: 'GITLAB', url: "${GIT_URL}")
           script {
              COMMIT_ID = sh(returnStdout: true, script: "git log -n 1 --pretty=format:'%h'").trim()
              TAG = BUILD_ID+'_'+COMMIT_ID
              println "Env is ${ENV}, App_name is ${APP_NAME}, Current branch is ${BRANCH}, Commit ID is ${COMMIT_ID}, Image TAG is ${TAG}, Giturl is ${GIT_URL}"
            }
          }
        }
      }
    }
   
  # 構(gòu)建應(yīng)用
  stage('Building') {
    steps {
      container(name: 'build') {
        sh """
          mvn clean package -U -Dmaven.test.skip=true -gs /root/settings/settings.xml   #構(gòu)建命令
          ls target/*
          """
          }
        }
    }
     
  #構(gòu)建鏡像并推送到代碼倉庫
  stage('Docker build for creating image') {
    environment {
      HARBOR_USER = credentials('HARBOR_ACCOUNT')
    }
    steps {
      container(name: 'docker') {
        sh """
          echo ${HARBOR_USER_USR} ${HARBOR_USER_PSW} ${TAG} ${ENV} ${APP_NAME}
          docker build -t ${HARBOR_ADDRESS}/${REGISTRY_DIR}/${APP_NAME}:${TAG} .   #構(gòu)建鏡像
          docker login -u ${HARBOR_USER_USR} -p ${HARBOR_USER_PSW} ${HARBOR_ADDRESS}
          docker push ${HARBOR_ADDRESS}/${REGISTRY_DIR}/${APP_NAME}:${TAG}  #push鏡像
          """
        }
      }
    }
     
  # 部署應(yīng)用到k8s集群
  stage('Deploying to K8s') {
    environment {
      HARBOR_USER = credentials('HARBOR_ACCOUNT')
    }
    steps {
      container(name: 'helm'){
       sh """
         helm repo add --username ${HARBOR_USER_USR} --password ${HARBOR_USER_PSW} ${REGISTRY_DIR}-${ENV} http://${HARBOR_ADDRESS}/chartrepo/${REGISTRY_DIR} \
         && cd chart \
         && helm cm-push ${APP_NAME} ${REGISTRY_DIR}-${ENV} \
         && helm repo update \
         && helm --kubeconfig /mnt/.kube/config upgrade -i ${APP_NAME} ${REGISTRY_DIR}-${ENV}/${APP_NAME} -n ${ENV} \
         --set image.registry=${HARBOR_ADDRESS} \
         --set image.repository=${REGISTRY_DIR}/${APP_NAME} \
         --set image.tag=${TAG} \
         --set replicaCount=1 \
         --set autoscaling.enabled=false \
         --set resources.limits.cpu=1000m,resources.limits.memory=2048Mi,resources.requests.cpu=100m,resources.requests.memory=1024Mi \
         --set env[0].name="CONFIG_SERVER_URI",env[0].value="${CONFIG_SERVER_URI}" \
         --set env[1].name="JAVA_OPTS",env[1].value="-Xms1024m -Xmx1024m -Dfile.encoding=UTF-8 -Dspring.profiles.active=${ENV} -Djava.awt.headless=true" \
         --set env[2].name="APP_PORT",env[2].value="--server.port=8080" \
         --set env[3].name="SW_AGENT_PATH",env[3].value="-javaagent:/home/test/Apps/skywalking/agent/skywalking-agent.jar -Dskywalking.collector.backend_service=${SKYWALKING_SERVICE} -Dapp.id=${APP_NAME} -Dskywalking.agent.service_name=${APP_NAME}"
         """
        }
      }
    }
  }
  # 定義一些全局的環(huán)境變量
  environment {
    COMMIT_ID = ""
    HARBOR_ADDRESS = "harbor.test.lo:5000" # Harbor地址
    REGISTRY_DIR = "test"  # Harbor的項目目錄,用于存放鏡像
    APP_NAME = sh(returnStdout: true, script: 'echo ${JOB_NAME} | awk -F. \'{print \$1}\'').trim()  # 應(yīng)用名，根據(jù)JOB_NAME獲取，JOB_NAME名實(shí)例appname.dev
    ENV= sh(returnStdout: true, script: 'echo ${JOB_NAME} | awk -F. \'{print \$2}\'').trim()  # 環(huán)境和命名空間，根據(jù)JOB_NAME獲取, JOB_NAME名實(shí)例appname.dev
    TAG = ""
    CONFIG_SERVER_URI = "http://nacos-cs.mdw:8848"  # nacos地址
    SKYWALKING_SERVICE = "skywalking-oap.skywalking:11800"  # skywalking地址
  }
  
  parameters {
    # GitParameter插件
    gitParameter(branch: '', branchFilter: 'origin/(.*)', defaultValue: '', description: 'Branch for build and deploy', name: 'BRANCH', quickFilterEnabled: false, selectedValue: 'NONE', sortMode: 'NONE',tagFilter: '*', type:'PT_BRANCH')
  }
}

Dockerfile應(yīng)用放置于代碼倉庫的根目錄下，具體內(nèi)容示例如下：

FROM harbor.test.lo:5000/dezhu/openjdk:8u345-b01-jre

RUN groupadd test -g 600 \
    && useradd -m -s /bin/bash test-u 600 -g 600

USER test 

RUN mkdir -p ~/Apps ~/Logs/Java ~/Data ~/Conf

COPY appname/target/*.jar Apps/appname.jar

CMD ["sh", "-c", "source /etc/profile;java ${JAVA_OPTS} ${SW_AGENT_PATH} ${SW_AGENT_COLLECTOR_BACKEND_SERVICES} ${SW_AGENT_NAME} -jar Apps/appname.jar ${APP_PORT}"]

7. 配置pipline流水線工程

登陸Jenkins -> 新建任務(wù) -> 流水線 -> 選擇‘Pineline script from SCM’

  • Repository URL: http://x.x.x:90/root/x.git
  • Credentials: （剛創(chuàng)建的gitlab憑證）
  • 指定分支（為空時代表any）: */master (按需修改)
  • 腳本路徑：Jenkinsfile（和gitlab中文件名對應(yīng)）
    注：由于Jenkins參數(shù)由Jenkinsfile生成，所以第一次執(zhí)行流水線會失敗，第二次構(gòu)建就能正常使用。

安裝Loki日志監(jiān)控

# 下載helm chart 
helm repo add grafana https://grafana.github.io/helm-charts
helm fetch grafana/loki-stack --version 2.1.2

# 準(zhǔn)備離線鏡像
grafana/loki:2.0.0
grafana/promtail:2.0.0

# helm安裝安裝Loki
helm upgrade --install loki ./loki-stack --namespace loki-stack --create-namespace \
--set loki.persistence.enabled=true --set loki.persistence.storageClassName=longhorn \
--set loki.persistence.size=100Gi \
--set loki.service.labels."app\.kubernetes\.io\/name"=loki \
--set loki.image.repository=harbor.dezhu.lo:5000/grafana/loki \
--set promtail.volumes[0].name=docker,promtail.volumes[0].hostPath.path="/home/finance/docker-data/containers" \
--set promtail.volumes[1].name=pods,promtail.volumes[1].hostPath.path="/var/log/pods" \
--set promtail.volumeMounts[0].name=docker,promtail.volumeMounts[0].mountPath="/home/finance/docker-data/containers",promtail.volumeMounts[0].readOnly=true \
--set promtail.volumeMounts[1].name=pods,promtail.volumeMounts[1].mountPath="/var/log/pods",promtail.volumeMounts[1].readOnly=true \
--set promtail.image.repository=harbor.dezhu.lo:5000/grafana/promtail \
--set loki.config.table_manager.retention_deletes_enabled=true \
--set loki.config.table_manager.retention_period=336h

安裝kube-prometheus監(jiān)控系統(tǒng)

# 也可以參考http://www.itdecent.cn/p/c01251cf881e安裝官方版本并配置HPA自動伸縮容
# 下載chart
helm repo add my-repo https://charts.bitnami.com/bitnami
helm search repo my-repo/kube-prometheus -l
helm fetch my-repo/kube-prometheus --version 8.1.11

# 準(zhǔn)備離線鏡像
bitnami/prometheus-operator:0.60.1-debian-11-r0
bitnami/prometheus:2.39.1-debian-11-r1
bitnami/thanos:0.28.1-scratch-r0
bitnami/alertmanager:0.24.0-debian-11-r46
bitnami/blackbox-exporter:0.22.0-debian-11-r23
bitnami/node-exporter:1.4.0-debian-11-r2
bitnami/kube-state-metrics:2.6.0-debian-11-r12
bitnami/nginx:1.16.1-debian-10-r63

# helm安裝kube-prometheus
helm upgrade --install kube-prometheus ./kube-prometheus --namespace monitoring --create-namespace \
--set global.imageRegistry=harbor.dezhu.lo:5000 --set global.storageClass=longhorn

# 暴露NodePort方式
helm upgrade --install kube-prometheus ./kube-prometheus --namespace monitoring --create-namespace \
--set global.imageRegistry=harbor.dezhu.lo:5000 --set global.storageClass=longhorn \
--set prometheus.service.type=NodePort --set prometheus.service.nodePort=30010 


# 【此版本無需手動添加，因為此版本的prometheus默認(rèn)使用clusterRole跨namespace訪問】kube-prometheus添加自定義監(jiān)控redis

# mdw 創(chuàng)建Role，此Role賦權(quán)限可以讀取mdw命名空間內(nèi)的api
cat<<EOF | kubectl apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  labels:
    app.kubernetes.io/component: prometheus
    app.kubernetes.io/name: prometheus
    app.kubernetes.io/part-of: kube-prometheus
    app.kubernetes.io/version: 2.29.1
  name: prometheus-k8s
  namespace: mdw
rules:
- apiGroups:
  - ""
  resources:
  - services
  - endpoints
  - pods
  - nodes
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - extensions
  resources:
  - ingresses
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - networking.k8s.io
  resources:
  - ingresses
  verbs:
  - get
  - list
  - watch
EOF

# mdw 創(chuàng)建RoleBingding，給prometheus的ServiceAccount綁定角色prometheus-k8s，這樣prometheus的serviceaccount可以讀取到mdw的api
cat<<EOF | kubectl apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  labels:
    app.kubernetes.io/name: mdw
    app.kubernetes.io/part-of: kube-prometheus
  name: 'prometheus-k8s'
  namespace: mdw
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: prometheus-k8s
subjects:
  - kind: ServiceAccount
    name: prometheus-k8s
    namespace: monitoring
EOF

# 創(chuàng)建自定義的PrometheusRule,來定義告警信息
cat<<EOF | kubectl apply -f -
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: redis-rules
  # If labels are defined in spec.ruleSelector.matchLabels of your deployed Prometheus object, make sure to include them here.
  labels:
   role: alert-rules
   app.kubernetes.io/component: redis-rules
   app.kubernetes.io/name: redis-rules
   app.kubernetes.io/part-of: kube-prometheus
   app.kubernetes.io/version: 6.2.6
   prometheus: k8s
  namespace: monitoring
spec:
  groups:
  - name: redisdown
    rules:
    - alert: RedisDown
      expr: redis_up == 0
      for: 1m
      labels:
        name: instance
        severity: Critical
      annotations:
        summary: " {{ $labels.alias }}"
        description: " 服務(wù)停止運(yùn)行 "
        value: "{{ $value }}"
    - alert: Redis linked too many clients
      expr: redis_connected_clients / redis_config_maxclients * 100 > 80
      for: 1m
      labels:
        name: instance
        severity: Warning
      annotations:
        summary: " {{ $labels.alias }}"
        description: " Redis連接數(shù)超過最大連接數(shù)的80%. "
        value: "{{ $value }}"
    - alert: Redis linked
      expr: redis_connected_clients / redis_config_maxclients * 100 > 80
      for: 1m
      labels:
        name: instance
        severity: Warning
      annotations:
        summary: " {{ $labels.alias }}"
        description: " Redis連接數(shù)超過最大連接數(shù)的80%. "
        value: "{{ $value }}"
EOF

# 監(jiān)控外部mysql 
# 需要在監(jiān)控的目標(biāo)數(shù)據(jù)庫創(chuàng)建賬號用于收集數(shù)據(jù)庫監(jiān)控指標(biāo)
CREATE USER 'exporter'@'localhost' IDENTIFIED BY 'pwd4test';
GRANT PROCESS, REPLICATION CLIENT, SELECT ON *.* TO 'exporter'@'localhost';
flush privileges;
# 安裝mysqld_exporter
wget https://github.com/prometheus/mysqld_exporter/releases/download/v0.14.0/mysqld_exporter-0.14.0.linux-amd64.tar.gz
sudo tar xf mysqld_exporter-0.14.0.linux-amd64.tar.gz -C /opt
sudo mv /opt/mysqld_exporter-0.14.0.linux-amd64 /opt/mysqld_exporter
# 創(chuàng)建配置文件，按需修改服務(wù)器信息
cat <<EOF | sudo tee /opt/mysqld_exporter/.my.cnf
[client]
host=127.0.0.1
port=3306
user=exporter
password=********
EOF
# 創(chuàng)建service
cat <<EOF | sudo tee /usr/lib/systemd/system/mysqld_exporter.service
[Unit]
Description=Mysqld Exporter for Prometheus
Wants=network-online.target
After=network-online.target

[Service]
User=root
Group=root
Type=simple
ExecStart=/opt/mysqld_exporter/mysqld_exporter \
--config.my-cnf=/opt/mysqld_exporter/.my.cnf \
--web.listen-address=:9104 \
--collect.perf_schema.replication_group_members \
--collect.perf_schema.replication_group_member_stats
KillMode=process
Restart=always

[Install]
WantedBy=multi-user.target
EOF
# 啟動服務(wù)
sudo systemctl daemon-reload
sudo systemctl enable --now mysqld_exporter
# 驗證指標(biāo)獲取
curl http://127.0.0.1:9104/metrics
# 創(chuàng)建service,監(jiān)聽endpoint
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  name: mysqld-exporter
  namespace: monitoring
  labels:
    app.kubernetes.io/name: mysqld-exporter
spec:
  type: ClusterIP
  ports:
  - name: http-metrics
    port: 9104
    protocol: TCP
    targetPort: 9104
EOF
# 創(chuàng)建service監(jiān)聽的endpoint （ip按需修改，為mysql服務(wù)器ip）
cat << EOF | kubectl apply -f -
apiVersion: v1
kind: Endpoints
metadata:
  name: mysqld-exporter
  namespace: monitoring
  labels:
    app.kubernetes.io/name: mysqld-exporter
subsets:
- addresses:
  - ip: 10.99.67.93
    nodeName: 10.99.67.93
  ports:
  - name: http-metrics
    port: 9104
    protocol: TCP
EOF
# 創(chuàng)建service monitor,接入prometheus獲取
cat << EOF | kubectl apply -f -
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: mysqld-exporter
  namespace: monitoring
  labels:
    app.kubernetes.io/name: mysqld-exporter
    release: prometheus
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: mysqld-exporter
  namespaceSelector:
    matchNames:
    - monitoring
  endpoints:
  - port: http-metrics
    interval: 30s
    path: /metrics
EOF
# 打開prometheus驗證endpoint是否添加

安裝rabbitmq

# 下載chart
helm repo add bitnami https://charts.bitnami.com/bitnami
helm pull bitnami/rabbitmq --version=8.25.0

# 準(zhǔn)備離線鏡像
bitnami/rabbitmq:3.9.11-debian-10-r28
bitnami/bitnami-shell:10-debian-10-r296
bitnami/nginx:1.16.1-debian-10-r63

# helm安裝rabbitmq (注意修改prometheus的namespace)
helm upgrade --install rabbitmq ./rabbitmq --namespace mdw --create-namespace \
--set global.storageClass=longhorn \
--set image.registry=harbor.dezhu.lo:5000 \
--set image.repository=bitnami/rabbitmq \
--set image.tag=3.9.11-debian-10-r28 \
--set replicaCount=3 \
--set metrics.enabled=true \
--set metrics.serviceMonitor.enabled=true \
--set metrics.serviceMonitor.namespace=monitoring \
--set ingress.enabled=true \
--set ingress.hostname=rabbitmq.test.lo \
--set ingress.annotations."kubernetes\.io\/ingress\.class"=nginx \
--set persistence.size=4G \
--set auth.username=dezhu \
--set auth.password=pwd4test \
--set extraEnvVars[0].name=TZ \
--set extraEnvVars[0].value="Asia/Shanghai" \
--set volumePermissions.enabled=true \
--set volumePermissions.image.registry=harbor.dezhu.lo:5000 \
--set volumePermissions.image.repository=bitnami/bitnami-shell \
--set volumePermissions.image.tag=10-debian-10-r296

# 配置本機(jī)/etc/hosts，訪問驗證
xx.xx.xx.xx rabbitmq.test.lo

http:// rabbitmq.test.lo/

安裝redis cluster (6.2.6)

# 下載helm chart 
helm pull bitnami/redis-cluster --version=7.5.0

# 準(zhǔn)備離線鏡像
bitnami/redis-cluster:6.2.6-debian-10-r193
bitnami/bitnami-shell:10-debian-10-r402
bitnami/redis-exporter:1.37.0-debian-10-r33
bitnami/nginx:1.16.1-debian-10-r63 

# helm安裝redis-cluster (注意修改prometheus的namespace)
helm upgrade --install redis-cluster ./redis-cluster --namespace mdw --create-namespace \
--set image.registry=harbor.dezhu.lo:5000 \
--set image.repository=bitnami/redis-cluster \
--set image.tag=6.2.6-debian-10-r193 \
--set usePassword=true \
--set password="pwd4test" \
--set persistence.enabled=true \
--set persistence.storageClass=longhorn \
--set persistence.size=2Gi \
--set metrics.enabled=true \
--set metrics.image.registry=harbor.dezhu.lo:5000  \
--set metrics.image.repository=bitnami/redis-exporter \
--set metrics.image.tag=1.37.0-debian-10-r33 \
--set metrics.serviceMonitor.enabled=true \
--set metrics.serviceMonitor.namespace=monitoring

# 驗證

redis-cli -c -h redis-cluster -a pwd4test

安裝mysql

# 拉取chart
elm pull bitnami/mysql --version 9.4.0

# 準(zhǔn)備離線鏡像
bitnami/mysql:8.0.30-debian-11-r27
bitnami/bitnami-shell:11-debian-11-r39
bitnami/mysqld-exporter:0.14.0-debian-11-r45
bitnami/nginx:1.16.1-debian-10-r63

 # helm安裝mysql (注意修改prometheus的namespace)
helm upgrade --install mysql ./mysql --namespace mdw --create-namespace \
--set global.storageClass=longhorn \
--set global.imageRegistry=harbor.dezhu.lo:5000 \
--set image.repository=bitnami/mysql \
--set image.tag=8.0.30-debian-11-r27 \
--set volumePermissions.image.repository=bitnami/bitnami-shell \
--set volumePermissions.image.tag=11-debian-11-r39 \
--set primary.persistence.size=10Gi \
--set secondary.persistence.size=10Gi \
--set auth.rootPassword=pwd4test \
--set metrics.enabled=true \
--set metrics.image.repository=bitnami/mysqld-exporter \
--set metrics.image.tag=0.14.0-debian-11-r45 \
--set metrics.prometheusRule.enabled=true \
--set metrics.serviceMonitor.enabled=true \
--set metrics.serviceMonitor.namespace=monitoring

# 登陸Mysql驗證
mysql -h mysql.mdw.svc.cluster.local -uroot -p

安裝grafana

# 下載helm chart 
helm fetch my-repo/grafana --version=8.2.11

# 準(zhǔn)備離線鏡像
bitnami/grafana:9.1.7-debian-11-r0
bitnami/grafana-image-renderer:3.6.1-debian-11-r10
bitnami/bitnami-shell:11-debian-11-r38

# helm安裝grafana
helm upgrade --install grafana ./grafana --namespace monitoring --create-namespace \
--set global.imageRegistry=harbor.dezhu.lo:5000 --set global.storageClass=longhorn \
--set admin.user=admin --set admin.password=pwd4test \
--set service.type=NodePort --set service.nodePorts.grafana=30002

# 登陸grafana驗證
http://10.99.73.137:30002/

# 添加kube-prometheus的datasource
setting -> Data source -> 刪除已存在的的prometheus
setting -> Data source -> prometheus 
  Name: Prometheus
  URL: http://kube-prometheus-prometheus.monitoring.svc.cluster.local:9090
save & test

# 下載prometheus 的dashboard模板 （https://grafana.com/grafana/dashboards/?search=starsl 搜索關(guān)鍵字starsl）
https://grafana.com/grafana/dashboards/13105
https://grafana.com/grafana/dashboards/13105-1-k8s-for-prometheus-dashboard-20211010/

# 導(dǎo)入dashboard模板
Dashboard -> Browse -> import -> 上傳文件 -> load -> 選擇VictoriaMetrics:Prometheus -> import 

# 添加Loki的datasource
setting -> Data source -> Loki 
  Name: Loki
  URL: http://loki.loki-stack.svc.cluster.local:3100
save & test

Export -> Data Source: Loki -> Labes：xxx -> Run Query -> 查看日志

# 導(dǎo)入dashboard模板
https://grafana.com/grafana/dashboards/15141


# 導(dǎo)入rabbitmq的dashboard
https://grafana.com/grafana/dashboards/10991
最后編輯于 ：
?著作權(quán)歸作者所有,轉(zhuǎn)載或內(nèi)容合作請聯(lián)系作者
【社區(qū)內(nèi)容提示】社區(qū)部分內(nèi)容疑似由AI輔助生成，瀏覽時請結(jié)合常識與多方信息審慎甄別。
平臺聲明：文章內(nèi)容（如有圖片或視頻亦包括在內(nèi)）由作者上傳并發(fā)布，文章內(nèi)容僅代表作者本人觀點(diǎn)，簡書系信息發(fā)布平臺，僅提供信息存儲服務(wù)。

相關(guān)閱讀更多精彩內(nèi)容

友情鏈接更多精彩內(nèi)容