**運(yùn)行環(huán)境: Python 2.7.5 , linux **

參考文檔

restfulAPI: https://docs.gitlab.com/ee/api/
官方Api: https://python-gitlab.readthedocs.io/en/v1.10.0/

python-gitlab簡(jiǎn)介

pip install python-gitlab
#官方建議使用配置文件
vi .python-gitlab.cfg
[global]
default = gitlab
ssh_verify = False
timeout = 8

[gitlab]
url = http://xxxxxxxxx
private_token = xxxxxxxxx

gitlab中private_token的獲取

gitlab.jpg

gitlab demo 簡(jiǎn)單使用，主要是理解api

# coding:utf-8
'''
用于上傳存儲(chǔ)過(guò)程以及本地文件到gitlab中
'''

import gitlab

def allprojects():
    #######獲取gitlab的所有projects###
    projects = gl.projects.list(all=True)
    for project in projects:
        print project.name, project.id


def allgroups():
    #######獲取gitlab的所有g(shù)roup名稱以及ID###
    all_groups = gl.groups.list(all=True)
    for group in all_groups:
        print group.name, group.id


def allusers():
    #######獲取gitlab的所有user名稱以及ID###
    users = gl.users.list(all=True)
    for user in users:
        print user.username, user.id, user.name, user.state


def assgroup():
    #######獲取gitlab指定組內(nèi)所有user以及project名稱以及ID信息，本例中組ID為58###
    gid = int(raw_input('Input the group ID: '))
    group = gl.groups.get(gid)
    print group.name
    # members = group.members.list(all=True)
    # for me in members:
    #    print me.username,me.id
    projects = group.projects.list(all=True)
    for project in projects:
        print group.name, project.name
    #######################################


def projectinfo():
    pid = int(raw_input('Input the pid: '))
    projects = gl.projects.get(pid)
    return projects


def projectid():
    gid = int(raw_input('Input the group ID: '))
    group = gl.groups.get(gid)
    repo = str(raw_input('Input your repo name: '))
    project = gl.projects.get(group.name + '/' + repo)
    print project.id


def assuser():
    #######獲取gitlab指定user###
    uid = int(raw_input('Input the user ID: '))
    user = gl.users.get(uid)
    print user.name


if __name__ == '__main__':
    ## login
    gl = gitlab.Gitlab.from_config('gitlab', ['.python-gitlab.cfg'])

    info = {1: 'allprojects()', 2: 'allgroups()', 3: 'allusers()', 4: 'projectinfo()', 5: 'projectid()', 6: 'assuser()',
            7: 'assgroup()'}
    serp = '-' * 20
    print '''%s
    1. 列出所有的projects
    2. 列出所有的groups
    3. 列出所有的users
    4. 根據(jù)project的ID列出project的所有信息
    5. 列出指定的project ID
    6. 列出指定的user
    7. 列出指定的組內(nèi)的信息
    %s''' % (serp, serp)
    num = int(raw_input('Input yout choice: '))
    exec info[num]

下面進(jìn)入mysql存儲(chǔ)過(guò)程的同步，多的話不說(shuō)，直接上代碼; 【這個(gè)是參考別人的文檔所寫(xiě)】
將同步后的文件寫(xiě)入到 /home/hadoop/shell/$db 目錄，用于后續(xù)用python-gitlab上傳

vi export.sh
#!/bin/bash
#set -x


#將mysql中存儲(chǔ)過(guò)程代碼直接導(dǎo)出為文件
dbcn="mysql -hrm-2zera5z4rl,com -udb_user -p*";
db=db_channel;
ii=0;
# 查詢當(dāng)前數(shù)據(jù)庫(kù)中的存儲(chǔ)過(guò)程
ct=`$dbcn -N -e " select count(1) from mysql.proc as p where 1=1 and p.db='$db' and p.type like 'P%';"`;
# 先刪除目錄，然后新建存放目錄
rm -rf /home/hadoop/shell/$db
mkdir -p /home/hadoop/shell/$db

 
while true;
do
if [ $ii -lt $ct ]
then 
p=$ii;
let ii++;

# 通過(guò)序號(hào)獲取每個(gè)存儲(chǔ)過(guò)程的名稱 
spname=`$dbcn -N -e " select p.name from mysql.proc as p where 1=1 and p.db='$db' and p.type like 'P%' limit $p,1;"`;

sleep 0;
ss=`$dbcn -N -e " 
SELECT   
CONCAT(
'
CREATE DEFINER=',CHAR(96),
LEFT(DEFINER, -1+LOCATE('@',DEFINER)),
CHAR(96),'@',CHAR(96),
RIGHT(DEFINER,LENGTH(DEFINER)-LOCATE('@',DEFINER)),
CHAR(96)
,' PROCEDURE ',CHAR(96),p.name ,CHAR(96),'(',p.param_list,')
', p.body ,REPEAT(CHAR(36),2)) AS sql_create 
FROM mysql.proc AS p 
WHERE 1=1 
and p.db='$db' 
and p.type LIKE 'P%'
AND p.name ='$spname'
;"
`
echo -e "$ss" > /home/hadoop/shell/$db/$spname.sql
if [ $? -ne 0 ]; then
    echo 'fail'
        exit 0;
else 
    echo 'success'
fi

# 循環(huán)結(jié)束
else 
echo '_while finished';
exit 0;
fi
done

用于將存儲(chǔ)過(guò)程以及python腳本&shell腳本上傳到服務(wù)器

# coding:utf-8
'''
用于上傳存儲(chǔ)過(guò)程以及本地文件到gitlab中
'''

import gitlab
import subprocess
import json
import requests
import time
import os


def projectinfo(pid):
    projects = gl.projects.get(pid)
    return projects
    # print projects.name, projects.http_url_to_repo


# 獲得project下單個(gè)文件   README.md
def getContent(pid):
    projects = projectinfo(pid)
    # 獲得文件
    f = projects.files.get(file_path='code/shell/export.sh', ref='master')
    # 第一次decode獲得bytes格式的內(nèi)容
    content = f.decode()
    # # 存到本地
    with open('export.sh', 'wb') as code:
        code.write(content)


def uploadeFile(pid):
    projects = projectinfo(pid)
    # projects.upload("export.sh", filepath="xxxxxx")
    # 這個(gè)方法上傳文件限制很大 ， file_path 文件名   f.content 用于生成文件內(nèi)容 ;一定要用 base64 
    f = projects.files.create({'file_path': 'export.sh',
                               'branch': 'master',
                               'content': 'test',
                               'author_email': 'xxx',
                               'author_name': 'xxx',
                               'encoding': 'base64',
                               'commit_message': 'Create export.sh'})
    # 將數(shù)據(jù)寫(xiě)入
    f.content = open('insert_channel_basic.sql').read()
    f.save(branch='master', commit_message='Update testfile')


# 【實(shí)際中用到這個(gè)方法】多文件提交參考 : https://docs.gitlab.com/ce/api/commits.html#create-a-commit-with-multiple-files-and-actions
def commit_file(projects, data):
    projects.commits.create(data)


def judgeType(projects, file_path):
    try:
        projects.files.get(file_path=file_path, ref='master')
        return True
    except Exception as e:
        return False


# jMap 用于生成最后的對(duì)象; path 文件路徑;  jsonList用于生成中間對(duì)象
def getData(path, projects, gitlab_source_dir, message):
    jsonList = []
    jMap = {}
    jMap["branch"] = 'master'
    jMap["commit_message"] = message
    for file in os.listdir(path):
        file_dir = os.path.join(path, file)
        # 判斷當(dāng)前路徑是文件還是目錄 ,排除 pyc造成的編譯問(wèn)題
        if os.path.isfile(file_dir) and 'pyc' not in file:
            gitlab_dir = '%s%s' % (gitlab_source_dir, file)
            aMap = {}
            # 存在就更新， 不存在就創(chuàng)建
            if judgeType(projects, gitlab_dir):
                aMap["action"] = "update"
            else:
                aMap["action"] = "create"
            aMap["file_path"] = gitlab_dir
            aMap["content"] = open(file_dir).read()
            jsonList.append(aMap)
    jMap["actions"] = jsonList
    return jMap


def run_it(cmd):
    '''
    通過(guò)python執(zhí)行shell命令
    :param cmd: sh /home/hadoop/shell/export.sh
    '''
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True,
                         stderr=subprocess.PIPE)
    # print ('running:%s' % cmd)
    out, err = p.communicate()
    if p.returncode != 0:
        print("Non zero exit code:%s executing: %s \nerr course ---> %s" % (p.returncode, cmd, err))
        # 如果出錯(cuò)直接告警
        raise RuntimeError('導(dǎo)出存儲(chǔ)過(guò)程失敗')


if __name__ == '__main__':
    # 用于釘釘告警
    url = "xxxx"
    header = {
        "Content-Type": "application/json",
        "charset": "utf-8"
    }

    ## login
    gl = gitlab.Gitlab.from_config('gitlab', ['.python-gitlab.cfg'])

    # 用于將gitlab上面文件寫(xiě)入本地
    # getContent(87)

    # 1. 將mysql存儲(chǔ)過(guò)程通過(guò)shell腳本寫(xiě)入到指定目錄
    try:
        cmd = 'sh /home/hadoop/shell/export.sh'
        path = '/home/hadoop/shell/db_channel'
        pid = 92
        run_it(cmd)
        # 獲取對(duì)應(yīng)project 對(duì)象
        projects = projectinfo(pid)
        # 將目錄下的文件循環(huán)讀取，然后批量提交到gitlab中
        # dir /home/hadoop/shell/db_channel
        # 用于存放拼接好的對(duì)象, 如果文件最開(kāi)始不存在，用update 會(huì)報(bào)錯(cuò)
        jsonList = []
        jMap = {}
        jMap["branch"] = 'master'
        jMap["commit_message"] = 'update production sql'
        for file in os.listdir(path):
            file_dir = os.path.join(path, file)
            gitlab_dir = '%s%s' % ("code/production/", file)
            aMap = {}
            # 存在就更新， 不存在就創(chuàng)建
            if judgeType(projects, gitlab_dir):
                aMap["action"] = "update"
            else:
                aMap["action"] = "create"
            aMap["file_path"] = gitlab_dir
            aMap["content"] = open(file_dir).read()
            jsonList.append(aMap)
        jMap["actions"] = jsonList
        # 用于存儲(chǔ)過(guò)程的同步
        commit_file(projects, jMap)

        # 2. 用于shell腳本的同步
        pathShell = "/home/hadoop/shell"
        gitlab_source_dirShell = "code/shell/"
        message_shell = "update shell file"
        shell_data = getData(pathShell, projects, gitlab_source_dirShell, message_shell)
        commit_file(projects, shell_data)

        # 3. 用于上傳python 腳本， 如果之后有新的路徑，只需要添加進(jìn)list中
        pythonList = ['/home/hadoop/py']
        for pathPython in pythonList:
            gitlab_source_dirPython = "code/Python/"
            message_python = "update python file"
            python_data = getData(pathPython, projects, gitlab_source_dirPython, message_python)
            commit_file(projects, python_data)

    except Exception as e:
        nowTime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
        data = {
            "msgtype": "text",
            "text": {
                "content": "【Fail:gitlab自動(dòng)同步失敗】:【" + nowTime + "】\n" + "【錯(cuò)誤原因】:\n" + str(e)
            }
        }
        sendData = json.dumps(data).encode("utf-8")
        requests.post(url=url, data=sendData, headers=header)