runsisi's

technical notes

镜像 github 项目到本地 gitlab 环境

2019-04-15 runsisi#git

mirror 脚本:

#!/bin/bash -x
# runsisi AT hust.edu.cn

export LANG=C
export LC_ALL=C

# mirror.conf
MIRROR_CONF="mirror.conf"
export HTTPS_PROXY="http://proxy123.example.com:12345"
export HTTP_PROXY=$HTTPS_PROXY
export NO_PROXY="src.example.com"

# parse a string like this:
# src [localdir] mirror [refspec], e.g.,
# https://github.com/ceph/ceph.git ceph http://src.example.com/runsisi/ceph.git --tags master firefly giant hammer
# or
# https://github.com/ceph/ceph.git ceph-ci http://src.example.com/runsisi/ceph.git --tags
# or
# https://gitbhu.com/ceph/ceph.git http://src.example.com/runsisi/ceph.git

do_parse() {
    local line_="$1"
    local i_=0

    line_=$(echo -e $line_ | sed -e 's/^\s*//' -e 's/\s*$//')
    line_=$(echo -e $line_ | sed 's/\s\+/ /g')
    [ ${line_:0:1} = "#" ] && return 1
    ((++i_))
    src=$(echo -e $line_ | cut -d' ' -f $i_)
    ((++i_))
    dir=$(echo -e $line_ | cut -d' ' -f $i_)
    if echo -e $dir | grep -q "^\(https\?\|ftp\|ssh\|file\)://"; then
        ((--i_))
        dir=$(echo -e "$src" | grep -o "[^/]*$")
    fi
    ((++i_))
    mirror=$(echo $line_ | cut -d' ' -f $i_)
    ((++i_))
    refspec=$(echo $line_ | cut -d' ' -f $i_-)
}

do_mirror() {
    src=""
    dir=""
    mirror=""
    refsepc=""

    local dir_=""

    if ! do_parse "$1"; then
        # do nothing
        return 0
    fi

    echo -e "\n- [mirror]: $1"

    # TODO: validate parameters

    # if dir does not exist, then clone it
    if [ ! -d "$dir" ]; then
        git clone --mirror $src $dir
        (
        cd "$dir"
        git remote set-url --push origin "$mirror"
        if [ -n "$refspec" ]; then
            git config remote.origin.mirror false
        fi
        )
    else # if dir does exist, then update it
        (
        cd "$dir"
        git remote update
        )
    fi

    # push to mirror
    (
    cd "$dir"

/usr/bin/expect <<-EOF
    # do not quote $refspec
    spawn git push --force origin $refspec
    expect "Username"
    send "runsisi\r"
    expect "Password"
    send "password-abc\r"
    expect eof
EOF
    )

    # sleep for a while
    sleep 5
}

./clean

export -f do_parse
export -f do_mirror

xargs -a "$MIRROR_CONF" -I line bash -e -c 'do_mirror "$@"' _ line

unset -f do_parse
unset -f do_mirror

echo -e "\n-- OK --"

clean 脚本:

#! /bin/bash

for i in $(ls); do
(
    if [ -d $i ]; then
        cd $i
        rm -f gc.log
        git prune
    fi
)
done

mirror.conf 配置文件:

### ceph core modules ###

# ceph -> runsisi/ceph
https://github.com/ceph/ceph.git runsisi-ceph.git http://src.example.com/runsisi/ceph.git master jewel luminous mimic

# ceph -> runsisi/ceph-dev
https://github.com/ceph/ceph.git runsisi-dev.git http://src.example.com/runsisi/ceph-dev.git --tags master hammer jewel luminous

### submodules ###
# ceph-object-corpus
https://github.com/ceph/ceph-object-corpus.git http://src.example.com/ceph/ceph-object-corpus.git

# civetweb
https://github.com/ceph/civetweb.git http://src.example.com/ceph/civetweb.git

# jerasure
https://github.com/ceph/jerasure.git http://src.example.com/ceph/jerasure.git

# gf-complete
https://github.com/ceph/gf-complete.git http://src.example.com/ceph/gf-complete.git

# rocksdb -> ceph/rocksdb
https://github.com/ceph/rocksdb.git http://src.example.com/ceph/rocksdb.git

# rocksdb -> runsisi/rocksdb
https://github.com/ceph/rocksdb.git runsisi-rocksdb.git http://src.example.com/runsisi/rocksdb.git --tags master luminous

# ceph-erasure-code-corpus
https://github.com/ceph/ceph-erasure-code-corpus.git http://src.example.com/ceph/ceph-erasure-code-corpus.git

# gmock
https://github.com/ceph/gmock.git http://src.example.com/ceph/gmock.git

# isa-l
https://github.com/ceph/isa-l.git http://src.example.com/ceph/isa-l.git

# libs3
https://github.com/ceph/libs3.git http://src.example.com/ceph/libs3.git

# lua
https://github.com/ceph/lua.git http://src.example.com/ceph/lua.git

# dpdk
https://github.com/ceph/dpdk.git http://src.example.com/ceph/dpdk.git

# spdk/dpdk
https://github.com/spdk/dpdk.git spdk-dpdk.git http://src.example.com/spdk/dpdk.git

# spdk
https://github.com/ceph/spdk.git http://src.example.com/ceph/spdk.git

# xxHash
https://github.com/ceph/xxHash.git http://src.example.com/ceph/xxHash.git

# gtest
https://github.com/ceph/gtest.git http://src.example.com/ceph/gtest.git

# googletest
https://github.com/ceph/googletest.git http://src.example.com/ceph/googletest.git

# googletest -> google/google-googletest
https://github.com/google/googletest.git google-googletest.git http://src.example.com/google/googletest.git

# Beast
https://github.com/ceph/Beast.git http://src.example.com/ceph/Beast.git

# zetascale
https://github.com/ceph/zetascale.git http://src.example.com/ceph/zetascale.git

# zstd
https://github.com/facebook/zstd.git http://src.example.com/facebook/zstd.git

# isa-l_crypto
https://github.com/01org/isa-l_crypto.git http://src.example.com/01org/isa-l_crypto.git

# blkin
https://github.com/ceph/blkin.git http://src.example.com/ceph/blkin.git

# rapidjson
https://github.com/ceph/rapidjson.git http://src.example.com/ceph/rapidjson.git

# dmclock -> ceph/dmclock
https://github.com/ceph/dmclock.git http://src.example.com/ceph/dmclock.git

# seastar
https://github.com/ceph/seastar.git http://src.example.com/ceph/seastar.git

# scylladb/dpdk
https://github.com/scylladb/dpdk.git scylladb-dpdk.git http://src.example.com/scylladb/dpdk.git

# fmt
https://github.com/ceph/fmt.git http://src.example.com/ceph/fmt.git

# c-ares
https://github.com/ceph/c-ares.git http://src.example.com/ceph/c-ares.git

# intel-ipsec-mb
https://github.com/spdk/intel-ipsec-mb.git http://src.example.com/spdk/intel-ipsec-mb.git


### others ###
# salt
https://github.com/saltstack/salt.git http://src.example.com/runsisi/salt.git --tags develop 2017.7 2017.7.1

# tcmu-runner
https://github.com/open-iscsi/tcmu-runner.git http://src.example.com/runsisi/tcmu-runner.git --tags master

# proftpd
https://github.com/proftpd/proftpd.git http://src.example.com/runsisi/proftpd.git --tags master

将 mirror 脚本,clean 脚本,以及 mirror.conf 配置文件放在同一个文件夹下,然后通过 cron 定时运行:

0   0 * * * cd /home/runsisi/mirrors && ./mirror > /dev/null 2>&1