From bf753e1255331c3a3dc2e10bb1da03558f309056 Mon Sep 17 00:00:00 2001 From: songtianlun Date: Fri, 3 Jan 2025 15:36:19 +0800 Subject: [PATCH] feat: add GitHub to Gitea mirror script - Create a new shell script for mirroring GitHub repositories to Gitea. - Implement configuration options for GitHub and Gitea credentials. - Add functionality to skip specific repositories during synchronization. - Include error handling and logging for better debugging. - Provide detailed usage instructions and environment variable setup. This commit introduces a new feature that automates the process of mirroring GitHub repositories to a Gitea instance. It supports batch synchronization, allows skipping certain repositories, and includes a robust error handling mechanism. The script is designed to be used with crontab for scheduled execution, making it suitable for regular backups of GitHub repositories to Gitea. --- REAEME.md | 126 +++++++++++++++++++++++++++++++++++ mirror.sh | 193 ++++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 300 insertions(+), 19 deletions(-) create mode 100644 REAEME.md diff --git a/REAEME.md b/REAEME.md new file mode 100644 index 0000000..6832a29 --- /dev/null +++ b/REAEME.md @@ -0,0 +1,126 @@ +# GitHub to Gitea Mirror Script + +这是一个自动将 GitHub 仓库镜像到 Gitea 的 Shell 脚本。支持批量同步所有仓库，可以设置跳过特定仓库，并具有良好的错误处理机制。 + +## 功能特性 + +- 自动同步 GitHub 所有仓库到 Gitea +- 支持设置跳过特定仓库 +- 分级推送策略（先尝试 mirror，失败后逐个推送分支） +- 详细的进度显示和错误提示 +- 支持通过环境变量配置 +- 适合配合 crontab 使用 + +## 必要条件 + +- Git +- curl +- jq +- GitHub Token（如需访问私有仓库） +- Gitea Token + +## 环境变量 + +| 变量名 | 必需 | 说明 | 示例 | +|--------|------|------|------| +| GITHUB_USER | 是 | GitHub 用户名 | `songtianlun` | +| GITHUB_TOKEN | 否 | GitHub 访问令牌 | `ghp_xxxxxxxxxxxx` | +| GITEA_URL | 是 | Gitea 实例地址 | `https://git.example.com` | +| GITEA_USER | 是 | Gitea 用户名 | `username` | +| GITEA_TOKEN | 是 | Gitea 访问令牌 | `d4209xxxxxxxxxxxxx` | +| SKIP_REPOS | 否 | 跳过的仓库列表（逗号分隔） | `repo1,repo2,repo3` | +| WORK_DIR | 否 | 临时工作目录 | `/tmp/git-mirror` | + +## 使用方法 + +### 1. 直接运行 + +```bash +GITHUB_USER=username \ +GITHUB_TOKEN=your-github-token \ +GITEA_URL=https://git.example.com \ +GITEA_USER=username \ +GITEA_TOKEN=your-gitea-token \ +bash mirror.sh +``` + +### 2. 配置环境变量后运行 + +```bash +# 设置环境变量 +export GITHUB_USER=username +export GITHUB_TOKEN=your-github-token +export GITEA_URL=https://git.example.com +export GITEA_USER=username +export GITEA_TOKEN=your-gitea-token + +# 运行脚本 +bash mirror.sh +``` + +### 3. 设置定时任务 + +编辑 crontab： +```bash +crontab -e +``` + +添加定时任务（每天凌晨 2 点运行）： +```cron +0 2 * * * GITHUB_USER=username GITHUB_TOKEN=xxx GITEA_URL=https://git.example.com GITEA_USER=username GITEA_TOKEN=xxx /path/to/mirror.sh >> /path/to/mirror.log 2>&1 +``` + +### 4. 跳过特定仓库 + +```bash +GITHUB_USER=username \ +GITEA_URL=https://git.example.com \ +GITEA_USER=username \ +GITEA_TOKEN=xxx \ +SKIP_REPOS="repo1,repo2,repo3" \ +bash mirror.sh +``` + +## 常见问题 + +1. **获取 GitHub Token** + - 访问 GitHub Settings -> Developer settings -> Personal access tokens + - 创建新的 Token，至少需要 `repo` 权限 + +2. **获取 Gitea Token** + - 访问 Gitea 设置 -> 应用 -> 创建新的令牌 + - 需要仓库的读写权限 + +3. **日志查看** + ```bash + # 如果配置了日志输出 + tail -f /path/to/mirror.log + ``` + +4. **错误处理** + - 检查令牌权限是否正确 + - 确保 Gitea 实例可访问 + - 验证用户名和 URL 是否正确 + +## 调试模式 + +添加 `-x` 参数启用调试模式： +```bash +bash -x mirror.sh +``` + +## 注意事项 + +- 建议使用专门的目录存放脚本和日志 +- 定期检查日志确保同步正常 +- 谨慎保管 Token，不要泄露 +- 建议先使用测试账号验证配置 +- 大型仓库同步可能需要较长时间 + +## License + +MIT License + +## 贡献 + +欢迎提交 Issue 和 Pull Request！ \ No newline at end of file diff --git a/mirror.sh b/mirror.sh index 6ba3224..b97fd62 100644 --- a/mirror.sh +++ b/mirror.sh @@ -7,14 +7,78 @@ YELLOW='\033[1;33m' NC='\033[0m' # No Color # 配置信息 -GITHUB_USER="your-github-username" -GITHUB_TOKEN="your-github-token" # 如果需要访问私有仓库 -GITEA_URL="https://your-gitea-instance" -GITEA_USER="your-gitea-username" -GITEA_TOKEN="your-gitea-token" +GITHUB_USER=${GITHUB_USER:-"default-github-username"} +GITHUB_TOKEN=${GITHUB_TOKEN:-""} # 可选的 GitHub Token +GITEA_URL=${GITEA_URL:-"https://your-gitea-instance"} +GITEA_USER=${GITEA_USER:-"default-gitea-username"} +GITEA_TOKEN=${GITEA_TOKEN:-"default-gitea-token"} +cur_path=$(readlink -f $(dirname $0)) # 工作目录 -WORK_DIR="/tmp/git-mirror" +WORK_DIR=${cur_path}/"./tmp" + +# 跳过的仓库列表（逗号分隔） +SKIP_REPOS=${SKIP_REPOS:-"archive,AutoApiSecret, \ + backup-openbilibili-go-common, \ + carrot,ChatGLM-6B,dokploy,hub-mirror, \ + Download-macOS, \ + songtianlun,songtianlun.github.io"} + + +# 将跳过的仓库字符串转换为数组 +IFS=',' read -ra SKIP_REPOS_ARRAY <<< "$SKIP_REPOS" +declare -A SKIP_REPOS_MAP # 声明关联数组 + +# 将跳过的仓库添加到映射中，以便快速查找 +for repo in "${SKIP_REPOS_ARRAY[@]}"; do + # 去除可能存在的空格 + repo=$(echo "$repo" | tr -d ' ') + if [ -n "$repo" ]; then + SKIP_REPOS_MAP[$repo]=1 + fi +done + +# 检查仓库是否在跳过列表中 +is_repo_skipped() { + local repo_name="$1" + [[ -n "${SKIP_REPOS_MAP[$repo_name]}" ]] +} + +# 显示配置信息 +show_config() { + echo -e "${BLUE}当前配置信息:${NC}" + echo -e "GitHub 用户: ${YELLOW}$GITHUB_USER${NC}" + echo -e "GitHub Token: ${YELLOW}${GITHUB_TOKEN:+已设置}${NC}" + echo -e "Gitea URL: ${YELLOW}$GITEA_URL${NC}" + echo -e "Gitea 用户: ${YELLOW}$GITEA_USER${NC}" + echo -e "工作目录: ${YELLOW}$WORK_DIR${NC}" + if [ ${#SKIP_REPOS_ARRAY[@]} -gt 0 ]; then + echo -e "跳过的仓库: ${YELLOW}${SKIP_REPOS}${NC}" + fi + echo +} + +# 检查必要的配置 +check_required_vars() { + local missing_vars=() + + [ "$GITHUB_USER" = "default-github-username" ] && missing_vars+=("GITHUB_USER") + [ "$GITEA_URL" = "https://your-gitea-instance" ] && missing_vars+=("GITEA_URL") + [ "$GITEA_USER" = "default-gitea-username" ] && missing_vars+=("GITEA_USER") + [ "$GITEA_TOKEN" = "default-gitea-token" ] && missing_vars+=("GITEA_TOKEN") + + if [ ${#missing_vars[@]} -ne 0 ]; then + echo -e "${RED}错误: 以下必需的环境变量未设置:${NC}" + printf '%s\n' "${missing_vars[@]}" + echo -e "\n${YELLOW}请设置环境变量后再运行，例如：${NC}" + echo "export GITHUB_USER=your-username" + echo "export GITEA_URL=https://your-gitea-instance" + echo "export GITEA_USER=your-gitea-username" + echo "export GITEA_TOKEN=your-gitea-token" + echo "export SKIP_REPOS=repo1,repo2,repo3" # 可选 + exit 1 + fi +} # 错误处理函数 error_exit() { @@ -22,6 +86,10 @@ error_exit() { exit 1 } +# 检查配置 +check_required_vars +show_config + # 检查必要的命令是否存在 command -v git >/dev/null 2>&1 || error_exit "需要安装 git" command -v curl >/dev/null 2>&1 || error_exit "需要安装 curl" @@ -33,23 +101,68 @@ cd "$WORK_DIR" || error_exit "无法进入工作目录" # 获取所有 GitHub 仓库列表 echo -e "${YELLOW}正在获取 GitHub 仓库列表...${NC}" -if [ -n "$GITHUB_TOKEN" ]; then - repos=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \ - "https://api.github.com/users/$GITHUB_USER/repos?per_page=100" | \ - jq -r '.[].name') -else - repos=$(curl -s "https://api.github.com/users/$GITHUB_USER/repos?per_page=100" | \ - jq -r '.[].name') -fi +# 检查 API 限制 +#if [ -n "$GITHUB_TOKEN" ]; then +# rate_limit=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \ +# "https://api.github.com/rate_limit") +# echo "API 限制信息：" +# echo "$rate_limit" | jq . +#fi +all_repos="" +page=1 +# 在获取仓库列表的循环中添加调试信息 +while true; do + #echo "获取第 $page 页的仓库..." + if [ -n "$GITHUB_TOKEN" ]; then + page_repos=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \ + "https://api.github.com/user/repos?per_page=100&page=$page&type=all" | \ + tee /tmp/github-response-$page.json | \ + jq -r '.[].name') + else + page_repos=$(curl -s \ + "https://api.github.com/user/repos?per_page=100&page=$page&type=all" | \ + tee /tmp/github-response-$page.json | \ + jq -r '.[].name') + fi + + # 显示获取到的仓库数量 + #count=$(echo "$page_repos" | grep -v '^$' | wc -l) + #echo "第 $page 页获取到 $count 个仓库" + + if [ -z "$page_repos" ] || [ "$page_repos" = "null" ]; then + # echo "没有更多仓库，退出循环" + break + fi + + all_repos="${all_repos}${page_repos}\n" + ((page++)) +done + +# 移除多余的空行并存储到 repos 变量 +repos=$(echo -e "$all_repos" | grep -v '^$') [ -z "$repos" ] && error_exit "无法获取仓库列表" +# 显示获取到的总仓库数 +total_repos=$(echo "$repos" | wc -l) +echo -e "${GREEN}总共获取到 $total_repos 个仓库${NC}" + # 计数器 total=$(echo "$repos" | wc -l) current=0 +skipped=0 +processed=0 for repo in $repos; do ((current++)) + + # 检查是否跳过该仓库 + if is_repo_skipped "$repo"; then + echo -e "\n${BLUE}跳过仓库 ($current/$total): $repo${NC}" + ((skipped++)) + continue + fi + echo -e "\n${YELLOW}处理仓库 ($current/$total): $repo${NC}" # 检查 Gitea 仓库是否存在 @@ -71,27 +184,69 @@ for repo in $repos; do # 克隆 GitHub 仓库 echo "克隆 GitHub 仓库..." if [ -n "$GITHUB_TOKEN" ]; then - git clone --mirror "https://$GITHUB_TOKEN@github.com/$GITHUB_USER/$repo.git" "$repo" || \ + git clone -q --mirror "https://$GITHUB_TOKEN@github.com/$GITHUB_USER/$repo.git" "$repo" || \ error_exit "无法克隆仓库 $repo" else - git clone --mirror "https://github.com/$GITHUB_USER/$repo.git" "$repo" || \ + git clone -q --mirror "https://github.com/$GITHUB_USER/$repo.git" "$repo" || \ error_exit "无法克隆仓库 $repo" fi cd "$repo" || error_exit "无法进入仓库目录 $repo" + # 确保获取所有分支和标签 + git fetch --all --tags + # 推送到 Gitea echo "推送到 Gitea..." - git push --mirror "https://$GITEA_USER:$GITEA_TOKEN@${GITEA_URL#https://}/$GITEA_USER/$repo.git" || \ - error_exit "无法推送到 Gitea 仓库 $repo" + # 尝试 mirror 推送 + if git push -q --mirror "https://$GITEA_USER:$GITEA_TOKEN@${GITEA_URL#https://}/$GITEA_USER/$repo.git"; then + echo "mirror 推送成功" + else + echo "mirror 推送失败，尝试逐个分支推送..." + + # 获取所有远程分支，去除 'origin/' 前缀 + branches=$(git branch -r | grep -v '\->' | sed 's/origin\///') + push_failed=false + + # 逐个推送分支 + for branch in $branches; do + echo "推送分支: $branch" + if ! git push -q "https://$GITEA_USER:$GITEA_TOKEN@${GITEA_URL#https://}/$GITEA_USER/$repo.git" "origin/$branch:$branch"; then + echo "警告: 推送分支 $branch 失败" + push_failed=true + fi + done + + # 推送所有标签 + tags=$(git tag) + if [ -n "$tags" ]; then + echo "推送标签..." + for tag in $tags; do + if ! git push -q "https://$GITEA_USER:$GITEA_TOKEN@${GITEA_URL#https://}/$GITEA_USER/$repo.git" "refs/tags/$tag"; then + echo "警告: 推送标签 $tag 失败" + push_failed=true + fi + done + fi + + # 如果有任何分支或标签推送失败，抛出错误 + if [ "$push_failed" = true ]; then + error_exit "部分分支或标签推送失败，请检查日志" + fi + fi + cd "$WORK_DIR" || error_exit "无法返回工作目录" rm -rf "$repo" echo -e "${GREEN}成功同步仓库: $repo${NC}" + ((processed++)) done -echo -e "\n${GREEN}所有仓库同步完成!${NC}" +echo -e "\n${GREEN}同步完成!${NC}" +echo -e "处理总数: $current" +echo -e "成功同步: $processed" +echo -e "跳过数量: $skipped" # 清理工作目录 cd / && rm -rf "$WORK_DIR" \ No newline at end of file