From 534a9ab0e4f89fa51f8540c74c0aa67c3026ac45 Mon Sep 17 00:00:00 2001 From: songtianlun Date: Fri, 3 Jan 2025 17:03:01 +0800 Subject: [PATCH] feat: add email notification and logging features - Introduce email notification configuration options in the README - Create a new config.sh file for managing environment variables - Implement mail.sh for sending email notifications - Add logging functionality to main.sh for tracking synchronization processes These changes enhance the functionality of the script by allowing users to receive email notifications about the synchronization status and maintain logs of the operations performed. The new configuration options provide flexibility in managing email settings and improve the overall usability of the tool. --- REAEME.md | 16 ++++ config.sh | 30 ++++++ mail.sh | 43 +++++++++ main.sh | 86 +++++++++++++++++ mirror.sh | 271 ++++++++++-------------------------------------------- 5 files changed, 222 insertions(+), 224 deletions(-) create mode 100644 config.sh create mode 100644 mail.sh create mode 100644 main.sh diff --git a/REAEME.md b/REAEME.md index 6832a29..3325f58 100644 --- a/REAEME.md +++ b/REAEME.md @@ -30,6 +30,22 @@ | GITEA_TOKEN | 是 | Gitea 访问令牌 | `d4209xxxxxxxxxxxxx` | | SKIP_REPOS | 否 | 跳过的仓库列表（逗号分隔） | `repo1,repo2,repo3` | | WORK_DIR | 否 | 临时工作目录 | `/tmp/git-mirror` | +| ENABLE_MAIL | 否 | 是否启用邮件通知 | `true` 或 `false` | `false` | +| SMTP_SERVER | 否 | SMTP 服务器地址 | `smtp.gmail.com` | - | +| SMTP_PORT | 否 | SMTP 端口 | `587` | `587` | +| SMTP_USER | 否 | SMTP 用户名 | `your-email@gmail.com` | - | +| SMTP_PASS | 否 | SMTP 密码 | `your-password` | - | +| MAIL_TO | 否 | 接收通知的邮箱 | `your-email@example.com` | - | +| MAIL_FROM | 否 | 发件人地址 | `noreply@example.com` | `$SMTP_USER` | + +## 日志文件 + +脚本会自动创建日志文件，包含完整的运行记录： + +- 默认日志目录：`/tmp/github-mirror-logs` +- 日志文件名格式：`mirror-YYYYMMDD-HHMMSS.log` +- 每次运行创建新的日志文件 +- 邮件通知中包含日志最后 50 行 ## 使用方法 diff --git a/config.sh b/config.sh new file mode 100644 index 0000000..1badea1 --- /dev/null +++ b/config.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# GitHub 配置 +GITHUB_USER=${GITHUB_USER:-"default-github-username"} +GITHUB_TOKEN=${GITHUB_TOKEN:-""} + +# Gitea 配置 +GITEA_URL=${GITEA_URL:-"https://your-gitea-instance"} +GITEA_USER=${GITEA_USER:-"default-gitea-username"} +GITEA_TOKEN=${GITEA_TOKEN:-"default-gitea-token"} + +# 工作目录配置 +WORK_DIR=${WORK_DIR:-"/tmp/github-mirror"} +LOG_DIR=${LOG_DIR:-"$WORK_DIR/logs"} + +# 邮件配置 +ENABLE_MAIL=${ENABLE_MAIL:-"false"} +SMTP_SERVER=${SMTP_SERVER:-""} +SMTP_PORT=${SMTP_PORT:-"587"} +SMTP_USER=${SMTP_USER:-""} +SMTP_PASS=${SMTP_PASS:-""} +MAIL_TO=${MAIL_TO:-""} +MAIL_FROM=${MAIL_FROM:-"$SMTP_USER"} + +# 跳过的仓库 +SKIP_REPOS=${SKIP_REPOS:-"archive,AutoApiSecret, \ + backup-openbilibili-go-common, \ + carrot,ChatGLM-6B,dokploy,hub-mirror, \ + Download-macOS, \ + songtianlun,songtianlun.github.io"} \ No newline at end of file diff --git a/mail.sh b/mail.sh new file mode 100644 index 0000000..ad00fda --- /dev/null +++ b/mail.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# 参数获取 +SMTP_SERVER="$1" +SMTP_PORT="$2" +SMTP_USER="$3" +SMTP_PASS="$4" +MAIL_TO="$5" +MAIL_FROM="$6" +SUBJECT="$7" +BODY="$8" + +# 发送邮件 +send_mail() { + local email_content="Subject: $SUBJECT +From: $MAIL_FROM +To: $MAIL_TO +Content-Type: text/plain; charset=UTF-8 +Date: $(date -R) + +$BODY" + + curl -s --url "smtps://$SMTP_SERVER:$SMTP_PORT" \ + --mail-from "$MAIL_FROM" \ + --mail-rcpt "$MAIL_TO" \ + --upload-file - \ + --ssl-reqd \ + --user "$SMTP_USER:$SMTP_PASS" \ + <<< "$email_content" +} + +# 主函数 +main() { + # 验证必要参数 + if [ -z "$SMTP_SERVER" ] || [ -z "$SMTP_USER" ] || [ -z "$SMTP_PASS" ] || [ -z "$MAIL_TO" ]; then + echo "错误: 缺少必要的邮件配置" + exit 1 + fi + + send_mail +} + +main "$@" \ No newline at end of file diff --git a/main.sh b/main.sh new file mode 100644 index 0000000..10bfcfa --- /dev/null +++ b/main.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +# 获取脚本所在目录的绝对路径 +SCRIPT_DIR=$(readlink -f $(dirname $0)) + +# 加载配置 +source "$SCRIPT_DIR/config.sh" + +# 颜色定义 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +# 初始化日志 +init_logging() { + mkdir -p "$LOG_DIR" + LOG_FILE="$LOG_DIR/mirror-$(date '+%Y%m%d-%H%M%S').log" + exec 1> >(tee -a "$LOG_FILE") + exec 2> >(tee -a "$LOG_FILE" >&2) +} + +# 日志函数 +log() { + local message="$1" + local timestamp=$(date '+%Y-%m-%d %H:%M:%S') + echo -e "$timestamp $message" +} + +# 检查必要的命令 +check_requirements() { + command -v git >/dev/null 2>&1 || error_exit "需要安装 git" + command -v curl >/dev/null 2>&1 || error_exit "需要安装 curl" + command -v jq >/dev/null 2>&1 || error_exit "需要安装 jq" +} + +# 主函数 +main() { + init_logging + check_requirements + + log "开始同步处理..." + + # 调用 mirror.sh 进行同步 + true || bash "$SCRIPT_DIR/mirror.sh" \ + "$GITHUB_USER" \ + "$GITHUB_TOKEN" \ + "$GITEA_URL" \ + "$GITEA_USER" \ + "$GITEA_TOKEN" \ + "$WORK_DIR" \ + "$SKIP_REPOS" + + mirror_exit_code=$? + + # 准备邮件内容 + summary="GitHub to Gitea 同步报告 + +运行时间: $(date '+%Y-%m-%d %H:%M:%S') +同步状态: $([ $mirror_exit_code -eq 0 ] && echo "成功" || echo "失败") + +详细日志: +$(cat "${LOG_FILE}")" + + # 如果启用了邮件通知，调用 mail.sh + if [ "$ENABLE_MAIL" = "true" ]; then + subject="GitHub 同步$([ $mirror_exit_code -eq 0 ] && echo "成功" || echo "失败") - $(date '+%Y-%m-%d')" + + bash "$SCRIPT_DIR/mail.sh" \ + "$SMTP_SERVER" \ + "$SMTP_PORT" \ + "$SMTP_USER" \ + "$SMTP_PASS" \ + "$MAIL_TO" \ + "$MAIL_FROM" \ + "$subject" \ + "$summary" + fi + + # 清理工作目录 + [ -d "$WORK_DIR" ] && rm -rf "$WORK_DIR" + + exit $mirror_exit_code +} + +main "$@" \ No newline at end of file diff --git a/mirror.sh b/mirror.sh index b97fd62..2f6a0c3 100644 --- a/mirror.sh +++ b/mirror.sh @@ -1,252 +1,75 @@ #!/bin/bash -# 颜色定义 -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color +# 参数获取 +GITHUB_USER="$1" +GITHUB_TOKEN="$2" +GITEA_URL="$3" +GITEA_USER="$4" +GITEA_TOKEN="$5" +WORK_DIR="$6" +SKIP_REPOS="$7" -# 配置信息 -GITHUB_USER=${GITHUB_USER:-"default-github-username"} -GITHUB_TOKEN=${GITHUB_TOKEN:-""} # 可选的 GitHub Token -GITEA_URL=${GITEA_URL:-"https://your-gitea-instance"} -GITEA_USER=${GITEA_USER:-"default-gitea-username"} -GITEA_TOKEN=${GITEA_TOKEN:-"default-gitea-token"} - -cur_path=$(readlink -f $(dirname $0)) -# 工作目录 -WORK_DIR=${cur_path}/"./tmp" - -# 跳过的仓库列表（逗号分隔） -SKIP_REPOS=${SKIP_REPOS:-"archive,AutoApiSecret, \ - backup-openbilibili-go-common, \ - carrot,ChatGLM-6B,dokploy,hub-mirror, \ - Download-macOS, \ - songtianlun,songtianlun.github.io"} - - -# 将跳过的仓库字符串转换为数组 -IFS=',' read -ra SKIP_REPOS_ARRAY <<< "$SKIP_REPOS" -declare -A SKIP_REPOS_MAP # 声明关联数组 - -# 将跳过的仓库添加到映射中，以便快速查找 -for repo in "${SKIP_REPOS_ARRAY[@]}"; do - # 去除可能存在的空格 - repo=$(echo "$repo" | tr -d ' ') - if [ -n "$repo" ]; then - SKIP_REPOS_MAP[$repo]=1 - fi -done - -# 检查仓库是否在跳过列表中 -is_repo_skipped() { - local repo_name="$1" - [[ -n "${SKIP_REPOS_MAP[$repo_name]}" ]] -} - -# 显示配置信息 -show_config() { - echo -e "${BLUE}当前配置信息:${NC}" - echo -e "GitHub 用户: ${YELLOW}$GITHUB_USER${NC}" - echo -e "GitHub Token: ${YELLOW}${GITHUB_TOKEN:+已设置}${NC}" - echo -e "Gitea URL: ${YELLOW}$GITEA_URL${NC}" - echo -e "Gitea 用户: ${YELLOW}$GITEA_USER${NC}" - echo -e "工作目录: ${YELLOW}$WORK_DIR${NC}" - if [ ${#SKIP_REPOS_ARRAY[@]} -gt 0 ]; then - echo -e "跳过的仓库: ${YELLOW}${SKIP_REPOS}${NC}" - fi - echo -} - -# 检查必要的配置 -check_required_vars() { - local missing_vars=() - - [ "$GITHUB_USER" = "default-github-username" ] && missing_vars+=("GITHUB_USER") - [ "$GITEA_URL" = "https://your-gitea-instance" ] && missing_vars+=("GITEA_URL") - [ "$GITEA_USER" = "default-gitea-username" ] && missing_vars+=("GITEA_USER") - [ "$GITEA_TOKEN" = "default-gitea-token" ] && missing_vars+=("GITEA_TOKEN") - - if [ ${#missing_vars[@]} -ne 0 ]; then - echo -e "${RED}错误: 以下必需的环境变量未设置:${NC}" - printf '%s\n' "${missing_vars[@]}" - echo -e "\n${YELLOW}请设置环境变量后再运行，例如：${NC}" - echo "export GITHUB_USER=your-username" - echo "export GITEA_URL=https://your-gitea-instance" - echo "export GITEA_USER=your-gitea-username" - echo "export GITEA_TOKEN=your-gitea-token" - echo "export SKIP_REPOS=repo1,repo2,repo3" # 可选 - exit 1 - fi -} - -# 错误处理函数 -error_exit() { - echo -e "${RED}错误: $1${NC}" >&2 - exit 1 -} - -# 检查配置 -check_required_vars -show_config - -# 检查必要的命令是否存在 -command -v git >/dev/null 2>&1 || error_exit "需要安装 git" -command -v curl >/dev/null 2>&1 || error_exit "需要安装 curl" -command -v jq >/dev/null 2>&1 || error_exit "需要安装 jq" - -# 创建工作目录 -mkdir -p "$WORK_DIR" || error_exit "无法创建工作目录" -cd "$WORK_DIR" || error_exit "无法进入工作目录" - -# 获取所有 GitHub 仓库列表 -echo -e "${YELLOW}正在获取 GitHub 仓库列表...${NC}" -# 检查 API 限制 -#if [ -n "$GITHUB_TOKEN" ]; then -# rate_limit=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \ -# "https://api.github.com/rate_limit") -# echo "API 限制信息：" -# echo "$rate_limit" | jq . -#fi -all_repos="" -page=1 -# 在获取仓库列表的循环中添加调试信息 -while true; do - #echo "获取第 $page 页的仓库..." - if [ -n "$GITHUB_TOKEN" ]; then - page_repos=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \ - "https://api.github.com/user/repos?per_page=100&page=$page&type=all" | \ - tee /tmp/github-response-$page.json | \ - jq -r '.[].name') - else - page_repos=$(curl -s \ - "https://api.github.com/user/repos?per_page=100&page=$page&type=all" | \ - tee /tmp/github-response-$page.json | \ - jq -r '.[].name') - fi - - # 显示获取到的仓库数量 - #count=$(echo "$page_repos" | grep -v '^$' | wc -l) - #echo "第 $page 页获取到 $count 个仓库" - - if [ -z "$page_repos" ] || [ "$page_repos" = "null" ]; then - # echo "没有更多仓库，退出循环" - break - fi - - all_repos="${all_repos}${page_repos}\n" - ((page++)) -done - -# 移除多余的空行并存储到 repos 变量 -repos=$(echo -e "$all_repos" | grep -v '^$') - -[ -z "$repos" ] && error_exit "无法获取仓库列表" - -# 显示获取到的总仓库数 -total_repos=$(echo "$repos" | wc -l) -echo -e "${GREEN}总共获取到 $total_repos 个仓库${NC}" - -# 计数器 -total=$(echo "$repos" | wc -l) -current=0 -skipped=0 -processed=0 - -for repo in $repos; do - ((current++)) - - # 检查是否跳过该仓库 - if is_repo_skipped "$repo"; then - echo -e "\n${BLUE}跳过仓库 ($current/$total): $repo${NC}" - ((skipped++)) - continue - fi - - echo -e "\n${YELLOW}处理仓库 ($current/$total): $repo${NC}" +# 同步单个仓库 +sync_repository() { + local repo="$1" # 检查 Gitea 仓库是否存在 - if curl -s -I -H "Authorization: token $GITEA_TOKEN" \ - "$GITEA_URL/api/v1/repos/$GITEA_USER/$repo" | \ - grep -q "404 Not Found"; then + if ! curl -s -o /dev/null -f -H "Authorization: token $GITEA_TOKEN" \ + "$GITEA_URL/api/v1/repos/$GITEA_USER/$repo"; then echo "在 Gitea 上创建仓库 $repo" curl -X POST -H "Authorization: token $GITEA_TOKEN" \ -H "Content-Type: application/json" \ -d "{\"name\":\"$repo\",\"private\":false}" \ - "$GITEA_URL/api/v1/user/repos" || \ - error_exit "无法在 Gitea 上创建仓库 $repo" + "$GITEA_URL/api/v1/user/repos" fi - # 如果目录已存在，先删除 + # 克隆和推送 [ -d "$repo" ] && rm -rf "$repo" + git clone --mirror "https://${GITHUB_TOKEN:+$GITHUB_TOKEN@}github.com/$GITHUB_USER/$repo.git" "$repo" + cd "$repo" - # 克隆 GitHub 仓库 - echo "克隆 GitHub 仓库..." - if [ -n "$GITHUB_TOKEN" ]; then - git clone -q --mirror "https://$GITHUB_TOKEN@github.com/$GITHUB_USER/$repo.git" "$repo" || \ - error_exit "无法克隆仓库 $repo" - else - git clone -q --mirror "https://github.com/$GITHUB_USER/$repo.git" "$repo" || \ - error_exit "无法克隆仓库 $repo" - fi - - cd "$repo" || error_exit "无法进入仓库目录 $repo" - - # 确保获取所有分支和标签 - git fetch --all --tags - - # 推送到 Gitea - echo "推送到 Gitea..." # 尝试 mirror 推送 - if git push -q --mirror "https://$GITEA_USER:$GITEA_TOKEN@${GITEA_URL#https://}/$GITEA_USER/$repo.git"; then - echo "mirror 推送成功" - else + if ! git push --mirror "https://$GITEA_USER:$GITEA_TOKEN@${GITEA_URL#https://}/$GITEA_USER/$repo.git"; then echo "mirror 推送失败，尝试逐个分支推送..." - # 获取所有远程分支，去除 'origin/' 前缀 - branches=$(git branch -r | grep -v '\->' | sed 's/origin\///') - push_failed=false + # 获取所有分支 + git fetch --all - # 逐个推送分支 - for branch in $branches; do + # 推送每个分支 + git for-each-ref --format='%(refname:short)' refs/heads/ | while read branch; do echo "推送分支: $branch" - if ! git push -q "https://$GITEA_USER:$GITEA_TOKEN@${GITEA_URL#https://}/$GITEA_USER/$repo.git" "origin/$branch:$branch"; then - echo "警告: 推送分支 $branch 失败" - push_failed=true - fi + git push "https://$GITEA_USER:$GITEA_TOKEN@${GITEA_URL#https://}/$GITEA_USER/$repo.git" "$branch:$branch" done # 推送所有标签 - tags=$(git tag) - if [ -n "$tags" ]; then - echo "推送标签..." - for tag in $tags; do - if ! git push -q "https://$GITEA_USER:$GITEA_TOKEN@${GITEA_URL#https://}/$GITEA_USER/$repo.git" "refs/tags/$tag"; then - echo "警告: 推送标签 $tag 失败" - push_failed=true - fi - done - fi - - # 如果有任何分支或标签推送失败，抛出错误 - if [ "$push_failed" = true ]; then - error_exit "部分分支或标签推送失败，请检查日志" - fi + git push "https://$GITEA_USER:$GITEA_TOKEN@${GITEA_URL#https://}/$GITEA_USER/$repo.git" --tags fi + cd .. +} - cd "$WORK_DIR" || error_exit "无法返回工作目录" - rm -rf "$repo" +# 主同步逻辑 +main() { + mkdir -p "$WORK_DIR" + cd "$WORK_DIR" - echo -e "${GREEN}成功同步仓库: $repo${NC}" - ((processed++)) -done + # 获取仓库列表 + repos=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \ + "https://api.github.com/user/repos?per_page=100&type=all" | \ + jq -r '.[].name') -echo -e "\n${GREEN}同步完成!${NC}" -echo -e "处理总数: $current" -echo -e "成功同步: $processed" -echo -e "跳过数量: $skipped" + # 同步每个仓库 + for repo in $repos; do + # 检查是否跳过 + if echo "$SKIP_REPOS" | grep -q "$repo"; then + echo "跳过仓库: $repo" + continue + fi -# 清理工作目录 -cd / && rm -rf "$WORK_DIR" \ No newline at end of file + echo "处理仓库: $repo" + sync_repository "$repo" + done +} + +main "$@" \ No newline at end of file