From ca6a80da97bca7588bc8c490a24ba345bfaa28ed Mon Sep 17 00:00:00 2001 From: songtianlun Date: Fri, 3 Jan 2025 17:26:32 +0800 Subject: [PATCH] feat: add synchronization statistics logging - Initialize statistics logging in the mirror script - Update logging to include total, processed, skipped, success, and failed repositories - Enhance summary report with detailed statistics after synchronization This commit introduces a comprehensive logging mechanism for the repository synchronization process. It tracks various metrics such as total repositories, processed repositories, and their success or failure status. This enhancement provides better visibility into the synchronization process and helps in debugging and monitoring. --- config.sh | 6 ++- main.sh | 32 ++++++++++++++-- mirror.sh | 107 +++++++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 123 insertions(+), 22 deletions(-) diff --git a/config.sh b/config.sh index 1badea1..caa5443 100644 --- a/config.sh +++ b/config.sh @@ -27,4 +27,8 @@ SKIP_REPOS=${SKIP_REPOS:-"archive,AutoApiSecret, \ backup-openbilibili-go-common, \ carrot,ChatGLM-6B,dokploy,hub-mirror, \ Download-macOS, \ - songtianlun,songtianlun.github.io"} \ No newline at end of file + songtianlun,songtianlun.github.io"} + +# 系统配置 +LOG_FILE="$LOG_DIR/mirror-$(date '+%Y%m%d-%H%M%S').log" +STATS_FILE="$LOG_DIR/sync_stats-$(date '+%Y%m%d-%H%M%S').json" diff --git a/main.sh b/main.sh index 29a2d73..7687f85 100644 --- a/main.sh +++ b/main.sh @@ -15,7 +15,6 @@ NC='\033[0m' # 初始化日志 init_logging() { mkdir -p "$LOG_DIR" - LOG_FILE="$LOG_DIR/mirror-$(date '+%Y%m%d-%H%M%S').log" exec 1> >(tee -a "$LOG_FILE") exec 2> >(tee -a "$LOG_FILE" >&2) } @@ -49,7 +48,8 @@ main() { "$GITEA_USER" \ "$GITEA_TOKEN" \ "$WORK_DIR" \ - "$SKIP_REPOS" + "$SKIP_REPOS" \ + "$STATS_FILE" mirror_exit_code=$? @@ -57,11 +57,37 @@ main() { summary="GitHub to Gitea 同步报告 运行时间: $(date '+%Y-%m-%d %H:%M:%S') -同步状态: $([ $mirror_exit_code -eq 0 ] && echo "成功" || echo "失败") 详细日志: $(cat "${LOG_FILE}")" + if [ -f "$STATS_FILE" ]; then + stats=$(cat "$STATS_FILE") + summary="GitHub to Gitea 同步报告 + + 开始时间: $(echo "$stats" | jq -r '.start_time') + 结束时间: $(echo "$stats" | jq -r '.end_time') + 同步状态: $([ $mirror_exit_code -eq 0 ] && echo "成功" || echo "失败") + + 统计信息: + - 总仓库数: $(echo "$stats" | jq -r '.total_repos') + - 处理数量: $(echo "$stats" | jq -r '.processed') + - 成功数量: $(echo "$stats" | jq -r '.success') + - 失败数量: $(echo "$stats" | jq -r '.failed') + - 跳过数量: $(echo "$stats" | jq -r '.skipped') + + 跳过的仓库: + $(echo "$stats" | jq -r '.details.skipped_repos[]' | sed 's/^/- /') + + 失败的仓库: + $(echo "$stats" | jq -r '.details.failed_repos[]' | sed 's/^/- /') + + 详细日志 (最后 50 行): + $(tail -n 50 "$LOG_FILE")" + else + summary="无法获取同步统计信息" + fi + # 如果启用了邮件通知，调用 mail.sh if [ "$ENABLE_MAIL" = "true" ]; then subject="GitHub 同步$([ $mirror_exit_code -eq 0 ] && echo "成功" || echo "失败") - $(date '+%Y-%m-%d')" diff --git a/mirror.sh b/mirror.sh index 2f6a0c3..a99eeeb 100644 --- a/mirror.sh +++ b/mirror.sh @@ -8,45 +8,101 @@ GITEA_USER="$4" GITEA_TOKEN="$5" WORK_DIR="$6" SKIP_REPOS="$7" +STATS_FILE="$8" + +# 初始化统计 +init_stats() { + cat > "$STATS_FILE" << EOF +{ + "total_repos": 0, + "processed": 0, + "skipped": 0, + "success": 0, + "failed": 0, + "start_time": "$(date '+%Y-%m-%d %H:%M:%S')", + "end_time": "", + "details": { + "skipped_repos": [], + "success_repos": [], + "failed_repos": [] + } +} +EOF +} + +# 更新统计信息 +update_stats() { + local key="$1" + local value="$2" + local type="$3" # 可以是 number 或 array + + if [ "$type" = "array" ]; then + # 添加到数组 + jq --arg key "$key" --arg value "$value" \ + ".details[$key] += [\$value]" "$STATS_FILE" > "$STATS_FILE.tmp" + else + # 更新数值 + jq --arg key "$key" --arg value "$value" \ + ".[$key] = ($value | tonumber)" "$STATS_FILE" > "$STATS_FILE.tmp" + fi + mv "$STATS_FILE.tmp" "$STATS_FILE" +} # 同步单个仓库 sync_repository() { local repo="$1" + local success=true # 检查 Gitea 仓库是否存在 if ! curl -s -o /dev/null -f -H "Authorization: token $GITEA_TOKEN" \ "$GITEA_URL/api/v1/repos/$GITEA_USER/$repo"; then - echo "在 Gitea 上创建仓库 $repo" - curl -X POST -H "Authorization: token $GITEA_TOKEN" \ + if ! curl -X POST -H "Authorization: token $GITEA_TOKEN" \ -H "Content-Type: application/json" \ -d "{\"name\":\"$repo\",\"private\":false}" \ - "$GITEA_URL/api/v1/user/repos" + "$GITEA_URL/api/v1/user/repos"; then + success=false + fi fi # 克隆和推送 [ -d "$repo" ] && rm -rf "$repo" - git clone --mirror "https://${GITHUB_TOKEN:+$GITHUB_TOKEN@}github.com/$GITHUB_USER/$repo.git" "$repo" - cd "$repo" + if ! git clone --mirror "https://${GITHUB_TOKEN:+$GITHUB_TOKEN@}github.com/$GITHUB_USER/$repo.git" "$repo"; then + success=false + else + cd "$repo" - # 尝试 mirror 推送 - if ! git push --mirror "https://$GITEA_USER:$GITEA_TOKEN@${GITEA_URL#https://}/$GITEA_USER/$repo.git"; then - echo "mirror 推送失败，尝试逐个分支推送..." + # 尝试 mirror 推送 + if ! git push --mirror "https://$GITEA_USER:$GITEA_TOKEN@${GITEA_URL#https://}/$GITEA_USER/$repo.git"; then + echo "mirror 推送失败，尝试逐个分支推送..." - # 获取所有分支 - git fetch --all + # 获取所有分支 + git fetch --all - # 推送每个分支 - git for-each-ref --format='%(refname:short)' refs/heads/ | while read branch; do - echo "推送分支: $branch" - git push "https://$GITEA_USER:$GITEA_TOKEN@${GITEA_URL#https://}/$GITEA_USER/$repo.git" "$branch:$branch" - done + # 推送每个分支 + git for-each-ref --format='%(refname:short)' refs/heads/ | while read branch; do + echo "推送分支: $branch" + if ! git push "https://$GITEA_USER:$GITEA_TOKEN@${GITEA_URL#https://}/$GITEA_USER/$repo.git" "$branch:$branch"; then + success=false + fi + done - # 推送所有标签 - git push "https://$GITEA_USER:$GITEA_TOKEN@${GITEA_URL#https://}/$GITEA_USER/$repo.git" --tags + # 推送所有标签 + if ! git push "https://$GITEA_USER:$GITEA_TOKEN@${GITEA_URL#https://}/$GITEA_USER/$repo.git" --tags; then + success=false + fi + fi + cd .. fi - cd .. + # 更新统计 + if [ "$success" = true ]; then + update_stats "success" "$(( $(jq '.success' "$STATS_FILE") + 1 ))" "number" + update_stats "success_repos" "$repo" "array" + else + update_stats "failed" "$(( $(jq '.failed' "$STATS_FILE") + 1 ))" "number" + update_stats "failed_repos" "$repo" "array" + fi } # 主同步逻辑 @@ -54,22 +110,37 @@ main() { mkdir -p "$WORK_DIR" cd "$WORK_DIR" + # 初始化统计 + init_stats + # 获取仓库列表 repos=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \ "https://api.github.com/user/repos?per_page=100&type=all" | \ jq -r '.[].name') + # 更新总仓库数 + total_repos=$(echo "$repos" | wc -l) + update_stats "total_repos" "$total_repos" "number" + # 同步每个仓库 for repo in $repos; do # 检查是否跳过 if echo "$SKIP_REPOS" | grep -q "$repo"; then echo "跳过仓库: $repo" + update_stats "skipped" "$(( $(jq '.skipped' "$STATS_FILE") + 1 ))" "number" + update_stats "skipped_repos" "$repo" "array" continue fi echo "处理仓库: $repo" + update_stats "processed" "$(( $(jq '.processed' "$STATS_FILE") + 1 ))" "number" sync_repository "$repo" done + + # 更新结束时间 + jq --arg time "$(date '+%Y-%m-%d %H:%M:%S')" \ + '.end_time = $time' "$STATS_FILE" > "$STATS_FILE.tmp" + mv "$STATS_FILE.tmp" "$STATS_FILE" } main "$@" \ No newline at end of file