diff --git a/backups/daemonset+cronjob/cfg.yml b/backups/daemonset+cronjob/archive-by-minio/cfg.yml similarity index 100% rename from backups/daemonset+cronjob/cfg.yml rename to backups/daemonset+cronjob/archive-by-minio/cfg.yml diff --git a/backups/daemonset+cronjob/archive-by-minio/crontab.yaml b/backups/daemonset+cronjob/archive-by-minio/crontab.yaml new file mode 100644 index 0000000..7fae3f9 --- /dev/null +++ b/backups/daemonset+cronjob/archive-by-minio/crontab.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: backup-system +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: node-backup-job + namespace: backup-system +spec: + # 每天凌晨2点运行 + schedule: "0 2 * * *" + concurrencyPolicy: Forbid + jobTemplate: + spec: + ttlSecondsAfterFinished: 86400 # 1天后删除已完成的任务 + template: + spec: + serviceAccountName: backup-service-account + nodeSelector: + kubernetes.io/hostname: "vkvm-us1" + containers: + - name: backup-trigger + image: bitnami/kubectl:latest + command: + - /bin/sh + - -c + - | + kubectl label daemonset/node-backup-daemon trigger-backup=true --overwrite -n backup-system && \ + sleep 60 && \ + kubectl label daemonset/node-backup-daemon trigger-backup- -n backup-system + restartPolicy: OnFailure diff --git a/backups/daemonset+cronjob/archive-by-minio/daemonset.yaml b/backups/daemonset+cronjob/archive-by-minio/daemonset.yaml new file mode 100644 index 0000000..d73b8a9 --- /dev/null +++ b/backups/daemonset+cronjob/archive-by-minio/daemonset.yaml @@ -0,0 +1,140 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-backup-daemon + namespace: backup-system +spec: + selector: + matchLabels: + app: node-backup + template: + metadata: + labels: + app: node-backup + spec: + nodeSelector: + kubernetes.io/hostname: "vkvm-us1" + containers: + - name: backup-container + image: minio/mc:latest + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - | + # 添加依赖的工具 + apk add --no-cache jq bash findutils tar curl + + # 等待触发备份 + while true; do + if [ "$(curl -s -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \ + https://kubernetes.default.svc/apis/apps/v1/namespaces/backup-system/daemonsets/node-backup-daemon \ + --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt \ + -X GET | jq -r '.metadata.labels["trigger-backup"]')" = "true" ]; then + echo "备份触发，执行备份脚本..." + bash /scripts/backup.sh + echo "备份完成" + fi + date + # 每分钟检查一次 + sleep 60 + done + env: + - name: MINIO_ENDPOINT + valueFrom: + secretKeyRef: + name: minio-credentials + key: endpoint + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: access-key + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: secret-key + - name: MINIO_BUCKET + valueFrom: + secretKeyRef: + name: minio-credentials + key: bucket + - name: MINIO_SUBPATH + valueFrom: + configMapKeyRef: + name: backup-config + key: subpath + optional: true + - name: BACKUPS_TO_KEEP + valueFrom: + configMapKeyRef: + name: backup-config + key: backups-to-keep + optional: true + volumeMounts: + - name: host-data + mountPath: /data + - name: scripts + mountPath: /scripts + volumes: + - name: host-data + hostPath: + path: /data + - name: scripts + configMap: + name: backup-script + defaultMode: 0755 +--- +apiVersion: v1 +kind: Secret +metadata: + name: minio-credentials + namespace: backup-system +type: Opaque +data: + # 这些值需要使用base64编码替换 + endpoint: aHR0cHM6Ly9hcGkubWluaW8uc2t5Ynl0ZS5tZQ== # https://api.minio.skybyte.me + access-key: RVZuWFViR2xld2t0dFF0em9XUWs= # EVnXUbGlewkttQtzoWQk + secret-key: THNxVFRmc0VEVzBFY3Buc09aOUxyTnhwc21zajdIMGxlR2R0WHBwRg== # LsqTTfsEDW0EcpnsOZ9LrNxpsmsj7H0leGdtXppF + bucket: YmFja3Vwcw== # backups +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: backup-config + namespace: backup-system +data: + subpath: "backups" + backups-to-keep: "3" +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: backup-service-account + namespace: backup-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: backup-role + namespace: backup-system +rules: +- apiGroups: ["apps"] + resources: ["daemonsets"] + verbs: ["get", "patch", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: backup-role-binding + namespace: backup-system +subjects: +- kind: ServiceAccount + name: backup-service-account + namespace: backup-system +roleRef: + kind: Role + name: backup-role + apiGroup: rbac.authorization.k8s.io diff --git a/backups/daemonset+cronjob/cm-account.yaml b/backups/daemonset+cronjob/cm-account.yaml new file mode 100644 index 0000000..4cacb52 --- /dev/null +++ b/backups/daemonset+cronjob/cm-account.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: backup-config + namespace: backup-system +data: + subpath: "nodes" + backups-to-keep: "3" + use-https: "True" + signature-v2: "False" # 设置为 "True" 如果 S3 服务需要 V2 签名 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: backup-service-account + namespace: backup-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: backup-role + namespace: backup-system +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list"] +- apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: backup-role-binding + namespace: backup-system +subjects: +- kind: ServiceAccount + name: backup-service-account + namespace: backup-system +roleRef: + kind: Role + name: backup-role + apiGroup: rbac.authorization.k8s.io diff --git a/backups/daemonset+cronjob/cm-script.yaml b/backups/daemonset+cronjob/cm-script.yaml new file mode 100644 index 0000000..db491a8 --- /dev/null +++ b/backups/daemonset+cronjob/cm-script.yaml @@ -0,0 +1,111 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: backup-script + namespace: backup-system +data: + backup.sh: | + #!/bin/bash + set -e + + # 获取环境变量 + S3_ENDPOINT="${S3_ENDPOINT}" + S3_ACCESS_KEY="${S3_ACCESS_KEY}" + S3_SECRET_KEY="${S3_SECRET_KEY}" + S3_BUCKET="${S3_BUCKET}" + S3_SUBPATH="${S3_SUBPATH:-backups}" + BACKUPS_TO_KEEP="${BACKUPS_TO_KEEP:-7}" + + # 获取主机名 + HOSTNAME=$(hostname) + HOSTNAME="${NODE_NAME:-$HOSTNAME}" + + + # 配置 s3cmd + cat > ~/.s3cfg << EOF + [default] + access_key = ${S3_ACCESS_KEY} + secret_key = ${S3_SECRET_KEY} + host_base = ${S3_ENDPOINT} + host_bucket = ${S3_ENDPOINT} + use_https = ${USE_HTTPS:-True} + signature_v2 = ${SIGNATURE_V2:-False} + EOF + + # 检查 /data/local-csi 路径是否存在 + if [ ! -d "/data/local-csi" ]; then + echo "目录 /data/local-csi 不存在，退出备份" + exit 0 + fi + + # 检查目录下是否有至少一个子目录 + DIR_COUNT=$(find /data/local-csi -mindepth 1 -maxdepth 1 -type d | wc -l) + if [ "$DIR_COUNT" -eq 0 ]; then + echo "目录 /data/local-csi 中没有子目录，退出备份" + exit 0 + fi + + # 遍历所有目录 + find /data/local-csi -mindepth 1 -maxdepth 1 -type d | while read dir; do + DIR_NAME=$(basename "$dir") + TIMESTAMP=$(date +%Y-%m-%d-%H-%M-%S) + BACKUP_NAME="backup-${DIR_NAME}-${TIMESTAMP}.tar.gz" + TEMP_DIR=$(mktemp -d) + BACKUP_PATH="/tmp/${BACKUP_NAME}" + + echo "正在备份目录: $dir" + + rsync -a "/data/local-csi/$DIR_NAME/" "$TEMP_DIR/" + # 创建备份压缩包 + # tar -czf "$BACKUP_PATH" -C "/data/local-csi" "$DIR_NAME" + tar -czf "$BACKUP_PATH" -C "/data/local-csi" "$TEMP_DIR" + + # 清理临时目录 + rm -rf "$TEMP_DIR" + + # 计算上传路径 + BACKUP_TARGET_PATH="s3://${S3_BUCKET}/${S3_SUBPATH}/${HOSTNAME}/${DIR_NAME}/" + echo "上传备份 $BACKUP_PATH 到 S3 路径: $BACKUP_TARGET_PATH" + + # 上传备份文件 + s3cmd put "$BACKUP_PATH" "${BACKUP_TARGET_PATH}${BACKUP_NAME}" + + # 删除本地临时备份 + rm "$BACKUP_PATH" + + # 清理旧备份 + echo "清理旧备份，保留最新的 $BACKUPS_TO_KEEP 个文件" + + # 获取所有备份文件并按时间排序 + BACKUP_FILES=$(s3cmd ls "${BACKUP_TARGET_PATH}" | sort -k1,2) + BACKUP_COUNT=$(echo "$BACKUP_FILES" | grep -c "backup-${DIR_NAME}") + + if [ "$BACKUP_COUNT" -gt "$BACKUPS_TO_KEEP" ]; then + # 计算需要删除的文件数量 + DELETE_COUNT=$((BACKUP_COUNT - BACKUPS_TO_KEEP)) + + # 获取需要删除的文件列表 + FILES_TO_DELETE=$(echo "$BACKUP_FILES" | head -n $DELETE_COUNT | awk '{print $4}') + + # 删除文件 + for FILE in $FILES_TO_DELETE; do + echo "删除旧备份: $FILE" + s3cmd rm "$FILE" + done + fi + done + + echo "备份完成" + + s3cmd-trigger.sh: | + #!/bin/bash + set -e + + # 检查标记文件 + if [ -f "/tmp/backup-triggered" ]; then + echo "发现备份触发标记，执行备份脚本..." + bash /scripts/backup.sh + rm /tmp/backup-triggered + echo "备份完成" + fi + diff --git a/backups/daemonset+cronjob/crontab.yaml b/backups/daemonset+cronjob/crontab.yaml index 7fae3f9..f4b674c 100644 --- a/backups/daemonset+cronjob/crontab.yaml +++ b/backups/daemonset+cronjob/crontab.yaml @@ -1,8 +1,3 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: backup-system ---- apiVersion: batch/v1 kind: CronJob metadata: @@ -19,7 +14,8 @@ spec: spec: serviceAccountName: backup-service-account nodeSelector: - kubernetes.io/hostname: "vkvm-us1" + #kubernetes.io/hostname: "vkvm-us1" + region: us containers: - name: backup-trigger image: bitnami/kubectl:latest @@ -27,7 +23,10 @@ spec: - /bin/sh - -c - | - kubectl label daemonset/node-backup-daemon trigger-backup=true --overwrite -n backup-system && \ - sleep 60 && \ - kubectl label daemonset/node-backup-daemon trigger-backup- -n backup-system + # 创建触发文件到所有备份 Pod 中 + for pod in $(kubectl get pods -n backup-system -l app=node-backup -o jsonpath='{.items[*].metadata.name}'); do + echo "触发 Pod $pod 执行备份" + kubectl exec $pod -n backup-system -- touch /tmp/backup-triggered + done + echo "所有节点备份已触发" restartPolicy: OnFailure diff --git a/backups/daemonset+cronjob/daemonset.yaml b/backups/daemonset+cronjob/daemonset.yaml index d73b8a9..1ee9cae 100644 --- a/backups/daemonset+cronjob/daemonset.yaml +++ b/backups/daemonset+cronjob/daemonset.yaml @@ -1,4 +1,3 @@ ---- apiVersion: apps/v1 kind: DaemonSet metadata: @@ -13,55 +12,64 @@ spec: labels: app: node-backup spec: - nodeSelector: - kubernetes.io/hostname: "vkvm-us1" + #nodeSelector: + # kubernetes.io/hostname: "vkvm-us1" + #affinity: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # #- key: region + # # operator: NotIn + # # values: + # # - cn + # - key: kubernetes.io/hostname + # operator: In + # values: + # - zgocloud-us1 containers: - name: backup-container - image: minio/mc:latest + image: alpine:latest imagePullPolicy: IfNotPresent command: - /bin/sh - -c - | - # 添加依赖的工具 - apk add --no-cache jq bash findutils tar curl - - # 等待触发备份 + # 安装依赖工具 + apk add --no-cache bash s3cmd tar rsync curl + + # 启动一个循环，每分钟检查一次触发文件 while true; do - if [ "$(curl -s -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \ - https://kubernetes.default.svc/apis/apps/v1/namespaces/backup-system/daemonsets/node-backup-daemon \ - --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt \ - -X GET | jq -r '.metadata.labels["trigger-backup"]')" = "true" ]; then - echo "备份触发，执行备份脚本..." - bash /scripts/backup.sh - echo "备份完成" - fi + bash /scripts/s3cmd-trigger.sh date - # 每分钟检查一次 sleep 60 done env: - - name: MINIO_ENDPOINT + - name: NODE_NAME # 使用 Downward API 获取 Pod 运行的节点名 + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: S3_ENDPOINT valueFrom: secretKeyRef: - name: minio-credentials + name: s3-credentials key: endpoint - - name: MINIO_ACCESS_KEY + - name: S3_ACCESS_KEY valueFrom: secretKeyRef: - name: minio-credentials + name: s3-credentials key: access-key - - name: MINIO_SECRET_KEY + - name: S3_SECRET_KEY valueFrom: secretKeyRef: - name: minio-credentials + name: s3-credentials key: secret-key - - name: MINIO_BUCKET + - name: S3_BUCKET valueFrom: secretKeyRef: - name: minio-credentials + name: s3-credentials key: bucket - - name: MINIO_SUBPATH + - name: S3_SUBPATH valueFrom: configMapKeyRef: name: backup-config @@ -73,6 +81,18 @@ spec: name: backup-config key: backups-to-keep optional: true + - name: USE_HTTPS + valueFrom: + configMapKeyRef: + name: backup-config + key: use-https + optional: true + - name: SIGNATURE_V2 + valueFrom: + configMapKeyRef: + name: backup-config + key: signature-v2 + optional: true volumeMounts: - name: host-data mountPath: /data @@ -86,55 +106,3 @@ spec: configMap: name: backup-script defaultMode: 0755 ---- -apiVersion: v1 -kind: Secret -metadata: - name: minio-credentials - namespace: backup-system -type: Opaque -data: - # 这些值需要使用base64编码替换 - endpoint: aHR0cHM6Ly9hcGkubWluaW8uc2t5Ynl0ZS5tZQ== # https://api.minio.skybyte.me - access-key: RVZuWFViR2xld2t0dFF0em9XUWs= # EVnXUbGlewkttQtzoWQk - secret-key: THNxVFRmc0VEVzBFY3Buc09aOUxyTnhwc21zajdIMGxlR2R0WHBwRg== # LsqTTfsEDW0EcpnsOZ9LrNxpsmsj7H0leGdtXppF - bucket: YmFja3Vwcw== # backups ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: backup-config - namespace: backup-system -data: - subpath: "backups" - backups-to-keep: "3" ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: backup-service-account - namespace: backup-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: backup-role - namespace: backup-system -rules: -- apiGroups: ["apps"] - resources: ["daemonsets"] - verbs: ["get", "patch", "update"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: backup-role-binding - namespace: backup-system -subjects: -- kind: ServiceAccount - name: backup-service-account - namespace: backup-system -roleRef: - kind: Role - name: backup-role - apiGroup: rbac.authorization.k8s.io diff --git a/backups/daemonset+cronjob/ns.yaml b/backups/daemonset+cronjob/ns.yaml new file mode 100644 index 0000000..cd98a83 --- /dev/null +++ b/backups/daemonset+cronjob/ns.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: backup-system diff --git a/backups/daemonset+cronjob/secret.yaml b/backups/daemonset+cronjob/secret.yaml new file mode 100644 index 0000000..5b696bd --- /dev/null +++ b/backups/daemonset+cronjob/secret.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Secret +metadata: + name: s3-credentials + namespace: backup-system +type: Opaque +data: + # 这些值需要使用base64编码替换 + endpoint: aHR0cHM6Ly9hcGkubWluaW8uc2t5Ynl0ZS5tZQ== # https://api.minio.skybyte.me + access-key: RVZuWFViR2xld2t0dFF0em9XUWs= # EVnXUbGlewkttQtzoWQk + secret-key: THNxVFRmc0VEVzBFY3Buc09aOUxyTnhwc21zajdIMGxlR2R0WHBwRg== # LsqTTfsEDW0EcpnsOZ9LrNxpsmsj7H0leGdtXppF + bucket: YmFja3Vwcw== # backups +