Publish: review-model → main #731
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sync Model CN to Aliyun OSS | |
| on: | |
| pull_request: | |
| types: | |
| - closed | |
| branches: | |
| - main | |
| paths: | |
| - 'model/cn/**' | |
| workflow_dispatch: | |
| inputs: | |
| manual_run: | |
| description: '手动触发全量同步 model/cn 数据到 OSS' | |
| required: false | |
| default: 'true' | |
| jobs: | |
| sync: | |
| runs-on: ubuntu-latest | |
| # PR 触发时仅在合并后运行，手动触发时始终运行 | |
| if: >- | |
| github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'pull_request' && github.event.pull_request.merged == true) | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Detect changed model/cn files | |
| id: changes | |
| env: | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| run: | | |
| set -euo pipefail | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| echo "手动触发：全量同步模式" | |
| echo "SYNC_MODE=full" >> "$GITHUB_ENV" | |
| echo "sync_mode=full" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "PR 合并触发：增量同步模式" | |
| echo "SYNC_MODE=incremental" >> "$GITHUB_ENV" | |
| echo "sync_mode=incremental" >> "$GITHUB_OUTPUT" | |
| # 获取 PR 中 model/cn/ 下变更的文件 | |
| CHANGED_FILES=$(git diff --name-only "$BASE_SHA" "$HEAD_SHA" -- 'model/cn/' || true) | |
| DELETED_FILES=$(git diff --name-only --diff-filter=D "$BASE_SHA" "$HEAD_SHA" -- 'model/cn/' || true) | |
| ADDED_MODIFIED_FILES=$(git diff --name-only --diff-filter=d "$BASE_SHA" "$HEAD_SHA" -- 'model/cn/' || true) | |
| if [ -z "$CHANGED_FILES" ]; then | |
| echo "没有 model/cn/ 下的文件变更，跳过同步" | |
| echo "HAS_CHANGES=false" >> "$GITHUB_ENV" | |
| echo "has_changes=false" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| echo "变更的文件:" | |
| echo "$CHANGED_FILES" | |
| # 保存文件列表供后续步骤使用 | |
| echo "$ADDED_MODIFIED_FILES" > /tmp/changed_model_files.txt | |
| echo "$DELETED_FILES" > /tmp/deleted_model_files.txt | |
| echo "HAS_CHANGES=true" >> "$GITHUB_ENV" | |
| echo "has_changes=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Validate secrets | |
| if: env.SYNC_MODE == 'full' || env.HAS_CHANGES == 'true' | |
| env: | |
| OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }} | |
| OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }} | |
| OSS_ENDPOINT: ${{ secrets.OSS_ENDPOINT }} | |
| OSS_BUCKET: ${{ secrets.OSS_BUCKET }} | |
| run: | | |
| set -euo pipefail | |
| echo "Validating required secrets..." | |
| for var in OSS_ACCESS_KEY_ID OSS_ACCESS_KEY_SECRET OSS_ENDPOINT OSS_BUCKET; do | |
| if [ -z "${!var}" ]; then | |
| echo "ERROR: $var is not set" | |
| exit 1 | |
| fi | |
| done | |
| echo "All required secrets are present" | |
| - name: Setup ossutil | |
| if: env.SYNC_MODE == 'full' || env.HAS_CHANGES == 'true' | |
| run: | | |
| set -euo pipefail | |
| wget -q https://gosspublic.alicdn.com/ossutil/1.7.15/ossutil64 | |
| chmod +x ossutil64 | |
| sudo mv ossutil64 /usr/local/bin/ossutil | |
| ossutil --version | |
| - name: Configure ossutil | |
| if: env.SYNC_MODE == 'full' || env.HAS_CHANGES == 'true' | |
| env: | |
| OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }} | |
| OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }} | |
| OSS_ENDPOINT: ${{ secrets.OSS_ENDPOINT }} | |
| run: | | |
| set -euo pipefail | |
| ossutil config -e $OSS_ENDPOINT -i $OSS_ACCESS_KEY_ID -k $OSS_ACCESS_KEY_SECRET | |
| ossutil ls oss:// || echo "Warning: Could not list OSS buckets" | |
| - name: Extract resource URLs and download files | |
| if: env.SYNC_MODE == 'full' || env.HAS_CHANGES == 'true' | |
| run: | | |
| set -euo pipefail | |
| mkdir -p downloads | |
| DOWNLOAD_COUNT=0 | |
| FAIL_COUNT=0 | |
| echo "=========================================" | |
| echo "Starting resource file download process" | |
| echo "=========================================" | |
| DOWNLOAD_EXTENSIONS='\.(png|jpg|jpeg|gif|webp|bmp|svg|ico|3mf|stl|gcode|snap3dp|snapcnc|snaplaser|zip|tar|gz|tgz|rar|7z|pdf|mp3|mp4|wav|avi|mov|exe|bin|apk|dmg|deb|rpm|msi|iso)$' | |
| ALL_URLS_FILE="all_resource_urls.txt" | |
| > "$ALL_URLS_FILE" | |
| if [ "$SYNC_MODE" = "incremental" ]; then | |
| # 增量模式：仅从变更的文件中提取 URL | |
| echo "增量模式：仅处理变更文件的资源" | |
| while IFS= read -r changed_file; do | |
| [ -z "$changed_file" ] && continue | |
| if [ -f "$changed_file" ]; then | |
| echo "Extracting URLs from: $changed_file" | |
| grep -oE 'https?://[^"'\''[:space:]<>]+' "$changed_file" 2>/dev/null >> "$ALL_URLS_FILE" || true | |
| fi | |
| done < /tmp/changed_model_files.txt | |
| else | |
| # 全量模式：处理所有文件 | |
| while IFS= read -r -d '' html_file; do | |
| echo "Extracting URLs from: $html_file" | |
| grep -oE 'https?://[^"'\''[:space:]<>]+' "$html_file" 2>/dev/null >> "$ALL_URLS_FILE" || true | |
| done < <(find model/cn/ -name "*.html" -type f -print0 2>/dev/null) | |
| fi | |
| SORTED_URLS_FILE="sorted_resource_urls.txt" | |
| sort -u "$ALL_URLS_FILE" > "$SORTED_URLS_FILE" | |
| rm -f "$ALL_URLS_FILE" | |
| while IFS= read -r url; do | |
| [ -z "$url" ] && continue | |
| url=$(echo "$url" | sed 's/[,;:]*$//') | |
| url_main=$(echo "$url" | cut -d'?' -f1) | |
| if ! echo "$url_main" | grep -qiE "$DOWNLOAD_EXTENSIONS"; then | |
| continue | |
| fi | |
| url_without_protocol=$(echo "$url" | sed 's|https\?://||') | |
| url_path=$(echo "$url_without_protocol" | cut -d'/' -f2-) | |
| target_dir="downloads/download/$(dirname "$url_path")" | |
| target_file="downloads/download/$url_path" | |
| mkdir -p "$target_dir" | |
| if wget -q --timeout=60 --tries=3 -O "$target_file" "$url"; then | |
| DOWNLOAD_COUNT=$((DOWNLOAD_COUNT + 1)) | |
| echo " [下载] $url_path" | |
| else | |
| echo "DOWNLOAD_FAILED: $url" >> sync_errors.txt | |
| rm -f "$target_file" | |
| FAIL_COUNT=$((FAIL_COUNT + 1)) | |
| echo " [失败] $url" | |
| fi | |
| done < "$SORTED_URLS_FILE" | |
| rm -f "$SORTED_URLS_FILE" | |
| echo "DOWNLOAD_COUNT=$DOWNLOAD_COUNT" >> "$GITHUB_ENV" | |
| echo "FAIL_COUNT=$FAIL_COUNT" >> "$GITHUB_ENV" | |
| echo "=========================================" | |
| echo "下载完成: ${DOWNLOAD_COUNT} 个, 失败 ${FAIL_COUNT} 个" | |
| echo "=========================================" | |
| - name: Create modified copies and inject missing JSON data | |
| if: env.SYNC_MODE == 'full' || env.HAS_CHANGES == 'true' | |
| env: | |
| CDN_DOMAIN: ${{ secrets.CDN_DOMAIN }} | |
| run: | | |
| set -euo pipefail | |
| rm -rf temp_oss | |
| mkdir -p temp_oss/model | |
| EXTENSIONS="png|jpg|jpeg|gif|webp|bmp|svg|ico|3mf|stl|gcode|snap3dp|snapcnc|snaplaser|zip|tar|gz|tgz|rar|7z|pdf|mp3|mp4|wav|avi|mov|exe|bin|apk|dmg|deb|rpm|msi|iso" | |
| if [ "$SYNC_MODE" = "incremental" ]; then | |
| # 增量模式：仅复制和处理变更的文件 | |
| echo "增量模式：仅处理变更的文件" | |
| while IFS= read -r changed_file; do | |
| [ -z "$changed_file" ] && continue | |
| if [ -f "$changed_file" ]; then | |
| # changed_file 格式: model/cn/xxx.html -> temp_oss/model/xxx.html | |
| rel_path="${changed_file#model/cn/}" | |
| target_dir="temp_oss/model/$(dirname "$rel_path")" | |
| mkdir -p "$target_dir" | |
| cp -a "$changed_file" "temp_oss/model/$rel_path" | |
| fi | |
| done < /tmp/changed_model_files.txt | |
| else | |
| # 全量模式：复制所有文件 | |
| if [ ! -d "model/cn" ] || [ -z "$(ls -A model/cn 2>/dev/null)" ]; then | |
| echo "ERROR: model/cn directory is empty or not found" | |
| exit 1 | |
| fi | |
| cp -a model/cn/* temp_oss/model/ | |
| fi | |
| # 去掉 .html 扩展名 | |
| while IFS= read -r -d '' f; do | |
| mv "$f" "${f%.html}" | |
| done < <(find temp_oss -type f -name "*.html" -print0) | |
| echo "Applying URL replacements..." | |
| while IFS= read -r -d '' html_file; do | |
| sed -i -E "s#https://[a-zA-Z0-9.-]+/([^\"]+\.($EXTENSIONS))#https://$CDN_DOMAIN/download/\1#gi" "$html_file" | |
| sed -i -E "s#https://[a-zA-Z0-9.-]+/api/model/detail/([0-9]+)#https://$CDN_DOMAIN/model/detail/\1#g" "$html_file" | |
| sed -i 's/snapmaker\.com/snapmaker.cn/g' "$html_file" | |
| done < <(find temp_oss -type f -print0) | |
| - name: Sync model files to OSS (incremental) | |
| if: env.SYNC_MODE == 'incremental' && env.HAS_CHANGES == 'true' | |
| env: | |
| OSS_BUCKET: ${{ secrets.OSS_BUCKET }} | |
| run: | | |
| set -euo pipefail | |
| MODEL_UPLOAD_COUNT=0 | |
| MODEL_DELETE_COUNT=0 | |
| # 上传变更的文件 | |
| if [ -d "temp_oss/model" ] && [ -n "$(ls -A temp_oss/model 2>/dev/null)" ]; then | |
| echo "增量上传变更的 model 文件..." | |
| while IFS= read -r -d '' file; do | |
| rel_path=${file#temp_oss/} | |
| ossutil cp -f "$file" "oss://$OSS_BUCKET/$rel_path" \ | |
| --meta "Content-Type:application/json#Cache-Control:public, max-age=3600#Content-Disposition:inline" | |
| MODEL_UPLOAD_COUNT=$((MODEL_UPLOAD_COUNT + 1)) | |
| echo " [上传] $rel_path" | |
| done < <(find temp_oss/model -type f -print0) | |
| fi | |
| # 删除在 PR 中被删除的文件 | |
| if [ -f /tmp/deleted_model_files.txt ]; then | |
| while IFS= read -r deleted_file; do | |
| [ -z "$deleted_file" ] && continue | |
| # deleted_file 格式: model/cn/detail/xx.html -> OSS key: model/detail/xx (去掉 cn/ 和 .html) | |
| rel_path="${deleted_file#model/cn/}" | |
| oss_key="model/${rel_path%.html}" | |
| echo " [删除] oss://$OSS_BUCKET/$oss_key" | |
| ossutil rm -f "oss://$OSS_BUCKET/$oss_key" || true | |
| MODEL_DELETE_COUNT=$((MODEL_DELETE_COUNT + 1)) | |
| done < /tmp/deleted_model_files.txt | |
| fi | |
| echo "MODEL_UPLOAD_COUNT=$MODEL_UPLOAD_COUNT" >> "$GITHUB_ENV" | |
| echo "MODEL_DELETE_COUNT=$MODEL_DELETE_COUNT" >> "$GITHUB_ENV" | |
| echo "增量同步完成: 上传 $MODEL_UPLOAD_COUNT 个, 删除 $MODEL_DELETE_COUNT 个" | |
| - name: Sync model files to OSS (full) and delete stale files | |
| if: env.SYNC_MODE == 'full' | |
| env: | |
| OSS_BUCKET: ${{ secrets.OSS_BUCKET }} | |
| run: | | |
| set -euo pipefail | |
| MODEL_UPLOAD_COUNT=0 | |
| MODEL_DELETE_COUNT=0 | |
| if [ ! -d "temp_oss/model" ] || [ -z "$(ls -A temp_oss/model 2>/dev/null)" ]; then | |
| echo "No model files to sync" | |
| else | |
| echo "全量同步 model 文件到 OSS..." | |
| while IFS= read -r -d '' file; do | |
| rel_path=${file#temp_oss/} | |
| ossutil cp -f "$file" "oss://$OSS_BUCKET/$rel_path" \ | |
| --meta "Content-Type:application/json#Cache-Control:public, max-age=3600#Content-Disposition:inline" | |
| MODEL_UPLOAD_COUNT=$((MODEL_UPLOAD_COUNT + 1)) | |
| done < <(find temp_oss/model -type f -print0) | |
| # 构建本地 key 集合 | |
| LOCAL_MODEL_KEYS="local_model_keys.txt" | |
| > "$LOCAL_MODEL_KEYS" | |
| while IFS= read -r -d '' file; do | |
| echo "${file#temp_oss/}" >> "$LOCAL_MODEL_KEYS" | |
| done < <(find temp_oss/model -type f -print0) | |
| # 删除 OSS 上多余的文件 | |
| OSS_MODEL_KEYS="oss_model_keys.txt" | |
| ossutil ls "oss://$OSS_BUCKET/model/" -s 2>/dev/null \ | |
| | grep -v '^$' | sed "s|oss://$OSS_BUCKET/||" > "$OSS_MODEL_KEYS" || true | |
| while IFS= read -r oss_key; do | |
| [ -z "$oss_key" ] && continue | |
| if ! grep -qxF "$oss_key" "$LOCAL_MODEL_KEYS"; then | |
| echo " [删除] oss://$OSS_BUCKET/$oss_key" | |
| ossutil rm -f "oss://$OSS_BUCKET/$oss_key" | |
| MODEL_DELETE_COUNT=$((MODEL_DELETE_COUNT + 1)) | |
| fi | |
| done < "$OSS_MODEL_KEYS" | |
| rm -f "$LOCAL_MODEL_KEYS" "$OSS_MODEL_KEYS" | |
| echo "全量同步完成: 上传 $MODEL_UPLOAD_COUNT 个, 删除 $MODEL_DELETE_COUNT 个" | |
| fi | |
| echo "MODEL_UPLOAD_COUNT=$MODEL_UPLOAD_COUNT" >> "$GITHUB_ENV" | |
| echo "MODEL_DELETE_COUNT=$MODEL_DELETE_COUNT" >> "$GITHUB_ENV" | |
| - name: Sync downloaded resource files to OSS | |
| if: env.SYNC_MODE == 'full' || env.HAS_CHANGES == 'true' | |
| env: | |
| OSS_BUCKET: ${{ secrets.OSS_BUCKET }} | |
| run: | | |
| set -euo pipefail | |
| RESOURCE_UPLOAD_COUNT=0 | |
| if [ -d "downloads" ] && [ -n "$(ls -A downloads 2>/dev/null)" ]; then | |
| echo "Syncing downloaded resource files to OSS..." | |
| while IFS= read -r -d '' file; do | |
| rel_path=${file#downloads/} | |
| ossutil cp -f "$file" "oss://$OSS_BUCKET/$rel_path" \ | |
| --meta "Cache-Control:public, max-age=86400" | |
| RESOURCE_UPLOAD_COUNT=$((RESOURCE_UPLOAD_COUNT + 1)) | |
| done < <(find downloads/ -type f -print0) | |
| echo "资源文件上传完成: $RESOURCE_UPLOAD_COUNT 个" | |
| fi | |
| echo "RESOURCE_UPLOAD_COUNT=$RESOURCE_UPLOAD_COUNT" >> "$GITHUB_ENV" | |
| - name: Cleanup temporary files | |
| if: always() | |
| run: | | |
| rm -rf temp_oss downloads | |
| rm -f sync_errors.txt all_resource_urls.txt sorted_resource_urls.txt | |
| rm -f local_model_keys.txt oss_model_keys.txt | |
| rm -f /tmp/changed_model_files.txt /tmp/deleted_model_files.txt | |
| - name: Setup aliyun-cli for CDN | |
| if: success() && (env.SYNC_MODE == 'full' || env.HAS_CHANGES == 'true') | |
| run: | | |
| set -euo pipefail | |
| wget -q https://aliyuncli.alicdn.com/aliyun-cli-linux-latest-amd64.tgz | |
| tar -xzf aliyun-cli-linux-latest-amd64.tgz | |
| sudo mv aliyun /usr/local/bin/ | |
| rm -f aliyun-cli-linux-latest-amd64.tgz | |
| - name: Refresh CDN cache | |
| if: success() && (env.SYNC_MODE == 'full' || env.HAS_CHANGES == 'true') | |
| env: | |
| CDN_ACCESS_KEY_ID: ${{ secrets.CDN_ACCESS_KEY_ID }} | |
| CDN_ACCESS_KEY_SECRET: ${{ secrets.CDN_ACCESS_KEY_SECRET }} | |
| CDN_DOMAIN: ${{ secrets.CDN_DOMAIN }} | |
| run: | | |
| set -euo pipefail | |
| if [ -z "${CDN_ACCESS_KEY_ID:-}" ] || [ -z "${CDN_ACCESS_KEY_SECRET:-}" ]; then | |
| echo "CDN secrets not set, skipping CDN refresh" | |
| exit 0 | |
| fi | |
| aliyun configure set \ | |
| --profile cdn-profile \ | |
| --mode AK \ | |
| --region cn-hangzhou \ | |
| --access-key-id "$CDN_ACCESS_KEY_ID" \ | |
| --access-key-secret "$CDN_ACCESS_KEY_SECRET" | |
| if [ "$SYNC_MODE" = "incremental" ]; then | |
| # 增量模式：仅刷新变更文件对应的 CDN 路径 | |
| echo "增量刷新 CDN 缓存..." | |
| REFRESH_URLS="" | |
| # 刷新变更的 model 文件 | |
| if [ -d "temp_oss/model" ]; then | |
| while IFS= read -r -d '' file; do | |
| rel_path=${file#temp_oss/} | |
| REFRESH_URLS="${REFRESH_URLS}https://$CDN_DOMAIN/$rel_path | |
| " | |
| done < <(find temp_oss -type f -print0 2>/dev/null) | |
| fi | |
| # 刷新被删除的文件 | |
| if [ -f /tmp/deleted_model_files.txt ]; then | |
| while IFS= read -r deleted_file; do | |
| [ -z "$deleted_file" ] && continue | |
| rel_path="${deleted_file#model/cn/}" | |
| oss_key="model/${rel_path%.html}" | |
| REFRESH_URLS="${REFRESH_URLS}https://$CDN_DOMAIN/$oss_key | |
| " | |
| done < /tmp/deleted_model_files.txt | |
| fi | |
| if [ -n "$REFRESH_URLS" ]; then | |
| aliyun --profile cdn-profile cdn RefreshObjectCaches \ | |
| --ObjectPath "$REFRESH_URLS" \ | |
| --ObjectType File 2>&1 || true | |
| fi | |
| # 资源文件目录刷新 | |
| if [ -d "downloads" ] && [ -n "$(ls -A downloads 2>/dev/null)" ]; then | |
| aliyun --profile cdn-profile cdn RefreshObjectCaches \ | |
| --ObjectPath "https://$CDN_DOMAIN/download/" \ | |
| --ObjectType Directory 2>&1 || true | |
| fi | |
| else | |
| # 全量模式：刷新整个目录 | |
| REFRESH_PATHS="https://$CDN_DOMAIN/model/ | |
| https://$CDN_DOMAIN/download/" | |
| aliyun --profile cdn-profile cdn RefreshObjectCaches \ | |
| --ObjectPath "$REFRESH_PATHS" \ | |
| --ObjectType Directory 2>&1 || true | |
| fi | |
| - name: Job summary | |
| if: always() | |
| run: | | |
| echo "=========================================" | |
| echo "同步完成汇总" | |
| echo "=========================================" | |
| echo "同步模式: ${SYNC_MODE:-unknown}" | |
| echo "资源文件: 下载 ${DOWNLOAD_COUNT:-0} 个, 失败 ${FAIL_COUNT:-0} 个" | |
| echo "Model 文件: 上传 ${MODEL_UPLOAD_COUNT:-0} 个, 删除 ${MODEL_DELETE_COUNT:-0} 个" | |
| echo "资源文件上传: ${RESOURCE_UPLOAD_COUNT:-0} 个" | |
| if [ -f sync_errors.txt ] && [ -s sync_errors.txt ]; then | |
| echo "" | |
| echo "警告：以下文件下载失败：" | |
| cat sync_errors.txt | |
| fi |