diff --git a/.gitea/workflows/ci-pipeline.yml b/.gitea/workflows/ci-pipeline.yml index dbae83666..1710dc2b9 100755 --- a/.gitea/workflows/ci-pipeline.yml +++ b/.gitea/workflows/ci-pipeline.yml @@ -1470,7 +1470,6 @@ jobs: - validate-security - validate-python - unit-tests - - frontend-lint - frontend-unit-test if: github.event_name == 'push' && github.ref_name == 'main' && !failure() && !cancelled() strategy: @@ -2124,3 +2123,4 @@ jobs: START_TIME="" [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) curl -sfH "Authorization: token ${GITHUB_TOKEN:-$GITEA_TOKEN}" -o /tmp/_ci_trace.py "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/ci_trace_report.py?ref=${GITHUB_SHA}" 2>/dev/null && python3 /tmp/_ci_trace.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true +# CI retry trigger diff --git a/EMPTY_COMMIT.txt b/EMPTY_COMMIT.txt new file mode 100644 index 000000000..cd5f7fe0b --- /dev/null +++ b/EMPTY_COMMIT.txt @@ -0,0 +1 @@ +retrigger3 \ No newline at end of file diff --git a/README.md b/README.md index d2189ef3e..bd73c324c 100644 --- a/README.md +++ b/README.md @@ -263,3 +263,4 @@ pytest --cov=packages --cov-report=html --- **License**: MIT + \ No newline at end of file diff --git a/scripts/ci/sync_base_images.sh b/scripts/ci/sync_base_images.sh new file mode 100755 index 000000000..b0f2bf072 --- /dev/null +++ b/scripts/ci/sync_base_images.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# ============================================ +# 基础镜像同步脚本 - 从公共镜像源同步到私有ACR +# 用法: +# ACR_USERNAME=xxx ACR_PASSWORD=yyy bash scripts/ci/sync_base_images.sh +# ============================================ + +set -euo pipefail + +ACR_REGISTRY="${ACR_REGISTRY:-xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji}" +ACR_USERNAME="${ACR_USERNAME:-}" +ACR_PASSWORD="${ACR_PASSWORD:-}" +SOURCE_PREFIX="${SOURCE_PREFIX:-docker.m.daocloud.io/library}" + +# 需要同步的镜像列表 (源镜像名:tag => ACR目标名:tag) +IMAGES=( + "python:3.12-slim-bookworm" + "python:3.12-slim" + "node:20" + "nginx:alpine" +) + +echo "============================================" +echo " 基础镜像同步到 ACR" +echo " ACR: $ACR_REGISTRY" +echo " 源: $SOURCE_PREFIX" +echo "============================================" +echo "" + +# 登录 ACR +if [ -n "$ACR_PASSWORD" ] && [ -n "$ACR_USERNAME" ]; then + echo "登录 ACR..." + ACR_HOST=$(echo "$ACR_REGISTRY" | cut -d/ -f1) + printf '%s' "$ACR_PASSWORD" | docker login "$ACR_HOST" -u "$ACR_USERNAME" --password-stdin + echo "ACR 登录成功" + echo "" +fi + +success=0 +failed=0 + +for image in "${IMAGES[@]}"; do + source_image="${SOURCE_PREFIX}/${image}" + target_image="${ACR_REGISTRY}/base/${image}" + + echo "--- 同步: $image ---" + echo " 源: $source_image" + echo " 目标: $target_image" + + # Pull 源镜像(带重试) + pulled=0 + for attempt in 1 2 3; do + echo " Pull 尝试 $attempt/3..." + if docker pull "$source_image"; then + pulled=1 + break + fi + echo " Pull 失败,5s 后重试..." + sleep 5 + done + + if [ "$pulled" -eq 0 ]; then + echo " ❌ Pull 失败: $image" + failed=$((failed + 1)) + continue + fi + + # Tag + docker tag "$source_image" "$target_image" + echo " Tag 完成" + + # Push 到 ACR + pushed=0 + for attempt in 1 2 3; do + echo " Push 尝试 $attempt/3..." + if docker push "$target_image"; then + pushed=1 + break + fi + echo " Push 失败,5s 后重试..." + sleep 5 + done + + if [ "$pushed" -eq 1 ]; then + echo " ✅ 同步成功: $image" + success=$((success + 1)) + else + echo " ❌ Push 失败: $image" + failed=$((failed + 1)) + fi + + echo "" +done + +echo "============================================" +echo " 同步完成" +echo " 成功: $success" +echo " 失败: $failed" +echo "============================================" + +if [ "$failed" -gt 0 ]; then + exit 1 +fi diff --git a/scripts/ci/validate_code_quality.sh b/scripts/ci/validate_code_quality.sh new file mode 100644 index 000000000..874c9c6cf --- /dev/null +++ b/scripts/ci/validate_code_quality.sh @@ -0,0 +1,157 @@ +#!/bin/bash +# CI Validate: 代码质量与安全扫描(并行Job 1/3) +# 包含:密钥扫描、格式检查、安全扫描、依赖漏洞、死代码检测、脚本语法校验 +set -eu + +echo "=== CI Validate: 代码质量与安全扫描 ===" + +# --- 密钥检测 --- +echo "" +echo "=== [1/6] Secret detection (detect-secrets) ===" +python3 -m pip install -q detect-secrets +detect-secrets --version + +detect-secrets scan \ + --all-files \ + --exclude-files '(^|/)(tests|test|e2e|__tests__|spec|docs|node_modules|site-packages|migrations|alembic|.gitea|.git|.pytest_cache|.next|dist|build)/' \ + --exclude-files '\.(md|rst|txt|lock|example|sample|min\.js|min\.css|spec\.ts|test\.ts|test\.py)$' \ + --exclude-files '(package-lock|yarn\.lock|poetry\.lock|Pipfile\.lock)$' \ + --disable-plugin Base64HighEntropyString \ + --disable-plugin HexHighEntropyString \ + --disable-plugin BasicAuthDetector \ + --disable-plugin KeywordDetector \ + --disable-plugin IPPublicDetector \ + > /tmp/secrets-scan.json 2>&1 + +FOUND=$(python3 -c " +import json +try: + with open('/tmp/secrets-scan.json') as f: + data = json.load(f) + results = data.get('results', {}) + total = sum(len(v) for v in results.values()) + print(total) +except Exception: + print('error') +") + +echo "Secrets detected: $FOUND" +if [ "$FOUND" != "0" ] && [ "$FOUND" != "error" ]; then + echo "" + echo "=== Secret details ===" + python3 -c " +import json +with open('/tmp/secrets-scan.json') as f: + data = json.load(f) +for fpath, items in data.get('results', {}).items(): + for item in items: + line = item.get('line_number', '?') + stype = item.get('type', '?') + hashed = item.get('hashed_secret', '')[:16] + print(f' {fpath}:{line} [{stype}] {hashed}...') +" + echo "" + echo "ERROR: Potential secrets detected in code!" + exit 1 +fi +echo "✅ Secret scan passed" + +# --- 代码质量检查(全量,PR 和 push 统一标准)--- +# 历史:PR 侧用增量检查以加速,但会导致 push 侧全量检查失败时 PR 侧感知不到 +# 现在统一全量检查,确保 CI 真正保护主分支(black/isort/ruff 全量仅多几十秒) +echo "" +echo "=== [2/6] Code quality checks (full scan) ===" +SCAN_MODE="full" +echo "Full scan mode" +python3 -m compileall -q alembic apps packages tests scripts +python3 -m black --check --fast alembic apps packages tests scripts +python3 -m isort --check-only alembic apps packages tests scripts +python3 -m ruff check apps packages tests --statistics + +echo "✅ Code quality checks passed" + +# --- Bandit 安全扫描(仅告警) --- +echo "" +echo "=== [3/6] Security scan (bandit, advisory only) ===" +set +e +bandit -r apps packages -q -ll +BANDIT_EXIT=$? +set -e +if [ "$BANDIT_EXIT" -ne 0 ]; then + echo "⚠️ Bandit found security issues (advisory mode - not blocking CI)" +else + echo "✅ Bandit security scan passed" +fi + +# --- Pip-audit 依赖漏洞扫描(仅告警) --- +echo "" +echo "=== [4/6] Python dependency vulnerability scan (pip-audit, advisory only) ===" +python3 -m pip install -q pip-audit +pip-audit --version +EXIT_CODE=0 +for req_file in requirements.txt requirements-base.txt requirements-dev.txt; do + if [ -f "$req_file" ]; then + echo "--- Scanning $req_file ---" + pip-audit -r "$req_file" --desc on 2>&1 | head -40 || EXIT_CODE=$? + echo "" + fi +done +echo "pip-audit scan completed (advisory mode - warnings only, not blocking CI)" + +# --- Vulture 死代码检测(仅告警) --- +echo "" +echo "=== [5/6] Dead code detection (vulture, advisory only) ===" +set +e +python3 -m pip install -q vulture +vulture --version +echo "告警模式,不阻断CI。置信度>=90%建议尽快确认。" +echo "" +vulture apps packages scripts \ + --exclude "tests,test,migrations,.gitea,docs,node_modules,site-packages,*/test_*.py,*/conftest.py" \ + --min-confidence 70 \ + 2>&1 | sort -t'(' -k2 -rn | head -80 +echo "" +echo "=== vulture scan summary ===" +echo "发现潜在死代码(可能包含框架装饰器注册的函数,为误报)" +echo "建议:定期人工审查高置信度(>=90%)条目" +set -e + +# --- CI脚本语法校验 --- +echo "" +echo "=== [6/6] CI & shell scripts syntax validation ===" +SYNTAX_ERROR=0 +# 检查所有 CI shell 脚本 +for script in scripts/ci/*.sh; do + if [ -f "$script" ]; then + if ! bash -n "$script" 2>&1; then + echo "❌ 语法错误: $script" + SYNTAX_ERROR=1 + fi + fi +done +# 检查所有 CI Python 脚本语法 +for script in scripts/ci/*.py; do + if [ -f "$script" ]; then + if ! python3 -m py_compile "$script" 2>&1; then + echo "❌ Python语法错误: $script" + SYNTAX_ERROR=1 + fi + fi +done +# 检查 .gitea/workflows 下的脚本(如果有) +for script in .gitea/workflows/*.sh; do + if [ -f "$script" ]; then + if ! bash -n "$script" 2>&1; then + echo "❌ 语法错误: $script" + SYNTAX_ERROR=1 + fi + fi +done +if [ "$SYNTAX_ERROR" -ne 0 ]; then + echo "❌ CI脚本语法校验失败,见上方错误" + exit 1 +fi +echo "✅ All CI scripts syntax OK" + +echo "" +echo "=== CI Validate: 代码质量与安全扫描 全部通过 ✅ ===" diff --git a/scripts/ci_production_deploy.sh b/scripts/ci_production_deploy.sh index c9a592892..f3804f489 100644 --- a/scripts/ci_production_deploy.sh +++ b/scripts/ci_production_deploy.sh @@ -170,16 +170,17 @@ rollback() { --name xiaoxia-api-production \ --env-file "$ENV_FILE" \ --network xiaoxia-net-production \ + --network-alias xiaoxia-api \ -p 127.0.0.1:8001:8000 \ -e APP_ENV=production \ -e APP_VERSION="$(echo $PREV_API_IMAGE | grep -oE '[^:]+$')" \ -e GENERATED_FILES_DIR=/app/generated \ - -e GENERATED_FILES_URL_PREFIX=/generated-files \ - -e PUBLIC_API_BASE_URL=https://production-api.xiaoxiajianji.com \ + -e GENERATED_FILES_URL_PREFIX=https://saas-api.xiaoxiajianji.com/generated-files \ + -e PUBLIC_API_BASE_URL=https://saas-api.xiaoxiajianji.com \ -v "$GENERATED_DIR:/app/generated" \ --restart unless-stopped \ - --cpus 2 \ - --memory 2g \ + --cpus 2 \ + --memory 2g \ --health-cmd "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/health', timeout=5)\"" \ --health-interval 30s \ --health-timeout 10s \ @@ -196,26 +197,29 @@ rollback() { echo "Rolling back Worker to: $PREV_WORKER_IMAGE" docker run -d \ --name xiaoxia-worker-production \ - --env-file "$ENV_FILE" \ --network xiaoxia-net-production \ + --network-alias xiaoxia-worker \ + --network-alias xiaoxia-api \ + --env-file "$ENV_FILE" \ -e APP_ENV=production \ -e APP_VERSION="$(echo $PREV_WORKER_IMAGE | grep -oE '[^:]+$')" \ - -e WORKER_CONCURRENCY=1 \ - -e WORKER_MAX_TASKS_PER_CHILD=100 \ -e GENERATED_FILES_DIR=/app/generated \ - -e GENERATED_FILES_URL_PREFIX=/generated-files \ - -e PUBLIC_API_BASE_URL=https://production-api.xiaoxiajianji.com \ + -e GENERATED_FILES_URL_PREFIX=https://saas-api.xiaoxiajianji.com/generated-files \ + -e PYTHONPATH=/app:/app/apps/api:/app/packages \ -v "$GENERATED_DIR:/app/generated" \ + -v "$LEGACY_ASSETS_DIR:/app/legacy-assets" \ + -w /app/apps/worker \ --restart unless-stopped \ - --cpus 2 \ - --memory 2g \ - --health-cmd "sh -c \"for pid in /proc/[0-9]*/cmdline; do if grep -ql celery \"$pid\" 2>/dev/null; then exit 0; fi; done; exit 1\"" \ + --cpus 2 \ + --memory 3g \ + --health-cmd "sh -c 'PYTHONPATH=/app:/app/apps/api:/app/packages celery -A worker_app.celery_app inspect ping -t 5 2>&1 | grep -q pong'" \ --health-interval 30s \ - --health-timeout 10s \ + --health-timeout 15s \ --health-retries 3 \ - --health-start-period 30s \ - $LOG_OPTS \ - "$PREV_WORKER_IMAGE" + --health-start-period 60s \ + --log-driver json-file --log-opt max-size=200m --log-opt max-file=5 \ + "$PREV_WORKER_IMAGE" \ + /usr/local/bin/entrypoint-worker.sh else echo "No previous Worker image to roll back to" fi @@ -230,12 +234,15 @@ rollback() { docker run -d \ --name xiaoxia-web-production \ --network xiaoxia-net-production \ + --network-alias xiaoxia-web \ -p 127.0.0.1:3002:80 \ - --restart unless-stopped \ - --cpus 0.5 \ - --memory 512m \ - $LEGACY_VOLUME \ + -e APP_ENV=production \ + -e API_BASE_URL=https://saas-api.xiaoxiajianji.com \ -v "$NGINX_CONF_FILE:/etc/nginx/conf.d/default.conf:ro" \ + $LEGACY_VOLUME \ + --restart unless-stopped \ + --cpus 1 \ + --memory 512m \ --health-cmd "wget --spider -q http://127.0.0.1:80" \ --health-interval 30s \ --health-timeout 5s \ @@ -373,12 +380,13 @@ docker run -d \ --name xiaoxia-api-production \ --env-file "$ENV_FILE" \ --network xiaoxia-net-production \ + --network-alias xiaoxia-api \ -p 127.0.0.1:8001:8000 \ -e APP_ENV=production \ -e APP_VERSION="$IMAGE_TAG" \ -e GENERATED_FILES_DIR=/app/generated \ - -e GENERATED_FILES_URL_PREFIX=/generated-files \ - -e PUBLIC_API_BASE_URL=https://production-api.xiaoxiajianji.com \ + -e GENERATED_FILES_URL_PREFIX=https://saas-api.xiaoxiajianji.com/generated-files \ + -e PUBLIC_API_BASE_URL=https://saas-api.xiaoxiajianji.com \ -v "$GENERATED_DIR:/app/generated" \ --restart unless-stopped \ --cpus 2 \ @@ -395,26 +403,29 @@ docker run -d \ echo "Starting Worker container..." docker run -d \ --name xiaoxia-worker-production \ - --env-file "$ENV_FILE" \ --network xiaoxia-net-production \ + --network-alias xiaoxia-worker \ + --network-alias xiaoxia-api \ + --env-file "$ENV_FILE" \ -e APP_ENV=production \ -e APP_VERSION="$IMAGE_TAG" \ - -e WORKER_CONCURRENCY=1 \ - -e WORKER_MAX_TASKS_PER_CHILD=100 \ -e GENERATED_FILES_DIR=/app/generated \ - -e GENERATED_FILES_URL_PREFIX=/generated-files \ - -e PUBLIC_API_BASE_URL=https://production-api.xiaoxiajianji.com \ + -e GENERATED_FILES_URL_PREFIX=https://saas-api.xiaoxiajianji.com/generated-files \ + -e PYTHONPATH=/app:/app/apps/api:/app/packages \ -v "$GENERATED_DIR:/app/generated" \ + -v "$LEGACY_ASSETS_DIR:/app/legacy-assets" \ + -w /app/apps/worker \ --restart unless-stopped \ --cpus 2 \ - --memory 2g \ - --health-cmd "sh -c \"for pid in /proc/[0-9]*/cmdline; do if grep -ql celery \"$pid\" 2>/dev/null; then exit 0; fi; done; exit 1\"" \ + --memory 3g \ + --health-cmd "sh -c 'PYTHONPATH=/app:/app/apps/api:/app/packages celery -A worker_app.celery_app inspect ping -t 5 2>&1 | grep -q pong'" \ --health-interval 30s \ - --health-timeout 10s \ + --health-timeout 15s \ --health-retries 3 \ - --health-start-period 30s \ - $LOG_OPTS \ - "$REGISTRY_WORKER" || rollback + --health-start-period 60s \ + --log-driver json-file --log-opt max-size=200m --log-opt max-file=5 \ + "$REGISTRY_WORKER" \ + /usr/local/bin/entrypoint-worker.sh || rollback # ---- 启动 Web ---- LEGACY_VOLUME="" @@ -429,12 +440,15 @@ echo "Starting Web container..." docker run -d \ --name xiaoxia-web-production \ --network xiaoxia-net-production \ + --network-alias xiaoxia-web \ -p 127.0.0.1:3002:80 \ - --restart unless-stopped \ - --cpus 0.5 \ - --memory 512m \ + -e APP_ENV=production \ + -e API_BASE_URL=https://saas-api.xiaoxiajianji.com \ -v "$NGINX_CONF_FILE:/etc/nginx/conf.d/default.conf:ro" \ $LEGACY_VOLUME \ + --restart unless-stopped \ + --cpus 1 \ + --memory 512m \ --health-cmd "wget --spider -q http://127.0.0.1:80" \ --health-interval 30s \ --health-timeout 5s \ @@ -487,7 +501,7 @@ docker builder prune -af --filter "until=168h" 2>/dev/null || true echo "" echo "=== Production deployment complete ===" -echo "API: http://127.0.0.1:8000" -echo "Web: http://127.0.0.1:3001" +echo "API: http://127.0.0.1:8001" +echo "Web: http://127.0.0.1:3002" echo "Version: $IMAGE_TAG" docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep production