Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f30af27762 | |||
| ab921517f6 | |||
| 8afa8b6b29 | |||
| d7138010fc |
@@ -172,6 +172,250 @@ jobs:
|
|||||||
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
|
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
|
||||||
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
|
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
|
||||||
|
|
||||||
|
|
||||||
|
validate-code-quality:
|
||||||
|
name: Validate - Code Quality
|
||||||
|
runs-on: ci-l2
|
||||||
|
timeout-minutes: 8
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
shell: sh
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ github.token }}
|
||||||
|
run: |
|
||||||
|
curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash
|
||||||
|
- name: Record job start time
|
||||||
|
shell: sh
|
||||||
|
run: bash scripts/ci/step_timer_start.sh
|
||||||
|
- name: Install dependencies
|
||||||
|
shell: sh
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
for i in 1 2 3; do
|
||||||
|
python3 -m pip install -q -r requirements-base.txt && break
|
||||||
|
echo "pip install requirements-base.txt 失败,重试 $i/3..."
|
||||||
|
[ $i -eq 3 ] && exit 1
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
for i in 1 2 3; do
|
||||||
|
python3 -m pip install -q -r requirements.txt && break
|
||||||
|
echo "pip install requirements.txt 失败,重试 $i/3..."
|
||||||
|
[ $i -eq 3 ] && exit 1
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
for i in 1 2 3; do
|
||||||
|
python3 -m pip install -q -r requirements-dev.txt && break
|
||||||
|
echo "pip install requirements-dev.txt 失败,重试 $i/3..."
|
||||||
|
[ $i -eq 3 ] && exit 1
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
for i in 1 2 3; do
|
||||||
|
python3 -m pip install --no-binary :all: black==26.5.1 isort==8.0.1 && break
|
||||||
|
echo "pip install black/isort 失败,重试 $i/3..."
|
||||||
|
[ $i -eq 3 ] && exit 1
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
- name: Run code quality and security checks
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ github.token }}
|
||||||
|
run: bash scripts/ci/validate_code_quality.sh
|
||||||
|
- name: Auto-fix formatting (black + isort)
|
||||||
|
if: failure()
|
||||||
|
shell: sh
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ github.token }}
|
||||||
|
run: python3 scripts/ci/auto_fix_formatting.py
|
||||||
|
- name: CI failure notification
|
||||||
|
if: failure()
|
||||||
|
shell: sh
|
||||||
|
env:
|
||||||
|
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||||
|
CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }}
|
||||||
|
run: |
|
||||||
|
set +e
|
||||||
|
FAILED_JOB="Validate - Code Quality" python3 scripts/ci_notify_failure.py
|
||||||
|
- name: Job duration summary
|
||||||
|
if: always()
|
||||||
|
shell: sh
|
||||||
|
run: bash scripts/ci/step_timer_end.sh
|
||||||
|
- name: Notify on failure
|
||||||
|
continue-on-error: true
|
||||||
|
if: failure()
|
||||||
|
shell: sh
|
||||||
|
env:
|
||||||
|
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
|
||||||
|
run: |
|
||||||
|
set +e
|
||||||
|
NOTIFY_MODE=failure JOB_NAME="Validate - Code Quality" python3 scripts/ci_notify.py
|
||||||
|
- name: Report CI trace
|
||||||
|
if: always()
|
||||||
|
shell: sh
|
||||||
|
env:
|
||||||
|
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
|
||||||
|
run: |
|
||||||
|
STATUS="ok"
|
||||||
|
[ ${{ job.status }} = "success" ] || STATUS="error"
|
||||||
|
START_TIME=""
|
||||||
|
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
|
||||||
|
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
|
||||||
|
|
||||||
|
validate-type-check:
|
||||||
|
name: Validate - Type Check (mypy)
|
||||||
|
runs-on: ci-l2
|
||||||
|
timeout-minutes: 8
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
shell: sh
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ github.token }}
|
||||||
|
run: |
|
||||||
|
curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash
|
||||||
|
- name: Record job start time
|
||||||
|
shell: sh
|
||||||
|
run: bash scripts/ci/step_timer_start.sh
|
||||||
|
- name: Install dependencies
|
||||||
|
shell: sh
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
for i in 1 2 3; do
|
||||||
|
python3 -m pip install -q -r requirements-base.txt && break
|
||||||
|
echo "pip install requirements-base.txt 失败,重试 $i/3..."
|
||||||
|
[ $i -eq 3 ] && exit 1
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
for i in 1 2 3; do
|
||||||
|
python3 -m pip install -q -r requirements.txt && break
|
||||||
|
echo "pip install requirements.txt 失败,重试 $i/3..."
|
||||||
|
[ $i -eq 3 ] && exit 1
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
for i in 1 2 3; do
|
||||||
|
python3 -m pip install -q -r requirements-dev.txt && break
|
||||||
|
echo "pip install requirements-dev.txt 失败,重试 $i/3..."
|
||||||
|
[ $i -eq 3 ] && exit 1
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
- name: Run mypy type check
|
||||||
|
shell: bash
|
||||||
|
run: bash scripts/ci/validate_mypy.sh
|
||||||
|
- name: CI failure notification
|
||||||
|
if: failure()
|
||||||
|
shell: sh
|
||||||
|
env:
|
||||||
|
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||||
|
CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }}
|
||||||
|
run: |
|
||||||
|
set +e
|
||||||
|
FAILED_JOB="Validate - Type Check (mypy)" python3 scripts/ci_notify_failure.py
|
||||||
|
- name: Job duration summary
|
||||||
|
if: always()
|
||||||
|
shell: sh
|
||||||
|
run: bash scripts/ci/step_timer_end.sh
|
||||||
|
- name: Notify on failure
|
||||||
|
continue-on-error: true
|
||||||
|
if: failure()
|
||||||
|
shell: sh
|
||||||
|
env:
|
||||||
|
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
|
||||||
|
run: |
|
||||||
|
set +e
|
||||||
|
NOTIFY_MODE=failure JOB_NAME="Validate - Type Check (mypy)" python3 scripts/ci_notify.py
|
||||||
|
- name: Report CI trace
|
||||||
|
if: always()
|
||||||
|
shell: sh
|
||||||
|
env:
|
||||||
|
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
|
||||||
|
run: |
|
||||||
|
STATUS="ok"
|
||||||
|
[ ${{ job.status }} = "success" ] || STATUS="error"
|
||||||
|
START_TIME=""
|
||||||
|
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
|
||||||
|
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
|
||||||
|
|
||||||
|
validate-migration:
|
||||||
|
name: Validate - Migration (alembic)
|
||||||
|
runs-on: ci-l2
|
||||||
|
timeout-minutes: 8
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
env:
|
||||||
|
DATABASE_URL: postgresql+psycopg://postgres:postgres@host.docker.internal:5432/xiaoxia_saas
|
||||||
|
USE_IN_MEMORY_DB: 'false'
|
||||||
|
CI_USE_SHARED_PG: 'true'
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
shell: sh
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ github.token }}
|
||||||
|
run: |
|
||||||
|
curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash
|
||||||
|
- name: Record job start time
|
||||||
|
shell: sh
|
||||||
|
run: bash scripts/ci/step_timer_start.sh
|
||||||
|
- name: Install dependencies
|
||||||
|
shell: sh
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
for i in 1 2 3; do
|
||||||
|
python3 -m pip install -q -r requirements-base.txt && break
|
||||||
|
echo "pip install requirements-base.txt 失败,重试 $i/3..."
|
||||||
|
[ $i -eq 3 ] && exit 1
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
for i in 1 2 3; do
|
||||||
|
python3 -m pip install -q -r requirements.txt && break
|
||||||
|
echo "pip install requirements.txt 失败,重试 $i/3..."
|
||||||
|
[ $i -eq 3 ] && exit 1
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
for i in 1 2 3; do
|
||||||
|
python3 -m pip install -q -r requirements-dev.txt && break
|
||||||
|
echo "pip install requirements-dev.txt 失败,重试 $i/3..."
|
||||||
|
[ $i -eq 3 ] && exit 1
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
- name: Run alembic migration validation
|
||||||
|
shell: bash
|
||||||
|
run: bash scripts/ci/validate_migration.sh
|
||||||
|
- name: CI failure notification
|
||||||
|
if: failure()
|
||||||
|
shell: sh
|
||||||
|
env:
|
||||||
|
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||||
|
CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }}
|
||||||
|
run: |
|
||||||
|
set +e
|
||||||
|
FAILED_JOB="Validate - Migration (alembic)" python3 scripts/ci_notify_failure.py
|
||||||
|
- name: Job duration summary
|
||||||
|
if: always()
|
||||||
|
shell: sh
|
||||||
|
run: bash scripts/ci/step_timer_end.sh
|
||||||
|
- name: Notify on failure
|
||||||
|
continue-on-error: true
|
||||||
|
if: failure()
|
||||||
|
shell: sh
|
||||||
|
env:
|
||||||
|
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
|
||||||
|
run: |
|
||||||
|
set +e
|
||||||
|
NOTIFY_MODE=failure JOB_NAME="Validate - Migration (alembic)" python3 scripts/ci_notify.py
|
||||||
|
- name: Report CI trace
|
||||||
|
if: always()
|
||||||
|
shell: sh
|
||||||
|
env:
|
||||||
|
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
|
||||||
|
run: |
|
||||||
|
STATUS="ok"
|
||||||
|
[ ${{ job.status }} = "success" ] || STATUS="error"
|
||||||
|
START_TIME=""
|
||||||
|
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
|
||||||
|
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
|
||||||
|
|
||||||
unit-tests:
|
unit-tests:
|
||||||
needs: check-frontend-only
|
needs: check-frontend-only
|
||||||
if: always() && needs.check-frontend-only.outputs.skip_backend != 'true'
|
if: always() && needs.check-frontend-only.outputs.skip_backend != 'true'
|
||||||
|
|||||||
@@ -0,0 +1,186 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# CI Validate: 代码质量与安全扫描(并行Job 1/3)
|
||||||
|
# 包含:密钥扫描、格式检查、安全扫描、依赖漏洞、死代码检测、脚本语法校验
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
echo "=== CI Validate: 代码质量与安全扫描 ==="
|
||||||
|
|
||||||
|
# --- 密钥检测 ---
|
||||||
|
echo ""
|
||||||
|
echo "=== [1/6] Secret detection (detect-secrets) ==="
|
||||||
|
python3 -m pip install -q detect-secrets
|
||||||
|
detect-secrets --version
|
||||||
|
|
||||||
|
detect-secrets scan \
|
||||||
|
--all-files \
|
||||||
|
--exclude-files '(^|/)(tests|test|e2e|__tests__|spec|docs|node_modules|site-packages|migrations|alembic|.gitea|.git|.pytest_cache|.next|dist|build)/' \
|
||||||
|
--exclude-files '\.(md|rst|txt|lock|example|sample|min\.js|min\.css|spec\.ts|test\.ts|test\.py)$' \
|
||||||
|
--exclude-files '(package-lock|yarn\.lock|poetry\.lock|Pipfile\.lock)$' \
|
||||||
|
--disable-plugin Base64HighEntropyString \
|
||||||
|
--disable-plugin HexHighEntropyString \
|
||||||
|
--disable-plugin BasicAuthDetector \
|
||||||
|
--disable-plugin KeywordDetector \
|
||||||
|
--disable-plugin IPPublicDetector \
|
||||||
|
> /tmp/secrets-scan.json 2>&1
|
||||||
|
|
||||||
|
FOUND=$(python3 -c "
|
||||||
|
import json
|
||||||
|
try:
|
||||||
|
with open('/tmp/secrets-scan.json') as f:
|
||||||
|
data = json.load(f)
|
||||||
|
results = data.get('results', {})
|
||||||
|
total = sum(len(v) for v in results.values())
|
||||||
|
print(total)
|
||||||
|
except Exception:
|
||||||
|
print('error')
|
||||||
|
")
|
||||||
|
|
||||||
|
echo "Secrets detected: $FOUND"
|
||||||
|
if [ "$FOUND" != "0" ] && [ "$FOUND" != "error" ]; then
|
||||||
|
echo ""
|
||||||
|
echo "=== Secret details ==="
|
||||||
|
python3 -c "
|
||||||
|
import json
|
||||||
|
with open('/tmp/secrets-scan.json') as f:
|
||||||
|
data = json.load(f)
|
||||||
|
for fpath, items in data.get('results', {}).items():
|
||||||
|
for item in items:
|
||||||
|
line = item.get('line_number', '?')
|
||||||
|
stype = item.get('type', '?')
|
||||||
|
hashed = item.get('hashed_secret', '')[:16]
|
||||||
|
print(f' {fpath}:{line} [{stype}] {hashed}...')
|
||||||
|
"
|
||||||
|
echo ""
|
||||||
|
echo "ERROR: Potential secrets detected in code!"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "✅ Secret scan passed"
|
||||||
|
|
||||||
|
# --- 增量/全量模式判断 ---
|
||||||
|
echo ""
|
||||||
|
echo "=== [2/6] Code quality checks ==="
|
||||||
|
SCAN_MODE="full"
|
||||||
|
CHANGED_PY_FILES=""
|
||||||
|
|
||||||
|
if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ] && [ -n "${GITHUB_REF_NAME:-}" ] && [ -n "${GITHUB_TOKEN:-}" ]; then
|
||||||
|
PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||')
|
||||||
|
API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=100"
|
||||||
|
set +e
|
||||||
|
RESPONSE=$(curl -s -w "\n%{http_code}" -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL")
|
||||||
|
HTTP_CODE=$(echo "$RESPONSE" | tail -n1)
|
||||||
|
BODY=$(echo "$RESPONSE" | sed '$d')
|
||||||
|
set -e
|
||||||
|
if [ "$HTTP_CODE" = "200" ]; then
|
||||||
|
CHANGED_PY_FILES=$(echo "$BODY" | python3 -c "
|
||||||
|
import json, sys
|
||||||
|
try:
|
||||||
|
files = json.load(sys.stdin)
|
||||||
|
py_files = [f['filename'] for f in files if f['filename'].endswith('.py') and f['status'] != 'removed']
|
||||||
|
print(' '.join(py_files))
|
||||||
|
except Exception:
|
||||||
|
print('')
|
||||||
|
")
|
||||||
|
if [ -n "$CHANGED_PY_FILES" ]; then
|
||||||
|
SCAN_MODE="incremental"
|
||||||
|
echo "Incremental mode: $(echo "$CHANGED_PY_FILES" | wc -w) Python files changed"
|
||||||
|
else
|
||||||
|
SCAN_MODE="skip_py"
|
||||||
|
echo "No Python files changed in this PR"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo "WARN: API returned HTTP $HTTP_CODE, falling back to full scan"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo "Full scan mode (not a PR event)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$SCAN_MODE" = "incremental" ]; then
|
||||||
|
# 防御性过滤
|
||||||
|
EXISTING_PY_FILES=""
|
||||||
|
for f in $CHANGED_PY_FILES; do
|
||||||
|
if [ -f "$f" ]; then
|
||||||
|
if [ -z "$EXISTING_PY_FILES" ]; then
|
||||||
|
EXISTING_PY_FILES="$f"
|
||||||
|
else
|
||||||
|
EXISTING_PY_FILES="$EXISTING_PY_FILES $f"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
CHANGED_PY_FILES="$EXISTING_PY_FILES"
|
||||||
|
|
||||||
|
python3 -m compileall -q $CHANGED_PY_FILES
|
||||||
|
python3 -m black --check --fast $CHANGED_PY_FILES
|
||||||
|
python3 -m isort --check-only $CHANGED_PY_FILES
|
||||||
|
RUFF_FILES=$(echo "$CHANGED_PY_FILES" | tr ' ' '\n' | grep -v '^scripts/' | grep -v '^$' | xargs)
|
||||||
|
if [ -n "$RUFF_FILES" ]; then
|
||||||
|
python3 -m ruff check $RUFF_FILES --statistics
|
||||||
|
else
|
||||||
|
echo "No ruff-checkable files changed, skipping"
|
||||||
|
fi
|
||||||
|
elif [ "$SCAN_MODE" = "skip_py" ]; then
|
||||||
|
echo "No Python files changed - skipping Python lint checks"
|
||||||
|
else
|
||||||
|
echo "Full scan mode"
|
||||||
|
python3 -m compileall -q alembic apps packages tests scripts
|
||||||
|
python3 -m black --check --fast alembic apps packages tests scripts
|
||||||
|
python3 -m isort --check-only alembic apps packages tests scripts
|
||||||
|
python3 -m ruff check apps packages tests --statistics
|
||||||
|
fi
|
||||||
|
echo "✅ Code quality checks passed"
|
||||||
|
|
||||||
|
# --- Bandit 安全扫描(仅告警) ---
|
||||||
|
echo ""
|
||||||
|
echo "=== [3/6] Security scan (bandit, advisory only) ==="
|
||||||
|
set +e
|
||||||
|
bandit -r apps packages -q -ll
|
||||||
|
BANDIT_EXIT=$?
|
||||||
|
set -e
|
||||||
|
if [ "$BANDIT_EXIT" -ne 0 ]; then
|
||||||
|
echo "⚠️ Bandit found security issues (advisory mode - not blocking CI)"
|
||||||
|
else
|
||||||
|
echo "✅ Bandit security scan passed"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- Pip-audit 依赖漏洞扫描(仅告警) ---
|
||||||
|
echo ""
|
||||||
|
echo "=== [4/6] Python dependency vulnerability scan (pip-audit, advisory only) ==="
|
||||||
|
python3 -m pip install -q pip-audit
|
||||||
|
pip-audit --version
|
||||||
|
EXIT_CODE=0
|
||||||
|
for req_file in requirements.txt requirements-base.txt requirements-dev.txt; do
|
||||||
|
if [ -f "$req_file" ]; then
|
||||||
|
echo "--- Scanning $req_file ---"
|
||||||
|
pip-audit -r "$req_file" --desc on 2>&1 | head -40 || EXIT_CODE=$?
|
||||||
|
echo ""
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
echo "pip-audit scan completed (advisory mode - warnings only, not blocking CI)"
|
||||||
|
|
||||||
|
# --- Vulture 死代码检测(仅告警) ---
|
||||||
|
echo ""
|
||||||
|
echo "=== [5/6] Dead code detection (vulture, advisory only) ==="
|
||||||
|
set +e
|
||||||
|
python3 -m pip install -q vulture
|
||||||
|
vulture --version
|
||||||
|
echo "告警模式,不阻断CI。置信度>=90%建议尽快确认。"
|
||||||
|
echo ""
|
||||||
|
vulture apps packages scripts \
|
||||||
|
--exclude "tests,test,migrations,.gitea,docs,node_modules,site-packages,*/test_*.py,*/conftest.py" \
|
||||||
|
--min-confidence 70 \
|
||||||
|
2>&1 | sort -t'(' -k2 -rn | head -80
|
||||||
|
echo ""
|
||||||
|
echo "=== vulture scan summary ==="
|
||||||
|
echo "发现潜在死代码(可能包含框架装饰器注册的函数,为误报)"
|
||||||
|
echo "建议:定期人工审查高置信度(>=90%)条目"
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# --- Release 脚本语法校验 ---
|
||||||
|
echo ""
|
||||||
|
echo "=== [6/6] Release scripts syntax validation ==="
|
||||||
|
bash -n scripts/backup_postgres.sh
|
||||||
|
bash -n scripts/restore_postgres_plan.sh
|
||||||
|
bash -n scripts/init_production_env.sh
|
||||||
|
echo "✅ Release scripts syntax OK"
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "=== CI Validate: 代码质量与安全扫描 全部通过 ✅ ==="
|
||||||
@@ -0,0 +1,182 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# CI Validate: Alembic迁移验证(并行Job 3/3)
|
||||||
|
# 需要PostgreSQL数据库
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
echo "=== CI Validate: Alembic迁移验证 ==="
|
||||||
|
|
||||||
|
# --- DooD模式检测:确定宿主机访问地址 ---
|
||||||
|
detect_docker_host() {
|
||||||
|
local test_port="${1:-5432}"
|
||||||
|
|
||||||
|
local candidates=()
|
||||||
|
|
||||||
|
# 1. host.docker.internal
|
||||||
|
if python3 -c "import socket; socket.gethostbyname('host.docker.internal')" 2>/dev/null; then
|
||||||
|
candidates+=("host.docker.internal")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 2. docker0 桥接网关
|
||||||
|
candidates+=("172.17.0.1")
|
||||||
|
|
||||||
|
# 3. 默认网关
|
||||||
|
local gw=""
|
||||||
|
gw=$(ip route 2>/dev/null | grep default | awk '{print $3}' | head -1)
|
||||||
|
if [ -n "$gw" ] && [ "$gw" != "127.0.0.1" ]; then
|
||||||
|
candidates+=("$gw")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 4. 宿主机同网段的.1或.254
|
||||||
|
local my_ip=""
|
||||||
|
my_ip=$(hostname -I 2>/dev/null | awk '{print $1}')
|
||||||
|
if [ -n "$my_ip" ]; then
|
||||||
|
local subnet=$(echo "$my_ip" | cut -d. -f1-3)
|
||||||
|
candidates+=("${subnet}.1")
|
||||||
|
candidates+=("${subnet}.254")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 5. 127.0.0.1 最后尝试
|
||||||
|
candidates+=("127.0.0.1")
|
||||||
|
|
||||||
|
for candidate in "${candidates[@]}"; do
|
||||||
|
if python3 -c "
|
||||||
|
import socket
|
||||||
|
s = socket.socket()
|
||||||
|
s.settimeout(2)
|
||||||
|
try:
|
||||||
|
s.connect(('$candidate', $test_port))
|
||||||
|
s.close()
|
||||||
|
print('ok')
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
" 2>/dev/null | grep -q ok; then
|
||||||
|
echo "$candidate"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "127.0.0.1"
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# 获取宿主机IP
|
||||||
|
if [ -S /var/run/docker.sock ]; then
|
||||||
|
DOCKER_HOST_IP=$(detect_docker_host 5433)
|
||||||
|
if [ "$DOCKER_HOST_IP" = "127.0.0.1" ]; then
|
||||||
|
DOCKER_HOST_IP=$(detect_docker_host 22)
|
||||||
|
fi
|
||||||
|
echo "检测到DooD模式,宿主机地址: $DOCKER_HOST_IP"
|
||||||
|
else
|
||||||
|
DOCKER_HOST_IP="127.0.0.1"
|
||||||
|
echo "非DooD模式,使用 127.0.0.1"
|
||||||
|
fi
|
||||||
|
PG_HOST="$DOCKER_HOST_IP"
|
||||||
|
echo "PG host: $PG_HOST"
|
||||||
|
|
||||||
|
# 指数退避TCP连接检查
|
||||||
|
wait_tcp_ready() {
|
||||||
|
local host="$1"
|
||||||
|
local port="$2"
|
||||||
|
local max_attempts="${3:-5}"
|
||||||
|
local delay=1
|
||||||
|
local attempt=1
|
||||||
|
while [ "$attempt" -le "$max_attempts" ]; do
|
||||||
|
if python3 -c "import socket; s=socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
echo "TCP连接尝试 $attempt/$max_attempts 失败,${delay}s后重试..."
|
||||||
|
sleep "$delay"
|
||||||
|
delay=$((delay * 2))
|
||||||
|
attempt=$((attempt + 1))
|
||||||
|
done
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
USE_SHARED_PG="${CI_USE_SHARED_PG:-false}"
|
||||||
|
|
||||||
|
if [ "$USE_SHARED_PG" = "true" ]; then
|
||||||
|
# 使用常驻共享PG实例
|
||||||
|
echo "使用常驻共享PG实例(CI_USE_SHARED_PG=true)"
|
||||||
|
SHARED_PG_HOST="$PG_HOST"
|
||||||
|
SHARED_PG_PORT="5433"
|
||||||
|
SHARED_PG_USER="postgres"
|
||||||
|
SHARED_PG_PASSWORD="ci_pg_2026!"
|
||||||
|
CI_DB_NAME="ci_run_${GITHUB_RUN_ID:-$$}"
|
||||||
|
|
||||||
|
echo "等待共享PG连接就绪..."
|
||||||
|
wait_tcp_ready "$SHARED_PG_HOST" "$SHARED_PG_PORT" 5
|
||||||
|
|
||||||
|
echo "创建测试数据库: $CI_DB_NAME"
|
||||||
|
PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c "
|
||||||
|
import psycopg2
|
||||||
|
conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres')
|
||||||
|
conn.autocommit = True
|
||||||
|
cur = conn.cursor()
|
||||||
|
cur.execute(f'CREATE DATABASE \"$CI_DB_NAME\"')
|
||||||
|
cur.close()
|
||||||
|
conn.close()
|
||||||
|
"
|
||||||
|
export DATABASE_URL="postgresql+psycopg://${SHARED_PG_USER}:${SHARED_PG_PASSWORD}@${SHARED_PG_HOST}:${SHARED_PG_PORT}/${CI_DB_NAME}"
|
||||||
|
echo "✅ 共享PG数据库已创建: $CI_DB_NAME"
|
||||||
|
|
||||||
|
# 执行迁移
|
||||||
|
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head
|
||||||
|
echo "✅ Alembic migrations applied successfully"
|
||||||
|
|
||||||
|
# 清理数据库
|
||||||
|
echo "清理测试数据库: $CI_DB_NAME"
|
||||||
|
PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c "
|
||||||
|
import psycopg2
|
||||||
|
conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres')
|
||||||
|
conn.autocommit = True
|
||||||
|
cur = conn.cursor()
|
||||||
|
cur.execute(f'DROP DATABASE IF EXISTS \"$CI_DB_NAME\" WITH (FORCE)')
|
||||||
|
cur.close()
|
||||||
|
conn.close()
|
||||||
|
" 2>/dev/null || echo "WARN: 数据库清理失败"
|
||||||
|
echo "✅ 共享PG数据库已清理"
|
||||||
|
else
|
||||||
|
# 使用临时PG容器(默认模式)
|
||||||
|
echo "使用临时PG容器模式"
|
||||||
|
PG_CONTAINER=ci-pg-validate-migration-${GITHUB_RUN_ID:-$$}
|
||||||
|
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||||
|
docker run -d --name "$PG_CONTAINER" \
|
||||||
|
--shm-size=256m \
|
||||||
|
-e POSTGRES_USER=postgres \
|
||||||
|
-e POSTGRES_PASSWORD=postgres \
|
||||||
|
-e POSTGRES_DB=xiaoxia_saas \
|
||||||
|
-P \
|
||||||
|
--health-cmd "pg_isready -U postgres" \
|
||||||
|
--health-interval 3s \
|
||||||
|
--health-timeout 3s \
|
||||||
|
--health-retries 20 \
|
||||||
|
postgres:16-alpine
|
||||||
|
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | cut -d: -f2)
|
||||||
|
echo "PostgreSQL port: $PG_PORT"
|
||||||
|
export DATABASE_URL="postgresql+psycopg://postgres:postgres@${PG_HOST}:${PG_PORT}/xiaoxia_saas"
|
||||||
|
|
||||||
|
# 等待容器健康
|
||||||
|
for i in $(seq 1 30); do
|
||||||
|
if docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then
|
||||||
|
echo "PostgreSQL container is healthy on port $PG_PORT"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
echo "Waiting for PostgreSQL container health... ($i/30)"
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" | grep -q healthy
|
||||||
|
|
||||||
|
# TCP连通性检查
|
||||||
|
echo "验证TCP连通性 ($PG_HOST:$PG_PORT)..."
|
||||||
|
wait_tcp_ready "$PG_HOST" "$PG_PORT" 5
|
||||||
|
echo "TCP connectivity to PostgreSQL confirmed on port $PG_PORT"
|
||||||
|
|
||||||
|
# 执行迁移
|
||||||
|
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head
|
||||||
|
echo "✅ Alembic migrations applied successfully"
|
||||||
|
|
||||||
|
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "=== CI Validate: Alembic迁移验证 通过 ✅ ==="
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# CI Validate: Mypy类型检查(并行Job 2/3)
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
echo "=== CI Validate: Mypy类型检查 ==="
|
||||||
|
|
||||||
|
bash scripts/ci/mypy_check.sh
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "=== CI Validate: Mypy类型检查 通过 ✅ ==="
|
||||||
Reference in New Issue
Block a user