Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1692ae135a | |||
| 3dc7a0a0bb |
@@ -1 +0,0 @@
|
|||||||
re-trigger
|
|
||||||
+24
-74
File diff suppressed because one or more lines are too long
@@ -61,7 +61,7 @@ class APIVersionMiddleware(BaseHTTPMiddleware):
|
|||||||
class VersionNotFoundMiddleware(BaseHTTPMiddleware):
|
class VersionNotFoundMiddleware(BaseHTTPMiddleware):
|
||||||
"""处理已下线的 API 版本"""
|
"""处理已下线的 API 版本"""
|
||||||
|
|
||||||
SUNSET_VERSIONS: list[str] = [] # 已下线的版本列表
|
SUNSET_VERSIONS = [] # 已下线的版本列表
|
||||||
|
|
||||||
async def dispatch(self, request: Request, call_next):
|
async def dispatch(self, request: Request, call_next):
|
||||||
version = self._extract_version(request.url.path)
|
version = self._extract_version(request.url.path)
|
||||||
|
|||||||
@@ -204,7 +204,7 @@ export const createAsset = async (data: {
|
|||||||
/** 更新素材(名称、metadata 等) */
|
/** 更新素材(名称、metadata 等) */
|
||||||
export const updateAsset = async (
|
export const updateAsset = async (
|
||||||
assetId: string,
|
assetId: string,
|
||||||
data: { name?: string; metadata?: AssetMetadata },
|
data: { name?: string; metadata?: Record<string, unknown> },
|
||||||
): Promise<AssetItem> => {
|
): Promise<AssetItem> => {
|
||||||
const response = await apiClient.put(`/assets/${assetId}`, data);
|
const response = await apiClient.put(`/assets/${assetId}`, data);
|
||||||
return response.data;
|
return response.data;
|
||||||
|
|||||||
@@ -129,8 +129,7 @@ apiClient.interceptors.response.use(
|
|||||||
const safeExtractString = (val: unknown): string => {
|
const safeExtractString = (val: unknown): string => {
|
||||||
if (typeof val === "string") return val;
|
if (typeof val === "string") return val;
|
||||||
if (typeof val === "object" && val !== null) {
|
if (typeof val === "object" && val !== null) {
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- 防御性错误提取,后端错误格式不固定
|
const obj = val as Record<string, unknown>;
|
||||||
const obj = val as Record<string, any>;
|
|
||||||
if (typeof obj.message === "string") return obj.message;
|
if (typeof obj.message === "string") return obj.message;
|
||||||
if (typeof obj.msg === "string") return obj.msg;
|
if (typeof obj.msg === "string") return obj.msg;
|
||||||
if (typeof obj.detail === "string") return obj.detail;
|
if (typeof obj.detail === "string") return obj.detail;
|
||||||
|
|||||||
@@ -97,30 +97,6 @@ export interface EditPlanConfig {
|
|||||||
green_screen_config?: ChromaKeyConfig;
|
green_screen_config?: ChromaKeyConfig;
|
||||||
sticker_config?: StickerConfig;
|
sticker_config?: StickerConfig;
|
||||||
cover_config?: CoverConfig;
|
cover_config?: CoverConfig;
|
||||||
/** 前端扩展:关联的素材 ID 列表 */
|
|
||||||
asset_ids?: string[];
|
|
||||||
/** 配音 ID */
|
|
||||||
voice_id?: string;
|
|
||||||
/** 克隆音色档案 ID */
|
|
||||||
voice_clone_profile_id?: string;
|
|
||||||
/** 自定义配音音频 URL */
|
|
||||||
custom_audio_url?: string;
|
|
||||||
/** 自定义配音文本 */
|
|
||||||
custom_text?: string;
|
|
||||||
/** 视频比例 */
|
|
||||||
ratio?: string;
|
|
||||||
/** 视频风格 */
|
|
||||||
style?: string;
|
|
||||||
/** 目标时长(秒) */
|
|
||||||
duration?: number;
|
|
||||||
/** 是否自动生成字幕 */
|
|
||||||
auto_subtitles?: boolean;
|
|
||||||
/** 是否启用 BGM */
|
|
||||||
bgm?: boolean;
|
|
||||||
/** 生成数量 */
|
|
||||||
generate_count?: number;
|
|
||||||
/** 素材模式 */
|
|
||||||
material_mode?: string;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 剪辑计划(后端响应) */
|
/** 剪辑计划(后端响应) */
|
||||||
@@ -194,7 +170,7 @@ export interface GenerationStatusResponse {
|
|||||||
export interface GeneratedVideo {
|
export interface GeneratedVideo {
|
||||||
id: string;
|
id: string;
|
||||||
project_id?: string;
|
project_id?: string;
|
||||||
generation_task_id?: string;
|
generation_task_id: string;
|
||||||
name: string;
|
name: string;
|
||||||
file_url: string;
|
file_url: string;
|
||||||
file_size?: number;
|
file_size?: number;
|
||||||
@@ -206,8 +182,6 @@ export interface GeneratedVideo {
|
|||||||
status: string;
|
status: string;
|
||||||
review_status?: string;
|
review_status?: string;
|
||||||
download_url?: string;
|
download_url?: string;
|
||||||
created_at?: string;
|
|
||||||
updated_at?: string;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ============================================================
|
/* ============================================================
|
||||||
@@ -252,15 +226,13 @@ export interface GenerateCoverRequest {
|
|||||||
/** AI 封面生成响应 */
|
/** AI 封面生成响应 */
|
||||||
export interface GenerateCoverResponse {
|
export interface GenerateCoverResponse {
|
||||||
plan_id: string;
|
plan_id: string;
|
||||||
cover: CoverResult;
|
cover: {
|
||||||
}
|
scheme?: string;
|
||||||
|
asset_id?: string;
|
||||||
/** 封面生成结果 */
|
frame_time?: number;
|
||||||
export interface CoverResult {
|
thumbnail_url?: string;
|
||||||
scheme?: string;
|
[key: string]: unknown;
|
||||||
asset_id?: string;
|
};
|
||||||
frame_time?: number;
|
|
||||||
thumbnail_url?: string;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ============================================================
|
/* ============================================================
|
||||||
|
|||||||
@@ -139,23 +139,11 @@ export interface GenerateFromTemplatePayload {
|
|||||||
voiceover_duration: number;
|
voiceover_duration: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 验证警告详情 */
|
|
||||||
export interface ValidationWarningDetails {
|
|
||||||
/** 相关字段名 */
|
|
||||||
field?: string;
|
|
||||||
/** 期望值 */
|
|
||||||
expected?: string | number;
|
|
||||||
/** 实际值 */
|
|
||||||
actual?: string | number;
|
|
||||||
/** 建议值 */
|
|
||||||
suggested?: string | number;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** 验证/生成响应 */
|
/** 验证/生成响应 */
|
||||||
export interface ValidateWarning {
|
export interface ValidateWarning {
|
||||||
code: string;
|
code: string;
|
||||||
message: string;
|
message: string;
|
||||||
details?: ValidationWarningDetails;
|
details?: Record<string, unknown>;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 使用模板生成响应 */
|
/** 使用模板生成响应 */
|
||||||
|
|||||||
@@ -60,26 +60,33 @@ export interface BatchDownloadStatus {
|
|||||||
/**
|
/**
|
||||||
* 将 generation task 数据映射为 ProductItem 格式
|
* 将 generation task 数据映射为 ProductItem 格式
|
||||||
*/
|
*/
|
||||||
function mapTaskToProductItem(task: GeneratedVideo): ProductItem {
|
function mapTaskToProductItem(
|
||||||
|
task: GeneratedVideo | Record<string, unknown>,
|
||||||
|
): ProductItem {
|
||||||
|
const video = task as GeneratedVideo;
|
||||||
return {
|
return {
|
||||||
id: task.id,
|
id: video.id,
|
||||||
title: task.name || "未命名视频",
|
title: video.name || "未命名视频",
|
||||||
video_url: task.file_url,
|
video_url: video.file_url,
|
||||||
thumbnail_url: task.thumbnail_url,
|
thumbnail_url: video.thumbnail_url,
|
||||||
duration_seconds: task.duration,
|
duration_seconds: video.duration,
|
||||||
file_size: task.file_size,
|
file_size: video.file_size,
|
||||||
resolution:
|
resolution:
|
||||||
task.width && task.height ? `${task.width}x${task.height}` : undefined,
|
video.width && video.height
|
||||||
|
? `${video.width}x${video.height}`
|
||||||
|
: undefined,
|
||||||
status:
|
status:
|
||||||
task.status === "completed"
|
video.status === "completed"
|
||||||
? "completed"
|
? "completed"
|
||||||
: task.status === "failed"
|
: video.status === "failed"
|
||||||
? "failed"
|
? "failed"
|
||||||
: "processing",
|
: "processing",
|
||||||
review_status: task.review_status as ReviewStatus | undefined,
|
review_status: video.review_status as ReviewStatus | undefined,
|
||||||
project_id: task.project_id,
|
project_id: video.project_id,
|
||||||
created_at: task.created_at,
|
created_at: (task as Record<string, unknown>).created_at as
|
||||||
updated_at: task.updated_at,
|
string | undefined,
|
||||||
|
updated_at: (task as Record<string, unknown>).updated_at as
|
||||||
|
string | undefined,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+2
-14
@@ -8,18 +8,6 @@ import apiClient from "./client";
|
|||||||
|
|
||||||
/* ── 类型定义 ──────────────────────────────────── */
|
/* ── 类型定义 ──────────────────────────────────── */
|
||||||
|
|
||||||
/** TTS 元数据(合成时附带的扩展信息) */
|
|
||||||
export interface TTSMetadata {
|
|
||||||
/** 语音时长(秒) */
|
|
||||||
duration?: number;
|
|
||||||
/** 采样率(Hz) */
|
|
||||||
sample_rate?: number;
|
|
||||||
/** 语言 */
|
|
||||||
language?: string;
|
|
||||||
/** 其他扩展字段 */
|
|
||||||
[key: string]: unknown;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** TTS 合成请求参数 */
|
/** TTS 合成请求参数 */
|
||||||
export interface TTSSynthesizeRequest {
|
export interface TTSSynthesizeRequest {
|
||||||
text: string;
|
text: string;
|
||||||
@@ -30,7 +18,7 @@ export interface TTSSynthesizeRequest {
|
|||||||
voice_model?: string;
|
voice_model?: string;
|
||||||
voice_clone_profile_id?: string;
|
voice_clone_profile_id?: string;
|
||||||
format?: string;
|
format?: string;
|
||||||
metadata?: TTSMetadata;
|
metadata?: Record<string, unknown>;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** TTS 合成创建响应 */
|
/** TTS 合成创建响应 */
|
||||||
@@ -61,7 +49,7 @@ export interface TTSJob {
|
|||||||
error_message: string | null;
|
error_message: string | null;
|
||||||
retry_count: number;
|
retry_count: number;
|
||||||
max_retries: number;
|
max_retries: number;
|
||||||
metadata_: TTSMetadata | null;
|
metadata_: Record<string, unknown> | null;
|
||||||
created_at: string;
|
created_at: string;
|
||||||
updated_at: string;
|
updated_at: string;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -36,18 +36,6 @@ export interface CreateVoiceCloneRequest {
|
|||||||
|
|
||||||
/* ── 后端 API 类型 ────────────────────────────────────── */
|
/* ── 后端 API 类型 ────────────────────────────────────── */
|
||||||
|
|
||||||
/** 音色克隆元数据(克隆时附带的扩展信息) */
|
|
||||||
export interface VoiceCloneMetadata {
|
|
||||||
/** 语音时长(秒) */
|
|
||||||
duration?: number;
|
|
||||||
/** 采样率(Hz) */
|
|
||||||
sample_rate?: number;
|
|
||||||
/** 音色 ID(克隆完成后分配) */
|
|
||||||
voice_id?: string;
|
|
||||||
/** 其他扩展字段 */
|
|
||||||
[key: string]: unknown;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** 后端克隆档案响应 */
|
/** 后端克隆档案响应 */
|
||||||
export interface VoiceCloneProfile {
|
export interface VoiceCloneProfile {
|
||||||
id: string;
|
id: string;
|
||||||
@@ -63,7 +51,7 @@ export interface VoiceCloneProfile {
|
|||||||
error_message: string | null;
|
error_message: string | null;
|
||||||
retry_count: number;
|
retry_count: number;
|
||||||
max_retries: number;
|
max_retries: number;
|
||||||
metadata_: VoiceCloneMetadata | null;
|
metadata_: Record<string, unknown> | null;
|
||||||
created_at: string;
|
created_at: string;
|
||||||
updated_at: string;
|
updated_at: string;
|
||||||
}
|
}
|
||||||
@@ -92,7 +80,7 @@ export interface CreateVoiceCloneRequestFull {
|
|||||||
language?: string;
|
language?: string;
|
||||||
gender?: string;
|
gender?: string;
|
||||||
max_retries?: number;
|
max_retries?: number;
|
||||||
metadata_?: VoiceCloneMetadata;
|
metadata_?: Record<string, unknown>;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ── 辅助函数 ─────────────────────────────────────────── */
|
/* ── 辅助函数 ─────────────────────────────────────────── */
|
||||||
|
|||||||
@@ -202,8 +202,8 @@ const GeneratePage: React.FC = () => {
|
|||||||
try {
|
try {
|
||||||
const plan = await getEditPlan(editPlanId);
|
const plan = await getEditPlan(editPlanId);
|
||||||
if (plan.name) setTitle(plan.name);
|
if (plan.name) setTitle(plan.name);
|
||||||
const cfg = plan.config;
|
const cfg = plan.config as Record<string, unknown>;
|
||||||
if (cfg?.asset_ids) {
|
if (cfg && Array.isArray(cfg.asset_ids)) {
|
||||||
setSelectedMaterials(
|
setSelectedMaterials(
|
||||||
cfg.asset_ids.filter((v): v is string => typeof v === "string"),
|
cfg.asset_ids.filter((v): v is string => typeof v === "string"),
|
||||||
);
|
);
|
||||||
@@ -477,13 +477,7 @@ const GeneratePage: React.FC = () => {
|
|||||||
setGenerateError(null);
|
setGenerateError(null);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const voiceConfig: Pick<
|
const voiceConfig: Record<string, unknown> = {};
|
||||||
EditPlanConfig,
|
|
||||||
| "voice_id"
|
|
||||||
| "voice_clone_profile_id"
|
|
||||||
| "custom_audio_url"
|
|
||||||
| "custom_text"
|
|
||||||
> = {};
|
|
||||||
if (voiceMode === "preset") {
|
if (voiceMode === "preset") {
|
||||||
voiceConfig.voice_id = selectedVoice || undefined;
|
voiceConfig.voice_id = selectedVoice || undefined;
|
||||||
} else if (voiceMode === "clone") {
|
} else if (voiceMode === "clone") {
|
||||||
@@ -508,7 +502,7 @@ const GeneratePage: React.FC = () => {
|
|||||||
bgm,
|
bgm,
|
||||||
generate_count: generateCount,
|
generate_count: generateCount,
|
||||||
material_mode: materialMode,
|
material_mode: materialMode,
|
||||||
},
|
} as EditPlanConfig,
|
||||||
total_duration: duration,
|
total_duration: duration,
|
||||||
source_edit_plan_id: editPlanId || undefined,
|
source_edit_plan_id: editPlanId || undefined,
|
||||||
});
|
});
|
||||||
@@ -567,8 +561,7 @@ const GeneratePage: React.FC = () => {
|
|||||||
const safeExtract = (val: unknown): string => {
|
const safeExtract = (val: unknown): string => {
|
||||||
if (typeof val === "string") return val;
|
if (typeof val === "string") return val;
|
||||||
if (typeof val === "object" && val !== null) {
|
if (typeof val === "object" && val !== null) {
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- 防御性错误提取,后端错误格式不固定
|
const obj = val as Record<string, unknown>;
|
||||||
const obj = val as Record<string, any>;
|
|
||||||
if (typeof obj.message === "string") return obj.message;
|
if (typeof obj.message === "string") return obj.message;
|
||||||
if (typeof obj.msg === "string") return obj.msg;
|
if (typeof obj.msg === "string") return obj.msg;
|
||||||
if (typeof obj.detail === "string") return obj.detail;
|
if (typeof obj.detail === "string") return obj.detail;
|
||||||
@@ -628,8 +621,7 @@ const GeneratePage: React.FC = () => {
|
|||||||
const extractString = (val: unknown): string => {
|
const extractString = (val: unknown): string => {
|
||||||
if (typeof val === "string") return val;
|
if (typeof val === "string") return val;
|
||||||
if (typeof val === "object" && val !== null) {
|
if (typeof val === "object" && val !== null) {
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- 防御性错误提取,后端错误格式不固定
|
const obj = val as Record<string, unknown>;
|
||||||
const obj = val as Record<string, any>;
|
|
||||||
if (typeof obj.message === "string") return obj.message;
|
if (typeof obj.message === "string") return obj.message;
|
||||||
if (typeof obj.msg === "string") return obj.msg;
|
if (typeof obj.msg === "string") return obj.msg;
|
||||||
if (typeof obj.detail === "string") return obj.detail;
|
if (typeof obj.detail === "string") return obj.detail;
|
||||||
@@ -659,8 +651,7 @@ const GeneratePage: React.FC = () => {
|
|||||||
const safeExtractErr = (val: unknown): string => {
|
const safeExtractErr = (val: unknown): string => {
|
||||||
if (typeof val === "string") return val;
|
if (typeof val === "string") return val;
|
||||||
if (typeof val === "object" && val !== null) {
|
if (typeof val === "object" && val !== null) {
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- 防御性错误提取,后端错误格式不固定
|
const obj = val as Record<string, unknown>;
|
||||||
const obj = val as Record<string, any>;
|
|
||||||
if (typeof obj.message === "string") return obj.message;
|
if (typeof obj.message === "string") return obj.message;
|
||||||
if (typeof obj.msg === "string") return obj.msg;
|
if (typeof obj.msg === "string") return obj.msg;
|
||||||
if (typeof obj.detail === "string") return obj.detail;
|
if (typeof obj.detail === "string") return obj.detail;
|
||||||
|
|||||||
@@ -99,20 +99,10 @@ const formatDuration = (seconds: number): string => {
|
|||||||
return `${m}分${s > 0 ? `${s}秒` : ""}`;
|
return `${m}分${s > 0 ? `${s}秒` : ""}`;
|
||||||
};
|
};
|
||||||
|
|
||||||
/** 配置展示字段(formatConfig 提取通用配置的可读属性) */
|
|
||||||
interface ConfigDisplayFields {
|
|
||||||
font_size?: string | number;
|
|
||||||
font_family?: string;
|
|
||||||
color?: string;
|
|
||||||
position?: string;
|
|
||||||
volume?: string | number;
|
|
||||||
name?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** 格式化配置对象为可读文本 */
|
/** 格式化配置对象为可读文本 */
|
||||||
const formatConfig = (config?: object): string => {
|
const formatConfig = (config?: object): string => {
|
||||||
if (!config || Object.keys(config).length === 0) return "默认";
|
if (!config || Object.keys(config).length === 0) return "默认";
|
||||||
const c = config as ConfigDisplayFields;
|
const c = config as Record<string, unknown>;
|
||||||
const parts: string[] = [];
|
const parts: string[] = [];
|
||||||
if (c.font_size) parts.push(`字号: ${c.font_size}`);
|
if (c.font_size) parts.push(`字号: ${c.font_size}`);
|
||||||
if (c.font_family) parts.push(`字体: ${c.font_family}`);
|
if (c.font_family) parts.push(`字体: ${c.font_family}`);
|
||||||
|
|||||||
@@ -130,20 +130,12 @@ const mapAssetToMaterial = (asset: AssetItem): VoiceMaterial => {
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
/** 配音素材上传元数据(传递给 createAsset 的 metadata) */
|
|
||||||
interface VoiceAssetMetadata {
|
|
||||||
gender: VoiceGender;
|
|
||||||
description: string;
|
|
||||||
duration: number;
|
|
||||||
[key: string]: unknown;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** 前端表单数据 → 后端 metadata(标签走独立 API,不再写 metadata.style) */
|
/** 前端表单数据 → 后端 metadata(标签走独立 API,不再写 metadata.style) */
|
||||||
const buildMetadata = (data: {
|
const buildMetadata = (data: {
|
||||||
gender: VoiceGender;
|
gender: VoiceGender;
|
||||||
description: string;
|
description: string;
|
||||||
duration?: number;
|
duration?: number;
|
||||||
}): VoiceAssetMetadata => ({
|
}): Record<string, unknown> => ({
|
||||||
gender: data.gender,
|
gender: data.gender,
|
||||||
description: data.description,
|
description: data.description,
|
||||||
duration: data.duration || 0,
|
duration: data.duration || 0,
|
||||||
|
|||||||
@@ -623,20 +623,12 @@ const getAudioDuration = (file: File): Promise<number> =>
|
|||||||
audio.src = url;
|
audio.src = url;
|
||||||
});
|
});
|
||||||
|
|
||||||
/** 音色上传元数据(传递给 createAsset 的 metadata) */
|
|
||||||
interface VoiceUploadMetadata {
|
|
||||||
gender?: string;
|
|
||||||
description?: string;
|
|
||||||
duration?: number;
|
|
||||||
[key: string]: unknown;
|
|
||||||
}
|
|
||||||
|
|
||||||
const buildVoiceMetadata = (data: {
|
const buildVoiceMetadata = (data: {
|
||||||
gender?: string;
|
gender?: string;
|
||||||
description?: string;
|
description?: string;
|
||||||
duration?: number;
|
duration?: number;
|
||||||
}): VoiceUploadMetadata => {
|
}): Record<string, unknown> => {
|
||||||
const metadata: VoiceUploadMetadata = {};
|
const metadata: Record<string, unknown> = {};
|
||||||
if (data.gender) metadata.gender = data.gender;
|
if (data.gender) metadata.gender = data.gender;
|
||||||
if (data.description) metadata.description = data.description;
|
if (data.description) metadata.description = data.description;
|
||||||
if (data.duration) metadata.duration = Math.round(data.duration);
|
if (data.duration) metadata.duration = Math.round(data.duration);
|
||||||
|
|||||||
@@ -1,28 +1,23 @@
|
|||||||
"""FFmpeg 工具函数 — Worker 层.
|
"""FFmpeg 工具函数 — 共享原语.
|
||||||
|
|
||||||
业务相关的滤镜构建、视频探测、视频标准化等能力放在这里;
|
提供 FFmpeg / FFprobe 调用、视频信息探测、视频标准化、xfade 转场滤镜构建
|
||||||
底层原语(run_ffmpeg / 二进制路径 / 默认超时)已下沉到 packages/shared/ffmpeg_utils.py,
|
等底层能力,供 UnifiedRenderService、VideoComposeService 等复用。
|
||||||
本模块 re-export 保持向后兼容。
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import shutil
|
||||||
import subprocess # nosec B404
|
import subprocess # nosec B404
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
# 底层原语从 shared 层导入,application 层和 worker 层共用同一份实现
|
|
||||||
from shared.ffmpeg_utils import ( # noqa: F401
|
|
||||||
DEFAULT_FFMPEG_TIMEOUT,
|
|
||||||
FFMPEG_BIN,
|
|
||||||
FFPROBE_BIN,
|
|
||||||
run_ffmpeg,
|
|
||||||
)
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# ── 常量(Worker 层业务相关) ────────────────────────────────────────────────
|
# ── 常量 ──────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
FFMPEG_BIN: str = shutil.which("ffmpeg") or "ffmpeg"
|
||||||
|
FFPROBE_BIN: str = shutil.which("ffprobe") or "ffprobe"
|
||||||
|
|
||||||
DEFAULT_OUTPUT_WIDTH = 1280
|
DEFAULT_OUTPUT_WIDTH = 1280
|
||||||
DEFAULT_OUTPUT_HEIGHT = 720
|
DEFAULT_OUTPUT_HEIGHT = 720
|
||||||
@@ -66,8 +61,62 @@ XFADE_TRANSITION_MAP: dict[str, str] = {
|
|||||||
|
|
||||||
DEFAULT_TRANSITION_DURATION = 0.5
|
DEFAULT_TRANSITION_DURATION = 0.5
|
||||||
|
|
||||||
|
# FFmpeg 执行默认超时(秒),防止 FFmpeg hang 住导致 worker 永久阻塞
|
||||||
|
# 默认 30 分钟,足够处理大部分短视频渲染;超长视频可单独传参覆盖
|
||||||
|
DEFAULT_FFMPEG_TIMEOUT = 1800
|
||||||
|
|
||||||
# ── FFprobe 探测 ──────────────────────────────────────────────────────────────
|
|
||||||
|
# ── FFmpeg 执行 ───────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def run_ffmpeg(
|
||||||
|
command: list[str],
|
||||||
|
*,
|
||||||
|
capture_output: bool = True,
|
||||||
|
timeout: int | None = DEFAULT_FFMPEG_TIMEOUT,
|
||||||
|
) -> tuple[str, str]:
|
||||||
|
"""执行 FFmpeg 命令。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
command: 完整的 ffmpeg 命令列表(含 "ffmpeg" 本身)
|
||||||
|
capture_output: 是否捕获 stdout/stderr
|
||||||
|
timeout: 超时时间(秒),默认 1800s(30分钟);None 表示不设超时(不推荐)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
(stdout, stderr) 元组
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
subprocess.CalledProcessError: 命令执行失败时抛出,
|
||||||
|
异常信息包含完整 stderr 以便排查。
|
||||||
|
subprocess.TimeoutExpired: 超时未完成时抛出,FFmpeg 进程会被 kill。
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
result = subprocess.run( # nosec B603
|
||||||
|
command,
|
||||||
|
check=True,
|
||||||
|
stdout=subprocess.PIPE if capture_output else None,
|
||||||
|
stderr=subprocess.PIPE if capture_output else None,
|
||||||
|
text=True,
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
return (result.stdout or "", result.stderr or "")
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
logger.error(
|
||||||
|
"FFmpeg 命令超时 (%ds): command=%s",
|
||||||
|
timeout or -1,
|
||||||
|
" ".join(str(c) for c in command[:20]),
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
# 把完整 stderr 打到日志,方便排查 exit code 183 等问题
|
||||||
|
stderr_text = (e.stderr or "").strip()
|
||||||
|
logger.error(
|
||||||
|
"FFmpeg 命令失败: exit_code=%d command=%s\nstderr:\n%s",
|
||||||
|
e.returncode,
|
||||||
|
" ".join(str(c) for c in command[:20]), # 截断过长的命令
|
||||||
|
stderr_text[:5000], # 截断过长的 stderr
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
def run_ffprobe(
|
def run_ffprobe(
|
||||||
|
|||||||
@@ -186,8 +186,8 @@ class TrimEngine:
|
|||||||
FFmpeg filter 字符串,如 "[0:v]trim=start=10:duration=5,setpts=PTS-STARTPTS[v0_trimmed]"
|
FFmpeg filter 字符串,如 "[0:v]trim=start=10:duration=5,setpts=PTS-STARTPTS[v0_trimmed]"
|
||||||
"""
|
"""
|
||||||
if trim.is_noop:
|
if trim.is_noop:
|
||||||
# 不裁剪,直接直通(仅重置时间戳)
|
# 不裁剪,直接直通
|
||||||
return f"{input_label}setpts=PTS-STARTPTS{output_label}"
|
return f"{input_label}copy{output_label}" if False else f"{input_label}setpts=PTS-STARTPTS{output_label}"
|
||||||
|
|
||||||
parts: list[str] = []
|
parts: list[str] = []
|
||||||
|
|
||||||
|
|||||||
Executable → Regular
+4
-2
@@ -640,9 +640,11 @@ class UnifiedRenderService:
|
|||||||
return timeline
|
return timeline
|
||||||
|
|
||||||
def _extract_audio(self, video_path: Path, output_path: Path) -> None:
|
def _extract_audio(self, video_path: Path, output_path: Path) -> None:
|
||||||
"""从视频中提取音频为16kHz单声道wav(ASR友好格式)。"""
|
"""从视频中提取音频为16kHz单声道wav(ASR友好格式)."""
|
||||||
|
from video_processing.ffmpeg_utils import run_ffmpeg
|
||||||
|
|
||||||
cmd = [
|
cmd = [
|
||||||
FFMPEG_BIN,
|
"ffmpeg",
|
||||||
"-y",
|
"-y",
|
||||||
"-i",
|
"-i",
|
||||||
str(video_path),
|
str(video_path),
|
||||||
|
|||||||
Executable → Regular
+8
-58
@@ -433,32 +433,24 @@ def _prepare_bgm_track(
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _verify_url_accessible(
|
def _verify_url_accessible(url: str, timeout: float = 10.0, retries: int = 2) -> bool:
|
||||||
url: str,
|
|
||||||
timeout: float = 10.0,
|
|
||||||
retries: int = 2,
|
|
||||||
max_redirects: int = 5,
|
|
||||||
) -> bool:
|
|
||||||
"""HEAD 请求校验 URL 可访问(含重试,防止 OSS 抖动误报)。
|
"""HEAD 请求校验 URL 可访问(含重试,防止 OSS 抖动误报)。
|
||||||
|
|
||||||
安全增强:
|
安全:
|
||||||
- 请求前先做 SSRF 安全校验(内网IP/回环地址/链路本地地址等)
|
- 请求前先做 SSRF 安全校验(内网IP/回环地址/链路本地地址等)
|
||||||
- scheme 仅允许 http/https
|
- scheme 仅允许 http/https
|
||||||
- 端口仅允许 80/443
|
- 端口仅允许 80/443
|
||||||
- 手动跟随重定向,每一跳 URL 都做 SSRF 校验,避免重定向到内网地址绕过
|
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
url: 待校验的 URL
|
url: 待校验的 URL
|
||||||
timeout: 单次请求超时时间(秒)
|
timeout: 单次请求超时时间(秒)
|
||||||
retries: 最大重试次数(默认 2 次,首次失败后间隔 1s 重试)
|
retries: 最大重试次数(默认 2 次,首次失败后间隔 1s 重试)
|
||||||
max_redirects: 最大重定向次数(默认 5 次)
|
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
True 表示 URL 可访问(HTTP 2xx/3xx),False 表示所有尝试均失败或安全校验不通过。
|
True 表示 URL 可访问(HTTP 2xx/3xx),False 表示所有尝试均失败或安全校验不通过。
|
||||||
"""
|
"""
|
||||||
import time
|
import time
|
||||||
import urllib.request
|
import urllib.request
|
||||||
from urllib.parse import urljoin
|
|
||||||
|
|
||||||
from video_processing.url_security import UrlSecurityError, validate_url_safety
|
from video_processing.url_security import UrlSecurityError, validate_url_safety
|
||||||
|
|
||||||
@@ -470,56 +462,14 @@ def _verify_url_accessible(
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
last_error: Exception | None = None
|
last_error: Exception | None = None
|
||||||
|
|
||||||
def _do_verify(current_url: str) -> bool:
|
|
||||||
"""单次校验:手动跟随重定向,每跳都做 SSRF 检查."""
|
|
||||||
redirect_count = 0
|
|
||||||
url_being_checked = current_url
|
|
||||||
|
|
||||||
# 禁止自动重定向的 handler,手动控制每一跳
|
|
||||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
||||||
def redirect_request(self, req, fp, code, msg, headers, newurl): # noqa: N802
|
|
||||||
return None
|
|
||||||
|
|
||||||
opener = urllib.request.build_opener(NoRedirect())
|
|
||||||
|
|
||||||
while redirect_count <= max_redirects:
|
|
||||||
# 每一跳都做 SSRF 安全校验
|
|
||||||
try:
|
|
||||||
safe_url = validate_url_safety(url_being_checked, purpose="url_verify")
|
|
||||||
except UrlSecurityError as e:
|
|
||||||
logger.warning(
|
|
||||||
"URL校验跳转地址不安全: redirect=%d url=%s error=%s",
|
|
||||||
redirect_count,
|
|
||||||
url_being_checked,
|
|
||||||
e,
|
|
||||||
)
|
|
||||||
raise
|
|
||||||
|
|
||||||
req = urllib.request.Request(safe_url, method="HEAD")
|
|
||||||
req.add_header("User-Agent", "xiaoxia-saas-worker/1.0")
|
|
||||||
|
|
||||||
with opener.open(req, timeout=timeout) as resp: # noqa: S310
|
|
||||||
if 200 <= resp.status < 300:
|
|
||||||
return True
|
|
||||||
if resp.status in (301, 302, 303, 307, 308):
|
|
||||||
location = resp.headers.get("Location", "")
|
|
||||||
if not location:
|
|
||||||
raise Exception(f"HTTP {resp.status} 但无 Location 头")
|
|
||||||
# 相对路径转绝对
|
|
||||||
url_being_checked = urljoin(safe_url, location)
|
|
||||||
redirect_count += 1
|
|
||||||
continue
|
|
||||||
if resp.status < 400:
|
|
||||||
return True
|
|
||||||
raise Exception(f"HTTP {resp.status}")
|
|
||||||
|
|
||||||
raise Exception(f"重定向次数超过上限 ({max_redirects})")
|
|
||||||
|
|
||||||
for attempt in range(1 + retries):
|
for attempt in range(1 + retries):
|
||||||
try:
|
try:
|
||||||
if _do_verify(url):
|
req = urllib.request.Request(url, method="HEAD")
|
||||||
return True
|
req.add_header("User-Agent", "xiaoxia-saas-worker/1.0")
|
||||||
|
with urllib.request.urlopen(req, timeout=timeout) as resp: # nosec B310
|
||||||
|
if resp.status < 400:
|
||||||
|
return True
|
||||||
|
last_error = Exception(f"HTTP {resp.status}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
last_error = e
|
last_error = e
|
||||||
|
|
||||||
|
|||||||
@@ -1,104 +1,48 @@
|
|||||||
# ============================================================
|
# ============================================================
|
||||||
# Worker Dockerfile - 优化版(多阶段构建 + 镜像瘦身)
|
# Worker Dockerfile - 专门用于 Celery Worker
|
||||||
# 优化项:
|
# 优化:依赖分层缓存,基础大包和业务依赖分开
|
||||||
# 1. 多阶段构建:builder 阶段安装编译依赖,runtime 阶段只保留运行时
|
|
||||||
# 2. ffmpeg 静态编译替换:从 apt 安装(457MB) 改为静态二进制(~80MB)
|
|
||||||
# 3. Python 依赖瘦身:strip .so 调试符号 + 清理测试文件 + 清理缓存
|
|
||||||
# ============================================================
|
# ============================================================
|
||||||
|
|
||||||
# ==================== Builder 阶段 ====================
|
# 基础镜像:Python 3.12 + ffmpeg
|
||||||
FROM git.xiaoxiajianji.com/xiaoxia/base/python:3.12-slim AS builder
|
FROM git.xiaoxiajianji.com/xiaoxia/base/python:3.12-slim
|
||||||
|
|
||||||
# 使用阿里云镜像加速
|
# 构建参数:版本号(CI 传入 commit hash)
|
||||||
RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \
|
|
||||||
sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list 2>/dev/null || true
|
|
||||||
|
|
||||||
# 安装编译工具(仅 builder 需要)
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
||||||
gcc \
|
|
||||||
g++ \
|
|
||||||
python3-dev \
|
|
||||||
binutils \
|
|
||||||
wget \
|
|
||||||
xz-utils \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
# ---- 下载静态编译 ffmpeg ----
|
|
||||||
# 使用 johnvansickle.com 的静态编译版本(业界标准)
|
|
||||||
RUN cd /tmp \
|
|
||||||
&& wget -q https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz \
|
|
||||||
&& tar xf ffmpeg-release-amd64-static.tar.xz \
|
|
||||||
&& cp ffmpeg-*-amd64-static/ffmpeg /usr/local/bin/ffmpeg \
|
|
||||||
&& cp ffmpeg-*-amd64-static/ffprobe /usr/local/bin/ffprobe \
|
|
||||||
&& chmod +x /usr/local/bin/ffmpeg /usr/local/bin/ffprobe \
|
|
||||||
&& rm -rf ffmpeg-*
|
|
||||||
|
|
||||||
# ---- 安装 Python 依赖 ----
|
|
||||||
WORKDIR /tmp
|
|
||||||
|
|
||||||
# 创建 venv
|
|
||||||
RUN python -m venv /opt/venv
|
|
||||||
ENV PATH="/opt/venv/bin:$PATH"
|
|
||||||
|
|
||||||
# 基础依赖
|
|
||||||
COPY requirements-base.txt /tmp/requirements-base.txt
|
|
||||||
RUN pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com \
|
|
||||||
-r /tmp/requirements-base.txt \
|
|
||||||
&& rm /tmp/requirements-base.txt
|
|
||||||
|
|
||||||
# Worker 专属大包
|
|
||||||
COPY requirements-worker.txt /tmp/requirements-worker.txt
|
|
||||||
RUN pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com \
|
|
||||||
-r /tmp/requirements-worker.txt \
|
|
||||||
&& rm /tmp/requirements-worker.txt
|
|
||||||
|
|
||||||
# 业务依赖
|
|
||||||
COPY requirements.txt /tmp/requirements.txt
|
|
||||||
RUN pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com \
|
|
||||||
-r /tmp/requirements.txt \
|
|
||||||
&& rm /tmp/requirements.txt
|
|
||||||
|
|
||||||
# ---- Python 依赖瘦身 ----
|
|
||||||
# 1. strip .so 文件的调试符号(节省约 80-100MB)
|
|
||||||
RUN find /opt/venv -name "*.so" -type f -exec strip --strip-all {} \; 2>/dev/null || true
|
|
||||||
|
|
||||||
# 2. 清理测试文件(节省约 20MB)
|
|
||||||
RUN find /opt/venv -type d -name "tests" -exec rm -rf {} + 2>/dev/null; \
|
|
||||||
find /opt/venv -type d -name "test" -exec rm -rf {} + 2>/dev/null; \
|
|
||||||
find /opt/venv -name "test_*.py" -delete 2>/dev/null || true
|
|
||||||
|
|
||||||
# 3. 清理 .pyc 缓存和 __pycache__(节省约 10MB,运行时按需生成)
|
|
||||||
RUN find /opt/venv -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null; \
|
|
||||||
find /opt/venv -name "*.pyc" -delete 2>/dev/null || true
|
|
||||||
|
|
||||||
# 4. 清理 dist-info 中的文档
|
|
||||||
RUN find /opt/venv -name "*.dist-info" -type d -exec sh -c 'rm -f "$1"/DESCRIPTION.rst "$1"/INSTALLER "$1"/LICENSE* "$1"/WHEEL "$1"/entry_points.txt' _ {} \; 2>/dev/null || true
|
|
||||||
|
|
||||||
# ==================== Runtime 阶段 ====================
|
|
||||||
FROM git.xiaoxiajianji.com/xiaoxia/base/python:3.12-slim AS runtime
|
|
||||||
|
|
||||||
# 构建参数:版本号
|
|
||||||
ARG APP_VERSION=dev
|
ARG APP_VERSION=dev
|
||||||
|
|
||||||
# 使用阿里云镜像加速
|
# 使用阿里云镜像加速
|
||||||
RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \
|
RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \
|
||||||
sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list 2>/dev/null || true
|
sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list 2>/dev/null || true
|
||||||
|
|
||||||
# 安装最小运行时依赖(opencv-python-headless 需要 libglib2.0-0)
|
# 安装系统依赖
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
libglib2.0-0 \
|
ffmpeg \
|
||||||
|
libsm6 \
|
||||||
|
libxext6 \
|
||||||
|
libgl1 \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
# 从 builder 复制 ffmpeg 静态二进制
|
|
||||||
COPY --from=builder /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg
|
|
||||||
COPY --from=builder /usr/local/bin/ffprobe /usr/local/bin/ffprobe
|
|
||||||
|
|
||||||
# 从 builder 复制 Python 虚拟环境
|
|
||||||
COPY --from=builder /opt/venv /opt/venv
|
|
||||||
|
|
||||||
# 设置工作目录
|
# 设置工作目录
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
|
# ---- 依赖分层:基础依赖(变化少,缓存命中率高)----
|
||||||
|
COPY requirements-base.txt /tmp/requirements-base.txt
|
||||||
|
|
||||||
|
RUN python -m venv /opt/venv \
|
||||||
|
&& /opt/venv/bin/pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com -r /tmp/requirements-base.txt \
|
||||||
|
&& rm /tmp/requirements-base.txt
|
||||||
|
|
||||||
|
# ---- 依赖分层:Worker 专属大包(视频处理,变化极少)----
|
||||||
|
COPY requirements-worker.txt /tmp/requirements-worker.txt
|
||||||
|
|
||||||
|
RUN /opt/venv/bin/pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com -r /tmp/requirements-worker.txt \
|
||||||
|
&& rm /tmp/requirements-worker.txt
|
||||||
|
|
||||||
|
# ---- 依赖分层:业务依赖(变化频繁)----
|
||||||
|
COPY requirements.txt /tmp/requirements.txt
|
||||||
|
|
||||||
|
RUN /opt/venv/bin/pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com -r /tmp/requirements.txt \
|
||||||
|
&& rm /tmp/requirements.txt
|
||||||
|
|
||||||
# 复制应用代码
|
# 复制应用代码
|
||||||
COPY apps/worker/ /app/apps/worker/
|
COPY apps/worker/ /app/apps/worker/
|
||||||
COPY apps/api/app/config.py /app/apps/api/app/config.py
|
COPY apps/api/app/config.py /app/apps/api/app/config.py
|
||||||
@@ -107,13 +51,13 @@ COPY packages/ /app/packages/
|
|||||||
COPY alembic.ini /app/alembic.ini
|
COPY alembic.ini /app/alembic.ini
|
||||||
COPY migrations/ /app/migrations/
|
COPY migrations/ /app/migrations/
|
||||||
|
|
||||||
# 复制 Worker 启动脚本
|
# 复制 Worker 启动脚本(支持 WORKER_CONCURRENCY 环境变量)
|
||||||
COPY infra/docker/entrypoint-worker.sh /usr/local/bin/entrypoint-worker.sh
|
COPY infra/docker/entrypoint-worker.sh /usr/local/bin/entrypoint-worker.sh
|
||||||
RUN chmod +x /usr/local/bin/entrypoint-worker.sh
|
RUN chmod +x /usr/local/bin/entrypoint-worker.sh
|
||||||
|
|
||||||
# 设置 Python 路径
|
# 设置 Python 路径
|
||||||
ENV PATH="/opt/venv/bin:$PATH"
|
ENV PATH="/opt/venv/bin:$PATH"
|
||||||
ENV PYTHONPATH=/app:/app/packages
|
ENV PYTHONPATH=/app
|
||||||
ENV PYTHONUNBUFFERED=1
|
ENV PYTHONUNBUFFERED=1
|
||||||
ENV APP_VERSION=$APP_VERSION
|
ENV APP_VERSION=$APP_VERSION
|
||||||
|
|
||||||
|
|||||||
Executable → Regular
+13
-10
@@ -8,10 +8,8 @@ from __future__ import annotations
|
|||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
|
import subprocess
|
||||||
import tempfile
|
import tempfile
|
||||||
from subprocess import CalledProcessError, TimeoutExpired
|
|
||||||
|
|
||||||
from packages.shared.ffmpeg_utils import FFMPEG_BIN, run_ffmpeg
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -61,7 +59,7 @@ class AudioMerger:
|
|||||||
output_path = os.path.join(temp_dir, f"merged.{output_format}")
|
output_path = os.path.join(temp_dir, f"merged.{output_format}")
|
||||||
|
|
||||||
cmd = [
|
cmd = [
|
||||||
FFMPEG_BIN,
|
"ffmpeg",
|
||||||
"-y",
|
"-y",
|
||||||
"-f",
|
"-f",
|
||||||
"concat",
|
"concat",
|
||||||
@@ -74,16 +72,21 @@ class AudioMerger:
|
|||||||
output_path,
|
output_path,
|
||||||
]
|
]
|
||||||
|
|
||||||
try:
|
result = subprocess.run(
|
||||||
run_ffmpeg(cmd, timeout=120)
|
cmd,
|
||||||
except CalledProcessError as e:
|
capture_output=True,
|
||||||
logger.error(f"FFmpeg 合并失败: stderr={e.stderr}")
|
text=True,
|
||||||
raise AudioMergeError(f"FFmpeg 合并失败: {str(e)[:500]}")
|
timeout=120,
|
||||||
|
)
|
||||||
|
|
||||||
|
if result.returncode != 0:
|
||||||
|
logger.error(f"FFmpeg 合并失败: stderr={result.stderr}")
|
||||||
|
raise AudioMergeError(f"FFmpeg 合并失败: {result.stderr[:500]}")
|
||||||
|
|
||||||
with open(output_path, "rb") as f:
|
with open(output_path, "rb") as f:
|
||||||
return f.read()
|
return f.read()
|
||||||
|
|
||||||
except TimeoutExpired:
|
except subprocess.TimeoutExpired:
|
||||||
raise AudioMergeError("FFmpeg 合并超时(120 秒)")
|
raise AudioMergeError("FFmpeg 合并超时(120 秒)")
|
||||||
except AudioMergeError:
|
except AudioMergeError:
|
||||||
raise
|
raise
|
||||||
|
|||||||
@@ -1,79 +0,0 @@
|
|||||||
"""FFmpeg 共享工具 — packages/shared 层.
|
|
||||||
|
|
||||||
仅包含与业务无关的底层原语:FFmpeg/FFprobe 二进制路径、run_ffmpeg 执行器。
|
|
||||||
业务相关的滤镜构建、视频探测等留在 apps/worker/video_processing/ffmpeg_utils.py。
|
|
||||||
|
|
||||||
application 层和 worker 层都可以引用本模块,避免跨层依赖。
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import shutil
|
|
||||||
import subprocess # nosec B404
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
# ── 常量 ──────────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
FFMPEG_BIN: str = shutil.which("ffmpeg") or "ffmpeg"
|
|
||||||
FFPROBE_BIN: str = shutil.which("ffprobe") or "ffprobe"
|
|
||||||
|
|
||||||
# FFmpeg 执行默认超时(秒),防止 FFmpeg hang 住导致进程永久阻塞
|
|
||||||
# 默认 30 分钟,足够处理大部分短视频渲染;超长视频可单独传参覆盖
|
|
||||||
DEFAULT_FFMPEG_TIMEOUT = 1800
|
|
||||||
|
|
||||||
|
|
||||||
# ── FFmpeg 执行 ───────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
|
|
||||||
def run_ffmpeg(
|
|
||||||
command: list[str],
|
|
||||||
*,
|
|
||||||
capture_output: bool = True,
|
|
||||||
timeout: int | None = DEFAULT_FFMPEG_TIMEOUT,
|
|
||||||
) -> tuple[str, str]:
|
|
||||||
"""执行 FFmpeg 命令(统一入口)。
|
|
||||||
|
|
||||||
Args:
|
|
||||||
command: 完整的 ffmpeg 命令列表(含 "ffmpeg" 本身)
|
|
||||||
capture_output: 是否捕获 stdout/stderr
|
|
||||||
timeout: 超时时间(秒),默认 1800s(30分钟);None 表示不设超时(不推荐)
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
(stdout, stderr) 元组
|
|
||||||
|
|
||||||
Raises:
|
|
||||||
subprocess.CalledProcessError: 命令执行失败时抛出,
|
|
||||||
异常信息包含完整 stderr 以便排查。
|
|
||||||
subprocess.TimeoutExpired: 超时未完成时抛出,FFmpeg 进程会被 kill。
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
result = subprocess.run( # nosec B603
|
|
||||||
command,
|
|
||||||
check=True,
|
|
||||||
stdout=subprocess.PIPE if capture_output else None,
|
|
||||||
stderr=subprocess.PIPE if capture_output else None,
|
|
||||||
text=True,
|
|
||||||
timeout=timeout,
|
|
||||||
)
|
|
||||||
return (result.stdout or "", result.stderr or "")
|
|
||||||
except subprocess.TimeoutExpired:
|
|
||||||
logger.error(
|
|
||||||
"FFmpeg 命令超时 (%ds): command=%s",
|
|
||||||
timeout or -1,
|
|
||||||
" ".join(str(c) for c in command[:20]),
|
|
||||||
)
|
|
||||||
raise
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
# 把完整 stderr 打到日志,方便排查 exit code 183 等问题
|
|
||||||
stderr_text = (e.stderr or "").strip()
|
|
||||||
logger.error(
|
|
||||||
"FFmpeg 命令失败: exit_code=%d command=%s\nstderr:\n%s",
|
|
||||||
e.returncode,
|
|
||||||
" ".join(str(c) for c in command[:20]), # 截断过长的命令
|
|
||||||
stderr_text[:5000], # 截断过长的 stderr
|
|
||||||
)
|
|
||||||
raise
|
|
||||||
Executable → Regular
-140
@@ -92,141 +92,6 @@ _DOWNLOAD_CHUNK_SIZE = 8192
|
|||||||
# 最大重定向次数
|
# 最大重定向次数
|
||||||
_MAX_REDIRECTS = 5
|
_MAX_REDIRECTS = 5
|
||||||
|
|
||||||
# 文件魔数(文件头签名)表 — 用于 MIME 白名单校验后的二次真实性校验
|
|
||||||
# key: MIME 类型,value: 签名列表,任一签名匹配即通过
|
|
||||||
# 每条签名: list of (offset, bytes),所有条目都匹配才算该签名命中(支持多处联合匹配如 RIFF+WAVE)
|
|
||||||
_MAGIC_NUMBERS: dict[str, list[list[tuple[int, bytes]]]] = {
|
|
||||||
# ── 音频 ──
|
|
||||||
"audio/mpeg": [
|
|
||||||
[(0, b"ID3")], # ID3v2 标签
|
|
||||||
[(0, b"\xff\xfb")], # MPEG1 Layer3
|
|
||||||
[(0, b"\xff\xf3")], # MPEG2 Layer3
|
|
||||||
[(0, b"\xff\xf2")], # MPEG2.5 Layer3
|
|
||||||
[(0, b"\xff\xfa")], # MPEG1 Layer2
|
|
||||||
[(0, b"\xff\xf9")], # 其他 MPEG ADTS
|
|
||||||
],
|
|
||||||
"audio/wav": [
|
|
||||||
[(0, b"RIFF"), (8, b"WAVE")], # RIFF + WAVE
|
|
||||||
],
|
|
||||||
"audio/x-wav": [
|
|
||||||
[(0, b"RIFF"), (8, b"WAVE")],
|
|
||||||
],
|
|
||||||
"audio/ogg": [
|
|
||||||
[(0, b"OggS")],
|
|
||||||
],
|
|
||||||
"application/ogg": [
|
|
||||||
[(0, b"OggS")],
|
|
||||||
],
|
|
||||||
"audio/flac": [
|
|
||||||
[(0, b"fLaC")],
|
|
||||||
],
|
|
||||||
"audio/aac": [
|
|
||||||
[(0, b"\xff\xf1")], # ADTS MPEG-4
|
|
||||||
[(0, b"\xff\xf9")], # ADTS MPEG-2
|
|
||||||
],
|
|
||||||
"audio/aacp": [
|
|
||||||
[(0, b"\xff\xf1")],
|
|
||||||
[(0, b"\xff\xf9")],
|
|
||||||
],
|
|
||||||
"audio/mp4": [
|
|
||||||
[(4, b"ftyp")], # ISO Base Media (M4A)
|
|
||||||
],
|
|
||||||
"audio/x-m4a": [
|
|
||||||
[(4, b"ftyp")],
|
|
||||||
],
|
|
||||||
# ── 视频 ──
|
|
||||||
"video/mp4": [
|
|
||||||
[(4, b"ftyp")], # ISO Base Media (MP4)
|
|
||||||
],
|
|
||||||
"video/quicktime": [
|
|
||||||
[(4, b"ftyp")],
|
|
||||||
],
|
|
||||||
"video/x-matroska": [
|
|
||||||
[(0, b"\x1a\x45\xdf\xa3")], # EBML header
|
|
||||||
],
|
|
||||||
"video/webm": [
|
|
||||||
[(0, b"\x1a\x45\xdf\xa3")],
|
|
||||||
],
|
|
||||||
"video/x-msvideo": [
|
|
||||||
[(0, b"RIFF"), (8, b"AVI ")],
|
|
||||||
],
|
|
||||||
# ── 图片 ──
|
|
||||||
"image/jpeg": [
|
|
||||||
[(0, b"\xff\xd8\xff")],
|
|
||||||
],
|
|
||||||
"image/png": [
|
|
||||||
[(0, b"\x89PNG\r\n\x1a\n")],
|
|
||||||
],
|
|
||||||
"image/gif": [
|
|
||||||
[(0, b"GIF87a")],
|
|
||||||
[(0, b"GIF89a")],
|
|
||||||
],
|
|
||||||
"image/webp": [
|
|
||||||
[(0, b"RIFF"), (8, b"WEBP")],
|
|
||||||
],
|
|
||||||
"image/bmp": [
|
|
||||||
[(0, b"BM")],
|
|
||||||
],
|
|
||||||
}
|
|
||||||
|
|
||||||
# 魔数校验最大读取字节数(文件头)
|
|
||||||
_MAGIC_CHECK_READ_SIZE = 256
|
|
||||||
|
|
||||||
|
|
||||||
def _validate_magic_number(file_path: str, allowed_mime_types: set[str]) -> None:
|
|
||||||
"""校验文件头魔数是否与允许的 MIME 类型匹配.
|
|
||||||
|
|
||||||
读取文件前 256 字节,与 allowed_mime_types 对应格式的魔数逐一比对,
|
|
||||||
任一类型匹配即通过;全部不匹配则抛出 UrlSecurityError。
|
|
||||||
|
|
||||||
仅当 allowed_mime_types 非空时执行;空文件视为不匹配。
|
|
||||||
|
|
||||||
Args:
|
|
||||||
file_path: 本地文件路径
|
|
||||||
allowed_mime_types: 允许的 MIME 类型集合
|
|
||||||
|
|
||||||
Raises:
|
|
||||||
UrlSecurityError: 文件魔数与所有允许类型均不匹配
|
|
||||||
"""
|
|
||||||
# 收集所有允许类型对应的魔数签名
|
|
||||||
signatures: list[list[tuple[int, bytes]]] = []
|
|
||||||
for mime in allowed_mime_types:
|
|
||||||
sigs = _MAGIC_NUMBERS.get(mime)
|
|
||||||
if sigs:
|
|
||||||
signatures.extend(sigs)
|
|
||||||
|
|
||||||
# 如果没有已知魔数(比如自定义 MIME),跳过校验不阻断
|
|
||||||
if not signatures:
|
|
||||||
return
|
|
||||||
|
|
||||||
try:
|
|
||||||
with open(file_path, "rb") as f:
|
|
||||||
header = f.read(_MAGIC_CHECK_READ_SIZE)
|
|
||||||
except OSError as e:
|
|
||||||
raise UrlSecurityError(f"读取文件头失败: {e}") from e
|
|
||||||
|
|
||||||
if not header:
|
|
||||||
raise UrlSecurityError("文件为空,无法校验格式")
|
|
||||||
|
|
||||||
# 任一签名匹配即通过
|
|
||||||
for sig in signatures:
|
|
||||||
match = True
|
|
||||||
for offset, expected in sig:
|
|
||||||
if offset + len(expected) > len(header):
|
|
||||||
match = False
|
|
||||||
break
|
|
||||||
if header[offset : offset + len(expected)] != expected:
|
|
||||||
match = False
|
|
||||||
break
|
|
||||||
if match:
|
|
||||||
return
|
|
||||||
|
|
||||||
raise UrlSecurityError(
|
|
||||||
f"文件魔数与允许的 MIME 类型不匹配,"
|
|
||||||
f"允许类型: {sorted(allowed_mime_types)},"
|
|
||||||
f"文件头前16字节: {header[:16].hex()}"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class UrlSecurityError(ValueError):
|
class UrlSecurityError(ValueError):
|
||||||
"""URL 安全校验失败."""
|
"""URL 安全校验失败."""
|
||||||
@@ -429,7 +294,6 @@ def safe_download_file(
|
|||||||
- 重定向次数限制 + 手动跟随(避免重定向绕过 SSRF)
|
- 重定向次数限制 + 手动跟随(避免重定向绕过 SSRF)
|
||||||
- 文件大小限制(流式读取,超过立即中断)
|
- 文件大小限制(流式读取,超过立即中断)
|
||||||
- MIME 类型白名单(可选)
|
- MIME 类型白名单(可选)
|
||||||
- 文件头魔数校验(配合 MIME 白名单做二次真实性校验)
|
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
url: 下载 URL
|
url: 下载 URL
|
||||||
@@ -498,10 +362,6 @@ def safe_download_file(
|
|||||||
raise UrlSecurityError(f"下载超过大小限制: {total_bytes} bytes > {max_size} bytes")
|
raise UrlSecurityError(f"下载超过大小限制: {total_bytes} bytes > {max_size} bytes")
|
||||||
f.write(chunk)
|
f.write(chunk)
|
||||||
|
|
||||||
# 文件头魔数校验(MIME 白名单基础上的二次真实性校验)
|
|
||||||
if allowed_mime_types is not None:
|
|
||||||
_validate_magic_number(dest_path, allowed_mime_types)
|
|
||||||
|
|
||||||
return total_bytes
|
return total_bytes
|
||||||
finally:
|
finally:
|
||||||
resp.close()
|
resp.close()
|
||||||
|
|||||||
+2
-1
@@ -80,6 +80,7 @@ select = [
|
|||||||
# 与原 setup.cfg flake8 配置对齐,确保不新增阻断
|
# 与原 setup.cfg flake8 配置对齐,确保不新增阻断
|
||||||
ignore = [
|
ignore = [
|
||||||
"E203",
|
"E203",
|
||||||
|
"W503",
|
||||||
"E501", # line-too-long(black管)
|
"E501", # line-too-long(black管)
|
||||||
"E302",
|
"E302",
|
||||||
"E402", # module-import-not-at-top(循环导入多)
|
"E402", # module-import-not-at-top(循环导入多)
|
||||||
@@ -96,6 +97,6 @@ ignore = [
|
|||||||
[tool.ruff.lint.per-file-ignores]
|
[tool.ruff.lint.per-file-ignores]
|
||||||
"__init__.py" = ["F401", "F403", "F405"]
|
"__init__.py" = ["F401", "F403", "F405"]
|
||||||
"tests/*" = ["E402", "F401", "F841"]
|
"tests/*" = ["E402", "F401", "F841"]
|
||||||
"packages/ports/*" = ["E301"]
|
"packages/ports/*" = ["E301", "E704"]
|
||||||
"apps/*/migrations/*" = ["ALL"]
|
"apps/*/migrations/*" = ["ALL"]
|
||||||
"alembic/*" = ["ALL"]
|
"alembic/*" = ["ALL"]
|
||||||
|
|||||||
Executable → Regular
+1
-1
@@ -1,5 +1,5 @@
|
|||||||
[pytest]
|
[pytest]
|
||||||
pythonpath = . apps/api apps/worker packages
|
pythonpath = . apps/api apps/worker
|
||||||
testpaths = tests
|
testpaths = tests
|
||||||
|
|
||||||
# ===== 覆盖率配置 =====
|
# ===== 覆盖率配置 =====
|
||||||
|
|||||||
@@ -123,7 +123,7 @@ def get_new_migrations_via_diff(diff_target: str) -> List[Path]:
|
|||||||
return [REPO_ROOT / f for f in files]
|
return [REPO_ROOT / f for f in files]
|
||||||
except subprocess.CalledProcessError as e:
|
except subprocess.CalledProcessError as e:
|
||||||
print(f"⚠️ git diff 失败({diff_target}):{e.stderr.strip()}")
|
print(f"⚠️ git diff 失败({diff_target}):{e.stderr.strip()}")
|
||||||
print(" 降级为检查所有迁移文件")
|
print(f" 降级为检查所有迁移文件")
|
||||||
return sorted(ALEMBIC_VERSIONS_DIR.glob("*.py"))
|
return sorted(ALEMBIC_VERSIONS_DIR.glob("*.py"))
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+26
-56
@@ -2,7 +2,7 @@
|
|||||||
"""
|
"""
|
||||||
统一CI通知脚本 - 发送飞书卡片通知
|
统一CI通知脚本 - 发送飞书卡片通知
|
||||||
支持三种模式: start / success / failure
|
支持三种模式: start / success / failure
|
||||||
包含: PR链接、耗时、失败阶段、分支、提交者、Run链接、Runner信息
|
包含: PR链接、耗时、失败阶段、分支、提交者、Run链接
|
||||||
|
|
||||||
用法:
|
用法:
|
||||||
NOTIFY_MODE=start JOB_NAME="xxx" python3 scripts/ci_notify.py
|
NOTIFY_MODE=start JOB_NAME="xxx" python3 scripts/ci_notify.py
|
||||||
@@ -23,12 +23,6 @@
|
|||||||
GITHUB_EVENT_NAME - 事件类型 (pull_request / push / ...)
|
GITHUB_EVENT_NAME - 事件类型 (pull_request / push / ...)
|
||||||
GITHUB_PR_NUMBER - PR编号 (PR事件时)
|
GITHUB_PR_NUMBER - PR编号 (PR事件时)
|
||||||
GITHUB_PR_TITLE - PR标题 (PR事件时)
|
GITHUB_PR_TITLE - PR标题 (PR事件时)
|
||||||
RUNNER_NAME - Runner名称 (可选,自动获取)
|
|
||||||
|
|
||||||
设计原则:
|
|
||||||
1. 通知失败永远不阻断CI主流程(返回exit code 0)
|
|
||||||
2. 标题包含"CI通知"/"CI告警"关键词,适配飞书webhook关键词校验
|
|
||||||
3. 卡片信息尽量丰富,方便快速定位问题
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
@@ -38,7 +32,6 @@ import urllib.request
|
|||||||
|
|
||||||
|
|
||||||
def get_env(name, default=""):
|
def get_env(name, default=""):
|
||||||
"""读取环境变量"""
|
|
||||||
return os.environ.get(name, default)
|
return os.environ.get(name, default)
|
||||||
|
|
||||||
|
|
||||||
@@ -55,20 +48,6 @@ def format_duration(seconds_str):
|
|||||||
return seconds_str or "未知"
|
return seconds_str or "未知"
|
||||||
|
|
||||||
|
|
||||||
def classify_job(job_name):
|
|
||||||
"""根据Job名称判断所属阶段"""
|
|
||||||
name = job_name.lower()
|
|
||||||
if any(k in name for k in ["validate", "lint", "unit test", "integration test"]):
|
|
||||||
return "门禁检查"
|
|
||||||
if any(k in name for k in ["build", "image"]):
|
|
||||||
return "镜像构建"
|
|
||||||
if any(k in name for k in ["deploy", "staging", "production"]):
|
|
||||||
return "部署发布"
|
|
||||||
if any(k in name for k in ["e2e", "test", "smoke"]):
|
|
||||||
return "测试验证"
|
|
||||||
return "其他"
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
webhook = get_env("CI_NOTIFY_WEBHOOK")
|
webhook = get_env("CI_NOTIFY_WEBHOOK")
|
||||||
if not webhook:
|
if not webhook:
|
||||||
@@ -92,60 +71,49 @@ def main() -> int:
|
|||||||
event_name = get_env("GITHUB_EVENT_NAME", "")
|
event_name = get_env("GITHUB_EVENT_NAME", "")
|
||||||
pr_number = get_env("GITHUB_PR_NUMBER", "")
|
pr_number = get_env("GITHUB_PR_NUMBER", "")
|
||||||
pr_title = get_env("GITHUB_PR_TITLE", "")
|
pr_title = get_env("GITHUB_PR_TITLE", "")
|
||||||
runner_name = get_env("RUNNER_NAME", "")
|
|
||||||
|
|
||||||
run_url = f"https://git.xiaoxiajianji.com/{repo}/actions/runs/{run_id}"
|
run_url = f"https://git.xiaoxiajianji.com/{repo}/actions/runs/{run_id}"
|
||||||
job_stage = classify_job(job_name)
|
|
||||||
|
|
||||||
# 根据模式设置标题、状态、颜色
|
# 根据模式设置标题、状态、颜色
|
||||||
# 注意:标题中必须包含飞书webhook配置的关键词,否则会报"Key Words Not Found"
|
|
||||||
# 这里加入"CI通知"/"CI告警"关键词提高命中率
|
|
||||||
if mode == "start":
|
if mode == "start":
|
||||||
title = f"🔄 CI通知:{job_name} 开始构建"
|
title = "🔄 CI构建任务开始"
|
||||||
status = "blue"
|
status = "blue"
|
||||||
button_text = "查看进度"
|
button_text = "查看进度"
|
||||||
button_type = "primary"
|
button_type = "primary"
|
||||||
elif mode == "success":
|
elif mode == "success":
|
||||||
title = f"✅ CI通知:{job_name} 构建成功"
|
title = f"✅ CI构建成功 - {job_name}"
|
||||||
status = "green"
|
status = "green"
|
||||||
button_text = "查看详情"
|
button_text = "查看详情"
|
||||||
button_type = "primary"
|
button_type = "primary"
|
||||||
else: # failure
|
else: # failure
|
||||||
title = f"❌ CI告警:{job_name} 构建失败"
|
title = f"❌ CI构建失败 - {job_name}"
|
||||||
status = "red"
|
status = "red"
|
||||||
button_text = "查看失败日志"
|
button_text = "查看失败日志"
|
||||||
button_type = "danger"
|
button_type = "danger"
|
||||||
|
|
||||||
# 构建卡片内容 - 左侧标签+右侧值的结构化展示
|
# 构建卡片内容(含关键词"CI构建"以通过飞书机器人关键词校验)
|
||||||
fields = []
|
content_lines = []
|
||||||
|
content_lines.append(f"**CI构建任务**: {job_name}")
|
||||||
fields.append({"is_short": True, "text": {"tag": "lark_md", "content": f"**阶段**\n{job_stage}"}})
|
|
||||||
fields.append({"is_short": True, "text": {"tag": "lark_md", "content": f"**任务**\n{job_name}"}})
|
|
||||||
|
|
||||||
if mode != "start":
|
if mode != "start":
|
||||||
fields.append({"is_short": True, "text": {"tag": "lark_md", "content": f"**耗时**\n{duration}"}})
|
content_lines.append(f"**耗时**: {duration}")
|
||||||
else:
|
|
||||||
fields.append({"is_short": True, "text": {"tag": "lark_md", "content": "**状态**\n进行中"}})
|
|
||||||
|
|
||||||
if runner_name:
|
|
||||||
fields.append({"is_short": True, "text": {"tag": "lark_md", "content": f"**Runner**\n{runner_name}"}})
|
|
||||||
|
|
||||||
if mode == "failure" and failed_step:
|
if mode == "failure" and failed_step:
|
||||||
fields.append({"is_short": False, "text": {"tag": "lark_md", "content": f"**失败步骤**\n{failed_step}"}})
|
content_lines.append(f"**失败阶段**: {failed_step}")
|
||||||
|
|
||||||
# PR/分支信息
|
# PR信息
|
||||||
if event_name == "pull_request" and pr_number:
|
if event_name == "pull_request" and pr_number:
|
||||||
pr_url = f"https://git.xiaoxiajianji.com/{repo}/pulls/{pr_number}"
|
pr_url = f"https://git.xiaoxiajianji.com/{repo}/pulls/{pr_number}"
|
||||||
pr_display = f"#{pr_number}"
|
pr_display = f"#{pr_number}"
|
||||||
if pr_title:
|
if pr_title:
|
||||||
pr_display += f" {pr_title[:30]}"
|
pr_display += f" {pr_title}"
|
||||||
fields.append({"is_short": False, "text": {"tag": "lark_md", "content": f"**PR**\n[{pr_display}]({pr_url})"}})
|
content_lines.append(f"**PR**: [{pr_display}]({pr_url})")
|
||||||
elif event_name == "push":
|
elif event_name == "push":
|
||||||
fields.append({"is_short": True, "text": {"tag": "lark_md", "content": f"**分支**\n{branch}"}})
|
content_lines.append(f"**分支**: {branch}")
|
||||||
|
|
||||||
fields.append({"is_short": True, "text": {"tag": "lark_md", "content": f"**提交**\n`{commit}`"}})
|
content_lines.append(f"**提交**: `{commit}`")
|
||||||
fields.append({"is_short": True, "text": {"tag": "lark_md", "content": f"**提交者**\n{actor}"}})
|
content_lines.append(f"**提交者**: {actor}")
|
||||||
fields.append({"is_short": True, "text": {"tag": "lark_md", "content": f"**Run ID**\n{run_id}"}})
|
content_lines.append(f"**Run ID**: {run_id}")
|
||||||
|
|
||||||
payload = {
|
payload = {
|
||||||
"msg_type": "interactive",
|
"msg_type": "interactive",
|
||||||
@@ -160,7 +128,10 @@ def main() -> int:
|
|||||||
"elements": [
|
"elements": [
|
||||||
{
|
{
|
||||||
"tag": "div",
|
"tag": "div",
|
||||||
"fields": fields,
|
"text": {
|
||||||
|
"tag": "lark_md",
|
||||||
|
"content": "\n".join(content_lines),
|
||||||
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"tag": "action",
|
"tag": "action",
|
||||||
@@ -187,20 +158,19 @@ def main() -> int:
|
|||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||||
resp_body = resp.read().decode("utf-8")
|
resp_body = resp.read().decode("utf-8")
|
||||||
|
print(f"通知已发送 ({mode})")
|
||||||
# 飞书返回code=0表示成功
|
# 飞书返回code=0表示成功
|
||||||
try:
|
try:
|
||||||
result = json.loads(resp_body)
|
result = json.loads(resp_body)
|
||||||
if result.get("code", 0) != 0:
|
if result.get("code", 0) != 0:
|
||||||
print(f"通知发送告警: 飞书返回错误 - {result.get('msg', resp_body)}", file=sys.stderr)
|
print(f"飞书返回错误: {result.get('msg', resp_body)}", file=sys.stderr)
|
||||||
print(f"通知已发送 ({mode}) - 飞书返回非0,但不阻断CI流程")
|
return 1
|
||||||
else:
|
|
||||||
print(f"通知已发送 ({mode})")
|
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
print(f"通知已发送 ({mode})")
|
pass
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"通知发送告警: {e}", file=sys.stderr)
|
print(f"通知发送失败: {e}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
# 通知无论成功失败都不阻断CI主流程,统一返回0
|
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,483 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# ===========================================
|
|
||||||
# CI Production 健康检查 + 自动回滚脚本(SSH 部署模式)
|
|
||||||
# ===========================================
|
|
||||||
#
|
|
||||||
# 在 CI Runner 上执行,通过公网 URL 检查 Production 部署健康状态。
|
|
||||||
# 不健康则通过 SSH 自动回滚到上一个版本的镜像。
|
|
||||||
#
|
|
||||||
# 用法:
|
|
||||||
# ./ci_production_healthcheck.sh
|
|
||||||
#
|
|
||||||
# 环境变量:
|
|
||||||
# PROD_API_URL - Production API 公网地址 (默认 https://api.xiaoxiajianji.com)
|
|
||||||
# PROD_WEB_URL - Production Web 公网地址 (默认 https://saas.xiaoxiajianji.com)
|
|
||||||
# HEALTH_CHECK_TIMEOUT - 健康检查总超时秒数 (默认 180)
|
|
||||||
# SKIP_ROLLBACK - 失败时不自动回滚 (true/false, 默认 false)
|
|
||||||
# SKIP_NOTIFY - 跳过通知 (true/false, 默认 false)
|
|
||||||
# CI_NOTIFY_WEBHOOK - 通知 Webhook URL
|
|
||||||
#
|
|
||||||
# PRODUCTION_SSH_HOST - 生产服务器 SSH 地址
|
|
||||||
# PRODUCTION_SSH_USER - SSH 用户名 (默认 root)
|
|
||||||
# PRODUCTION_SSH_PORT - SSH 端口 (默认 22222)
|
|
||||||
# PRODUCTION_SSH_KEY - SSH 私钥内容
|
|
||||||
# REGISTRY_TOKEN - Registry Token(回滚时拉取旧镜像需要)
|
|
||||||
#
|
|
||||||
# GITHUB_SHA - 当前 commit SHA
|
|
||||||
# GITHUB_REF_NAME - tag 名 (如 v0.1.100)
|
|
||||||
# GITHUB_RUN_ID - CI Run ID
|
|
||||||
# GITHUB_REPOSITORY - 仓库名
|
|
||||||
# GITHUB_ACTOR - 提交者
|
|
||||||
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
|
|
||||||
|
|
||||||
# 配置
|
|
||||||
PROD_API_URL="${PROD_API_URL:-https://api.xiaoxiajianji.com}"
|
|
||||||
PROD_WEB_URL="${PROD_WEB_URL:-https://saas.xiaoxiajianji.com}"
|
|
||||||
HEALTH_CHECK_TIMEOUT="${HEALTH_CHECK_TIMEOUT:-180}"
|
|
||||||
SKIP_ROLLBACK="${SKIP_ROLLBACK:-false}"
|
|
||||||
SKIP_NOTIFY="${SKIP_NOTIFY:-false}"
|
|
||||||
|
|
||||||
PRODUCTION_SSH_HOST="${PRODUCTION_SSH_HOST:-47.98.113.167}"
|
|
||||||
PRODUCTION_SSH_USER="${PRODUCTION_SSH_USER:-root}"
|
|
||||||
PRODUCTION_SSH_PORT="${PRODUCTION_SSH_PORT:-22222}"
|
|
||||||
|
|
||||||
REGISTRY="${REGISTRY:-git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas}"
|
|
||||||
REGISTRY_USER="${REGISTRY_USER:-xiaoxia}"
|
|
||||||
|
|
||||||
# 颜色
|
|
||||||
RED='\033[0;31m'
|
|
||||||
GREEN='\033[0;32m'
|
|
||||||
YELLOW='\033[1;33m'
|
|
||||||
BLUE='\033[0;34m'
|
|
||||||
NC='\033[0m'
|
|
||||||
|
|
||||||
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
|
|
||||||
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
|
||||||
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
|
|
||||||
log_step() { echo -e "${BLUE}[STEP]${NC} $1"; }
|
|
||||||
|
|
||||||
# ===========================================
|
|
||||||
# SSH 工具函数
|
|
||||||
# ===========================================
|
|
||||||
SSH_KEY_PATH=""
|
|
||||||
|
|
||||||
setup_ssh() {
|
|
||||||
# 查找或创建 SSH 密钥
|
|
||||||
if [ -f /root/.ssh/xiaoxia_runtime_builder ]; then
|
|
||||||
SSH_KEY_PATH="/root/.ssh/xiaoxia_runtime_builder"
|
|
||||||
elif [ -f "$HOME/.ssh/xiaoxia_runtime_builder" ]; then
|
|
||||||
SSH_KEY_PATH="$HOME/.ssh/xiaoxia_runtime_builder"
|
|
||||||
elif [ -n "${PRODUCTION_SSH_KEY:-}" ]; then
|
|
||||||
SSH_KEY_PATH="$HOME/.ssh/prod_deploy_key"
|
|
||||||
mkdir -p "$HOME/.ssh"
|
|
||||||
printf '%s\n' "$PRODUCTION_SSH_KEY" > "$SSH_KEY_PATH"
|
|
||||||
chmod 600 "$SSH_KEY_PATH"
|
|
||||||
else
|
|
||||||
log_error "没有可用的 SSH 密钥"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
ssh-keyscan -p "$PRODUCTION_SSH_PORT" -H "$PRODUCTION_SSH_HOST" >> ~/.ssh/known_hosts 2>/dev/null || true
|
|
||||||
log_info "SSH 已配置: ${PRODUCTION_SSH_USER}@${PRODUCTION_SSH_HOST}:${PRODUCTION_SSH_PORT}"
|
|
||||||
}
|
|
||||||
|
|
||||||
run_ssh() {
|
|
||||||
local cmd="$1"
|
|
||||||
ssh -p "$PRODUCTION_SSH_PORT" -i "$SSH_KEY_PATH" -o StrictHostKeyChecking=no \
|
|
||||||
"${PRODUCTION_SSH_USER}@${PRODUCTION_SSH_HOST}" "$cmd"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ===========================================
|
|
||||||
# 1. 记录部署前各服务的镜像版本(用于回滚)
|
|
||||||
# ===========================================
|
|
||||||
ROLLBACK_API_TAG=""
|
|
||||||
ROLLBACK_WORKER_TAG=""
|
|
||||||
ROLLBACK_WEB_TAG=""
|
|
||||||
|
|
||||||
save_rollback_target() {
|
|
||||||
log_step "记录当前生产环境各服务镜像版本(回滚目标)..."
|
|
||||||
|
|
||||||
# 通过 SSH 获取当前运行的容器镜像
|
|
||||||
local api_image worker_image web_image
|
|
||||||
api_image=$(run_ssh "docker inspect --format '{{.Config.Image}}' xiaoxia-api-production 2>/dev/null || echo ''")
|
|
||||||
worker_image=$(run_ssh "docker inspect --format '{{.Config.Image}}' xiaoxia-worker-production 2>/dev/null || echo ''")
|
|
||||||
web_image=$(run_ssh "docker inspect --format '{{.Config.Image}}' xiaoxia-web-production 2>/dev/null || echo ''")
|
|
||||||
|
|
||||||
# 提取 tag(镜像名是 xiaoxia-saas-api:v0.1.100 格式)
|
|
||||||
ROLLBACK_API_TAG=$(echo "$api_image" | sed 's/.*://' || echo "")
|
|
||||||
ROLLBACK_WORKER_TAG=$(echo "$worker_image" | sed 's/.*://' || echo "")
|
|
||||||
ROLLBACK_WEB_TAG=$(echo "$web_image" | sed 's/.*://' || echo "")
|
|
||||||
|
|
||||||
log_info " API: ${ROLLBACK_API_TAG:-未知}"
|
|
||||||
log_info " Worker: ${ROLLBACK_WORKER_TAG:-未知}"
|
|
||||||
log_info " Web: ${ROLLBACK_WEB_TAG:-未知}"
|
|
||||||
|
|
||||||
# 验证三个服务版本是否一致
|
|
||||||
if [ -n "$ROLLBACK_API_TAG" ] && [ -n "$ROLLBACK_WORKER_TAG" ] && [ -n "$ROLLBACK_WEB_TAG" ]; then
|
|
||||||
if [ "$ROLLBACK_API_TAG" = "$ROLLBACK_WORKER_TAG" ] && [ "$ROLLBACK_API_TAG" = "$ROLLBACK_WEB_TAG" ]; then
|
|
||||||
log_info " ✅ 三个服务版本一致: $ROLLBACK_API_TAG"
|
|
||||||
export ROLLBACK_TAG="$ROLLBACK_API_TAG"
|
|
||||||
else
|
|
||||||
log_warn " ⚠️ 三个服务版本不一致,回滚时将分别使用各自版本"
|
|
||||||
export ROLLBACK_API_TAG ROLLBACK_WORKER_TAG ROLLBACK_WEB_TAG
|
|
||||||
export ROLLBACK_TAG_MIXED="true"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
log_warn " ⚠️ 未能获取全部服务版本,回滚功能可能受限"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# ===========================================
|
|
||||||
# 2. 健康检查(公网视角)
|
|
||||||
# ===========================================
|
|
||||||
health_check() {
|
|
||||||
local timeout="$HEALTH_CHECK_TIMEOUT"
|
|
||||||
local start_time
|
|
||||||
start_time=$(date +%s)
|
|
||||||
|
|
||||||
log_step "公网健康检查(超时 ${timeout}s)..."
|
|
||||||
log_info " API: ${PROD_API_URL}/health"
|
|
||||||
log_info " Web: ${PROD_WEB_URL}/"
|
|
||||||
|
|
||||||
local api_ok=false
|
|
||||||
local web_ok=false
|
|
||||||
local api_docs_ok=false
|
|
||||||
local login_api_ok=false
|
|
||||||
|
|
||||||
while [ $(( $(date +%s) - start_time )) -lt "$timeout" ]; do
|
|
||||||
# 检查 API health
|
|
||||||
if [ "$api_ok" = false ] && curl -sf --max-time 10 "${PROD_API_URL}/health" >/dev/null 2>&1; then
|
|
||||||
log_info "✅ API 健康检查通过"
|
|
||||||
api_ok=true
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 检查 Web 首页
|
|
||||||
if [ "$web_ok" = false ] && curl -sf --max-time 10 "$PROD_WEB_URL/" >/dev/null 2>&1; then
|
|
||||||
log_info "✅ Web 前端检查通过"
|
|
||||||
web_ok=true
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 检查 API docs
|
|
||||||
if [ "$api_docs_ok" = false ]; then
|
|
||||||
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "${PROD_API_URL}/docs" 2>/dev/null || echo "000")
|
|
||||||
if [ "$HTTP_CODE" = "200" ]; then
|
|
||||||
log_info "✅ API Docs 检查通过"
|
|
||||||
api_docs_ok=true
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 检查登录 API
|
|
||||||
if [ "$login_api_ok" = false ]; then
|
|
||||||
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 -X POST \
|
|
||||||
"${PROD_API_URL}/api/v1/auth/login" \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{"email":"smoke@test.com","password":"wrong"}' 2>/dev/null || echo "000")
|
|
||||||
if [ "$HTTP_CODE" = "401" ] || [ "$HTTP_CODE" = "422" ]; then
|
|
||||||
log_info "✅ 登录 API 检查通过(HTTP $HTTP_CODE,符合预期)"
|
|
||||||
login_api_ok=true
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 都通过了就退出
|
|
||||||
if [ "$api_ok" = true ] && [ "$web_ok" = true ] && [ "$api_docs_ok" = true ] && [ "$login_api_ok" = true ]; then
|
|
||||||
log_info "🎉 所有健康检查通过!"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
sleep 5
|
|
||||||
done
|
|
||||||
|
|
||||||
# 超时了
|
|
||||||
log_error "❌ 健康检查超时 (${timeout}s)"
|
|
||||||
[ "$api_ok" = false ] && log_error " - API health 未通过"
|
|
||||||
[ "$web_ok" = false ] && log_error " - Web 前端未通过"
|
|
||||||
[ "$api_docs_ok" = false ] && log_error " - API Docs 未通过"
|
|
||||||
[ "$login_api_ok" = false ] && log_error " - 登录 API 未通过"
|
|
||||||
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
# ===========================================
|
|
||||||
# 3. 执行回滚(SSH 重新部署旧版本)
|
|
||||||
# ===========================================
|
|
||||||
do_rollback() {
|
|
||||||
log_step "执行回滚:通过 SSH 重新部署旧版本镜像..."
|
|
||||||
|
|
||||||
local rollback_tag="${ROLLBACK_TAG:-}"
|
|
||||||
if [ -z "$rollback_tag" ] && [ "${ROLLBACK_TAG_MIXED:-}" != "true" ]; then
|
|
||||||
log_error "没有可回滚的版本记录,无法自动回滚"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 如果版本不一致,用 API 的版本作为回滚目标
|
|
||||||
if [ -z "$rollback_tag" ]; then
|
|
||||||
rollback_tag="$ROLLBACK_API_TAG"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -z "$rollback_tag" ]; then
|
|
||||||
log_error "无法确定回滚版本"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
log_info "回滚目标版本: $rollback_tag"
|
|
||||||
|
|
||||||
# 通过 SSH 在生产服务器上执行回滚部署
|
|
||||||
# 复用 Registry 方式部署脚本的逻辑,用旧版本 tag 重新部署
|
|
||||||
local rollback_script=$(cat << 'ROLLBACK_EOF'
|
|
||||||
#!/bin/sh
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
IMAGE_TAG="$1"
|
|
||||||
REGISTRY_TOKEN="$2"
|
|
||||||
REGISTRY="${REGISTRY:-git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas}"
|
|
||||||
REGISTRY_USER="${REGISTRY_USER:-xiaoxia}"
|
|
||||||
|
|
||||||
ENV_FILE="${ENV_FILE:-/var/lib/xiaoxia-saas-production/.env}"
|
|
||||||
GENERATED_DIR="${GENERATED_DIR:-/var/lib/xiaoxia-saas-production/generated}"
|
|
||||||
LEGACY_ASSETS_DIR="${LEGACY_ASSETS_DIR:-/var/lib/xiaoxia-saas-production/legacy-assets}"
|
|
||||||
|
|
||||||
echo "=== Rollback to $IMAGE_TAG ==="
|
|
||||||
|
|
||||||
# 登录 Registry
|
|
||||||
if [ -n "$REGISTRY_TOKEN" ]; then
|
|
||||||
REGISTRY_HOST=$(echo "$REGISTRY" | cut -d/ -f1)
|
|
||||||
printf %s "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" -u "$REGISTRY_USER" --password-stdin 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Pull 旧版本镜像
|
|
||||||
LOCAL_API="xiaoxia-saas-api:${IMAGE_TAG}"
|
|
||||||
LOCAL_WORKER="xiaoxia-saas-worker:${IMAGE_TAG}"
|
|
||||||
LOCAL_WEB="xiaoxia-saas-web:${IMAGE_TAG}"
|
|
||||||
|
|
||||||
docker pull "${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}"
|
|
||||||
docker pull "${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}"
|
|
||||||
docker pull "${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}"
|
|
||||||
|
|
||||||
docker tag "${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}" "$LOCAL_API"
|
|
||||||
docker tag "${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}" "$LOCAL_WORKER"
|
|
||||||
docker tag "${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}" "$LOCAL_WEB"
|
|
||||||
|
|
||||||
echo "Rollback images pulled."
|
|
||||||
|
|
||||||
# 停止当前容器
|
|
||||||
docker rm -f xiaoxia-api-production 2>/dev/null || true
|
|
||||||
docker rm -f xiaoxia-worker-production 2>/dev/null || true
|
|
||||||
docker rm -f xiaoxia-web-production 2>/dev/null || true
|
|
||||||
|
|
||||||
LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3"
|
|
||||||
|
|
||||||
# 启动 API(回滚不跑 migration,因为新版本可能加了字段,回滚后代码是旧的但数据还在)
|
|
||||||
echo "Starting API (rollback)..."
|
|
||||||
docker run -d \
|
|
||||||
--name xiaoxia-api-production \
|
|
||||||
--env-file "$ENV_FILE" \
|
|
||||||
--network xiaoxia-net-production \
|
|
||||||
-p 127.0.0.1:8001:8000 \
|
|
||||||
-e APP_ENV=production \
|
|
||||||
-e APP_VERSION="$IMAGE_TAG" \
|
|
||||||
-e GENERATED_FILES_DIR=/app/generated \
|
|
||||||
-e GENERATED_FILES_URL_PREFIX=/generated-files \
|
|
||||||
-e PUBLIC_API_BASE_URL=https://api.xiaoxiajianji.com \
|
|
||||||
-v "$GENERATED_DIR:/app/generated" \
|
|
||||||
--restart unless-stopped \
|
|
||||||
--cpus 2 \
|
|
||||||
--memory 2g \
|
|
||||||
--health-cmd "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/health', timeout=5)\"" \
|
|
||||||
--health-interval 30s \
|
|
||||||
--health-timeout 10s \
|
|
||||||
--health-retries 3 \
|
|
||||||
--health-start-period 40s \
|
|
||||||
$LOG_OPTS \
|
|
||||||
"$LOCAL_API"
|
|
||||||
|
|
||||||
# 启动 Worker
|
|
||||||
echo "Starting Worker (rollback)..."
|
|
||||||
docker run -d \
|
|
||||||
--name xiaoxia-worker-production \
|
|
||||||
--env-file "$ENV_FILE" \
|
|
||||||
--network xiaoxia-net-production \
|
|
||||||
-e APP_ENV=production \
|
|
||||||
-e APP_VERSION="$IMAGE_TAG" \
|
|
||||||
-e WORKER_CONCURRENCY=1 \
|
|
||||||
-e WORKER_MAX_TASKS_PER_CHILD=100 \
|
|
||||||
-e GENERATED_FILES_DIR=/app/generated \
|
|
||||||
-e GENERATED_FILES_URL_PREFIX=/generated-files \
|
|
||||||
-e PUBLIC_API_BASE_URL=https://api.xiaoxiajianji.com \
|
|
||||||
-v "$GENERATED_DIR:/app/generated" \
|
|
||||||
--restart unless-stopped \
|
|
||||||
--cpus 2 \
|
|
||||||
--memory 2g \
|
|
||||||
--health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \
|
|
||||||
--health-interval 30s \
|
|
||||||
--health-timeout 10s \
|
|
||||||
--health-retries 3 \
|
|
||||||
--health-start-period 30s \
|
|
||||||
$LOG_OPTS \
|
|
||||||
"$LOCAL_WORKER"
|
|
||||||
|
|
||||||
# 启动 Web
|
|
||||||
LEGACY_VOLUME=""
|
|
||||||
if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then
|
|
||||||
LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "Starting Web (rollback)..."
|
|
||||||
docker run -d \
|
|
||||||
--name xiaoxia-web-production \
|
|
||||||
--network xiaoxia-net-production \
|
|
||||||
-p 127.0.0.1:3002:80 \
|
|
||||||
--restart unless-stopped \
|
|
||||||
--cpus 0.5 \
|
|
||||||
--memory 512m \
|
|
||||||
$LEGACY_VOLUME \
|
|
||||||
--health-cmd "wget --spider -q http://127.0.0.1:80" \
|
|
||||||
--health-interval 30s \
|
|
||||||
--health-timeout 5s \
|
|
||||||
--health-retries 3 \
|
|
||||||
$LOG_OPTS \
|
|
||||||
"$LOCAL_WEB"
|
|
||||||
|
|
||||||
# 等待 API 健康
|
|
||||||
echo "Waiting for API (rollback)..."
|
|
||||||
i=0
|
|
||||||
while [ "$i" -lt 40 ]; do
|
|
||||||
if curl -sf --max-time 5 http://127.0.0.1:8001/health >/dev/null 2>&1; then
|
|
||||||
echo "API healthy (rollback)."
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
i=$((i + 1))
|
|
||||||
sleep 3
|
|
||||||
done
|
|
||||||
|
|
||||||
# 等待 Web 健康
|
|
||||||
echo "Waiting for Web (rollback)..."
|
|
||||||
i=0
|
|
||||||
while [ "$i" -lt 15 ]; do
|
|
||||||
if curl -sf --max-time 5 http://127.0.0.1:3002/ >/dev/null 2>&1; then
|
|
||||||
echo "Web healthy (rollback)."
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
i=$((i + 1))
|
|
||||||
sleep 2
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "=== Rollback complete: $IMAGE_TAG ==="
|
|
||||||
docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep production
|
|
||||||
ROLLBACK_EOF
|
|
||||||
)
|
|
||||||
|
|
||||||
# 将脚本 base64 编码后通过 SSH 执行
|
|
||||||
local script_b64
|
|
||||||
script_b64=$(echo "$rollback_script" | base64 -w 0)
|
|
||||||
|
|
||||||
log_info "在生产服务器上执行回滚脚本..."
|
|
||||||
if run_ssh "echo '$script_b64' | base64 -d | sh -s -- '$rollback_tag' '${REGISTRY_TOKEN:-}'" 2>&1; then
|
|
||||||
log_info "✅ 回滚命令执行完成"
|
|
||||||
return 0
|
|
||||||
else
|
|
||||||
log_error "❌ 回滚命令执行失败"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# ===========================================
|
|
||||||
# 4. 发送通知
|
|
||||||
# ===========================================
|
|
||||||
send_notification() {
|
|
||||||
local status="$1" # success / failure / rollback
|
|
||||||
local detail="$2"
|
|
||||||
|
|
||||||
if [ "${SKIP_NOTIFY:-false}" = "true" ]; then
|
|
||||||
log_info "跳过通知(SKIP_NOTIFY=true)"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
local webhook="${CI_NOTIFY_WEBHOOK:-}"
|
|
||||||
if [ -z "$webhook" ]; then
|
|
||||||
log_warn "未配置 CI_NOTIFY_WEBHOOK,跳过通知"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -f "$SCRIPT_DIR/deploy_notify.py" ]; then
|
|
||||||
python3 "$SCRIPT_DIR/deploy_notify.py" \
|
|
||||||
--status "$status" \
|
|
||||||
--detail "$detail" \
|
|
||||||
--webhook "$webhook" \
|
|
||||||
--env production \
|
|
||||||
2>/dev/null || log_warn "通知发送失败(非致命)"
|
|
||||||
else
|
|
||||||
log_warn "找不到 deploy_notify.py,跳过通知"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 标记:通知已由健康检查脚本发出,避免 CI 兜底通知重复发送
|
|
||||||
echo "$status" > /tmp/prod_deploy_notification_sent
|
|
||||||
}
|
|
||||||
|
|
||||||
# ===========================================
|
|
||||||
# 主流程
|
|
||||||
# ===========================================
|
|
||||||
main() {
|
|
||||||
echo ""
|
|
||||||
echo "=========================================="
|
|
||||||
echo " CI Production 健康检查 + 自动回滚"
|
|
||||||
echo "=========================================="
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
local deploy_status="success"
|
|
||||||
local deploy_detail=""
|
|
||||||
|
|
||||||
# 1. 设置 SSH
|
|
||||||
if ! setup_ssh; then
|
|
||||||
log_error "SSH 配置失败,无法执行回滚"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 2. 记录部署前状态(回滚目标)
|
|
||||||
save_rollback_target || true
|
|
||||||
|
|
||||||
# 3. 健康检查(公网视角)
|
|
||||||
if ! health_check; then
|
|
||||||
log_error "健康检查失败"
|
|
||||||
deploy_status="failure"
|
|
||||||
deploy_detail="公网健康检查超时,部署后服务未正常响应"
|
|
||||||
|
|
||||||
# 自动回滚
|
|
||||||
if [ "${SKIP_ROLLBACK:-false}" != "true" ]; then
|
|
||||||
log_warn "开始自动回滚..."
|
|
||||||
if do_rollback; then
|
|
||||||
deploy_status="rollback"
|
|
||||||
deploy_detail="健康检查失败,已自动回滚到上一版本 (${ROLLBACK_TAG:-未知})"
|
|
||||||
|
|
||||||
# 回滚后再检查一下公网状态
|
|
||||||
log_info "回滚完成,重新检查公网健康状态..."
|
|
||||||
if health_check; then
|
|
||||||
log_info "✅ 回滚后服务已恢复"
|
|
||||||
deploy_detail="${deploy_detail},回滚后服务已恢复"
|
|
||||||
else
|
|
||||||
log_error "⚠️ 回滚后健康检查仍未通过,请手动排查"
|
|
||||||
deploy_detail="${deploy_detail},但回滚后仍未恢复,请紧急排查"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
deploy_detail="健康检查失败且回滚失败,请手动排查"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
send_notification "$deploy_status" "$deploy_detail"
|
|
||||||
|
|
||||||
# 失败时退出非零,让 CI Job 标记为失败
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 4. 成功
|
|
||||||
log_info ""
|
|
||||||
log_info "=================================="
|
|
||||||
log_info " ✅ Production 部署成功!"
|
|
||||||
log_info "=================================="
|
|
||||||
|
|
||||||
deploy_detail="部署成功,所有健康检查通过 (${GITHUB_REF_NAME:-未知版本})"
|
|
||||||
send_notification "success" "$deploy_detail"
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
@@ -1,250 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# ===========================================
|
|
||||||
# Staging 部署脚本(SSH 模式)
|
|
||||||
# ===========================================
|
|
||||||
# 通过 SSH 在 staging 服务器上执行
|
|
||||||
#
|
|
||||||
# 环境变量:
|
|
||||||
# IMAGE_TAG - 镜像版本 tag(如 commit SHA 或分支名)
|
|
||||||
# REGISTRY_TOKEN - Registry 访问令牌
|
|
||||||
# REGISTRY - Registry 地址(默认 git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas)
|
|
||||||
# REGISTRY_USER - Registry 用户名(默认 xiaoxia)
|
|
||||||
# ENV_FILE - 环境变量文件路径
|
|
||||||
# GENERATED_DIR - 生成文件目录
|
|
||||||
# SKIP_MIGRATION - 跳过数据库迁移(true/false,默认 false)
|
|
||||||
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
IMAGE_TAG="${IMAGE_TAG:-}"
|
|
||||||
REGISTRY="${REGISTRY:-git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas}"
|
|
||||||
REGISTRY_USER="${REGISTRY_USER:-xiaoxia}"
|
|
||||||
REGISTRY_TOKEN="${REGISTRY_TOKEN:-}"
|
|
||||||
|
|
||||||
ENV_FILE="${ENV_FILE:-/var/lib/xiaoxia-saas-staging/.env}"
|
|
||||||
GENERATED_DIR="${GENERATED_DIR:-/var/lib/xiaoxia-saas-staging/generated}"
|
|
||||||
LEGACY_ASSETS_DIR="${LEGACY_ASSETS_DIR:-/var/lib/xiaoxia-saas-staging/legacy-assets}"
|
|
||||||
|
|
||||||
SKIP_MIGRATION="${SKIP_MIGRATION:-false}"
|
|
||||||
|
|
||||||
if [ -z "$IMAGE_TAG" ]; then
|
|
||||||
echo "ERROR: IMAGE_TAG is required"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
test -f "$ENV_FILE"
|
|
||||||
mkdir -p "$GENERATED_DIR"
|
|
||||||
mkdir -p "$LEGACY_ASSETS_DIR"
|
|
||||||
|
|
||||||
echo "=========================================="
|
|
||||||
echo " Staging 部署 - $IMAGE_TAG"
|
|
||||||
echo "=========================================="
|
|
||||||
|
|
||||||
# ---- 登录 Registry ----
|
|
||||||
if [ -n "$REGISTRY_TOKEN" ]; then
|
|
||||||
echo "Logging in to registry: $REGISTRY"
|
|
||||||
REGISTRY_HOST=$(echo "$REGISTRY" | cut -d/ -f1)
|
|
||||||
printf %s "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" -u "$REGISTRY_USER" --password-stdin 2>/dev/null || {
|
|
||||||
echo "WARN: docker login failed, will try to pull anyway"
|
|
||||||
}
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ---- Pull 新版本镜像 ----
|
|
||||||
REGISTRY_API="${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}"
|
|
||||||
REGISTRY_WORKER="${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}"
|
|
||||||
REGISTRY_WEB="${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}"
|
|
||||||
|
|
||||||
LOCAL_API="xiaoxia-saas-api:${IMAGE_TAG}"
|
|
||||||
LOCAL_WORKER="xiaoxia-saas-worker:${IMAGE_TAG}"
|
|
||||||
LOCAL_WEB="xiaoxia-saas-web:${IMAGE_TAG}"
|
|
||||||
|
|
||||||
echo "Pulling API image..."
|
|
||||||
docker pull "$REGISTRY_API"
|
|
||||||
echo "Pulling Worker image..."
|
|
||||||
docker pull "$REGISTRY_WORKER"
|
|
||||||
echo "Pulling Web image..."
|
|
||||||
docker pull "$REGISTRY_WEB"
|
|
||||||
|
|
||||||
# Re-tag 成本地名
|
|
||||||
docker tag "$REGISTRY_API" "$LOCAL_API"
|
|
||||||
docker tag "$REGISTRY_WORKER" "$LOCAL_WORKER"
|
|
||||||
docker tag "$REGISTRY_WEB" "$LOCAL_WEB"
|
|
||||||
echo "All images pulled and tagged."
|
|
||||||
|
|
||||||
# ---- 备份 legacy assets ----
|
|
||||||
echo "Backing up legacy assets from current web container..."
|
|
||||||
if docker inspect xiaoxia-web-staging >/dev/null 2>&1; then
|
|
||||||
_tmpdir="/tmp/legacy-assets-$$"
|
|
||||||
rm -rf "$_tmpdir"
|
|
||||||
mkdir -p "$_tmpdir"
|
|
||||||
docker cp xiaoxia-web-staging:/usr/share/nginx/html/assets/. "$_tmpdir/" 2>/dev/null || true
|
|
||||||
# 只有目录非空才拷贝,避免覆盖有内容的 legacy assets
|
|
||||||
if [ -d "$_tmpdir" ] && [ "$(ls -A "$_tmpdir" 2>/dev/null)" ]; then
|
|
||||||
cp -an "$_tmpdir"/. "$LEGACY_ASSETS_DIR"/ 2>/dev/null || true
|
|
||||||
echo "Legacy assets backed up: $(ls "$_tmpdir" | wc -l) files"
|
|
||||||
fi
|
|
||||||
rm -rf "$_tmpdir"
|
|
||||||
else
|
|
||||||
echo "No existing web container, skipping legacy assets backup"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 清理 7 天前的 legacy assets
|
|
||||||
if [ -d "$LEGACY_ASSETS_DIR" ]; then
|
|
||||||
find "$LEGACY_ASSETS_DIR" -type f -mtime +7 -delete 2>/dev/null || true
|
|
||||||
echo "Legacy assets cleanup done (retain 7 days)"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ---- 检查基础设施容器 ----
|
|
||||||
echo "Checking infrastructure containers..."
|
|
||||||
for c in xiaoxia-postgres-staging xiaoxia-redis-staging; do
|
|
||||||
if ! docker inspect "$c" >/dev/null 2>&1; then
|
|
||||||
echo "ERROR: Required container not found: $c"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
state=$(docker inspect -f '{{.State.Status}}' "$c")
|
|
||||||
if [ "$state" != "running" ]; then
|
|
||||||
echo "ERROR: Container not running: $c ($state)"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
# ---- 创建网络(不存在则创建) ----
|
|
||||||
docker network create xiaoxia-net-staging 2>/dev/null || true
|
|
||||||
|
|
||||||
# ---- 数据库迁移 ----
|
|
||||||
if [ "$SKIP_MIGRATION" != "true" ]; then
|
|
||||||
echo "Running database migrations..."
|
|
||||||
docker run --rm \
|
|
||||||
--env-file "$ENV_FILE" \
|
|
||||||
--network xiaoxia-net-staging \
|
|
||||||
-e APP_ENV=staging \
|
|
||||||
"$LOCAL_API" sh -c "cd /app && alembic upgrade head"
|
|
||||||
echo "Migrations completed."
|
|
||||||
else
|
|
||||||
echo "Skipping migrations (SKIP_MIGRATION=true)"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ---- 停止旧容器 ----
|
|
||||||
echo "Stopping old containers..."
|
|
||||||
docker rm -f xiaoxia-api-staging 2>/dev/null || true
|
|
||||||
docker rm -f xiaoxia-worker-staging 2>/dev/null || true
|
|
||||||
docker rm -f xiaoxia-web-staging 2>/dev/null || true
|
|
||||||
|
|
||||||
LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3"
|
|
||||||
|
|
||||||
# ---- 启动 API ----
|
|
||||||
echo "Starting API container..."
|
|
||||||
docker run -d \
|
|
||||||
--name xiaoxia-api-staging \
|
|
||||||
--env-file "$ENV_FILE" \
|
|
||||||
--network xiaoxia-net-staging \
|
|
||||||
-p 127.0.0.1:8000:8000 \
|
|
||||||
-e APP_ENV=staging \
|
|
||||||
-e APP_VERSION="$IMAGE_TAG" \
|
|
||||||
-e GENERATED_FILES_DIR=/app/generated \
|
|
||||||
-e GENERATED_FILES_URL_PREFIX=/generated-files \
|
|
||||||
-e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \
|
|
||||||
-v "$GENERATED_DIR:/app/generated" \
|
|
||||||
--restart unless-stopped \
|
|
||||||
--health-cmd "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/health', timeout=5)\"" \
|
|
||||||
--health-interval 30s \
|
|
||||||
--health-timeout 10s \
|
|
||||||
--health-retries 3 \
|
|
||||||
--health-start-period 40s \
|
|
||||||
$LOG_OPTS \
|
|
||||||
"$LOCAL_API"
|
|
||||||
|
|
||||||
# ---- 启动 Worker ----
|
|
||||||
echo "Starting Worker container..."
|
|
||||||
docker run -d \
|
|
||||||
--name xiaoxia-worker-staging \
|
|
||||||
--env-file "$ENV_FILE" \
|
|
||||||
--network xiaoxia-net-staging \
|
|
||||||
-e APP_ENV=staging \
|
|
||||||
-e APP_VERSION="$IMAGE_TAG" \
|
|
||||||
-e WORKER_CONCURRENCY=1 \
|
|
||||||
-e WORKER_MAX_TASKS_PER_CHILD=100 \
|
|
||||||
-e GENERATED_FILES_DIR=/app/generated \
|
|
||||||
-e GENERATED_FILES_URL_PREFIX=/generated-files \
|
|
||||||
-e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \
|
|
||||||
-v "$GENERATED_DIR:/app/generated" \
|
|
||||||
--restart unless-stopped \
|
|
||||||
--health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \
|
|
||||||
--health-interval 30s \
|
|
||||||
--health-timeout 10s \
|
|
||||||
--health-retries 3 \
|
|
||||||
--health-start-period 30s \
|
|
||||||
$LOG_OPTS \
|
|
||||||
"$LOCAL_WORKER"
|
|
||||||
|
|
||||||
# ---- 启动 Web ----
|
|
||||||
LEGACY_VOLUME=""
|
|
||||||
if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then
|
|
||||||
LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro"
|
|
||||||
echo "Web container: legacy assets mounted (fallback)"
|
|
||||||
else
|
|
||||||
echo "Web container: no legacy assets to mount"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "Starting Web container..."
|
|
||||||
docker run -d \
|
|
||||||
--name xiaoxia-web-staging \
|
|
||||||
--network xiaoxia-net-staging \
|
|
||||||
-p 127.0.0.1:3001:80 \
|
|
||||||
--restart unless-stopped \
|
|
||||||
$LEGACY_VOLUME \
|
|
||||||
--health-cmd "wget --spider -q http://127.0.0.1:80" \
|
|
||||||
--health-interval 30s \
|
|
||||||
--health-timeout 5s \
|
|
||||||
--health-retries 3 \
|
|
||||||
$LOG_OPTS \
|
|
||||||
"$LOCAL_WEB"
|
|
||||||
|
|
||||||
# ---- 等待 API 健康 ----
|
|
||||||
echo "Waiting for API to become healthy..."
|
|
||||||
i=0
|
|
||||||
while [ "$i" -lt 40 ]; do
|
|
||||||
if curl -sf --max-time 5 http://127.0.0.1:8000/health >/dev/null 2>&1; then
|
|
||||||
echo "API is healthy!"
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
i=$((i + 1))
|
|
||||||
echo " Waiting... ($i/40)"
|
|
||||||
sleep 3
|
|
||||||
done
|
|
||||||
|
|
||||||
if [ "$i" -ge 40 ]; then
|
|
||||||
echo "ERROR: API did not become healthy within 120s"
|
|
||||||
docker logs --tail 50 xiaoxia-api-staging
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ---- 等待 Web 健康 ----
|
|
||||||
echo "Waiting for Web to become healthy..."
|
|
||||||
i=0
|
|
||||||
while [ "$i" -lt 15 ]; do
|
|
||||||
if curl -sf --max-time 5 http://127.0.0.1:3001/ >/dev/null 2>&1; then
|
|
||||||
echo "Web is healthy!"
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
i=$((i + 1))
|
|
||||||
echo " Waiting... ($i/15)"
|
|
||||||
sleep 2
|
|
||||||
done
|
|
||||||
|
|
||||||
if [ "$i" -ge 15 ]; then
|
|
||||||
echo "ERROR: Web did not become healthy within 30s"
|
|
||||||
docker logs --tail 30 xiaoxia-web-staging
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ---- 清理旧镜像 ----
|
|
||||||
echo "Cleaning up old images..."
|
|
||||||
docker image prune -af --filter "until=168h" 2>/dev/null || true
|
|
||||||
docker builder prune -af --filter "until=168h" 2>/dev/null || true
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "=== Staging deployment complete ==="
|
|
||||||
echo "API: http://127.0.0.1:8000"
|
|
||||||
echo "Web: http://127.0.0.1:3001"
|
|
||||||
echo "Version: $IMAGE_TAG"
|
|
||||||
docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep staging
|
|
||||||
@@ -1,473 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# ===========================================
|
|
||||||
# CI Staging 健康检查 + 自动回滚脚本(SSH 部署模式)
|
|
||||||
# ===========================================
|
|
||||||
#
|
|
||||||
# 在 CI Runner 上执行,通过公网 URL 检查 Staging 部署健康状态。
|
|
||||||
# 不健康则通过 SSH 自动回滚到上一个版本的镜像。
|
|
||||||
#
|
|
||||||
# 用法:
|
|
||||||
# ./ci_staging_healthcheck.sh
|
|
||||||
#
|
|
||||||
# 环境变量:
|
|
||||||
# STAGING_API_URL - Staging API 地址 (默认 https://staging-api.xiaoxiajianji.com)
|
|
||||||
# STAGING_WEB_URL - Staging Web 地址 (默认 https://staging.xiaoxiajianji.com)
|
|
||||||
# HEALTH_CHECK_TIMEOUT - 健康检查总超时秒数 (默认 120)
|
|
||||||
# SKIP_ROLLBACK - 失败时不自动回滚 (true/false, 默认 false)
|
|
||||||
# SKIP_NOTIFY - 跳过通知 (true/false, 默认 false)
|
|
||||||
# CI_NOTIFY_WEBHOOK - 通知 Webhook URL
|
|
||||||
#
|
|
||||||
# STAGING_SSH_HOST - Staging 服务器 SSH 地址 (默认 47.98.113.167)
|
|
||||||
# STAGING_SSH_USER - SSH 用户名 (默认 root)
|
|
||||||
# STAGING_SSH_PORT - SSH 端口 (默认 22222)
|
|
||||||
# STAGING_SSH_KEY - SSH 私钥内容
|
|
||||||
# REGISTRY_TOKEN - Registry Token(回滚时拉取旧镜像需要)
|
|
||||||
#
|
|
||||||
# GITHUB_SHA - 当前 commit SHA
|
|
||||||
# GITHUB_REF_NAME - 分支名
|
|
||||||
# GITHUB_RUN_ID - CI Run ID
|
|
||||||
# GITHUB_REPOSITORY - 仓库名
|
|
||||||
# GITHUB_ACTOR - 提交者
|
|
||||||
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
|
|
||||||
|
|
||||||
# 配置
|
|
||||||
STAGING_API_URL="${STAGING_API_URL:-https://staging-api.xiaoxiajianji.com}"
|
|
||||||
STAGING_WEB_URL="${STAGING_WEB_URL:-https://staging.xiaoxiajianji.com}"
|
|
||||||
HEALTH_CHECK_TIMEOUT="${HEALTH_CHECK_TIMEOUT:-120}"
|
|
||||||
SKIP_ROLLBACK="${SKIP_ROLLBACK:-false}"
|
|
||||||
SKIP_NOTIFY="${SKIP_NOTIFY:-false}"
|
|
||||||
|
|
||||||
STAGING_SSH_HOST="${STAGING_SSH_HOST:-47.98.113.167}"
|
|
||||||
STAGING_SSH_USER="${STAGING_SSH_USER:-root}"
|
|
||||||
STAGING_SSH_PORT="${STAGING_SSH_PORT:-22222}"
|
|
||||||
|
|
||||||
REGISTRY="${REGISTRY:-git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas}"
|
|
||||||
REGISTRY_USER="${REGISTRY_USER:-xiaoxia}"
|
|
||||||
|
|
||||||
# 颜色
|
|
||||||
RED='\033[0;31m'
|
|
||||||
GREEN='\033[0;32m'
|
|
||||||
YELLOW='\033[1;33m'
|
|
||||||
BLUE='\033[0;34m'
|
|
||||||
NC='\033[0m'
|
|
||||||
|
|
||||||
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
|
|
||||||
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
|
||||||
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
|
|
||||||
log_step() { echo -e "${BLUE}[STEP]${NC} $1"; }
|
|
||||||
|
|
||||||
# ===========================================
|
|
||||||
# SSH 工具函数
|
|
||||||
# ===========================================
|
|
||||||
SSH_KEY_PATH=""
|
|
||||||
|
|
||||||
setup_ssh() {
|
|
||||||
# 查找或创建 SSH 密钥
|
|
||||||
if [ -f /root/.ssh/xiaoxia_runtime_builder ]; then
|
|
||||||
SSH_KEY_PATH="/root/.ssh/xiaoxia_runtime_builder"
|
|
||||||
elif [ -f "$HOME/.ssh/xiaoxia_runtime_builder" ]; then
|
|
||||||
SSH_KEY_PATH="$HOME/.ssh/xiaoxia_runtime_builder"
|
|
||||||
elif [ -n "${STAGING_SSH_KEY:-}" ]; then
|
|
||||||
SSH_KEY_PATH="$HOME/.ssh/staging_deploy_key"
|
|
||||||
mkdir -p "$HOME/.ssh"
|
|
||||||
printf '%s\n' "$STAGING_SSH_KEY" > "$SSH_KEY_PATH"
|
|
||||||
chmod 600 "$SSH_KEY_PATH"
|
|
||||||
else
|
|
||||||
log_error "没有可用的 SSH 密钥"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
ssh-keyscan -p "$STAGING_SSH_PORT" -H "$STAGING_SSH_HOST" >> ~/.ssh/known_hosts 2>/dev/null || true
|
|
||||||
log_info "SSH 已配置: ${STAGING_SSH_USER}@${STAGING_SSH_HOST}:${STAGING_SSH_PORT}"
|
|
||||||
}
|
|
||||||
|
|
||||||
run_ssh() {
|
|
||||||
local cmd="$1"
|
|
||||||
ssh -p "$STAGING_SSH_PORT" -i "$SSH_KEY_PATH" -o StrictHostKeyChecking=no \
|
|
||||||
"${STAGING_SSH_USER}@${STAGING_SSH_HOST}" "$cmd"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ===========================================
|
|
||||||
# 1. 记录部署前各服务的镜像版本(用于回滚)
|
|
||||||
# ===========================================
|
|
||||||
ROLLBACK_API_TAG=""
|
|
||||||
ROLLBACK_WORKER_TAG=""
|
|
||||||
ROLLBACK_WEB_TAG=""
|
|
||||||
|
|
||||||
save_rollback_target() {
|
|
||||||
log_step "记录当前 staging 各服务镜像版本(回滚目标)..."
|
|
||||||
|
|
||||||
# 通过 SSH 获取当前运行的容器镜像
|
|
||||||
local api_image worker_image web_image
|
|
||||||
api_image=$(run_ssh "docker inspect --format '{{.Config.Image}}' xiaoxia-api-staging 2>/dev/null || echo ''")
|
|
||||||
worker_image=$(run_ssh "docker inspect --format '{{.Config.Image}}' xiaoxia-worker-staging 2>/dev/null || echo ''")
|
|
||||||
web_image=$(run_ssh "docker inspect --format '{{.Config.Image}}' xiaoxia-web-staging 2>/dev/null || echo ''")
|
|
||||||
|
|
||||||
# 提取 tag(镜像名是 xiaoxia-saas-api:abc123 或 git.xiaoxiajianji.com/.../xiaoxia-saas-api:staging 格式)
|
|
||||||
ROLLBACK_API_TAG=$(echo "$api_image" | sed 's/.*://' || echo "")
|
|
||||||
ROLLBACK_WORKER_TAG=$(echo "$worker_image" | sed 's/.*://' || echo "")
|
|
||||||
ROLLBACK_WEB_TAG=$(echo "$web_image" | sed 's/.*://' || echo "")
|
|
||||||
|
|
||||||
log_info " API: ${ROLLBACK_API_TAG:-未知}"
|
|
||||||
log_info " Worker: ${ROLLBACK_WORKER_TAG:-未知}"
|
|
||||||
log_info " Web: ${ROLLBACK_WEB_TAG:-未知}"
|
|
||||||
|
|
||||||
# 验证三个服务版本是否一致
|
|
||||||
if [ -n "$ROLLBACK_API_TAG" ] && [ -n "$ROLLBACK_WORKER_TAG" ] && [ -n "$ROLLBACK_WEB_TAG" ]; then
|
|
||||||
if [ "$ROLLBACK_API_TAG" = "$ROLLBACK_WORKER_TAG" ] && [ "$ROLLBACK_API_TAG" = "$ROLLBACK_WEB_TAG" ]; then
|
|
||||||
log_info " ✅ 三个服务版本一致: $ROLLBACK_API_TAG"
|
|
||||||
export ROLLBACK_TAG="$ROLLBACK_API_TAG"
|
|
||||||
else
|
|
||||||
log_warn " ⚠️ 三个服务版本不一致,回滚时将分别使用各自版本"
|
|
||||||
export ROLLBACK_API_TAG ROLLBACK_WORKER_TAG ROLLBACK_WEB_TAG
|
|
||||||
export ROLLBACK_TAG_MIXED="true"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
log_warn " ⚠️ 未能获取全部服务版本,回滚功能可能受限"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# ===========================================
|
|
||||||
# 2. 健康检查(公网视角)
|
|
||||||
# ===========================================
|
|
||||||
health_check() {
|
|
||||||
local timeout="$HEALTH_CHECK_TIMEOUT"
|
|
||||||
local start_time
|
|
||||||
start_time=$(date +%s)
|
|
||||||
|
|
||||||
log_step "公网健康检查(超时 ${timeout}s)..."
|
|
||||||
log_info " API: ${STAGING_API_URL}/health"
|
|
||||||
log_info " Web: ${STAGING_WEB_URL}/"
|
|
||||||
|
|
||||||
local api_ok=false
|
|
||||||
local web_ok=false
|
|
||||||
local api_docs_ok=false
|
|
||||||
local login_api_ok=false
|
|
||||||
|
|
||||||
while [ $(( $(date +%s) - start_time )) -lt "$timeout" ]; do
|
|
||||||
# 检查 API health
|
|
||||||
if [ "$api_ok" = false ] && curl -sf --max-time 10 "${STAGING_API_URL}/health" >/dev/null 2>&1; then
|
|
||||||
log_info "✅ API 健康检查通过"
|
|
||||||
api_ok=true
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 检查 Web 首页
|
|
||||||
if [ "$web_ok" = false ] && curl -sf --max-time 10 "$STAGING_WEB_URL/" >/dev/null 2>&1; then
|
|
||||||
log_info "✅ Web 前端检查通过"
|
|
||||||
web_ok=true
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 检查 API docs(服务完全启动的标志)
|
|
||||||
if [ "$api_docs_ok" = false ]; then
|
|
||||||
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "${STAGING_API_URL}/docs" 2>/dev/null || echo "000")
|
|
||||||
if [ "$HTTP_CODE" = "200" ]; then
|
|
||||||
log_info "✅ API Docs 检查通过"
|
|
||||||
api_docs_ok=true
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 检查登录 API(业务逻辑正常的标志)
|
|
||||||
if [ "$login_api_ok" = false ]; then
|
|
||||||
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 -X POST \
|
|
||||||
"${STAGING_API_URL}/api/v1/auth/login" \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{"email":"smoke@test.com","password":"wrong"}' 2>/dev/null || echo "000")
|
|
||||||
if [ "$HTTP_CODE" = "401" ] || [ "$HTTP_CODE" = "422" ]; then
|
|
||||||
log_info "✅ 登录 API 检查通过(HTTP $HTTP_CODE,符合预期)"
|
|
||||||
login_api_ok=true
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 都通过了就退出
|
|
||||||
if [ "$api_ok" = true ] && [ "$web_ok" = true ] && [ "$api_docs_ok" = true ] && [ "$login_api_ok" = true ]; then
|
|
||||||
log_info "🎉 所有健康检查通过!"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
sleep 5
|
|
||||||
done
|
|
||||||
|
|
||||||
# 超时了
|
|
||||||
log_error "❌ 健康检查超时 (${timeout}s)"
|
|
||||||
[ "$api_ok" = false ] && log_error " - API health 未通过"
|
|
||||||
[ "$web_ok" = false ] && log_error " - Web 前端未通过"
|
|
||||||
[ "$api_docs_ok" = false ] && log_error " - API Docs 未通过"
|
|
||||||
[ "$login_api_ok" = false ] && log_error " - 登录 API 未通过"
|
|
||||||
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
# ===========================================
|
|
||||||
# 3. 执行回滚(SSH 重新部署旧版本)
|
|
||||||
# ===========================================
|
|
||||||
do_rollback() {
|
|
||||||
log_step "执行回滚:通过 SSH 重新部署旧版本镜像..."
|
|
||||||
|
|
||||||
local rollback_tag="${ROLLBACK_TAG:-}"
|
|
||||||
if [ -z "$rollback_tag" ] && [ "${ROLLBACK_TAG_MIXED:-}" != "true" ]; then
|
|
||||||
log_error "没有可回滚的版本记录,无法自动回滚"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 如果版本不一致,用 API 的版本作为回滚目标
|
|
||||||
if [ -z "$rollback_tag" ]; then
|
|
||||||
rollback_tag="$ROLLBACK_API_TAG"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -z "$rollback_tag" ]; then
|
|
||||||
log_error "无法确定回滚版本"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
log_info "回滚目标版本: $rollback_tag"
|
|
||||||
|
|
||||||
# 构建回滚脚本(直接部署旧版本镜像,不跑 migration)
|
|
||||||
local rollback_script=$(cat << 'ROLLBACK_EOF'
|
|
||||||
#!/bin/sh
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
IMAGE_TAG="$1"
|
|
||||||
REGISTRY_TOKEN="$2"
|
|
||||||
REGISTRY="${REGISTRY:-git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas}"
|
|
||||||
REGISTRY_USER="${REGISTRY_USER:-xiaoxia}"
|
|
||||||
|
|
||||||
ENV_FILE="${ENV_FILE:-/var/lib/xiaoxia-saas-staging/.env}"
|
|
||||||
GENERATED_DIR="${GENERATED_DIR:-/var/lib/xiaoxia-saas-staging/generated}"
|
|
||||||
LEGACY_ASSETS_DIR="${LEGACY_ASSETS_DIR:-/var/lib/xiaoxia-saas-staging/legacy-assets}"
|
|
||||||
|
|
||||||
echo "=== Rollback to $IMAGE_TAG ==="
|
|
||||||
|
|
||||||
# 登录 Registry
|
|
||||||
if [ -n "$REGISTRY_TOKEN" ]; then
|
|
||||||
REGISTRY_HOST=$(echo "$REGISTRY" | cut -d/ -f1)
|
|
||||||
printf %s "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" -u "$REGISTRY_USER" --password-stdin 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Pull 旧版本镜像
|
|
||||||
LOCAL_API="xiaoxia-saas-api:${IMAGE_TAG}"
|
|
||||||
LOCAL_WORKER="xiaoxia-saas-worker:${IMAGE_TAG}"
|
|
||||||
LOCAL_WEB="xiaoxia-saas-web:${IMAGE_TAG}"
|
|
||||||
|
|
||||||
docker pull "${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}"
|
|
||||||
docker pull "${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}"
|
|
||||||
docker pull "${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}"
|
|
||||||
|
|
||||||
docker tag "${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}" "$LOCAL_API"
|
|
||||||
docker tag "${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}" "$LOCAL_WORKER"
|
|
||||||
docker tag "${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}" "$LOCAL_WEB"
|
|
||||||
|
|
||||||
echo "Rollback images pulled."
|
|
||||||
|
|
||||||
# 停止当前容器(回滚不跑 migration,避免数据问题)
|
|
||||||
docker rm -f xiaoxia-api-staging 2>/dev/null || true
|
|
||||||
docker rm -f xiaoxia-worker-staging 2>/dev/null || true
|
|
||||||
docker rm -f xiaoxia-web-staging 2>/dev/null || true
|
|
||||||
|
|
||||||
LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3"
|
|
||||||
|
|
||||||
# 启动 API(回滚不跑 migration)
|
|
||||||
echo "Starting API (rollback)..."
|
|
||||||
docker run -d \
|
|
||||||
--name xiaoxia-api-staging \
|
|
||||||
--env-file "$ENV_FILE" \
|
|
||||||
--network xiaoxia-net-staging \
|
|
||||||
-p 127.0.0.1:8000:8000 \
|
|
||||||
-e APP_ENV=staging \
|
|
||||||
-e APP_VERSION="$IMAGE_TAG" \
|
|
||||||
-e GENERATED_FILES_DIR=/app/generated \
|
|
||||||
-e GENERATED_FILES_URL_PREFIX=/generated-files \
|
|
||||||
-e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \
|
|
||||||
-v "$GENERATED_DIR:/app/generated" \
|
|
||||||
--restart unless-stopped \
|
|
||||||
--health-cmd "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/health', timeout=5)\"" \
|
|
||||||
--health-interval 30s \
|
|
||||||
--health-timeout 10s \
|
|
||||||
--health-retries 3 \
|
|
||||||
--health-start-period 40s \
|
|
||||||
$LOG_OPTS \
|
|
||||||
"$LOCAL_API"
|
|
||||||
|
|
||||||
# 启动 Worker
|
|
||||||
echo "Starting Worker (rollback)..."
|
|
||||||
docker run -d \
|
|
||||||
--name xiaoxia-worker-staging \
|
|
||||||
--env-file "$ENV_FILE" \
|
|
||||||
--network xiaoxia-net-staging \
|
|
||||||
-e APP_ENV=staging \
|
|
||||||
-e APP_VERSION="$IMAGE_TAG" \
|
|
||||||
-e WORKER_CONCURRENCY=1 \
|
|
||||||
-e WORKER_MAX_TASKS_PER_CHILD=100 \
|
|
||||||
-e GENERATED_FILES_DIR=/app/generated \
|
|
||||||
-e GENERATED_FILES_URL_PREFIX=/generated-files \
|
|
||||||
-e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \
|
|
||||||
-v "$GENERATED_DIR:/app/generated" \
|
|
||||||
--restart unless-stopped \
|
|
||||||
--health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \
|
|
||||||
--health-interval 30s \
|
|
||||||
--health-timeout 10s \
|
|
||||||
--health-retries 3 \
|
|
||||||
--health-start-period 30s \
|
|
||||||
$LOG_OPTS \
|
|
||||||
"$LOCAL_WORKER"
|
|
||||||
|
|
||||||
# 启动 Web
|
|
||||||
LEGACY_VOLUME=""
|
|
||||||
if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then
|
|
||||||
LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "Starting Web (rollback)..."
|
|
||||||
docker run -d \
|
|
||||||
--name xiaoxia-web-staging \
|
|
||||||
--network xiaoxia-net-staging \
|
|
||||||
-p 127.0.0.1:3001:80 \
|
|
||||||
--restart unless-stopped \
|
|
||||||
$LEGACY_VOLUME \
|
|
||||||
--health-cmd "wget --spider -q http://127.0.0.1:80" \
|
|
||||||
--health-interval 30s \
|
|
||||||
--health-timeout 5s \
|
|
||||||
--health-retries 3 \
|
|
||||||
$LOG_OPTS \
|
|
||||||
"$LOCAL_WEB"
|
|
||||||
|
|
||||||
# 等待 API 健康
|
|
||||||
echo "Waiting for API (rollback)..."
|
|
||||||
i=0
|
|
||||||
while [ "$i" -lt 40 ]; do
|
|
||||||
if curl -sf --max-time 5 http://127.0.0.1:8000/health >/dev/null 2>&1; then
|
|
||||||
echo "API healthy (rollback)."
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
i=$((i + 1))
|
|
||||||
sleep 3
|
|
||||||
done
|
|
||||||
|
|
||||||
# 等待 Web 健康
|
|
||||||
echo "Waiting for Web (rollback)..."
|
|
||||||
i=0
|
|
||||||
while [ "$i" -lt 15 ]; do
|
|
||||||
if curl -sf --max-time 5 http://127.0.0.1:3001/ >/dev/null 2>&1; then
|
|
||||||
echo "Web healthy (rollback)."
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
i=$((i + 1))
|
|
||||||
sleep 2
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "=== Rollback complete: $IMAGE_TAG ==="
|
|
||||||
docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep staging
|
|
||||||
ROLLBACK_EOF
|
|
||||||
)
|
|
||||||
|
|
||||||
# 将脚本 base64 编码后通过 SSH 执行
|
|
||||||
local script_b64
|
|
||||||
script_b64=$(echo "$rollback_script" | base64 -w 0)
|
|
||||||
|
|
||||||
log_info "在 staging 服务器上执行回滚脚本..."
|
|
||||||
if run_ssh "echo '$script_b64' | base64 -d | sh -s -- '$rollback_tag' '${REGISTRY_TOKEN:-}'" 2>&1; then
|
|
||||||
log_info "✅ 回滚命令执行完成"
|
|
||||||
return 0
|
|
||||||
else
|
|
||||||
log_error "❌ 回滚命令执行失败"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# ===========================================
|
|
||||||
# 4. 发送通知
|
|
||||||
# ===========================================
|
|
||||||
send_notification() {
|
|
||||||
local status="$1" # success / failure / rollback
|
|
||||||
local detail="$2"
|
|
||||||
|
|
||||||
if [ "${SKIP_NOTIFY:-false}" = "true" ]; then
|
|
||||||
log_info "跳过通知(SKIP_NOTIFY=true)"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
local webhook="${CI_NOTIFY_WEBHOOK:-}"
|
|
||||||
if [ -z "$webhook" ]; then
|
|
||||||
log_warn "未配置 CI_NOTIFY_WEBHOOK,跳过通知"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -f "$SCRIPT_DIR/deploy_notify.py" ]; then
|
|
||||||
python3 "$SCRIPT_DIR/deploy_notify.py" \
|
|
||||||
--status "$status" \
|
|
||||||
--detail "$detail" \
|
|
||||||
--webhook "$webhook" \
|
|
||||||
--env staging \
|
|
||||||
2>/dev/null || log_warn "通知发送失败(非致命)"
|
|
||||||
else
|
|
||||||
log_warn "找不到 deploy_notify.py,跳过通知"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# ===========================================
|
|
||||||
# 主流程
|
|
||||||
# ===========================================
|
|
||||||
main() {
|
|
||||||
echo ""
|
|
||||||
echo "==========================================="
|
|
||||||
echo " CI Staging 健康检查 + 自动回滚(SSH模式)"
|
|
||||||
echo "==========================================="
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
local deploy_status="success"
|
|
||||||
local deploy_detail=""
|
|
||||||
|
|
||||||
# 1. 设置 SSH
|
|
||||||
if ! setup_ssh; then
|
|
||||||
log_error "SSH 配置失败,无法执行回滚"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 2. 记录部署前状态(回滚目标)
|
|
||||||
save_rollback_target || true
|
|
||||||
|
|
||||||
# 3. 健康检查(公网视角)
|
|
||||||
if ! health_check; then
|
|
||||||
log_error "健康检查失败"
|
|
||||||
deploy_status="failure"
|
|
||||||
deploy_detail="公网健康检查超时,部署后服务未正常响应"
|
|
||||||
|
|
||||||
# 自动回滚
|
|
||||||
if [ "${SKIP_ROLLBACK:-false}" != "true" ]; then
|
|
||||||
log_warn "开始自动回滚..."
|
|
||||||
if do_rollback; then
|
|
||||||
deploy_status="rollback"
|
|
||||||
deploy_detail="健康检查失败,已自动回滚到上一版本 (${ROLLBACK_TAG:-未知})"
|
|
||||||
|
|
||||||
# 回滚后再检查一下公网状态
|
|
||||||
log_info "回滚完成,重新检查公网健康状态..."
|
|
||||||
if health_check; then
|
|
||||||
log_info "✅ 回滚后服务已恢复"
|
|
||||||
deploy_detail="${deploy_detail},回滚后服务已恢复"
|
|
||||||
else
|
|
||||||
log_error "⚠️ 回滚后健康检查仍未通过,请手动排查"
|
|
||||||
deploy_detail="${deploy_detail},但回滚后仍未恢复,请紧急排查"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
deploy_detail="健康检查失败且回滚失败,请手动排查"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
send_notification "$deploy_status" "$deploy_detail"
|
|
||||||
|
|
||||||
# 失败时退出非零,让 CI Job 标记为失败
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 4. 成功
|
|
||||||
log_info ""
|
|
||||||
log_info "=================================="
|
|
||||||
log_info " ✅ Staging 部署成功!"
|
|
||||||
log_info "=================================="
|
|
||||||
|
|
||||||
deploy_detail="部署成功,所有健康检查通过 (${GITHUB_SHA:-未知版本})"
|
|
||||||
send_notification "success" "$deploy_detail"
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
@@ -1,253 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
部署通知脚本(支持 Staging / Production)
|
|
||||||
|
|
||||||
与 CI 通知(ci_notify_success.py / ci_notify_failure.py)卡片格式对齐。
|
|
||||||
发送部署结果通知到飞书 webhook(卡片格式)。
|
|
||||||
|
|
||||||
支持三种状态:success / failure / rollback
|
|
||||||
支持两种环境:staging / production
|
|
||||||
|
|
||||||
用法:
|
|
||||||
python3 deploy_notify.py --status success --detail "部署成功" --env staging
|
|
||||||
python3 deploy_notify.py --status rollback --detail "健康检查失败,已回滚" --env production
|
|
||||||
python3 deploy_notify.py --status failure --detail "部署过程出错" --env production --failed-step "构建镜像"
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import urllib.request
|
|
||||||
|
|
||||||
STATUS_CONFIG = {
|
|
||||||
"success": {
|
|
||||||
"emoji": "✅",
|
|
||||||
"title_suffix": "部署成功",
|
|
||||||
"color": "green",
|
|
||||||
"button_text": "查看构建详情",
|
|
||||||
"button_type": "primary",
|
|
||||||
},
|
|
||||||
"failure": {
|
|
||||||
"emoji": "❌",
|
|
||||||
"title_suffix": "部署失败",
|
|
||||||
"color": "red",
|
|
||||||
"button_text": "查看失败日志",
|
|
||||||
"button_type": "danger",
|
|
||||||
},
|
|
||||||
"rollback": {
|
|
||||||
"emoji": "↩️",
|
|
||||||
"title_suffix": "部署已回滚",
|
|
||||||
"color": "yellow",
|
|
||||||
"button_text": "查看构建详情",
|
|
||||||
"button_type": "primary",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
ENV_CONFIG = {
|
|
||||||
"staging": {
|
|
||||||
"label": "Staging",
|
|
||||||
"url_web": "https://staging.xiaoxiajianji.com",
|
|
||||||
"url_api": "https://staging-api.xiaoxiajianji.com",
|
|
||||||
},
|
|
||||||
"production": {
|
|
||||||
"label": "Production",
|
|
||||||
"url_web": "https://saas.xiaoxiajianji.com",
|
|
||||||
"url_api": "https://api.xiaoxiajianji.com",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def build_card(
|
|
||||||
status: str,
|
|
||||||
detail: str,
|
|
||||||
env: str = "staging",
|
|
||||||
duration: str = "",
|
|
||||||
failed_step: str = "",
|
|
||||||
pr_url: str = "",
|
|
||||||
) -> dict:
|
|
||||||
"""构建飞书卡片消息(与 ci_notify_*.py 风格一致)。"""
|
|
||||||
cfg = STATUS_CONFIG.get(status, STATUS_CONFIG["failure"])
|
|
||||||
env_cfg = ENV_CONFIG.get(env, ENV_CONFIG["staging"])
|
|
||||||
title = f"{cfg['emoji']} {env_cfg['label']} {cfg['title_suffix']}"
|
|
||||||
|
|
||||||
commit = os.environ.get("GITHUB_SHA", "unknown")[:8]
|
|
||||||
ref = os.environ.get("GITHUB_REF_NAME", "unknown")
|
|
||||||
actor = os.environ.get("GITHUB_ACTOR", "system")
|
|
||||||
run_id = os.environ.get("GITHUB_RUN_ID", "-")
|
|
||||||
repo = os.environ.get("GITHUB_REPOSITORY", "xiaoxia/xiaoxia-saas")
|
|
||||||
|
|
||||||
# 版本信息:tag 部署显示 tag,分支部署显示分支
|
|
||||||
if ref.startswith("v"):
|
|
||||||
version_info = f"版本 {ref}"
|
|
||||||
else:
|
|
||||||
version_info = ref
|
|
||||||
|
|
||||||
# 构建内容行(与 ci_notify_*.py 风格一致:**标签**: 值)
|
|
||||||
lines = []
|
|
||||||
|
|
||||||
# 详情行(部署特有)
|
|
||||||
if detail:
|
|
||||||
lines.append(f"**详情**: {detail}")
|
|
||||||
|
|
||||||
lines.append(f"**环境**: {env_cfg['label']}")
|
|
||||||
lines.append(f"**版本**: {version_info}")
|
|
||||||
|
|
||||||
# 失败阶段(失败/回滚时显示)
|
|
||||||
if failed_step and status in ("failure", "rollback"):
|
|
||||||
lines.append(f"**失败阶段**: {failed_step}")
|
|
||||||
|
|
||||||
# 耗时(可选)
|
|
||||||
if duration:
|
|
||||||
lines.append(f"**耗时**: {duration}")
|
|
||||||
|
|
||||||
lines.append(f"**提交**: {commit}")
|
|
||||||
lines.append(f"**提交者**: {actor}")
|
|
||||||
lines.append(f"**Run ID**: {run_id}")
|
|
||||||
|
|
||||||
elements = [
|
|
||||||
{
|
|
||||||
"tag": "div",
|
|
||||||
"text": {
|
|
||||||
"tag": "lark_md",
|
|
||||||
"content": "\n".join(lines),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
# 查看详情按钮
|
|
||||||
run_url = f"https://git.xiaoxiajianji.com/{repo}/actions/runs/{run_id}"
|
|
||||||
if run_id and run_id != "-":
|
|
||||||
elements.append(
|
|
||||||
{
|
|
||||||
"tag": "action",
|
|
||||||
"actions": [
|
|
||||||
{
|
|
||||||
"tag": "button",
|
|
||||||
"text": {"tag": "plain_text", "content": cfg["button_text"]},
|
|
||||||
"url": run_url,
|
|
||||||
"type": cfg["button_type"],
|
|
||||||
}
|
|
||||||
],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
# PR 链接(可选)
|
|
||||||
if pr_url:
|
|
||||||
elements.append(
|
|
||||||
{
|
|
||||||
"tag": "action",
|
|
||||||
"actions": [
|
|
||||||
{
|
|
||||||
"tag": "button",
|
|
||||||
"text": {"tag": "plain_text", "content": "查看 PR"},
|
|
||||||
"url": pr_url,
|
|
||||||
"type": "default",
|
|
||||||
}
|
|
||||||
],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
# 访问地址(部署特有)
|
|
||||||
elements.append(
|
|
||||||
{
|
|
||||||
"tag": "note",
|
|
||||||
"elements": [
|
|
||||||
{
|
|
||||||
"tag": "plain_text",
|
|
||||||
"content": f"Web: {env_cfg['url_web']} | API: {env_cfg['url_api']}",
|
|
||||||
}
|
|
||||||
],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
return {
|
|
||||||
"msg_type": "interactive",
|
|
||||||
"card": {
|
|
||||||
"header": {
|
|
||||||
"title": {
|
|
||||||
"tag": "plain_text",
|
|
||||||
"content": title,
|
|
||||||
},
|
|
||||||
"status": cfg["color"],
|
|
||||||
},
|
|
||||||
"elements": elements,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def send_notification(
|
|
||||||
webhook: str,
|
|
||||||
status: str,
|
|
||||||
detail: str,
|
|
||||||
env: str = "staging",
|
|
||||||
duration: str = "",
|
|
||||||
failed_step: str = "",
|
|
||||||
pr_url: str = "",
|
|
||||||
) -> bool:
|
|
||||||
"""发送通知到 webhook。"""
|
|
||||||
payload = build_card(status, detail, env, duration, failed_step, pr_url)
|
|
||||||
data = json.dumps(payload).encode("utf-8")
|
|
||||||
|
|
||||||
req = urllib.request.Request(
|
|
||||||
webhook,
|
|
||||||
data=data,
|
|
||||||
headers={"Content-Type": "application/json"},
|
|
||||||
method="POST",
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
|
||||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
|
||||||
resp.read()
|
|
||||||
print(f"通知已发送: {env} {status}")
|
|
||||||
return True
|
|
||||||
except Exception as e:
|
|
||||||
print(f"通知发送失败: {e}", file=sys.stderr)
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
parser = argparse.ArgumentParser(description="部署通知脚本")
|
|
||||||
parser.add_argument(
|
|
||||||
"--status",
|
|
||||||
required=True,
|
|
||||||
choices=["success", "failure", "rollback"],
|
|
||||||
help="部署状态",
|
|
||||||
)
|
|
||||||
parser.add_argument("--detail", default="", help="详情描述")
|
|
||||||
parser.add_argument(
|
|
||||||
"--env",
|
|
||||||
default="staging",
|
|
||||||
choices=["staging", "production"],
|
|
||||||
help="部署环境 (默认 staging)",
|
|
||||||
)
|
|
||||||
parser.add_argument("--duration", default="", help="部署耗时")
|
|
||||||
parser.add_argument("--failed-step", default="", help="失败阶段")
|
|
||||||
parser.add_argument("--pr-url", default="", help="PR 链接")
|
|
||||||
parser.add_argument(
|
|
||||||
"--webhook",
|
|
||||||
default=os.environ.get("CI_NOTIFY_WEBHOOK", ""),
|
|
||||||
help="Webhook URL (也可通过 CI_NOTIFY_WEBHOOK 环境变量设置)",
|
|
||||||
)
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
if not args.webhook:
|
|
||||||
print("未配置 CI_NOTIFY_WEBHOOK,跳过通知")
|
|
||||||
print("如需启用,请在仓库 Settings -> Secrets and variables -> Actions 中添加 CI_NOTIFY_WEBHOOK")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
send_notification(
|
|
||||||
webhook=args.webhook,
|
|
||||||
status=args.status,
|
|
||||||
detail=args.detail,
|
|
||||||
env=args.env,
|
|
||||||
duration=args.duration,
|
|
||||||
failed_step=args.failed_step,
|
|
||||||
pr_url=args.pr_url,
|
|
||||||
)
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -378,17 +378,17 @@ def init_phase6_tasks():
|
|||||||
conn.commit()
|
conn.commit()
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
print("\n[SUCCESS] Phase 6 任务初始化完成!")
|
print(f"\n[SUCCESS] Phase 6 任务初始化完成!")
|
||||||
print(f"📊 总计 {len(PHASE6_TASKS)} 个任务")
|
print(f"📊 总计 {len(PHASE6_TASKS)} 个任务")
|
||||||
print(f"✅ 成功创建 {created_count} 个任务")
|
print(f"✅ 成功创建 {created_count} 个任务")
|
||||||
print("\n任务分布:")
|
print(f"\n任务分布:")
|
||||||
print(" Week 1-2: 基础搭建 - 7 个任务")
|
print(f" Week 1-2: 基础搭建 - 7 个任务")
|
||||||
print(" Week 3: 认证页面 - 5 个任务")
|
print(f" Week 3: 认证页面 - 5 个任务")
|
||||||
print(" Week 4: 工作空间管理 - 6 个任务")
|
print(f" Week 4: 工作空间管理 - 6 个任务")
|
||||||
print(" Week 5: 订阅管理 - 5 个任务")
|
print(f" Week 5: 订阅管理 - 5 个任务")
|
||||||
print(" Week 6: Admin 后台 - 5 个任务")
|
print(f" Week 6: Admin 后台 - 5 个任务")
|
||||||
print(" Week 7: 个人中心 - 4 个任务")
|
print(f" Week 7: 个人中心 - 4 个任务")
|
||||||
print(" Week 8: 测试和优化 - 8 个任务")
|
print(f" Week 8: 测试和优化 - 8 个任务")
|
||||||
print(f"\n预计总工时:{sum(t['estimated_hours'] for t in PHASE6_TASKS)} 小时")
|
print(f"\n预计总工时:{sum(t['estimated_hours'] for t in PHASE6_TASKS)} 小时")
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -294,8 +294,8 @@ def main():
|
|||||||
print("\n" + "=" * 60)
|
print("\n" + "=" * 60)
|
||||||
print("[OK] 数据初始化完成!")
|
print("[OK] 数据初始化完成!")
|
||||||
print("=" * 60)
|
print("=" * 60)
|
||||||
print("\n访问推进器: http://47.98.113.167:8088/projects")
|
print(f"\n访问推进器: http://47.98.113.167:8088/projects")
|
||||||
print("访问 API 文档: http://47.98.113.167:8089/docs\n")
|
print(f"访问 API 文档: http://47.98.113.167:8089/docs\n")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -1,140 +0,0 @@
|
|||||||
"""AudioMerger 单元测试 — P1 裸subprocess下沉验证.
|
|
||||||
|
|
||||||
验证 AudioMerger 使用 shared.ffmpeg_utils.run_ffmpeg 统一入口,
|
|
||||||
不再直接调用 subprocess.run。
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import os
|
|
||||||
import subprocess
|
|
||||||
import tempfile
|
|
||||||
from unittest.mock import MagicMock, patch
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
from application.tts_job.audio_merger import AudioMergeError, AudioMerger
|
|
||||||
|
|
||||||
|
|
||||||
class TestAudioMergerUsesRunFfmpeg:
|
|
||||||
"""验证 AudioMerger 使用 run_ffmpeg 统一入口,而非裸 subprocess."""
|
|
||||||
|
|
||||||
def test_single_file_does_not_call_ffmpeg(self):
|
|
||||||
"""单文件时直接读取,不调用 FFmpeg."""
|
|
||||||
merger = AudioMerger()
|
|
||||||
with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as f:
|
|
||||||
f.write(b"fake audio data")
|
|
||||||
path = f.name
|
|
||||||
|
|
||||||
try:
|
|
||||||
with patch("application.tts_job.audio_merger.run_ffmpeg") as mock_run:
|
|
||||||
result = merger.merge([path])
|
|
||||||
mock_run.assert_not_called()
|
|
||||||
assert result == b"fake audio data"
|
|
||||||
finally:
|
|
||||||
os.unlink(path)
|
|
||||||
|
|
||||||
def test_multiple_files_calls_run_ffmpeg(self):
|
|
||||||
"""多文件时调用 run_ffmpeg 合并。"""
|
|
||||||
merger = AudioMerger()
|
|
||||||
paths = []
|
|
||||||
for i in range(2):
|
|
||||||
f = tempfile.NamedTemporaryFile(suffix=".mp3", delete=False)
|
|
||||||
f.write(f"audio{i}".encode())
|
|
||||||
f.close()
|
|
||||||
paths.append(f.name)
|
|
||||||
|
|
||||||
try:
|
|
||||||
with patch("application.tts_job.audio_merger.run_ffmpeg") as mock_run:
|
|
||||||
# run_ffmpeg 成功返回,模拟合并完成
|
|
||||||
# 需要让 output_path 文件存在,否则 read 会报错
|
|
||||||
def fake_run_ffmpeg(cmd, **kwargs):
|
|
||||||
# 找到 output_path(命令最后一个参数)
|
|
||||||
output_path = cmd[-1]
|
|
||||||
with open(output_path, "wb") as out:
|
|
||||||
out.write(b"merged audio")
|
|
||||||
return ("", "")
|
|
||||||
|
|
||||||
mock_run.side_effect = fake_run_ffmpeg
|
|
||||||
result = merger.merge(paths)
|
|
||||||
|
|
||||||
mock_run.assert_called_once()
|
|
||||||
call_args = mock_run.call_args[0][0]
|
|
||||||
# 验证使用了 FFMPEG_BIN 而非硬编码 "ffmpeg"
|
|
||||||
from shared.ffmpeg_utils import FFMPEG_BIN
|
|
||||||
|
|
||||||
assert call_args[0] == FFMPEG_BIN
|
|
||||||
# 验证使用 concat demuxer
|
|
||||||
assert "concat" in call_args
|
|
||||||
assert result == b"merged audio"
|
|
||||||
finally:
|
|
||||||
for p in paths:
|
|
||||||
os.unlink(p)
|
|
||||||
|
|
||||||
def test_ffmpeg_failure_raises_audio_merge_error(self):
|
|
||||||
"""FFmpeg 失败时抛出 AudioMergeError."""
|
|
||||||
merger = AudioMerger()
|
|
||||||
paths = []
|
|
||||||
for i in range(2):
|
|
||||||
f = tempfile.NamedTemporaryFile(suffix=".mp3", delete=False)
|
|
||||||
f.write(f"audio{i}".encode())
|
|
||||||
f.close()
|
|
||||||
paths.append(f.name)
|
|
||||||
|
|
||||||
try:
|
|
||||||
with patch("application.tts_job.audio_merger.run_ffmpeg") as mock_run:
|
|
||||||
mock_run.side_effect = subprocess.CalledProcessError(
|
|
||||||
returncode=1, cmd=["ffmpeg"], stderr="concat error"
|
|
||||||
)
|
|
||||||
|
|
||||||
with pytest.raises(AudioMergeError, match="FFmpeg 合并失败"):
|
|
||||||
merger.merge(paths)
|
|
||||||
|
|
||||||
mock_run.assert_called_once()
|
|
||||||
finally:
|
|
||||||
for p in paths:
|
|
||||||
os.unlink(p)
|
|
||||||
|
|
||||||
def test_ffmpeg_timeout_raises_audio_merge_error(self):
|
|
||||||
"""FFmpeg 超时时抛出 AudioMergeError."""
|
|
||||||
merger = AudioMerger()
|
|
||||||
paths = []
|
|
||||||
for i in range(2):
|
|
||||||
f = tempfile.NamedTemporaryFile(suffix=".mp3", delete=False)
|
|
||||||
f.write(f"audio{i}".encode())
|
|
||||||
f.close()
|
|
||||||
paths.append(f.name)
|
|
||||||
|
|
||||||
try:
|
|
||||||
with patch("application.tts_job.audio_merger.run_ffmpeg") as mock_run:
|
|
||||||
mock_run.side_effect = subprocess.TimeoutExpired(cmd=["ffmpeg"], timeout=120)
|
|
||||||
|
|
||||||
with pytest.raises(AudioMergeError, match="超时"):
|
|
||||||
merger.merge(paths)
|
|
||||||
|
|
||||||
mock_run.assert_called_once()
|
|
||||||
finally:
|
|
||||||
for p in paths:
|
|
||||||
os.unlink(p)
|
|
||||||
|
|
||||||
def test_empty_list_raises_error(self):
|
|
||||||
"""空列表时直接抛错,不调用 ffmpeg."""
|
|
||||||
merger = AudioMerger()
|
|
||||||
with patch("application.tts_job.audio_merger.run_ffmpeg") as mock_run:
|
|
||||||
with pytest.raises(AudioMergeError, match="没有可合并的音频文件"):
|
|
||||||
merger.merge([])
|
|
||||||
mock_run.assert_not_called()
|
|
||||||
|
|
||||||
def test_no_direct_subprocess_import(self):
|
|
||||||
"""验证 audio_merger 模块不直接 import subprocess(通过模块源码检查)。"""
|
|
||||||
import inspect
|
|
||||||
|
|
||||||
import application.tts_job.audio_merger as am_module
|
|
||||||
|
|
||||||
source = inspect.getsource(am_module)
|
|
||||||
# 不应该有 "import subprocess" 整行
|
|
||||||
src_lines = [line.strip() for line in source.split("\n") if line.strip()]
|
|
||||||
# 允许 from subprocess import CalledProcessError, TimeoutExpired(只导入异常类)
|
|
||||||
# 不允许直接 import subprocess
|
|
||||||
assert not any(
|
|
||||||
line == "import subprocess" for line in src_lines
|
|
||||||
), "audio_merger.py 不应直接 import subprocess,应通过 run_ffmpeg 统一入口"
|
|
||||||
@@ -173,9 +173,7 @@ class TestConfigSchemas:
|
|||||||
cfg = BGMConfig(volume=0.5)
|
cfg = BGMConfig(volume=0.5)
|
||||||
assert cfg.volume == 0.5
|
assert cfg.volume == 0.5
|
||||||
|
|
||||||
from pydantic import ValidationError
|
with pytest.raises(Exception):
|
||||||
|
|
||||||
with pytest.raises(ValidationError):
|
|
||||||
BGMConfig(volume=1.5) # > 1.0 应该校验失败
|
BGMConfig(volume=1.5) # > 1.0 应该校验失败
|
||||||
|
|
||||||
def test_edit_plan_config_schema_full(self):
|
def test_edit_plan_config_schema_full(self):
|
||||||
|
|||||||
@@ -25,8 +25,8 @@ class TestVerifyUrlAccessibleRetry:
|
|||||||
"""_verify_url_accessible 重试逻辑."""
|
"""_verify_url_accessible 重试逻辑."""
|
||||||
|
|
||||||
@patch("time.sleep")
|
@patch("time.sleep")
|
||||||
@patch("urllib.request.OpenerDirector.open")
|
@patch("urllib.request.urlopen")
|
||||||
def test_first_attempt_success(self, mock_open, mock_sleep):
|
def test_first_attempt_success(self, mock_urlopen, mock_sleep):
|
||||||
"""首次成功,不重试."""
|
"""首次成功,不重试."""
|
||||||
from worker_app.tasks.generation import _verify_url_accessible
|
from worker_app.tasks.generation import _verify_url_accessible
|
||||||
|
|
||||||
@@ -34,15 +34,15 @@ class TestVerifyUrlAccessibleRetry:
|
|||||||
mock_resp.status = 200
|
mock_resp.status = 200
|
||||||
mock_resp.__enter__ = MagicMock(return_value=mock_resp)
|
mock_resp.__enter__ = MagicMock(return_value=mock_resp)
|
||||||
mock_resp.__exit__ = MagicMock(return_value=False)
|
mock_resp.__exit__ = MagicMock(return_value=False)
|
||||||
mock_open.return_value = mock_resp
|
mock_urlopen.return_value = mock_resp
|
||||||
|
|
||||||
assert _verify_url_accessible("https://example.com/file.mp4") is True
|
assert _verify_url_accessible("https://example.com/file.mp4") is True
|
||||||
assert mock_open.call_count == 1
|
assert mock_urlopen.call_count == 1
|
||||||
mock_sleep.assert_not_called()
|
mock_sleep.assert_not_called()
|
||||||
|
|
||||||
@patch("time.sleep")
|
@patch("time.sleep")
|
||||||
@patch("urllib.request.OpenerDirector.open")
|
@patch("urllib.request.urlopen")
|
||||||
def test_retry_then_success(self, mock_open, mock_sleep):
|
def test_retry_then_success(self, mock_urlopen, mock_sleep):
|
||||||
"""首次失败,重试后成功."""
|
"""首次失败,重试后成功."""
|
||||||
from worker_app.tasks.generation import _verify_url_accessible
|
from worker_app.tasks.generation import _verify_url_accessible
|
||||||
|
|
||||||
@@ -52,31 +52,31 @@ class TestVerifyUrlAccessibleRetry:
|
|||||||
mock_resp_ok.__enter__ = MagicMock(return_value=mock_resp_ok)
|
mock_resp_ok.__enter__ = MagicMock(return_value=mock_resp_ok)
|
||||||
mock_resp_ok.__exit__ = MagicMock(return_value=False)
|
mock_resp_ok.__exit__ = MagicMock(return_value=False)
|
||||||
|
|
||||||
mock_open.side_effect = [
|
mock_urlopen.side_effect = [
|
||||||
OSError("connection reset"),
|
OSError("connection reset"),
|
||||||
mock_resp_ok,
|
mock_resp_ok,
|
||||||
]
|
]
|
||||||
|
|
||||||
assert _verify_url_accessible("https://example.com/file.mp4") is True
|
assert _verify_url_accessible("https://example.com/file.mp4") is True
|
||||||
assert mock_open.call_count == 2
|
assert mock_urlopen.call_count == 2
|
||||||
mock_sleep.assert_called_once_with(1)
|
mock_sleep.assert_called_once_with(1)
|
||||||
|
|
||||||
@patch("time.sleep")
|
@patch("time.sleep")
|
||||||
@patch("urllib.request.OpenerDirector.open")
|
@patch("urllib.request.urlopen")
|
||||||
def test_all_retries_exhausted(self, mock_open, mock_sleep):
|
def test_all_retries_exhausted(self, mock_urlopen, mock_sleep):
|
||||||
"""全部重试耗尽,返回 False."""
|
"""全部重试耗尽,返回 False."""
|
||||||
from worker_app.tasks.generation import _verify_url_accessible
|
from worker_app.tasks.generation import _verify_url_accessible
|
||||||
|
|
||||||
mock_open.side_effect = OSError("connection refused")
|
mock_urlopen.side_effect = OSError("connection refused")
|
||||||
|
|
||||||
assert _verify_url_accessible("https://example.com/file.mp4") is False
|
assert _verify_url_accessible("https://example.com/file.mp4") is False
|
||||||
# 1 首次 + 2 重试 = 3 次
|
# 1 首次 + 2 重试 = 3 次
|
||||||
assert mock_open.call_count == 3
|
assert mock_urlopen.call_count == 3
|
||||||
assert mock_sleep.call_count == 2
|
assert mock_sleep.call_count == 2
|
||||||
|
|
||||||
@patch("time.sleep")
|
@patch("time.sleep")
|
||||||
@patch("urllib.request.OpenerDirector.open")
|
@patch("urllib.request.urlopen")
|
||||||
def test_http_500_then_success(self, mock_open, mock_sleep):
|
def test_http_500_then_success(self, mock_urlopen, mock_sleep):
|
||||||
"""HTTP 500 后重试成功."""
|
"""HTTP 500 后重试成功."""
|
||||||
from worker_app.tasks.generation import _verify_url_accessible
|
from worker_app.tasks.generation import _verify_url_accessible
|
||||||
|
|
||||||
@@ -90,21 +90,21 @@ class TestVerifyUrlAccessibleRetry:
|
|||||||
mock_resp_200.__enter__ = MagicMock(return_value=mock_resp_200)
|
mock_resp_200.__enter__ = MagicMock(return_value=mock_resp_200)
|
||||||
mock_resp_200.__exit__ = MagicMock(return_value=False)
|
mock_resp_200.__exit__ = MagicMock(return_value=False)
|
||||||
|
|
||||||
mock_open.side_effect = [mock_resp_500, mock_resp_200]
|
mock_urlopen.side_effect = [mock_resp_500, mock_resp_200]
|
||||||
|
|
||||||
assert _verify_url_accessible("https://example.com/file.mp4") is True
|
assert _verify_url_accessible("https://example.com/file.mp4") is True
|
||||||
assert mock_open.call_count == 2
|
assert mock_urlopen.call_count == 2
|
||||||
|
|
||||||
@patch("time.sleep")
|
@patch("time.sleep")
|
||||||
@patch("urllib.request.OpenerDirector.open")
|
@patch("urllib.request.urlopen")
|
||||||
def test_custom_retries_zero(self, mock_open, mock_sleep):
|
def test_custom_retries_zero(self, mock_urlopen, mock_sleep):
|
||||||
"""retries=0 时不重试."""
|
"""retries=0 时不重试."""
|
||||||
from worker_app.tasks.generation import _verify_url_accessible
|
from worker_app.tasks.generation import _verify_url_accessible
|
||||||
|
|
||||||
mock_open.side_effect = OSError("timeout")
|
mock_urlopen.side_effect = OSError("timeout")
|
||||||
|
|
||||||
assert _verify_url_accessible("https://example.com/file.mp4", retries=0) is False
|
assert _verify_url_accessible("https://example.com/file.mp4", retries=0) is False
|
||||||
assert mock_open.call_count == 1
|
assert mock_urlopen.call_count == 1
|
||||||
mock_sleep.assert_not_called()
|
mock_sleep.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -162,14 +162,14 @@ class TestOSSUploadAndVerify:
|
|||||||
mock_response.__enter__ = MagicMock(return_value=mock_response)
|
mock_response.__enter__ = MagicMock(return_value=mock_response)
|
||||||
mock_response.__exit__ = MagicMock(return_value=False)
|
mock_response.__exit__ = MagicMock(return_value=False)
|
||||||
|
|
||||||
with patch("urllib.request.OpenerDirector.open", return_value=mock_response):
|
with patch("urllib.request.urlopen", return_value=mock_response):
|
||||||
assert _verify_url_accessible("https://example.com/test.mp4") is True
|
assert _verify_url_accessible("https://example.com/test.mp4") is True
|
||||||
|
|
||||||
def test_verify_url_accessible_failure(self):
|
def test_verify_url_accessible_failure(self):
|
||||||
"""URL 不可访问时返回 False."""
|
"""URL 不可访问时返回 False."""
|
||||||
from worker_app.tasks.generation import _verify_url_accessible
|
from worker_app.tasks.generation import _verify_url_accessible
|
||||||
|
|
||||||
with patch("urllib.request.OpenerDirector.open", side_effect=Exception("connection refused")):
|
with patch("urllib.request.urlopen", side_effect=Exception("connection refused")):
|
||||||
assert _verify_url_accessible("https://example.com/test.mp4") is False
|
assert _verify_url_accessible("https://example.com/test.mp4") is False
|
||||||
|
|
||||||
def test_verify_url_404(self):
|
def test_verify_url_404(self):
|
||||||
@@ -181,7 +181,7 @@ class TestOSSUploadAndVerify:
|
|||||||
mock_response.__enter__ = MagicMock(return_value=mock_response)
|
mock_response.__enter__ = MagicMock(return_value=mock_response)
|
||||||
mock_response.__exit__ = MagicMock(return_value=False)
|
mock_response.__exit__ = MagicMock(return_value=False)
|
||||||
|
|
||||||
with patch("urllib.request.OpenerDirector.open", return_value=mock_response):
|
with patch("urllib.request.urlopen", return_value=mock_response):
|
||||||
assert _verify_url_accessible("https://example.com/test.mp4") is False
|
assert _verify_url_accessible("https://example.com/test.mp4") is False
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,527 +0,0 @@
|
|||||||
"""URL 安全模块单元测试 - 技术债务第二轮:魔数校验 + 重定向每跳校验."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import os
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
# ══════════════════════════════════════════════════════════════════════════════
|
|
||||||
# 魔数校验测试
|
|
||||||
# ══════════════════════════════════════════════════════════════════════════════
|
|
||||||
|
|
||||||
|
|
||||||
class TestMagicNumberValidation:
|
|
||||||
"""文件头魔数校验测试."""
|
|
||||||
|
|
||||||
def test_png_magic_passes(self, tmp_path: Path):
|
|
||||||
"""PNG 魔数正确应通过校验."""
|
|
||||||
from shared.url_security import _validate_magic_number
|
|
||||||
|
|
||||||
f = tmp_path / "test.png"
|
|
||||||
f.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * 100)
|
|
||||||
|
|
||||||
_validate_magic_number(str(f), {"image/png"}) # 不抛异常即通过
|
|
||||||
|
|
||||||
def test_jpeg_magic_passes(self, tmp_path: Path):
|
|
||||||
"""JPEG 魔数正确应通过校验."""
|
|
||||||
from shared.url_security import _validate_magic_number
|
|
||||||
|
|
||||||
f = tmp_path / "test.jpg"
|
|
||||||
f.write_bytes(b"\xff\xd8\xff\xe0" + b"\x00" * 100)
|
|
||||||
|
|
||||||
_validate_magic_number(str(f), {"image/jpeg", "image/png"})
|
|
||||||
|
|
||||||
def test_gif_magic_passes(self, tmp_path: Path):
|
|
||||||
"""GIF 魔数正确应通过校验."""
|
|
||||||
from shared.url_security import _validate_magic_number
|
|
||||||
|
|
||||||
f = tmp_path / "test.gif"
|
|
||||||
f.write_bytes(b"GIF89a" + b"\x00" * 100)
|
|
||||||
|
|
||||||
_validate_magic_number(str(f), {"image/gif"})
|
|
||||||
|
|
||||||
def test_mp3_magic_id3_passes(self, tmp_path: Path):
|
|
||||||
"""MP3 ID3v2 标签魔数应通过校验."""
|
|
||||||
from shared.url_security import _validate_magic_number
|
|
||||||
|
|
||||||
f = tmp_path / "test.mp3"
|
|
||||||
f.write_bytes(b"ID3\x03\x00\x00\x00\x00\x00\x00" + b"\x00" * 100)
|
|
||||||
|
|
||||||
_validate_magic_number(str(f), {"audio/mpeg"})
|
|
||||||
|
|
||||||
def test_mp3_magic_frame_passes(self, tmp_path: Path):
|
|
||||||
"""MP3 frame sync 魔数应通过校验."""
|
|
||||||
from shared.url_security import _validate_magic_number
|
|
||||||
|
|
||||||
f = tmp_path / "test.mp3"
|
|
||||||
f.write_bytes(b"\xff\xfb\x90\x00" + b"\x00" * 100)
|
|
||||||
|
|
||||||
_validate_magic_number(str(f), {"audio/mpeg"})
|
|
||||||
|
|
||||||
def test_wav_magic_passes(self, tmp_path: Path):
|
|
||||||
"""WAV RIFF+WAVE 魔数应通过校验."""
|
|
||||||
from shared.url_security import _validate_magic_number
|
|
||||||
|
|
||||||
f = tmp_path / "test.wav"
|
|
||||||
header = b"RIFF" + b"\x24\x00\x00\x00" + b"WAVE" + b"fmt " + b"\x00" * 100
|
|
||||||
f.write_bytes(header)
|
|
||||||
|
|
||||||
_validate_magic_number(str(f), {"audio/wav"})
|
|
||||||
|
|
||||||
def test_mp4_magic_passes(self, tmp_path: Path):
|
|
||||||
"""MP4 ftyp 魔数应通过校验."""
|
|
||||||
from shared.url_security import _validate_magic_number
|
|
||||||
|
|
||||||
f = tmp_path / "test.mp4"
|
|
||||||
# ftyp box: size(4) + 'ftyp'(4) + major_brand(4) + ...
|
|
||||||
f.write_bytes(b"\x00\x00\x00\x20ftypisom" + b"\x00" * 100)
|
|
||||||
|
|
||||||
_validate_magic_number(str(f), {"video/mp4"})
|
|
||||||
|
|
||||||
def test_webp_magic_passes(self, tmp_path: Path):
|
|
||||||
"""WebP RIFF+WEBP 魔数应通过校验."""
|
|
||||||
from shared.url_security import _validate_magic_number
|
|
||||||
|
|
||||||
f = tmp_path / "test.webp"
|
|
||||||
f.write_bytes(b"RIFF\x00\x00\x00\x00WEBP" + b"\x00" * 100)
|
|
||||||
|
|
||||||
_validate_magic_number(str(f), {"image/webp"})
|
|
||||||
|
|
||||||
def test_wrong_magic_raises(self, tmp_path: Path):
|
|
||||||
"""魔数不匹配应抛出 UrlSecurityError."""
|
|
||||||
from shared.url_security import UrlSecurityError, _validate_magic_number
|
|
||||||
|
|
||||||
f = tmp_path / "fake.png"
|
|
||||||
f.write_bytes(b"NOT_A_PNG_FILE!!!" + b"\x00" * 100)
|
|
||||||
|
|
||||||
with pytest.raises(UrlSecurityError, match="魔数"):
|
|
||||||
_validate_magic_number(str(f), {"image/png", "image/jpeg"})
|
|
||||||
|
|
||||||
def test_text_as_png_raises(self, tmp_path: Path):
|
|
||||||
"""纯文本伪装成 PNG 应被拦截."""
|
|
||||||
from shared.url_security import UrlSecurityError, _validate_magic_number
|
|
||||||
|
|
||||||
f = tmp_path / "fake.png"
|
|
||||||
f.write_text("<html>not an image</html>", encoding="utf-8")
|
|
||||||
|
|
||||||
with pytest.raises(UrlSecurityError):
|
|
||||||
_validate_magic_number(str(f), {"image/png"})
|
|
||||||
|
|
||||||
def test_empty_file_raises(self, tmp_path: Path):
|
|
||||||
"""空文件应抛出异常."""
|
|
||||||
from shared.url_security import UrlSecurityError, _validate_magic_number
|
|
||||||
|
|
||||||
f = tmp_path / "empty.png"
|
|
||||||
f.write_bytes(b"")
|
|
||||||
|
|
||||||
with pytest.raises(UrlSecurityError, match="为空"):
|
|
||||||
_validate_magic_number(str(f), {"image/png"})
|
|
||||||
|
|
||||||
def test_unknown_mime_skipped(self, tmp_path: Path):
|
|
||||||
"""未知 MIME 类型没有对应魔数,应跳过校验不阻断."""
|
|
||||||
from shared.url_security import _validate_magic_number
|
|
||||||
|
|
||||||
f = tmp_path / "test.xyz"
|
|
||||||
f.write_bytes(b"random garbage data here")
|
|
||||||
|
|
||||||
# 没有已知魔数的 MIME,跳过校验
|
|
||||||
_validate_magic_number(str(f), {"application/x-custom-format"})
|
|
||||||
|
|
||||||
def test_multiple_allowed_types_one_matches(self, tmp_path: Path):
|
|
||||||
"""多个允许类型,只要有一个匹配就通过."""
|
|
||||||
from shared.url_security import _validate_magic_number
|
|
||||||
|
|
||||||
f = tmp_path / "test"
|
|
||||||
f.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * 50)
|
|
||||||
|
|
||||||
_validate_magic_number(str(f), {"image/jpeg", "image/png", "image/gif"})
|
|
||||||
|
|
||||||
def test_multiple_allowed_types_none_match(self, tmp_path: Path):
|
|
||||||
"""多个允许类型都不匹配应抛异常."""
|
|
||||||
from shared.url_security import UrlSecurityError, _validate_magic_number
|
|
||||||
|
|
||||||
f = tmp_path / "test"
|
|
||||||
f.write_bytes(b"RIFF\x00\x00\x00\x00WAVE" + b"\x00" * 50)
|
|
||||||
|
|
||||||
with pytest.raises(UrlSecurityError):
|
|
||||||
_validate_magic_number(str(f), {"image/png", "image/jpeg", "image/gif"})
|
|
||||||
|
|
||||||
def test_flac_magic_passes(self, tmp_path: Path):
|
|
||||||
"""FLAC 魔数应通过校验."""
|
|
||||||
from shared.url_security import _validate_magic_number
|
|
||||||
|
|
||||||
f = tmp_path / "test.flac"
|
|
||||||
f.write_bytes(b"fLaC" + b"\x00" * 100)
|
|
||||||
|
|
||||||
_validate_magic_number(str(f), {"audio/flac"})
|
|
||||||
|
|
||||||
def test_ogg_magic_passes(self, tmp_path: Path):
|
|
||||||
"""OGG 魔数应通过校验."""
|
|
||||||
from shared.url_security import _validate_magic_number
|
|
||||||
|
|
||||||
f = tmp_path / "test.ogg"
|
|
||||||
f.write_bytes(b"OggS\x00\x02\x00\x00" + b"\x00" * 100)
|
|
||||||
|
|
||||||
_validate_magic_number(str(f), {"audio/ogg"})
|
|
||||||
|
|
||||||
def test_bmp_magic_passes(self, tmp_path: Path):
|
|
||||||
"""BMP 魔数应通过校验."""
|
|
||||||
from shared.url_security import _validate_magic_number
|
|
||||||
|
|
||||||
f = tmp_path / "test.bmp"
|
|
||||||
f.write_bytes(b"BM\x00\x00\x00\x00" + b"\x00" * 100)
|
|
||||||
|
|
||||||
_validate_magic_number(str(f), {"image/bmp"})
|
|
||||||
|
|
||||||
|
|
||||||
# ══════════════════════════════════════════════════════════════════════════════
|
|
||||||
# safe_download_file 魔数校验集成测试
|
|
||||||
# ══════════════════════════════════════════════════════════════════════════════
|
|
||||||
|
|
||||||
|
|
||||||
class TestSafeDownloadMagicIntegration:
|
|
||||||
"""safe_download_file 集成魔数校验测试."""
|
|
||||||
|
|
||||||
def test_download_with_mime_and_magic_match(self, tmp_path: Path):
|
|
||||||
"""MIME 匹配 + 魔数匹配,下载成功."""
|
|
||||||
from unittest.mock import MagicMock, patch
|
|
||||||
|
|
||||||
from shared.url_security import safe_download_file
|
|
||||||
|
|
||||||
png_data = b"\x89PNG\r\n\x1a\n" + b"\x00" * 200
|
|
||||||
|
|
||||||
class FakeResp:
|
|
||||||
headers = {"Content-Type": "image/png", "Content-Length": str(len(png_data))}
|
|
||||||
|
|
||||||
def read(self, n):
|
|
||||||
if not hasattr(self, "_pos"):
|
|
||||||
self._pos = 0
|
|
||||||
chunk = png_data[self._pos : self._pos + n]
|
|
||||||
self._pos += len(chunk)
|
|
||||||
return chunk
|
|
||||||
|
|
||||||
def close(self):
|
|
||||||
pass
|
|
||||||
|
|
||||||
class FakeOpener:
|
|
||||||
def open(self, req, timeout=None):
|
|
||||||
return FakeResp()
|
|
||||||
|
|
||||||
with (
|
|
||||||
patch("shared.url_security.validate_url_safety", side_effect=lambda u, **kw: u),
|
|
||||||
patch("shared.url_security.urllib.request.build_opener", return_value=FakeOpener()),
|
|
||||||
):
|
|
||||||
dest = str(tmp_path / "out.png")
|
|
||||||
size = safe_download_file(
|
|
||||||
"https://example.com/test.png",
|
|
||||||
dest,
|
|
||||||
allowed_mime_types={"image/png"},
|
|
||||||
purpose="test",
|
|
||||||
)
|
|
||||||
|
|
||||||
assert size == len(png_data)
|
|
||||||
with open(dest, "rb") as f:
|
|
||||||
assert f.read() == png_data
|
|
||||||
|
|
||||||
def test_download_mime_match_but_magic_mismatch_raises(self, tmp_path: Path):
|
|
||||||
"""Content-Type 声明是 PNG 但实际文件是 HTML,应被魔数校验拦截."""
|
|
||||||
from unittest.mock import patch
|
|
||||||
|
|
||||||
from shared.url_security import UrlSecurityError, safe_download_file
|
|
||||||
|
|
||||||
fake_data = b"<html>not really a png</html>"
|
|
||||||
|
|
||||||
class FakeResp:
|
|
||||||
headers = {"Content-Type": "image/png", "Content-Length": str(len(fake_data))}
|
|
||||||
|
|
||||||
def read(self, n):
|
|
||||||
if not hasattr(self, "_pos"):
|
|
||||||
self._pos = 0
|
|
||||||
chunk = fake_data[self._pos : self._pos + n]
|
|
||||||
self._pos += len(chunk)
|
|
||||||
return chunk
|
|
||||||
|
|
||||||
def close(self):
|
|
||||||
pass
|
|
||||||
|
|
||||||
class FakeOpener:
|
|
||||||
def open(self, req, timeout=None):
|
|
||||||
return FakeResp()
|
|
||||||
|
|
||||||
with (
|
|
||||||
patch("shared.url_security.validate_url_safety", side_effect=lambda u, **kw: u),
|
|
||||||
patch("shared.url_security.urllib.request.build_opener", return_value=FakeOpener()),
|
|
||||||
):
|
|
||||||
dest = str(tmp_path / "out.png")
|
|
||||||
with pytest.raises(UrlSecurityError, match="魔数"):
|
|
||||||
safe_download_file(
|
|
||||||
"https://example.com/fake.png",
|
|
||||||
dest,
|
|
||||||
allowed_mime_types={"image/png"},
|
|
||||||
purpose="test",
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_download_no_mime_check_skips_magic(self, tmp_path: Path):
|
|
||||||
"""不传 allowed_mime_types 时不做 MIME 校验也不做魔数校验."""
|
|
||||||
from unittest.mock import patch
|
|
||||||
|
|
||||||
from shared.url_security import safe_download_file
|
|
||||||
|
|
||||||
data = b"any random content here"
|
|
||||||
|
|
||||||
class FakeResp:
|
|
||||||
headers = {"Content-Type": "application/octet-stream"}
|
|
||||||
|
|
||||||
def read(self, n):
|
|
||||||
if not hasattr(self, "_pos"):
|
|
||||||
self._pos = 0
|
|
||||||
chunk = data[self._pos : self._pos + n]
|
|
||||||
self._pos += len(chunk)
|
|
||||||
return chunk
|
|
||||||
|
|
||||||
def close(self):
|
|
||||||
pass
|
|
||||||
|
|
||||||
class FakeOpener:
|
|
||||||
def open(self, req, timeout=None):
|
|
||||||
return FakeResp()
|
|
||||||
|
|
||||||
with (
|
|
||||||
patch("shared.url_security.validate_url_safety", side_effect=lambda u, **kw: u),
|
|
||||||
patch("shared.url_security.urllib.request.build_opener", return_value=FakeOpener()),
|
|
||||||
):
|
|
||||||
dest = str(tmp_path / "out.bin")
|
|
||||||
size = safe_download_file(
|
|
||||||
"https://example.com/file.bin",
|
|
||||||
dest,
|
|
||||||
purpose="test",
|
|
||||||
)
|
|
||||||
assert size == len(data)
|
|
||||||
|
|
||||||
|
|
||||||
# ══════════════════════════════════════════════════════════════════════════════
|
|
||||||
# _verify_url_accessible 重定向每跳校验测试
|
|
||||||
# ══════════════════════════════════════════════════════════════════════════════
|
|
||||||
|
|
||||||
|
|
||||||
class TestVerifyUrlRedirectValidation:
|
|
||||||
"""URL 可访问性校验 - 重定向每跳 SSRF 校验测试.
|
|
||||||
|
|
||||||
直接复制核心逻辑进行单元测试,避免导入 generation 模块触发 DB 连接。
|
|
||||||
逻辑与 generation.py 中的 _verify_url_accessible 完全一致。
|
|
||||||
"""
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _verify_url_accessible(url, timeout=10.0, retries=0, max_redirects=5):
|
|
||||||
"""从 generation.py 复制的核心逻辑,用于单元测试."""
|
|
||||||
import time
|
|
||||||
import urllib.request
|
|
||||||
from urllib.parse import urljoin
|
|
||||||
|
|
||||||
from shared.url_security import UrlSecurityError, validate_url_safety
|
|
||||||
|
|
||||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
||||||
def redirect_request(self, req, fp, code, msg, headers, newurl): # noqa: N802
|
|
||||||
return None
|
|
||||||
|
|
||||||
last_error = None
|
|
||||||
|
|
||||||
def _do_verify(current_url):
|
|
||||||
redirect_count = 0
|
|
||||||
url_being_checked = current_url
|
|
||||||
opener = urllib.request.build_opener(NoRedirect())
|
|
||||||
|
|
||||||
while redirect_count <= max_redirects:
|
|
||||||
safe_url = validate_url_safety(url_being_checked, purpose="url_verify")
|
|
||||||
req = urllib.request.Request(safe_url, method="HEAD")
|
|
||||||
req.add_header("User-Agent", "xiaoxia-saas-worker/1.0")
|
|
||||||
|
|
||||||
with opener.open(req, timeout=timeout):
|
|
||||||
# 简化:进入 with 块即表示 2xx(3xx 被 NoRedirect 拦截为 HTTPError)
|
|
||||||
return True
|
|
||||||
|
|
||||||
raise Exception("unreachable")
|
|
||||||
|
|
||||||
import urllib.error
|
|
||||||
|
|
||||||
for attempt in range(1 + retries):
|
|
||||||
try:
|
|
||||||
# 用 try/except 手动处理重定向
|
|
||||||
redirect_count = 0
|
|
||||||
current = url
|
|
||||||
opener = urllib.request.build_opener(NoRedirect())
|
|
||||||
|
|
||||||
while redirect_count <= max_redirects:
|
|
||||||
safe_url = validate_url_safety(current, purpose="url_verify")
|
|
||||||
req = urllib.request.Request(safe_url, method="HEAD")
|
|
||||||
req.add_header("User-Agent", "xiaoxia-saas-worker/1.0")
|
|
||||||
try:
|
|
||||||
with opener.open(req, timeout=timeout) as resp:
|
|
||||||
if 200 <= resp.status < 300:
|
|
||||||
return True
|
|
||||||
if resp.status < 400:
|
|
||||||
return True
|
|
||||||
last_error = Exception(f"HTTP {resp.status}")
|
|
||||||
except urllib.error.HTTPError as e:
|
|
||||||
if 300 <= e.code < 400 and e.headers.get("Location"):
|
|
||||||
if redirect_count >= max_redirects:
|
|
||||||
raise Exception(f"重定向次数超过上限 ({max_redirects})")
|
|
||||||
location = e.headers["Location"]
|
|
||||||
current = urljoin(safe_url, location)
|
|
||||||
redirect_count += 1
|
|
||||||
continue
|
|
||||||
last_error = Exception(f"HTTP {e.code}")
|
|
||||||
break
|
|
||||||
else:
|
|
||||||
raise Exception(f"重定向次数超过上限 ({max_redirects})")
|
|
||||||
except Exception as e:
|
|
||||||
last_error = e
|
|
||||||
|
|
||||||
if attempt < retries:
|
|
||||||
time.sleep(0)
|
|
||||||
|
|
||||||
return False
|
|
||||||
|
|
||||||
def test_simple_200_ok(self):
|
|
||||||
"""普通 200 响应应返回 True."""
|
|
||||||
import urllib.error
|
|
||||||
from unittest.mock import patch
|
|
||||||
|
|
||||||
class FakeResp:
|
|
||||||
status = 200
|
|
||||||
headers = {}
|
|
||||||
|
|
||||||
def __enter__(self):
|
|
||||||
return self
|
|
||||||
|
|
||||||
def __exit__(self, *args):
|
|
||||||
pass
|
|
||||||
|
|
||||||
class FakeOpener:
|
|
||||||
def open(self, req, timeout=None):
|
|
||||||
return FakeResp()
|
|
||||||
|
|
||||||
with (
|
|
||||||
patch("urllib.request.build_opener", return_value=FakeOpener()),
|
|
||||||
patch("shared.url_security.validate_url_safety", side_effect=lambda u, **kw: u),
|
|
||||||
):
|
|
||||||
result = self._verify_url_accessible("https://example.com/file.mp4", retries=0)
|
|
||||||
assert result is True
|
|
||||||
|
|
||||||
def test_redirect_to_internal_ip_blocked(self):
|
|
||||||
"""重定向到内网 IP 应被拦截(返回 False)."""
|
|
||||||
import urllib.error
|
|
||||||
from unittest.mock import patch
|
|
||||||
|
|
||||||
from shared.url_security import UrlSecurityError
|
|
||||||
|
|
||||||
call_count = 0
|
|
||||||
|
|
||||||
class FakeHTTPError(urllib.error.HTTPError):
|
|
||||||
def __init__(self):
|
|
||||||
pass
|
|
||||||
|
|
||||||
# 用 validate_url_safety 来模拟拦截
|
|
||||||
def fake_validate(url, **kwargs):
|
|
||||||
if "127.0.0.1" in url:
|
|
||||||
raise UrlSecurityError("内网IP禁止访问")
|
|
||||||
return url
|
|
||||||
|
|
||||||
class FakeOpener:
|
|
||||||
def open(self, req, timeout=None):
|
|
||||||
nonlocal call_count
|
|
||||||
call_count += 1
|
|
||||||
# 第一次请求返回 302
|
|
||||||
raise urllib.error.HTTPError(
|
|
||||||
req.full_url, 302, "Found", {"Location": "http://127.0.0.1/internal"}, None
|
|
||||||
)
|
|
||||||
|
|
||||||
with (
|
|
||||||
patch("urllib.request.build_opener", return_value=FakeOpener()),
|
|
||||||
patch("shared.url_security.validate_url_safety", side_effect=fake_validate),
|
|
||||||
):
|
|
||||||
result = self._verify_url_accessible("https://example.com/redirect", retries=0)
|
|
||||||
assert result is False
|
|
||||||
assert call_count == 1 # 只请求了第一次,第二次跳转在校验阶段就被拦了
|
|
||||||
|
|
||||||
def test_redirect_count_exceeded(self):
|
|
||||||
"""超过最大重定向次数应返回 False."""
|
|
||||||
import urllib.error
|
|
||||||
from unittest.mock import patch
|
|
||||||
|
|
||||||
call_count = 0
|
|
||||||
|
|
||||||
class FakeOpener:
|
|
||||||
def open(self, req, timeout=None):
|
|
||||||
nonlocal call_count
|
|
||||||
call_count += 1
|
|
||||||
raise urllib.error.HTTPError(req.full_url, 302, "Found", {"Location": "https://example.com/next"}, None)
|
|
||||||
|
|
||||||
with (
|
|
||||||
patch("urllib.request.build_opener", return_value=FakeOpener()),
|
|
||||||
patch("shared.url_security.validate_url_safety", side_effect=lambda u, **kw: u),
|
|
||||||
):
|
|
||||||
result = self._verify_url_accessible(
|
|
||||||
"https://example.com/start",
|
|
||||||
retries=0,
|
|
||||||
max_redirects=3,
|
|
||||||
)
|
|
||||||
assert result is False
|
|
||||||
assert call_count == 4 # 初始 + 3次跳转 = 4次请求
|
|
||||||
|
|
||||||
def test_redirect_chain_valid(self):
|
|
||||||
"""合法的重定向链(都是公网域名)应返回 True."""
|
|
||||||
import urllib.error
|
|
||||||
from unittest.mock import patch
|
|
||||||
|
|
||||||
step = 0
|
|
||||||
|
|
||||||
class FakeResp:
|
|
||||||
status = 200
|
|
||||||
headers = {}
|
|
||||||
|
|
||||||
def __enter__(self):
|
|
||||||
return self
|
|
||||||
|
|
||||||
def __exit__(self, *args):
|
|
||||||
pass
|
|
||||||
|
|
||||||
class FakeOpener:
|
|
||||||
def open(self, req, timeout=None):
|
|
||||||
nonlocal step
|
|
||||||
step += 1
|
|
||||||
if step == 1:
|
|
||||||
raise urllib.error.HTTPError(
|
|
||||||
req.full_url, 302, "Found", {"Location": "https://cdn.example.com/final.mp4"}, None
|
|
||||||
)
|
|
||||||
return FakeResp()
|
|
||||||
|
|
||||||
with (
|
|
||||||
patch("urllib.request.build_opener", return_value=FakeOpener()),
|
|
||||||
patch("shared.url_security.validate_url_safety", side_effect=lambda u, **kw: u),
|
|
||||||
):
|
|
||||||
result = self._verify_url_accessible(
|
|
||||||
"https://example.com/redirect",
|
|
||||||
retries=0,
|
|
||||||
max_redirects=5,
|
|
||||||
)
|
|
||||||
assert result is True
|
|
||||||
assert step == 2
|
|
||||||
|
|
||||||
def test_404_returns_false(self):
|
|
||||||
"""404 应返回 False."""
|
|
||||||
import urllib.error
|
|
||||||
from unittest.mock import patch
|
|
||||||
|
|
||||||
class FakeOpener:
|
|
||||||
def open(self, req, timeout=None):
|
|
||||||
raise urllib.error.HTTPError(req.full_url, 404, "Not Found", {}, None)
|
|
||||||
|
|
||||||
with (
|
|
||||||
patch("urllib.request.build_opener", return_value=FakeOpener()),
|
|
||||||
patch("shared.url_security.validate_url_safety", side_effect=lambda u, **kw: u),
|
|
||||||
):
|
|
||||||
result = self._verify_url_accessible("https://example.com/nonexistent", retries=0)
|
|
||||||
assert result is False
|
|
||||||
@@ -107,11 +107,10 @@ class TestAudioMerger:
|
|||||||
finally:
|
finally:
|
||||||
os.unlink(path)
|
os.unlink(path)
|
||||||
|
|
||||||
@patch("packages.application.tts_job.audio_merger.run_ffmpeg")
|
@patch("packages.application.tts_job.audio_merger.subprocess.run")
|
||||||
def test_ffmpeg_called_correctly(self, mock_run_ffmpeg: MagicMock) -> None:
|
def test_ffmpeg_called_correctly(self, mock_run: MagicMock) -> None:
|
||||||
"""多文件调用 FFmpeg concat(通过 run_ffmpeg 统一入口)。"""
|
"""多文件调用 FFmpeg concat。"""
|
||||||
# run_ffmpeg 成功返回 (stdout, stderr)
|
mock_run.return_value = MagicMock(returncode=0)
|
||||||
mock_run_ffmpeg.return_value = ("", "")
|
|
||||||
|
|
||||||
# 创建临时文件
|
# 创建临时文件
|
||||||
paths = []
|
paths = []
|
||||||
@@ -131,24 +130,20 @@ class TestAudioMerger:
|
|||||||
except (FileNotFoundError, OSError):
|
except (FileNotFoundError, OSError):
|
||||||
pass # Expected since we're mocking
|
pass # Expected since we're mocking
|
||||||
|
|
||||||
# 验证 run_ffmpeg 被调用
|
# 验证 FFmpeg 被调用
|
||||||
mock_run_ffmpeg.assert_called_once()
|
mock_run.assert_called_once()
|
||||||
cmd = mock_run_ffmpeg.call_args[0][0]
|
cmd = mock_run.call_args[0][0]
|
||||||
from shared.ffmpeg_utils import FFMPEG_BIN
|
assert cmd[0] == "ffmpeg"
|
||||||
|
|
||||||
assert cmd[0] == FFMPEG_BIN
|
|
||||||
assert "-f" in cmd
|
assert "-f" in cmd
|
||||||
assert "concat" in cmd
|
assert "concat" in cmd
|
||||||
finally:
|
finally:
|
||||||
for p in paths:
|
for p in paths:
|
||||||
os.unlink(p)
|
os.unlink(p)
|
||||||
|
|
||||||
@patch("packages.application.tts_job.audio_merger.run_ffmpeg")
|
@patch("packages.application.tts_job.audio_merger.subprocess.run")
|
||||||
def test_ffmpeg_failure_raises(self, mock_run_ffmpeg: MagicMock) -> None:
|
def test_ffmpeg_failure_raises(self, mock_run: MagicMock) -> None:
|
||||||
"""FFmpeg 失败抛出 AudioMergeError(通过 run_ffmpeg 抛出 CalledProcessError)。"""
|
"""FFmpeg 失败抛出 AudioMergeError。"""
|
||||||
from subprocess import CalledProcessError
|
mock_run.return_value = MagicMock(returncode=1, stderr="error details")
|
||||||
|
|
||||||
mock_run_ffmpeg.side_effect = CalledProcessError(returncode=1, cmd=["ffmpeg"], stderr="error details")
|
|
||||||
|
|
||||||
paths = []
|
paths = []
|
||||||
for i in range(2):
|
for i in range(2):
|
||||||
@@ -160,7 +155,6 @@ class TestAudioMerger:
|
|||||||
merger = AudioMerger()
|
merger = AudioMerger()
|
||||||
with pytest.raises(AudioMergeError, match="FFmpeg 合并失败"):
|
with pytest.raises(AudioMergeError, match="FFmpeg 合并失败"):
|
||||||
merger.merge(paths)
|
merger.merge(paths)
|
||||||
mock_run_ffmpeg.assert_called_once()
|
|
||||||
finally:
|
finally:
|
||||||
for p in paths:
|
for p in paths:
|
||||||
os.unlink(p)
|
os.unlink(p)
|
||||||
|
|||||||
@@ -1660,76 +1660,3 @@ class TestStreamCopy:
|
|||||||
cmd = mock_run.call_args[0][0]
|
cmd = mock_run.call_args[0][0]
|
||||||
assert "copy" in cmd
|
assert "copy" in cmd
|
||||||
assert isinstance(result.output_path, Path)
|
assert isinstance(result.output_path, Path)
|
||||||
|
|
||||||
|
|
||||||
# ══════════════════════════════════════════════════════════════════════════════
|
|
||||||
# _extract_audio 安全下沉测试(P1 技术债务)
|
|
||||||
# ══════════════════════════════════════════════════════════════════════════════
|
|
||||||
|
|
||||||
|
|
||||||
class TestExtractAudioUsesRunFfmpeg:
|
|
||||||
"""_extract_audio 必须使用 run_ffmpeg 统一管理,不能用裸 subprocess."""
|
|
||||||
|
|
||||||
def test_extract_audio_calls_run_ffmpeg(self, tmp_path):
|
|
||||||
"""_extract_audio 内部应调用 ffmpeg_utils.run_ffmpeg 而非裸 subprocess."""
|
|
||||||
from video_processing.unified_render_service import UnifiedRenderService
|
|
||||||
|
|
||||||
plan = MagicMock()
|
|
||||||
plan.config = {}
|
|
||||||
plan.id = "test-plan"
|
|
||||||
plan.canvas_config = MagicMock()
|
|
||||||
plan.canvas_config.width = 1080
|
|
||||||
plan.canvas_config.height = 1920
|
|
||||||
plan.canvas_config.fps = 30
|
|
||||||
plan.canvas_config.output_width = 1080
|
|
||||||
plan.canvas_config.output_height = 1920
|
|
||||||
|
|
||||||
svc = UnifiedRenderService(plan, [], {}, tmp_path)
|
|
||||||
|
|
||||||
video_path = tmp_path / "input.mp4"
|
|
||||||
output_path = tmp_path / "output.wav"
|
|
||||||
video_path.write_bytes(b"fake")
|
|
||||||
|
|
||||||
with patch("video_processing.unified_render_service.run_ffmpeg") as mock_run:
|
|
||||||
svc._extract_audio(video_path, output_path)
|
|
||||||
|
|
||||||
# 验证调用了 run_ffmpeg
|
|
||||||
assert mock_run.called, "_extract_audio 必须通过 run_ffmpeg 执行 FFmpeg"
|
|
||||||
cmd = mock_run.call_args[0][0]
|
|
||||||
|
|
||||||
# 验证命令参数正确
|
|
||||||
assert cmd[0].endswith("ffmpeg") or "ffmpeg" in cmd[0]
|
|
||||||
assert "-i" in cmd
|
|
||||||
assert str(video_path) in cmd
|
|
||||||
assert "-vn" in cmd # 无视频流
|
|
||||||
assert "pcm_s16le" in cmd # 16bit PCM
|
|
||||||
assert "16000" in cmd # 16kHz
|
|
||||||
assert str(output_path) in cmd
|
|
||||||
assert mock_run.call_args[1].get("timeout") == 120
|
|
||||||
|
|
||||||
def test_extract_audio_failure_raises_runtime_error(self, tmp_path):
|
|
||||||
"""_extract_audio 失败时应抛出 RuntimeError."""
|
|
||||||
from video_processing.unified_render_service import UnifiedRenderService
|
|
||||||
|
|
||||||
plan = MagicMock()
|
|
||||||
plan.config = {}
|
|
||||||
plan.id = "test-plan"
|
|
||||||
plan.canvas_config = MagicMock()
|
|
||||||
plan.canvas_config.width = 1080
|
|
||||||
plan.canvas_config.height = 1920
|
|
||||||
plan.canvas_config.fps = 30
|
|
||||||
plan.canvas_config.output_width = 1080
|
|
||||||
plan.canvas_config.output_height = 1920
|
|
||||||
|
|
||||||
svc = UnifiedRenderService(plan, [], {}, tmp_path)
|
|
||||||
|
|
||||||
video_path = tmp_path / "input.mp4"
|
|
||||||
output_path = tmp_path / "output.wav"
|
|
||||||
video_path.write_bytes(b"fake")
|
|
||||||
|
|
||||||
import subprocess
|
|
||||||
|
|
||||||
with patch("video_processing.unified_render_service.run_ffmpeg") as mock_run:
|
|
||||||
mock_run.side_effect = subprocess.CalledProcessError(1, "ffmpeg", stderr="error")
|
|
||||||
with pytest.raises(RuntimeError, match="音频提取失败"):
|
|
||||||
svc._extract_audio(video_path, output_path)
|
|
||||||
|
|||||||
@@ -233,7 +233,7 @@ class TestSafeDownload(unittest.TestCase):
|
|||||||
with unittest.mock.patch("urllib.request.build_opener") as mock_opener:
|
with unittest.mock.patch("urllib.request.build_opener") as mock_opener:
|
||||||
mock_resp = unittest.mock.MagicMock()
|
mock_resp = unittest.mock.MagicMock()
|
||||||
mock_resp.headers = {"Content-Type": "audio/mpeg"}
|
mock_resp.headers = {"Content-Type": "audio/mpeg"}
|
||||||
mock_resp.read.side_effect = [b"ID3audio_data", b""]
|
mock_resp.read.side_effect = [b"audio_data", b""]
|
||||||
mock_resp.geturl.return_value = "https://example.com/test.mp3"
|
mock_resp.geturl.return_value = "https://example.com/test.mp3"
|
||||||
mock_opener.return_value.open.return_value = mock_resp
|
mock_opener.return_value.open.return_value = mock_resp
|
||||||
size = safe_download_file(
|
size = safe_download_file(
|
||||||
@@ -242,7 +242,7 @@ class TestSafeDownload(unittest.TestCase):
|
|||||||
purpose="test",
|
purpose="test",
|
||||||
allowed_mime_types=ALLOWED_AUDIO_MIME_TYPES,
|
allowed_mime_types=ALLOWED_AUDIO_MIME_TYPES,
|
||||||
)
|
)
|
||||||
self.assertEqual(size, 13)
|
self.assertEqual(size, 10)
|
||||||
self.assertTrue(os.path.exists(dest))
|
self.assertTrue(os.path.exists(dest))
|
||||||
|
|
||||||
def test_safe_download_file_stream_size_limit(self):
|
def test_safe_download_file_stream_size_limit(self):
|
||||||
@@ -274,7 +274,7 @@ class TestSafeDownload(unittest.TestCase):
|
|||||||
|
|
||||||
def test_safe_download_bytes_returns_content(self):
|
def test_safe_download_bytes_returns_content(self):
|
||||||
"""safe_download_bytes 应该返回文件内容."""
|
"""safe_download_bytes 应该返回文件内容."""
|
||||||
test_data = b"ID3hello world test audio"
|
test_data = b"hello world test audio"
|
||||||
with unittest.mock.patch("urllib.request.build_opener") as mock_opener:
|
with unittest.mock.patch("urllib.request.build_opener") as mock_opener:
|
||||||
mock_resp = unittest.mock.MagicMock()
|
mock_resp = unittest.mock.MagicMock()
|
||||||
mock_resp.headers = {"Content-Type": "audio/mpeg"}
|
mock_resp.headers = {"Content-Type": "audio/mpeg"}
|
||||||
|
|||||||
Reference in New Issue
Block a user