mirror of
https://github.com/harry0703/MoneyPrinterTurbo.git
synced 2026-02-21 16:37:21 +08:00
556 lines
21 KiB
Python
556 lines
21 KiB
Python
import math
|
||
import os.path
|
||
import re
|
||
from os import path
|
||
|
||
from loguru import logger
|
||
|
||
from app.config import config
|
||
from app.models import const
|
||
from app.models.schema import (
|
||
VideoConcatMode,
|
||
VideoParams,
|
||
VideoAspect,
|
||
MaterialInfo,
|
||
VideoSegment,
|
||
)
|
||
from app.services import llm, material, subtitle, voice, video
|
||
from app.services import video as video_utils
|
||
from app.services import state as sm
|
||
from app.utils import utils
|
||
import time
|
||
|
||
# ... 您已有的 start 函数 ...
|
||
|
||
# ===================================================================
|
||
# 新增的、实现音画同步的主任务函数
|
||
# ===================================================================
|
||
def start_storyboard_task(task_id, params: VideoParams):
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_PROCESSING)
|
||
workdir = utils.task_dir(task_id)
|
||
|
||
# 1. Generate Storyboard
|
||
logger.info("--- Step 1: Generating Storyboard ---")
|
||
video_script = params.video_script
|
||
if not video_script:
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_FAILED, status_message="Video script is empty.")
|
||
return
|
||
|
||
storyboard = llm.generate_storyboard(params.video_subject, video_script)
|
||
if not storyboard:
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_FAILED, status_message="Failed to generate storyboard.")
|
||
return
|
||
|
||
# 2. Process each segment
|
||
logger.info(f"--- Step 2: Processing {len(storyboard)} video segments ---")
|
||
segment_video_paths = []
|
||
segment_audio_paths = []
|
||
segment_srt_paths = []
|
||
total_duration = 0
|
||
last_used_keywords = set()
|
||
|
||
for i, segment in enumerate(storyboard):
|
||
try:
|
||
logger.info(f"--- Processing segment {i + 1} ---")
|
||
segment_script = segment.get("script")
|
||
if not segment_script:
|
||
logger.warning(f"Segment {i + 1} has no script, skipping")
|
||
continue
|
||
|
||
search_terms_str = segment.get("search_terms", "")
|
||
search_terms = [term.strip() for term in search_terms_str.split(',') if term.strip()]
|
||
if not search_terms:
|
||
logger.warning(f"Segment {i + 1} has no search terms, skipping")
|
||
continue
|
||
|
||
# Keyword Guard: Check for repetitive keywords
|
||
current_keywords = set(search_terms)
|
||
if i > 0 and current_keywords == last_used_keywords:
|
||
logger.warning(f"Segment {i + 1} uses the same keywords as the previous one ({search_terms_str}). Reusing last video clip to avoid visual repetition.")
|
||
if segment_video_paths:
|
||
segment_video_paths.append(segment_video_paths[-1]) # Reuse the last processed video clip
|
||
segment_audio_paths.append(segment_audio_paths[-1]) # Reuse the last audio clip
|
||
continue # Skip processing for this segment
|
||
|
||
last_used_keywords = current_keywords
|
||
|
||
# a. Generate audio and subtitles for the segment
|
||
segment_audio_file = path.join(workdir, f"segment_{i + 1}.mp3")
|
||
segment_srt_file = path.join(workdir, f"segment_{i + 1}.srt")
|
||
sub_maker = voice.tts(
|
||
text=segment_script,
|
||
voice_name=voice.parse_voice_name(params.voice_name),
|
||
voice_rate=params.voice_rate,
|
||
voice_file=segment_audio_file,
|
||
)
|
||
if not sub_maker:
|
||
raise Exception(f"Failed to generate audio for segment {i + 1}")
|
||
|
||
# Trim silence from the generated audio
|
||
trimmed_audio_file = path.join(workdir, f"segment_{i + 1}_trimmed.mp3")
|
||
if voice.trim_audio_silence(segment_audio_file, trimmed_audio_file):
|
||
logger.info(f"Silence trimmed for segment {i+1}, using trimmed audio.")
|
||
audio_to_process = trimmed_audio_file
|
||
else:
|
||
logger.warning(f"Failed to trim silence for segment {i+1}, using original audio.")
|
||
audio_to_process = segment_audio_file
|
||
|
||
voice.create_subtitle(
|
||
sub_maker=sub_maker, text=segment_script, subtitle_file=segment_srt_file
|
||
)
|
||
audio_duration = video.get_video_duration(audio_to_process)
|
||
total_duration += audio_duration
|
||
|
||
# b. Calculate the number of clips needed and download them
|
||
num_clips = math.ceil(audio_duration / params.max_clip_duration) if params.max_clip_duration > 0 else 1
|
||
logger.info(f"Segment {i+1} audio duration: {audio_duration:.2f}s, max_clip_duration: {params.max_clip_duration}s. Calculated number of clips: {num_clips}")
|
||
|
||
video_materials = material.download_videos_for_clips(
|
||
video_search_terms=search_terms,
|
||
num_clips=num_clips,
|
||
source=params.video_source
|
||
)
|
||
if not video_materials or len(video_materials) < num_clips:
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_FAILED, status_message=f"Failed to download enough video materials for segment {i + 1}")
|
||
return
|
||
|
||
# c. Create video clip by combining materials with precise durations
|
||
video_segments = []
|
||
remaining_audio_duration = audio_duration
|
||
for video_material in video_materials:
|
||
if remaining_audio_duration <= 0:
|
||
break
|
||
clip_duration = min(remaining_audio_duration, params.max_clip_duration)
|
||
video_segments.append(VideoSegment(path=video_material.path, duration=clip_duration))
|
||
remaining_audio_duration -= clip_duration
|
||
|
||
# If the total duration of the clips is still less than the audio duration, adjust the last clip
|
||
if remaining_audio_duration > 0.01 and video_segments:
|
||
video_segments[-1].duration += remaining_audio_duration
|
||
|
||
segment_video_path = os.path.join(workdir, f"segment_video_{i + 1}.mp4")
|
||
video_created = video.create_video_clip_from_segments(
|
||
segments=video_segments,
|
||
video_aspect=params.video_aspect,
|
||
output_path=segment_video_path
|
||
)
|
||
|
||
if not video_created:
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_FAILED, status_message=f"Video clip creation failed for segment {i + 1}")
|
||
return
|
||
|
||
segment_video_paths.append(segment_video_path)
|
||
segment_audio_paths.append(audio_to_process)
|
||
segment_srt_paths.append(segment_srt_file)
|
||
|
||
except Exception as e:
|
||
logger.error(f"Error processing segment {i + 1}: {e}")
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_FAILED, status_message=f"Error in segment {i + 1}: {e}")
|
||
return
|
||
|
||
# Check if any segments were processed
|
||
if not segment_video_paths:
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_FAILED, status_message="Failed to process any segments.")
|
||
logger.error("Failed to process any segments. Aborting video generation.")
|
||
return
|
||
|
||
# 3. Combine all segments
|
||
logger.info("--- Step 3: Combining all video segments ---")
|
||
# a. Combine audios
|
||
combined_audio_path = path.join(workdir, "voice.mp3")
|
||
if not voice.combine_audio_files(segment_audio_paths, combined_audio_path):
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_FAILED, status_message="Failed to combine audio files.")
|
||
return
|
||
|
||
# b. Combine videos
|
||
video_transition_mode = params.video_transition_mode
|
||
concatenated_video_path = path.join(workdir, "concatenated_video.mp4")
|
||
if not video.concatenate_videos(segment_video_paths, concatenated_video_path, transition_mode=video_transition_mode):
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_FAILED, status_message="Failed to concatenate videos.")
|
||
return
|
||
|
||
# c. Combine subtitles
|
||
combined_srt_path = path.join(workdir, "subtitles.srt")
|
||
subtitle.combine_srt_files(segment_srt_paths, combined_srt_path)
|
||
|
||
# 4. Final video assembly
|
||
logger.info("--- Step 4: Final video assembly ---")
|
||
# a. Add audio to concatenated video
|
||
video_with_audio_path = path.join(workdir, "video_with_audio.mp4")
|
||
if not video.add_audio_to_video(concatenated_video_path, combined_audio_path, video_with_audio_path):
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_FAILED, status_message="Failed to add audio to video.")
|
||
return
|
||
|
||
# b. Add background music
|
||
video_with_bgm_path = path.join(workdir, "video_with_bgm.mp4")
|
||
bgm_file = video.get_bgm_file(bgm_type=params.bgm_type, bgm_file=params.bgm_file)
|
||
if bgm_file:
|
||
if not video.add_bgm_to_video(
|
||
input_video_path=video_with_audio_path,
|
||
bgm_path=bgm_file,
|
||
bgm_volume=params.bgm_volume,
|
||
output_video_path=video_with_bgm_path
|
||
):
|
||
logger.warning("Failed to mix BGM. Proceeding without it.")
|
||
video_with_bgm_path = video_with_audio_path # Fallback
|
||
else:
|
||
video_with_bgm_path = video_with_audio_path # No BGM requested
|
||
|
||
# c. Add subtitles
|
||
final_video_path = path.join(workdir, f"final_{task_id}.mp4")
|
||
# video.add_subtitles_to_video(
|
||
# video_path=video_with_bgm_path,
|
||
# srt_path=combined_srt_path,
|
||
# font_name=params.font_name,
|
||
# font_size=params.font_size,
|
||
# text_fore_color=params.text_fore_color,
|
||
# stroke_color=params.stroke_color,
|
||
# stroke_width=params.stroke_width,
|
||
# subtitle_position=params.subtitle_position,
|
||
# custom_position=params.custom_position,
|
||
# output_path=final_video_path
|
||
# )
|
||
import shutil
|
||
shutil.copy(video_with_bgm_path, final_video_path)
|
||
|
||
# 5. Cleanup
|
||
logger.info("--- Step 5: Cleaning up temporary files ---")
|
||
cleanup_files = segment_video_paths + segment_audio_paths + segment_srt_paths + [combined_audio_path, concatenated_video_path, combined_srt_path, video_with_audio_path, video_with_bgm_path]
|
||
for item in cleanup_files:
|
||
if item and item != final_video_path and os.path.exists(item):
|
||
os.remove(item)
|
||
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_COMPLETE, progress=100, video_path=final_video_path)
|
||
logger.success(f"Task {task_id} completed successfully. Final video: {final_video_path}")
|
||
|
||
|
||
|
||
return {"videos": [final_video_path]}
|
||
|
||
|
||
def generate_script(task_id, params):
|
||
logger.info("\n\n## generating video script")
|
||
video_script = params.video_script.strip()
|
||
if not video_script:
|
||
video_script = llm.generate_script(
|
||
video_subject=params.video_subject,
|
||
language=params.video_language,
|
||
paragraph_number=params.paragraph_number,
|
||
)
|
||
else:
|
||
logger.debug(f"video script: \n{video_script}")
|
||
|
||
if not video_script:
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_FAILED)
|
||
logger.error("failed to generate video script.")
|
||
return None
|
||
|
||
return video_script
|
||
|
||
|
||
def generate_terms(task_id, params, video_script):
|
||
logger.info("\n\n## generating video terms")
|
||
video_terms = params.video_terms
|
||
if not video_terms:
|
||
video_terms = llm.generate_terms(
|
||
video_subject=params.video_subject, video_script=video_script
|
||
)
|
||
else:
|
||
if isinstance(video_terms, str):
|
||
video_terms = [term.strip() for term in re.split(r"[,，]", video_terms)]
|
||
elif isinstance(video_terms, list):
|
||
video_terms = [term.strip() for term in video_terms]
|
||
else:
|
||
raise ValueError("video_terms must be a string or a list of strings.")
|
||
|
||
logger.debug(f"video terms: {utils.to_json(video_terms)}")
|
||
|
||
if not video_terms:
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_FAILED)
|
||
logger.error("failed to generate video terms.")
|
||
return None
|
||
|
||
return video_terms
|
||
|
||
|
||
def save_script_data(task_id, video_script, video_terms, params):
|
||
script_file = path.join(utils.task_dir(task_id), "script.json")
|
||
script_data = {
|
||
"script": video_script,
|
||
"search_terms": video_terms,
|
||
"params": params,
|
||
}
|
||
|
||
with open(script_file, "w", encoding="utf-8") as f:
|
||
f.write(utils.to_json(script_data))
|
||
|
||
|
||
def generate_audio(task_id, params, video_script):
|
||
logger.info("\n\n## generating audio")
|
||
audio_file = path.join(utils.task_dir(task_id), "audio.mp3")
|
||
sub_maker = voice.tts(
|
||
text=video_script,
|
||
voice_name=voice.parse_voice_name(params.voice_name),
|
||
voice_rate=params.voice_rate,
|
||
voice_file=audio_file,
|
||
)
|
||
if sub_maker is None:
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_FAILED)
|
||
logger.error(
|
||
"""failed to generate audio:
|
||
1. check if the language of the voice matches the language of the video script.
|
||
2. check if the network is available. If you are in China, it is recommended to use a VPN and enable the global traffic mode.
|
||
""".strip()
|
||
)
|
||
return None, None, None
|
||
|
||
audio_duration = math.ceil(voice.get_audio_duration(sub_maker))
|
||
return audio_file, audio_duration, sub_maker
|
||
|
||
|
||
def generate_subtitle(task_id, params, video_script, sub_maker, audio_file):
|
||
if not params.subtitle_enabled:
|
||
return ""
|
||
|
||
subtitle_path = path.join(utils.task_dir(task_id), "subtitle.srt")
|
||
subtitle_provider = config.app.get("subtitle_provider", "edge").strip().lower()
|
||
logger.info(f"\n\n## generating subtitle, provider: {subtitle_provider}")
|
||
|
||
subtitle_fallback = False
|
||
if subtitle_provider == "edge":
|
||
voice.create_subtitle(
|
||
text=video_script, sub_maker=sub_maker, subtitle_file=subtitle_path
|
||
)
|
||
if not os.path.exists(subtitle_path):
|
||
subtitle_fallback = True
|
||
logger.warning("subtitle file not found, fallback to whisper")
|
||
|
||
if subtitle_provider == "whisper" or subtitle_fallback:
|
||
subtitle.create(audio_file=audio_file, subtitle_file=subtitle_path)
|
||
logger.info("\n\n## correcting subtitle")
|
||
subtitle.correct(subtitle_file=subtitle_path, video_script=video_script)
|
||
|
||
subtitle_lines = subtitle.file_to_subtitles(subtitle_path)
|
||
if not subtitle_lines:
|
||
logger.warning(f"subtitle file is invalid: {subtitle_path}")
|
||
return ""
|
||
|
||
return subtitle_path
|
||
|
||
|
||
def get_video_materials(task_id, params, video_terms, audio_duration):
|
||
if params.video_source == "local":
|
||
logger.info("\n\n## preprocess local materials")
|
||
materials = video.preprocess_video(
|
||
materials=params.video_materials, clip_duration=params.max_clip_duration
|
||
)
|
||
if not materials:
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_FAILED)
|
||
logger.error(
|
||
"no valid materials found, please check the materials and try again."
|
||
)
|
||
return None
|
||
return [material_info.url for material_info in materials]
|
||
else:
|
||
logger.info(f"\n\n## downloading videos from {params.video_source}")
|
||
downloaded_videos = material.download_videos(
|
||
task_id=task_id,
|
||
video_subject=params.video_subject,
|
||
search_terms=video_terms,
|
||
source=params.video_source,
|
||
video_aspect=params.video_aspect,
|
||
video_contact_mode=params.video_concat_mode,
|
||
audio_duration=audio_duration * params.video_count,
|
||
max_clip_duration=params.max_clip_duration,
|
||
)
|
||
if not downloaded_videos:
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_FAILED)
|
||
logger.error(
|
||
"failed to download videos, maybe the network is not available. if you are in China, please use a VPN."
|
||
)
|
||
return None
|
||
return downloaded_videos
|
||
|
||
|
||
def generate_final_videos(
|
||
task_id, params, downloaded_videos, audio_file, subtitle_path
|
||
):
|
||
final_video_paths = []
|
||
combined_video_paths = []
|
||
video_concat_mode = (
|
||
params.video_concat_mode if params.video_count == 1 else VideoConcatMode.random
|
||
)
|
||
video_transition_mode = params.video_transition_mode
|
||
|
||
_progress = 50
|
||
for i in range(params.video_count):
|
||
index = i + 1
|
||
combined_video_path = path.join(
|
||
utils.task_dir(task_id), f"combined-{index}.mp4"
|
||
)
|
||
logger.info(f"\n\n## combining video: {index} => {combined_video_path}")
|
||
video_utils.combine_videos_ffmpeg(
|
||
combined_video_path=combined_video_path,
|
||
video_paths=downloaded_videos,
|
||
audio_file=audio_file,
|
||
video_aspect=params.video_aspect,
|
||
video_concat_mode=video_concat_mode,
|
||
video_transition_mode=video_transition_mode,
|
||
max_clip_duration=params.max_clip_duration,
|
||
threads=params.n_threads,
|
||
)
|
||
|
||
_progress += 50 / params.video_count / 2
|
||
sm.state.update_task(task_id, progress=_progress)
|
||
|
||
final_video_path = path.join(utils.task_dir(task_id), f"final-{index}.mp4")
|
||
|
||
logger.info(f"\n\n## generating video: {index} => {final_video_path}")
|
||
video_utils.generate_video(
|
||
video_path=combined_video_path,
|
||
audio_path=audio_file,
|
||
subtitle_path=subtitle_path,
|
||
output_file=final_video_path,
|
||
params=params,
|
||
)
|
||
|
||
_progress += 50 / params.video_count / 2
|
||
sm.state.update_task(task_id, progress=_progress)
|
||
|
||
final_video_paths.append(final_video_path)
|
||
combined_video_paths.append(combined_video_path)
|
||
|
||
return final_video_paths, combined_video_paths
|
||
|
||
|
||
def start(task_id, params: VideoParams, stop_at: str = "video"):
|
||
logger.info(f"start task: {task_id}, stop_at: {stop_at}")
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_PROCESSING, progress=5)
|
||
|
||
if type(params.video_concat_mode) is str:
|
||
params.video_concat_mode = VideoConcatMode(params.video_concat_mode)
|
||
|
||
# 1. Generate script
|
||
video_script = generate_script(task_id, params)
|
||
if not video_script or "Error: " in video_script:
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_FAILED)
|
||
return
|
||
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_PROCESSING, progress=10)
|
||
|
||
if stop_at == "script":
|
||
sm.state.update_task(
|
||
task_id, state=const.TASK_STATE_COMPLETE, progress=100, script=video_script
|
||
)
|
||
return {"script": video_script}
|
||
|
||
# 2. Generate terms
|
||
video_terms = ""
|
||
if params.video_source != "local":
|
||
video_terms = generate_terms(task_id, params, video_script)
|
||
if not video_terms:
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_FAILED)
|
||
return
|
||
|
||
save_script_data(task_id, video_script, video_terms, params)
|
||
|
||
if stop_at == "terms":
|
||
sm.state.update_task(
|
||
task_id, state=const.TASK_STATE_COMPLETE, progress=100, terms=video_terms
|
||
)
|
||
return {"script": video_script, "terms": video_terms}
|
||
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_PROCESSING, progress=20)
|
||
|
||
# 3. Generate audio
|
||
audio_file, audio_duration, sub_maker = generate_audio(
|
||
task_id, params, video_script
|
||
)
|
||
if not audio_file:
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_FAILED)
|
||
return
|
||
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_PROCESSING, progress=30)
|
||
|
||
if stop_at == "audio":
|
||
sm.state.update_task(
|
||
task_id,
|
||
state=const.TASK_STATE_COMPLETE,
|
||
progress=100,
|
||
audio_file=audio_file,
|
||
)
|
||
return {"audio_file": audio_file, "audio_duration": audio_duration}
|
||
|
||
# 4. Generate subtitle
|
||
subtitle_path = generate_subtitle(
|
||
task_id, params, video_script, sub_maker, audio_file
|
||
)
|
||
|
||
if stop_at == "subtitle":
|
||
sm.state.update_task(
|
||
task_id,
|
||
state=const.TASK_STATE_COMPLETE,
|
||
progress=100,
|
||
subtitle_path=subtitle_path,
|
||
)
|
||
return {"subtitle_path": subtitle_path}
|
||
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_PROCESSING, progress=40)
|
||
|
||
# 5. Get video materials
|
||
downloaded_videos = get_video_materials(
|
||
task_id, params, video_terms, audio_duration
|
||
)
|
||
if not downloaded_videos:
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_FAILED)
|
||
return
|
||
|
||
if stop_at == "materials":
|
||
sm.state.update_task(
|
||
task_id,
|
||
state=const.TASK_STATE_COMPLETE,
|
||
progress=100,
|
||
materials=downloaded_videos,
|
||
)
|
||
return {"materials": downloaded_videos}
|
||
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_PROCESSING, progress=50)
|
||
|
||
# 6. Generate final videos
|
||
final_video_paths, combined_video_paths = generate_final_videos(
|
||
task_id, params, downloaded_videos, audio_file, subtitle_path
|
||
)
|
||
|
||
if not final_video_paths:
|
||
sm.state.update_task(task_id, state=const.TASK_STATE_FAILED)
|
||
return
|
||
|
||
logger.success(
|
||
f"task {task_id} finished, generated {len(final_video_paths)} videos."
|
||
)
|
||
|
||
kwargs = {
|
||
"videos": final_video_paths,
|
||
"combined_videos": combined_video_paths,
|
||
"script": video_script,
|
||
"terms": video_terms,
|
||
"audio_file": audio_file,
|
||
"audio_duration": audio_duration,
|
||
"subtitle_path": subtitle_path,
|
||
"materials": downloaded_videos,
|
||
}
|
||
sm.state.update_task(
|
||
task_id, state=const.TASK_STATE_COMPLETE, progress=100, **kwargs
|
||
)
|
||
return kwargs
|
||
|
||
|
||
if __name__ == "__main__":
|
||
task_id = "task_id"
|
||
params = VideoParams(
|
||
video_subject="金钱的作用",
|
||
voice_name="zh-CN-XiaoyiNeural-Female",
|
||
voice_rate=1.0,
|
||
)
|
||
start(task_id, params, stop_at="video")
|