jacquie/batchtube
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: initial Tauri desktop shell for BatchTube v1.0

Browse Source
This commit is contained in:
Agent Zero
2026-03-07 01:27:25 +01:00
commit a1853bb68a
25 changed files with 3665 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

443
backend/api_server.py Normal file
View File

@@ -0,0 +1,443 @@
"""
api_server.py - FastAPI backend for YouTube SIPOC Tool
Serves the Web GUI, REST API, and Server-Sent Events (SSE) for real-time progress.
"""
import sys
import os
from pathlib import Path
# Ensure project directory is always on sys.path (needed for some Python installations)
_PROJECT_DIR = Path(__file__).parent.resolve()
if str(_PROJECT_DIR) not in sys.path:
sys.path.insert(0, str(_PROJECT_DIR))
import json
import asyncio
import logging
from typing import Optional, List
from fastapi import FastAPI, HTTPException, BackgroundTasks, Request
from fastapi.responses import HTMLResponse, FileResponse, StreamingResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
import uvicorn
import database as db
import worker
import config
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
)
logger = logging.getLogger("api_server")
app = FastAPI(title="YouTube SIPOC Tool", version="1.0.0")
# CORS - allow localhost GUI access
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_methods=["*"],
allow_headers=["*"],
)
# Serve static web files
WEB_DIR = Path(__file__).parent / "web"
# ─────────────────────────────────────────────────────────────────────────────
# Request / Response Models
# ─────────────────────────────────────────────────────────────────────────────
class CreateJobRequest(BaseModel):
input_type: str # url | playlist | csv | handle
input_value: str # the URL/handle/CSV text
output_folder: str # absolute path to save files
selections: List[str] # artifact types to extract
resolution: str = "best" # for MP4
delay_min: float = 2.0
delay_max: float = 7.0
openrouter_api_key: str = ""
openrouter_model: str = "openai/gpt-4o-mini"
summary_prompt: str = ""
class ResumeJobRequest(BaseModel):
openrouter_api_key: str = ""
class BrowseFolderRequest(BaseModel):
path: str = ""
# ─────────────────────────────────────────────────────────────────────────────
# Web GUI Serving
# ─────────────────────────────────────────────────────────────────────────────
@app.get("/", response_class=HTMLResponse)
async def serve_gui():
index_path = WEB_DIR / "index.html"
if not index_path.exists():
return HTMLResponse("<h1>Web GUI not found. Check the web/ directory.</h1>", status_code=404)
return HTMLResponse(index_path.read_text(encoding="utf-8"))
@app.get("/app.js")
async def serve_js():
return FileResponse(WEB_DIR / "app.js", media_type="application/javascript")
@app.get("/style.css")
async def serve_css():
return FileResponse(WEB_DIR / "style.css", media_type="text/css")
# ─────────────────────────────────────────────────────────────────────────────
# Job Management API
# ─────────────────────────────────────────────────────────────────────────────
@app.post("/api/jobs")
async def create_job(req: CreateJobRequest):
"""Create and immediately launch a new extraction job."""
# Validate output folder
output_folder = req.output_folder.strip()
if not output_folder:
raise HTTPException(status_code=400, detail="output_folder is required")
# Validate selections
valid_types = set(config.ARTIFACT_TYPES)
invalid = [s for s in req.selections if s not in valid_types]
if invalid:
raise HTTPException(status_code=400, detail=f"Invalid artifact types: {invalid}")
if not req.selections:
raise HTTPException(status_code=400, detail="At least one artifact type must be selected")
# Validate input
if not req.input_value.strip():
raise HTTPException(status_code=400, detail="input_value is required")
# Create the folder if it doesn't exist
try:
Path(output_folder).mkdir(parents=True, exist_ok=True)
except Exception as e:
raise HTTPException(status_code=400, detail=f"Cannot create output folder: {e}")
# Create job in DB
job_id = db.create_job(
output_folder=output_folder,
selections=req.selections,
resolution=req.resolution,
delay_min=req.delay_min,
delay_max=req.delay_max,
input_type=req.input_type,
input_value=req.input_value,
summary_prompt=req.summary_prompt or config.SUMMARY_SYSTEM_PROMPT,
openrouter_model=req.openrouter_model or config.OPENROUTER_MODEL,
)
# Launch background worker thread
worker.launch_job(job_id, openrouter_api_key=req.openrouter_api_key)
logger.info(f"Created and launched job {job_id}")
return {"job_id": job_id, "status": "running"}
@app.get("/api/jobs")
async def list_jobs():
"""List all jobs (most recent first)."""
jobs = db.list_jobs()
result = []
for j in jobs:
stats = db.get_job_stats(j["job_id"])
j.update(stats)
result.append(j)
return result
@app.get("/api/jobs/{job_id}")
async def get_job(job_id: str):
"""Get job details + stats."""
job = db.get_job(job_id)
if not job:
raise HTTPException(status_code=404, detail="Job not found")
stats = db.get_job_stats(job_id)
job.update(stats)
return job
@app.post("/api/jobs/{job_id}/pause")
async def pause_job(job_id: str):
job = db.get_job(job_id)
if not job:
raise HTTPException(status_code=404, detail="Job not found")
if job["status"] not in ("running",):
raise HTTPException(status_code=400, detail=f"Cannot pause a job with status: {job['status']}")
worker.pause_job(job_id)
return {"job_id": job_id, "status": "paused"}
@app.post("/api/jobs/{job_id}/resume")
async def resume_job(job_id: str, req: ResumeJobRequest = ResumeJobRequest()):
job = db.get_job(job_id)
if not job:
raise HTTPException(status_code=404, detail="Job not found")
if job["status"] == "paused":
# Job thread is still alive, just unpause it
worker.resume_job(job_id)
return {"job_id": job_id, "status": "running"}
elif job["status"] in ("stopped", "failed", "done_with_errors"):
# Re-launch the job from where it left off
db.update_job_status(job_id, "pending")
worker.launch_job(job_id, openrouter_api_key=req.openrouter_api_key)
return {"job_id": job_id, "status": "running", "note": "Job re-launched to resume from last checkpoint"}
else:
raise HTTPException(status_code=400, detail=f"Cannot resume job with status: {job['status']}")
@app.post("/api/jobs/{job_id}/stop")
async def stop_job(job_id: str):
job = db.get_job(job_id)
if not job:
raise HTTPException(status_code=404, detail="Job not found")
worker.stop_job(job_id)
return {"job_id": job_id, "status": "stopped"}
@app.post("/api/jobs/{job_id}/skip")
async def skip_video(job_id: str):
"""Skip the currently processing video."""
job = db.get_job(job_id)
if not job:
raise HTTPException(status_code=404, detail="Job not found")
worker.skip_video(job_id)
return {"job_id": job_id, "action": "skip_sent"}
# ─────────────────────────────────────────────────────────────────────────────
# Videos & Artifacts
# ─────────────────────────────────────────────────────────────────────────────
@app.get("/api/jobs/{job_id}/videos")
async def get_videos(job_id: str):
videos = db.get_videos(job_id)
if not videos and not db.get_job(job_id):
raise HTTPException(status_code=404, detail="Job not found")
# Enrich with artifact statuses
result = []
for v in videos:
arts = db.get_artifacts_for_video(job_id, v["video_id"])
v["artifacts"] = arts
result.append(v)
return result
@app.get("/api/jobs/{job_id}/artifacts")
async def get_artifacts(job_id: str):
return db.get_all_artifacts(job_id)
# ─────────────────────────────────────────────────────────────────────────────
# Logging
# ─────────────────────────────────────────────────────────────────────────────
@app.get("/api/jobs/{job_id}/logs")
async def get_logs(job_id: str, limit: int = 500):
if not db.get_job(job_id):
raise HTTPException(status_code=404, detail="Job not found")
return db.get_logs(job_id, limit=limit)
@app.get("/api/jobs/{job_id}/logs/csv")
async def download_log_csv(job_id: str):
"""Download artifact log as CSV."""
if not db.get_job(job_id):
raise HTTPException(status_code=404, detail="Job not found")
csv_content = db.get_log_csv(job_id)
return StreamingResponse(
iter([csv_content]),
media_type="text/csv",
headers={"Content-Disposition": f'attachment; filename="job_{job_id[:8]}_log.csv"'},
)
# ─────────────────────────────────────────────────────────────────────────────
# Server-Sent Events (SSE) for real-time updates
# ─────────────────────────────────────────────────────────────────────────────
@app.get("/api/jobs/{job_id}/events")
async def job_events(job_id: str, request: Request):
"""
SSE endpoint. Client subscribes and receives real-time events
as the worker processes artifacts.
"""
if not db.get_job(job_id):
raise HTTPException(status_code=404, detail="Job not found")
queue = worker.emitter.subscribe(job_id)
async def event_generator():
try:
# Send initial state snapshot
job = db.get_job(job_id)
stats = db.get_job_stats(job_id)
if job:
job.update(stats)
yield _sse("job_status", {"job_id": job_id, "status": job["status"], "stats": stats})
while True:
if await request.is_disconnected():
break
# Poll the queue with a short timeout
event = await asyncio.get_event_loop().run_in_executor(
None, lambda: queue.get(timeout=1.0)
)
if event:
yield _sse(event["type"], event["data"])
else:
# Send a keepalive comment
yield ": keepalive\n\n"
# Check if job is complete
job = db.get_job(job_id)
if job and job["status"] in ("done", "done_with_errors", "stopped", "failed"):
# Give a moment for final events to flush
await asyncio.sleep(0.2)
# Drain remaining queue events
while True:
event = await asyncio.get_event_loop().run_in_executor(
None, lambda: queue.get(timeout=0.1)
)
if event:
yield _sse(event["type"], event["data"])
else:
break
yield _sse("stream_end", {"job_id": job_id})
break
except asyncio.CancelledError:
pass
finally:
worker.emitter.unsubscribe(job_id, queue)
return StreamingResponse(
event_generator(),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
"X-Accel-Buffering": "no",
"Connection": "keep-alive",
},
)
def _sse(event_type: str, data: dict) -> str:
"""Format a Server-Sent Event string."""
return f"event: {event_type}\ndata: {json.dumps(data)}\n\n"
# ─────────────────────────────────────────────────────────────────────────────
# File System Browsing Helper
# ─────────────────────────────────────────────────────────────────────────────
@app.get("/api/browse")
async def browse_folder(path: str = ""):
"""
Return directory listing for folder picking in the GUI.
Defaults to user home directory.
"""
if not path:
path = str(Path.home())
target = Path(path)
if not target.exists() or not target.is_dir():
# Fallback to home
target = Path.home()
try:
entries = []
for entry in sorted(target.iterdir()):
try:
entries.append({
"name": entry.name,
"path": str(entry.absolute()),
"is_dir": entry.is_dir(),
})
except PermissionError:
pass
# Add parent directory option
parent = str(target.parent.absolute())
return {
"current": str(target.absolute()),
"parent": parent if parent != str(target.absolute()) else None,
"entries": entries,
}
except PermissionError:
raise HTTPException(status_code=403, detail="Permission denied")
@app.post("/api/mkdir")
async def make_dir(req: BrowseFolderRequest):
"""Create a new directory."""
if not req.path:
raise HTTPException(status_code=400, detail="path is required")
try:
Path(req.path).mkdir(parents=True, exist_ok=True)
return {"created": req.path}
except Exception as e:
raise HTTPException(status_code=400, detail=str(e))
# ─────────────────────────────────────────────────────────────────────────────
# Config / Settings API
# ─────────────────────────────────────────────────────────────────────────────
@app.get("/api/config")
async def get_config():
"""Return non-sensitive defaults for the GUI."""
return {
"available_resolutions": config.AVAILABLE_RESOLUTIONS,
"artifact_types": config.ARTIFACT_TYPES,
"default_resolution": config.DEFAULT_VIDEO_RESOLUTION,
"delay_min": config.DELAY_MIN_SECONDS,
"delay_max": config.DELAY_MAX_SECONDS,
"max_retries": config.MAX_RETRIES,
"default_model": config.OPENROUTER_MODEL,
"summary_prompt": config.SUMMARY_SYSTEM_PROMPT,
"openrouter_base_url": config.OPENROUTER_BASE_URL,
}
# ─────────────────────────────────────────────────────────────────────────────
# Health Check
# ─────────────────────────────────────────────────────────────────────────────
@app.get("/api/health")
async def health():
return {"status": "ok", "version": "1.0.0"}
# ─────────────────────────────────────────────────────────────────────────────
# Entry Point
# ─────────────────────────────────────────────────────────────────────────────
if __name__ == "__main__":
print(f"\n{'='*60}")
print(" YouTube SIPOC Tool")
print(f" Web GUI: http://{config.API_HOST}:{config.API_PORT}")
print(f" API: http://{config.API_HOST}:{config.API_PORT}/api")
print(f"{'='*60}\n")
uvicorn.run(
"api_server:app",
host=config.API_HOST,
port=config.API_PORT,
reload=False,
log_level="info",
)

64
backend/batchtube-server.spec Normal file
View File

@@ -0,0 +1,64 @@
# batchtube-server.spec
# PyInstaller spec to bundle the BatchTube FastAPI backend
# Run: pyinstaller batchtube-server.spec
import sys
from pathlib import Path
block_cipher = None
a = Analysis(
['api_server.py'],
pathex=[str(Path('.'). resolve())],
binaries=[],
datas=[
('web', 'web'),
('youtube_sipoc.db', '.'),
],
hiddenimports=[
'uvicorn.logging',
'uvicorn.loops',
'uvicorn.loops.auto',
'uvicorn.protocols',
'uvicorn.protocols.http',
'uvicorn.protocols.http.auto',
'uvicorn.protocols.websockets',
'uvicorn.protocols.websockets.auto',
'uvicorn.lifespan',
'uvicorn.lifespan.on',
'fastapi',
'yt_dlp',
],
hookspath=[],
hooksconfig={},
runtime_hooks=[],
excludes=[],
win_no_prefer_redirects=False,
win_private_assemblies=False,
cipher=block_cipher,
noarchive=False,
)
pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
exe = EXE(
pyz,
a.scripts,
a.binaries,
a.zipfiles,
a.datas,
[],
name='batchtube-server',
debug=False,
bootloader_ignore_signals=False,
strip=False,
upx=True,
upx_exclude=[],
runtime_tmpdir=None,
console=False, # No console window
disable_windowed_traceback=False,
argv_emulation=False,
target_arch=None,
codesign_identity=None,
entitlements_file=None,
)

38
backend/config.py Normal file
View File

@@ -0,0 +1,38 @@
"""
config.py - Default configuration for YouTube SIPOC Tool
"""
import os
# ── API Settings ──────────────────────────────────────────────────────────────
OPENROUTER_API_KEY = os.environ.get("OPENROUTER_API_KEY", "")
OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1"
OPENROUTER_MODEL = os.environ.get("OPENROUTER_MODEL", "openai/gpt-4o-mini")
# ── Server Settings ───────────────────────────────────────────────────────────
API_HOST = "127.0.0.1"
API_PORT = 8765
# ── Download / Bot Avoidance ──────────────────────────────────────────────────
DELAY_MIN_SECONDS = 2.0 # minimum random pause between artifacts
DELAY_MAX_SECONDS = 7.0 # maximum random pause between artifacts
MAX_RETRIES = 3 # max retry attempts per failed artifact
# ── Video Download ────────────────────────────────────────────────────────────
DEFAULT_VIDEO_RESOLUTION = "1080" # used if not specified by user
AVAILABLE_RESOLUTIONS = ["best", "4320", "2160", "1440", "1080", "720", "480", "360", "240", "144"]
# ── Summary ───────────────────────────────────────────────────────────────────
SUMMARY_SYSTEM_PROMPT = (
"You are a helpful assistant that creates concise, well-structured summaries of video transcripts. "
"Include key points, main topics, and any actionable insights. Format with bullet points where appropriate."
)
SUMMARY_MAX_TOKENS = 1024
# ── File Naming ───────────────────────────────────────────────────────────────
MAX_TITLE_LENGTH = 80 # characters - truncate long titles in filenames
# ── Database ──────────────────────────────────────────────────────────────────
DB_PATH = os.path.join(os.path.dirname(__file__), "youtube_sipoc.db")
# ── Artifact Types ────────────────────────────────────────────────────────────
ARTIFACT_TYPES = ["title", "stats", "transcript", "summary", "thumbnail", "mp3", "mp4"]

371
backend/database.py Normal file
View File

@@ -0,0 +1,371 @@
"""
database.py - SQLite state management for YouTube SIPOC Tool
Handles jobs, videos, artifacts, and logs with resume capability.
"""
import sqlite3
import json
import uuid
from datetime import datetime
from contextlib import contextmanager
from typing import Optional, List, Dict, Any
import config
@contextmanager
def get_conn():
"""Context manager providing a thread-safe SQLite connection."""
conn = sqlite3.connect(config.DB_PATH, check_same_thread=False)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA journal_mode=WAL") # WAL mode for concurrent reads
conn.execute("PRAGMA foreign_keys=ON")
try:
yield conn
conn.commit()
except Exception:
conn.rollback()
raise
finally:
conn.close()
def init_db():
"""Create all tables if they don't exist."""
with get_conn() as conn:
conn.executescript("""
CREATE TABLE IF NOT EXISTS jobs (
job_id TEXT PRIMARY KEY,
status TEXT NOT NULL DEFAULT 'pending',
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
output_folder TEXT NOT NULL,
selections TEXT NOT NULL,
resolution TEXT NOT NULL DEFAULT 'best',
delay_min REAL NOT NULL DEFAULT 2.0,
delay_max REAL NOT NULL DEFAULT 7.0,
input_type TEXT NOT NULL DEFAULT 'url',
input_value TEXT NOT NULL DEFAULT '',
summary_prompt TEXT NOT NULL DEFAULT '',
openrouter_model TEXT NOT NULL DEFAULT '',
total_videos INTEGER NOT NULL DEFAULT 0,
done_videos INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE IF NOT EXISTS videos (
id INTEGER PRIMARY KEY AUTOINCREMENT,
job_id TEXT NOT NULL REFERENCES jobs(job_id),
video_id TEXT NOT NULL,
title TEXT NOT NULL DEFAULT '',
url TEXT NOT NULL,
position INTEGER NOT NULL DEFAULT 0,
status TEXT NOT NULL DEFAULT 'pending',
UNIQUE(job_id, video_id)
);
CREATE TABLE IF NOT EXISTS artifacts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
job_id TEXT NOT NULL REFERENCES jobs(job_id),
video_id TEXT NOT NULL,
artifact_type TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending',
file_path TEXT,
error_msg TEXT,
attempts INTEGER NOT NULL DEFAULT 0,
started_at TEXT,
finished_at TEXT,
UNIQUE(job_id, video_id, artifact_type)
);
CREATE TABLE IF NOT EXISTS logs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
job_id TEXT NOT NULL REFERENCES jobs(job_id),
video_id TEXT,
artifact_type TEXT,
level TEXT NOT NULL DEFAULT 'INFO',
message TEXT NOT NULL,
timestamp TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_artifacts_job ON artifacts(job_id);
CREATE INDEX IF NOT EXISTS idx_videos_job ON videos(job_id);
CREATE INDEX IF NOT EXISTS idx_logs_job ON logs(job_id);
""")
# ── Job Operations ────────────────────────────────────────────────────────────
def create_job(
output_folder: str,
selections: List[str],
resolution: str,
delay_min: float,
delay_max: float,
input_type: str,
input_value: str,
summary_prompt: str,
openrouter_model: str,
) -> str:
"""Create a new job and return its job_id."""
job_id = str(uuid.uuid4())
now = datetime.utcnow().isoformat()
with get_conn() as conn:
conn.execute(
"""INSERT INTO jobs
(job_id, status, created_at, updated_at, output_folder, selections,
resolution, delay_min, delay_max, input_type, input_value,
summary_prompt, openrouter_model)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)""",
(
job_id, "pending", now, now, output_folder,
json.dumps(selections), resolution,
delay_min, delay_max, input_type, input_value,
summary_prompt, openrouter_model,
),
)
return job_id
def get_job(job_id: str) -> Optional[Dict]:
with get_conn() as conn:
row = conn.execute("SELECT * FROM jobs WHERE job_id=?", (job_id,)).fetchone()
if row:
d = dict(row)
d["selections"] = json.loads(d["selections"])
return d
return None
def list_jobs() -> List[Dict]:
with get_conn() as conn:
rows = conn.execute(
"SELECT * FROM jobs ORDER BY created_at DESC"
).fetchall()
result = []
for row in rows:
d = dict(row)
d["selections"] = json.loads(d["selections"])
result.append(d)
return result
def update_job_status(job_id: str, status: str):
now = datetime.utcnow().isoformat()
with get_conn() as conn:
conn.execute(
"UPDATE jobs SET status=?, updated_at=? WHERE job_id=?",
(status, now, job_id),
)
def update_job_progress(job_id: str, total: int, done: int):
now = datetime.utcnow().isoformat()
with get_conn() as conn:
conn.execute(
"UPDATE jobs SET total_videos=?, done_videos=?, updated_at=? WHERE job_id=?",
(total, done, now, job_id),
)
def set_job_field(job_id: str, field: str, value: Any):
"""Update a single field on the jobs table (safe whitelist)."""
allowed = {"status", "total_videos", "done_videos", "updated_at"}
if field not in allowed:
raise ValueError(f"Field '{field}' not allowed")
now = datetime.utcnow().isoformat()
with get_conn() as conn:
conn.execute(f"UPDATE jobs SET {field}=?, updated_at=? WHERE job_id=?", (value, now, job_id))
# ── Video Operations ──────────────────────────────────────────────────────────
def upsert_video(job_id: str, video_id: str, url: str, title: str = "", position: int = 0):
with get_conn() as conn:
conn.execute(
"""INSERT INTO videos (job_id, video_id, title, url, position, status)
VALUES (?,?,?,?,?,'pending')
ON CONFLICT(job_id, video_id) DO UPDATE SET
title=excluded.title,
position=excluded.position""",
(job_id, video_id, title, url, position),
)
def update_video_title(job_id: str, video_id: str, title: str):
with get_conn() as conn:
conn.execute(
"UPDATE videos SET title=? WHERE job_id=? AND video_id=?",
(title, job_id, video_id),
)
def update_video_status(job_id: str, video_id: str, status: str):
with get_conn() as conn:
conn.execute(
"UPDATE videos SET status=? WHERE job_id=? AND video_id=?",
(status, job_id, video_id),
)
def get_videos(job_id: str) -> List[Dict]:
with get_conn() as conn:
rows = conn.execute(
"SELECT * FROM videos WHERE job_id=? ORDER BY position ASC",
(job_id,),
).fetchall()
return [dict(r) for r in rows]
# ── Artifact Operations ───────────────────────────────────────────────────────
def upsert_artifact(job_id: str, video_id: str, artifact_type: str):
"""Create artifact record if not exists (skip if already done/skipped)."""
with get_conn() as conn:
conn.execute(
"""INSERT INTO artifacts (job_id, video_id, artifact_type, status)
VALUES (?,?,?,'pending')
ON CONFLICT(job_id, video_id, artifact_type) DO NOTHING""",
(job_id, video_id, artifact_type),
)
def get_artifact(job_id: str, video_id: str, artifact_type: str) -> Optional[Dict]:
with get_conn() as conn:
row = conn.execute(
"SELECT * FROM artifacts WHERE job_id=? AND video_id=? AND artifact_type=?",
(job_id, video_id, artifact_type),
).fetchone()
return dict(row) if row else None
def artifact_is_done(job_id: str, video_id: str, artifact_type: str) -> bool:
"""Return True if artifact already succeeded or was skipped."""
art = get_artifact(job_id, video_id, artifact_type)
return art is not None and art["status"] in ("done", "skipped")
def update_artifact_status(
job_id: str,
video_id: str,
artifact_type: str,
status: str,
file_path: Optional[str] = None,
error_msg: Optional[str] = None,
):
now = datetime.utcnow().isoformat()
with get_conn() as conn:
conn.execute(
"""UPDATE artifacts
SET status=?, file_path=?, error_msg=?, finished_at=?,
attempts = attempts + CASE WHEN ? IN ('done','failed','skipped') THEN 1 ELSE 0 END
WHERE job_id=? AND video_id=? AND artifact_type=?""",
(status, file_path, error_msg, now, status, job_id, video_id, artifact_type),
)
def mark_artifact_started(job_id: str, video_id: str, artifact_type: str):
now = datetime.utcnow().isoformat()
with get_conn() as conn:
conn.execute(
"UPDATE artifacts SET status='running', started_at=? WHERE job_id=? AND video_id=? AND artifact_type=?",
(now, job_id, video_id, artifact_type),
)
def get_artifacts_for_video(job_id: str, video_id: str) -> List[Dict]:
with get_conn() as conn:
rows = conn.execute(
"SELECT * FROM artifacts WHERE job_id=? AND video_id=? ORDER BY id ASC",
(job_id, video_id),
).fetchall()
return [dict(r) for r in rows]
def get_all_artifacts(job_id: str) -> List[Dict]:
with get_conn() as conn:
rows = conn.execute(
"""SELECT a.*, v.title as video_title
FROM artifacts a
JOIN videos v ON a.job_id=v.job_id AND a.video_id=v.video_id
WHERE a.job_id=?
ORDER BY v.position ASC, a.artifact_type ASC""",
(job_id,),
).fetchall()
return [dict(r) for r in rows]
# ── Logging ───────────────────────────────────────────────────────────────────
def add_log(
job_id: str,
message: str,
level: str = "INFO",
video_id: Optional[str] = None,
artifact_type: Optional[str] = None,
):
now = datetime.utcnow().isoformat()
with get_conn() as conn:
conn.execute(
"""INSERT INTO logs (job_id, video_id, artifact_type, level, message, timestamp)
VALUES (?,?,?,?,?,?)""",
(job_id, video_id, artifact_type, level, message, now),
)
def get_logs(job_id: str, limit: int = 500) -> List[Dict]:
with get_conn() as conn:
rows = conn.execute(
"SELECT * FROM logs WHERE job_id=? ORDER BY id DESC LIMIT ?",
(job_id, limit),
).fetchall()
return [dict(r) for r in reversed(rows)]
def get_log_csv(job_id: str) -> str:
"""Export artifacts as CSV string for download."""
import io, csv
artifacts = get_all_artifacts(job_id)
output = io.StringIO()
writer = csv.writer(output)
writer.writerow(["video_id", "video_title", "artifact_type", "status", "file_path", "error_msg", "attempts", "started_at", "finished_at"])
for a in artifacts:
writer.writerow([
a.get("video_id", ""),
a.get("video_title", ""),
a.get("artifact_type", ""),
a.get("status", ""),
a.get("file_path", ""),
a.get("error_msg", ""),
a.get("attempts", 0),
a.get("started_at", ""),
a.get("finished_at", ""),
])
return output.getvalue()
def get_job_stats(job_id: str) -> Dict:
"""Return aggregate counts for a job."""
with get_conn() as conn:
total_videos = conn.execute(
"SELECT COUNT(*) FROM videos WHERE job_id=?", (job_id,)
).fetchone()[0]
done_videos = conn.execute(
"SELECT COUNT(*) FROM videos WHERE job_id=? AND status='done'", (job_id,)
).fetchone()[0]
artifact_counts = conn.execute(
"""SELECT status, COUNT(*) as cnt FROM artifacts WHERE job_id=? GROUP BY status""",
(job_id,),
).fetchall()
counts = {row["status"]: row["cnt"] for row in artifact_counts}
return {
"total_videos": total_videos,
"done_videos": done_videos,
"artifact_done": counts.get("done", 0),
"artifact_failed": counts.get("failed", 0),
"artifact_pending": counts.get("pending", 0),
"artifact_running": counts.get("running", 0),
"artifact_skipped": counts.get("skipped", 0),
}
# Initialize on import
init_db()

749
backend/extractors.py Normal file
View File

@@ -0,0 +1,749 @@
"""
extractors.py - Per-artifact extraction logic for YouTube SIPOC Tool
Uses yt-dlp, youtube-transcript-api, and OpenRouter LLM.
"""
import os
import re
import json
import time
import logging
import requests
from pathlib import Path
from typing import Optional, Dict, Tuple, List
import yt_dlp
from youtube_transcript_api import YouTubeTranscriptApi
try:
from youtube_transcript_api import TranscriptsDisabled, NoTranscriptFound
except ImportError:
# v1.x moved/renamed exceptions
TranscriptsDisabled = Exception
NoTranscriptFound = Exception
from openai import OpenAI
import config
logger = logging.getLogger("extractors")
# ─────────────────────────────────────────────────────────────────────────────
# Utilities
# ─────────────────────────────────────────────────────────────────────────────
def sanitize_filename(name: str, max_length: int = config.MAX_TITLE_LENGTH) -> str:
"""Make a string safe to use in a file name."""
name = re.sub(r'[\\/:*?"<>|]', "_", name)
name = re.sub(r'\s+', "_", name.strip())
name = re.sub(r'_+', "_", name)
return name[:max_length].rstrip("._")
def make_base_name(title: str, video_id: str) -> str:
"""Return the standard base filename (no extension)."""
safe_title = sanitize_filename(title) if title else "untitled"
return f"{safe_title}__{video_id}"
def ensure_dir(path: str):
Path(path).mkdir(parents=True, exist_ok=True)
def video_url(video_id: str) -> str:
return f"https://www.youtube.com/watch?v={video_id}"
# ─────────────────────────────────────────────────────────────────────────────
# Input Resolution: URL → list of video IDs
# ─────────────────────────────────────────────────────────────────────────────
def resolve_input(input_type: str, input_value: str) -> List[Dict]:
"""
Resolve any input type to a list of dicts: {video_id, url, title, position}
Supported input_types: url, playlist, csv, handle
"""
if input_type == "url":
return _resolve_single_url(input_value)
elif input_type == "playlist":
return _resolve_playlist(input_value)
elif input_type == "csv":
return _resolve_csv(input_value)
elif input_type == "handle":
return _resolve_handle(input_value)
else:
raise ValueError(f"Unknown input_type: {input_type}")
def _extract_video_id(url: str) -> Optional[str]:
"""Extract YouTube video ID from various URL formats."""
patterns = [
r'(?:v=|youtu\.be/|embed/|shorts/)([A-Za-z0-9_-]{11})',
]
for pat in patterns:
m = re.search(pat, url)
if m:
return m.group(1)
return None
def _resolve_single_url(url: str) -> List[Dict]:
"""Handle single YouTube URL (video or short)."""
url = url.strip()
vid = _extract_video_id(url)
if vid:
return [{"video_id": vid, "url": video_url(vid), "title": "", "position": 0}]
# Fall through to yt-dlp for unusual URLs
return _resolve_with_ytdlp(url)
def _resolve_playlist(playlist_url: str) -> List[Dict]:
return _resolve_with_ytdlp(playlist_url.strip())
def _resolve_handle(handle: str) -> List[Dict]:
"""Resolve @handle or channel URL to video list."""
handle = handle.strip()
if not handle.startswith("http"):
if handle.startswith("@"):
url = f"https://www.youtube.com/{handle}/videos"
else:
url = f"https://www.youtube.com/@{handle}/videos"
else:
url = handle
return _resolve_with_ytdlp(url)
def _resolve_csv(csv_content: str) -> List[Dict]:
"""Parse CSV content: each line may be a URL or video_id."""
lines = []
for line in csv_content.splitlines():
for part in line.split(","):
part = part.strip().strip('"').strip("'")
if part:
lines.append(part)
results = []
for i, entry in enumerate(lines):
if entry.startswith("http"):
items = _resolve_single_url(entry)
for item in items:
item["position"] = len(results)
results.append(item)
else:
# Treat as raw video_id
vid = entry[:11]
results.append({"video_id": vid, "url": video_url(vid), "title": "", "position": len(results)})
return results
def _resolve_with_ytdlp(url: str) -> List[Dict]:
"""Use yt-dlp to extract video list from any URL (playlist, channel, etc.)."""
captured_errors = []
class ErrorCollector:
"""Custom yt-dlp logger to capture warning/error messages."""
def debug(self, msg): pass
def info(self, msg): pass
def warning(self, msg):
captured_errors.append(f"WARNING: {msg}")
def error(self, msg):
captured_errors.append(f"ERROR: {msg}")
ydl_opts = {
"quiet": True,
"extract_flat": True,
"skip_download": True,
"ignoreerrors": False, # Don't swallow errors silently
"logger": ErrorCollector(),
}
results = []
info = None
try:
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
info = ydl.extract_info(url, download=False)
except Exception as e:
captured_errors.append(str(e))
if not info:
# Build a meaningful error message
err_summary = "; ".join(captured_errors) if captured_errors else "No information returned by yt-dlp."
# Surface private/login-required as a clear user-facing error
lower = err_summary.lower()
if "private" in lower:
raise ValueError(
f"This playlist or channel is PRIVATE. "
f"YouTube requires you to be logged in to access it. "
f"To process private content, add your YouTube cookies file path in Settings. "
f"Details: {err_summary}"
)
elif "sign in" in lower or "login" in lower or "confirm your age" in lower:
raise ValueError(
f"YouTube requires authentication (sign-in or age verification). "
f"Add your YouTube cookies file path in Settings. "
f"Details: {err_summary}"
)
elif "not exist" in lower or "removed" in lower or "unavailable" in lower:
raise ValueError(f"Content not available or removed. Details: {err_summary}")
else:
raise ValueError(f"Could not resolve URL '{url}'. Details: {err_summary}")
entries = info.get("entries") or [info]
for i, entry in enumerate(entries):
if not entry:
continue
vid = entry.get("id") or entry.get("video_id")
if not vid:
continue
title = entry.get("title", "")
results.append({
"video_id": vid,
"url": video_url(vid),
"title": title,
"position": i,
})
if not results:
raise ValueError(
f"URL resolved successfully but contained 0 videos. "
f"The playlist/channel may be empty. URL: {url}"
)
return results
# ─────────────────────────────────────────────────────────────────────────────
# Metadata Fetch
# ─────────────────────────────────────────────────────────────────────────────
def fetch_metadata(video_id: str) -> Dict:
"""Fetch full video metadata using yt-dlp."""
ydl_opts = {
"quiet": True,
"skip_download": True,
"ignoreerrors": True,
}
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
info = ydl.extract_info(video_url(video_id), download=False)
return info or {}
# ─────────────────────────────────────────────────────────────────────────────
# Artifact: Title
# ─────────────────────────────────────────────────────────────────────────────
def extract_title(video_id: str, meta: Dict, output_folder: str) -> Tuple[str, str]:
"""
Save title as a .txt file.
Returns (file_path, title_string)
"""
title = meta.get("title", "")
base = make_base_name(title, video_id)
file_path = os.path.join(output_folder, f"{base}__title.txt")
ensure_dir(output_folder)
with open(file_path, "w", encoding="utf-8") as f:
f.write(f"Title: {title}\n")
f.write(f"Video ID: {video_id}\n")
f.write(f"URL: {video_url(video_id)}\n")
return file_path, title
# ─────────────────────────────────────────────────────────────────────────────
# Artifact: Stats
# ─────────────────────────────────────────────────────────────────────────────
def extract_stats(video_id: str, meta: Dict, output_folder: str) -> str:
"""Save video stats + full metadata as a human-readable .txt file."""
title = meta.get("title", "")
base = make_base_name(title, video_id)
file_path = os.path.join(output_folder, f"{base}__stats.txt")
ensure_dir(output_folder)
# Format helpers
def fmt_num(n) -> str:
if n is None:
return "N/A"
try:
return f"{int(n):,}"
except Exception:
return str(n)
def fmt_duration(secs) -> str:
if not secs:
return "N/A"
try:
secs = int(secs)
h, rem = divmod(secs, 3600)
m, s = divmod(rem, 60)
return f"{h}h {m}m {s}s" if h else f"{m}m {s}s"
except Exception:
return str(secs)
def fmt_date(d) -> str:
if not d:
return "N/A"
try:
d = str(d)
return f"{d[:4]}-{d[4:6]}-{d[6:8]}"
except Exception:
return str(d)
tags = meta.get("tags") or []
categories = meta.get("categories") or []
chapters = meta.get("chapters") or []
description = meta.get("description", "") or ""
formats = meta.get("formats") or []
subtitles = meta.get("subtitles") or {}
automatic_captions = meta.get("automatic_captions") or {}
with open(file_path, "w", encoding="utf-8") as f:
f.write("=" * 70 + "\n")
f.write(f" VIDEO STATS & METADATA\n")
f.write("=" * 70 + "\n\n")
f.write(f"Title : {meta.get('title', '')}\n")
f.write(f"Video ID : {video_id}\n")
f.write(f"URL : {video_url(video_id)}\n")
f.write(f"Webpage URL : {meta.get('webpage_url', '')}\n\n")
f.write("── Channel ──────────────────────────────────────────────────\n")
f.write(f"Channel : {meta.get('channel', meta.get('uploader', ''))}\n")
f.write(f"Channel ID : {meta.get('channel_id', '')}\n")
f.write(f"Channel URL : {meta.get('channel_url', meta.get('uploader_url', ''))}\n")
f.write(f"Subscribers : {fmt_num(meta.get('channel_follower_count'))}\n\n")
f.write("── Content ──────────────────────────────────────────────────\n")
f.write(f"Upload Date : {fmt_date(meta.get('upload_date'))}\n")
f.write(f"Duration : {fmt_duration(meta.get('duration'))} ({meta.get('duration_string', '')})\n")
f.write(f"Age Limit : {meta.get('age_limit', 0)}\n")
f.write(f"Is Live : {meta.get('is_live', False)}\n")
f.write(f"Was Live : {meta.get('was_live', False)}\n")
f.write(f"Live Status : {meta.get('live_status', '')}\n\n")
f.write("── Engagement ───────────────────────────────────────────────\n")
f.write(f"Views : {fmt_num(meta.get('view_count'))}\n")
f.write(f"Likes : {fmt_num(meta.get('like_count'))}\n")
f.write(f"Comments : {fmt_num(meta.get('comment_count'))}\n")
f.write(f"Average Rating : {meta.get('average_rating', 'N/A')}\n\n")
f.write("── Media Info ───────────────────────────────────────────────\n")
f.write(f"Thumbnail URL : {meta.get('thumbnail', '')}\n")
# Best format info
best_fmt = next((fmt for fmt in reversed(formats) if fmt.get("vcodec") not in (None, "none")), None)
if best_fmt:
f.write(f"Best Resolution : {best_fmt.get('width', '?')}x{best_fmt.get('height', '?')} "
f"@ {best_fmt.get('fps', '?')}fps\n")
f.write(f"Best Video Codec : {best_fmt.get('vcodec', 'N/A')}\n")
f.write(f"Best Audio Codec : {best_fmt.get('acodec', 'N/A')}\n")
f.write(f"Available Formats: {len(formats)}\n\n")
f.write("── Classification ───────────────────────────────────────────\n")
f.write(f"Categories : {', '.join(categories) if categories else 'N/A'}\n")
if tags:
tag_str = ", ".join(tags[:30])
if len(tags) > 30:
tag_str += f" ... (+{len(tags)-30} more)"
f.write(f"Tags : {tag_str}\n")
else:
f.write(f"Tags : N/A\n")
f.write("\n")
# Subtitles / transcript languages
all_sub_langs = sorted(set(list(subtitles.keys()) + list(automatic_captions.keys())))
if all_sub_langs:
f.write("── Subtitles / Captions ─────────────────────────────────────\n")
f.write(f"Languages ({len(all_sub_langs)}): {', '.join(all_sub_langs[:20])}")
if len(all_sub_langs) > 20:
f.write(f" ... (+{len(all_sub_langs)-20} more)")
f.write("\n\n")
# Chapters
if chapters:
f.write("── Chapters ─────────────────────────────────────────────────\n")
for ch in chapters:
start = ch.get("start_time", 0)
m, s = divmod(int(start), 60)
h, m = divmod(m, 60)
ts = f"{h:02d}:{m:02d}:{s:02d}" if h else f"{m:02d}:{s:02d}"
f.write(f" [{ts}] {ch.get('title', '')}\n")
f.write("\n")
# Description
if description:
f.write("── Description ──────────────────────────────────────────────\n")
# Limit description to 3000 chars to keep the file manageable
if len(description) > 3000:
f.write(description[:3000] + "\n... [truncated]\n")
else:
f.write(description + "\n")
f.write("\n")
f.write("── Extraction Info ──────────────────────────────────────────\n")
f.write(f"Extracted At : {time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime())}\n")
f.write(f"Extractor : {meta.get('extractor', 'youtube')}\n")
f.write(f"Extractor Key : {meta.get('extractor_key', 'Youtube')}\n")
f.write("=" * 70 + "\n")
return file_path
# ─────────────────────────────────────────────────────────────────────────────
# Artifact: Transcript
# ─────────────────────────────────────────────────────────────────────────────
def _get_snippet_text(snippet) -> str:
"""Safely extract text from a transcript snippet (dict or object)."""
if isinstance(snippet, dict):
return snippet.get("text", "")
return getattr(snippet, "text", "") or ""
def _get_snippet_start(snippet) -> float:
"""Safely extract start time from a transcript snippet (dict or object)."""
if isinstance(snippet, dict):
return snippet.get("start", 0.0)
return getattr(snippet, "start", 0.0) or 0.0
def _fetch_transcript(video_id: str):
"""
Fetch transcript snippets using youtube-transcript-api v1.x.
v1.x API: YouTubeTranscriptApi().list(video_id) and .fetch(video_id, languages=[...])
Returns (FetchedTranscript iterable, errors_list) — iterable may be None on failure.
"""
errors = []
api = YouTubeTranscriptApi()
# Fast path: try direct fetch with preferred languages
for lang_group in [["en", "en-US", "en-GB"], ["en-US"], ["en-GB"]]:
try:
fetched = api.fetch(video_id, languages=lang_group)
return fetched, errors
except Exception:
pass
# Slower path: list all available transcripts and pick the best
try:
transcript_list = api.list(video_id)
except Exception as e:
errors.append(f"list error: {e}")
# Last-ditch: fetch with no language preference
try:
fetched = api.fetch(video_id)
return fetched, errors
except Exception as e2:
errors.append(f"fetch error: {e2}")
return None, errors
# Prefer any manual transcript
all_transcripts = list(transcript_list)
manual = [t for t in all_transcripts if not getattr(t, "is_generated", True)]
auto = [t for t in all_transcripts if getattr(t, "is_generated", False)]
for t in (manual + auto):
try:
# Try fetching via the transcript object's own fetch method
fetched = t.fetch()
return fetched, errors
except Exception as e:
errors.append(f"t.fetch() error for {getattr(t, 'language_code','?')}: {e}")
continue
errors.append("No fetchable transcripts found")
return None, errors
def extract_transcript(video_id: str, meta: Dict, output_folder: str) -> str:
"""
Download transcript using youtube-transcript-api.
Falls back to auto-generated captions. Saves as .txt with timestamps.
"""
title = meta.get("title", "")
base = make_base_name(title, video_id)
file_path = os.path.join(output_folder, f"{base}__transcript.txt")
ensure_dir(output_folder)
transcript, errors = _fetch_transcript(video_id)
if not transcript:
raise RuntimeError(f"Could not retrieve transcript for {video_id}. Errors: {'; '.join(errors)}")
# Write formatted transcript
with open(file_path, "w", encoding="utf-8") as f:
f.write(f"Transcript: {meta.get('title', video_id)}\n")
f.write(f"Video ID: {video_id}\n")
f.write(f"URL: {video_url(video_id)}\n")
f.write("=" * 60 + "\n\n")
for entry in transcript:
start = _get_snippet_start(entry)
text = _get_snippet_text(entry).replace("\n", " ")
minutes, seconds = divmod(int(start), 60)
hours, minutes = divmod(minutes, 60)
ts = f"[{hours:02d}:{minutes:02d}:{seconds:02d}]" if hours else f"[{minutes:02d}:{seconds:02d}]"
f.write(f"{ts} {text}\n")
return file_path
def get_transcript_text(video_id: str) -> str:
"""Get plain transcript text for summarization."""
try:
transcript, _ = _fetch_transcript(video_id)
if transcript:
return " ".join(_get_snippet_text(e) for e in transcript)
except Exception:
pass
return ""
# ─────────────────────────────────────────────────────────────────────────────
# Artifact: Summary
# ─────────────────────────────────────────────────────────────────────────────
def extract_summary(
video_id: str,
meta: Dict,
output_folder: str,
api_key: str,
model: str,
system_prompt: str,
) -> str:
"""Generate a summary using OpenRouter LLM and save as .txt."""
if not api_key:
raise RuntimeError("OpenRouter API key is not configured. Set it in the Settings panel.")
title = meta.get("title", "")
base = make_base_name(title, video_id)
file_path = os.path.join(output_folder, f"{base}__summary.txt")
ensure_dir(output_folder)
# Get transcript text for summarization
transcript_text = get_transcript_text(video_id)
if not transcript_text:
transcript_text = meta.get("description", "")
if not transcript_text:
raise RuntimeError(f"No transcript or description available for summarization of {video_id}")
# Truncate very long transcripts to ~12000 chars (token budget)
max_chars = 12000
if len(transcript_text) > max_chars:
transcript_text = transcript_text[:max_chars] + "\n\n[Transcript truncated for length]"
client = OpenAI(
api_key=api_key,
base_url=config.OPENROUTER_BASE_URL,
)
prompt = (
f"Title: {title}\n"
f"Video ID: {video_id}\n"
f"URL: {video_url(video_id)}\n\n"
f"Transcript:\n{transcript_text}"
)
response = client.chat.completions.create(
model=model or config.OPENROUTER_MODEL,
messages=[
{"role": "system", "content": system_prompt or config.SUMMARY_SYSTEM_PROMPT},
{"role": "user", "content": prompt},
],
max_tokens=config.SUMMARY_MAX_TOKENS,
)
summary_text = response.choices[0].message.content or ""
with open(file_path, "w", encoding="utf-8") as f:
f.write(f"Summary: {title}\n")
f.write(f"Video ID: {video_id}\n")
f.write(f"URL: {video_url(video_id)}\n")
f.write("=" * 60 + "\n\n")
f.write(summary_text)
f.write("\n")
return file_path
# ─────────────────────────────────────────────────────────────────────────────
# Artifact: Thumbnail
# ─────────────────────────────────────────────────────────────────────────────
def extract_thumbnail(video_id: str, meta: Dict, output_folder: str) -> str:
"""Download the highest-resolution thumbnail available."""
title = meta.get("title", "")
base = make_base_name(title, video_id)
ensure_dir(output_folder)
# Build priority list of thumbnail URLs
thumbnails = meta.get("thumbnails", [])
# Sort by resolution (width * height), highest first
thumbnails_sorted = sorted(
[t for t in thumbnails if t.get("url")],
key=lambda t: (t.get("width", 0) or 0) * (t.get("height", 0) or 0),
reverse=True,
)
# Fallback URLs
fallback_urls = [
f"https://img.youtube.com/vi/{video_id}/maxresdefault.jpg",
f"https://img.youtube.com/vi/{video_id}/sddefault.jpg",
f"https://img.youtube.com/vi/{video_id}/hqdefault.jpg",
f"https://img.youtube.com/vi/{video_id}/mqdefault.jpg",
f"https://img.youtube.com/vi/{video_id}/default.jpg",
]
candidate_urls = [t["url"] for t in thumbnails_sorted] + fallback_urls
for url in candidate_urls:
try:
response = requests.get(url, timeout=15)
if response.status_code == 200 and len(response.content) > 1000:
# Determine extension from content-type or URL
ct = response.headers.get("content-type", "")
if "webp" in ct or url.endswith(".webp"):
ext = "webp"
elif "png" in ct or url.endswith(".png"):
ext = "png"
else:
ext = "jpg"
file_path = os.path.join(output_folder, f"{base}__thumbnail.{ext}")
with open(file_path, "wb") as f:
f.write(response.content)
return file_path
except Exception:
continue
raise RuntimeError(f"Could not download thumbnail for {video_id}")
# ─────────────────────────────────────────────────────────────────────────────
# Artifact: MP3
# ─────────────────────────────────────────────────────────────────────────────
def extract_mp3(video_id: str, meta: Dict, output_folder: str) -> str:
"""Download audio and convert to MP3 using yt-dlp + ffmpeg."""
title = meta.get("title", "")
base = make_base_name(title, video_id)
ensure_dir(output_folder)
ffmpeg_dir = _find_ffmpeg()
# yt-dlp will append extension itself; we use outtmpl without extension
out_template = os.path.join(output_folder, f"{base}__audio")
expected_path = os.path.join(output_folder, f"{base}__audio.mp3")
ydl_opts = {
"quiet": True,
"no_warnings": True,
"format": "bestaudio/best",
"outtmpl": out_template,
"postprocessors": [
{
"key": "FFmpegExtractAudio",
"preferredcodec": "mp3",
"preferredquality": "192",
}
],
"ignoreerrors": False,
}
if ffmpeg_dir:
ydl_opts["ffmpeg_location"] = ffmpeg_dir
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
ydl.download([video_url(video_id)])
# Rename to standard naming convention
if os.path.exists(expected_path):
final_path = os.path.join(output_folder, f"{base}__mp3.mp3")
if not os.path.exists(final_path):
os.rename(expected_path, final_path)
return final_path
# Search for any mp3 file created in this run
for fname in os.listdir(output_folder):
if fname.startswith(base) and fname.endswith(".mp3"):
final_path = os.path.join(output_folder, f"{base}__mp3.mp3")
src = os.path.join(output_folder, fname)
if src != final_path and not os.path.exists(final_path):
os.rename(src, final_path)
return final_path
raise RuntimeError(f"MP3 file not found after download for {video_id}")
# ─────────────────────────────────────────────────────────────────────────────
# Artifact: MP4
# ─────────────────────────────────────────────────────────────────────────────
def _find_ffmpeg() -> Optional[str]:
"""Find ffmpeg binary location."""
# Common Windows locations
candidates = [
r"C:\ProgramData\chocolatey\bin",
r"C:\ffmpeg\bin",
r"C:\Program Files\ffmpeg\bin",
r"C:\Program Files (x86)\ffmpeg\bin",
]
for folder in candidates:
if os.path.isfile(os.path.join(folder, "ffmpeg.exe")):
return folder
# Try PATH
import shutil
found = shutil.which("ffmpeg")
if found:
return str(Path(found).parent)
return None
def extract_mp4(video_id: str, meta: Dict, output_folder: str, resolution: str = "best") -> str:
"""Download video as MP4 using yt-dlp. Explicitly uses ffmpeg for merging."""
title = meta.get("title", "")
base = make_base_name(title, video_id)
ensure_dir(output_folder)
ffmpeg_dir = _find_ffmpeg()
# Build format selector — always require vcodec to avoid audio-only streams
if resolution == "best":
fmt = (
"bestvideo[vcodec^=avc1][ext=mp4]+bestaudio[ext=m4a]"
"/bestvideo[ext=mp4]+bestaudio[ext=m4a]"
"/bestvideo+bestaudio"
"/best[vcodec!=none]"
"/best"
)
else:
fmt = (
f"bestvideo[vcodec^=avc1][height<={resolution}][ext=mp4]+bestaudio[ext=m4a]"
f"/bestvideo[height<={resolution}][ext=mp4]+bestaudio[ext=m4a]"
f"/bestvideo[height<={resolution}]+bestaudio"
f"/best[height<={resolution}][vcodec!=none]"
f"/best[height<={resolution}]"
f"/best[vcodec!=none]"
f"/best"
)
out_template = os.path.join(output_folder, f"{base}__mp4.%(ext)s")
ydl_opts = {
"quiet": True,
"no_warnings": True,
"format": fmt,
"outtmpl": out_template,
"merge_output_format": "mp4",
"ignoreerrors": False,
"postprocessors": [
{"key": "FFmpegVideoConvertor", "preferedformat": "mp4"},
],
}
# Explicitly tell yt-dlp where ffmpeg is
if ffmpeg_dir:
ydl_opts["ffmpeg_location"] = ffmpeg_dir
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
ydl.download([video_url(video_id)])
# Find the downloaded file
for fname in sorted(os.listdir(output_folder)):
if fname.startswith(f"{base}__mp4") and fname.endswith(".mp4"):
return os.path.join(output_folder, fname)
raise RuntimeError(f"MP4 file not found after download for {video_id}. ffmpeg_dir={ffmpeg_dir}")

30
backend/requirements.txt Normal file
View File

@@ -0,0 +1,30 @@
# Core
fastapi>=0.110.0
uvicorn[standard]>=0.29.0
sse-starlette>=1.8.2
python-multipart>=0.0.9
# YouTube downloading
yt-dlp>=2024.3.10
# Transcripts
youtube-transcript-api>=0.6.2
# HTTP / utilities
requests>=2.31.0
aiofiles>=23.2.1
# LLM (OpenRouter via OpenAI-compatible API)
openai>=1.14.0
# Database
# sqlite3 is built-in to Python
# MCP
mcp>=1.0.0
# Misc
python-slugify>=8.0.4
pydantic>=2.6.0
tqdm>=4.66.2

BIN
backend/web/app.js Normal file
View File

Binary file not shown.

265
backend/web/index.html Normal file
View File

@@ -0,0 +1,265 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>YouTube SIPOC Tool</title>
<link rel="stylesheet" href="/style.css" />
</head>
<body>
<!-- ═══════════════════════════ HEADER ═══════════════════════════ -->
<header class="app-header">
<div class="app-logo">
<svg width="28" height="28" viewBox="0 0 24 24" fill="currentColor">
<path d="M19.59 6.69a4.83 4.83 0 01-3.77-4.25V2h-3.45v13.67a2.89 2.89 0 01-2.88 2.5 2.89 2.89 0 01-2.89-2.89 2.89 2.89 0 012.89-2.89c.28 0 .54.04.79.1V9.01a6.33 6.33 0 00-.79-.05 6.34 6.34 0 00-6.34 6.34 6.34 6.34 0 006.34 6.34 6.34 6.34 0 006.33-6.34v-7a8.15 8.15 0 004.77 1.52v-3.4a4.85 4.85 0 01-1-.09z"/>
</svg>
<span class="app-title">YouTube <span>SIPOC</span></span>
</div>
<nav class="header-nav">
<button class="nav-btn active" onclick="showView('jobs')" id="nav-jobs">Jobs</button>
<button class="nav-btn" onclick="showView('new')" id="nav-new">+ New Job</button>
<button class="nav-btn" onclick="showView('settings')" id="nav-settings">⚙ Settings</button>
</nav>
</header>
<div class="app-body">
<!-- ═══════════════════════ SIDEBAR ═══════════════════════ -->
<aside class="sidebar">
<div class="sidebar-section">
<button class="new-job-btn" onclick="showView('new')">
<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5"><line x1="12" y1="5" x2="12" y2="19"/><line x1="5" y1="12" x2="19" y2="12"/></svg>
New Extraction Job
</button>
</div>
<div class="sidebar-section" style="flex:1; overflow-y:auto; border-bottom:none;">
<div class="sidebar-label">Recent Jobs</div>
<div class="job-list" id="sidebar-job-list">
<div class="text-muted" style="font-size:12px;padding:8px 4px;">No jobs yet</div>
</div>
</div>
</aside>
<!-- ═══════════════════════ MAIN CONTENT ═══════════════════════ -->
<main class="main-content">
<!-- ─── VIEW: NEW JOB ─── -->
<div id="view-new" class="hidden">
<div class="card">
<div class="card-header">
<svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="var(--accent)" stroke-width="2"><circle cx="12" cy="12" r="10"/><line x1="12" y1="8" x2="12" y2="16"/><line x1="8" y1="12" x2="16" y2="12"/></svg>
<div>
<div class="card-title">New Extraction Job</div>
<div class="card-subtitle">Configure YouTube source and what to extract</div>
</div>
</div>
<!-- Input Type -->
<div class="form-group">
<label class="form-label">Input Type</label>
<div class="type-pills">
<div class="type-pill selected" data-type="url" onclick="selectInputType('url')">🔗 Single URL</div>
<div class="type-pill" data-type="playlist" onclick="selectInputType('playlist')">📋 Playlist</div>
<div class="type-pill" data-type="handle" onclick="selectInputType('handle')">👤 Channel Handle</div>
<div class="type-pill" data-type="csv" onclick="selectInputType('csv')">📄 CSV List</div>
</div>
</div>
<!-- Input Value -->
<div class="form-group" id="input-url-group">
<label class="form-label" id="input-label">YouTube URL</label>
<input type="text" class="form-input" id="input-value"
placeholder="https://www.youtube.com/watch?v=..." />
</div>
<!-- CSV textarea (only shown for csv type) -->
<div class="form-group hidden" id="input-csv-group">
<label class="form-label">CSV / List of URLs (one per line or comma-separated)</label>
<textarea class="form-textarea" id="input-csv-value" rows="5"
placeholder="https://youtu.be/abc123&#10;https://youtu.be/xyz456&#10;dQw4w9WgXcQ"></textarea>
</div>
<hr class="divider" />
<!-- Artifact Selection -->
<div class="form-group">
<label class="form-label">What to Extract (select all that apply)</label>
<div class="artifact-grid" id="artifact-grid">
<!-- populated by JS -->
</div>
</div>
<hr class="divider" />
<!-- Output Folder -->
<div class="form-group">
<label class="form-label">Output Folder</label>
<div class="folder-display" id="folder-display" onclick="openFolderBrowser()">
<span>📁</span>
<span class="folder-path" id="folder-path-text">Click to choose a folder…</span>
</div>
<input type="hidden" id="output-folder" value="" />
</div>
<!-- Resolution (shown only when mp4 selected) -->
<div class="form-group hidden" id="resolution-group">
<label class="form-label">MP4 Resolution</label>
<select class="form-select" id="resolution">
<option value="best">Best Available</option>
<option value="2160">4K (2160p)</option>
<option value="1440">2K (1440p)</option>
<option value="1080" selected>1080p</option>
<option value="720">720p</option>
<option value="480">480p</option>
<option value="360">360p</option>
<option value="240">240p</option>
<option value="144">144p</option>
</select>
</div>
<!-- Bot Avoidance Delays -->
<div class="form-group">
<label class="form-label">Random Delay Between Downloads (seconds, for bot detection avoidance)</label>
<div class="range-row">
<div class="form-group" style="margin:0">
<label class="form-label" style="font-size:11px">Min Delay</label>
<input type="number" class="form-input" id="delay-min" value="2" min="0" max="60" step="0.5" style="width:100px" />
</div>
<div class="form-group" style="margin:0">
<label class="form-label" style="font-size:11px">Max Delay</label>
<input type="number" class="form-input" id="delay-max" value="7" min="0" max="120" step="0.5" style="width:100px" />
</div>
</div>
</div>
<hr class="divider" />
<!-- API Settings -->
<div class="form-group">
<label class="form-label">OpenRouter API Key <span class="text-muted">(required for summaries)</span></label>
<input type="password" class="form-input" id="api-key"
placeholder="sk-or-v1-..." autocomplete="off" />
<div style="font-size:11px;color:var(--text-muted);margin-top:4px;">
Not stored — only used for this session's summary requests
</div>
</div>
<div class="form-group">
<label class="form-label">OpenRouter Model</label>
<input type="text" class="form-input" id="or-model"
value="openai/gpt-4o-mini"
placeholder="openai/gpt-4o-mini" />
</div>
<div class="form-group">
<label class="form-label">Summary Prompt <span class="text-muted">(leave blank for default)</span></label>
<textarea class="form-textarea" id="summary-prompt" rows="3"
placeholder="Custom system prompt for LLM summarization (optional)…"></textarea>
</div>
<div class="btn-row">
<button class="btn btn-primary" onclick="createJob()" id="create-btn">
<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5"><polyline points="5 12 12 5 19 12"/><polyline points="12 19 12 5"/></svg>
Start Extraction
</button>
<button class="btn btn-outline" onclick="showView('jobs')">Cancel</button>
</div>
</div>
</div>
<!-- ─── VIEW: JOB DETAIL ─── -->
<div id="view-jobs">
<div id="job-detail">
<!-- Welcome / empty state -->
<div class="empty-state">
<div class="empty-state-icon">🎬</div>
<div class="empty-state-title">YouTube SIPOC Tool</div>
<div class="empty-state-body">
Extract titles, stats, transcripts, summaries, thumbnails, MP3, and MP4 from YouTube videos.<br/><br/>
Click <strong>+ New Job</strong> to get started.
</div>
</div>
</div>
</div>
<!-- ─── VIEW: SETTINGS ─── -->
<div id="view-settings" class="hidden">
<div class="card">
<div class="card-header">
<div class="card-title">⚙ Settings & Defaults</div>
<div class="card-subtitle">These defaults are used when creating new jobs</div>
</div>
<div class="settings-grid">
<div class="form-group">
<label class="form-label">Default Delay Min (s)</label>
<input type="number" class="form-input" id="s-delay-min" value="2" min="0" />
</div>
<div class="form-group">
<label class="form-label">Default Delay Max (s)</label>
<input type="number" class="form-input" id="s-delay-max" value="7" min="0" />
</div>
<div class="form-group">
<label class="form-label">Default MP4 Resolution</label>
<select class="form-select" id="s-resolution">
<option value="best">Best</option>
<option value="1080" selected>1080p</option>
<option value="720">720p</option>
<option value="480">480p</option>
</select>
</div>
<div class="form-group">
<label class="form-label">Default OpenRouter Model</label>
<input type="text" class="form-input" id="s-model" value="openai/gpt-4o-mini" />
</div>
</div>
<div class="form-group">
<label class="form-label">Default Summary Prompt</label>
<textarea class="form-textarea" id="s-prompt" rows="4"></textarea>
</div>
<div class="btn-row">
<button class="btn btn-blue" onclick="saveSettings()">Save Defaults</button>
</div>
</div>
<div class="card">
<div class="card-header">
<div class="card-title">🔌 MCP Server Registration</div>
<div class="card-subtitle">Add this to your MCP settings to use with AI agents</div>
</div>
<pre id="mcp-config-snippet" style="background:var(--bg);border:1px solid var(--border);border-radius:4px;padding:14px;font-size:12px;color:var(--text-dim);overflow-x:auto;white-space:pre-wrap;word-break:break-all;"></pre>
<div class="mt-8">
<button class="btn btn-outline btn-sm" onclick="copyMcpConfig()">📋 Copy to Clipboard</button>
</div>
</div>
</div>
</main>
</div>
<!-- ═══════════════════════ FOLDER BROWSER MODAL ═══════════════════════ -->
<div class="modal-overlay hidden" id="folder-modal">
<div class="modal" style="max-width:600px">
<div class="modal-title">📁 Choose Output Folder</div>
<div style="margin-bottom:10px;">
<div class="input-with-btn">
<input type="text" class="form-input" id="folder-modal-path" placeholder="Type path or browse below…" />
<button class="btn btn-blue btn-sm" onclick="navigateToTyped()">Go</button>
<button class="btn btn-outline btn-sm" onclick="createNewFolder()">+ New Folder</button>
</div>
</div>
<div class="folder-current" id="folder-current-path"></div>
<div class="folder-browser" id="folder-browser-list"></div>
<div class="btn-row mt-16">
<button class="btn btn-green" onclick="selectCurrentFolder()">✔ Select This Folder</button>
<button class="btn btn-outline" onclick="closeFolderModal()">Cancel</button>
</div>
</div>
</div>
<!-- ═══════════════════════ TOAST CONTAINER ═══════════════════════ -->
<div class="toast-container" id="toast-container"></div>
<script src="/app.js"></script>
</body>
</html>

778
backend/web/style.css Normal file
View File

@@ -0,0 +1,778 @@
/* ============================================================
YouTube SIPOC Tool - Dark Mode UI
============================================================ */
:root {
--bg: #0f0f0f;
--bg2: #1a1a1a;
--bg3: #242424;
--bg4: #2e2e2e;
--border: #3a3a3a;
--text: #e8e8e8;
--text-dim: #888;
--text-muted: #555;
--accent: #ff4444;
--accent2: #cc2222;
--blue: #4a9eff;
--green: #22cc66;
--yellow: #ffcc00;
--orange: #ff8822;
--purple: #9966ff;
--red: #ff4444;
--radius: 8px;
--radius-sm: 4px;
--shadow: 0 4px 20px rgba(0,0,0,0.4);
--font: 'Segoe UI', system-ui, -apple-system, sans-serif;
--font-mono: 'Cascadia Code', 'Fira Code', 'Consolas', monospace;
}
* { box-sizing: border-box; margin: 0; padding: 0; }
body {
font-family: var(--font);
background: var(--bg);
color: var(--text);
font-size: 14px;
line-height: 1.5;
min-height: 100vh;
}
/* ── Layout ───────────────────────────────────────────────── */
.app-header {
background: var(--bg2);
border-bottom: 1px solid var(--border);
padding: 0 24px;
display: flex;
align-items: center;
gap: 16px;
height: 56px;
position: sticky;
top: 0;
z-index: 100;
}
.app-logo {
display: flex;
align-items: center;
gap: 10px;
text-decoration: none;
}
.app-logo svg { color: var(--accent); }
.app-title {
font-size: 18px;
font-weight: 700;
color: var(--text);
letter-spacing: -0.3px;
}
.app-title span { color: var(--accent); }
.header-nav {
display: flex;
gap: 4px;
margin-left: auto;
}
.nav-btn {
background: none;
border: 1px solid transparent;
color: var(--text-dim);
padding: 6px 14px;
border-radius: var(--radius-sm);
cursor: pointer;
font-size: 13px;
transition: all 0.15s;
}
.nav-btn:hover { background: var(--bg3); color: var(--text); border-color: var(--border); }
.nav-btn.active { background: var(--bg3); color: var(--text); border-color: var(--border); }
.app-body {
display: flex;
min-height: calc(100vh - 56px);
}
/* Sidebar */
.sidebar {
width: 260px;
background: var(--bg2);
border-right: 1px solid var(--border);
display: flex;
flex-direction: column;
flex-shrink: 0;
}
.sidebar-section {
padding: 12px;
border-bottom: 1px solid var(--border);
}
.sidebar-label {
font-size: 10px;
font-weight: 700;
text-transform: uppercase;
letter-spacing: 0.8px;
color: var(--text-muted);
margin-bottom: 8px;
padding: 0 4px;
}
.job-list { display: flex; flex-direction: column; gap: 4px; }
.job-item {
padding: 8px 10px;
border-radius: var(--radius-sm);
cursor: pointer;
transition: background 0.12s;
border: 1px solid transparent;
}
.job-item:hover { background: var(--bg3); }
.job-item.active { background: var(--bg3); border-color: var(--border); }
.job-item-title {
font-size: 12px;
font-weight: 600;
color: var(--text);
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.job-item-meta {
font-size: 11px;
color: var(--text-dim);
display: flex;
align-items: center;
gap: 6px;
margin-top: 2px;
}
.new-job-btn {
width: 100%;
background: var(--accent);
color: #fff;
border: none;
padding: 9px;
border-radius: var(--radius-sm);
cursor: pointer;
font-size: 13px;
font-weight: 600;
display: flex;
align-items: center;
justify-content: center;
gap: 6px;
transition: background 0.15s;
}
.new-job-btn:hover { background: var(--accent2); }
/* Main content */
.main-content {
flex: 1;
overflow-y: auto;
padding: 24px;
max-width: 1100px;
}
/* ── Cards & Panels ──────────────────────────────────────── */
.card {
background: var(--bg2);
border: 1px solid var(--border);
border-radius: var(--radius);
padding: 20px;
margin-bottom: 16px;
}
.card-header {
display: flex;
align-items: center;
gap: 10px;
margin-bottom: 16px;
padding-bottom: 12px;
border-bottom: 1px solid var(--border);
}
.card-title {
font-size: 15px;
font-weight: 700;
color: var(--text);
}
.card-subtitle {
font-size: 12px;
color: var(--text-dim);
margin-top: 1px;
}
/* ── Form Controls ────────────────────────────────────────── */
.form-group {
margin-bottom: 16px;
}
.form-label {
display: block;
font-size: 12px;
font-weight: 600;
color: var(--text-dim);
margin-bottom: 6px;
text-transform: uppercase;
letter-spacing: 0.4px;
}
.form-input,
.form-select,
.form-textarea {
width: 100%;
background: var(--bg3);
border: 1px solid var(--border);
color: var(--text);
padding: 8px 12px;
border-radius: var(--radius-sm);
font-family: var(--font);
font-size: 13px;
transition: border-color 0.15s;
outline: none;
}
.form-input:focus,
.form-select:focus,
.form-textarea:focus { border-color: var(--blue); }
.form-textarea { resize: vertical; min-height: 80px; }
.form-input.error { border-color: var(--red); }
.input-with-btn {
display: flex;
gap: 8px;
}
.input-with-btn .form-input { flex: 1; }
/* Type pills */
.type-pills {
display: flex;
gap: 6px;
flex-wrap: wrap;
}
.type-pill {
padding: 5px 12px;
border-radius: 20px;
border: 1px solid var(--border);
background: var(--bg3);
color: var(--text-dim);
font-size: 12px;
cursor: pointer;
transition: all 0.15s;
user-select: none;
}
.type-pill:hover { border-color: var(--blue); color: var(--text); }
.type-pill.selected { background: var(--blue); border-color: var(--blue); color: #fff; }
/* Artifact checkboxes */
.artifact-grid {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(130px, 1fr));
gap: 8px;
}
.artifact-check {
display: flex;
align-items: center;
gap: 8px;
padding: 8px 12px;
background: var(--bg3);
border: 1px solid var(--border);
border-radius: var(--radius-sm);
cursor: pointer;
transition: all 0.15s;
user-select: none;
}
.artifact-check:hover { border-color: var(--blue); }
.artifact-check.checked { border-color: var(--green); background: rgba(34,204,102,0.08); }
.artifact-check input { display: none; }
.artifact-icon { font-size: 16px; width: 20px; text-align: center; }
.artifact-label { font-size: 12px; font-weight: 600; color: var(--text); }
/* Range inputs */
.range-row {
display: flex;
gap: 12px;
}
.range-row .form-group { flex: 1; }
/* Folder picker */
.folder-display {
display: flex;
align-items: center;
gap: 8px;
padding: 8px 12px;
background: var(--bg3);
border: 1px solid var(--border);
border-radius: var(--radius-sm);
font-size: 13px;
color: var(--text-dim);
font-family: var(--font-mono);
cursor: pointer;
transition: border-color 0.15s;
min-height: 36px;
}
.folder-display:hover { border-color: var(--blue); }
.folder-display .folder-path { flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; color: var(--text); }
/* ── Buttons ──────────────────────────────────────────────── */
.btn {
display: inline-flex;
align-items: center;
gap: 6px;
padding: 8px 16px;
border-radius: var(--radius-sm);
border: 1px solid transparent;
font-size: 13px;
font-weight: 600;
cursor: pointer;
transition: all 0.15s;
font-family: var(--font);
}
.btn:disabled { opacity: 0.4; cursor: not-allowed; }
.btn-primary { background: var(--accent); color: #fff; }
.btn-primary:hover:not(:disabled) { background: var(--accent2); }
.btn-blue { background: var(--blue); color: #fff; }
.btn-blue:hover:not(:disabled) { background: #3a8ee0; }
.btn-green { background: var(--green); color: #000; }
.btn-green:hover:not(:disabled) { background: #1ab055; }
.btn-outline {
background: transparent;
border-color: var(--border);
color: var(--text-dim);
}
.btn-outline:hover:not(:disabled) { border-color: var(--text-dim); color: var(--text); }
.btn-danger { background: rgba(255,68,68,0.15); border-color: var(--red); color: var(--red); }
.btn-danger:hover:not(:disabled) { background: rgba(255,68,68,0.25); }
.btn-sm { padding: 5px 10px; font-size: 12px; }
.btn-row {
display: flex;
gap: 8px;
align-items: center;
flex-wrap: wrap;
}
/* ── Status Badges ────────────────────────────────────────── */
.badge {
display: inline-flex;
align-items: center;
gap: 4px;
padding: 2px 8px;
border-radius: 20px;
font-size: 11px;
font-weight: 700;
text-transform: uppercase;
letter-spacing: 0.4px;
}
.badge-running { background: rgba(74,158,255,0.15); color: var(--blue); border: 1px solid rgba(74,158,255,0.3); }
.badge-done { background: rgba(34,204,102,0.15); color: var(--green); border: 1px solid rgba(34,204,102,0.3); }
.badge-done_with_errors { background: rgba(255,136,34,0.15); color: var(--orange); border: 1px solid rgba(255,136,34,0.3); }
.badge-failed { background: rgba(255,68,68,0.15); color: var(--red); border: 1px solid rgba(255,68,68,0.3); }
.badge-paused { background: rgba(255,204,0,0.15); color: var(--yellow); border: 1px solid rgba(255,204,0,0.3); }
.badge-stopped { background: rgba(136,136,136,0.15); color: var(--text-dim); border: 1px solid var(--border); }
.badge-pending { background: rgba(153,102,255,0.15); color: var(--purple); border: 1px solid rgba(153,102,255,0.3); }
/* Artifact status */
.art-dot {
display: inline-block;
width: 8px;
height: 8px;
border-radius: 50%;
flex-shrink: 0;
}
.art-dot-done { background: var(--green); }
.art-dot-failed { background: var(--red); box-shadow: 0 0 4px var(--red); }
.art-dot-running { background: var(--blue); animation: pulse 1s infinite; }
.art-dot-pending { background: var(--border); }
.art-dot-skipped { background: var(--text-muted); }
@keyframes pulse {
0%, 100% { opacity: 1; }
50% { opacity: 0.4; }
}
/* ── Progress ─────────────────────────────────────────────── */
.progress-bar {
width: 100%;
height: 6px;
background: var(--bg4);
border-radius: 3px;
overflow: hidden;
margin: 8px 0;
}
.progress-fill {
height: 100%;
border-radius: 3px;
background: var(--blue);
transition: width 0.4s ease;
}
.progress-fill.green { background: var(--green); }
.progress-fill.yellow { background: var(--yellow); }
.progress-fill.red { background: var(--red); }
.stats-row {
display: flex;
gap: 16px;
flex-wrap: wrap;
}
.stat-item {
display: flex;
flex-direction: column;
gap: 2px;
}
.stat-value {
font-size: 22px;
font-weight: 700;
color: var(--text);
line-height: 1;
}
.stat-label {
font-size: 11px;
color: var(--text-muted);
text-transform: uppercase;
letter-spacing: 0.5px;
}
/* ── Video Table ──────────────────────────────────────────── */
.video-table-wrap {
overflow-x: auto;
border-radius: var(--radius-sm);
border: 1px solid var(--border);
}
.video-table {
width: 100%;
border-collapse: collapse;
font-size: 13px;
}
.video-table th {
text-align: left;
padding: 10px 12px;
background: var(--bg3);
color: var(--text-muted);
font-size: 11px;
font-weight: 700;
text-transform: uppercase;
letter-spacing: 0.5px;
border-bottom: 1px solid var(--border);
white-space: nowrap;
}
.video-table td {
padding: 10px 12px;
border-bottom: 1px solid var(--border);
vertical-align: middle;
}
.video-table tr:last-child td { border-bottom: none; }
.video-table tr:hover td { background: rgba(255,255,255,0.02); }
.video-table tr.active-row td { background: rgba(74,158,255,0.06); }
.video-title-cell {
max-width: 300px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.video-id-badge {
font-family: var(--font-mono);
font-size: 11px;
color: var(--text-muted);
background: var(--bg3);
padding: 2px 6px;
border-radius: 3px;
}
.artifact-pills {
display: flex;
gap: 4px;
flex-wrap: wrap;
}
.art-pill {
display: inline-flex;
align-items: center;
gap: 3px;
padding: 2px 7px;
border-radius: 10px;
font-size: 11px;
font-weight: 600;
border: 1px solid var(--border);
background: var(--bg3);
color: var(--text-muted);
white-space: nowrap;
}
.art-pill-done { border-color: rgba(34,204,102,0.4); background: rgba(34,204,102,0.08); color: var(--green); }
.art-pill-failed { border-color: rgba(255,68,68,0.4); background: rgba(255,68,68,0.08); color: var(--red); }
.art-pill-running { border-color: rgba(74,158,255,0.4); background: rgba(74,158,255,0.08); color: var(--blue); }
.art-pill-pending { color: var(--text-muted); }
/* ── Log Panel ────────────────────────────────────────────── */
.log-panel {
background: var(--bg);
border: 1px solid var(--border);
border-radius: var(--radius-sm);
height: 300px;
overflow-y: auto;
font-family: var(--font-mono);
font-size: 12px;
padding: 12px;
}
.log-entry {
display: flex;
gap: 10px;
padding: 2px 0;
line-height: 1.4;
}
.log-ts { color: var(--text-muted); flex-shrink: 0; }
.log-level { flex-shrink: 0; font-weight: 700; min-width: 40px; }
.log-level-INFO { color: var(--blue); }
.log-level-WARN { color: var(--yellow); }
.log-level-ERROR { color: var(--red); }
.log-level-DEBUG { color: var(--text-muted); }
.log-msg { color: var(--text-dim); word-break: break-all; }
/* ── Modal ────────────────────────────────────────────────── */
.modal-overlay {
position: fixed;
inset: 0;
background: rgba(0,0,0,0.7);
display: flex;
align-items: center;
justify-content: center;
z-index: 999;
backdrop-filter: blur(4px);
}
.modal {
background: var(--bg2);
border: 1px solid var(--border);
border-radius: var(--radius);
padding: 24px;
width: 90%;
max-width: 520px;
max-height: 80vh;
overflow-y: auto;
box-shadow: var(--shadow);
}
.modal-title {
font-size: 16px;
font-weight: 700;
margin-bottom: 16px;
color: var(--text);
}
/* Folder browser */
.folder-browser {
background: var(--bg3);
border: 1px solid var(--border);
border-radius: var(--radius-sm);
max-height: 320px;
overflow-y: auto;
}
.folder-entry {
display: flex;
align-items: center;
gap: 8px;
padding: 8px 12px;
cursor: pointer;
border-bottom: 1px solid var(--border);
transition: background 0.1s;
font-size: 13px;
}
.folder-entry:last-child { border-bottom: none; }
.folder-entry:hover { background: var(--bg4); }
.folder-entry.is-file { color: var(--text-muted); cursor: default; }
.folder-entry .entry-icon { width: 18px; text-align: center; font-size: 15px; }
.folder-current {
font-family: var(--font-mono);
font-size: 11px;
color: var(--text-dim);
padding: 8px 12px;
border-bottom: 1px solid var(--border);
background: var(--bg4);
word-break: break-all;
}
/* ── Toast Notifications ─────────────────────────────────── */
.toast-container {
position: fixed;
bottom: 24px;
right: 24px;
display: flex;
flex-direction: column;
gap: 8px;
z-index: 9999;
}
.toast {
background: var(--bg3);
border: 1px solid var(--border);
border-radius: var(--radius-sm);
padding: 12px 16px;
min-width: 260px;
max-width: 400px;
box-shadow: var(--shadow);
display: flex;
align-items: flex-start;
gap: 10px;
animation: slide-in 0.2s ease;
}
.toast.success { border-left: 3px solid var(--green); }
.toast.error { border-left: 3px solid var(--red); }
.toast.info { border-left: 3px solid var(--blue); }
.toast-msg { font-size: 13px; color: var(--text); flex: 1; }
@keyframes slide-in {
from { transform: translateX(20px); opacity: 0; }
to { transform: translateX(0); opacity: 1; }
}
/* ── Settings ─────────────────────────────────────────────── */
.settings-grid {
display: grid;
grid-template-columns: 1fr 1fr;
gap: 12px;
}
@media (max-width: 600px) { .settings-grid { grid-template-columns: 1fr; } }
/* ── Empty States ─────────────────────────────────────────── */
.empty-state {
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
padding: 60px 24px;
text-align: center;
gap: 12px;
color: var(--text-muted);
}
.empty-state-icon { font-size: 48px; }
.empty-state-title { font-size: 16px; font-weight: 700; color: var(--text-dim); }
.empty-state-body { font-size: 13px; max-width: 360px; }
/* ── Misc ─────────────────────────────────────────────────── */
.divider {
border: none;
border-top: 1px solid var(--border);
margin: 16px 0;
}
.section-title {
font-size: 13px;
font-weight: 700;
color: var(--text-dim);
margin-bottom: 12px;
text-transform: uppercase;
letter-spacing: 0.5px;
}
.text-muted { color: var(--text-muted); }
.text-dim { color: var(--text-dim); }
.text-green { color: var(--green); }
.text-red { color: var(--red); }
.text-blue { color: var(--blue); }
.text-yellow { color: var(--yellow); }
.monospace { font-family: var(--font-mono); }
.flex-row { display: flex; align-items: center; gap: 8px; }
.flex-1 { flex: 1; }
.mt-8 { margin-top: 8px; }
.mt-16 { margin-top: 16px; }
.mb-8 { margin-bottom: 8px; }
.hidden { display: none !important; }
/* Spinner */
.spinner {
display: inline-block;
width: 14px;
height: 14px;
border: 2px solid rgba(255,255,255,0.2);
border-top-color: #fff;
border-radius: 50%;
animation: spin 0.7s linear infinite;
flex-shrink: 0;
}
@keyframes spin { to { transform: rotate(360deg); } }
/* ── Job Control Bar ──────────────────────────────────────── */
.job-controls {
display: flex;
align-items: center;
gap: 8px;
padding: 12px 0;
flex-wrap: wrap;
}
.current-video-info {
flex: 1;
font-size: 12px;
color: var(--text-dim);
font-family: var(--font-mono);
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
/* Scrollbar */
::-webkit-scrollbar { width: 6px; height: 6px; }
::-webkit-scrollbar-track { background: var(--bg); }
::-webkit-scrollbar-thumb { background: var(--bg4); border-radius: 3px; }
::-webkit-scrollbar-thumb:hover { background: var(--border); }
/* View tabs */
.view-tabs {
display: flex;
gap: 0;
border-bottom: 1px solid var(--border);
margin-bottom: 16px;
}
.view-tab {
padding: 8px 16px;
background: none;
border: none;
color: var(--text-muted);
font-size: 13px;
font-weight: 600;
cursor: pointer;
border-bottom: 2px solid transparent;
margin-bottom: -1px;
transition: all 0.15s;
}
.view-tab:hover { color: var(--text); }
.view-tab.active { color: var(--blue); border-bottom-color: var(--blue); }

507
backend/worker.py Normal file
View File

@@ -0,0 +1,507 @@
"""
worker.py - Background job pipeline for YouTube SIPOC Tool
Handles pause/stop/resume, per-artifact saving, duplicate skipping,
random delays, retry logic, and resume-on-failure.
"""
import time
import random
import logging
import threading
from typing import Dict, Callable, Optional
import database as db
import extractors
import config
logger = logging.getLogger("worker")
# ─────────────────────────────────────────────────────────────────────────────
# Job Control Registry
# Stores threading.Event objects per job_id for pause/stop/skip signals.
# ─────────────────────────────────────────────────────────────────────────────
_job_controls: Dict[str, Dict] = {}
_registry_lock = threading.Lock()
def _get_controls(job_id: str) -> Dict:
with _registry_lock:
if job_id not in _job_controls:
_job_controls[job_id] = {
"pause": threading.Event(), # set = paused
"stop": threading.Event(), # set = stop requested
"skip": threading.Event(), # set = skip current video
}
return _job_controls[job_id]
def _cleanup_controls(job_id: str):
with _registry_lock:
_job_controls.pop(job_id, None)
def pause_job(job_id: str):
_get_controls(job_id)["pause"].set()
db.update_job_status(job_id, "paused")
db.add_log(job_id, "Job paused by user.", level="INFO")
def resume_job(job_id: str):
_get_controls(job_id)["pause"].clear()
db.update_job_status(job_id, "running")
db.add_log(job_id, "Job resumed by user.", level="INFO")
def stop_job(job_id: str):
_get_controls(job_id)["stop"].set()
_get_controls(job_id)["pause"].clear() # Unpause so thread can exit
db.update_job_status(job_id, "stopped")
db.add_log(job_id, "Job stopped by user.", level="INFO")
def skip_video(job_id: str):
"""Signal the worker to skip the current video and move to the next."""
_get_controls(job_id)["skip"].set()
db.add_log(job_id, "Skip requested for current video.", level="INFO")
# ─────────────────────────────────────────────────────────────────────────────
# SSE Event Emitter
# ─────────────────────────────────────────────────────────────────────────────
class EventEmitter:
"""Thread-safe event queue for Server-Sent Events."""
def __init__(self):
self._listeners: Dict[str, list] = {}
self._lock = threading.Lock()
def subscribe(self, job_id: str) -> "EventQueue":
queue = EventQueue()
with self._lock:
if job_id not in self._listeners:
self._listeners[job_id] = []
self._listeners[job_id].append(queue)
return queue
def unsubscribe(self, job_id: str, queue: "EventQueue"):
with self._lock:
if job_id in self._listeners:
try:
self._listeners[job_id].remove(queue)
except ValueError:
pass
def emit(self, job_id: str, event_type: str, data: dict):
with self._lock:
listeners = list(self._listeners.get(job_id, []))
for q in listeners:
q.put(event_type, data)
class EventQueue:
"""Per-subscriber event queue with thread-safe get."""
def __init__(self):
import queue
self._q = queue.Queue(maxsize=500)
def put(self, event_type: str, data: dict):
try:
self._q.put_nowait({"type": event_type, "data": data})
except Exception:
pass # Drop events if queue is full
def get(self, timeout: float = 30.0):
try:
return self._q.get(timeout=timeout)
except Exception:
return None
# Global emitter instance
emitter = EventEmitter()
# ─────────────────────────────────────────────────────────────────────────────
# Worker Thread Entry Point
# ─────────────────────────────────────────────────────────────────────────────
def run_job(job_id: str):
"""
Main worker function. Runs in a background thread.
Resolves inputs, iterates videos, processes each artifact,
emits SSE events, handles pause/stop/skip/retry.
"""
controls = _get_controls(job_id)
job = db.get_job(job_id)
if not job:
logger.error(f"Job {job_id} not found in database.")
return
db.update_job_status(job_id, "running")
_emit(job_id, "job_status", {"job_id": job_id, "status": "running"})
db.add_log(job_id, "Job started.", level="INFO")
try:
# ── Step 1: Resolve input to video list ──────────────────────────────
_emit(job_id, "progress", {"message": "Resolving input URLs…"})
db.add_log(job_id, f"Resolving input: type={job['input_type']}", level="INFO")
try:
videos = extractors.resolve_input(job["input_type"], job["input_value"])
except Exception as e:
db.add_log(job_id, f"Failed to resolve input: {e}", level="ERROR")
db.update_job_status(job_id, "failed")
_emit(job_id, "job_status", {"job_id": job_id, "status": "failed", "error": str(e)})
return
if not videos:
db.add_log(job_id, "No videos found for input.", level="WARN")
db.update_job_status(job_id, "done")
_emit(job_id, "job_status", {"job_id": job_id, "status": "done"})
return
# Deduplicate by video_id (preserve order)
seen = set()
unique_videos = []
for v in videos:
if v["video_id"] not in seen:
seen.add(v["video_id"])
unique_videos.append(v)
videos = unique_videos
# ── Step 2: Register videos in DB ────────────────────────────────────
for i, v in enumerate(videos):
db.upsert_video(job_id, v["video_id"], v["url"], v.get("title", ""), i)
db.update_job_progress(job_id, len(videos), 0)
_emit(job_id, "video_list", {
"job_id": job_id,
"total": len(videos),
"videos": [{"video_id": v["video_id"], "title": v.get("title", ""), "position": i}
for i, v in enumerate(videos)]
})
db.add_log(job_id, f"Found {len(videos)} unique videos.", level="INFO")
# ── Step 3: Process each video ────────────────────────────────────────
selections = job["selections"]
done_count = 0
for idx, video in enumerate(videos):
# Check stop
if controls["stop"].is_set():
db.add_log(job_id, "Job stopped before processing video.", level="INFO")
break
# Handle pause (blocking)
_wait_if_paused(controls)
if controls["stop"].is_set():
break
# Reset skip flag for this video
controls["skip"].clear()
video_id = video["video_id"]
db.add_log(job_id, f"Processing video {idx+1}/{len(videos)}: {video_id}", level="INFO")
_emit(job_id, "video_start", {"job_id": job_id, "video_id": video_id, "position": idx})
# ── Step 3a: Fetch metadata (once per video) ──────────────────
meta = {}
try:
_emit(job_id, "artifact_start", {
"job_id": job_id, "video_id": video_id, "artifact_type": "metadata"
})
meta = extractors.fetch_metadata(video_id)
title = meta.get("title", "")
if title:
db.update_video_title(job_id, video_id, title)
_emit(job_id, "video_title", {"job_id": job_id, "video_id": video_id, "title": title})
db.add_log(job_id, f"Metadata fetched for {video_id}: {title or '(no title)'}", level="INFO")
except Exception as e:
db.add_log(job_id, f"Failed to fetch metadata for {video_id}: {e}", level="ERROR")
_emit(job_id, "video_error", {"job_id": job_id, "video_id": video_id, "error": str(e)})
db.update_video_status(job_id, video_id, "failed")
continue
# ── Step 3b: Initialize artifact records ──────────────────────
for artifact_type in selections:
db.upsert_artifact(job_id, video_id, artifact_type)
# ── Step 3c: Process each selected artifact ───────────────────
video_failed = False
for artifact_type in selections:
# Check stop or skip
if controls["stop"].is_set():
break
if controls["skip"].is_set():
db.add_log(job_id, f"Skipping remaining artifacts for {video_id} (skip requested).", level="INFO")
break
# Handle pause
_wait_if_paused(controls)
if controls["stop"].is_set():
break
# Skip if already done
if db.artifact_is_done(job_id, video_id, artifact_type):
db.add_log(job_id, f"[SKIP-DUP] {artifact_type} for {video_id} already done.", level="INFO")
_emit(job_id, "artifact_skip", {
"job_id": job_id, "video_id": video_id, "artifact_type": artifact_type,
"reason": "already_done"
})
continue
# Process with retry
success = _process_artifact_with_retry(
job_id=job_id,
video_id=video_id,
artifact_type=artifact_type,
meta=meta,
job=job,
controls=controls,
)
if not success:
video_failed = True
# Random delay between artifacts (bot avoidance)
if not controls["stop"].is_set() and not controls["skip"].is_set():
delay = random.uniform(job["delay_min"], job["delay_max"])
db.add_log(job_id, f"Waiting {delay:.1f}s before next artifact…", level="DEBUG")
_interruptible_sleep(delay, controls)
# ── Step 3d: Mark video done/failed ──────────────────────────
if not controls["stop"].is_set():
status = "failed" if video_failed else "done"
db.update_video_status(job_id, video_id, status)
done_count += 1
db.update_job_progress(job_id, len(videos), done_count)
_emit(job_id, "video_done", {
"job_id": job_id,
"video_id": video_id,
"status": status,
"done": done_count,
"total": len(videos),
})
# Random delay between videos
if idx < len(videos) - 1 and not controls["stop"].is_set():
delay = random.uniform(job["delay_min"], job["delay_max"])
db.add_log(job_id, f"Waiting {delay:.1f}s before next video…", level="DEBUG")
_interruptible_sleep(delay, controls)
# ── Step 4: Finalize job ──────────────────────────────────────────────
if controls["stop"].is_set():
final_status = "stopped"
else:
stats = db.get_job_stats(job_id)
final_status = "done" if stats["artifact_failed"] == 0 else "done_with_errors"
db.update_job_status(job_id, final_status)
stats = db.get_job_stats(job_id)
db.add_log(job_id, f"Job finished with status: {final_status}. Stats: {stats}", level="INFO")
_emit(job_id, "job_status", {"job_id": job_id, "status": final_status, "stats": stats})
except Exception as e:
logger.exception(f"Unexpected error in job {job_id}: {e}")
db.add_log(job_id, f"Unexpected error: {e}", level="ERROR")
db.update_job_status(job_id, "failed")
_emit(job_id, "job_status", {"job_id": job_id, "status": "failed", "error": str(e)})
finally:
_cleanup_controls(job_id)
# ─────────────────────────────────────────────────────────────────────────────
# Artifact Processing with Retry
# ─────────────────────────────────────────────────────────────────────────────
def _process_artifact_with_retry(
job_id: str,
video_id: str,
artifact_type: str,
meta: Dict,
job: Dict,
controls: Dict,
) -> bool:
"""
Attempt to process one artifact, retrying up to MAX_RETRIES times.
Returns True on success, False on final failure.
"""
max_retries = config.MAX_RETRIES
output_folder = job["output_folder"]
resolution = job.get("resolution", "best")
api_key = job.get("openrouter_model", "") # model field
# We store the API key in the job separately; retrieve from job dict
# (api_key is set in the job via the POST body - see api_server.py)
openrouter_api_key = job.get("_openrouter_api_key", "") # injected by api_server
openrouter_model = job.get("openrouter_model", config.OPENROUTER_MODEL)
summary_prompt = job.get("summary_prompt", config.SUMMARY_SYSTEM_PROMPT)
for attempt in range(1, max_retries + 1):
if controls["stop"].is_set() or controls["skip"].is_set():
return False
db.mark_artifact_started(job_id, video_id, artifact_type)
_emit(job_id, "artifact_start", {
"job_id": job_id,
"video_id": video_id,
"artifact_type": artifact_type,
"attempt": attempt,
})
db.add_log(job_id, f"[{artifact_type}] Starting (attempt {attempt}/{max_retries})", level="INFO",
video_id=video_id, artifact_type=artifact_type)
try:
file_path = _run_extractor(
artifact_type=artifact_type,
video_id=video_id,
meta=meta,
output_folder=output_folder,
resolution=resolution,
openrouter_api_key=openrouter_api_key,
openrouter_model=openrouter_model,
summary_prompt=summary_prompt,
)
db.update_artifact_status(job_id, video_id, artifact_type, "done", file_path=file_path)
db.add_log(job_id, f"[{artifact_type}] Done → {file_path}", level="INFO",
video_id=video_id, artifact_type=artifact_type)
_emit(job_id, "artifact_done", {
"job_id": job_id,
"video_id": video_id,
"artifact_type": artifact_type,
"file_path": file_path,
})
return True
except Exception as e:
err_msg = str(e)
logger.warning(f"[{job_id}][{video_id}][{artifact_type}] Attempt {attempt} failed: {e}")
db.add_log(job_id, f"[{artifact_type}] Attempt {attempt} failed: {e}", level="WARN",
video_id=video_id, artifact_type=artifact_type)
if attempt < max_retries:
# Exponential backoff between retries
backoff = min(2 ** attempt * 3, 30) + random.uniform(0, 2)
db.add_log(job_id, f"[{artifact_type}] Retry in {backoff:.1f}s…", level="INFO",
video_id=video_id, artifact_type=artifact_type)
_interruptible_sleep(backoff, controls)
else:
db.update_artifact_status(
job_id, video_id, artifact_type, "failed", error_msg=err_msg
)
db.add_log(job_id, f"[{artifact_type}] FAILED after {max_retries} attempts: {e}", level="ERROR",
video_id=video_id, artifact_type=artifact_type)
_emit(job_id, "artifact_failed", {
"job_id": job_id,
"video_id": video_id,
"artifact_type": artifact_type,
"error": err_msg,
"attempts": max_retries,
})
return False
return False
def _run_extractor(
artifact_type: str,
video_id: str,
meta: Dict,
output_folder: str,
resolution: str,
openrouter_api_key: str,
openrouter_model: str,
summary_prompt: str,
) -> str:
"""Dispatch to the correct extractor function."""
if artifact_type == "title":
file_path, _ = extractors.extract_title(video_id, meta, output_folder)
return file_path
elif artifact_type == "stats":
return extractors.extract_stats(video_id, meta, output_folder)
elif artifact_type == "transcript":
return extractors.extract_transcript(video_id, meta, output_folder)
elif artifact_type == "summary":
return extractors.extract_summary(
video_id, meta, output_folder,
api_key=openrouter_api_key,
model=openrouter_model,
system_prompt=summary_prompt,
)
elif artifact_type == "thumbnail":
return extractors.extract_thumbnail(video_id, meta, output_folder)
elif artifact_type == "mp3":
return extractors.extract_mp3(video_id, meta, output_folder)
elif artifact_type == "mp4":
return extractors.extract_mp4(video_id, meta, output_folder, resolution=resolution)
else:
raise ValueError(f"Unknown artifact type: {artifact_type}")
# ─────────────────────────────────────────────────────────────────────────────
# Helpers
# ─────────────────────────────────────────────────────────────────────────────
def _emit(job_id: str, event_type: str, data: dict):
"""Emit an SSE event."""
emitter.emit(job_id, event_type, data)
def _wait_if_paused(controls: Dict):
"""Block worker thread while paused, check every 0.5s."""
while controls["pause"].is_set() and not controls["stop"].is_set():
time.sleep(0.5)
def _interruptible_sleep(seconds: float, controls: Dict):
"""Sleep for `seconds` but wake early if stop/skip is set."""
end = time.monotonic() + seconds
while time.monotonic() < end:
if controls["stop"].is_set() or controls["skip"].is_set():
return
time.sleep(min(0.3, end - time.monotonic()))
# ─────────────────────────────────────────────────────────────────────────────
# Job Launcher
# ─────────────────────────────────────────────────────────────────────────────
def launch_job(job_id: str, openrouter_api_key: str = ""):
"""
Launch job in a daemon background thread.
openrouter_api_key is injected at runtime (not stored in DB for security).
"""
# Inject API key into job dict via a side channel stored in controls
controls = _get_controls(job_id)
controls["_openrouter_api_key"] = openrouter_api_key
# Patch the run_job function to use the injected key
def _patched_run():
job = db.get_job(job_id)
if job:
job["_openrouter_api_key"] = openrouter_api_key
run_job_with_key(job_id, openrouter_api_key)
t = threading.Thread(target=_patched_run, daemon=True, name=f"job-{job_id[:8]}")
t.start()
return t
def run_job_with_key(job_id: str, openrouter_api_key: str):
"""Wrapper that injects the API key into job processing."""
# Monkey-patch the job dict to include the API key during this run
original_get_job = db.get_job
def patched_get_job(jid):
result = original_get_job(jid)
if result and jid == job_id:
result["_openrouter_api_key"] = openrouter_api_key
return result
# Temporarily patch db.get_job for this thread
import database
_orig = database.get_job
database.get_job = patched_get_job
try:
run_job(job_id)
finally:
database.get_job = _orig

BIN
backend/youtube_sipoc.db Normal file
View File

Binary file not shown.