444 lines
18 KiB
Python
444 lines
18 KiB
Python
"""
|
|
api_server.py - FastAPI backend for YouTube SIPOC Tool
|
|
Serves the Web GUI, REST API, and Server-Sent Events (SSE) for real-time progress.
|
|
"""
|
|
import sys
|
|
import os
|
|
from pathlib import Path
|
|
|
|
# Ensure project directory is always on sys.path (needed for some Python installations)
|
|
_PROJECT_DIR = Path(__file__).parent.resolve()
|
|
if str(_PROJECT_DIR) not in sys.path:
|
|
sys.path.insert(0, str(_PROJECT_DIR))
|
|
|
|
import json
|
|
import asyncio
|
|
import logging
|
|
from typing import Optional, List
|
|
|
|
from fastapi import FastAPI, HTTPException, BackgroundTasks, Request
|
|
from fastapi.responses import HTMLResponse, FileResponse, StreamingResponse, JSONResponse
|
|
from fastapi.staticfiles import StaticFiles
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
from pydantic import BaseModel
|
|
import uvicorn
|
|
|
|
import database as db
|
|
import worker
|
|
import config
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
|
)
|
|
logger = logging.getLogger("api_server")
|
|
|
|
app = FastAPI(title="YouTube SIPOC Tool", version="1.0.0")
|
|
|
|
# CORS - allow localhost GUI access
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=["*"],
|
|
allow_methods=["*"],
|
|
allow_headers=["*"],
|
|
)
|
|
|
|
# Serve static web files
|
|
WEB_DIR = Path(__file__).parent / "web"
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Request / Response Models
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
class CreateJobRequest(BaseModel):
|
|
input_type: str # url | playlist | csv | handle
|
|
input_value: str # the URL/handle/CSV text
|
|
output_folder: str # absolute path to save files
|
|
selections: List[str] # artifact types to extract
|
|
resolution: str = "best" # for MP4
|
|
delay_min: float = 2.0
|
|
delay_max: float = 7.0
|
|
openrouter_api_key: str = ""
|
|
openrouter_model: str = "openai/gpt-4o-mini"
|
|
summary_prompt: str = ""
|
|
|
|
|
|
class ResumeJobRequest(BaseModel):
|
|
openrouter_api_key: str = ""
|
|
|
|
|
|
class BrowseFolderRequest(BaseModel):
|
|
path: str = ""
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Web GUI Serving
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
@app.get("/", response_class=HTMLResponse)
|
|
async def serve_gui():
|
|
index_path = WEB_DIR / "index.html"
|
|
if not index_path.exists():
|
|
return HTMLResponse("<h1>Web GUI not found. Check the web/ directory.</h1>", status_code=404)
|
|
return HTMLResponse(index_path.read_text(encoding="utf-8"))
|
|
|
|
|
|
@app.get("/app.js")
|
|
async def serve_js():
|
|
return FileResponse(WEB_DIR / "app.js", media_type="application/javascript")
|
|
|
|
|
|
@app.get("/style.css")
|
|
async def serve_css():
|
|
return FileResponse(WEB_DIR / "style.css", media_type="text/css")
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Job Management API
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
@app.post("/api/jobs")
|
|
async def create_job(req: CreateJobRequest):
|
|
"""Create and immediately launch a new extraction job."""
|
|
# Validate output folder
|
|
output_folder = req.output_folder.strip()
|
|
if not output_folder:
|
|
raise HTTPException(status_code=400, detail="output_folder is required")
|
|
|
|
# Validate selections
|
|
valid_types = set(config.ARTIFACT_TYPES)
|
|
invalid = [s for s in req.selections if s not in valid_types]
|
|
if invalid:
|
|
raise HTTPException(status_code=400, detail=f"Invalid artifact types: {invalid}")
|
|
if not req.selections:
|
|
raise HTTPException(status_code=400, detail="At least one artifact type must be selected")
|
|
|
|
# Validate input
|
|
if not req.input_value.strip():
|
|
raise HTTPException(status_code=400, detail="input_value is required")
|
|
|
|
# Create the folder if it doesn't exist
|
|
try:
|
|
Path(output_folder).mkdir(parents=True, exist_ok=True)
|
|
except Exception as e:
|
|
raise HTTPException(status_code=400, detail=f"Cannot create output folder: {e}")
|
|
|
|
# Create job in DB
|
|
job_id = db.create_job(
|
|
output_folder=output_folder,
|
|
selections=req.selections,
|
|
resolution=req.resolution,
|
|
delay_min=req.delay_min,
|
|
delay_max=req.delay_max,
|
|
input_type=req.input_type,
|
|
input_value=req.input_value,
|
|
summary_prompt=req.summary_prompt or config.SUMMARY_SYSTEM_PROMPT,
|
|
openrouter_model=req.openrouter_model or config.OPENROUTER_MODEL,
|
|
)
|
|
|
|
# Launch background worker thread
|
|
worker.launch_job(job_id, openrouter_api_key=req.openrouter_api_key)
|
|
|
|
logger.info(f"Created and launched job {job_id}")
|
|
return {"job_id": job_id, "status": "running"}
|
|
|
|
|
|
@app.get("/api/jobs")
|
|
async def list_jobs():
|
|
"""List all jobs (most recent first)."""
|
|
jobs = db.list_jobs()
|
|
result = []
|
|
for j in jobs:
|
|
stats = db.get_job_stats(j["job_id"])
|
|
j.update(stats)
|
|
result.append(j)
|
|
return result
|
|
|
|
|
|
@app.get("/api/jobs/{job_id}")
|
|
async def get_job(job_id: str):
|
|
"""Get job details + stats."""
|
|
job = db.get_job(job_id)
|
|
if not job:
|
|
raise HTTPException(status_code=404, detail="Job not found")
|
|
stats = db.get_job_stats(job_id)
|
|
job.update(stats)
|
|
return job
|
|
|
|
|
|
@app.post("/api/jobs/{job_id}/pause")
|
|
async def pause_job(job_id: str):
|
|
job = db.get_job(job_id)
|
|
if not job:
|
|
raise HTTPException(status_code=404, detail="Job not found")
|
|
if job["status"] not in ("running",):
|
|
raise HTTPException(status_code=400, detail=f"Cannot pause a job with status: {job['status']}")
|
|
worker.pause_job(job_id)
|
|
return {"job_id": job_id, "status": "paused"}
|
|
|
|
|
|
@app.post("/api/jobs/{job_id}/resume")
|
|
async def resume_job(job_id: str, req: ResumeJobRequest = ResumeJobRequest()):
|
|
job = db.get_job(job_id)
|
|
if not job:
|
|
raise HTTPException(status_code=404, detail="Job not found")
|
|
|
|
if job["status"] == "paused":
|
|
# Job thread is still alive, just unpause it
|
|
worker.resume_job(job_id)
|
|
return {"job_id": job_id, "status": "running"}
|
|
|
|
elif job["status"] in ("stopped", "failed", "done_with_errors"):
|
|
# Re-launch the job from where it left off
|
|
db.update_job_status(job_id, "pending")
|
|
worker.launch_job(job_id, openrouter_api_key=req.openrouter_api_key)
|
|
return {"job_id": job_id, "status": "running", "note": "Job re-launched to resume from last checkpoint"}
|
|
|
|
else:
|
|
raise HTTPException(status_code=400, detail=f"Cannot resume job with status: {job['status']}")
|
|
|
|
|
|
@app.post("/api/jobs/{job_id}/stop")
|
|
async def stop_job(job_id: str):
|
|
job = db.get_job(job_id)
|
|
if not job:
|
|
raise HTTPException(status_code=404, detail="Job not found")
|
|
worker.stop_job(job_id)
|
|
return {"job_id": job_id, "status": "stopped"}
|
|
|
|
|
|
@app.post("/api/jobs/{job_id}/skip")
|
|
async def skip_video(job_id: str):
|
|
"""Skip the currently processing video."""
|
|
job = db.get_job(job_id)
|
|
if not job:
|
|
raise HTTPException(status_code=404, detail="Job not found")
|
|
worker.skip_video(job_id)
|
|
return {"job_id": job_id, "action": "skip_sent"}
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Videos & Artifacts
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
@app.get("/api/jobs/{job_id}/videos")
|
|
async def get_videos(job_id: str):
|
|
videos = db.get_videos(job_id)
|
|
if not videos and not db.get_job(job_id):
|
|
raise HTTPException(status_code=404, detail="Job not found")
|
|
# Enrich with artifact statuses
|
|
result = []
|
|
for v in videos:
|
|
arts = db.get_artifacts_for_video(job_id, v["video_id"])
|
|
v["artifacts"] = arts
|
|
result.append(v)
|
|
return result
|
|
|
|
|
|
@app.get("/api/jobs/{job_id}/artifacts")
|
|
async def get_artifacts(job_id: str):
|
|
return db.get_all_artifacts(job_id)
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Logging
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
@app.get("/api/jobs/{job_id}/logs")
|
|
async def get_logs(job_id: str, limit: int = 500):
|
|
if not db.get_job(job_id):
|
|
raise HTTPException(status_code=404, detail="Job not found")
|
|
return db.get_logs(job_id, limit=limit)
|
|
|
|
|
|
@app.get("/api/jobs/{job_id}/logs/csv")
|
|
async def download_log_csv(job_id: str):
|
|
"""Download artifact log as CSV."""
|
|
if not db.get_job(job_id):
|
|
raise HTTPException(status_code=404, detail="Job not found")
|
|
csv_content = db.get_log_csv(job_id)
|
|
return StreamingResponse(
|
|
iter([csv_content]),
|
|
media_type="text/csv",
|
|
headers={"Content-Disposition": f'attachment; filename="job_{job_id[:8]}_log.csv"'},
|
|
)
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Server-Sent Events (SSE) for real-time updates
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
@app.get("/api/jobs/{job_id}/events")
|
|
async def job_events(job_id: str, request: Request):
|
|
"""
|
|
SSE endpoint. Client subscribes and receives real-time events
|
|
as the worker processes artifacts.
|
|
"""
|
|
if not db.get_job(job_id):
|
|
raise HTTPException(status_code=404, detail="Job not found")
|
|
|
|
queue = worker.emitter.subscribe(job_id)
|
|
|
|
async def event_generator():
|
|
try:
|
|
# Send initial state snapshot
|
|
job = db.get_job(job_id)
|
|
stats = db.get_job_stats(job_id)
|
|
if job:
|
|
job.update(stats)
|
|
yield _sse("job_status", {"job_id": job_id, "status": job["status"], "stats": stats})
|
|
|
|
while True:
|
|
if await request.is_disconnected():
|
|
break
|
|
|
|
# Poll the queue with a short timeout
|
|
event = await asyncio.get_event_loop().run_in_executor(
|
|
None, lambda: queue.get(timeout=1.0)
|
|
)
|
|
|
|
if event:
|
|
yield _sse(event["type"], event["data"])
|
|
else:
|
|
# Send a keepalive comment
|
|
yield ": keepalive\n\n"
|
|
|
|
# Check if job is complete
|
|
job = db.get_job(job_id)
|
|
if job and job["status"] in ("done", "done_with_errors", "stopped", "failed"):
|
|
# Give a moment for final events to flush
|
|
await asyncio.sleep(0.2)
|
|
# Drain remaining queue events
|
|
while True:
|
|
event = await asyncio.get_event_loop().run_in_executor(
|
|
None, lambda: queue.get(timeout=0.1)
|
|
)
|
|
if event:
|
|
yield _sse(event["type"], event["data"])
|
|
else:
|
|
break
|
|
yield _sse("stream_end", {"job_id": job_id})
|
|
break
|
|
|
|
except asyncio.CancelledError:
|
|
pass
|
|
finally:
|
|
worker.emitter.unsubscribe(job_id, queue)
|
|
|
|
return StreamingResponse(
|
|
event_generator(),
|
|
media_type="text/event-stream",
|
|
headers={
|
|
"Cache-Control": "no-cache",
|
|
"X-Accel-Buffering": "no",
|
|
"Connection": "keep-alive",
|
|
},
|
|
)
|
|
|
|
|
|
def _sse(event_type: str, data: dict) -> str:
|
|
"""Format a Server-Sent Event string."""
|
|
return f"event: {event_type}\ndata: {json.dumps(data)}\n\n"
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# File System Browsing Helper
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
@app.get("/api/browse")
|
|
async def browse_folder(path: str = ""):
|
|
"""
|
|
Return directory listing for folder picking in the GUI.
|
|
Defaults to user home directory.
|
|
"""
|
|
if not path:
|
|
path = str(Path.home())
|
|
|
|
target = Path(path)
|
|
if not target.exists() or not target.is_dir():
|
|
# Fallback to home
|
|
target = Path.home()
|
|
|
|
try:
|
|
entries = []
|
|
for entry in sorted(target.iterdir()):
|
|
try:
|
|
entries.append({
|
|
"name": entry.name,
|
|
"path": str(entry.absolute()),
|
|
"is_dir": entry.is_dir(),
|
|
})
|
|
except PermissionError:
|
|
pass
|
|
|
|
# Add parent directory option
|
|
parent = str(target.parent.absolute())
|
|
return {
|
|
"current": str(target.absolute()),
|
|
"parent": parent if parent != str(target.absolute()) else None,
|
|
"entries": entries,
|
|
}
|
|
except PermissionError:
|
|
raise HTTPException(status_code=403, detail="Permission denied")
|
|
|
|
|
|
@app.post("/api/mkdir")
|
|
async def make_dir(req: BrowseFolderRequest):
|
|
"""Create a new directory."""
|
|
if not req.path:
|
|
raise HTTPException(status_code=400, detail="path is required")
|
|
try:
|
|
Path(req.path).mkdir(parents=True, exist_ok=True)
|
|
return {"created": req.path}
|
|
except Exception as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Config / Settings API
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
@app.get("/api/config")
|
|
async def get_config():
|
|
"""Return non-sensitive defaults for the GUI."""
|
|
return {
|
|
"available_resolutions": config.AVAILABLE_RESOLUTIONS,
|
|
"artifact_types": config.ARTIFACT_TYPES,
|
|
"default_resolution": config.DEFAULT_VIDEO_RESOLUTION,
|
|
"delay_min": config.DELAY_MIN_SECONDS,
|
|
"delay_max": config.DELAY_MAX_SECONDS,
|
|
"max_retries": config.MAX_RETRIES,
|
|
"default_model": config.OPENROUTER_MODEL,
|
|
"summary_prompt": config.SUMMARY_SYSTEM_PROMPT,
|
|
"openrouter_base_url": config.OPENROUTER_BASE_URL,
|
|
}
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Health Check
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
@app.get("/api/health")
|
|
async def health():
|
|
return {"status": "ok", "version": "1.0.0"}
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Entry Point
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
if __name__ == "__main__":
|
|
print(f"\n{'='*60}")
|
|
print(" YouTube SIPOC Tool")
|
|
print(f" Web GUI: http://{config.API_HOST}:{config.API_PORT}")
|
|
print(f" API: http://{config.API_HOST}:{config.API_PORT}/api")
|
|
print(f"{'='*60}\n")
|
|
uvicorn.run(
|
|
"api_server:app",
|
|
host=config.API_HOST,
|
|
port=config.API_PORT,
|
|
reload=False,
|
|
log_level="info",
|
|
)
|