From 8b21034d78ebb261cfca30c12b52b433b473eefe Mon Sep 17 00:00:00 2001 From: ben Date: Tue, 31 Mar 2026 21:20:46 -0400 Subject: [PATCH] configured initial yt-dlp worker fallback download location --- pm/tasks-v2.org | 22 +++++++++++------- youdis/main.py | 58 ++++++++++++++++++++++++++++++++++++++++++++---- youdis/models.py | 2 ++ 3 files changed, 70 insertions(+), 12 deletions(-) diff --git a/pm/tasks-v2.org b/pm/tasks-v2.org index c751369..1fea622 100644 --- a/pm/tasks-v2.org +++ b/pm/tasks-v2.org @@ -36,7 +36,7 @@ define the target architecture for a private backend yt-dlp worker with thin cha ** notes - first architecture draft captured in `docs/architecture-v2.org` -* [ ] 2.0.1: build backend yt-dlp worker (3) +* [X] 2.0.1: build backend yt-dlp worker (3) create the minimal backend/service skeleton and establish a working yt-dlp baseline with clean hooks for future frontends ** pm notes - foundation; don't need the full finished service here, just the basic shape plus enough real yt-dlp execution to validate the seam and build on it. @@ -55,10 +55,13 @@ create the minimal backend/service skeleton and establish a working yt-dlp basel - submit/request path exists - status/progress hook exists - basic health/version visibility exists -4f. +4. make local testing practical without breaking container defaults + - backend can run when `/config` or `/downloads` are not writable in local dev + - env vars may override executable/config/download paths + - status makes the effective runtime command/paths inspectable ** evidence -- commit: +- commit: 7926534,2a56485 - tests: 1. `python3 -m py_compile ./youdis/main.py ./youdis/models.py ./youdis/adapters/__init__.py ./youdis/adapters/discord.py` 2. `python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000` @@ -73,11 +76,12 @@ create the minimal backend/service skeleton and establish a working yt-dlp basel 11. if testing in container: `docker run --rm -p 8000:8000 -v [config]:/config -v [downloads]:/downloads youdis:v2` :OUTPUT_STEP1-9: user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/health -{"status":"ok"}user@paladin:~/proj/youdis$ -user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/version -{"version":"20250829-ec72c56","active_job":false}user@paladin:~/proj/youdis$ +{"status":"ok"} + user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/version +{"version":"20250829-ec72c56","active_job":false} user@paladin:~/proj/youdis$ curl -X POST http://127.0.0.1:8000/jobs -H 'content-type: application/json' -d '{"url":"https://www.youtube.com/watch?v=3i72yY_LaW4"}' -{"job_id":"cc85165e-d906-4eee-864f-1a398b6de2e0","state":"accepted","url":"https://www.youtube.com/watch?v=3i72yY_LaW4","message":"accepted","phase":"queued","disposition":null,"requester_id":null,"requester_name":null,"origin":null,"result_path":null,"command":[],"returncode":null,"created_at":"2026-04-01T00:44:35.657196","updated_at":"2026-04-01T00:44:35.657198"}user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/jobs/current +{"job_id":"cc85165e-d906-4eee-864f-1a398b6de2e0","state":"accepted","url":"https://www.youtube.com/watch?v=3i72yY_LaW4","message":"accepted","phase":"queued","disposition":null,"requester_id":null,"requester_name":null,"origin":null,"result_path":null,"command":[],"returncode":null,"created_at":"2026-04-01T00:44:35.657196","updated_at":"2026-04-01T00:44:35.657198"} +user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/jobs/current {"active":false,"job":{"job_id":"cc85165e-d906-4eee-864f-1a398b6de2e0","state":"failed","url":"https://www.youtube.com/watch?v=3i72yY_LaW4","message":"ERROR: unable to create directory [Errno 13] Permission denied: '/downloads'","phase":null,"disposition":null,"requester_id":null,"requester_name":null,"origin":null,"result_path":null,"command":["yt-dlp","--config-locations","/home/user/proj/youdis/default-yt-dlp.conf","https://www.youtube.com/watch?v=3i72yY_LaW4"],"returncode":1,"created_at":"2026-04-01T00:44:35.657196","updated_at":"2026-03-31T20:44:36.653353"}}user@paladin:~/proj/youdis$ :END: - datetime: [2026-03-31 Tue 20:45] @@ -87,7 +91,9 @@ user@paladin:~/proj/youdis$ curl -X POST http://127.0.0.1:8000/jobs -H 'content- - confirm `--config-locations` behavior against the installed `yt-dlp` version during integration testing - current backend scaffold is not yet wired into `dockerfile` or `run-youdis.sh` - archive-hit and result-path parsing currently depend on `yt-dlp` stdout text patterns, so treat them as provisional until integration-tested - +- local dev now falls back to repo-local `.runtime/{config,downloads}` when `/config` or `/downloads` are not writable +- had to uninstall yt-dlp python pkg from the venv, which resulted in a '403 Forbidden' + * [ ] 2.0.2: update discord bot to use new backend (3) update the discord bot into a thin frontend that talks to the backend and verify the flow end to end ** pm notes diff --git a/youdis/main.py b/youdis/main.py index e616544..31eef27 100644 --- a/youdis/main.py +++ b/youdis/main.py @@ -5,6 +5,7 @@ from dataclasses import dataclass, field from datetime import datetime from os import getenv from pathlib import Path +import tempfile from uuid import uuid4 from fastapi import FastAPI, HTTPException @@ -16,6 +17,12 @@ REPO_ROOT = Path(__file__).resolve().parent.parent DEFAULT_CONFIG = REPO_ROOT / "default-yt-dlp.conf" VERSION_FILE = REPO_ROOT / "youdis-version.txt" YTDLP_EXECUTABLE = getenv("YOUDIS_YTDLP_EXECUTABLE", "yt-dlp") +PREFERRED_CONFIG_ROOT = Path(getenv("YOUDIS_CONFIG_DIR", "/config")) +PREFERRED_DOWNLOAD_ROOT = Path(getenv("YOUDIS_DOWNLOAD_DIR", "/downloads")) +LOCAL_RUNTIME_ROOT = REPO_ROOT / ".runtime" +FALLBACK_CONFIG_ROOT = LOCAL_RUNTIME_ROOT / "config" +FALLBACK_DOWNLOAD_ROOT = LOCAL_RUNTIME_ROOT / "downloads" +OUTPUT_TEMPLATE = "%(uploader)s/%(playlist_title)s/%(playlist_index)s%(playlist_index& - )s%(title)s.%(ext)s" @dataclass @@ -43,11 +50,33 @@ def read_version() -> str: return "unknown" -def build_ytdlp_command(request: JobRequest) -> list[str]: +def ensure_writable_directory(preferred: Path, fallback: Path) -> Path: + for candidate in (preferred, fallback): + try: + candidate.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=candidate, prefix=".youdis-write-", delete=True): + pass + return candidate + except OSError: + continue + raise OSError(f"no writable runtime directory available for {preferred} or {fallback}") + + +def resolve_runtime_paths() -> tuple[Path, Path]: + config_root = ensure_writable_directory(PREFERRED_CONFIG_ROOT, FALLBACK_CONFIG_ROOT) + download_root = ensure_writable_directory(PREFERRED_DOWNLOAD_ROOT, FALLBACK_DOWNLOAD_ROOT) + return config_root, download_root + + +def build_ytdlp_command(request: JobRequest, config_root: Path, download_root: Path) -> list[str]: return [ YTDLP_EXECUTABLE, "--config-locations", str(DEFAULT_CONFIG), + "--download-archive", + str(config_root / "archive.txt"), + "--output", + str(download_root / OUTPUT_TEMPLATE), request.url, ] @@ -121,13 +150,11 @@ async def finalize_job(job: ManagedJob, returncode: int) -> None: async def run_job(job: ManagedJob, request: JobRequest) -> None: - command = build_ytdlp_command(request) update_status( job, state="running", phase="starting", message="starting yt-dlp", - command=command, ) try: @@ -142,6 +169,15 @@ async def run_job(job: ManagedJob, request: JobRequest) -> None: await finalize_job(job, 78) return + config_root, download_root = resolve_runtime_paths() + command = build_ytdlp_command(request, config_root, download_root) + update_status( + job, + command=command, + archive_path=str(config_root / "archive.txt"), + result_path=str(download_root), + ) + process = await asyncio.create_subprocess_exec( *command, stdout=PIPE, @@ -157,6 +193,16 @@ async def run_job(job: ManagedJob, request: JobRequest) -> None: ) await finalize_job(job, 127) return + except OSError as exc: + update_status( + job, + state="failed", + phase=None, + message=f"runtime path setup failed: {exc}", + returncode=73, + ) + await finalize_job(job, 73) + return try: job.process = process @@ -220,7 +266,11 @@ async def health() -> HealthResponse: @app.get("/version", response_model=VersionResponse) async def version() -> VersionResponse: - return VersionResponse(version=read_version(), active_job=active_job is not None) + return VersionResponse( + version=read_version(), + active_job=active_job is not None, + ytdlp_executable=YTDLP_EXECUTABLE, + ) @app.get("/jobs/current", response_model=CurrentJobResponse) diff --git a/youdis/models.py b/youdis/models.py index 97fb836..21364df 100644 --- a/youdis/models.py +++ b/youdis/models.py @@ -26,6 +26,7 @@ class JobStatus(BaseModel): requester_name: str | None = None origin: str | None = None result_path: str | None = None + archive_path: str | None = None command: list[str] = Field(default_factory=list) returncode: int | None = None created_at: datetime = Field(default_factory=datetime.utcnow) @@ -44,3 +45,4 @@ class HealthResponse(BaseModel): class VersionResponse(BaseModel): version: str active_job: bool + ytdlp_executable: str