configured initial yt-dlp worker fallback download location

pm/tasks-v2.org

@@ -36,7 +36,7 @@ define the target architecture for a private backend yt-dlp worker with thin cha
 ** notes
 - first architecture draft captured in `docs/architecture-v2.org`
 
-* [ ] 2.0.1: build backend yt-dlp worker (3)
+* [X] 2.0.1: build backend yt-dlp worker (3)
 create the minimal backend/service skeleton and establish a working yt-dlp baseline with clean hooks for future frontends
 ** pm notes
 - foundation; don't need the full finished service here, just the basic shape plus enough real yt-dlp execution to validate the seam and build on it.
@@ -55,10 +55,13 @@ create the minimal backend/service skeleton and establish a working yt-dlp basel
    - submit/request path exists
    - status/progress hook exists
    - basic health/version visibility exists
-4f.      
+4. make local testing practical without breaking container defaults
+   - backend can run when `/config` or `/downloads` are not writable in local dev
+   - env vars may override executable/config/download paths
+   - status makes the effective runtime command/paths inspectable
 
 ** evidence
-- commit:
+- commit: 7926534,2a56485
 - tests:
   1. `python3 -m py_compile ./youdis/main.py ./youdis/models.py ./youdis/adapters/__init__.py ./youdis/adapters/discord.py`
   2. `python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000`
@@ -73,11 +76,12 @@ create the minimal backend/service skeleton and establish a working yt-dlp basel
   11. if testing in container: `docker run --rm -p 8000:8000 -v [config]:/config -v [downloads]:/downloads youdis:v2`
   :OUTPUT_STEP1-9:
   user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/health
-{"status":"ok"}user@paladin:~/proj/youdis$
+{"status":"ok"}
-user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/version
+  user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/version
-{"version":"20250829-ec72c56","active_job":false}user@paladin:~/proj/youdis$
+{"version":"20250829-ec72c56","active_job":false}
 user@paladin:~/proj/youdis$ curl -X POST http://127.0.0.1:8000/jobs -H 'content-type: application/json' -d '{"url":"https://www.youtube.com/watch?v=3i72yY_LaW4"}'
-{"job_id":"cc85165e-d906-4eee-864f-1a398b6de2e0","state":"accepted","url":"https://www.youtube.com/watch?v=3i72yY_LaW4","message":"accepted","phase":"queued","disposition":null,"requester_id":null,"requester_name":null,"origin":null,"result_path":null,"command":[],"returncode":null,"created_at":"2026-04-01T00:44:35.657196","updated_at":"2026-04-01T00:44:35.657198"}user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/jobs/current
+{"job_id":"cc85165e-d906-4eee-864f-1a398b6de2e0","state":"accepted","url":"https://www.youtube.com/watch?v=3i72yY_LaW4","message":"accepted","phase":"queued","disposition":null,"requester_id":null,"requester_name":null,"origin":null,"result_path":null,"command":[],"returncode":null,"created_at":"2026-04-01T00:44:35.657196","updated_at":"2026-04-01T00:44:35.657198"}
+user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/jobs/current
 {"active":false,"job":{"job_id":"cc85165e-d906-4eee-864f-1a398b6de2e0","state":"failed","url":"https://www.youtube.com/watch?v=3i72yY_LaW4","message":"ERROR: unable to create directory [Errno 13] Permission denied: '/downloads'","phase":null,"disposition":null,"requester_id":null,"requester_name":null,"origin":null,"result_path":null,"command":["yt-dlp","--config-locations","/home/user/proj/youdis/default-yt-dlp.conf","https://www.youtube.com/watch?v=3i72yY_LaW4"],"returncode":1,"created_at":"2026-04-01T00:44:35.657196","updated_at":"2026-03-31T20:44:36.653353"}}user@paladin:~/proj/youdis$
   :END:
 - datetime: [2026-03-31 Tue 20:45]
@@ -87,7 +91,9 @@ user@paladin:~/proj/youdis$ curl -X POST http://127.0.0.1:8000/jobs -H 'content-
 - confirm `--config-locations` behavior against the installed `yt-dlp` version during integration testing
 - current backend scaffold is not yet wired into `dockerfile` or `run-youdis.sh`
 - archive-hit and result-path parsing currently depend on `yt-dlp` stdout text patterns, so treat them as provisional until integration-tested
-
+- local dev now falls back to repo-local `.runtime/{config,downloads}` when `/config` or `/downloads` are not writable
+- had to uninstall yt-dlp python pkg from the venv, which resulted in a '403 Forbidden'
+  
 * [ ] 2.0.2: update discord bot to use new backend (3)
 update the discord bot into a thin frontend that talks to the backend and verify the flow end to end
 ** pm notes

youdis/main.py

@@ -5,6 +5,7 @@ from dataclasses import dataclass, field
 from datetime import datetime
 from os import getenv
 from pathlib import Path
+import tempfile
 from uuid import uuid4
 
 from fastapi import FastAPI, HTTPException
@@ -16,6 +17,12 @@ REPO_ROOT = Path(__file__).resolve().parent.parent
 DEFAULT_CONFIG = REPO_ROOT / "default-yt-dlp.conf"
 VERSION_FILE = REPO_ROOT / "youdis-version.txt"
 YTDLP_EXECUTABLE = getenv("YOUDIS_YTDLP_EXECUTABLE", "yt-dlp")
+PREFERRED_CONFIG_ROOT = Path(getenv("YOUDIS_CONFIG_DIR", "/config"))
+PREFERRED_DOWNLOAD_ROOT = Path(getenv("YOUDIS_DOWNLOAD_DIR", "/downloads"))
+LOCAL_RUNTIME_ROOT = REPO_ROOT / ".runtime"
+FALLBACK_CONFIG_ROOT = LOCAL_RUNTIME_ROOT / "config"
+FALLBACK_DOWNLOAD_ROOT = LOCAL_RUNTIME_ROOT / "downloads"
+OUTPUT_TEMPLATE = "%(uploader)s/%(playlist_title)s/%(playlist_index)s%(playlist_index& - )s%(title)s.%(ext)s"
 
 
 @dataclass
@@ -43,11 +50,33 @@ def read_version() -> str:
     return "unknown"
 
 
-def build_ytdlp_command(request: JobRequest) -> list[str]:
+def ensure_writable_directory(preferred: Path, fallback: Path) -> Path:
+    for candidate in (preferred, fallback):
+        try:
+            candidate.mkdir(parents=True, exist_ok=True)
+            with tempfile.NamedTemporaryFile(dir=candidate, prefix=".youdis-write-", delete=True):
+                pass
+            return candidate
+        except OSError:
+            continue
+    raise OSError(f"no writable runtime directory available for {preferred} or {fallback}")
+
+
+def resolve_runtime_paths() -> tuple[Path, Path]:
+    config_root = ensure_writable_directory(PREFERRED_CONFIG_ROOT, FALLBACK_CONFIG_ROOT)
+    download_root = ensure_writable_directory(PREFERRED_DOWNLOAD_ROOT, FALLBACK_DOWNLOAD_ROOT)
+    return config_root, download_root
+
+
+def build_ytdlp_command(request: JobRequest, config_root: Path, download_root: Path) -> list[str]:
     return [
         YTDLP_EXECUTABLE,
         "--config-locations",
         str(DEFAULT_CONFIG),
+        "--download-archive",
+        str(config_root / "archive.txt"),
+        "--output",
+        str(download_root / OUTPUT_TEMPLATE),
         request.url,
     ]
 
@@ -121,13 +150,11 @@ async def finalize_job(job: ManagedJob, returncode: int) -> None:
 
 
 async def run_job(job: ManagedJob, request: JobRequest) -> None:
-    command = build_ytdlp_command(request)
     update_status(
         job,
         state="running",
         phase="starting",
         message="starting yt-dlp",
-        command=command,
     )
 
     try:
@@ -142,6 +169,15 @@ async def run_job(job: ManagedJob, request: JobRequest) -> None:
             await finalize_job(job, 78)
             return
 
+        config_root, download_root = resolve_runtime_paths()
+        command = build_ytdlp_command(request, config_root, download_root)
+        update_status(
+            job,
+            command=command,
+            archive_path=str(config_root / "archive.txt"),
+            result_path=str(download_root),
+        )
+
         process = await asyncio.create_subprocess_exec(
             *command,
             stdout=PIPE,
@@ -157,6 +193,16 @@ async def run_job(job: ManagedJob, request: JobRequest) -> None:
         )
         await finalize_job(job, 127)
         return
+    except OSError as exc:
+        update_status(
+            job,
+            state="failed",
+            phase=None,
+            message=f"runtime path setup failed: {exc}",
+            returncode=73,
+        )
+        await finalize_job(job, 73)
+        return
 
     try:
         job.process = process
@@ -220,7 +266,11 @@ async def health() -> HealthResponse:
 
 @app.get("/version", response_model=VersionResponse)
 async def version() -> VersionResponse:
-    return VersionResponse(version=read_version(), active_job=active_job is not None)
+    return VersionResponse(
+        version=read_version(),
+        active_job=active_job is not None,
+        ytdlp_executable=YTDLP_EXECUTABLE,
+    )
 
 
 @app.get("/jobs/current", response_model=CurrentJobResponse)

youdis/models.py

@@ -26,6 +26,7 @@ class JobStatus(BaseModel):
     requester_name: str | None = None
     origin: str | None = None
     result_path: str | None = None
+    archive_path: str | None = None
     command: list[str] = Field(default_factory=list)
     returncode: int | None = None
     created_at: datetime = Field(default_factory=datetime.utcnow)
@@ -44,3 +45,4 @@ class HealthResponse(BaseModel):
 class VersionResponse(BaseModel):
     version: str
     active_job: bool
+    ytdlp_executable: str