Compare commits

...

7 Commits

Author SHA1 Message Date
ben
18537b9de9 removed duplicate command registration 2026-04-02 17:54:45 -04:00
ben
1757318523 consolidated env vars and .env 2026-04-02 17:14:05 -04:00
ben
90b9dad59c Add configurable adapter selection to app runner 2026-04-02 14:47:03 -04:00
ben
309ce87abb Add app runner for backend and discord stack 2026-04-02 12:41:53 -04:00
ben
4abd39f884 finished discord > adapter functionality 2026-04-02 12:27:31 -04:00
ben
266ed86217 added docstring 2026-04-02 12:27:22 -04:00
ben
0aa9950e18 cleanup discord adapter artifacts 2026-04-02 12:19:56 -04:00
9 changed files with 364 additions and 47 deletions

12
.env.example Normal file
View File

@@ -0,0 +1,12 @@
YOUDIS_ENABLE_BACKEND=1
YOUDIS_BACKEND_HOST=127.0.0.1
YOUDIS_BACKEND_PORT=8000
YOUDIS_BACKEND_HEALTH_TIMEOUT=20
YOUDIS_BACKEND_HEALTH_INTERVAL=0.5
YOUDIS_POLL_INTERVAL_SECONDS=2
YOUDIS_YTDLP_EXECUTABLE=yt-dlp
YOUDIS_CONFIG_DIR=/home/user/proj/youdis/test/
YOUDIS_DOWNLOAD_DIR=/home/user/proj/youdis/downloads
ENABLE_DISCORD=1
DISCORD_BOT_TOKEN=
DISCORD_BOT_SCOPE=2147491904

1
.gitignore vendored
View File

@@ -11,6 +11,7 @@ env/
# --- environment files --- # --- environment files ---
.env .env
.env.* .env.*
!.env.example
*.local *.local
# --- emacs --- # --- emacs ---

View File

@@ -1,10 +1,60 @@
v2 architecture draft: see `docs/architecture-v2.org` # Running youdis
build and run the docker container build and run the docker container
``` ```
api_token = [discord bot token] -v [host_downloads]:/downloads
-v [downloads]:/downloads -v [host_config]:/config
-v [config]:/config -e ENABLE_DISCORD=1
-e DISCORD_BOT_TOKEN=<token>
-e DISCORD_BOT_SCOPE=<scope-int>
``` ```
config contains data to persist across container updates, i.e., unraid appdata, config contains data to persist across container updates, i.e., unraid appdata,
such as users.json (authorized users) and yt-dlp's archive.txt such as yt-dlp's archive.txt
# Development
v2 architecture draft: `docs/architecture-v2.org`
The app runs with `youdis.py`.
This starts the backend first, waits for health, then starts each enabled adapter explicitly.
Test components directly with uvicorn:
```
python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000
python3 -m youdis.adapters.discord
```
Key runner/config vars:
```
YOUDIS_ENABLE_BACKEND=1 #default enabled, dont disable unless testing
YOUDIS_BACKEND_HOST=127.0.0.1
YOUDIS_BACKEND_PORT=8000
YOUDIS_BACKEND_HEALTH_TIMEOUT=20
YOUDIS_BACKEND_HEALTH_INTERVAL=0.5
YOUDIS_POLL_INTERVAL_SECONDS=2
YOUDIS_YTDLP_EXECUTABLE=yt-dlp
YOUDIS_CONFIG_DIR=/path/to/config/
YOUDIS_DOWNLOAD_DIR=/path/to/downloads/
ENABLE_DISCORD=1
DISCORD_BOT_TOKEN=<token-string>
DISCORD_BOT_SCOPE=123456789
```
## Add Adapter
Configure adapter selection in .env, like so:
```
YOUDIS_RUN_BACKEND=1
YOUDIS_BACKEND_HOST=127.0.0.1
YOUDIS_BACKEND_PORT=8000
ENABLE_DISCORD=1
DISCORD_BOT_TOKEN=<api_key>
DISCORD_BOT_SCOPE=123456789
```
to add a new adapter:
1. add `youdis/adapters/<adapter>.py`
2. make it independently runnable
3. add `start_<adapter>()` to `youdis.py`
4. register it in the explicit adapter starter map in `youdis.py`
5. add `ENABLE_<adapter>` and supporting env vars to `.env.example`

View File

@@ -98,7 +98,7 @@ user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/jobs/current
- local dev now falls back to repo-local `.runtime/{config,downloads}` when `/config` or `/downloads` are not writable - local dev now falls back to repo-local `.runtime/{config,downloads}` when `/config` or `/downloads` are not writable
- had to uninstall yt-dlp python pkg from the venv, which resulted in a '403 Forbidden' - had to uninstall yt-dlp python pkg from the venv, which resulted in a '403 Forbidden'
* [ ] 2.0.2: update discord bot to use new backend (3) * [X] 2.0.2: update discord bot to use new backend (3)
update the discord bot into a thin frontend that talks to the backend and verify the flow end to end update the discord bot into a thin frontend that talks to the backend and verify the flow end to end
** pm notes ** pm notes
- this is the first real frontend proof. once this works cleanly, zulip/xmpp should mostly be adapter work rather than downloader rewrites. - this is the first real frontend proof. once this works cleanly, zulip/xmpp should mostly be adapter work rather than downloader rewrites.
@@ -121,7 +121,7 @@ update the discord bot into a thin frontend that talks to the backend and verify
- populate .env with dev env defaults - populate .env with dev env defaults
** evidence ** evidence
- commit: - commit: 5210d2c, 043cb4
- tests: https://youtu.be/20HxMMSqRyg?si=3v7mN2L88c_FxpQR 18m - tests: https://youtu.be/20HxMMSqRyg?si=3v7mN2L88c_FxpQR 18m
1. start backend: `python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000` 1. start backend: `python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000`
2. create local env file: `cp .env.example .env` 2. create local env file: `cp .env.example .env`
@@ -205,10 +205,10 @@ python ./youdis.py
- discord adapter is now a thin HTTP client of the backend; it no longer imports or configures yt-dlp - discord adapter is now a thin HTTP client of the backend; it no longer imports or configures yt-dlp
- `YOUDIS_BACKEND_URL` controls which backend the adapter targets - `YOUDIS_BACKEND_URL` controls which backend the adapter targets
- progress updates are currently implemented by polling `/jobs/current` and DMing only when the summary changes - progress updates are currently implemented by polling `/jobs/current` and DMing only when the summary changes
- legacy auth/user-management commands were removed from the active adapter path and should be cleaned up formally in `2.0.3` - legacy auth/user-management commands were removed from the active adapter path
- `.env` is now supported for local/dev convenience, while real environment variables still override it in prod/docker - `.env` is now supported for local/dev convenience, while real environment variables still override it in prod/docker
- submitting via DM doesn't work - command registration required explicit binding plus `@bot.listen()` listeners in this adapter structure
* [ ] 2.0.3: remove deprecated discord-bot functionality (2) * [X] 2.0.3: remove deprecated discord-bot functionality (2)
delete or retire legacy bot behaviors that no longer fit once the backend split is in place delete or retire legacy bot behaviors that no longer fit once the backend split is in place
** pm notes ** pm notes
- only remove this after the new path works. this is cleanup, not pioneering work. - only remove this after the new path works. this is cleanup, not pioneering work.
@@ -227,13 +227,73 @@ delete or retire legacy bot behaviors that no longer fit once the backend split
- deprecated artifacts are clearly removed or marked - deprecated artifacts are clearly removed or marked
** evidence ** evidence
- commit: - commit: 0aa9950
- tests: - tests:
- datetime: 1. `python3 -m py_compile ./youdis.py ./youdis/adapters/discord.py`
2. `rg -n "users.json|api_token" README.md unraid-ca-template.xml youdis.py youdis/adapters/discord.py`
3. start backend: `python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000`
4. start adapter: `python3 ./youdis.py`
5. in discord, run `/youtube`, `/status`, and `/interrupt` and confirm the adapter still works after cleanup
- datetime: [2026-04-02 Thu 12:09]
** notes ** notes
- active bot path no longer includes local auth or user-management behavior
- top-level `youdis.py` remains as a thin launcher so existing operator habits and scripts do not break during the refactor
- cleanup updated user-facing deployment artifacts to match the v2 naming and architecture, including `DISCORD_BOT_TOKEN` and removal of `users.json` references
- archived planning docs were intentionally left untouched: `pm/tasks.org` is historical and `pm/notes.org` is personal working notes
* [ ] 2.0.4: fix automation and build pipeline (3) * [ ] 2.0.4: make youdis.py app runner for backend + adapters (3)
Build a simple Python orchestration layer in youdis.py so the standard app stack can be launched from one entrypoint while backend and adapters remain independently runnable for testing.
** pm notes
- keep components independently runnable
- youdis.py as app-level orchestrator; let Docker just run python3 /app/youdis.py
- no plugin discovery or hot-loading
- Docker should package and launch the app, not decide internal process topology
- preserve the ability to run backend or adapters directly for debugging
- optimize for one obvious default run path
** Acceptance Criteria
1. define the default app startup path
- youdis.py launches the standard stack for v2
- backend and Discord adapter startup order is explicit
- shutdown behavior is coherent enough for local/dev use
2. preserve modular run paths
- backend can still be run directly
- Discord adapter can still be run directly
- orchestration layer does not bury component-level testing
3. keep orchestration simple
- no dynamic adapter discovery
- no hot reloading framework
- configured components are started explicitly
4. document runtime ownership
- clarify what Python orchestrates vs what Docker orchestrates
- identify env vars or flags that control which components start
- leave room for future Zulip/XMPP adapters without redesigning the runner
** evidence
- commit: 309ce87
- tests:
1. `python3 -m py_compile ./youdis.py ./youdis/adapters/discord.py`
2. backend direct run still works: `python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000`
3. discord direct run still works: `python3 -m youdis.adapters.discord`
4. app runner backend-only smoke test: `ENABLE_DISCORD=0 timeout 5s python3 ./youdis.py`
5. app runner default path: `python3 ./youdis.py`
- date: [2026-04-02 Thu 14:13]
** notes
- `youdis.py` is now the default v2 app runner and starts the standard stack explicitly rather than dynamically discovering adapters
- backend starts first and must pass a health check before the Discord adapter is launched
- backend and Discord adapter remain directly runnable for debugging and tests
- Docker is intended to invoke `python3 /app/youdis.py`; Python owns app orchestration while Docker owns packaging and runtime environment
- adapter startup remains explicit in `youdis.py`, while each adapter gets its own `ENABLE_<ADAPTER>` flag
- adding an adapter means creating `youdis/adapters/<adapter>.py`, adding `start_<adapter>()` in `youdis.py`, wiring its `ENABLE_<ADAPTER>` flag, and documenting any supporting env vars in `.env.example`
- runner flags currently include `YOUDIS_RUN_BACKEND`, `YOUDIS_BACKEND_HOST`, `YOUDIS_BACKEND_PORT`, and `ENABLE_DISCORD`
* ==== BACKLOG ====
Tasks below this line are inactive and should not be touched.
* [ ] 2.0.X: fix automation and build pipeline (3)
repair and simplify the build/update/deploy path so it matches the new backend-plus-frontend structure repair and simplify the build/update/deploy path so it matches the new backend-plus-frontend structure
** pm notes ** pm notes
- this should come after architecture and discord integration stabilize. no point polishing the pipeline for the wrong shape. - this should come after architecture and discord integration stabilize. no point polishing the pipeline for the wrong shape.
@@ -257,3 +317,33 @@ repair and simplify the build/update/deploy path so it matches the new backend-p
- datetime: - datetime:
** notes ** notes
* [ ] X.x.x: clean up discord adapter UI
** acceptance criteria
1. fix interaction pattern so it doesnt time out - prefer "command accepted" or somehting
2. remove all intermediate messages between "accepted/running" and "complete" - /status handles this!
- discord can also output a "busy" signal, research this
3. fix output syntax, we dont need to get crazy with discord cards
** evidence
- commit:
- tests:
- date:
** notes
* [ ] X.x.x: fix youtube -> plex default output
** acceptance criteria
1.
-
** evidence
- commit:
- tests:
- date:
** notes
* /

View File

@@ -8,11 +8,11 @@
<Privileged>false</Privileged> <Privileged>false</Privileged>
<Support>[Unraid Support Thread]</Support> <Support>[Unraid Support Thread]</Support>
<Project>https://github.com/eulaly/youdis</Project> <Project>https://github.com/eulaly/youdis</Project>
<Overview>Discord bot-based wrapper for yt-dlp. Let your friends download videos to your server! Supports playlists, requires a configured Discord bot.</Overview> <Overview>Private yt-dlp worker with a Discord adapter. Submit downloads through Discord while the backend owns yt-dlp execution and job state.</Overview>
<Category>Downloaders: Tools:</Category> <Category>Downloaders: Tools:</Category>
<TemplateURL>https://raw.githubusercontent.com/eulaly/unraid-templates/refs/heads/master/unraid-ca-template.xml</TemplateURL> <TemplateURL>https://raw.githubusercontent.com/eulaly/unraid-templates/refs/heads/master/unraid-ca-template.xml</TemplateURL>
<Icon>https://github.com/eulaly/youdis/blob/c978a2326984efa9670678687ed1a1473478d753/yt_dlp.png</Icon> <Icon>https://github.com/eulaly/youdis/blob/c978a2326984efa9670678687ed1a1473478d753/yt_dlp.png</Icon>
<Config Name="api_token" Target="api_token" Default="" Mode="" Description="Discord bot token" Type="Variable" Display="always" Required="true" Mask="true"/> <Config Name="DISCORD_BOT_TOKEN" Target="DISCORD_BOT_TOKEN" Default="" Mode="" Description="Discord bot token" Type="Variable" Display="always" Required="true" Mask="true"/>
<Config Name="Downloads" Target="/downloads" Default="" Mode="rw" Description="Video download location" Type="Path" Display="always" Required="false"/> <Config Name="Downloads" Target="/downloads" Default="" Mode="rw" Description="Video download location" Type="Path" Display="always" Required="false"/>
<Config Name="Config" Target="/config" Default="/mnt/user/appdata/youdis/config" Mode="rw" Description="Config location (archive.txt, users.json)" Type="Path" Display="always" Required="false"/> <Config Name="Config" Target="/config" Default="/mnt/user/appdata/youdis/config" Mode="rw" Description="Config location (archive.txt and backend runtime data)" Type="Path" Display="always" Required="false"/>
</Container> </Container>

156
youdis.py
View File

@@ -1,6 +1,160 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""App runner for FastAPI backend and discord adapter."""
from youdis.adapters.discord import main from __future__ import annotations
import asyncio
import os
import sys
from dataclasses import dataclass
from pathlib import Path
from urllib.error import URLError
from urllib.request import urlopen
from youdis.env import load_project_dotenv
load_project_dotenv()
REPO_ROOT = Path(__file__).resolve().parent
DEFAULT_BACKEND_HOST = os.getenv("YOUDIS_BACKEND_HOST", "127.0.0.1")
DEFAULT_BACKEND_PORT = int(os.getenv("YOUDIS_BACKEND_PORT", "8000"))
ENABLE_BACKEND = os.getenv("YOUDIS_RUN_BACKEND", "1") not in {"0", "false", "False"}
ENABLE_DISCORD = os.getenv("ENABLE_DISCORD", "1") not in {"0", "false", "False"}
BACKEND_HEALTH_TIMEOUT = float(os.getenv("YOUDIS_BACKEND_HEALTH_TIMEOUT", "20"))
BACKEND_HEALTH_INTERVAL = float(os.getenv("YOUDIS_BACKEND_HEALTH_INTERVAL", "0.5"))
@dataclass
class ManagedProcess:
name: str
process: asyncio.subprocess.Process
def backend_url() -> str:
return f"http://{DEFAULT_BACKEND_HOST}:{DEFAULT_BACKEND_PORT}"
def build_child_env() -> dict[str, str]:
env = os.environ.copy()
env.setdefault("YOUDIS_BACKEND_URL", backend_url())
return env
async def start_backend(env: dict[str, str]) -> ManagedProcess:
process = await asyncio.create_subprocess_exec(
sys.executable,
"-m",
"uvicorn",
"youdis.main:app",
"--host",
DEFAULT_BACKEND_HOST,
"--port",
str(DEFAULT_BACKEND_PORT),
cwd=str(REPO_ROOT),
env=env,
)
return ManagedProcess(name="backend", process=process)
async def start_discord(env: dict[str, str]) -> ManagedProcess:
process = await asyncio.create_subprocess_exec(
sys.executable,
"-m",
"youdis.adapters.discord",
cwd=str(REPO_ROOT),
env=env,
)
return ManagedProcess(name="discord", process=process)
ADAPTER_STARTERS = {
"discord": start_discord,
}
def selected_adapters() -> list[str]:
requested: list[str] = []
if ENABLE_DISCORD:
requested.append("discord")
return requested
async def wait_for_backend_health(timeout_seconds: float) -> None:
deadline = asyncio.get_running_loop().time() + timeout_seconds
health_url = f"{backend_url()}/health"
while True:
try:
with urlopen(health_url, timeout=2) as response:
if response.status == 200:
return
except URLError:
pass
if asyncio.get_running_loop().time() >= deadline:
raise TimeoutError(f"backend did not become healthy at {health_url}")
await asyncio.sleep(BACKEND_HEALTH_INTERVAL)
async def stop_process(proc: ManagedProcess) -> None:
if proc.process.returncode is not None:
return
proc.process.terminate()
try:
await asyncio.wait_for(proc.process.wait(), timeout=10)
except asyncio.TimeoutError:
proc.process.kill()
await proc.process.wait()
async def run() -> int:
managed: list[ManagedProcess] = []
env = build_child_env()
adapters = selected_adapters()
if not ENABLE_BACKEND and not adapters:
print("nothing to start: backend is disabled and no adapters are enabled")
return 1
try:
if ENABLE_BACKEND:
backend = await start_backend(env)
managed.append(backend)
print(f"started backend on {backend_url()}")
await wait_for_backend_health(BACKEND_HEALTH_TIMEOUT)
print("backend is healthy")
for adapter_name in adapters:
process = await ADAPTER_STARTERS[adapter_name](env)
managed.append(process)
print(f"started {adapter_name} adapter")
wait_tasks = [asyncio.create_task(proc.process.wait()) for proc in managed]
done, pending = await asyncio.wait(wait_tasks, return_when=asyncio.FIRST_COMPLETED)
for task in pending:
task.cancel()
first_returncode = 0
for task in done:
first_returncode = task.result()
break
return first_returncode
finally:
for proc in reversed(managed):
await stop_process(proc)
def main() -> None:
try:
raise SystemExit(asyncio.run(run()))
except KeyboardInterrupt:
raise SystemExit(130)
if __name__ == "__main__": if __name__ == "__main__":

View File

@@ -1,3 +1,5 @@
"""Discord adapter for interacting with FastAPI backend worker."""
import asyncio import asyncio
from os import getenv from os import getenv
@@ -10,7 +12,9 @@ from ..env import load_project_dotenv
load_project_dotenv() load_project_dotenv()
BACKEND_URL = getenv("YOUDIS_BACKEND_URL", "http://127.0.0.1:8000").rstrip("/") BACKEND_HOST = getenv("YOUDIS_BACKEND_HOST", "127.0.0.1")
BACKEND_PORT = int(getenv("YOUDIS_BACKEND_PORT", "8000"))
BACKEND_URL = f"http://{BACKEND_HOST}:{BACKEND_PORT}".rstrip("/")
POLL_INTERVAL_SECONDS = float(getenv("YOUDIS_POLL_INTERVAL_SECONDS", "2")) POLL_INTERVAL_SECONDS = float(getenv("YOUDIS_POLL_INTERVAL_SECONDS", "2"))
DEFAULT_SCOPE = int(getenv("DISCORD_BOT_SCOPE", "2147491904")) DEFAULT_SCOPE = int(getenv("DISCORD_BOT_SCOPE", "2147491904"))
@@ -112,28 +116,6 @@ def ensure_poll_task(ctx: interactions.SlashContext, job_id: str) -> None:
return return
poll_tasks[job_id] = asyncio.create_task(poll_job_updates(ctx, job_id)) poll_tasks[job_id] = asyncio.create_task(poll_job_updates(ctx, job_id))
@bot.listen()
async def on_startup():
await get_session()
print(f"discord adapter configured for backend {BACKEND_URL}")
print(f"discord adapter default scope: {DEFAULT_SCOPE}")
print(f"discord adapter command cache keys: {sorted(bot._interaction_lookup.keys())}")
@bot.listen()
async def on_ready():
print(f"registered commands: {bot.application_commands}")
@bot.listen()
async def on_shutdown():
global http_session
for task in list(poll_tasks.values()):
task.cancel()
poll_tasks.clear()
if http_session is not None and not http_session.closed:
await http_session.close()
http_session = None
@interactions.slash_command(name="youtube", description="submit a youtube download to the backend") @interactions.slash_command(name="youtube", description="submit a youtube download to the backend")
@interactions.slash_option( @interactions.slash_option(
name="url", name="url",
@@ -217,12 +199,33 @@ async def status(ctx: interactions.SlashContext):
prefix = "active" if active else "last" prefix = "active" if active else "last"
await dm(ctx, f"{prefix} job: {format_status_message(job)}") await dm(ctx, f"{prefix} job: {format_status_message(job)}")
@bot.listen()
async def on_startup():
await get_session()
print(f"discord adapter configured for backend {BACKEND_URL}")
print(f"{bot.application_commands.count} registered commands:")
for i, x in enumerate(bot.application_commands,start=1):
print(f" {i}. {x.name}")
@bot.listen()
async def on_shutdown():
global http_session
for task in list(poll_tasks.values()):
task.cancel()
poll_tasks.clear()
if http_session is not None and not http_session.closed:
await http_session.close()
http_session = None
def main() -> None: def main() -> None:
api_token = getenv("DISCORD_BOT_TOKEN") api_token = getenv("DISCORD_BOT_TOKEN")
if not api_token: if not api_token:
raise ValueError("API token not set. Retrieve from your Discord bot.") raise ValueError("API token not set. Retrieve from your Discord bot.")
bot.add_command(youtube) # bot.add_command(youtube)
bot.add_command(status) # bot.add_command(status)
bot.add_command(interrupt) # bot.add_command(interrupt)
bot.start(api_token) bot.start(api_token)
if __name__ == "__main__":
main()

View File

@@ -1,8 +1,13 @@
from pathlib import Path from pathlib import Path
try:
from dotenv import load_dotenv from dotenv import load_dotenv
except ModuleNotFoundError: # pragma: no cover - optional local convenience dependency
load_dotenv = None
def load_project_dotenv() -> None: def load_project_dotenv() -> None:
if load_dotenv is None:
return
repo_root = Path(__file__).resolve().parent.parent repo_root = Path(__file__).resolve().parent.parent
load_dotenv(repo_root / ".env", override=False) load_dotenv(repo_root / ".env", override=False)

View File

@@ -1,3 +1,5 @@
"""FastAPI backend worker managing yt-dlp subprocess jobs and exposing job state."""
import asyncio import asyncio
from asyncio.subprocess import PIPE, STDOUT from asyncio.subprocess import PIPE, STDOUT
from collections import deque from collections import deque