Compare commits

..

12 Commits

Author SHA1 Message Date
ben
18537b9de9 removed duplicate command registration 2026-04-02 17:54:45 -04:00
ben
1757318523 consolidated env vars and .env 2026-04-02 17:14:05 -04:00
ben
90b9dad59c Add configurable adapter selection to app runner 2026-04-02 14:47:03 -04:00
ben
309ce87abb Add app runner for backend and discord stack 2026-04-02 12:41:53 -04:00
ben
4abd39f884 finished discord > adapter functionality 2026-04-02 12:27:31 -04:00
ben
266ed86217 added docstring 2026-04-02 12:27:22 -04:00
ben
0aa9950e18 cleanup discord adapter artifacts 2026-04-02 12:19:56 -04:00
ben
043c8b4df3 implemented discord adapter 2026-04-02 11:57:54 -04:00
ben
5210d2c032 moved discord functionality into adapter 2026-04-02 10:53:48 -04:00
ben
8b21034d78 configured initial yt-dlp worker fallback download location 2026-03-31 21:20:46 -04:00
ben
2a5648506e initial ytdlp worker 2026-03-31 20:54:56 -04:00
ben
7926534e54 initial ytdlp worker 2026-03-31 20:54:23 -04:00
15 changed files with 1165 additions and 303 deletions

12
.env.example Normal file
View File

@@ -0,0 +1,12 @@
YOUDIS_ENABLE_BACKEND=1
YOUDIS_BACKEND_HOST=127.0.0.1
YOUDIS_BACKEND_PORT=8000
YOUDIS_BACKEND_HEALTH_TIMEOUT=20
YOUDIS_BACKEND_HEALTH_INTERVAL=0.5
YOUDIS_POLL_INTERVAL_SECONDS=2
YOUDIS_YTDLP_EXECUTABLE=yt-dlp
YOUDIS_CONFIG_DIR=/home/user/proj/youdis/test/
YOUDIS_DOWNLOAD_DIR=/home/user/proj/youdis/downloads
ENABLE_DISCORD=1
DISCORD_BOT_TOKEN=
DISCORD_BOT_SCOPE=2147491904

70
.gitignore vendored
View File

@@ -1,34 +1,36 @@
# --- python bytecode --- # --- python bytecode ---
__pycache__/ __pycache__/
*.py[cod] *.py[cod]
*$py.class *$py.class
# --- virtual environments --- # --- virtual environments ---
.venv/ .venv/
venv/ venv/
env/ env/
# --- environment files --- # --- environment files ---
.env .env
.env.* .env.*
*.local !.env.example
*.local
# --- emacs ---
*~ # --- emacs ---
\#*\# *~
.\#* \#*\#
*.elc .\#*
*.elc
# --- project private data ---
/private/ # --- project private data ---
archive/ /private/
downloads/ archive/
data.json downloads/
test/
# --- django --- data.json
db.sqlite3
staticfiles/ # --- django ---
media/ db.sqlite3
staticfiles/
# --- misc --- media/
.DS_Store
# --- misc ---
.DS_Store

View File

@@ -1,10 +1,60 @@
v2 architecture draft: see `docs/architecture-v2.org` # Running youdis
build and run the docker container build and run the docker container
``` ```
api_token = [discord bot token] -v [host_downloads]:/downloads
-v [downloads]:/downloads -v [host_config]:/config
-v [config]:/config -e ENABLE_DISCORD=1
``` -e DISCORD_BOT_TOKEN=<token>
config contains data to persist across container updates, i.e., unraid appdata, -e DISCORD_BOT_SCOPE=<scope-int>
such as users.json (authorized users) and yt-dlp's archive.txt ```
config contains data to persist across container updates, i.e., unraid appdata,
such as yt-dlp's archive.txt
# Development
v2 architecture draft: `docs/architecture-v2.org`
The app runs with `youdis.py`.
This starts the backend first, waits for health, then starts each enabled adapter explicitly.
Test components directly with uvicorn:
```
python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000
python3 -m youdis.adapters.discord
```
Key runner/config vars:
```
YOUDIS_ENABLE_BACKEND=1 #default enabled, dont disable unless testing
YOUDIS_BACKEND_HOST=127.0.0.1
YOUDIS_BACKEND_PORT=8000
YOUDIS_BACKEND_HEALTH_TIMEOUT=20
YOUDIS_BACKEND_HEALTH_INTERVAL=0.5
YOUDIS_POLL_INTERVAL_SECONDS=2
YOUDIS_YTDLP_EXECUTABLE=yt-dlp
YOUDIS_CONFIG_DIR=/path/to/config/
YOUDIS_DOWNLOAD_DIR=/path/to/downloads/
ENABLE_DISCORD=1
DISCORD_BOT_TOKEN=<token-string>
DISCORD_BOT_SCOPE=123456789
```
## Add Adapter
Configure adapter selection in .env, like so:
```
YOUDIS_RUN_BACKEND=1
YOUDIS_BACKEND_HOST=127.0.0.1
YOUDIS_BACKEND_PORT=8000
ENABLE_DISCORD=1
DISCORD_BOT_TOKEN=<api_key>
DISCORD_BOT_SCOPE=123456789
```
to add a new adapter:
1. add `youdis/adapters/<adapter>.py`
2. make it independently runnable
3. add `start_<adapter>()` to `youdis.py`
4. register it in the explicit adapter starter map in `youdis.py`
5. add `ENABLE_<adapter>` and supporting env vars to `.env.example`

View File

@@ -1,11 +1,8 @@
# yt-dlp config file (yt-dlp.conf or .config/yt-dlp/config) # yt-dlp config file
--simulate -f "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best"
-f "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best" --embed-chapters
--embed-chapters --embed-info-json
--embed-info-json --write-playlist-metafiles
--write-playlist-metafiles --download-archive "/config/archive.txt"
#--paths "{"home":"./downloads"}" --restrict-filenames
--download-archive "/config/archive.txt" --output "/downloads/%(uploader)s/%(playlist_title)s/%(playlist_index)s%(playlist_index& - )s%(title)s.%(ext)s"
--restrict-filenames
--output "/downloads/%(uploader)s/%(playlist_title)s/%(playlist_index)s%(playlist_index& - )s%(title)s.%(ext)s"
# --split-chapters

View File

@@ -2,15 +2,17 @@
#+updated: [2026-03-31 Tue 16:03] #+updated: [2026-03-31 Tue 16:03]
Use the template below, which should be a top-level org-mode header. Use the template below, which should be a top-level org-mode header.
A sample task is below the template
* [ ] M.m.m: Task Title (# commits) * [ ] M.m.m: Task Title (<estimated # of commits>)
description of the task description of the task
** pm notes: amplifying information ** pm notes: amplifying information
** Acceptance Criteria ** Acceptance Criteria
1. Criterion 1. Criterion
- expanded data - expanded data
2. Criterion 2. Criterion
3f. Criterion added after initial task completion
** evidence ** evidence
- commit: abc123, bcd234 - commit: abc123, bcd234
@@ -19,3 +21,61 @@ description of the task
** notes ** notes
- explanation of work done, decisions made, reasoning - explanation of work done, decisions made, reasoning
* [ ] 2.0.1: build backend yt-dlp worker (3)
create the minimal backend/service skeleton and establish a working yt-dlp baseline with clean hooks for future frontends
** pm notes
- foundation; don't need the full finished service here, just the basic shape plus enough real yt-dlp execution to validate the seam and build on it.
- keep single-job semantics
- prioritize inspectable behavior over polish
** Acceptance Criteria
1. create the backend/service skeleton
- app/module layout exists
- core request path is stubbed or minimally working
2. establish a working yt-dlp baseline
- archive behavior is preserved
- output path behavior is preserved or intentionally updated
- use yt-dlp .conf and set reasonable default
3. expose basic hooks/interfaces for future frontends
- submit/request path exists
- status/progress hook exists
- basic health/version visibility exists
4f. print env vars to stdout on fastapi launch
- repo_root
- default_config
- ytdlp_executable
5f. create ytdlp conf to facilitate tests
** evidence
- commit:
- tests:
1. `python3 -m py_compile ./youdis/main.py ./youdis/models.py ./youdis/adapters/__init__.py ./youdis/adapters/discord.py`
2. `python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000`
3. `sudo curl -L https://github.com/yt-dlp/yt-dlp/releases/download/2026.03.17/yt-dlp -o /usr/local/bin/yt-dlp`
4. `sudo chmod +x /usr/local/bin/yt-dlp`
5. `yt-dlp --version`
6. `curl http://127.0.0.1:8000/health`
7. `curl http://127.0.0.1:8000/version`
8. `curl -X POST http://127.0.0.1:8000/jobs -H 'content-type: application/json' -d '{"url":"https://www.youtube.com/watch?v=dQw4w9WgXcQ"}'`
9. `curl http://127.0.0.1:8000/jobs/current`
10. if testing in container: `docker build -t youdis:v2 .`
11. if testing in container: `docker run --rm -p 8000:8000 -v [config]:/config -v [downloads]:/downloads youdis:v2`
:OUTPUT_STEP1-9:
user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/health
{"status":"ok"}user@paladin:~/proj/youdis$
user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/version
{"version":"20250829-ec72c56","active_job":false}user@paladin:~/proj/youdis$
user@paladin:~/proj/youdis$ curl -X POST http://127.0.0.1:8000/jobs -H 'content-type: application/json' -d '{"url":"https://www.youtube.com/watch?v=3i72yY_LaW4"}'
{"job_id":"cc85165e-d906-4eee-864f-1a398b6de2e0","state":"accepted","url":"https://www.youtube.com/watch?v=3i72yY_LaW4","message":"accepted","phase":"queued","disposition":null,"requester_id":null,"requester_name":null,"origin":null,"result_path":null,"command":[],"returncode":null,"created_at":"2026-04-01T00:44:35.657196","updated_at":"2026-04-01T00:44:35.657198"}user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/jobs/current
{"active":false,"job":{"job_id":"cc85165e-d906-4eee-864f-1a398b6de2e0","state":"failed","url":"https://www.youtube.com/watch?v=3i72yY_LaW4","message":"ERROR: unable to create directory [Errno 13] Permission denied: '/downloads'","phase":null,"disposition":null,"requester_id":null,"requester_name":null,"origin":null,"result_path":null,"command":["yt-dlp","--config-locations","/home/user/proj/youdis/default-yt-dlp.conf","https://www.youtube.com/watch?v=3i72yY_LaW4"],"returncode":1,"created_at":"2026-04-01T00:44:35.657196","updated_at":"2026-03-31T20:44:36.653353"}}user@paladin:~/proj/youdis$
:END:
- datetime: [2026-03-31 Tue 20:45]
** notes
- validate `yt-dlp` subprocess invocation in-container; not verifiable in the current shell because `yt-dlp` is not installed here
- confirm `--config-locations` behavior against the installed `yt-dlp` version during integration testing
- current backend scaffold is not yet wired into `dockerfile` or `run-youdis.sh`
- archive-hit and result-path parsing currently depend on `yt-dlp` stdout text patterns, so treat them as provisional until integration-tested

View File

@@ -1,5 +1,10 @@
#+title: Task Log #+title: Task Log
#+updated: [2026-03-31 Tue 16:03] #+updated: [2026-03-31 Tue 16:03]
#+startup: overview
# Local Variables:
# org-babel-python-command: "/home/user/proj/youdis/.venv/bin/python"
# project-dir: "~/proj/youdis"
# End:
* youdis v2 goals: * youdis v2 goals:
1. Separate backend from frontend 1. Separate backend from frontend
@@ -8,7 +13,7 @@
4. Default output format supports plex browsing youtube channels as "tv shows" 4. Default output format supports plex browsing youtube channels as "tv shows"
5. Facilitate multiple GUI inbounds: Discord, Zulip, XMPP 5. Facilitate multiple GUI inbounds: Discord, Zulip, XMPP
* [ ] 2.0.0: define architecture (2-4) * [X] 2.0.0: define architecture (2-4)
define the target architecture for a private backend yt-dlp worker with thin chat frontends define the target architecture for a private backend yt-dlp worker with thin chat frontends
** pm notes: ** pm notes:
- keep this iterative. the point is to choose the shape and seam, not prematurely implement infra. likely decisions include backend framework, request/status model, and how thin the discord shim should be. - keep this iterative. the point is to choose the shape and seam, not prematurely implement infra. likely decisions include backend framework, request/status model, and how thin the discord shim should be.
@@ -28,14 +33,14 @@ define the target architecture for a private backend yt-dlp worker with thin cha
- next implementation task is unblocked - next implementation task is unblocked
** evidence ** evidence
- commit: - commit: 0ed16ec
- tests: - tests: n/a
- datetime: - datetime: [2026-03-31 Tue 18:48]
** notes ** notes
- first architecture draft captured in `docs/architecture-v2.org` - first architecture draft captured in `docs/architecture-v2.org`
* [ ] 2.0.1: build backend yt-dlp worker (3) * [X] 2.0.1: build backend yt-dlp worker (3)
create the minimal backend/service skeleton and establish a working yt-dlp baseline with clean hooks for future frontends create the minimal backend/service skeleton and establish a working yt-dlp baseline with clean hooks for future frontends
** pm notes ** pm notes
- foundation; don't need the full finished service here, just the basic shape plus enough real yt-dlp execution to validate the seam and build on it. - foundation; don't need the full finished service here, just the basic shape plus enough real yt-dlp execution to validate the seam and build on it.
@@ -54,15 +59,46 @@ create the minimal backend/service skeleton and establish a working yt-dlp basel
- submit/request path exists - submit/request path exists
- status/progress hook exists - status/progress hook exists
- basic health/version visibility exists - basic health/version visibility exists
4. make local testing practical without breaking container defaults
- backend can run when `/config` or `/downloads` are not writable in local dev
- env vars may override executable/config/download paths
- status makes the effective runtime command/paths inspectable
** evidence ** evidence
- commit: - commit: 7926534,2a56485
- tests: - tests:
- datetime: 1. `python3 -m py_compile ./youdis/main.py ./youdis/models.py ./youdis/adapters/__init__.py ./youdis/adapters/discord.py`
2. `python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000`
3. `sudo curl -L https://github.com/yt-dlp/yt-dlp/releases/download/2026.03.17/yt-dlp -o /usr/local/bin/yt-dlp`
4. `sudo chmod +x /usr/local/bin/yt-dlp`
5. `yt-dlp --version`
6. `curl http://127.0.0.1:8000/health`
7. `curl http://127.0.0.1:8000/version`
8. `curl -X POST http://127.0.0.1:8000/jobs -H 'content-type: application/json' -d '{"url":"https://www.youtube.com/watch?v=dQw4w9WgXcQ"}'`
9. `curl http://127.0.0.1:8000/jobs/current`
10. if testing in container: `docker build -t youdis:v2 .`
11. if testing in container: `docker run --rm -p 8000:8000 -v [config]:/config -v [downloads]:/downloads youdis:v2`
:OUTPUT_STEP1-9:
user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/health
{"status":"ok"}
user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/version
{"version":"20250829-ec72c56","active_job":false}
user@paladin:~/proj/youdis$ curl -X POST http://127.0.0.1:8000/jobs -H 'content-type: application/json' -d '{"url":"https://www.youtube.com/watch?v=3i72yY_LaW4"}'
{"job_id":"cc85165e-d906-4eee-864f-1a398b6de2e0","state":"accepted","url":"https://www.youtube.com/watch?v=3i72yY_LaW4","message":"accepted","phase":"queued","disposition":null,"requester_id":null,"requester_name":null,"origin":null,"result_path":null,"command":[],"returncode":null,"created_at":"2026-04-01T00:44:35.657196","updated_at":"2026-04-01T00:44:35.657198"}
user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/jobs/current
{"active":false,"job":{"job_id":"cc85165e-d906-4eee-864f-1a398b6de2e0","state":"failed","url":"https://www.youtube.com/watch?v=3i72yY_LaW4","message":"ERROR: unable to create directory [Errno 13] Permission denied: '/downloads'","phase":null,"disposition":null,"requester_id":null,"requester_name":null,"origin":null,"result_path":null,"command":["yt-dlp","--config-locations","/home/user/proj/youdis/default-yt-dlp.conf","https://www.youtube.com/watch?v=3i72yY_LaW4"],"returncode":1,"created_at":"2026-04-01T00:44:35.657196","updated_at":"2026-03-31T20:44:36.653353"}}user@paladin:~/proj/youdis$
:END:
- datetime: [2026-03-31 Tue 20:45]
** notes ** notes
- validate `yt-dlp` subprocess invocation in-container; not verifiable in the current shell because `yt-dlp` is not installed here
* [ ] 2.0.2: update discord bot to use new backend (3) - confirm `--config-locations` behavior against the installed `yt-dlp` version during integration testing
- current backend scaffold is not yet wired into `dockerfile` or `run-youdis.sh`
- archive-hit and result-path parsing currently depend on `yt-dlp` stdout text patterns, so treat them as provisional until integration-tested
- local dev now falls back to repo-local `.runtime/{config,downloads}` when `/config` or `/downloads` are not writable
- had to uninstall yt-dlp python pkg from the venv, which resulted in a '403 Forbidden'
* [X] 2.0.2: update discord bot to use new backend (3)
update the discord bot into a thin frontend that talks to the backend and verify the flow end to end update the discord bot into a thin frontend that talks to the backend and verify the flow end to end
** pm notes ** pm notes
- this is the first real frontend proof. once this works cleanly, zulip/xmpp should mostly be adapter work rather than downloader rewrites. - this is the first real frontend proof. once this works cleanly, zulip/xmpp should mostly be adapter work rather than downloader rewrites.
@@ -80,15 +116,99 @@ update the discord bot into a thin frontend that talks to the backend and verify
- valid request path tested - valid request path tested
- busy or conflict behavior tested - busy or conflict behavior tested
- failure path tested - failure path tested
4. add dotenv support to ease dev
- os.getenv methods remain standard for prod/docker build
- populate .env with dev env defaults
** evidence ** evidence
- commit: - commit: 5210d2c, 043cb4
- tests: - tests: https://youtu.be/20HxMMSqRyg?si=3v7mN2L88c_FxpQR 18m
- datetime: 1. start backend: `python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000`
2. create local env file: `cp .env.example .env`
3. add `api_token` to `.env`
4. start adapter: `python3 ./youdis.py`
5. in discord, run `/youtube url:https://www.youtube.com/watch?v=dQw4w9WgXcQ`
:out:
#+begin_src shell
(venv) user@paladin:~/proj/youdis$ python ./youdis.py
Task exception was never retrieved
future: <Task finished name='Task-43' coro=<Client._dispatch_interaction() done, defined at /home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/client/client.py:1773> exception=KeyError('youtube')>
Traceback (most recent call last):
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/client/client.py", line 1798, in _dispatch_interaction
if ctx.command:
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/models/internal/context.py", line 329, in command
return self.client._interaction_lookup[self._command_name]
KeyError: 'youtube'
#+end_src
:end:
6. confirm channel response says the job was submitted to backend
7. confirm requester receives DM updates for accepted/running/completed or failed
:out:
accepted job 4ef6fde5-57cc-478c-b0e2-18458c693fa4 for https://www.youtube.com/watch?v=dQw4w9WgXcQ
state=running | phase=running | [youtube] dQw4w9WgXcQ: Downloading webpage | path=/home/user/proj/youdis/downloads
state=running | phase=downloading | [download] Destination: /home/user/proj/youdis/downloads/Rick_Astley/NA/NANARickAstley-_Never_Gonna_Give_You_Up_Official_Video_4K_Remaster.f401.mp4 | path=/home/user/proj/youdis/downloads/Rick_Astley/NA/NANARickAstley-_Never_Gonna_Give_You_Up_Official_Video_4K_Remaster.f401.mp4
state=completed | [Metadata] There isn't any metadata to add | path=/home/user/proj/youdis/downloads/Rick_Astley/NA/NANARickAstley-_Never_Gonna_Give_You_Up_Official_Video_4K_Remaster.mp4
:end:
8. while first job is active, submit another `/youtube` and confirm busy behavior
:out:
#+begin_src shell
Error: AttributeError
Traceback (most recent call last):
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/client/client.py", line 1900, in __dispatch_interaction
response = await callback
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/client/client.py", line 1771, in _run_slash_command
return await command(ctx, **ctx.kwargs)
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/models/internal/command.py", line 132, in __call__
await self.call_callback(self.callback, context)
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/models/internal/application_commands.py", line 833, in call_callback
return await self.call_with_binding(callback, ctx, *new_args, **new_kwargs)
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/models/internal/callback.py", line 43, in call_with_binding
return await callback(*args, **kwargs)
File "/home/user/proj/youdis/youdis/adapters/discord.py", line 167, in youtube
await ctx.channel.send(f"Submitted <{url}> to the backend. Status updates via DM.")
AttributeError: 'NoneType' object has no attribute 'send'
#+end_src
busy: busy with https://youtu.be/20HxMMSqRyg?si=3v7mN2L88c_FxpQR
state=running | phase=running | cancel requested | path=/home/user/proj/youdis/downloads/James_Hoffmann/NA/NANAThe_Beginner_s_Guide_To_Latte_Art.f401.mp4
:end:
9. run `/status` and confirm it reflects current or last backend job
:out:
last job: state=completed | [Metadata] There isn't any metadata to add | path=/home/user/proj/youdis/downloads/Rick_Astley/NA/NANARickAstley-_Never_Gonna_Give_You_Up_Official_Video_4K_Remaster.mp4
:end:
10. run `/interrupt` as owner and confirm cancellation is surfaced via DM
:out:
last job: state=cancelled | cancelled | path=/home/user/proj/youdis/downloads/James_Hoffmann/NA/NANAThe_Beginner_s_Guide_To_Latte_Art.f401.mp4
:end:
- datetime:[2026-04-02 Thu 11:52]
*** org-block tests
#+begin_src shell :dir ~/proj/youdis :results output verbatim
source ./venv/bin/activate
python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000
#+end_src
#+begin_src shell :results output
echo ok
#+end_src
#+RESULTS:
: ok
#+begin_src python :dir ~/proj/youdis :results output verbatim
python ./youdis.py
#+end_src
#+RESULTS:
** notes ** notes
- discord adapter is now a thin HTTP client of the backend; it no longer imports or configures yt-dlp
* [ ] 2.0.3: remove deprecated discord-bot functionality (2) - `YOUDIS_BACKEND_URL` controls which backend the adapter targets
- progress updates are currently implemented by polling `/jobs/current` and DMing only when the summary changes
- legacy auth/user-management commands were removed from the active adapter path
- `.env` is now supported for local/dev convenience, while real environment variables still override it in prod/docker
- command registration required explicit binding plus `@bot.listen()` listeners in this adapter structure
* [X] 2.0.3: remove deprecated discord-bot functionality (2)
delete or retire legacy bot behaviors that no longer fit once the backend split is in place delete or retire legacy bot behaviors that no longer fit once the backend split is in place
** pm notes ** pm notes
- only remove this after the new path works. this is cleanup, not pioneering work. - only remove this after the new path works. this is cleanup, not pioneering work.
@@ -107,13 +227,73 @@ delete or retire legacy bot behaviors that no longer fit once the backend split
- deprecated artifacts are clearly removed or marked - deprecated artifacts are clearly removed or marked
** evidence ** evidence
- commit: - commit: 0aa9950
- tests: - tests:
- datetime: 1. `python3 -m py_compile ./youdis.py ./youdis/adapters/discord.py`
2. `rg -n "users.json|api_token" README.md unraid-ca-template.xml youdis.py youdis/adapters/discord.py`
3. start backend: `python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000`
4. start adapter: `python3 ./youdis.py`
5. in discord, run `/youtube`, `/status`, and `/interrupt` and confirm the adapter still works after cleanup
- datetime: [2026-04-02 Thu 12:09]
** notes ** notes
- active bot path no longer includes local auth or user-management behavior
- top-level `youdis.py` remains as a thin launcher so existing operator habits and scripts do not break during the refactor
- cleanup updated user-facing deployment artifacts to match the v2 naming and architecture, including `DISCORD_BOT_TOKEN` and removal of `users.json` references
- archived planning docs were intentionally left untouched: `pm/tasks.org` is historical and `pm/notes.org` is personal working notes
* [ ] 2.0.5: fix automation and build pipeline (3) * [ ] 2.0.4: make youdis.py app runner for backend + adapters (3)
Build a simple Python orchestration layer in youdis.py so the standard app stack can be launched from one entrypoint while backend and adapters remain independently runnable for testing.
** pm notes
- keep components independently runnable
- youdis.py as app-level orchestrator; let Docker just run python3 /app/youdis.py
- no plugin discovery or hot-loading
- Docker should package and launch the app, not decide internal process topology
- preserve the ability to run backend or adapters directly for debugging
- optimize for one obvious default run path
** Acceptance Criteria
1. define the default app startup path
- youdis.py launches the standard stack for v2
- backend and Discord adapter startup order is explicit
- shutdown behavior is coherent enough for local/dev use
2. preserve modular run paths
- backend can still be run directly
- Discord adapter can still be run directly
- orchestration layer does not bury component-level testing
3. keep orchestration simple
- no dynamic adapter discovery
- no hot reloading framework
- configured components are started explicitly
4. document runtime ownership
- clarify what Python orchestrates vs what Docker orchestrates
- identify env vars or flags that control which components start
- leave room for future Zulip/XMPP adapters without redesigning the runner
** evidence
- commit: 309ce87
- tests:
1. `python3 -m py_compile ./youdis.py ./youdis/adapters/discord.py`
2. backend direct run still works: `python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000`
3. discord direct run still works: `python3 -m youdis.adapters.discord`
4. app runner backend-only smoke test: `ENABLE_DISCORD=0 timeout 5s python3 ./youdis.py`
5. app runner default path: `python3 ./youdis.py`
- date: [2026-04-02 Thu 14:13]
** notes
- `youdis.py` is now the default v2 app runner and starts the standard stack explicitly rather than dynamically discovering adapters
- backend starts first and must pass a health check before the Discord adapter is launched
- backend and Discord adapter remain directly runnable for debugging and tests
- Docker is intended to invoke `python3 /app/youdis.py`; Python owns app orchestration while Docker owns packaging and runtime environment
- adapter startup remains explicit in `youdis.py`, while each adapter gets its own `ENABLE_<ADAPTER>` flag
- adding an adapter means creating `youdis/adapters/<adapter>.py`, adding `start_<adapter>()` in `youdis.py`, wiring its `ENABLE_<ADAPTER>` flag, and documenting any supporting env vars in `.env.example`
- runner flags currently include `YOUDIS_RUN_BACKEND`, `YOUDIS_BACKEND_HOST`, `YOUDIS_BACKEND_PORT`, and `ENABLE_DISCORD`
* ==== BACKLOG ====
Tasks below this line are inactive and should not be touched.
* [ ] 2.0.X: fix automation and build pipeline (3)
repair and simplify the build/update/deploy path so it matches the new backend-plus-frontend structure repair and simplify the build/update/deploy path so it matches the new backend-plus-frontend structure
** pm notes ** pm notes
- this should come after architecture and discord integration stabilize. no point polishing the pipeline for the wrong shape. - this should come after architecture and discord integration stabilize. no point polishing the pipeline for the wrong shape.
@@ -137,3 +317,33 @@ repair and simplify the build/update/deploy path so it matches the new backend-p
- datetime: - datetime:
** notes ** notes
* [ ] X.x.x: clean up discord adapter UI
** acceptance criteria
1. fix interaction pattern so it doesnt time out - prefer "command accepted" or somehting
2. remove all intermediate messages between "accepted/running" and "complete" - /status handles this!
- discord can also output a "busy" signal, research this
3. fix output syntax, we dont need to get crazy with discord cards
** evidence
- commit:
- tests:
- date:
** notes
* [ ] X.x.x: fix youtube -> plex default output
** acceptance criteria
1.
-
** evidence
- commit:
- tests:
- date:
** notes
* /

Binary file not shown.

View File

@@ -8,11 +8,11 @@
<Privileged>false</Privileged> <Privileged>false</Privileged>
<Support>[Unraid Support Thread]</Support> <Support>[Unraid Support Thread]</Support>
<Project>https://github.com/eulaly/youdis</Project> <Project>https://github.com/eulaly/youdis</Project>
<Overview>Discord bot-based wrapper for yt-dlp. Let your friends download videos to your server! Supports playlists, requires a configured Discord bot.</Overview> <Overview>Private yt-dlp worker with a Discord adapter. Submit downloads through Discord while the backend owns yt-dlp execution and job state.</Overview>
<Category>Downloaders: Tools:</Category> <Category>Downloaders: Tools:</Category>
<TemplateURL>https://raw.githubusercontent.com/eulaly/unraid-templates/refs/heads/master/unraid-ca-template.xml</TemplateURL> <TemplateURL>https://raw.githubusercontent.com/eulaly/unraid-templates/refs/heads/master/unraid-ca-template.xml</TemplateURL>
<Icon>https://github.com/eulaly/youdis/blob/c978a2326984efa9670678687ed1a1473478d753/yt_dlp.png</Icon> <Icon>https://github.com/eulaly/youdis/blob/c978a2326984efa9670678687ed1a1473478d753/yt_dlp.png</Icon>
<Config Name="api_token" Target="api_token" Default="" Mode="" Description="Discord bot token" Type="Variable" Display="always" Required="true" Mask="true"/> <Config Name="DISCORD_BOT_TOKEN" Target="DISCORD_BOT_TOKEN" Default="" Mode="" Description="Discord bot token" Type="Variable" Display="always" Required="true" Mask="true"/>
<Config Name="Downloads" Target="/downloads" Default="" Mode="rw" Description="Video download location" Type="Path" Display="always" Required="false"/> <Config Name="Downloads" Target="/downloads" Default="" Mode="rw" Description="Video download location" Type="Path" Display="always" Required="false"/>
<Config Name="Config" Target="/config" Default="/mnt/user/appdata/youdis/config" Mode="rw" Description="Config location (archive.txt, users.json)" Type="Path" Display="always" Required="false"/> <Config Name="Config" Target="/config" Default="/mnt/user/appdata/youdis/config" Mode="rw" Description="Config location (archive.txt and backend runtime data)" Type="Path" Display="always" Required="false"/>
</Container> </Container>

382
youdis.py
View File

@@ -1,235 +1,161 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
''' """App runner for FastAPI backend and discord adapter."""
youdis v1.1
bot for downloading youtube videos using yt-dlp from __future__ import annotations
discord-py-interactions 5.11.0 has new option
requires python>=3.9
'''
# match_filter: info_dict -> Raise utils.DownloadCancelled(msg) ? interrupt
import interactions
from os import getenv
from pathlib import Path
import yt_dlp
import json
import asyncio import asyncio
import threading import os
import sys
from dataclasses import dataclass
from pathlib import Path
from urllib.error import URLError
from urllib.request import urlopen
userFile = Path('/config/users.json') from youdis.env import load_project_dotenv
userFile.parent.mkdir(exist_ok=True, parents=True)
bot = interactions.Client(intents=interactions.Intents.DEFAULT,default_scope=2147491904)
def save_authorized_users(authorized_users):
with open(userFile, 'w') as f:
json.dump({'authorized_users': authorized_users}, f)
def load_authorized_users():
if not userFile.exists():
save_authorized_users([])
print(f'users.json not found; saving to {userFile}')
return []
try:
with open(userFile, 'r') as f:
data = json.load(f)
except (json.JSONDecodeError, OSError):
save_authorized_users([])
print(f'users.json invalid; resetting {userFile}')
return []
authorized_users = data.get('authorized_users', [])
if not isinstance(authorized_users, list):
authorized_users = []
authorized_users = [str(user_id) for user_id in authorized_users]
save_authorized_users(authorized_users)
print(f'authorized_users:{authorized_users}')
return authorized_users
authorized_users = load_authorized_users()
active_job_lock = threading.Lock()
active_job = None
async def send_message(ctx, message):
await ctx.author.send(message)
def claim_active_job(job):
global active_job
with active_job_lock:
if active_job is not None:
return active_job
active_job = job
return None
def get_active_job():
with active_job_lock:
return active_job
def clear_active_job(job):
global active_job
with active_job_lock:
if active_job is job:
active_job = None
def download_video(url, options):
with yt_dlp.YoutubeDL(options) as ydl:
ydl.download(url)
def create_hook(ctx, loop, cancel_event):
seen_updates = set()
def hook(d):
if cancel_event.is_set():
raise yt_dlp.utils.DownloadCancelled('download canceled by /interrupt')
status = d.get('status')
info = d.get('info_dict') or {}
if status not in {'downloading', 'finished'}:
return
filename = d.get('filename') or info.get('_filename') or info.get('title')
update_key = (status, filename)
if update_key in seen_updates:
return
seen_updates.add(update_key)
playlist_index = info.get('playlist_index')
playlist_count = info.get('playlist_count')
url = info.get('webpage_url')
prefix = status
if playlist_index and playlist_count:
prefix = f'{status} {playlist_index} of {playlist_count}'
msg = f'{prefix}: {filename}'
if url:
msg = f'{msg} <{url}>'
asyncio.run_coroutine_threadsafe(send_message(ctx, msg), loop)
return hook
@interactions.slash_command(name="youtube",description="download video from youtube to server")
@interactions.slash_option(
name='url',
opt_type=interactions.OptionType.STRING,
required=True,
description='url target'
)
async def youtube(ctx: interactions.SlashContext, url:str):
print(f'{ctx.author.id} requested {url}')
# check that user is authorized
if str(ctx.author.id) not in authorized_users:
if ctx.author.id == 127831327012683776:
await ctx.author.send('potato stop')
await ctx.author.send('you are not authorized to use this command. message my owner to be added.')
return
loop = asyncio.get_running_loop()
cancel_event = threading.Event()
hook = create_hook(ctx, loop, cancel_event)
job = {
'requester_id': str(ctx.author.id),
'request_url': url,
'cancel_event': cancel_event,
}
existing_job = claim_active_job(job)
if existing_job:
await ctx.author.send(
f'already downloading for <@{existing_job["requester_id"]}>. '
'single-job mode is enabled right now; try again after it finishes.'
)
return
# use api_to_cli and paste cli options to get the output you need
yoptions = {
'format':'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best',
'fragment_tries': 10,
'restrictfilenames':True,
'paths': {'home':'/downloads'},
'retries':10,
'writeinfojson':False,
'allow_playlist_files':True,
'noplaylist':True,
'download_archive':'/config/archive.txt',
'progress_hooks':[hook],
'outtmpl': '%(uploader)s/%(playlist_title)s/%(playlist_index)s%(playlist_index& - )s%(title)s.%(ext)s',
'outtmpl_na_placeholder':'',
}
await ctx.channel.send(f'Downloading from <{url}>. Status updates via DM. Single-job mode is enabled.')
try:
await asyncio.to_thread(download_video, url, yoptions)
except yt_dlp.utils.DownloadCancelled as exc:
print(f'download canceled: {exc}')
await ctx.author.send(f'download canceled: {exc}')
except yt_dlp.utils.DownloadError as exc:
print(f'download failed: {exc}')
await ctx.author.send(f'download failed: {exc}')
except Exception as exc:
print(f'unexpected download failure: {exc}')
await ctx.author.send(f'unexpected download failure: {exc}')
else:
await ctx.author.send(f'download complete for <{url}>')
finally:
clear_active_job(job)
@interactions.slash_command(name="interrupt",description="cancel current job") load_project_dotenv()
@interactions.check(interactions.is_owner())
async def _interrupt(ctx):
job = get_active_job()
if not job:
await ctx.author.send('no active download to interrupt')
return
job['cancel_event'].set()
print(f'interrupt requested for {job["request_url"]}') REPO_ROOT = Path(__file__).resolve().parent
await ctx.author.send( DEFAULT_BACKEND_HOST = os.getenv("YOUDIS_BACKEND_HOST", "127.0.0.1")
f'interrupt requested for <{job["request_url"]}>; ' DEFAULT_BACKEND_PORT = int(os.getenv("YOUDIS_BACKEND_PORT", "8000"))
'cancellation is coarse and will stop on the next yt-dlp progress update' ENABLE_BACKEND = os.getenv("YOUDIS_RUN_BACKEND", "1") not in {"0", "false", "False"}
ENABLE_DISCORD = os.getenv("ENABLE_DISCORD", "1") not in {"0", "false", "False"}
BACKEND_HEALTH_TIMEOUT = float(os.getenv("YOUDIS_BACKEND_HEALTH_TIMEOUT", "20"))
BACKEND_HEALTH_INTERVAL = float(os.getenv("YOUDIS_BACKEND_HEALTH_INTERVAL", "0.5"))
@dataclass
class ManagedProcess:
name: str
process: asyncio.subprocess.Process
def backend_url() -> str:
return f"http://{DEFAULT_BACKEND_HOST}:{DEFAULT_BACKEND_PORT}"
def build_child_env() -> dict[str, str]:
env = os.environ.copy()
env.setdefault("YOUDIS_BACKEND_URL", backend_url())
return env
async def start_backend(env: dict[str, str]) -> ManagedProcess:
process = await asyncio.create_subprocess_exec(
sys.executable,
"-m",
"uvicorn",
"youdis.main:app",
"--host",
DEFAULT_BACKEND_HOST,
"--port",
str(DEFAULT_BACKEND_PORT),
cwd=str(REPO_ROOT),
env=env,
) )
return ManagedProcess(name="backend", process=process)
@interactions.slash_command(name="adduser",description="authorize target user")
@interactions.slash_option(
name="user",
opt_type=interactions.OptionType.USER,
required=True,
description='enable this bot for target user',
)
@interactions.check(interactions.is_owner())
async def _adduser(ctx: interactions.SlashContext, user:interactions.OptionType.USER):
user_id = str(user.id)
if user_id not in authorized_users:
authorized_users.append(user_id)
save_authorized_users(authorized_users)
print(f'authorized {user_id}')
await ctx.author.send(f'authorized {user.mention}')
else:
await ctx.author.send(f'{user.mention} is already authorized')
@interactions.slash_command(name="removeuser",description="deauthorize target user")
@interactions.slash_option(
name="user",
opt_type=interactions.OptionType.USER,
required=True,
description='disable this bot for target user',
)
@interactions.check(interactions.is_owner())
async def _removeuser(ctx: interactions.SlashContext, user:interactions.OptionType.USER):
user_id = str(user.id)
if user_id in authorized_users:
authorized_users.remove(user_id)
save_authorized_users(authorized_users)
print(f'deauthorized {user_id}')
await ctx.author.send(f'deauthorized {user.mention}')
else:
await ctx.author.send(f'{user.mention} is not currently authorized')
api_token = getenv('api_token')
if not api_token: async def start_discord(env: dict[str, str]) -> ManagedProcess:
raise ValueError('API token not set. Retrieve from your Discord bot.') process = await asyncio.create_subprocess_exec(
bot.start(api_token) sys.executable,
"-m",
"youdis.adapters.discord",
cwd=str(REPO_ROOT),
env=env,
)
return ManagedProcess(name="discord", process=process)
ADAPTER_STARTERS = {
"discord": start_discord,
}
def selected_adapters() -> list[str]:
requested: list[str] = []
if ENABLE_DISCORD:
requested.append("discord")
return requested
async def wait_for_backend_health(timeout_seconds: float) -> None:
deadline = asyncio.get_running_loop().time() + timeout_seconds
health_url = f"{backend_url()}/health"
while True:
try:
with urlopen(health_url, timeout=2) as response:
if response.status == 200:
return
except URLError:
pass
if asyncio.get_running_loop().time() >= deadline:
raise TimeoutError(f"backend did not become healthy at {health_url}")
await asyncio.sleep(BACKEND_HEALTH_INTERVAL)
async def stop_process(proc: ManagedProcess) -> None:
if proc.process.returncode is not None:
return
proc.process.terminate()
try:
await asyncio.wait_for(proc.process.wait(), timeout=10)
except asyncio.TimeoutError:
proc.process.kill()
await proc.process.wait()
async def run() -> int:
managed: list[ManagedProcess] = []
env = build_child_env()
adapters = selected_adapters()
if not ENABLE_BACKEND and not adapters:
print("nothing to start: backend is disabled and no adapters are enabled")
return 1
try:
if ENABLE_BACKEND:
backend = await start_backend(env)
managed.append(backend)
print(f"started backend on {backend_url()}")
await wait_for_backend_health(BACKEND_HEALTH_TIMEOUT)
print("backend is healthy")
for adapter_name in adapters:
process = await ADAPTER_STARTERS[adapter_name](env)
managed.append(process)
print(f"started {adapter_name} adapter")
wait_tasks = [asyncio.create_task(proc.process.wait()) for proc in managed]
done, pending = await asyncio.wait(wait_tasks, return_when=asyncio.FIRST_COMPLETED)
for task in pending:
task.cancel()
first_returncode = 0
for task in done:
first_returncode = task.result()
break
return first_returncode
finally:
for proc in reversed(managed):
await stop_process(proc)
def main() -> None:
try:
raise SystemExit(asyncio.run(run()))
except KeyboardInterrupt:
raise SystemExit(130)
if __name__ == "__main__":
main()

1
youdis/__init__.py Normal file
View File

@@ -0,0 +1 @@
"""Youdis v2 backend package."""

View File

@@ -0,0 +1 @@
"""Frontend adapters for the youdis backend."""

231
youdis/adapters/discord.py Normal file
View File

@@ -0,0 +1,231 @@
"""Discord adapter for interacting with FastAPI backend worker."""
import asyncio
from os import getenv
import aiohttp
import interactions
from ..env import load_project_dotenv
load_project_dotenv()
BACKEND_HOST = getenv("YOUDIS_BACKEND_HOST", "127.0.0.1")
BACKEND_PORT = int(getenv("YOUDIS_BACKEND_PORT", "8000"))
BACKEND_URL = f"http://{BACKEND_HOST}:{BACKEND_PORT}".rstrip("/")
POLL_INTERVAL_SECONDS = float(getenv("YOUDIS_POLL_INTERVAL_SECONDS", "2"))
DEFAULT_SCOPE = int(getenv("DISCORD_BOT_SCOPE", "2147491904"))
bot = interactions.Client(
intents=interactions.Intents.DEFAULT,
default_scope=DEFAULT_SCOPE,
)
http_session: aiohttp.ClientSession | None = None
poll_tasks: dict[str, asyncio.Task] = {}
def backend_url(path: str) -> str:
return f"{BACKEND_URL}{path}"
async def get_session() -> aiohttp.ClientSession:
global http_session
if http_session is None or http_session.closed:
http_session = aiohttp.ClientSession()
return http_session
async def request_json(method: str, path: str, **kwargs):
session = await get_session()
async with session.request(method, backend_url(path), **kwargs) as response:
data = await response.json()
return response.status, data
def format_status_message(job: dict) -> str:
state = job.get("state")
phase = job.get("phase")
disposition = job.get("disposition")
message = job.get("message")
result_path = job.get("result_path")
parts = [f"state={state}"]
if phase:
parts.append(f"phase={phase}")
if disposition:
parts.append(f"disposition={disposition}")
if message:
parts.append(message)
if result_path:
parts.append(f"path={result_path}")
return " | ".join(parts)
async def dm(ctx: interactions.SlashContext, message: str) -> None:
await ctx.author.send(message)
async def respond(ctx: interactions.SlashContext, message: str) -> None:
if ctx.channel is not None:
await ctx.channel.send(message)
return
await dm(ctx, message)
async def poll_job_updates(ctx: interactions.SlashContext, job_id: str) -> None:
last_sent = None
try:
while True:
status_code, payload = await request_json("GET", "/jobs/current")
if status_code != 200:
await dm(ctx, f"backend status check failed: HTTP {status_code}")
return
job = payload.get("job")
if not job:
await dm(ctx, f"job {job_id} is no longer visible from the backend")
return
if job.get("job_id") != job_id:
await dm(ctx, f"job {job_id} is no longer the current backend job")
return
summary = format_status_message(job)
if summary != last_sent:
await dm(ctx, summary)
last_sent = summary
if job.get("state") in {"completed", "failed", "cancelled"}:
return
await asyncio.sleep(POLL_INTERVAL_SECONDS)
except asyncio.CancelledError:
raise
except aiohttp.ClientError as exc:
await dm(ctx, f"backend poll failed: {exc}")
finally:
poll_tasks.pop(job_id, None)
def ensure_poll_task(ctx: interactions.SlashContext, job_id: str) -> None:
existing = poll_tasks.get(job_id)
if existing and not existing.done():
return
poll_tasks[job_id] = asyncio.create_task(poll_job_updates(ctx, job_id))
@interactions.slash_command(name="youtube", description="submit a youtube download to the backend")
@interactions.slash_option(
name="url",
opt_type=interactions.OptionType.STRING,
required=True,
description="url target",
)
async def youtube(ctx: interactions.SlashContext, url: str):
payload = {
"url": url,
"requester_id": str(ctx.author.id),
"requester_name": ctx.author.username,
"origin": f"discord:{ctx.guild_id or 'dm'}:{ctx.channel_id}",
}
try:
status_code, job = await request_json("POST", "/jobs", json=payload)
except aiohttp.ClientError as exc:
await dm(ctx, f"backend request failed: {exc}")
return
if status_code != 200:
await dm(ctx, f"backend request failed: HTTP {status_code}")
return
state = job.get("state")
job_id = job.get("job_id", "unknown")
if state == "busy":
await respond(ctx, "Backend is busy with another job. Details via DM.")
await dm(ctx, f"busy: {job.get('message')}")
return
if state != "accepted":
await respond(ctx, "Backend rejected the request. Details via DM.")
await dm(ctx, format_status_message(job))
return
await respond(ctx, f"Submitted <{url}> to the backend. Status updates via DM.")
await dm(ctx, f"accepted job {job_id} for <{url}>")
ensure_poll_task(ctx, job_id)
@interactions.slash_command(name="interrupt", description="cancel the current backend job")
@interactions.check(interactions.is_owner())
async def interrupt(ctx: interactions.SlashContext):
try:
status_code, payload = await request_json("POST", "/jobs/current/cancel")
except aiohttp.ClientError as exc:
await dm(ctx, f"backend cancel failed: {exc}")
return
if status_code == 404:
await dm(ctx, "no active backend job to interrupt")
return
if status_code != 200:
await dm(ctx, f"backend cancel failed: HTTP {status_code}")
return
await dm(ctx, format_status_message(payload))
@interactions.slash_command(name="status", description="show the current backend job status")
async def status(ctx: interactions.SlashContext):
try:
status_code, payload = await request_json("GET", "/jobs/current")
except aiohttp.ClientError as exc:
await dm(ctx, f"backend status failed: {exc}")
return
if status_code != 200:
await dm(ctx, f"backend status failed: HTTP {status_code}")
return
job = payload.get("job")
if not job:
await dm(ctx, "backend has no active or recent job")
return
active = payload.get("active")
prefix = "active" if active else "last"
await dm(ctx, f"{prefix} job: {format_status_message(job)}")
@bot.listen()
async def on_startup():
await get_session()
print(f"discord adapter configured for backend {BACKEND_URL}")
print(f"{bot.application_commands.count} registered commands:")
for i, x in enumerate(bot.application_commands,start=1):
print(f" {i}. {x.name}")
@bot.listen()
async def on_shutdown():
global http_session
for task in list(poll_tasks.values()):
task.cancel()
poll_tasks.clear()
if http_session is not None and not http_session.closed:
await http_session.close()
http_session = None
def main() -> None:
api_token = getenv("DISCORD_BOT_TOKEN")
if not api_token:
raise ValueError("API token not set. Retrieve from your Discord bot.")
# bot.add_command(youtube)
# bot.add_command(status)
# bot.add_command(interrupt)
bot.start(api_token)
if __name__ == "__main__":
main()

13
youdis/env.py Normal file
View File

@@ -0,0 +1,13 @@
from pathlib import Path
try:
from dotenv import load_dotenv
except ModuleNotFoundError: # pragma: no cover - optional local convenience dependency
load_dotenv = None
def load_project_dotenv() -> None:
if load_dotenv is None:
return
repo_root = Path(__file__).resolve().parent.parent
load_dotenv(repo_root / ".env", override=False)

311
youdis/main.py Normal file
View File

@@ -0,0 +1,311 @@
"""FastAPI backend worker managing yt-dlp subprocess jobs and exposing job state."""
import asyncio
from asyncio.subprocess import PIPE, STDOUT
from collections import deque
from dataclasses import dataclass, field
from datetime import datetime
from os import getenv
from pathlib import Path
import tempfile
from uuid import uuid4
from fastapi import FastAPI, HTTPException
from .env import load_project_dotenv
from .models import CurrentJobResponse, HealthResponse, JobRequest, JobStatus, VersionResponse
load_project_dotenv()
REPO_ROOT = Path(__file__).resolve().parent.parent
DEFAULT_CONFIG = REPO_ROOT / "default-yt-dlp.conf"
VERSION_FILE = REPO_ROOT / "youdis-version.txt"
YTDLP_EXECUTABLE = getenv("YOUDIS_YTDLP_EXECUTABLE", "yt-dlp")
PREFERRED_CONFIG_ROOT = Path(getenv("YOUDIS_CONFIG_DIR", "/config"))
PREFERRED_DOWNLOAD_ROOT = Path(getenv("YOUDIS_DOWNLOAD_DIR", "/downloads"))
LOCAL_RUNTIME_ROOT = REPO_ROOT / ".runtime"
FALLBACK_CONFIG_ROOT = LOCAL_RUNTIME_ROOT / "config"
FALLBACK_DOWNLOAD_ROOT = LOCAL_RUNTIME_ROOT / "downloads"
OUTPUT_TEMPLATE = "%(uploader)s/%(playlist_title)s/%(playlist_index)s%(playlist_index& - )s%(title)s.%(ext)s"
@dataclass
class ManagedJob:
status: JobStatus
process: asyncio.subprocess.Process | None = None
task: asyncio.Task | None = None
cancel_requested: bool = False
output_lines: deque[str] = field(default_factory=lambda: deque(maxlen=25))
app = FastAPI(title="youdis", version="2")
job_lock = asyncio.Lock()
active_job: ManagedJob | None = None
last_job: JobStatus | None = None
def now_utc() -> datetime:
return datetime.now()
def read_version() -> str:
if VERSION_FILE.exists():
return VERSION_FILE.read_text().strip()
return "unknown"
def ensure_writable_directory(preferred: Path, fallback: Path) -> Path:
for candidate in (preferred, fallback):
try:
candidate.mkdir(parents=True, exist_ok=True)
with tempfile.NamedTemporaryFile(dir=candidate, prefix=".youdis-write-", delete=True):
pass
return candidate
except OSError:
continue
raise OSError(f"no writable runtime directory available for {preferred} or {fallback}")
def resolve_runtime_paths() -> tuple[Path, Path]:
config_root = ensure_writable_directory(PREFERRED_CONFIG_ROOT, FALLBACK_CONFIG_ROOT)
download_root = ensure_writable_directory(PREFERRED_DOWNLOAD_ROOT, FALLBACK_DOWNLOAD_ROOT)
return config_root, download_root
def build_ytdlp_command(request: JobRequest, config_root: Path, download_root: Path) -> list[str]:
return [
YTDLP_EXECUTABLE,
"--config-locations",
str(DEFAULT_CONFIG),
"--download-archive",
str(config_root / "archive.txt"),
"--output",
str(download_root / OUTPUT_TEMPLATE),
request.url,
]
def clone_status(status: JobStatus) -> JobStatus:
return JobStatus(**status.model_dump())
def update_status(job: ManagedJob, **changes: object) -> None:
for key, value in changes.items():
setattr(job.status, key, value)
job.status.updated_at = now_utc()
def classify_output_line(job: ManagedJob, line: str) -> None:
if not line:
return
job.output_lines.append(line)
message = line.strip()
if not message:
return
lowered = message.lower()
if "has already been recorded in the archive" in lowered:
update_status(
job,
disposition="archive_hit",
phase="downloading",
message="already in archive",
)
return
if "[download]" in lowered:
update_status(job, phase="downloading", message=message)
if "destination:" in lowered:
result_path = message.split(":", 1)[1].strip()
update_status(job, result_path=result_path)
return
if "merging formats into" in lowered:
result_path = message.split("into", 1)[1].strip().strip('"')
update_status(job, phase="postprocessing", message=message, result_path=result_path)
return
update_status(job, message=message)
async def finalize_job(job: ManagedJob, returncode: int) -> None:
disposition = job.status.disposition
if job.cancel_requested:
state = "cancelled"
message = "cancelled"
elif returncode == 0 and disposition == "archive_hit":
state = "completed"
message = "already in archive"
elif returncode == 0:
state = "completed"
message = job.status.message or "completed"
else:
state = "failed"
message = job.status.message or f"yt-dlp exited with code {returncode}"
update_status(job, state=state, phase=None, returncode=returncode, message=message)
global active_job, last_job
async with job_lock:
if active_job is job:
active_job = None
last_job = clone_status(job.status)
async def run_job(job: ManagedJob, request: JobRequest) -> None:
update_status(
job,
state="running",
phase="starting",
message="starting yt-dlp",
)
try:
if not DEFAULT_CONFIG.exists():
update_status(
job,
state="failed",
phase=None,
message=f"default config not found: {DEFAULT_CONFIG}",
returncode=78,
)
await finalize_job(job, 78)
return
config_root, download_root = resolve_runtime_paths()
command = build_ytdlp_command(request, config_root, download_root)
update_status(
job,
command=command,
archive_path=str(config_root / "archive.txt"),
result_path=str(download_root),
)
process = await asyncio.create_subprocess_exec(
*command,
stdout=PIPE,
stderr=STDOUT,
)
except FileNotFoundError:
update_status(
job,
state="failed",
phase=None,
message="yt-dlp executable not found",
returncode=127,
)
await finalize_job(job, 127)
return
except OSError as exc:
update_status(
job,
state="failed",
phase=None,
message=f"runtime path setup failed: {exc}",
returncode=73,
)
await finalize_job(job, 73)
return
try:
job.process = process
update_status(job, phase="running", message="yt-dlp running")
assert process.stdout is not None
while True:
line = await process.stdout.readline()
if not line:
break
classify_output_line(job, line.decode(errors="replace").strip())
returncode = await process.wait()
await finalize_job(job, returncode)
except Exception as exc:
update_status(
job,
state="failed",
phase=None,
message=f"backend runner error: {exc}",
returncode=1,
)
await finalize_job(job, 1)
async def create_job(request: JobRequest) -> JobStatus:
global active_job
async with job_lock:
if active_job is not None:
busy_job = active_job.status
return JobStatus(
job_id=busy_job.job_id,
state="busy",
url=request.url,
message=f"busy with {busy_job.url}",
requester_id=request.requester_id,
requester_name=request.requester_name,
origin=request.origin,
)
status = JobStatus(
job_id=str(uuid4()),
state="accepted",
url=request.url,
message="accepted",
phase="queued",
requester_id=request.requester_id,
requester_name=request.requester_name,
origin=request.origin,
)
managed_job = ManagedJob(status=status)
managed_job.task = asyncio.create_task(run_job(managed_job, request))
active_job = managed_job
return clone_status(status)
@app.get("/health", response_model=HealthResponse)
async def health() -> HealthResponse:
return HealthResponse(status="ok")
@app.get("/version", response_model=VersionResponse)
async def version() -> VersionResponse:
return VersionResponse(
version=read_version(),
active_job=active_job is not None,
ytdlp_executable=YTDLP_EXECUTABLE,
)
@app.get("/jobs/current", response_model=CurrentJobResponse)
async def current_job() -> CurrentJobResponse:
async with job_lock:
if active_job is not None:
return CurrentJobResponse(active=True, job=clone_status(active_job.status))
if last_job is not None:
return CurrentJobResponse(active=False, job=clone_status(last_job))
return CurrentJobResponse(active=False, job=None)
@app.post("/jobs", response_model=JobStatus)
async def submit_job(request: JobRequest) -> JobStatus:
return await create_job(request)
@app.post("/jobs/current/cancel", response_model=JobStatus)
async def cancel_current_job() -> JobStatus:
async with job_lock:
if active_job is None:
raise HTTPException(status_code=404, detail="no active job")
job = active_job
if job.process is None:
update_status(job, message="cancel requested before yt-dlp started")
job.cancel_requested = True
return clone_status(job.status)
job.cancel_requested = True
update_status(job, message="cancel requested", phase="running")
job.process.terminate()
return clone_status(job.status)

48
youdis/models.py Normal file
View File

@@ -0,0 +1,48 @@
from datetime import datetime
from typing import Literal
from pydantic import BaseModel, Field
JobState = Literal["accepted", "busy", "running", "completed", "failed", "cancelled"]
class JobRequest(BaseModel):
url: str
requester_id: str | None = None
requester_name: str | None = None
origin: str | None = None
requested_at: datetime | None = None
class JobStatus(BaseModel):
job_id: str
state: JobState
url: str
message: str | None = None
phase: str | None = None
disposition: str | None = None
requester_id: str | None = None
requester_name: str | None = None
origin: str | None = None
result_path: str | None = None
archive_path: str | None = None
command: list[str] = Field(default_factory=list)
returncode: int | None = None
created_at: datetime = Field(default_factory=datetime.utcnow)
updated_at: datetime = Field(default_factory=datetime.utcnow)
class CurrentJobResponse(BaseModel):
active: bool
job: JobStatus | None = None
class HealthResponse(BaseModel):
status: Literal["ok"]
class VersionResponse(BaseModel):
version: str
active_job: bool
ytdlp_executable: str