Compare commits

...

10 Commits

11 changed files with 770 additions and 295 deletions

12
.env.example Normal file
View File

@@ -0,0 +1,12 @@
YOUDIS_ENABLE_BACKEND=1
YOUDIS_BACKEND_HOST=127.0.0.1
YOUDIS_BACKEND_PORT=8000
YOUDIS_BACKEND_HEALTH_TIMEOUT=20
YOUDIS_BACKEND_HEALTH_INTERVAL=0.5
YOUDIS_POLL_INTERVAL_SECONDS=2
YOUDIS_YTDLP_EXECUTABLE=yt-dlp
YOUDIS_CONFIG_DIR=/home/user/proj/youdis/test/
YOUDIS_DOWNLOAD_DIR=/home/user/proj/youdis/downloads
ENABLE_DISCORD=1
DISCORD_BOT_TOKEN=
DISCORD_BOT_SCOPE=2147491904

70
.gitignore vendored
View File

@@ -1,34 +1,36 @@
# --- python bytecode ---
__pycache__/
*.py[cod]
*$py.class
# --- virtual environments ---
.venv/
venv/
env/
# --- environment files ---
.env
.env.*
*.local
# --- emacs ---
*~
\#*\#
.\#*
*.elc
# --- project private data ---
/private/
archive/
downloads/
data.json
# --- django ---
db.sqlite3
staticfiles/
media/
# --- misc ---
.DS_Store
# --- python bytecode ---
__pycache__/
*.py[cod]
*$py.class
# --- virtual environments ---
.venv/
venv/
env/
# --- environment files ---
.env
.env.*
!.env.example
*.local
# --- emacs ---
*~
\#*\#
.\#*
*.elc
# --- project private data ---
/private/
archive/
downloads/
test/
data.json
# --- django ---
db.sqlite3
staticfiles/
media/
# --- misc ---
.DS_Store

View File

@@ -1,10 +1,60 @@
v2 architecture draft: see `docs/architecture-v2.org`
# Running youdis
build and run the docker container
```
api_token = [discord bot token]
-v [downloads]:/downloads
-v [config]:/config
```
config contains data to persist across container updates, i.e., unraid appdata,
such as users.json (authorized users) and yt-dlp's archive.txt
-v [host_downloads]:/downloads
-v [host_config]:/config
-e ENABLE_DISCORD=1
-e DISCORD_BOT_TOKEN=<token>
-e DISCORD_BOT_SCOPE=<scope-int>
```
config contains data to persist across container updates, i.e., unraid appdata,
such as yt-dlp's archive.txt
# Development
v2 architecture draft: `docs/architecture-v2.org`
The app runs with `youdis.py`.
This starts the backend first, waits for health, then starts each enabled adapter explicitly.
Test components directly with uvicorn:
```
python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000
python3 -m youdis.adapters.discord
```
Key runner/config vars:
```
YOUDIS_ENABLE_BACKEND=1 #default enabled, dont disable unless testing
YOUDIS_BACKEND_HOST=127.0.0.1
YOUDIS_BACKEND_PORT=8000
YOUDIS_BACKEND_HEALTH_TIMEOUT=20
YOUDIS_BACKEND_HEALTH_INTERVAL=0.5
YOUDIS_POLL_INTERVAL_SECONDS=2
YOUDIS_YTDLP_EXECUTABLE=yt-dlp
YOUDIS_CONFIG_DIR=/path/to/config/
YOUDIS_DOWNLOAD_DIR=/path/to/downloads/
ENABLE_DISCORD=1
DISCORD_BOT_TOKEN=<token-string>
DISCORD_BOT_SCOPE=123456789
```
## Add Adapter
Configure adapter selection in .env, like so:
```
YOUDIS_RUN_BACKEND=1
YOUDIS_BACKEND_HOST=127.0.0.1
YOUDIS_BACKEND_PORT=8000
ENABLE_DISCORD=1
DISCORD_BOT_TOKEN=<api_key>
DISCORD_BOT_SCOPE=123456789
```
to add a new adapter:
1. add `youdis/adapters/<adapter>.py`
2. make it independently runnable
3. add `start_<adapter>()` to `youdis.py`
4. register it in the explicit adapter starter map in `youdis.py`
5. add `ENABLE_<adapter>` and supporting env vars to `.env.example`

View File

@@ -1,6 +1,10 @@
#+title: Task Log
#+updated: [2026-03-31 Tue 16:03]
#+startup: overview
# Local Variables:
# org-babel-python-command: "/home/user/proj/youdis/.venv/bin/python"
# project-dir: "~/proj/youdis"
# End:
* youdis v2 goals:
1. Separate backend from frontend
@@ -36,7 +40,7 @@ define the target architecture for a private backend yt-dlp worker with thin cha
** notes
- first architecture draft captured in `docs/architecture-v2.org`
* [ ] 2.0.1: build backend yt-dlp worker (3)
* [X] 2.0.1: build backend yt-dlp worker (3)
create the minimal backend/service skeleton and establish a working yt-dlp baseline with clean hooks for future frontends
** pm notes
- foundation; don't need the full finished service here, just the basic shape plus enough real yt-dlp execution to validate the seam and build on it.
@@ -55,10 +59,13 @@ create the minimal backend/service skeleton and establish a working yt-dlp basel
- submit/request path exists
- status/progress hook exists
- basic health/version visibility exists
4f.
4. make local testing practical without breaking container defaults
- backend can run when `/config` or `/downloads` are not writable in local dev
- env vars may override executable/config/download paths
- status makes the effective runtime command/paths inspectable
** evidence
- commit:
- commit: 7926534,2a56485
- tests:
1. `python3 -m py_compile ./youdis/main.py ./youdis/models.py ./youdis/adapters/__init__.py ./youdis/adapters/discord.py`
2. `python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000`
@@ -73,11 +80,12 @@ create the minimal backend/service skeleton and establish a working yt-dlp basel
11. if testing in container: `docker run --rm -p 8000:8000 -v [config]:/config -v [downloads]:/downloads youdis:v2`
:OUTPUT_STEP1-9:
user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/health
{"status":"ok"}user@paladin:~/proj/youdis$
user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/version
{"version":"20250829-ec72c56","active_job":false}user@paladin:~/proj/youdis$
{"status":"ok"}
user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/version
{"version":"20250829-ec72c56","active_job":false}
user@paladin:~/proj/youdis$ curl -X POST http://127.0.0.1:8000/jobs -H 'content-type: application/json' -d '{"url":"https://www.youtube.com/watch?v=3i72yY_LaW4"}'
{"job_id":"cc85165e-d906-4eee-864f-1a398b6de2e0","state":"accepted","url":"https://www.youtube.com/watch?v=3i72yY_LaW4","message":"accepted","phase":"queued","disposition":null,"requester_id":null,"requester_name":null,"origin":null,"result_path":null,"command":[],"returncode":null,"created_at":"2026-04-01T00:44:35.657196","updated_at":"2026-04-01T00:44:35.657198"}user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/jobs/current
{"job_id":"cc85165e-d906-4eee-864f-1a398b6de2e0","state":"accepted","url":"https://www.youtube.com/watch?v=3i72yY_LaW4","message":"accepted","phase":"queued","disposition":null,"requester_id":null,"requester_name":null,"origin":null,"result_path":null,"command":[],"returncode":null,"created_at":"2026-04-01T00:44:35.657196","updated_at":"2026-04-01T00:44:35.657198"}
user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/jobs/current
{"active":false,"job":{"job_id":"cc85165e-d906-4eee-864f-1a398b6de2e0","state":"failed","url":"https://www.youtube.com/watch?v=3i72yY_LaW4","message":"ERROR: unable to create directory [Errno 13] Permission denied: '/downloads'","phase":null,"disposition":null,"requester_id":null,"requester_name":null,"origin":null,"result_path":null,"command":["yt-dlp","--config-locations","/home/user/proj/youdis/default-yt-dlp.conf","https://www.youtube.com/watch?v=3i72yY_LaW4"],"returncode":1,"created_at":"2026-04-01T00:44:35.657196","updated_at":"2026-03-31T20:44:36.653353"}}user@paladin:~/proj/youdis$
:END:
- datetime: [2026-03-31 Tue 20:45]
@@ -87,8 +95,10 @@ user@paladin:~/proj/youdis$ curl -X POST http://127.0.0.1:8000/jobs -H 'content-
- confirm `--config-locations` behavior against the installed `yt-dlp` version during integration testing
- current backend scaffold is not yet wired into `dockerfile` or `run-youdis.sh`
- archive-hit and result-path parsing currently depend on `yt-dlp` stdout text patterns, so treat them as provisional until integration-tested
* [ ] 2.0.2: update discord bot to use new backend (3)
- local dev now falls back to repo-local `.runtime/{config,downloads}` when `/config` or `/downloads` are not writable
- had to uninstall yt-dlp python pkg from the venv, which resulted in a '403 Forbidden'
* [X] 2.0.2: update discord bot to use new backend (3)
update the discord bot into a thin frontend that talks to the backend and verify the flow end to end
** pm notes
- this is the first real frontend proof. once this works cleanly, zulip/xmpp should mostly be adapter work rather than downloader rewrites.
@@ -106,15 +116,99 @@ update the discord bot into a thin frontend that talks to the backend and verify
- valid request path tested
- busy or conflict behavior tested
- failure path tested
4. add dotenv support to ease dev
- os.getenv methods remain standard for prod/docker build
- populate .env with dev env defaults
** evidence
- commit:
- tests:
- datetime:
- commit: 5210d2c, 043cb4
- tests: https://youtu.be/20HxMMSqRyg?si=3v7mN2L88c_FxpQR 18m
1. start backend: `python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000`
2. create local env file: `cp .env.example .env`
3. add `api_token` to `.env`
4. start adapter: `python3 ./youdis.py`
5. in discord, run `/youtube url:https://www.youtube.com/watch?v=dQw4w9WgXcQ`
:out:
#+begin_src shell
(venv) user@paladin:~/proj/youdis$ python ./youdis.py
Task exception was never retrieved
future: <Task finished name='Task-43' coro=<Client._dispatch_interaction() done, defined at /home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/client/client.py:1773> exception=KeyError('youtube')>
Traceback (most recent call last):
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/client/client.py", line 1798, in _dispatch_interaction
if ctx.command:
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/models/internal/context.py", line 329, in command
return self.client._interaction_lookup[self._command_name]
KeyError: 'youtube'
#+end_src
:end:
6. confirm channel response says the job was submitted to backend
7. confirm requester receives DM updates for accepted/running/completed or failed
:out:
accepted job 4ef6fde5-57cc-478c-b0e2-18458c693fa4 for https://www.youtube.com/watch?v=dQw4w9WgXcQ
state=running | phase=running | [youtube] dQw4w9WgXcQ: Downloading webpage | path=/home/user/proj/youdis/downloads
state=running | phase=downloading | [download] Destination: /home/user/proj/youdis/downloads/Rick_Astley/NA/NANARickAstley-_Never_Gonna_Give_You_Up_Official_Video_4K_Remaster.f401.mp4 | path=/home/user/proj/youdis/downloads/Rick_Astley/NA/NANARickAstley-_Never_Gonna_Give_You_Up_Official_Video_4K_Remaster.f401.mp4
state=completed | [Metadata] There isn't any metadata to add | path=/home/user/proj/youdis/downloads/Rick_Astley/NA/NANARickAstley-_Never_Gonna_Give_You_Up_Official_Video_4K_Remaster.mp4
:end:
8. while first job is active, submit another `/youtube` and confirm busy behavior
:out:
#+begin_src shell
Error: AttributeError
Traceback (most recent call last):
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/client/client.py", line 1900, in __dispatch_interaction
response = await callback
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/client/client.py", line 1771, in _run_slash_command
return await command(ctx, **ctx.kwargs)
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/models/internal/command.py", line 132, in __call__
await self.call_callback(self.callback, context)
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/models/internal/application_commands.py", line 833, in call_callback
return await self.call_with_binding(callback, ctx, *new_args, **new_kwargs)
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/models/internal/callback.py", line 43, in call_with_binding
return await callback(*args, **kwargs)
File "/home/user/proj/youdis/youdis/adapters/discord.py", line 167, in youtube
await ctx.channel.send(f"Submitted <{url}> to the backend. Status updates via DM.")
AttributeError: 'NoneType' object has no attribute 'send'
#+end_src
busy: busy with https://youtu.be/20HxMMSqRyg?si=3v7mN2L88c_FxpQR
state=running | phase=running | cancel requested | path=/home/user/proj/youdis/downloads/James_Hoffmann/NA/NANAThe_Beginner_s_Guide_To_Latte_Art.f401.mp4
:end:
9. run `/status` and confirm it reflects current or last backend job
:out:
last job: state=completed | [Metadata] There isn't any metadata to add | path=/home/user/proj/youdis/downloads/Rick_Astley/NA/NANARickAstley-_Never_Gonna_Give_You_Up_Official_Video_4K_Remaster.mp4
:end:
10. run `/interrupt` as owner and confirm cancellation is surfaced via DM
:out:
last job: state=cancelled | cancelled | path=/home/user/proj/youdis/downloads/James_Hoffmann/NA/NANAThe_Beginner_s_Guide_To_Latte_Art.f401.mp4
:end:
- datetime:[2026-04-02 Thu 11:52]
*** org-block tests
#+begin_src shell :dir ~/proj/youdis :results output verbatim
source ./venv/bin/activate
python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000
#+end_src
#+begin_src shell :results output
echo ok
#+end_src
#+RESULTS:
: ok
#+begin_src python :dir ~/proj/youdis :results output verbatim
python ./youdis.py
#+end_src
#+RESULTS:
** notes
* [ ] 2.0.3: remove deprecated discord-bot functionality (2)
- discord adapter is now a thin HTTP client of the backend; it no longer imports or configures yt-dlp
- `YOUDIS_BACKEND_URL` controls which backend the adapter targets
- progress updates are currently implemented by polling `/jobs/current` and DMing only when the summary changes
- legacy auth/user-management commands were removed from the active adapter path
- `.env` is now supported for local/dev convenience, while real environment variables still override it in prod/docker
- command registration required explicit binding plus `@bot.listen()` listeners in this adapter structure
* [X] 2.0.3: remove deprecated discord-bot functionality (2)
delete or retire legacy bot behaviors that no longer fit once the backend split is in place
** pm notes
- only remove this after the new path works. this is cleanup, not pioneering work.
@@ -133,13 +227,73 @@ delete or retire legacy bot behaviors that no longer fit once the backend split
- deprecated artifacts are clearly removed or marked
** evidence
- commit:
- commit: 0aa9950
- tests:
- datetime:
1. `python3 -m py_compile ./youdis.py ./youdis/adapters/discord.py`
2. `rg -n "users.json|api_token" README.md unraid-ca-template.xml youdis.py youdis/adapters/discord.py`
3. start backend: `python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000`
4. start adapter: `python3 ./youdis.py`
5. in discord, run `/youtube`, `/status`, and `/interrupt` and confirm the adapter still works after cleanup
- datetime: [2026-04-02 Thu 12:09]
** notes
- active bot path no longer includes local auth or user-management behavior
- top-level `youdis.py` remains as a thin launcher so existing operator habits and scripts do not break during the refactor
- cleanup updated user-facing deployment artifacts to match the v2 naming and architecture, including `DISCORD_BOT_TOKEN` and removal of `users.json` references
- archived planning docs were intentionally left untouched: `pm/tasks.org` is historical and `pm/notes.org` is personal working notes
* [ ] 2.0.5: fix automation and build pipeline (3)
* [ ] 2.0.4: make youdis.py app runner for backend + adapters (3)
Build a simple Python orchestration layer in youdis.py so the standard app stack can be launched from one entrypoint while backend and adapters remain independently runnable for testing.
** pm notes
- keep components independently runnable
- youdis.py as app-level orchestrator; let Docker just run python3 /app/youdis.py
- no plugin discovery or hot-loading
- Docker should package and launch the app, not decide internal process topology
- preserve the ability to run backend or adapters directly for debugging
- optimize for one obvious default run path
** Acceptance Criteria
1. define the default app startup path
- youdis.py launches the standard stack for v2
- backend and Discord adapter startup order is explicit
- shutdown behavior is coherent enough for local/dev use
2. preserve modular run paths
- backend can still be run directly
- Discord adapter can still be run directly
- orchestration layer does not bury component-level testing
3. keep orchestration simple
- no dynamic adapter discovery
- no hot reloading framework
- configured components are started explicitly
4. document runtime ownership
- clarify what Python orchestrates vs what Docker orchestrates
- identify env vars or flags that control which components start
- leave room for future Zulip/XMPP adapters without redesigning the runner
** evidence
- commit: 309ce87
- tests:
1. `python3 -m py_compile ./youdis.py ./youdis/adapters/discord.py`
2. backend direct run still works: `python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000`
3. discord direct run still works: `python3 -m youdis.adapters.discord`
4. app runner backend-only smoke test: `ENABLE_DISCORD=0 timeout 5s python3 ./youdis.py`
5. app runner default path: `python3 ./youdis.py`
- date: [2026-04-02 Thu 14:13]
** notes
- `youdis.py` is now the default v2 app runner and starts the standard stack explicitly rather than dynamically discovering adapters
- backend starts first and must pass a health check before the Discord adapter is launched
- backend and Discord adapter remain directly runnable for debugging and tests
- Docker is intended to invoke `python3 /app/youdis.py`; Python owns app orchestration while Docker owns packaging and runtime environment
- adapter startup remains explicit in `youdis.py`, while each adapter gets its own `ENABLE_<ADAPTER>` flag
- adding an adapter means creating `youdis/adapters/<adapter>.py`, adding `start_<adapter>()` in `youdis.py`, wiring its `ENABLE_<ADAPTER>` flag, and documenting any supporting env vars in `.env.example`
- runner flags currently include `YOUDIS_RUN_BACKEND`, `YOUDIS_BACKEND_HOST`, `YOUDIS_BACKEND_PORT`, and `ENABLE_DISCORD`
* ==== BACKLOG ====
Tasks below this line are inactive and should not be touched.
* [ ] 2.0.X: fix automation and build pipeline (3)
repair and simplify the build/update/deploy path so it matches the new backend-plus-frontend structure
** pm notes
- this should come after architecture and discord integration stabilize. no point polishing the pipeline for the wrong shape.
@@ -163,3 +317,33 @@ repair and simplify the build/update/deploy path so it matches the new backend-p
- datetime:
** notes
* [ ] X.x.x: clean up discord adapter UI
** acceptance criteria
1. fix interaction pattern so it doesnt time out - prefer "command accepted" or somehting
2. remove all intermediate messages between "accepted/running" and "complete" - /status handles this!
- discord can also output a "busy" signal, research this
3. fix output syntax, we dont need to get crazy with discord cards
** evidence
- commit:
- tests:
- date:
** notes
* [ ] X.x.x: fix youtube -> plex default output
** acceptance criteria
1.
-
** evidence
- commit:
- tests:
- date:
** notes
* /

Binary file not shown.

View File

@@ -8,11 +8,11 @@
<Privileged>false</Privileged>
<Support>[Unraid Support Thread]</Support>
<Project>https://github.com/eulaly/youdis</Project>
<Overview>Discord bot-based wrapper for yt-dlp. Let your friends download videos to your server! Supports playlists, requires a configured Discord bot.</Overview>
<Overview>Private yt-dlp worker with a Discord adapter. Submit downloads through Discord while the backend owns yt-dlp execution and job state.</Overview>
<Category>Downloaders: Tools:</Category>
<TemplateURL>https://raw.githubusercontent.com/eulaly/unraid-templates/refs/heads/master/unraid-ca-template.xml</TemplateURL>
<Icon>https://github.com/eulaly/youdis/blob/c978a2326984efa9670678687ed1a1473478d753/yt_dlp.png</Icon>
<Config Name="api_token" Target="api_token" Default="" Mode="" Description="Discord bot token" Type="Variable" Display="always" Required="true" Mask="true"/>
<Config Name="DISCORD_BOT_TOKEN" Target="DISCORD_BOT_TOKEN" Default="" Mode="" Description="Discord bot token" Type="Variable" Display="always" Required="true" Mask="true"/>
<Config Name="Downloads" Target="/downloads" Default="" Mode="rw" Description="Video download location" Type="Path" Display="always" Required="false"/>
<Config Name="Config" Target="/config" Default="/mnt/user/appdata/youdis/config" Mode="rw" Description="Config location (archive.txt, users.json)" Type="Path" Display="always" Required="false"/>
<Config Name="Config" Target="/config" Default="/mnt/user/appdata/youdis/config" Mode="rw" Description="Config location (archive.txt and backend runtime data)" Type="Path" Display="always" Required="false"/>
</Container>

382
youdis.py
View File

@@ -1,235 +1,161 @@
#!/usr/bin/env python3
'''
youdis v1.1
bot for downloading youtube videos using yt-dlp
discord-py-interactions 5.11.0 has new option
requires python>=3.9
'''
# match_filter: info_dict -> Raise utils.DownloadCancelled(msg) ? interrupt
import interactions
from os import getenv
from pathlib import Path
import yt_dlp
import json
#!/usr/bin/env python3
"""App runner for FastAPI backend and discord adapter."""
from __future__ import annotations
import asyncio
import threading
import os
import sys
from dataclasses import dataclass
from pathlib import Path
from urllib.error import URLError
from urllib.request import urlopen
userFile = Path('/config/users.json')
userFile.parent.mkdir(exist_ok=True, parents=True)
bot = interactions.Client(intents=interactions.Intents.DEFAULT,default_scope=2147491904)
def save_authorized_users(authorized_users):
with open(userFile, 'w') as f:
json.dump({'authorized_users': authorized_users}, f)
def load_authorized_users():
if not userFile.exists():
save_authorized_users([])
print(f'users.json not found; saving to {userFile}')
return []
try:
with open(userFile, 'r') as f:
data = json.load(f)
except (json.JSONDecodeError, OSError):
save_authorized_users([])
print(f'users.json invalid; resetting {userFile}')
return []
authorized_users = data.get('authorized_users', [])
if not isinstance(authorized_users, list):
authorized_users = []
authorized_users = [str(user_id) for user_id in authorized_users]
save_authorized_users(authorized_users)
print(f'authorized_users:{authorized_users}')
return authorized_users
authorized_users = load_authorized_users()
active_job_lock = threading.Lock()
active_job = None
async def send_message(ctx, message):
await ctx.author.send(message)
def claim_active_job(job):
global active_job
with active_job_lock:
if active_job is not None:
return active_job
active_job = job
return None
def get_active_job():
with active_job_lock:
return active_job
def clear_active_job(job):
global active_job
with active_job_lock:
if active_job is job:
active_job = None
def download_video(url, options):
with yt_dlp.YoutubeDL(options) as ydl:
ydl.download(url)
def create_hook(ctx, loop, cancel_event):
seen_updates = set()
def hook(d):
if cancel_event.is_set():
raise yt_dlp.utils.DownloadCancelled('download canceled by /interrupt')
status = d.get('status')
info = d.get('info_dict') or {}
if status not in {'downloading', 'finished'}:
return
filename = d.get('filename') or info.get('_filename') or info.get('title')
update_key = (status, filename)
if update_key in seen_updates:
return
seen_updates.add(update_key)
playlist_index = info.get('playlist_index')
playlist_count = info.get('playlist_count')
url = info.get('webpage_url')
prefix = status
if playlist_index and playlist_count:
prefix = f'{status} {playlist_index} of {playlist_count}'
msg = f'{prefix}: {filename}'
if url:
msg = f'{msg} <{url}>'
asyncio.run_coroutine_threadsafe(send_message(ctx, msg), loop)
return hook
@interactions.slash_command(name="youtube",description="download video from youtube to server")
@interactions.slash_option(
name='url',
opt_type=interactions.OptionType.STRING,
required=True,
description='url target'
)
async def youtube(ctx: interactions.SlashContext, url:str):
print(f'{ctx.author.id} requested {url}')
# check that user is authorized
if str(ctx.author.id) not in authorized_users:
if ctx.author.id == 127831327012683776:
await ctx.author.send('potato stop')
await ctx.author.send('you are not authorized to use this command. message my owner to be added.')
return
loop = asyncio.get_running_loop()
cancel_event = threading.Event()
hook = create_hook(ctx, loop, cancel_event)
job = {
'requester_id': str(ctx.author.id),
'request_url': url,
'cancel_event': cancel_event,
}
existing_job = claim_active_job(job)
if existing_job:
await ctx.author.send(
f'already downloading for <@{existing_job["requester_id"]}>. '
'single-job mode is enabled right now; try again after it finishes.'
)
return
# use api_to_cli and paste cli options to get the output you need
yoptions = {
'format':'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best',
'fragment_tries': 10,
'restrictfilenames':True,
'paths': {'home':'/downloads'},
'retries':10,
'writeinfojson':False,
'allow_playlist_files':True,
'noplaylist':True,
'download_archive':'/config/archive.txt',
'progress_hooks':[hook],
'outtmpl': '%(uploader)s/%(playlist_title)s/%(playlist_index)s%(playlist_index& - )s%(title)s.%(ext)s',
'outtmpl_na_placeholder':'',
}
await ctx.channel.send(f'Downloading from <{url}>. Status updates via DM. Single-job mode is enabled.')
try:
await asyncio.to_thread(download_video, url, yoptions)
except yt_dlp.utils.DownloadCancelled as exc:
print(f'download canceled: {exc}')
await ctx.author.send(f'download canceled: {exc}')
except yt_dlp.utils.DownloadError as exc:
print(f'download failed: {exc}')
await ctx.author.send(f'download failed: {exc}')
except Exception as exc:
print(f'unexpected download failure: {exc}')
await ctx.author.send(f'unexpected download failure: {exc}')
else:
await ctx.author.send(f'download complete for <{url}>')
finally:
clear_active_job(job)
from youdis.env import load_project_dotenv
@interactions.slash_command(name="interrupt",description="cancel current job")
@interactions.check(interactions.is_owner())
async def _interrupt(ctx):
job = get_active_job()
if not job:
await ctx.author.send('no active download to interrupt')
return
load_project_dotenv()
job['cancel_event'].set()
print(f'interrupt requested for {job["request_url"]}')
await ctx.author.send(
f'interrupt requested for <{job["request_url"]}>; '
'cancellation is coarse and will stop on the next yt-dlp progress update'
REPO_ROOT = Path(__file__).resolve().parent
DEFAULT_BACKEND_HOST = os.getenv("YOUDIS_BACKEND_HOST", "127.0.0.1")
DEFAULT_BACKEND_PORT = int(os.getenv("YOUDIS_BACKEND_PORT", "8000"))
ENABLE_BACKEND = os.getenv("YOUDIS_RUN_BACKEND", "1") not in {"0", "false", "False"}
ENABLE_DISCORD = os.getenv("ENABLE_DISCORD", "1") not in {"0", "false", "False"}
BACKEND_HEALTH_TIMEOUT = float(os.getenv("YOUDIS_BACKEND_HEALTH_TIMEOUT", "20"))
BACKEND_HEALTH_INTERVAL = float(os.getenv("YOUDIS_BACKEND_HEALTH_INTERVAL", "0.5"))
@dataclass
class ManagedProcess:
name: str
process: asyncio.subprocess.Process
def backend_url() -> str:
return f"http://{DEFAULT_BACKEND_HOST}:{DEFAULT_BACKEND_PORT}"
def build_child_env() -> dict[str, str]:
env = os.environ.copy()
env.setdefault("YOUDIS_BACKEND_URL", backend_url())
return env
async def start_backend(env: dict[str, str]) -> ManagedProcess:
process = await asyncio.create_subprocess_exec(
sys.executable,
"-m",
"uvicorn",
"youdis.main:app",
"--host",
DEFAULT_BACKEND_HOST,
"--port",
str(DEFAULT_BACKEND_PORT),
cwd=str(REPO_ROOT),
env=env,
)
@interactions.slash_command(name="adduser",description="authorize target user")
@interactions.slash_option(
name="user",
opt_type=interactions.OptionType.USER,
required=True,
description='enable this bot for target user',
)
@interactions.check(interactions.is_owner())
async def _adduser(ctx: interactions.SlashContext, user:interactions.OptionType.USER):
user_id = str(user.id)
if user_id not in authorized_users:
authorized_users.append(user_id)
save_authorized_users(authorized_users)
print(f'authorized {user_id}')
await ctx.author.send(f'authorized {user.mention}')
else:
await ctx.author.send(f'{user.mention} is already authorized')
@interactions.slash_command(name="removeuser",description="deauthorize target user")
@interactions.slash_option(
name="user",
opt_type=interactions.OptionType.USER,
required=True,
description='disable this bot for target user',
)
@interactions.check(interactions.is_owner())
async def _removeuser(ctx: interactions.SlashContext, user:interactions.OptionType.USER):
user_id = str(user.id)
if user_id in authorized_users:
authorized_users.remove(user_id)
save_authorized_users(authorized_users)
print(f'deauthorized {user_id}')
await ctx.author.send(f'deauthorized {user.mention}')
else:
await ctx.author.send(f'{user.mention} is not currently authorized')
return ManagedProcess(name="backend", process=process)
api_token = getenv('api_token')
if not api_token:
raise ValueError('API token not set. Retrieve from your Discord bot.')
bot.start(api_token)
async def start_discord(env: dict[str, str]) -> ManagedProcess:
process = await asyncio.create_subprocess_exec(
sys.executable,
"-m",
"youdis.adapters.discord",
cwd=str(REPO_ROOT),
env=env,
)
return ManagedProcess(name="discord", process=process)
ADAPTER_STARTERS = {
"discord": start_discord,
}
def selected_adapters() -> list[str]:
requested: list[str] = []
if ENABLE_DISCORD:
requested.append("discord")
return requested
async def wait_for_backend_health(timeout_seconds: float) -> None:
deadline = asyncio.get_running_loop().time() + timeout_seconds
health_url = f"{backend_url()}/health"
while True:
try:
with urlopen(health_url, timeout=2) as response:
if response.status == 200:
return
except URLError:
pass
if asyncio.get_running_loop().time() >= deadline:
raise TimeoutError(f"backend did not become healthy at {health_url}")
await asyncio.sleep(BACKEND_HEALTH_INTERVAL)
async def stop_process(proc: ManagedProcess) -> None:
if proc.process.returncode is not None:
return
proc.process.terminate()
try:
await asyncio.wait_for(proc.process.wait(), timeout=10)
except asyncio.TimeoutError:
proc.process.kill()
await proc.process.wait()
async def run() -> int:
managed: list[ManagedProcess] = []
env = build_child_env()
adapters = selected_adapters()
if not ENABLE_BACKEND and not adapters:
print("nothing to start: backend is disabled and no adapters are enabled")
return 1
try:
if ENABLE_BACKEND:
backend = await start_backend(env)
managed.append(backend)
print(f"started backend on {backend_url()}")
await wait_for_backend_health(BACKEND_HEALTH_TIMEOUT)
print("backend is healthy")
for adapter_name in adapters:
process = await ADAPTER_STARTERS[adapter_name](env)
managed.append(process)
print(f"started {adapter_name} adapter")
wait_tasks = [asyncio.create_task(proc.process.wait()) for proc in managed]
done, pending = await asyncio.wait(wait_tasks, return_when=asyncio.FIRST_COMPLETED)
for task in pending:
task.cancel()
first_returncode = 0
for task in done:
first_returncode = task.result()
break
return first_returncode
finally:
for proc in reversed(managed):
await stop_process(proc)
def main() -> None:
try:
raise SystemExit(asyncio.run(run()))
except KeyboardInterrupt:
raise SystemExit(130)
if __name__ == "__main__":
main()

View File

@@ -1 +1,231 @@
"""Discord adapter placeholder for the v2 backend."""
"""Discord adapter for interacting with FastAPI backend worker."""
import asyncio
from os import getenv
import aiohttp
import interactions
from ..env import load_project_dotenv
load_project_dotenv()
BACKEND_HOST = getenv("YOUDIS_BACKEND_HOST", "127.0.0.1")
BACKEND_PORT = int(getenv("YOUDIS_BACKEND_PORT", "8000"))
BACKEND_URL = f"http://{BACKEND_HOST}:{BACKEND_PORT}".rstrip("/")
POLL_INTERVAL_SECONDS = float(getenv("YOUDIS_POLL_INTERVAL_SECONDS", "2"))
DEFAULT_SCOPE = int(getenv("DISCORD_BOT_SCOPE", "2147491904"))
bot = interactions.Client(
intents=interactions.Intents.DEFAULT,
default_scope=DEFAULT_SCOPE,
)
http_session: aiohttp.ClientSession | None = None
poll_tasks: dict[str, asyncio.Task] = {}
def backend_url(path: str) -> str:
return f"{BACKEND_URL}{path}"
async def get_session() -> aiohttp.ClientSession:
global http_session
if http_session is None or http_session.closed:
http_session = aiohttp.ClientSession()
return http_session
async def request_json(method: str, path: str, **kwargs):
session = await get_session()
async with session.request(method, backend_url(path), **kwargs) as response:
data = await response.json()
return response.status, data
def format_status_message(job: dict) -> str:
state = job.get("state")
phase = job.get("phase")
disposition = job.get("disposition")
message = job.get("message")
result_path = job.get("result_path")
parts = [f"state={state}"]
if phase:
parts.append(f"phase={phase}")
if disposition:
parts.append(f"disposition={disposition}")
if message:
parts.append(message)
if result_path:
parts.append(f"path={result_path}")
return " | ".join(parts)
async def dm(ctx: interactions.SlashContext, message: str) -> None:
await ctx.author.send(message)
async def respond(ctx: interactions.SlashContext, message: str) -> None:
if ctx.channel is not None:
await ctx.channel.send(message)
return
await dm(ctx, message)
async def poll_job_updates(ctx: interactions.SlashContext, job_id: str) -> None:
last_sent = None
try:
while True:
status_code, payload = await request_json("GET", "/jobs/current")
if status_code != 200:
await dm(ctx, f"backend status check failed: HTTP {status_code}")
return
job = payload.get("job")
if not job:
await dm(ctx, f"job {job_id} is no longer visible from the backend")
return
if job.get("job_id") != job_id:
await dm(ctx, f"job {job_id} is no longer the current backend job")
return
summary = format_status_message(job)
if summary != last_sent:
await dm(ctx, summary)
last_sent = summary
if job.get("state") in {"completed", "failed", "cancelled"}:
return
await asyncio.sleep(POLL_INTERVAL_SECONDS)
except asyncio.CancelledError:
raise
except aiohttp.ClientError as exc:
await dm(ctx, f"backend poll failed: {exc}")
finally:
poll_tasks.pop(job_id, None)
def ensure_poll_task(ctx: interactions.SlashContext, job_id: str) -> None:
existing = poll_tasks.get(job_id)
if existing and not existing.done():
return
poll_tasks[job_id] = asyncio.create_task(poll_job_updates(ctx, job_id))
@interactions.slash_command(name="youtube", description="submit a youtube download to the backend")
@interactions.slash_option(
name="url",
opt_type=interactions.OptionType.STRING,
required=True,
description="url target",
)
async def youtube(ctx: interactions.SlashContext, url: str):
payload = {
"url": url,
"requester_id": str(ctx.author.id),
"requester_name": ctx.author.username,
"origin": f"discord:{ctx.guild_id or 'dm'}:{ctx.channel_id}",
}
try:
status_code, job = await request_json("POST", "/jobs", json=payload)
except aiohttp.ClientError as exc:
await dm(ctx, f"backend request failed: {exc}")
return
if status_code != 200:
await dm(ctx, f"backend request failed: HTTP {status_code}")
return
state = job.get("state")
job_id = job.get("job_id", "unknown")
if state == "busy":
await respond(ctx, "Backend is busy with another job. Details via DM.")
await dm(ctx, f"busy: {job.get('message')}")
return
if state != "accepted":
await respond(ctx, "Backend rejected the request. Details via DM.")
await dm(ctx, format_status_message(job))
return
await respond(ctx, f"Submitted <{url}> to the backend. Status updates via DM.")
await dm(ctx, f"accepted job {job_id} for <{url}>")
ensure_poll_task(ctx, job_id)
@interactions.slash_command(name="interrupt", description="cancel the current backend job")
@interactions.check(interactions.is_owner())
async def interrupt(ctx: interactions.SlashContext):
try:
status_code, payload = await request_json("POST", "/jobs/current/cancel")
except aiohttp.ClientError as exc:
await dm(ctx, f"backend cancel failed: {exc}")
return
if status_code == 404:
await dm(ctx, "no active backend job to interrupt")
return
if status_code != 200:
await dm(ctx, f"backend cancel failed: HTTP {status_code}")
return
await dm(ctx, format_status_message(payload))
@interactions.slash_command(name="status", description="show the current backend job status")
async def status(ctx: interactions.SlashContext):
try:
status_code, payload = await request_json("GET", "/jobs/current")
except aiohttp.ClientError as exc:
await dm(ctx, f"backend status failed: {exc}")
return
if status_code != 200:
await dm(ctx, f"backend status failed: HTTP {status_code}")
return
job = payload.get("job")
if not job:
await dm(ctx, "backend has no active or recent job")
return
active = payload.get("active")
prefix = "active" if active else "last"
await dm(ctx, f"{prefix} job: {format_status_message(job)}")
@bot.listen()
async def on_startup():
await get_session()
print(f"discord adapter configured for backend {BACKEND_URL}")
print(f"{bot.application_commands.count} registered commands:")
for i, x in enumerate(bot.application_commands,start=1):
print(f" {i}. {x.name}")
@bot.listen()
async def on_shutdown():
global http_session
for task in list(poll_tasks.values()):
task.cancel()
poll_tasks.clear()
if http_session is not None and not http_session.closed:
await http_session.close()
http_session = None
def main() -> None:
api_token = getenv("DISCORD_BOT_TOKEN")
if not api_token:
raise ValueError("API token not set. Retrieve from your Discord bot.")
# bot.add_command(youtube)
# bot.add_command(status)
# bot.add_command(interrupt)
bot.start(api_token)
if __name__ == "__main__":
main()

13
youdis/env.py Normal file
View File

@@ -0,0 +1,13 @@
from pathlib import Path
try:
from dotenv import load_dotenv
except ModuleNotFoundError: # pragma: no cover - optional local convenience dependency
load_dotenv = None
def load_project_dotenv() -> None:
if load_dotenv is None:
return
repo_root = Path(__file__).resolve().parent.parent
load_dotenv(repo_root / ".env", override=False)

View File

@@ -1,3 +1,5 @@
"""FastAPI backend worker managing yt-dlp subprocess jobs and exposing job state."""
import asyncio
from asyncio.subprocess import PIPE, STDOUT
from collections import deque
@@ -5,17 +7,28 @@ from dataclasses import dataclass, field
from datetime import datetime
from os import getenv
from pathlib import Path
import tempfile
from uuid import uuid4
from fastapi import FastAPI, HTTPException
from .env import load_project_dotenv
from .models import CurrentJobResponse, HealthResponse, JobRequest, JobStatus, VersionResponse
load_project_dotenv()
REPO_ROOT = Path(__file__).resolve().parent.parent
DEFAULT_CONFIG = REPO_ROOT / "default-yt-dlp.conf"
VERSION_FILE = REPO_ROOT / "youdis-version.txt"
YTDLP_EXECUTABLE = getenv("YOUDIS_YTDLP_EXECUTABLE", "yt-dlp")
PREFERRED_CONFIG_ROOT = Path(getenv("YOUDIS_CONFIG_DIR", "/config"))
PREFERRED_DOWNLOAD_ROOT = Path(getenv("YOUDIS_DOWNLOAD_DIR", "/downloads"))
LOCAL_RUNTIME_ROOT = REPO_ROOT / ".runtime"
FALLBACK_CONFIG_ROOT = LOCAL_RUNTIME_ROOT / "config"
FALLBACK_DOWNLOAD_ROOT = LOCAL_RUNTIME_ROOT / "downloads"
OUTPUT_TEMPLATE = "%(uploader)s/%(playlist_title)s/%(playlist_index)s%(playlist_index& - )s%(title)s.%(ext)s"
@dataclass
@@ -43,11 +56,33 @@ def read_version() -> str:
return "unknown"
def build_ytdlp_command(request: JobRequest) -> list[str]:
def ensure_writable_directory(preferred: Path, fallback: Path) -> Path:
for candidate in (preferred, fallback):
try:
candidate.mkdir(parents=True, exist_ok=True)
with tempfile.NamedTemporaryFile(dir=candidate, prefix=".youdis-write-", delete=True):
pass
return candidate
except OSError:
continue
raise OSError(f"no writable runtime directory available for {preferred} or {fallback}")
def resolve_runtime_paths() -> tuple[Path, Path]:
config_root = ensure_writable_directory(PREFERRED_CONFIG_ROOT, FALLBACK_CONFIG_ROOT)
download_root = ensure_writable_directory(PREFERRED_DOWNLOAD_ROOT, FALLBACK_DOWNLOAD_ROOT)
return config_root, download_root
def build_ytdlp_command(request: JobRequest, config_root: Path, download_root: Path) -> list[str]:
return [
YTDLP_EXECUTABLE,
"--config-locations",
str(DEFAULT_CONFIG),
"--download-archive",
str(config_root / "archive.txt"),
"--output",
str(download_root / OUTPUT_TEMPLATE),
request.url,
]
@@ -121,13 +156,11 @@ async def finalize_job(job: ManagedJob, returncode: int) -> None:
async def run_job(job: ManagedJob, request: JobRequest) -> None:
command = build_ytdlp_command(request)
update_status(
job,
state="running",
phase="starting",
message="starting yt-dlp",
command=command,
)
try:
@@ -142,6 +175,15 @@ async def run_job(job: ManagedJob, request: JobRequest) -> None:
await finalize_job(job, 78)
return
config_root, download_root = resolve_runtime_paths()
command = build_ytdlp_command(request, config_root, download_root)
update_status(
job,
command=command,
archive_path=str(config_root / "archive.txt"),
result_path=str(download_root),
)
process = await asyncio.create_subprocess_exec(
*command,
stdout=PIPE,
@@ -157,6 +199,16 @@ async def run_job(job: ManagedJob, request: JobRequest) -> None:
)
await finalize_job(job, 127)
return
except OSError as exc:
update_status(
job,
state="failed",
phase=None,
message=f"runtime path setup failed: {exc}",
returncode=73,
)
await finalize_job(job, 73)
return
try:
job.process = process
@@ -220,7 +272,11 @@ async def health() -> HealthResponse:
@app.get("/version", response_model=VersionResponse)
async def version() -> VersionResponse:
return VersionResponse(version=read_version(), active_job=active_job is not None)
return VersionResponse(
version=read_version(),
active_job=active_job is not None,
ytdlp_executable=YTDLP_EXECUTABLE,
)
@app.get("/jobs/current", response_model=CurrentJobResponse)

View File

@@ -26,6 +26,7 @@ class JobStatus(BaseModel):
requester_name: str | None = None
origin: str | None = None
result_path: str | None = None
archive_path: str | None = None
command: list[str] = Field(default_factory=list)
returncode: int | None = None
created_at: datetime = Field(default_factory=datetime.utcnow)
@@ -44,3 +45,4 @@ class HealthResponse(BaseModel):
class VersionResponse(BaseModel):
version: str
active_job: bool
ytdlp_executable: str