Compare commits
3 Commits
2a5648506e
...
043c8b4df3
| Author | SHA1 | Date | |
|---|---|---|---|
| 043c8b4df3 | |||
| 5210d2c032 | |||
| 8b21034d78 |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -23,6 +23,7 @@ env/
|
||||
/private/
|
||||
archive/
|
||||
downloads/
|
||||
test/
|
||||
data.json
|
||||
|
||||
# --- django ---
|
||||
|
||||
114
pm/tasks-v2.org
114
pm/tasks-v2.org
@@ -1,6 +1,10 @@
|
||||
#+title: Task Log
|
||||
#+updated: [2026-03-31 Tue 16:03]
|
||||
#+startup: overview
|
||||
# Local Variables:
|
||||
# org-babel-python-command: "/home/user/proj/youdis/.venv/bin/python"
|
||||
# project-dir: "~/proj/youdis"
|
||||
# End:
|
||||
|
||||
* youdis v2 goals:
|
||||
1. Separate backend from frontend
|
||||
@@ -36,7 +40,7 @@ define the target architecture for a private backend yt-dlp worker with thin cha
|
||||
** notes
|
||||
- first architecture draft captured in `docs/architecture-v2.org`
|
||||
|
||||
* [ ] 2.0.1: build backend yt-dlp worker (3)
|
||||
* [X] 2.0.1: build backend yt-dlp worker (3)
|
||||
create the minimal backend/service skeleton and establish a working yt-dlp baseline with clean hooks for future frontends
|
||||
** pm notes
|
||||
- foundation; don't need the full finished service here, just the basic shape plus enough real yt-dlp execution to validate the seam and build on it.
|
||||
@@ -55,10 +59,13 @@ create the minimal backend/service skeleton and establish a working yt-dlp basel
|
||||
- submit/request path exists
|
||||
- status/progress hook exists
|
||||
- basic health/version visibility exists
|
||||
4f.
|
||||
4. make local testing practical without breaking container defaults
|
||||
- backend can run when `/config` or `/downloads` are not writable in local dev
|
||||
- env vars may override executable/config/download paths
|
||||
- status makes the effective runtime command/paths inspectable
|
||||
|
||||
** evidence
|
||||
- commit:
|
||||
- commit: 7926534,2a56485
|
||||
- tests:
|
||||
1. `python3 -m py_compile ./youdis/main.py ./youdis/models.py ./youdis/adapters/__init__.py ./youdis/adapters/discord.py`
|
||||
2. `python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000`
|
||||
@@ -73,11 +80,12 @@ create the minimal backend/service skeleton and establish a working yt-dlp basel
|
||||
11. if testing in container: `docker run --rm -p 8000:8000 -v [config]:/config -v [downloads]:/downloads youdis:v2`
|
||||
:OUTPUT_STEP1-9:
|
||||
user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/health
|
||||
{"status":"ok"}user@paladin:~/proj/youdis$
|
||||
{"status":"ok"}
|
||||
user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/version
|
||||
{"version":"20250829-ec72c56","active_job":false}user@paladin:~/proj/youdis$
|
||||
{"version":"20250829-ec72c56","active_job":false}
|
||||
user@paladin:~/proj/youdis$ curl -X POST http://127.0.0.1:8000/jobs -H 'content-type: application/json' -d '{"url":"https://www.youtube.com/watch?v=3i72yY_LaW4"}'
|
||||
{"job_id":"cc85165e-d906-4eee-864f-1a398b6de2e0","state":"accepted","url":"https://www.youtube.com/watch?v=3i72yY_LaW4","message":"accepted","phase":"queued","disposition":null,"requester_id":null,"requester_name":null,"origin":null,"result_path":null,"command":[],"returncode":null,"created_at":"2026-04-01T00:44:35.657196","updated_at":"2026-04-01T00:44:35.657198"}user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/jobs/current
|
||||
{"job_id":"cc85165e-d906-4eee-864f-1a398b6de2e0","state":"accepted","url":"https://www.youtube.com/watch?v=3i72yY_LaW4","message":"accepted","phase":"queued","disposition":null,"requester_id":null,"requester_name":null,"origin":null,"result_path":null,"command":[],"returncode":null,"created_at":"2026-04-01T00:44:35.657196","updated_at":"2026-04-01T00:44:35.657198"}
|
||||
user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/jobs/current
|
||||
{"active":false,"job":{"job_id":"cc85165e-d906-4eee-864f-1a398b6de2e0","state":"failed","url":"https://www.youtube.com/watch?v=3i72yY_LaW4","message":"ERROR: unable to create directory [Errno 13] Permission denied: '/downloads'","phase":null,"disposition":null,"requester_id":null,"requester_name":null,"origin":null,"result_path":null,"command":["yt-dlp","--config-locations","/home/user/proj/youdis/default-yt-dlp.conf","https://www.youtube.com/watch?v=3i72yY_LaW4"],"returncode":1,"created_at":"2026-04-01T00:44:35.657196","updated_at":"2026-03-31T20:44:36.653353"}}user@paladin:~/proj/youdis$
|
||||
:END:
|
||||
- datetime: [2026-03-31 Tue 20:45]
|
||||
@@ -87,6 +95,8 @@ user@paladin:~/proj/youdis$ curl -X POST http://127.0.0.1:8000/jobs -H 'content-
|
||||
- confirm `--config-locations` behavior against the installed `yt-dlp` version during integration testing
|
||||
- current backend scaffold is not yet wired into `dockerfile` or `run-youdis.sh`
|
||||
- archive-hit and result-path parsing currently depend on `yt-dlp` stdout text patterns, so treat them as provisional until integration-tested
|
||||
- local dev now falls back to repo-local `.runtime/{config,downloads}` when `/config` or `/downloads` are not writable
|
||||
- had to uninstall yt-dlp python pkg from the venv, which resulted in a '403 Forbidden'
|
||||
|
||||
* [ ] 2.0.2: update discord bot to use new backend (3)
|
||||
update the discord bot into a thin frontend that talks to the backend and verify the flow end to end
|
||||
@@ -106,14 +116,98 @@ update the discord bot into a thin frontend that talks to the backend and verify
|
||||
- valid request path tested
|
||||
- busy or conflict behavior tested
|
||||
- failure path tested
|
||||
4. add dotenv support to ease dev
|
||||
- os.getenv methods remain standard for prod/docker build
|
||||
- populate .env with dev env defaults
|
||||
|
||||
** evidence
|
||||
- commit:
|
||||
- tests:
|
||||
- datetime:
|
||||
- tests: https://youtu.be/20HxMMSqRyg?si=3v7mN2L88c_FxpQR 18m
|
||||
1. start backend: `python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000`
|
||||
2. create local env file: `cp .env.example .env`
|
||||
3. add `api_token` to `.env`
|
||||
4. start adapter: `python3 ./youdis.py`
|
||||
5. in discord, run `/youtube url:https://www.youtube.com/watch?v=dQw4w9WgXcQ`
|
||||
:out:
|
||||
#+begin_src shell
|
||||
(venv) user@paladin:~/proj/youdis$ python ./youdis.py
|
||||
Task exception was never retrieved
|
||||
future: <Task finished name='Task-43' coro=<Client._dispatch_interaction() done, defined at /home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/client/client.py:1773> exception=KeyError('youtube')>
|
||||
Traceback (most recent call last):
|
||||
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/client/client.py", line 1798, in _dispatch_interaction
|
||||
if ctx.command:
|
||||
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/models/internal/context.py", line 329, in command
|
||||
return self.client._interaction_lookup[self._command_name]
|
||||
KeyError: 'youtube'
|
||||
#+end_src
|
||||
:end:
|
||||
|
||||
6. confirm channel response says the job was submitted to backend
|
||||
7. confirm requester receives DM updates for accepted/running/completed or failed
|
||||
:out:
|
||||
accepted job 4ef6fde5-57cc-478c-b0e2-18458c693fa4 for https://www.youtube.com/watch?v=dQw4w9WgXcQ
|
||||
state=running | phase=running | [youtube] dQw4w9WgXcQ: Downloading webpage | path=/home/user/proj/youdis/downloads
|
||||
state=running | phase=downloading | [download] Destination: /home/user/proj/youdis/downloads/Rick_Astley/NA/NANARickAstley-_Never_Gonna_Give_You_Up_Official_Video_4K_Remaster.f401.mp4 | path=/home/user/proj/youdis/downloads/Rick_Astley/NA/NANARickAstley-_Never_Gonna_Give_You_Up_Official_Video_4K_Remaster.f401.mp4
|
||||
state=completed | [Metadata] There isn't any metadata to add | path=/home/user/proj/youdis/downloads/Rick_Astley/NA/NANARickAstley-_Never_Gonna_Give_You_Up_Official_Video_4K_Remaster.mp4
|
||||
:end:
|
||||
8. while first job is active, submit another `/youtube` and confirm busy behavior
|
||||
:out:
|
||||
#+begin_src shell
|
||||
Error: AttributeError
|
||||
Traceback (most recent call last):
|
||||
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/client/client.py", line 1900, in __dispatch_interaction
|
||||
response = await callback
|
||||
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/client/client.py", line 1771, in _run_slash_command
|
||||
return await command(ctx, **ctx.kwargs)
|
||||
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/models/internal/command.py", line 132, in __call__
|
||||
await self.call_callback(self.callback, context)
|
||||
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/models/internal/application_commands.py", line 833, in call_callback
|
||||
return await self.call_with_binding(callback, ctx, *new_args, **new_kwargs)
|
||||
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/models/internal/callback.py", line 43, in call_with_binding
|
||||
return await callback(*args, **kwargs)
|
||||
File "/home/user/proj/youdis/youdis/adapters/discord.py", line 167, in youtube
|
||||
await ctx.channel.send(f"Submitted <{url}> to the backend. Status updates via DM.")
|
||||
AttributeError: 'NoneType' object has no attribute 'send'
|
||||
#+end_src
|
||||
busy: busy with https://youtu.be/20HxMMSqRyg?si=3v7mN2L88c_FxpQR
|
||||
state=running | phase=running | cancel requested | path=/home/user/proj/youdis/downloads/James_Hoffmann/NA/NANAThe_Beginner_s_Guide_To_Latte_Art.f401.mp4
|
||||
:end:
|
||||
9. run `/status` and confirm it reflects current or last backend job
|
||||
:out:
|
||||
last job: state=completed | [Metadata] There isn't any metadata to add | path=/home/user/proj/youdis/downloads/Rick_Astley/NA/NANARickAstley-_Never_Gonna_Give_You_Up_Official_Video_4K_Remaster.mp4
|
||||
:end:
|
||||
10. run `/interrupt` as owner and confirm cancellation is surfaced via DM
|
||||
:out:
|
||||
last job: state=cancelled | cancelled | path=/home/user/proj/youdis/downloads/James_Hoffmann/NA/NANAThe_Beginner_s_Guide_To_Latte_Art.f401.mp4
|
||||
:end:
|
||||
- datetime:[2026-04-02 Thu 11:52]
|
||||
|
||||
*** org-block tests
|
||||
#+begin_src shell :dir ~/proj/youdis :results output verbatim
|
||||
source ./venv/bin/activate
|
||||
python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000
|
||||
#+end_src
|
||||
|
||||
#+begin_src shell :results output
|
||||
echo ok
|
||||
#+end_src
|
||||
|
||||
#+RESULTS:
|
||||
: ok
|
||||
|
||||
#+begin_src python :dir ~/proj/youdis :results output verbatim
|
||||
python ./youdis.py
|
||||
#+end_src
|
||||
|
||||
#+RESULTS:
|
||||
|
||||
** notes
|
||||
|
||||
- discord adapter is now a thin HTTP client of the backend; it no longer imports or configures yt-dlp
|
||||
- `YOUDIS_BACKEND_URL` controls which backend the adapter targets
|
||||
- progress updates are currently implemented by polling `/jobs/current` and DMing only when the summary changes
|
||||
- legacy auth/user-management commands were removed from the active adapter path and should be cleaned up formally in `2.0.3`
|
||||
- `.env` is now supported for local/dev convenience, while real environment variables still override it in prod/docker
|
||||
- submitting via DM doesn't work
|
||||
* [ ] 2.0.3: remove deprecated discord-bot functionality (2)
|
||||
delete or retire legacy bot behaviors that no longer fit once the backend split is in place
|
||||
** pm notes
|
||||
@@ -139,7 +233,7 @@ delete or retire legacy bot behaviors that no longer fit once the backend split
|
||||
|
||||
** notes
|
||||
|
||||
* [ ] 2.0.5: fix automation and build pipeline (3)
|
||||
* [ ] 2.0.4: fix automation and build pipeline (3)
|
||||
repair and simplify the build/update/deploy path so it matches the new backend-plus-frontend structure
|
||||
** pm notes
|
||||
- this should come after architecture and discord integration stabilize. no point polishing the pipeline for the wrong shape.
|
||||
|
||||
BIN
requirements.txt
BIN
requirements.txt
Binary file not shown.
234
youdis.py
234
youdis.py
@@ -1,235 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
'''
|
||||
youdis v1.1
|
||||
bot for downloading youtube videos using yt-dlp
|
||||
discord-py-interactions 5.11.0 has new option
|
||||
requires python>=3.9
|
||||
'''
|
||||
# match_filter: info_dict -> Raise utils.DownloadCancelled(msg) ? interrupt
|
||||
|
||||
import interactions
|
||||
from os import getenv
|
||||
from pathlib import Path
|
||||
import yt_dlp
|
||||
import json
|
||||
import asyncio
|
||||
import threading
|
||||
|
||||
userFile = Path('/config/users.json')
|
||||
userFile.parent.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
bot = interactions.Client(intents=interactions.Intents.DEFAULT,default_scope=2147491904)
|
||||
|
||||
def save_authorized_users(authorized_users):
|
||||
with open(userFile, 'w') as f:
|
||||
json.dump({'authorized_users': authorized_users}, f)
|
||||
|
||||
def load_authorized_users():
|
||||
if not userFile.exists():
|
||||
save_authorized_users([])
|
||||
print(f'users.json not found; saving to {userFile}')
|
||||
return []
|
||||
|
||||
try:
|
||||
with open(userFile, 'r') as f:
|
||||
data = json.load(f)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
save_authorized_users([])
|
||||
print(f'users.json invalid; resetting {userFile}')
|
||||
return []
|
||||
|
||||
authorized_users = data.get('authorized_users', [])
|
||||
if not isinstance(authorized_users, list):
|
||||
authorized_users = []
|
||||
|
||||
authorized_users = [str(user_id) for user_id in authorized_users]
|
||||
save_authorized_users(authorized_users)
|
||||
print(f'authorized_users:{authorized_users}')
|
||||
return authorized_users
|
||||
|
||||
authorized_users = load_authorized_users()
|
||||
|
||||
active_job_lock = threading.Lock()
|
||||
active_job = None
|
||||
|
||||
async def send_message(ctx, message):
|
||||
await ctx.author.send(message)
|
||||
|
||||
def claim_active_job(job):
|
||||
global active_job
|
||||
with active_job_lock:
|
||||
if active_job is not None:
|
||||
return active_job
|
||||
active_job = job
|
||||
return None
|
||||
|
||||
def get_active_job():
|
||||
with active_job_lock:
|
||||
return active_job
|
||||
|
||||
def clear_active_job(job):
|
||||
global active_job
|
||||
with active_job_lock:
|
||||
if active_job is job:
|
||||
active_job = None
|
||||
|
||||
def download_video(url, options):
|
||||
with yt_dlp.YoutubeDL(options) as ydl:
|
||||
ydl.download(url)
|
||||
|
||||
def create_hook(ctx, loop, cancel_event):
|
||||
seen_updates = set()
|
||||
|
||||
def hook(d):
|
||||
if cancel_event.is_set():
|
||||
raise yt_dlp.utils.DownloadCancelled('download canceled by /interrupt')
|
||||
|
||||
status = d.get('status')
|
||||
info = d.get('info_dict') or {}
|
||||
|
||||
if status not in {'downloading', 'finished'}:
|
||||
return
|
||||
|
||||
filename = d.get('filename') or info.get('_filename') or info.get('title')
|
||||
update_key = (status, filename)
|
||||
if update_key in seen_updates:
|
||||
return
|
||||
|
||||
seen_updates.add(update_key)
|
||||
playlist_index = info.get('playlist_index')
|
||||
playlist_count = info.get('playlist_count')
|
||||
url = info.get('webpage_url')
|
||||
|
||||
prefix = status
|
||||
if playlist_index and playlist_count:
|
||||
prefix = f'{status} {playlist_index} of {playlist_count}'
|
||||
|
||||
msg = f'{prefix}: {filename}'
|
||||
if url:
|
||||
msg = f'{msg} <{url}>'
|
||||
|
||||
asyncio.run_coroutine_threadsafe(send_message(ctx, msg), loop)
|
||||
|
||||
return hook
|
||||
|
||||
@interactions.slash_command(name="youtube",description="download video from youtube to server")
|
||||
@interactions.slash_option(
|
||||
name='url',
|
||||
opt_type=interactions.OptionType.STRING,
|
||||
required=True,
|
||||
description='url target'
|
||||
)
|
||||
async def youtube(ctx: interactions.SlashContext, url:str):
|
||||
print(f'{ctx.author.id} requested {url}')
|
||||
# check that user is authorized
|
||||
if str(ctx.author.id) not in authorized_users:
|
||||
if ctx.author.id == 127831327012683776:
|
||||
await ctx.author.send('potato stop')
|
||||
await ctx.author.send('you are not authorized to use this command. message my owner to be added.')
|
||||
return
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
cancel_event = threading.Event()
|
||||
hook = create_hook(ctx, loop, cancel_event)
|
||||
job = {
|
||||
'requester_id': str(ctx.author.id),
|
||||
'request_url': url,
|
||||
'cancel_event': cancel_event,
|
||||
}
|
||||
existing_job = claim_active_job(job)
|
||||
if existing_job:
|
||||
await ctx.author.send(
|
||||
f'already downloading for <@{existing_job["requester_id"]}>. '
|
||||
'single-job mode is enabled right now; try again after it finishes.'
|
||||
)
|
||||
return
|
||||
|
||||
# use api_to_cli and paste cli options to get the output you need
|
||||
yoptions = {
|
||||
'format':'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best',
|
||||
'fragment_tries': 10,
|
||||
'restrictfilenames':True,
|
||||
'paths': {'home':'/downloads'},
|
||||
'retries':10,
|
||||
'writeinfojson':False,
|
||||
'allow_playlist_files':True,
|
||||
'noplaylist':True,
|
||||
'download_archive':'/config/archive.txt',
|
||||
'progress_hooks':[hook],
|
||||
'outtmpl': '%(uploader)s/%(playlist_title)s/%(playlist_index)s%(playlist_index& - )s%(title)s.%(ext)s',
|
||||
'outtmpl_na_placeholder':'',
|
||||
}
|
||||
await ctx.channel.send(f'Downloading from <{url}>. Status updates via DM. Single-job mode is enabled.')
|
||||
|
||||
try:
|
||||
await asyncio.to_thread(download_video, url, yoptions)
|
||||
except yt_dlp.utils.DownloadCancelled as exc:
|
||||
print(f'download canceled: {exc}')
|
||||
await ctx.author.send(f'download canceled: {exc}')
|
||||
except yt_dlp.utils.DownloadError as exc:
|
||||
print(f'download failed: {exc}')
|
||||
await ctx.author.send(f'download failed: {exc}')
|
||||
except Exception as exc:
|
||||
print(f'unexpected download failure: {exc}')
|
||||
await ctx.author.send(f'unexpected download failure: {exc}')
|
||||
else:
|
||||
await ctx.author.send(f'download complete for <{url}>')
|
||||
finally:
|
||||
clear_active_job(job)
|
||||
from youdis.adapters.discord import main
|
||||
|
||||
|
||||
@interactions.slash_command(name="interrupt",description="cancel current job")
|
||||
@interactions.check(interactions.is_owner())
|
||||
async def _interrupt(ctx):
|
||||
job = get_active_job()
|
||||
if not job:
|
||||
await ctx.author.send('no active download to interrupt')
|
||||
return
|
||||
|
||||
job['cancel_event'].set()
|
||||
print(f'interrupt requested for {job["request_url"]}')
|
||||
await ctx.author.send(
|
||||
f'interrupt requested for <{job["request_url"]}>; '
|
||||
'cancellation is coarse and will stop on the next yt-dlp progress update'
|
||||
)
|
||||
|
||||
@interactions.slash_command(name="adduser",description="authorize target user")
|
||||
@interactions.slash_option(
|
||||
name="user",
|
||||
opt_type=interactions.OptionType.USER,
|
||||
required=True,
|
||||
description='enable this bot for target user',
|
||||
)
|
||||
@interactions.check(interactions.is_owner())
|
||||
async def _adduser(ctx: interactions.SlashContext, user:interactions.OptionType.USER):
|
||||
user_id = str(user.id)
|
||||
if user_id not in authorized_users:
|
||||
authorized_users.append(user_id)
|
||||
save_authorized_users(authorized_users)
|
||||
print(f'authorized {user_id}')
|
||||
await ctx.author.send(f'authorized {user.mention}')
|
||||
else:
|
||||
await ctx.author.send(f'{user.mention} is already authorized')
|
||||
|
||||
@interactions.slash_command(name="removeuser",description="deauthorize target user")
|
||||
@interactions.slash_option(
|
||||
name="user",
|
||||
opt_type=interactions.OptionType.USER,
|
||||
required=True,
|
||||
description='disable this bot for target user',
|
||||
)
|
||||
@interactions.check(interactions.is_owner())
|
||||
async def _removeuser(ctx: interactions.SlashContext, user:interactions.OptionType.USER):
|
||||
user_id = str(user.id)
|
||||
if user_id in authorized_users:
|
||||
authorized_users.remove(user_id)
|
||||
save_authorized_users(authorized_users)
|
||||
print(f'deauthorized {user_id}')
|
||||
await ctx.author.send(f'deauthorized {user.mention}')
|
||||
else:
|
||||
await ctx.author.send(f'{user.mention} is not currently authorized')
|
||||
|
||||
api_token = getenv('api_token')
|
||||
if not api_token:
|
||||
raise ValueError('API token not set. Retrieve from your Discord bot.')
|
||||
bot.start(api_token)
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -1 +1,228 @@
|
||||
"""Discord adapter placeholder for the v2 backend."""
|
||||
import asyncio
|
||||
from os import getenv
|
||||
|
||||
import aiohttp
|
||||
import interactions
|
||||
|
||||
from ..env import load_project_dotenv
|
||||
|
||||
|
||||
load_project_dotenv()
|
||||
|
||||
|
||||
BACKEND_URL = getenv("YOUDIS_BACKEND_URL", "http://127.0.0.1:8000").rstrip("/")
|
||||
POLL_INTERVAL_SECONDS = float(getenv("YOUDIS_POLL_INTERVAL_SECONDS", "2"))
|
||||
DEFAULT_SCOPE = int(getenv("DISCORD_BOT_SCOPE", "2147491904"))
|
||||
|
||||
|
||||
bot = interactions.Client(
|
||||
intents=interactions.Intents.DEFAULT,
|
||||
default_scope=DEFAULT_SCOPE,
|
||||
)
|
||||
http_session: aiohttp.ClientSession | None = None
|
||||
poll_tasks: dict[str, asyncio.Task] = {}
|
||||
|
||||
|
||||
def backend_url(path: str) -> str:
|
||||
return f"{BACKEND_URL}{path}"
|
||||
|
||||
|
||||
async def get_session() -> aiohttp.ClientSession:
|
||||
global http_session
|
||||
if http_session is None or http_session.closed:
|
||||
http_session = aiohttp.ClientSession()
|
||||
return http_session
|
||||
|
||||
|
||||
async def request_json(method: str, path: str, **kwargs):
|
||||
session = await get_session()
|
||||
async with session.request(method, backend_url(path), **kwargs) as response:
|
||||
data = await response.json()
|
||||
return response.status, data
|
||||
|
||||
|
||||
def format_status_message(job: dict) -> str:
|
||||
state = job.get("state")
|
||||
phase = job.get("phase")
|
||||
disposition = job.get("disposition")
|
||||
message = job.get("message")
|
||||
result_path = job.get("result_path")
|
||||
|
||||
parts = [f"state={state}"]
|
||||
if phase:
|
||||
parts.append(f"phase={phase}")
|
||||
if disposition:
|
||||
parts.append(f"disposition={disposition}")
|
||||
if message:
|
||||
parts.append(message)
|
||||
if result_path:
|
||||
parts.append(f"path={result_path}")
|
||||
return " | ".join(parts)
|
||||
|
||||
|
||||
async def dm(ctx: interactions.SlashContext, message: str) -> None:
|
||||
await ctx.author.send(message)
|
||||
|
||||
|
||||
async def respond(ctx: interactions.SlashContext, message: str) -> None:
|
||||
if ctx.channel is not None:
|
||||
await ctx.channel.send(message)
|
||||
return
|
||||
await dm(ctx, message)
|
||||
|
||||
|
||||
async def poll_job_updates(ctx: interactions.SlashContext, job_id: str) -> None:
|
||||
last_sent = None
|
||||
try:
|
||||
while True:
|
||||
status_code, payload = await request_json("GET", "/jobs/current")
|
||||
if status_code != 200:
|
||||
await dm(ctx, f"backend status check failed: HTTP {status_code}")
|
||||
return
|
||||
|
||||
job = payload.get("job")
|
||||
if not job:
|
||||
await dm(ctx, f"job {job_id} is no longer visible from the backend")
|
||||
return
|
||||
|
||||
if job.get("job_id") != job_id:
|
||||
await dm(ctx, f"job {job_id} is no longer the current backend job")
|
||||
return
|
||||
|
||||
summary = format_status_message(job)
|
||||
if summary != last_sent:
|
||||
await dm(ctx, summary)
|
||||
last_sent = summary
|
||||
|
||||
if job.get("state") in {"completed", "failed", "cancelled"}:
|
||||
return
|
||||
|
||||
await asyncio.sleep(POLL_INTERVAL_SECONDS)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except aiohttp.ClientError as exc:
|
||||
await dm(ctx, f"backend poll failed: {exc}")
|
||||
finally:
|
||||
poll_tasks.pop(job_id, None)
|
||||
|
||||
|
||||
def ensure_poll_task(ctx: interactions.SlashContext, job_id: str) -> None:
|
||||
existing = poll_tasks.get(job_id)
|
||||
if existing and not existing.done():
|
||||
return
|
||||
poll_tasks[job_id] = asyncio.create_task(poll_job_updates(ctx, job_id))
|
||||
|
||||
@bot.listen()
|
||||
async def on_startup():
|
||||
await get_session()
|
||||
print(f"discord adapter configured for backend {BACKEND_URL}")
|
||||
print(f"discord adapter default scope: {DEFAULT_SCOPE}")
|
||||
print(f"discord adapter command cache keys: {sorted(bot._interaction_lookup.keys())}")
|
||||
|
||||
@bot.listen()
|
||||
async def on_ready():
|
||||
print(f"registered commands: {bot.application_commands}")
|
||||
|
||||
@bot.listen()
|
||||
async def on_shutdown():
|
||||
global http_session
|
||||
for task in list(poll_tasks.values()):
|
||||
task.cancel()
|
||||
poll_tasks.clear()
|
||||
if http_session is not None and not http_session.closed:
|
||||
await http_session.close()
|
||||
http_session = None
|
||||
|
||||
|
||||
@interactions.slash_command(name="youtube", description="submit a youtube download to the backend")
|
||||
@interactions.slash_option(
|
||||
name="url",
|
||||
opt_type=interactions.OptionType.STRING,
|
||||
required=True,
|
||||
description="url target",
|
||||
)
|
||||
async def youtube(ctx: interactions.SlashContext, url: str):
|
||||
payload = {
|
||||
"url": url,
|
||||
"requester_id": str(ctx.author.id),
|
||||
"requester_name": ctx.author.username,
|
||||
"origin": f"discord:{ctx.guild_id or 'dm'}:{ctx.channel_id}",
|
||||
}
|
||||
|
||||
try:
|
||||
status_code, job = await request_json("POST", "/jobs", json=payload)
|
||||
except aiohttp.ClientError as exc:
|
||||
await dm(ctx, f"backend request failed: {exc}")
|
||||
return
|
||||
|
||||
if status_code != 200:
|
||||
await dm(ctx, f"backend request failed: HTTP {status_code}")
|
||||
return
|
||||
|
||||
state = job.get("state")
|
||||
job_id = job.get("job_id", "unknown")
|
||||
if state == "busy":
|
||||
await respond(ctx, "Backend is busy with another job. Details via DM.")
|
||||
await dm(ctx, f"busy: {job.get('message')}")
|
||||
return
|
||||
|
||||
if state != "accepted":
|
||||
await respond(ctx, "Backend rejected the request. Details via DM.")
|
||||
await dm(ctx, format_status_message(job))
|
||||
return
|
||||
|
||||
await respond(ctx, f"Submitted <{url}> to the backend. Status updates via DM.")
|
||||
await dm(ctx, f"accepted job {job_id} for <{url}>")
|
||||
ensure_poll_task(ctx, job_id)
|
||||
|
||||
|
||||
@interactions.slash_command(name="interrupt", description="cancel the current backend job")
|
||||
@interactions.check(interactions.is_owner())
|
||||
async def interrupt(ctx: interactions.SlashContext):
|
||||
try:
|
||||
status_code, payload = await request_json("POST", "/jobs/current/cancel")
|
||||
except aiohttp.ClientError as exc:
|
||||
await dm(ctx, f"backend cancel failed: {exc}")
|
||||
return
|
||||
|
||||
if status_code == 404:
|
||||
await dm(ctx, "no active backend job to interrupt")
|
||||
return
|
||||
|
||||
if status_code != 200:
|
||||
await dm(ctx, f"backend cancel failed: HTTP {status_code}")
|
||||
return
|
||||
|
||||
await dm(ctx, format_status_message(payload))
|
||||
|
||||
|
||||
@interactions.slash_command(name="status", description="show the current backend job status")
|
||||
async def status(ctx: interactions.SlashContext):
|
||||
try:
|
||||
status_code, payload = await request_json("GET", "/jobs/current")
|
||||
except aiohttp.ClientError as exc:
|
||||
await dm(ctx, f"backend status failed: {exc}")
|
||||
return
|
||||
|
||||
if status_code != 200:
|
||||
await dm(ctx, f"backend status failed: HTTP {status_code}")
|
||||
return
|
||||
|
||||
job = payload.get("job")
|
||||
if not job:
|
||||
await dm(ctx, "backend has no active or recent job")
|
||||
return
|
||||
|
||||
active = payload.get("active")
|
||||
prefix = "active" if active else "last"
|
||||
await dm(ctx, f"{prefix} job: {format_status_message(job)}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
api_token = getenv("DISCORD_BOT_TOKEN")
|
||||
if not api_token:
|
||||
raise ValueError("API token not set. Retrieve from your Discord bot.")
|
||||
bot.add_command(youtube)
|
||||
bot.add_command(status)
|
||||
bot.add_command(interrupt)
|
||||
bot.start(api_token)
|
||||
|
||||
8
youdis/env.py
Normal file
8
youdis/env.py
Normal file
@@ -0,0 +1,8 @@
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
|
||||
def load_project_dotenv() -> None:
|
||||
repo_root = Path(__file__).resolve().parent.parent
|
||||
load_dotenv(repo_root / ".env", override=False)
|
||||
@@ -5,17 +5,28 @@ from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from os import getenv
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import FastAPI, HTTPException
|
||||
|
||||
from .env import load_project_dotenv
|
||||
from .models import CurrentJobResponse, HealthResponse, JobRequest, JobStatus, VersionResponse
|
||||
|
||||
|
||||
load_project_dotenv()
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
DEFAULT_CONFIG = REPO_ROOT / "default-yt-dlp.conf"
|
||||
VERSION_FILE = REPO_ROOT / "youdis-version.txt"
|
||||
YTDLP_EXECUTABLE = getenv("YOUDIS_YTDLP_EXECUTABLE", "yt-dlp")
|
||||
PREFERRED_CONFIG_ROOT = Path(getenv("YOUDIS_CONFIG_DIR", "/config"))
|
||||
PREFERRED_DOWNLOAD_ROOT = Path(getenv("YOUDIS_DOWNLOAD_DIR", "/downloads"))
|
||||
LOCAL_RUNTIME_ROOT = REPO_ROOT / ".runtime"
|
||||
FALLBACK_CONFIG_ROOT = LOCAL_RUNTIME_ROOT / "config"
|
||||
FALLBACK_DOWNLOAD_ROOT = LOCAL_RUNTIME_ROOT / "downloads"
|
||||
OUTPUT_TEMPLATE = "%(uploader)s/%(playlist_title)s/%(playlist_index)s%(playlist_index& - )s%(title)s.%(ext)s"
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -43,11 +54,33 @@ def read_version() -> str:
|
||||
return "unknown"
|
||||
|
||||
|
||||
def build_ytdlp_command(request: JobRequest) -> list[str]:
|
||||
def ensure_writable_directory(preferred: Path, fallback: Path) -> Path:
|
||||
for candidate in (preferred, fallback):
|
||||
try:
|
||||
candidate.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.NamedTemporaryFile(dir=candidate, prefix=".youdis-write-", delete=True):
|
||||
pass
|
||||
return candidate
|
||||
except OSError:
|
||||
continue
|
||||
raise OSError(f"no writable runtime directory available for {preferred} or {fallback}")
|
||||
|
||||
|
||||
def resolve_runtime_paths() -> tuple[Path, Path]:
|
||||
config_root = ensure_writable_directory(PREFERRED_CONFIG_ROOT, FALLBACK_CONFIG_ROOT)
|
||||
download_root = ensure_writable_directory(PREFERRED_DOWNLOAD_ROOT, FALLBACK_DOWNLOAD_ROOT)
|
||||
return config_root, download_root
|
||||
|
||||
|
||||
def build_ytdlp_command(request: JobRequest, config_root: Path, download_root: Path) -> list[str]:
|
||||
return [
|
||||
YTDLP_EXECUTABLE,
|
||||
"--config-locations",
|
||||
str(DEFAULT_CONFIG),
|
||||
"--download-archive",
|
||||
str(config_root / "archive.txt"),
|
||||
"--output",
|
||||
str(download_root / OUTPUT_TEMPLATE),
|
||||
request.url,
|
||||
]
|
||||
|
||||
@@ -121,13 +154,11 @@ async def finalize_job(job: ManagedJob, returncode: int) -> None:
|
||||
|
||||
|
||||
async def run_job(job: ManagedJob, request: JobRequest) -> None:
|
||||
command = build_ytdlp_command(request)
|
||||
update_status(
|
||||
job,
|
||||
state="running",
|
||||
phase="starting",
|
||||
message="starting yt-dlp",
|
||||
command=command,
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -142,6 +173,15 @@ async def run_job(job: ManagedJob, request: JobRequest) -> None:
|
||||
await finalize_job(job, 78)
|
||||
return
|
||||
|
||||
config_root, download_root = resolve_runtime_paths()
|
||||
command = build_ytdlp_command(request, config_root, download_root)
|
||||
update_status(
|
||||
job,
|
||||
command=command,
|
||||
archive_path=str(config_root / "archive.txt"),
|
||||
result_path=str(download_root),
|
||||
)
|
||||
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*command,
|
||||
stdout=PIPE,
|
||||
@@ -157,6 +197,16 @@ async def run_job(job: ManagedJob, request: JobRequest) -> None:
|
||||
)
|
||||
await finalize_job(job, 127)
|
||||
return
|
||||
except OSError as exc:
|
||||
update_status(
|
||||
job,
|
||||
state="failed",
|
||||
phase=None,
|
||||
message=f"runtime path setup failed: {exc}",
|
||||
returncode=73,
|
||||
)
|
||||
await finalize_job(job, 73)
|
||||
return
|
||||
|
||||
try:
|
||||
job.process = process
|
||||
@@ -220,7 +270,11 @@ async def health() -> HealthResponse:
|
||||
|
||||
@app.get("/version", response_model=VersionResponse)
|
||||
async def version() -> VersionResponse:
|
||||
return VersionResponse(version=read_version(), active_job=active_job is not None)
|
||||
return VersionResponse(
|
||||
version=read_version(),
|
||||
active_job=active_job is not None,
|
||||
ytdlp_executable=YTDLP_EXECUTABLE,
|
||||
)
|
||||
|
||||
|
||||
@app.get("/jobs/current", response_model=CurrentJobResponse)
|
||||
|
||||
@@ -26,6 +26,7 @@ class JobStatus(BaseModel):
|
||||
requester_name: str | None = None
|
||||
origin: str | None = None
|
||||
result_path: str | None = None
|
||||
archive_path: str | None = None
|
||||
command: list[str] = Field(default_factory=list)
|
||||
returncode: int | None = None
|
||||
created_at: datetime = Field(default_factory=datetime.utcnow)
|
||||
@@ -44,3 +45,4 @@ class HealthResponse(BaseModel):
|
||||
class VersionResponse(BaseModel):
|
||||
version: str
|
||||
active_job: bool
|
||||
ytdlp_executable: str
|
||||
|
||||
Reference in New Issue
Block a user