Compare commits
3 Commits
2a5648506e
...
043c8b4df3
| Author | SHA1 | Date | |
|---|---|---|---|
| 043c8b4df3 | |||
| 5210d2c032 | |||
| 8b21034d78 |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -23,6 +23,7 @@ env/
|
|||||||
/private/
|
/private/
|
||||||
archive/
|
archive/
|
||||||
downloads/
|
downloads/
|
||||||
|
test/
|
||||||
data.json
|
data.json
|
||||||
|
|
||||||
# --- django ---
|
# --- django ---
|
||||||
|
|||||||
114
pm/tasks-v2.org
114
pm/tasks-v2.org
@@ -1,6 +1,10 @@
|
|||||||
#+title: Task Log
|
#+title: Task Log
|
||||||
#+updated: [2026-03-31 Tue 16:03]
|
#+updated: [2026-03-31 Tue 16:03]
|
||||||
#+startup: overview
|
#+startup: overview
|
||||||
|
# Local Variables:
|
||||||
|
# org-babel-python-command: "/home/user/proj/youdis/.venv/bin/python"
|
||||||
|
# project-dir: "~/proj/youdis"
|
||||||
|
# End:
|
||||||
|
|
||||||
* youdis v2 goals:
|
* youdis v2 goals:
|
||||||
1. Separate backend from frontend
|
1. Separate backend from frontend
|
||||||
@@ -36,7 +40,7 @@ define the target architecture for a private backend yt-dlp worker with thin cha
|
|||||||
** notes
|
** notes
|
||||||
- first architecture draft captured in `docs/architecture-v2.org`
|
- first architecture draft captured in `docs/architecture-v2.org`
|
||||||
|
|
||||||
* [ ] 2.0.1: build backend yt-dlp worker (3)
|
* [X] 2.0.1: build backend yt-dlp worker (3)
|
||||||
create the minimal backend/service skeleton and establish a working yt-dlp baseline with clean hooks for future frontends
|
create the minimal backend/service skeleton and establish a working yt-dlp baseline with clean hooks for future frontends
|
||||||
** pm notes
|
** pm notes
|
||||||
- foundation; don't need the full finished service here, just the basic shape plus enough real yt-dlp execution to validate the seam and build on it.
|
- foundation; don't need the full finished service here, just the basic shape plus enough real yt-dlp execution to validate the seam and build on it.
|
||||||
@@ -55,10 +59,13 @@ create the minimal backend/service skeleton and establish a working yt-dlp basel
|
|||||||
- submit/request path exists
|
- submit/request path exists
|
||||||
- status/progress hook exists
|
- status/progress hook exists
|
||||||
- basic health/version visibility exists
|
- basic health/version visibility exists
|
||||||
4f.
|
4. make local testing practical without breaking container defaults
|
||||||
|
- backend can run when `/config` or `/downloads` are not writable in local dev
|
||||||
|
- env vars may override executable/config/download paths
|
||||||
|
- status makes the effective runtime command/paths inspectable
|
||||||
|
|
||||||
** evidence
|
** evidence
|
||||||
- commit:
|
- commit: 7926534,2a56485
|
||||||
- tests:
|
- tests:
|
||||||
1. `python3 -m py_compile ./youdis/main.py ./youdis/models.py ./youdis/adapters/__init__.py ./youdis/adapters/discord.py`
|
1. `python3 -m py_compile ./youdis/main.py ./youdis/models.py ./youdis/adapters/__init__.py ./youdis/adapters/discord.py`
|
||||||
2. `python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000`
|
2. `python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000`
|
||||||
@@ -73,11 +80,12 @@ create the minimal backend/service skeleton and establish a working yt-dlp basel
|
|||||||
11. if testing in container: `docker run --rm -p 8000:8000 -v [config]:/config -v [downloads]:/downloads youdis:v2`
|
11. if testing in container: `docker run --rm -p 8000:8000 -v [config]:/config -v [downloads]:/downloads youdis:v2`
|
||||||
:OUTPUT_STEP1-9:
|
:OUTPUT_STEP1-9:
|
||||||
user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/health
|
user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/health
|
||||||
{"status":"ok"}user@paladin:~/proj/youdis$
|
{"status":"ok"}
|
||||||
user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/version
|
user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/version
|
||||||
{"version":"20250829-ec72c56","active_job":false}user@paladin:~/proj/youdis$
|
{"version":"20250829-ec72c56","active_job":false}
|
||||||
user@paladin:~/proj/youdis$ curl -X POST http://127.0.0.1:8000/jobs -H 'content-type: application/json' -d '{"url":"https://www.youtube.com/watch?v=3i72yY_LaW4"}'
|
user@paladin:~/proj/youdis$ curl -X POST http://127.0.0.1:8000/jobs -H 'content-type: application/json' -d '{"url":"https://www.youtube.com/watch?v=3i72yY_LaW4"}'
|
||||||
{"job_id":"cc85165e-d906-4eee-864f-1a398b6de2e0","state":"accepted","url":"https://www.youtube.com/watch?v=3i72yY_LaW4","message":"accepted","phase":"queued","disposition":null,"requester_id":null,"requester_name":null,"origin":null,"result_path":null,"command":[],"returncode":null,"created_at":"2026-04-01T00:44:35.657196","updated_at":"2026-04-01T00:44:35.657198"}user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/jobs/current
|
{"job_id":"cc85165e-d906-4eee-864f-1a398b6de2e0","state":"accepted","url":"https://www.youtube.com/watch?v=3i72yY_LaW4","message":"accepted","phase":"queued","disposition":null,"requester_id":null,"requester_name":null,"origin":null,"result_path":null,"command":[],"returncode":null,"created_at":"2026-04-01T00:44:35.657196","updated_at":"2026-04-01T00:44:35.657198"}
|
||||||
|
user@paladin:~/proj/youdis$ curl http://127.0.0.1:8000/jobs/current
|
||||||
{"active":false,"job":{"job_id":"cc85165e-d906-4eee-864f-1a398b6de2e0","state":"failed","url":"https://www.youtube.com/watch?v=3i72yY_LaW4","message":"ERROR: unable to create directory [Errno 13] Permission denied: '/downloads'","phase":null,"disposition":null,"requester_id":null,"requester_name":null,"origin":null,"result_path":null,"command":["yt-dlp","--config-locations","/home/user/proj/youdis/default-yt-dlp.conf","https://www.youtube.com/watch?v=3i72yY_LaW4"],"returncode":1,"created_at":"2026-04-01T00:44:35.657196","updated_at":"2026-03-31T20:44:36.653353"}}user@paladin:~/proj/youdis$
|
{"active":false,"job":{"job_id":"cc85165e-d906-4eee-864f-1a398b6de2e0","state":"failed","url":"https://www.youtube.com/watch?v=3i72yY_LaW4","message":"ERROR: unable to create directory [Errno 13] Permission denied: '/downloads'","phase":null,"disposition":null,"requester_id":null,"requester_name":null,"origin":null,"result_path":null,"command":["yt-dlp","--config-locations","/home/user/proj/youdis/default-yt-dlp.conf","https://www.youtube.com/watch?v=3i72yY_LaW4"],"returncode":1,"created_at":"2026-04-01T00:44:35.657196","updated_at":"2026-03-31T20:44:36.653353"}}user@paladin:~/proj/youdis$
|
||||||
:END:
|
:END:
|
||||||
- datetime: [2026-03-31 Tue 20:45]
|
- datetime: [2026-03-31 Tue 20:45]
|
||||||
@@ -87,6 +95,8 @@ user@paladin:~/proj/youdis$ curl -X POST http://127.0.0.1:8000/jobs -H 'content-
|
|||||||
- confirm `--config-locations` behavior against the installed `yt-dlp` version during integration testing
|
- confirm `--config-locations` behavior against the installed `yt-dlp` version during integration testing
|
||||||
- current backend scaffold is not yet wired into `dockerfile` or `run-youdis.sh`
|
- current backend scaffold is not yet wired into `dockerfile` or `run-youdis.sh`
|
||||||
- archive-hit and result-path parsing currently depend on `yt-dlp` stdout text patterns, so treat them as provisional until integration-tested
|
- archive-hit and result-path parsing currently depend on `yt-dlp` stdout text patterns, so treat them as provisional until integration-tested
|
||||||
|
- local dev now falls back to repo-local `.runtime/{config,downloads}` when `/config` or `/downloads` are not writable
|
||||||
|
- had to uninstall yt-dlp python pkg from the venv, which resulted in a '403 Forbidden'
|
||||||
|
|
||||||
* [ ] 2.0.2: update discord bot to use new backend (3)
|
* [ ] 2.0.2: update discord bot to use new backend (3)
|
||||||
update the discord bot into a thin frontend that talks to the backend and verify the flow end to end
|
update the discord bot into a thin frontend that talks to the backend and verify the flow end to end
|
||||||
@@ -106,14 +116,98 @@ update the discord bot into a thin frontend that talks to the backend and verify
|
|||||||
- valid request path tested
|
- valid request path tested
|
||||||
- busy or conflict behavior tested
|
- busy or conflict behavior tested
|
||||||
- failure path tested
|
- failure path tested
|
||||||
|
4. add dotenv support to ease dev
|
||||||
|
- os.getenv methods remain standard for prod/docker build
|
||||||
|
- populate .env with dev env defaults
|
||||||
|
|
||||||
** evidence
|
** evidence
|
||||||
- commit:
|
- commit:
|
||||||
- tests:
|
- tests: https://youtu.be/20HxMMSqRyg?si=3v7mN2L88c_FxpQR 18m
|
||||||
- datetime:
|
1. start backend: `python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000`
|
||||||
|
2. create local env file: `cp .env.example .env`
|
||||||
|
3. add `api_token` to `.env`
|
||||||
|
4. start adapter: `python3 ./youdis.py`
|
||||||
|
5. in discord, run `/youtube url:https://www.youtube.com/watch?v=dQw4w9WgXcQ`
|
||||||
|
:out:
|
||||||
|
#+begin_src shell
|
||||||
|
(venv) user@paladin:~/proj/youdis$ python ./youdis.py
|
||||||
|
Task exception was never retrieved
|
||||||
|
future: <Task finished name='Task-43' coro=<Client._dispatch_interaction() done, defined at /home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/client/client.py:1773> exception=KeyError('youtube')>
|
||||||
|
Traceback (most recent call last):
|
||||||
|
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/client/client.py", line 1798, in _dispatch_interaction
|
||||||
|
if ctx.command:
|
||||||
|
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/models/internal/context.py", line 329, in command
|
||||||
|
return self.client._interaction_lookup[self._command_name]
|
||||||
|
KeyError: 'youtube'
|
||||||
|
#+end_src
|
||||||
|
:end:
|
||||||
|
|
||||||
|
6. confirm channel response says the job was submitted to backend
|
||||||
|
7. confirm requester receives DM updates for accepted/running/completed or failed
|
||||||
|
:out:
|
||||||
|
accepted job 4ef6fde5-57cc-478c-b0e2-18458c693fa4 for https://www.youtube.com/watch?v=dQw4w9WgXcQ
|
||||||
|
state=running | phase=running | [youtube] dQw4w9WgXcQ: Downloading webpage | path=/home/user/proj/youdis/downloads
|
||||||
|
state=running | phase=downloading | [download] Destination: /home/user/proj/youdis/downloads/Rick_Astley/NA/NANARickAstley-_Never_Gonna_Give_You_Up_Official_Video_4K_Remaster.f401.mp4 | path=/home/user/proj/youdis/downloads/Rick_Astley/NA/NANARickAstley-_Never_Gonna_Give_You_Up_Official_Video_4K_Remaster.f401.mp4
|
||||||
|
state=completed | [Metadata] There isn't any metadata to add | path=/home/user/proj/youdis/downloads/Rick_Astley/NA/NANARickAstley-_Never_Gonna_Give_You_Up_Official_Video_4K_Remaster.mp4
|
||||||
|
:end:
|
||||||
|
8. while first job is active, submit another `/youtube` and confirm busy behavior
|
||||||
|
:out:
|
||||||
|
#+begin_src shell
|
||||||
|
Error: AttributeError
|
||||||
|
Traceback (most recent call last):
|
||||||
|
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/client/client.py", line 1900, in __dispatch_interaction
|
||||||
|
response = await callback
|
||||||
|
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/client/client.py", line 1771, in _run_slash_command
|
||||||
|
return await command(ctx, **ctx.kwargs)
|
||||||
|
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/models/internal/command.py", line 132, in __call__
|
||||||
|
await self.call_callback(self.callback, context)
|
||||||
|
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/models/internal/application_commands.py", line 833, in call_callback
|
||||||
|
return await self.call_with_binding(callback, ctx, *new_args, **new_kwargs)
|
||||||
|
File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/models/internal/callback.py", line 43, in call_with_binding
|
||||||
|
return await callback(*args, **kwargs)
|
||||||
|
File "/home/user/proj/youdis/youdis/adapters/discord.py", line 167, in youtube
|
||||||
|
await ctx.channel.send(f"Submitted <{url}> to the backend. Status updates via DM.")
|
||||||
|
AttributeError: 'NoneType' object has no attribute 'send'
|
||||||
|
#+end_src
|
||||||
|
busy: busy with https://youtu.be/20HxMMSqRyg?si=3v7mN2L88c_FxpQR
|
||||||
|
state=running | phase=running | cancel requested | path=/home/user/proj/youdis/downloads/James_Hoffmann/NA/NANAThe_Beginner_s_Guide_To_Latte_Art.f401.mp4
|
||||||
|
:end:
|
||||||
|
9. run `/status` and confirm it reflects current or last backend job
|
||||||
|
:out:
|
||||||
|
last job: state=completed | [Metadata] There isn't any metadata to add | path=/home/user/proj/youdis/downloads/Rick_Astley/NA/NANARickAstley-_Never_Gonna_Give_You_Up_Official_Video_4K_Remaster.mp4
|
||||||
|
:end:
|
||||||
|
10. run `/interrupt` as owner and confirm cancellation is surfaced via DM
|
||||||
|
:out:
|
||||||
|
last job: state=cancelled | cancelled | path=/home/user/proj/youdis/downloads/James_Hoffmann/NA/NANAThe_Beginner_s_Guide_To_Latte_Art.f401.mp4
|
||||||
|
:end:
|
||||||
|
- datetime:[2026-04-02 Thu 11:52]
|
||||||
|
|
||||||
|
*** org-block tests
|
||||||
|
#+begin_src shell :dir ~/proj/youdis :results output verbatim
|
||||||
|
source ./venv/bin/activate
|
||||||
|
python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000
|
||||||
|
#+end_src
|
||||||
|
|
||||||
|
#+begin_src shell :results output
|
||||||
|
echo ok
|
||||||
|
#+end_src
|
||||||
|
|
||||||
|
#+RESULTS:
|
||||||
|
: ok
|
||||||
|
|
||||||
|
#+begin_src python :dir ~/proj/youdis :results output verbatim
|
||||||
|
python ./youdis.py
|
||||||
|
#+end_src
|
||||||
|
|
||||||
|
#+RESULTS:
|
||||||
|
|
||||||
** notes
|
** notes
|
||||||
|
- discord adapter is now a thin HTTP client of the backend; it no longer imports or configures yt-dlp
|
||||||
|
- `YOUDIS_BACKEND_URL` controls which backend the adapter targets
|
||||||
|
- progress updates are currently implemented by polling `/jobs/current` and DMing only when the summary changes
|
||||||
|
- legacy auth/user-management commands were removed from the active adapter path and should be cleaned up formally in `2.0.3`
|
||||||
|
- `.env` is now supported for local/dev convenience, while real environment variables still override it in prod/docker
|
||||||
|
- submitting via DM doesn't work
|
||||||
* [ ] 2.0.3: remove deprecated discord-bot functionality (2)
|
* [ ] 2.0.3: remove deprecated discord-bot functionality (2)
|
||||||
delete or retire legacy bot behaviors that no longer fit once the backend split is in place
|
delete or retire legacy bot behaviors that no longer fit once the backend split is in place
|
||||||
** pm notes
|
** pm notes
|
||||||
@@ -139,7 +233,7 @@ delete or retire legacy bot behaviors that no longer fit once the backend split
|
|||||||
|
|
||||||
** notes
|
** notes
|
||||||
|
|
||||||
* [ ] 2.0.5: fix automation and build pipeline (3)
|
* [ ] 2.0.4: fix automation and build pipeline (3)
|
||||||
repair and simplify the build/update/deploy path so it matches the new backend-plus-frontend structure
|
repair and simplify the build/update/deploy path so it matches the new backend-plus-frontend structure
|
||||||
** pm notes
|
** pm notes
|
||||||
- this should come after architecture and discord integration stabilize. no point polishing the pipeline for the wrong shape.
|
- this should come after architecture and discord integration stabilize. no point polishing the pipeline for the wrong shape.
|
||||||
|
|||||||
BIN
requirements.txt
BIN
requirements.txt
Binary file not shown.
234
youdis.py
234
youdis.py
@@ -1,235 +1,7 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
'''
|
|
||||||
youdis v1.1
|
|
||||||
bot for downloading youtube videos using yt-dlp
|
|
||||||
discord-py-interactions 5.11.0 has new option
|
|
||||||
requires python>=3.9
|
|
||||||
'''
|
|
||||||
# match_filter: info_dict -> Raise utils.DownloadCancelled(msg) ? interrupt
|
|
||||||
|
|
||||||
import interactions
|
from youdis.adapters.discord import main
|
||||||
from os import getenv
|
|
||||||
from pathlib import Path
|
|
||||||
import yt_dlp
|
|
||||||
import json
|
|
||||||
import asyncio
|
|
||||||
import threading
|
|
||||||
|
|
||||||
userFile = Path('/config/users.json')
|
|
||||||
userFile.parent.mkdir(exist_ok=True, parents=True)
|
|
||||||
|
|
||||||
bot = interactions.Client(intents=interactions.Intents.DEFAULT,default_scope=2147491904)
|
|
||||||
|
|
||||||
def save_authorized_users(authorized_users):
|
|
||||||
with open(userFile, 'w') as f:
|
|
||||||
json.dump({'authorized_users': authorized_users}, f)
|
|
||||||
|
|
||||||
def load_authorized_users():
|
|
||||||
if not userFile.exists():
|
|
||||||
save_authorized_users([])
|
|
||||||
print(f'users.json not found; saving to {userFile}')
|
|
||||||
return []
|
|
||||||
|
|
||||||
try:
|
|
||||||
with open(userFile, 'r') as f:
|
|
||||||
data = json.load(f)
|
|
||||||
except (json.JSONDecodeError, OSError):
|
|
||||||
save_authorized_users([])
|
|
||||||
print(f'users.json invalid; resetting {userFile}')
|
|
||||||
return []
|
|
||||||
|
|
||||||
authorized_users = data.get('authorized_users', [])
|
|
||||||
if not isinstance(authorized_users, list):
|
|
||||||
authorized_users = []
|
|
||||||
|
|
||||||
authorized_users = [str(user_id) for user_id in authorized_users]
|
|
||||||
save_authorized_users(authorized_users)
|
|
||||||
print(f'authorized_users:{authorized_users}')
|
|
||||||
return authorized_users
|
|
||||||
|
|
||||||
authorized_users = load_authorized_users()
|
|
||||||
|
|
||||||
active_job_lock = threading.Lock()
|
|
||||||
active_job = None
|
|
||||||
|
|
||||||
async def send_message(ctx, message):
|
|
||||||
await ctx.author.send(message)
|
|
||||||
|
|
||||||
def claim_active_job(job):
|
|
||||||
global active_job
|
|
||||||
with active_job_lock:
|
|
||||||
if active_job is not None:
|
|
||||||
return active_job
|
|
||||||
active_job = job
|
|
||||||
return None
|
|
||||||
|
|
||||||
def get_active_job():
|
|
||||||
with active_job_lock:
|
|
||||||
return active_job
|
|
||||||
|
|
||||||
def clear_active_job(job):
|
|
||||||
global active_job
|
|
||||||
with active_job_lock:
|
|
||||||
if active_job is job:
|
|
||||||
active_job = None
|
|
||||||
|
|
||||||
def download_video(url, options):
|
|
||||||
with yt_dlp.YoutubeDL(options) as ydl:
|
|
||||||
ydl.download(url)
|
|
||||||
|
|
||||||
def create_hook(ctx, loop, cancel_event):
|
|
||||||
seen_updates = set()
|
|
||||||
|
|
||||||
def hook(d):
|
|
||||||
if cancel_event.is_set():
|
|
||||||
raise yt_dlp.utils.DownloadCancelled('download canceled by /interrupt')
|
|
||||||
|
|
||||||
status = d.get('status')
|
|
||||||
info = d.get('info_dict') or {}
|
|
||||||
|
|
||||||
if status not in {'downloading', 'finished'}:
|
|
||||||
return
|
|
||||||
|
|
||||||
filename = d.get('filename') or info.get('_filename') or info.get('title')
|
|
||||||
update_key = (status, filename)
|
|
||||||
if update_key in seen_updates:
|
|
||||||
return
|
|
||||||
|
|
||||||
seen_updates.add(update_key)
|
|
||||||
playlist_index = info.get('playlist_index')
|
|
||||||
playlist_count = info.get('playlist_count')
|
|
||||||
url = info.get('webpage_url')
|
|
||||||
|
|
||||||
prefix = status
|
|
||||||
if playlist_index and playlist_count:
|
|
||||||
prefix = f'{status} {playlist_index} of {playlist_count}'
|
|
||||||
|
|
||||||
msg = f'{prefix}: {filename}'
|
|
||||||
if url:
|
|
||||||
msg = f'{msg} <{url}>'
|
|
||||||
|
|
||||||
asyncio.run_coroutine_threadsafe(send_message(ctx, msg), loop)
|
|
||||||
|
|
||||||
return hook
|
|
||||||
|
|
||||||
@interactions.slash_command(name="youtube",description="download video from youtube to server")
|
|
||||||
@interactions.slash_option(
|
|
||||||
name='url',
|
|
||||||
opt_type=interactions.OptionType.STRING,
|
|
||||||
required=True,
|
|
||||||
description='url target'
|
|
||||||
)
|
|
||||||
async def youtube(ctx: interactions.SlashContext, url:str):
|
|
||||||
print(f'{ctx.author.id} requested {url}')
|
|
||||||
# check that user is authorized
|
|
||||||
if str(ctx.author.id) not in authorized_users:
|
|
||||||
if ctx.author.id == 127831327012683776:
|
|
||||||
await ctx.author.send('potato stop')
|
|
||||||
await ctx.author.send('you are not authorized to use this command. message my owner to be added.')
|
|
||||||
return
|
|
||||||
|
|
||||||
loop = asyncio.get_running_loop()
|
|
||||||
cancel_event = threading.Event()
|
|
||||||
hook = create_hook(ctx, loop, cancel_event)
|
|
||||||
job = {
|
|
||||||
'requester_id': str(ctx.author.id),
|
|
||||||
'request_url': url,
|
|
||||||
'cancel_event': cancel_event,
|
|
||||||
}
|
|
||||||
existing_job = claim_active_job(job)
|
|
||||||
if existing_job:
|
|
||||||
await ctx.author.send(
|
|
||||||
f'already downloading for <@{existing_job["requester_id"]}>. '
|
|
||||||
'single-job mode is enabled right now; try again after it finishes.'
|
|
||||||
)
|
|
||||||
return
|
|
||||||
|
|
||||||
# use api_to_cli and paste cli options to get the output you need
|
|
||||||
yoptions = {
|
|
||||||
'format':'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best',
|
|
||||||
'fragment_tries': 10,
|
|
||||||
'restrictfilenames':True,
|
|
||||||
'paths': {'home':'/downloads'},
|
|
||||||
'retries':10,
|
|
||||||
'writeinfojson':False,
|
|
||||||
'allow_playlist_files':True,
|
|
||||||
'noplaylist':True,
|
|
||||||
'download_archive':'/config/archive.txt',
|
|
||||||
'progress_hooks':[hook],
|
|
||||||
'outtmpl': '%(uploader)s/%(playlist_title)s/%(playlist_index)s%(playlist_index& - )s%(title)s.%(ext)s',
|
|
||||||
'outtmpl_na_placeholder':'',
|
|
||||||
}
|
|
||||||
await ctx.channel.send(f'Downloading from <{url}>. Status updates via DM. Single-job mode is enabled.')
|
|
||||||
|
|
||||||
try:
|
|
||||||
await asyncio.to_thread(download_video, url, yoptions)
|
|
||||||
except yt_dlp.utils.DownloadCancelled as exc:
|
|
||||||
print(f'download canceled: {exc}')
|
|
||||||
await ctx.author.send(f'download canceled: {exc}')
|
|
||||||
except yt_dlp.utils.DownloadError as exc:
|
|
||||||
print(f'download failed: {exc}')
|
|
||||||
await ctx.author.send(f'download failed: {exc}')
|
|
||||||
except Exception as exc:
|
|
||||||
print(f'unexpected download failure: {exc}')
|
|
||||||
await ctx.author.send(f'unexpected download failure: {exc}')
|
|
||||||
else:
|
|
||||||
await ctx.author.send(f'download complete for <{url}>')
|
|
||||||
finally:
|
|
||||||
clear_active_job(job)
|
|
||||||
|
|
||||||
|
|
||||||
@interactions.slash_command(name="interrupt",description="cancel current job")
|
if __name__ == "__main__":
|
||||||
@interactions.check(interactions.is_owner())
|
main()
|
||||||
async def _interrupt(ctx):
|
|
||||||
job = get_active_job()
|
|
||||||
if not job:
|
|
||||||
await ctx.author.send('no active download to interrupt')
|
|
||||||
return
|
|
||||||
|
|
||||||
job['cancel_event'].set()
|
|
||||||
print(f'interrupt requested for {job["request_url"]}')
|
|
||||||
await ctx.author.send(
|
|
||||||
f'interrupt requested for <{job["request_url"]}>; '
|
|
||||||
'cancellation is coarse and will stop on the next yt-dlp progress update'
|
|
||||||
)
|
|
||||||
|
|
||||||
@interactions.slash_command(name="adduser",description="authorize target user")
|
|
||||||
@interactions.slash_option(
|
|
||||||
name="user",
|
|
||||||
opt_type=interactions.OptionType.USER,
|
|
||||||
required=True,
|
|
||||||
description='enable this bot for target user',
|
|
||||||
)
|
|
||||||
@interactions.check(interactions.is_owner())
|
|
||||||
async def _adduser(ctx: interactions.SlashContext, user:interactions.OptionType.USER):
|
|
||||||
user_id = str(user.id)
|
|
||||||
if user_id not in authorized_users:
|
|
||||||
authorized_users.append(user_id)
|
|
||||||
save_authorized_users(authorized_users)
|
|
||||||
print(f'authorized {user_id}')
|
|
||||||
await ctx.author.send(f'authorized {user.mention}')
|
|
||||||
else:
|
|
||||||
await ctx.author.send(f'{user.mention} is already authorized')
|
|
||||||
|
|
||||||
@interactions.slash_command(name="removeuser",description="deauthorize target user")
|
|
||||||
@interactions.slash_option(
|
|
||||||
name="user",
|
|
||||||
opt_type=interactions.OptionType.USER,
|
|
||||||
required=True,
|
|
||||||
description='disable this bot for target user',
|
|
||||||
)
|
|
||||||
@interactions.check(interactions.is_owner())
|
|
||||||
async def _removeuser(ctx: interactions.SlashContext, user:interactions.OptionType.USER):
|
|
||||||
user_id = str(user.id)
|
|
||||||
if user_id in authorized_users:
|
|
||||||
authorized_users.remove(user_id)
|
|
||||||
save_authorized_users(authorized_users)
|
|
||||||
print(f'deauthorized {user_id}')
|
|
||||||
await ctx.author.send(f'deauthorized {user.mention}')
|
|
||||||
else:
|
|
||||||
await ctx.author.send(f'{user.mention} is not currently authorized')
|
|
||||||
|
|
||||||
api_token = getenv('api_token')
|
|
||||||
if not api_token:
|
|
||||||
raise ValueError('API token not set. Retrieve from your Discord bot.')
|
|
||||||
bot.start(api_token)
|
|
||||||
|
|||||||
@@ -1 +1,228 @@
|
|||||||
"""Discord adapter placeholder for the v2 backend."""
|
import asyncio
|
||||||
|
from os import getenv
|
||||||
|
|
||||||
|
import aiohttp
|
||||||
|
import interactions
|
||||||
|
|
||||||
|
from ..env import load_project_dotenv
|
||||||
|
|
||||||
|
|
||||||
|
load_project_dotenv()
|
||||||
|
|
||||||
|
|
||||||
|
BACKEND_URL = getenv("YOUDIS_BACKEND_URL", "http://127.0.0.1:8000").rstrip("/")
|
||||||
|
POLL_INTERVAL_SECONDS = float(getenv("YOUDIS_POLL_INTERVAL_SECONDS", "2"))
|
||||||
|
DEFAULT_SCOPE = int(getenv("DISCORD_BOT_SCOPE", "2147491904"))
|
||||||
|
|
||||||
|
|
||||||
|
bot = interactions.Client(
|
||||||
|
intents=interactions.Intents.DEFAULT,
|
||||||
|
default_scope=DEFAULT_SCOPE,
|
||||||
|
)
|
||||||
|
http_session: aiohttp.ClientSession | None = None
|
||||||
|
poll_tasks: dict[str, asyncio.Task] = {}
|
||||||
|
|
||||||
|
|
||||||
|
def backend_url(path: str) -> str:
|
||||||
|
return f"{BACKEND_URL}{path}"
|
||||||
|
|
||||||
|
|
||||||
|
async def get_session() -> aiohttp.ClientSession:
|
||||||
|
global http_session
|
||||||
|
if http_session is None or http_session.closed:
|
||||||
|
http_session = aiohttp.ClientSession()
|
||||||
|
return http_session
|
||||||
|
|
||||||
|
|
||||||
|
async def request_json(method: str, path: str, **kwargs):
|
||||||
|
session = await get_session()
|
||||||
|
async with session.request(method, backend_url(path), **kwargs) as response:
|
||||||
|
data = await response.json()
|
||||||
|
return response.status, data
|
||||||
|
|
||||||
|
|
||||||
|
def format_status_message(job: dict) -> str:
|
||||||
|
state = job.get("state")
|
||||||
|
phase = job.get("phase")
|
||||||
|
disposition = job.get("disposition")
|
||||||
|
message = job.get("message")
|
||||||
|
result_path = job.get("result_path")
|
||||||
|
|
||||||
|
parts = [f"state={state}"]
|
||||||
|
if phase:
|
||||||
|
parts.append(f"phase={phase}")
|
||||||
|
if disposition:
|
||||||
|
parts.append(f"disposition={disposition}")
|
||||||
|
if message:
|
||||||
|
parts.append(message)
|
||||||
|
if result_path:
|
||||||
|
parts.append(f"path={result_path}")
|
||||||
|
return " | ".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
async def dm(ctx: interactions.SlashContext, message: str) -> None:
|
||||||
|
await ctx.author.send(message)
|
||||||
|
|
||||||
|
|
||||||
|
async def respond(ctx: interactions.SlashContext, message: str) -> None:
|
||||||
|
if ctx.channel is not None:
|
||||||
|
await ctx.channel.send(message)
|
||||||
|
return
|
||||||
|
await dm(ctx, message)
|
||||||
|
|
||||||
|
|
||||||
|
async def poll_job_updates(ctx: interactions.SlashContext, job_id: str) -> None:
|
||||||
|
last_sent = None
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
status_code, payload = await request_json("GET", "/jobs/current")
|
||||||
|
if status_code != 200:
|
||||||
|
await dm(ctx, f"backend status check failed: HTTP {status_code}")
|
||||||
|
return
|
||||||
|
|
||||||
|
job = payload.get("job")
|
||||||
|
if not job:
|
||||||
|
await dm(ctx, f"job {job_id} is no longer visible from the backend")
|
||||||
|
return
|
||||||
|
|
||||||
|
if job.get("job_id") != job_id:
|
||||||
|
await dm(ctx, f"job {job_id} is no longer the current backend job")
|
||||||
|
return
|
||||||
|
|
||||||
|
summary = format_status_message(job)
|
||||||
|
if summary != last_sent:
|
||||||
|
await dm(ctx, summary)
|
||||||
|
last_sent = summary
|
||||||
|
|
||||||
|
if job.get("state") in {"completed", "failed", "cancelled"}:
|
||||||
|
return
|
||||||
|
|
||||||
|
await asyncio.sleep(POLL_INTERVAL_SECONDS)
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
raise
|
||||||
|
except aiohttp.ClientError as exc:
|
||||||
|
await dm(ctx, f"backend poll failed: {exc}")
|
||||||
|
finally:
|
||||||
|
poll_tasks.pop(job_id, None)
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_poll_task(ctx: interactions.SlashContext, job_id: str) -> None:
|
||||||
|
existing = poll_tasks.get(job_id)
|
||||||
|
if existing and not existing.done():
|
||||||
|
return
|
||||||
|
poll_tasks[job_id] = asyncio.create_task(poll_job_updates(ctx, job_id))
|
||||||
|
|
||||||
|
@bot.listen()
|
||||||
|
async def on_startup():
|
||||||
|
await get_session()
|
||||||
|
print(f"discord adapter configured for backend {BACKEND_URL}")
|
||||||
|
print(f"discord adapter default scope: {DEFAULT_SCOPE}")
|
||||||
|
print(f"discord adapter command cache keys: {sorted(bot._interaction_lookup.keys())}")
|
||||||
|
|
||||||
|
@bot.listen()
|
||||||
|
async def on_ready():
|
||||||
|
print(f"registered commands: {bot.application_commands}")
|
||||||
|
|
||||||
|
@bot.listen()
|
||||||
|
async def on_shutdown():
|
||||||
|
global http_session
|
||||||
|
for task in list(poll_tasks.values()):
|
||||||
|
task.cancel()
|
||||||
|
poll_tasks.clear()
|
||||||
|
if http_session is not None and not http_session.closed:
|
||||||
|
await http_session.close()
|
||||||
|
http_session = None
|
||||||
|
|
||||||
|
|
||||||
|
@interactions.slash_command(name="youtube", description="submit a youtube download to the backend")
|
||||||
|
@interactions.slash_option(
|
||||||
|
name="url",
|
||||||
|
opt_type=interactions.OptionType.STRING,
|
||||||
|
required=True,
|
||||||
|
description="url target",
|
||||||
|
)
|
||||||
|
async def youtube(ctx: interactions.SlashContext, url: str):
|
||||||
|
payload = {
|
||||||
|
"url": url,
|
||||||
|
"requester_id": str(ctx.author.id),
|
||||||
|
"requester_name": ctx.author.username,
|
||||||
|
"origin": f"discord:{ctx.guild_id or 'dm'}:{ctx.channel_id}",
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
status_code, job = await request_json("POST", "/jobs", json=payload)
|
||||||
|
except aiohttp.ClientError as exc:
|
||||||
|
await dm(ctx, f"backend request failed: {exc}")
|
||||||
|
return
|
||||||
|
|
||||||
|
if status_code != 200:
|
||||||
|
await dm(ctx, f"backend request failed: HTTP {status_code}")
|
||||||
|
return
|
||||||
|
|
||||||
|
state = job.get("state")
|
||||||
|
job_id = job.get("job_id", "unknown")
|
||||||
|
if state == "busy":
|
||||||
|
await respond(ctx, "Backend is busy with another job. Details via DM.")
|
||||||
|
await dm(ctx, f"busy: {job.get('message')}")
|
||||||
|
return
|
||||||
|
|
||||||
|
if state != "accepted":
|
||||||
|
await respond(ctx, "Backend rejected the request. Details via DM.")
|
||||||
|
await dm(ctx, format_status_message(job))
|
||||||
|
return
|
||||||
|
|
||||||
|
await respond(ctx, f"Submitted <{url}> to the backend. Status updates via DM.")
|
||||||
|
await dm(ctx, f"accepted job {job_id} for <{url}>")
|
||||||
|
ensure_poll_task(ctx, job_id)
|
||||||
|
|
||||||
|
|
||||||
|
@interactions.slash_command(name="interrupt", description="cancel the current backend job")
|
||||||
|
@interactions.check(interactions.is_owner())
|
||||||
|
async def interrupt(ctx: interactions.SlashContext):
|
||||||
|
try:
|
||||||
|
status_code, payload = await request_json("POST", "/jobs/current/cancel")
|
||||||
|
except aiohttp.ClientError as exc:
|
||||||
|
await dm(ctx, f"backend cancel failed: {exc}")
|
||||||
|
return
|
||||||
|
|
||||||
|
if status_code == 404:
|
||||||
|
await dm(ctx, "no active backend job to interrupt")
|
||||||
|
return
|
||||||
|
|
||||||
|
if status_code != 200:
|
||||||
|
await dm(ctx, f"backend cancel failed: HTTP {status_code}")
|
||||||
|
return
|
||||||
|
|
||||||
|
await dm(ctx, format_status_message(payload))
|
||||||
|
|
||||||
|
|
||||||
|
@interactions.slash_command(name="status", description="show the current backend job status")
|
||||||
|
async def status(ctx: interactions.SlashContext):
|
||||||
|
try:
|
||||||
|
status_code, payload = await request_json("GET", "/jobs/current")
|
||||||
|
except aiohttp.ClientError as exc:
|
||||||
|
await dm(ctx, f"backend status failed: {exc}")
|
||||||
|
return
|
||||||
|
|
||||||
|
if status_code != 200:
|
||||||
|
await dm(ctx, f"backend status failed: HTTP {status_code}")
|
||||||
|
return
|
||||||
|
|
||||||
|
job = payload.get("job")
|
||||||
|
if not job:
|
||||||
|
await dm(ctx, "backend has no active or recent job")
|
||||||
|
return
|
||||||
|
|
||||||
|
active = payload.get("active")
|
||||||
|
prefix = "active" if active else "last"
|
||||||
|
await dm(ctx, f"{prefix} job: {format_status_message(job)}")
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
api_token = getenv("DISCORD_BOT_TOKEN")
|
||||||
|
if not api_token:
|
||||||
|
raise ValueError("API token not set. Retrieve from your Discord bot.")
|
||||||
|
bot.add_command(youtube)
|
||||||
|
bot.add_command(status)
|
||||||
|
bot.add_command(interrupt)
|
||||||
|
bot.start(api_token)
|
||||||
|
|||||||
8
youdis/env.py
Normal file
8
youdis/env.py
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
|
||||||
|
|
||||||
|
def load_project_dotenv() -> None:
|
||||||
|
repo_root = Path(__file__).resolve().parent.parent
|
||||||
|
load_dotenv(repo_root / ".env", override=False)
|
||||||
@@ -5,17 +5,28 @@ from dataclasses import dataclass, field
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from os import getenv
|
from os import getenv
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
from fastapi import FastAPI, HTTPException
|
from fastapi import FastAPI, HTTPException
|
||||||
|
|
||||||
|
from .env import load_project_dotenv
|
||||||
from .models import CurrentJobResponse, HealthResponse, JobRequest, JobStatus, VersionResponse
|
from .models import CurrentJobResponse, HealthResponse, JobRequest, JobStatus, VersionResponse
|
||||||
|
|
||||||
|
|
||||||
|
load_project_dotenv()
|
||||||
|
|
||||||
|
|
||||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||||
DEFAULT_CONFIG = REPO_ROOT / "default-yt-dlp.conf"
|
DEFAULT_CONFIG = REPO_ROOT / "default-yt-dlp.conf"
|
||||||
VERSION_FILE = REPO_ROOT / "youdis-version.txt"
|
VERSION_FILE = REPO_ROOT / "youdis-version.txt"
|
||||||
YTDLP_EXECUTABLE = getenv("YOUDIS_YTDLP_EXECUTABLE", "yt-dlp")
|
YTDLP_EXECUTABLE = getenv("YOUDIS_YTDLP_EXECUTABLE", "yt-dlp")
|
||||||
|
PREFERRED_CONFIG_ROOT = Path(getenv("YOUDIS_CONFIG_DIR", "/config"))
|
||||||
|
PREFERRED_DOWNLOAD_ROOT = Path(getenv("YOUDIS_DOWNLOAD_DIR", "/downloads"))
|
||||||
|
LOCAL_RUNTIME_ROOT = REPO_ROOT / ".runtime"
|
||||||
|
FALLBACK_CONFIG_ROOT = LOCAL_RUNTIME_ROOT / "config"
|
||||||
|
FALLBACK_DOWNLOAD_ROOT = LOCAL_RUNTIME_ROOT / "downloads"
|
||||||
|
OUTPUT_TEMPLATE = "%(uploader)s/%(playlist_title)s/%(playlist_index)s%(playlist_index& - )s%(title)s.%(ext)s"
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -43,11 +54,33 @@ def read_version() -> str:
|
|||||||
return "unknown"
|
return "unknown"
|
||||||
|
|
||||||
|
|
||||||
def build_ytdlp_command(request: JobRequest) -> list[str]:
|
def ensure_writable_directory(preferred: Path, fallback: Path) -> Path:
|
||||||
|
for candidate in (preferred, fallback):
|
||||||
|
try:
|
||||||
|
candidate.mkdir(parents=True, exist_ok=True)
|
||||||
|
with tempfile.NamedTemporaryFile(dir=candidate, prefix=".youdis-write-", delete=True):
|
||||||
|
pass
|
||||||
|
return candidate
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
raise OSError(f"no writable runtime directory available for {preferred} or {fallback}")
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_runtime_paths() -> tuple[Path, Path]:
|
||||||
|
config_root = ensure_writable_directory(PREFERRED_CONFIG_ROOT, FALLBACK_CONFIG_ROOT)
|
||||||
|
download_root = ensure_writable_directory(PREFERRED_DOWNLOAD_ROOT, FALLBACK_DOWNLOAD_ROOT)
|
||||||
|
return config_root, download_root
|
||||||
|
|
||||||
|
|
||||||
|
def build_ytdlp_command(request: JobRequest, config_root: Path, download_root: Path) -> list[str]:
|
||||||
return [
|
return [
|
||||||
YTDLP_EXECUTABLE,
|
YTDLP_EXECUTABLE,
|
||||||
"--config-locations",
|
"--config-locations",
|
||||||
str(DEFAULT_CONFIG),
|
str(DEFAULT_CONFIG),
|
||||||
|
"--download-archive",
|
||||||
|
str(config_root / "archive.txt"),
|
||||||
|
"--output",
|
||||||
|
str(download_root / OUTPUT_TEMPLATE),
|
||||||
request.url,
|
request.url,
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -121,13 +154,11 @@ async def finalize_job(job: ManagedJob, returncode: int) -> None:
|
|||||||
|
|
||||||
|
|
||||||
async def run_job(job: ManagedJob, request: JobRequest) -> None:
|
async def run_job(job: ManagedJob, request: JobRequest) -> None:
|
||||||
command = build_ytdlp_command(request)
|
|
||||||
update_status(
|
update_status(
|
||||||
job,
|
job,
|
||||||
state="running",
|
state="running",
|
||||||
phase="starting",
|
phase="starting",
|
||||||
message="starting yt-dlp",
|
message="starting yt-dlp",
|
||||||
command=command,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -142,6 +173,15 @@ async def run_job(job: ManagedJob, request: JobRequest) -> None:
|
|||||||
await finalize_job(job, 78)
|
await finalize_job(job, 78)
|
||||||
return
|
return
|
||||||
|
|
||||||
|
config_root, download_root = resolve_runtime_paths()
|
||||||
|
command = build_ytdlp_command(request, config_root, download_root)
|
||||||
|
update_status(
|
||||||
|
job,
|
||||||
|
command=command,
|
||||||
|
archive_path=str(config_root / "archive.txt"),
|
||||||
|
result_path=str(download_root),
|
||||||
|
)
|
||||||
|
|
||||||
process = await asyncio.create_subprocess_exec(
|
process = await asyncio.create_subprocess_exec(
|
||||||
*command,
|
*command,
|
||||||
stdout=PIPE,
|
stdout=PIPE,
|
||||||
@@ -157,6 +197,16 @@ async def run_job(job: ManagedJob, request: JobRequest) -> None:
|
|||||||
)
|
)
|
||||||
await finalize_job(job, 127)
|
await finalize_job(job, 127)
|
||||||
return
|
return
|
||||||
|
except OSError as exc:
|
||||||
|
update_status(
|
||||||
|
job,
|
||||||
|
state="failed",
|
||||||
|
phase=None,
|
||||||
|
message=f"runtime path setup failed: {exc}",
|
||||||
|
returncode=73,
|
||||||
|
)
|
||||||
|
await finalize_job(job, 73)
|
||||||
|
return
|
||||||
|
|
||||||
try:
|
try:
|
||||||
job.process = process
|
job.process = process
|
||||||
@@ -220,7 +270,11 @@ async def health() -> HealthResponse:
|
|||||||
|
|
||||||
@app.get("/version", response_model=VersionResponse)
|
@app.get("/version", response_model=VersionResponse)
|
||||||
async def version() -> VersionResponse:
|
async def version() -> VersionResponse:
|
||||||
return VersionResponse(version=read_version(), active_job=active_job is not None)
|
return VersionResponse(
|
||||||
|
version=read_version(),
|
||||||
|
active_job=active_job is not None,
|
||||||
|
ytdlp_executable=YTDLP_EXECUTABLE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@app.get("/jobs/current", response_model=CurrentJobResponse)
|
@app.get("/jobs/current", response_model=CurrentJobResponse)
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ class JobStatus(BaseModel):
|
|||||||
requester_name: str | None = None
|
requester_name: str | None = None
|
||||||
origin: str | None = None
|
origin: str | None = None
|
||||||
result_path: str | None = None
|
result_path: str | None = None
|
||||||
|
archive_path: str | None = None
|
||||||
command: list[str] = Field(default_factory=list)
|
command: list[str] = Field(default_factory=list)
|
||||||
returncode: int | None = None
|
returncode: int | None = None
|
||||||
created_at: datetime = Field(default_factory=datetime.utcnow)
|
created_at: datetime = Field(default_factory=datetime.utcnow)
|
||||||
@@ -44,3 +45,4 @@ class HealthResponse(BaseModel):
|
|||||||
class VersionResponse(BaseModel):
|
class VersionResponse(BaseModel):
|
||||||
version: str
|
version: str
|
||||||
active_job: bool
|
active_job: bool
|
||||||
|
ytdlp_executable: str
|
||||||
|
|||||||
Reference in New Issue
Block a user