diff --git a/.gitignore b/.gitignore index 02cbff9..e164256 100644 --- a/.gitignore +++ b/.gitignore @@ -1,34 +1,35 @@ -# --- python bytecode --- -__pycache__/ -*.py[cod] -*$py.class - -# --- virtual environments --- -.venv/ -venv/ -env/ - -# --- environment files --- -.env -.env.* -*.local - -# --- emacs --- -*~ -\#*\# -.\#* -*.elc - -# --- project private data --- -/private/ -archive/ -downloads/ -data.json - -# --- django --- -db.sqlite3 -staticfiles/ -media/ - -# --- misc --- +# --- python bytecode --- +__pycache__/ +*.py[cod] +*$py.class + +# --- virtual environments --- +.venv/ +venv/ +env/ + +# --- environment files --- +.env +.env.* +*.local + +# --- emacs --- +*~ +\#*\# +.\#* +*.elc + +# --- project private data --- +/private/ +archive/ +downloads/ +test/ +data.json + +# --- django --- +db.sqlite3 +staticfiles/ +media/ + +# --- misc --- .DS_Store \ No newline at end of file diff --git a/pm/tasks-v2.org b/pm/tasks-v2.org index 1fea622..853d464 100644 --- a/pm/tasks-v2.org +++ b/pm/tasks-v2.org @@ -1,6 +1,10 @@ #+title: Task Log #+updated: [2026-03-31 Tue 16:03] #+startup: overview +# Local Variables: +# org-babel-python-command: "/home/user/proj/youdis/.venv/bin/python" +# project-dir: "~/proj/youdis" +# End: * youdis v2 goals: 1. Separate backend from frontend @@ -112,13 +116,67 @@ update the discord bot into a thin frontend that talks to the backend and verify - valid request path tested - busy or conflict behavior tested - failure path tested - +4. add dotenv support to ease dev + - os.getenv methods remain standard for prod/docker build + - populate .env with dev env defaults + ** evidence - commit: - tests: + 1. start backend: `python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000` + 2. create local env file: `cp .env.example .env` + 3. add `api_token` to `.env` + 4. start adapter: `python3 ./youdis.py` + 5. in discord, run `/youtube url:https://www.youtube.com/watch?v=dQw4w9WgXcQ` + :out: + #+begin_src shell + (venv) user@paladin:~/proj/youdis$ python ./youdis.py +Task exception was never retrieved +future: exception=KeyError('youtube')> +Traceback (most recent call last): + File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/client/client.py", line 1798, in _dispatch_interaction + if ctx.command: + File "/home/user/proj/youdis/venv/lib/python3.10/site-packages/interactions/models/internal/context.py", line 329, in command + return self.client._interaction_lookup[self._command_name] +KeyError: 'youtube' + #+end_src + :end: + +#+end_:out: +#+begin_src + + 6. confirm channel response says the job was submitted to backend + 7. confirm requester receives DM updates for accepted/running/completed or failed + 8. while first job is active, submit another `/youtube` and confirm busy behavior + 9. run `/status` and confirm it reflects current or last backend job + 10. run `/interrupt` as owner and confirm cancellation is surfaced via DM - datetime: +*** testing tests +#+begin_src shell :dir ~/proj/youdis :results output verbatim + source ./venv/bin/activate + python3 -m uvicorn youdis.main:app --host 127.0.0.1 --port 8000 +#+end_src + +#+begin_src shell :results output +echo ok +#+end_src + +#+RESULTS: +: ok + +#+begin_src python :dir ~/proj/youdis :results output verbatim +python ./youdis.py +#+end_src + +#+RESULTS: + ** notes +- discord adapter is now a thin HTTP client of the backend; it no longer imports or configures yt-dlp +- `YOUDIS_BACKEND_URL` controls which backend the adapter targets +- progress updates are currently implemented by polling `/jobs/current` and DMing only when the summary changes +- legacy auth/user-management commands were removed from the active adapter path and should be cleaned up formally in `2.0.3` +- `.env` is now supported for local/dev convenience, while real environment variables still override it in prod/docker * [ ] 2.0.3: remove deprecated discord-bot functionality (2) delete or retire legacy bot behaviors that no longer fit once the backend split is in place @@ -145,7 +203,7 @@ delete or retire legacy bot behaviors that no longer fit once the backend split ** notes -* [ ] 2.0.5: fix automation and build pipeline (3) +* [ ] 2.0.4: fix automation and build pipeline (3) repair and simplify the build/update/deploy path so it matches the new backend-plus-frontend structure ** pm notes - this should come after architecture and discord integration stabilize. no point polishing the pipeline for the wrong shape. diff --git a/requirements.txt b/requirements.txt index 049f12a..43d0a5a 100644 Binary files a/requirements.txt and b/requirements.txt differ diff --git a/youdis.py b/youdis.py index d530bdd..87c0f1e 100644 --- a/youdis.py +++ b/youdis.py @@ -1,235 +1,7 @@ -#!/usr/bin/env python3 -''' -youdis v1.1 -bot for downloading youtube videos using yt-dlp -discord-py-interactions 5.11.0 has new option - requires python>=3.9 -''' -# match_filter: info_dict -> Raise utils.DownloadCancelled(msg) ? interrupt - -import interactions -from os import getenv -from pathlib import Path -import yt_dlp -import json -import asyncio -import threading +#!/usr/bin/env python3 -userFile = Path('/config/users.json') -userFile.parent.mkdir(exist_ok=True, parents=True) - -bot = interactions.Client(intents=interactions.Intents.DEFAULT,default_scope=2147491904) - -def save_authorized_users(authorized_users): - with open(userFile, 'w') as f: - json.dump({'authorized_users': authorized_users}, f) - -def load_authorized_users(): - if not userFile.exists(): - save_authorized_users([]) - print(f'users.json not found; saving to {userFile}') - return [] - - try: - with open(userFile, 'r') as f: - data = json.load(f) - except (json.JSONDecodeError, OSError): - save_authorized_users([]) - print(f'users.json invalid; resetting {userFile}') - return [] - - authorized_users = data.get('authorized_users', []) - if not isinstance(authorized_users, list): - authorized_users = [] - - authorized_users = [str(user_id) for user_id in authorized_users] - save_authorized_users(authorized_users) - print(f'authorized_users:{authorized_users}') - return authorized_users - -authorized_users = load_authorized_users() - -active_job_lock = threading.Lock() -active_job = None - -async def send_message(ctx, message): - await ctx.author.send(message) - -def claim_active_job(job): - global active_job - with active_job_lock: - if active_job is not None: - return active_job - active_job = job - return None - -def get_active_job(): - with active_job_lock: - return active_job - -def clear_active_job(job): - global active_job - with active_job_lock: - if active_job is job: - active_job = None - -def download_video(url, options): - with yt_dlp.YoutubeDL(options) as ydl: - ydl.download(url) - -def create_hook(ctx, loop, cancel_event): - seen_updates = set() - - def hook(d): - if cancel_event.is_set(): - raise yt_dlp.utils.DownloadCancelled('download canceled by /interrupt') - - status = d.get('status') - info = d.get('info_dict') or {} - - if status not in {'downloading', 'finished'}: - return - - filename = d.get('filename') or info.get('_filename') or info.get('title') - update_key = (status, filename) - if update_key in seen_updates: - return - - seen_updates.add(update_key) - playlist_index = info.get('playlist_index') - playlist_count = info.get('playlist_count') - url = info.get('webpage_url') - - prefix = status - if playlist_index and playlist_count: - prefix = f'{status} {playlist_index} of {playlist_count}' - - msg = f'{prefix}: {filename}' - if url: - msg = f'{msg} <{url}>' - - asyncio.run_coroutine_threadsafe(send_message(ctx, msg), loop) - - return hook - -@interactions.slash_command(name="youtube",description="download video from youtube to server") -@interactions.slash_option( - name='url', - opt_type=interactions.OptionType.STRING, - required=True, - description='url target' -) -async def youtube(ctx: interactions.SlashContext, url:str): - print(f'{ctx.author.id} requested {url}') - # check that user is authorized - if str(ctx.author.id) not in authorized_users: - if ctx.author.id == 127831327012683776: - await ctx.author.send('potato stop') - await ctx.author.send('you are not authorized to use this command. message my owner to be added.') - return - - loop = asyncio.get_running_loop() - cancel_event = threading.Event() - hook = create_hook(ctx, loop, cancel_event) - job = { - 'requester_id': str(ctx.author.id), - 'request_url': url, - 'cancel_event': cancel_event, - } - existing_job = claim_active_job(job) - if existing_job: - await ctx.author.send( - f'already downloading for <@{existing_job["requester_id"]}>. ' - 'single-job mode is enabled right now; try again after it finishes.' - ) - return - - # use api_to_cli and paste cli options to get the output you need - yoptions = { - 'format':'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best', - 'fragment_tries': 10, - 'restrictfilenames':True, - 'paths': {'home':'/downloads'}, - 'retries':10, - 'writeinfojson':False, - 'allow_playlist_files':True, - 'noplaylist':True, - 'download_archive':'/config/archive.txt', - 'progress_hooks':[hook], - 'outtmpl': '%(uploader)s/%(playlist_title)s/%(playlist_index)s%(playlist_index& - )s%(title)s.%(ext)s', - 'outtmpl_na_placeholder':'', - } - await ctx.channel.send(f'Downloading from <{url}>. Status updates via DM. Single-job mode is enabled.') - - try: - await asyncio.to_thread(download_video, url, yoptions) - except yt_dlp.utils.DownloadCancelled as exc: - print(f'download canceled: {exc}') - await ctx.author.send(f'download canceled: {exc}') - except yt_dlp.utils.DownloadError as exc: - print(f'download failed: {exc}') - await ctx.author.send(f'download failed: {exc}') - except Exception as exc: - print(f'unexpected download failure: {exc}') - await ctx.author.send(f'unexpected download failure: {exc}') - else: - await ctx.author.send(f'download complete for <{url}>') - finally: - clear_active_job(job) +from youdis.adapters.discord import main -@interactions.slash_command(name="interrupt",description="cancel current job") -@interactions.check(interactions.is_owner()) -async def _interrupt(ctx): - job = get_active_job() - if not job: - await ctx.author.send('no active download to interrupt') - return - - job['cancel_event'].set() - print(f'interrupt requested for {job["request_url"]}') - await ctx.author.send( - f'interrupt requested for <{job["request_url"]}>; ' - 'cancellation is coarse and will stop on the next yt-dlp progress update' - ) - -@interactions.slash_command(name="adduser",description="authorize target user") -@interactions.slash_option( - name="user", - opt_type=interactions.OptionType.USER, - required=True, - description='enable this bot for target user', -) -@interactions.check(interactions.is_owner()) -async def _adduser(ctx: interactions.SlashContext, user:interactions.OptionType.USER): - user_id = str(user.id) - if user_id not in authorized_users: - authorized_users.append(user_id) - save_authorized_users(authorized_users) - print(f'authorized {user_id}') - await ctx.author.send(f'authorized {user.mention}') - else: - await ctx.author.send(f'{user.mention} is already authorized') - -@interactions.slash_command(name="removeuser",description="deauthorize target user") -@interactions.slash_option( - name="user", - opt_type=interactions.OptionType.USER, - required=True, - description='disable this bot for target user', -) -@interactions.check(interactions.is_owner()) -async def _removeuser(ctx: interactions.SlashContext, user:interactions.OptionType.USER): - user_id = str(user.id) - if user_id in authorized_users: - authorized_users.remove(user_id) - save_authorized_users(authorized_users) - print(f'deauthorized {user_id}') - await ctx.author.send(f'deauthorized {user.mention}') - else: - await ctx.author.send(f'{user.mention} is not currently authorized') - -api_token = getenv('api_token') -if not api_token: - raise ValueError('API token not set. Retrieve from your Discord bot.') -bot.start(api_token) +if __name__ == "__main__": + main() diff --git a/youdis/adapters/discord.py b/youdis/adapters/discord.py index 7beab65..e7aa553 100644 --- a/youdis/adapters/discord.py +++ b/youdis/adapters/discord.py @@ -1 +1,214 @@ -"""Discord adapter placeholder for the v2 backend.""" +import asyncio +from os import getenv + +import aiohttp +import interactions + +from ..env import load_project_dotenv + + +load_project_dotenv() + + +BACKEND_URL = getenv("YOUDIS_BACKEND_URL", "http://127.0.0.1:8000").rstrip("/") +POLL_INTERVAL_SECONDS = float(getenv("YOUDIS_POLL_INTERVAL_SECONDS", "2")) +DEFAULT_SCOPE = int(getenv("DISCORD_BOT_SCOPE", "2147491904")) + + +bot = interactions.Client( + intents=interactions.Intents.DEFAULT, + default_scope=DEFAULT_SCOPE, +) +http_session: aiohttp.ClientSession | None = None +poll_tasks: dict[str, asyncio.Task] = {} + + +def backend_url(path: str) -> str: + return f"{BACKEND_URL}{path}" + + +async def get_session() -> aiohttp.ClientSession: + global http_session + if http_session is None or http_session.closed: + http_session = aiohttp.ClientSession() + return http_session + + +async def request_json(method: str, path: str, **kwargs): + session = await get_session() + async with session.request(method, backend_url(path), **kwargs) as response: + data = await response.json() + return response.status, data + + +def format_status_message(job: dict) -> str: + state = job.get("state") + phase = job.get("phase") + disposition = job.get("disposition") + message = job.get("message") + result_path = job.get("result_path") + + parts = [f"state={state}"] + if phase: + parts.append(f"phase={phase}") + if disposition: + parts.append(f"disposition={disposition}") + if message: + parts.append(message) + if result_path: + parts.append(f"path={result_path}") + return " | ".join(parts) + + +async def dm(ctx: interactions.SlashContext, message: str) -> None: + await ctx.author.send(message) + + +async def poll_job_updates(ctx: interactions.SlashContext, job_id: str) -> None: + last_sent = None + try: + while True: + status_code, payload = await request_json("GET", "/jobs/current") + if status_code != 200: + await dm(ctx, f"backend status check failed: HTTP {status_code}") + return + + job = payload.get("job") + if not job: + await dm(ctx, f"job {job_id} is no longer visible from the backend") + return + + if job.get("job_id") != job_id: + await dm(ctx, f"job {job_id} is no longer the current backend job") + return + + summary = format_status_message(job) + if summary != last_sent: + await dm(ctx, summary) + last_sent = summary + + if job.get("state") in {"completed", "failed", "cancelled"}: + return + + await asyncio.sleep(POLL_INTERVAL_SECONDS) + except asyncio.CancelledError: + raise + except aiohttp.ClientError as exc: + await dm(ctx, f"backend poll failed: {exc}") + finally: + poll_tasks.pop(job_id, None) + + +def ensure_poll_task(ctx: interactions.SlashContext, job_id: str) -> None: + existing = poll_tasks.get(job_id) + if existing and not existing.done(): + return + poll_tasks[job_id] = asyncio.create_task(poll_job_updates(ctx, job_id)) + + +@interactions.listen() +async def on_startup(): + await get_session() + print(f"discord adapter configured for backend {BACKEND_URL}") + + +@interactions.listen() +async def on_shutdown(): + global http_session + for task in list(poll_tasks.values()): + task.cancel() + poll_tasks.clear() + if http_session is not None and not http_session.closed: + await http_session.close() + http_session = None + + +@interactions.slash_command(name="youtube", description="submit a youtube download to the backend") +@interactions.slash_option( + name="url", + opt_type=interactions.OptionType.STRING, + required=True, + description="url target", +) +async def youtube(ctx: interactions.SlashContext, url: str): + payload = { + "url": url, + "requester_id": str(ctx.author.id), + "requester_name": ctx.author.username, + "origin": f"discord:{ctx.guild_id or 'dm'}:{ctx.channel_id}", + } + + try: + status_code, job = await request_json("POST", "/jobs", json=payload) + except aiohttp.ClientError as exc: + await dm(ctx, f"backend request failed: {exc}") + return + + if status_code != 200: + await dm(ctx, f"backend request failed: HTTP {status_code}") + return + + state = job.get("state") + job_id = job.get("job_id", "unknown") + if state == "busy": + await ctx.channel.send(f"Backend is busy with another job. Details via DM.") + await dm(ctx, f"busy: {job.get('message')}") + return + + if state != "accepted": + await ctx.channel.send("Backend rejected the request. Details via DM.") + await dm(ctx, format_status_message(job)) + return + + await ctx.channel.send(f"Submitted <{url}> to the backend. Status updates via DM.") + await dm(ctx, f"accepted job {job_id} for <{url}>") + ensure_poll_task(ctx, job_id) + + +@interactions.slash_command(name="interrupt", description="cancel the current backend job") +@interactions.check(interactions.is_owner()) +async def interrupt(ctx: interactions.SlashContext): + try: + status_code, payload = await request_json("POST", "/jobs/current/cancel") + except aiohttp.ClientError as exc: + await dm(ctx, f"backend cancel failed: {exc}") + return + + if status_code == 404: + await dm(ctx, "no active backend job to interrupt") + return + + if status_code != 200: + await dm(ctx, f"backend cancel failed: HTTP {status_code}") + return + + await dm(ctx, format_status_message(payload)) + + +@interactions.slash_command(name="status", description="show the current backend job status") +async def status(ctx: interactions.SlashContext): + try: + status_code, payload = await request_json("GET", "/jobs/current") + except aiohttp.ClientError as exc: + await dm(ctx, f"backend status failed: {exc}") + return + + if status_code != 200: + await dm(ctx, f"backend status failed: HTTP {status_code}") + return + + job = payload.get("job") + if not job: + await dm(ctx, "backend has no active or recent job") + return + + active = payload.get("active") + prefix = "active" if active else "last" + await dm(ctx, f"{prefix} job: {format_status_message(job)}") + + +def main() -> None: + api_token = getenv("DISCORD_BOT_TOKEN") + if not api_token: + raise ValueError("API token not set. Retrieve from your Discord bot.") + bot.start(api_token) diff --git a/youdis/env.py b/youdis/env.py new file mode 100644 index 0000000..82934c6 --- /dev/null +++ b/youdis/env.py @@ -0,0 +1,8 @@ +from pathlib import Path + +from dotenv import load_dotenv + + +def load_project_dotenv() -> None: + repo_root = Path(__file__).resolve().parent.parent + load_dotenv(repo_root / ".env", override=False) diff --git a/youdis/main.py b/youdis/main.py index 31eef27..8faf8e1 100644 --- a/youdis/main.py +++ b/youdis/main.py @@ -10,9 +10,13 @@ from uuid import uuid4 from fastapi import FastAPI, HTTPException +from .env import load_project_dotenv from .models import CurrentJobResponse, HealthResponse, JobRequest, JobStatus, VersionResponse +load_project_dotenv() + + REPO_ROOT = Path(__file__).resolve().parent.parent DEFAULT_CONFIG = REPO_ROOT / "default-yt-dlp.conf" VERSION_FILE = REPO_ROOT / "youdis-version.txt"