From 7e8f45fe649b132c897608ba99c3d5ccdba16fb8 Mon Sep 17 00:00:00 2001 From: sanj <67624670+iodrift@users.noreply.github.com> Date: Fri, 2 Aug 2024 11:15:52 -0700 Subject: [PATCH] Major update - database sync methods finally work reliably --- README.md | 2 +- requirements.txt | 2 +- sijapi/config/tts.yaml-example | 10 +++ sijapi/data/ms365/.cert.key | 28 ++++++ sijapi/data/ms365/.cert.pem | 19 ++++ sijapi/data/ms365/.token.txt | 1 + sijapi/helpers/cli.py | 57 ------------ sijapi/routers/asr.py | 1 + sijapi/routers/cal.py | 2 + sijapi/routers/cf.py | 2 + sijapi/routers/dist.py | 65 -------------- sijapi/routers/email.py | 2 + sijapi/routers/forward.py | 124 ++++++++++++++++++++++++++ sijapi/routers/ghost.py | 16 ++++ sijapi/routers/gis.py | 2 + sijapi/routers/health.py | 2 + sijapi/routers/ig.py | 2 + sijapi/routers/img.py | 3 +- sijapi/routers/llm.py | 1 + sijapi/routers/news.py | 4 + sijapi/routers/note.py | 5 +- sijapi/routers/rag.py | 1 + sijapi/routers/scrape.py | 5 ++ sijapi/routers/serve.py | 74 +-------------- sijapi/routers/signal.py | 33 ------- sijapi/routers/{time.py => timing.py} | 22 ++--- sijapi/routers/tts.py | 2 + sijapi/routers/weather.py | 2 + 28 files changed, 247 insertions(+), 242 deletions(-) create mode 100644 sijapi/config/tts.yaml-example create mode 100644 sijapi/data/ms365/.cert.key create mode 100644 sijapi/data/ms365/.cert.pem create mode 100644 sijapi/data/ms365/.token.txt delete mode 100644 sijapi/helpers/cli.py delete mode 100644 sijapi/routers/dist.py create mode 100644 sijapi/routers/forward.py delete mode 100644 sijapi/routers/signal.py rename sijapi/routers/{time.py => timing.py} (98%) diff --git a/README.md b/README.md index 9b4b063..626ad96 100644 --- a/README.md +++ b/README.md @@ -44,4 +44,4 @@ IN DRAFT IN DRAFT ## Extras -[Apple Shortcut for companion Pythonista script](https://www.icloud.com/shortcuts/d63b179f9c664b0cbbacb1a607767ef7) +[Apple Shortcut for location tracking Pythonista script](https://www.icloud.com/shortcuts/d63b179f9c664b0cbbacb1a607767ef7) diff --git a/requirements.txt b/requirements.txt index 99ca5ab..1a93536 100644 --- a/requirements.txt +++ b/requirements.txt @@ -51,4 +51,4 @@ urllib3 anyio semaphore location -SRTM.py +SRTM.py \ No newline at end of file diff --git a/sijapi/config/tts.yaml-example b/sijapi/config/tts.yaml-example new file mode 100644 index 0000000..c4e1d74 --- /dev/null +++ b/sijapi/config/tts.yaml-example @@ -0,0 +1,10 @@ +default: xtts +email: xtts +webclip: 11L +rss: xtts +podcast_dir: '{{ DIR.HOME }}/Library/Mobile Documents/iCloud~co~supertop~castro/Documents/Sideloads' +xtts: + voice: joanne +elevenlabs: + voice: luna + api_key: '{{ SECRET.ELEVENLABS_API_KEY }}' \ No newline at end of file diff --git a/sijapi/data/ms365/.cert.key b/sijapi/data/ms365/.cert.key new file mode 100644 index 0000000..8a4d6ab --- /dev/null +++ b/sijapi/data/ms365/.cert.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCikW67UW0RpncJ +h4Ha9HumZ/WzgEZWRWkgksVJIOJ8t1PftctizLUlz+xMWNl+Volp4crxnPpnczis +pOXU4g65XoFHHpF9nhF/3YDgxo5BDEM/mIIKEO9LFkIBQVBdE85qXnIVot5MfuNj +HeyEs7doRMBxilOSR/DkT8bTWu7m5yeHlF58iYVOxxssGhP3bo7CcAcaZD1LJBnP +Df+UBqzWQ9as903p5bFixHy2kVz8Qkd5k5tyIQ/tXqlhRfLLHG4AYHmBbS06CAg0 +nEpKUeQx4l1J/ykAjQTwhHf70xv1z0p28mHcr5ib4UvpYK9fMM6FKWenwlqA3qrK +zQUJQ7E/AgMBAAECggEAQ5H/XIxzsSpnv+Y66y9DVd9QGNPwaFthXtCif8rTWNM6 +YXnGl8JOaPELXpBvljuR0hivqc19pxIVNG01uk5boGDPiygBgRz6WRNQRh1Bc3gN +W5mgM17ml2cg+DSVmppo6X1oHeYcT99N1BzT+jRYv1YURx0fr2WHkt413hOlyQMR +b8ir/TOBx3olg4KBPDuysRC5BCIr3Mkz4jsh+9wVIOReKVezsy7nxJVzipcxOyZO +9VGgvlw4XLrRTOJEv4e3ldcg219j5KEGsJ4FFSziSmpj5fN4Vt+JmY7nueSHyL6P +3hX52lRfOcTXTEeiEV2cXkm3h8uQ3zfiZRYi3P0DQQKBgQDXGBZc3WnfXim0u1EV +JzZFwxBS7SHkyGgnd50ak6e9yDbdxOuYIOo4mBlc3ofd20EfT4TvR7Xyw+PD2fWJ ++isdwCEb9JZZ1H6RDGIzSDYXGNeGId4kMKBZdmKpEeLgStihsrYp/nxtwcE/8A7N +jCEKZj1ld7QfbQlGT/NJ4Jj80wKBgQDBfBpth6vMyCETKMJVqYd2qhVnJKiFLfRn +OD/Ck6xwUuedbfe9M34wNO3Pn2Xvu1xVsQGb2dmlT345Iq9Z1nbZCGXyY9yfLnTV +fz7F2utjUjaJtuiSb52SgX7MWZ8E4nbqqKnC4SYSIlaeuL9KK3r/x6bcNLAYPcdk +qKHexDkGZQKBgF0JGyshzhiChzGYUBMBOfVk0Ru9XAq0MHDZyQdk1Io/HpRAB9Nu +cUD3cQj9a/EnU/yyDYLeFrIhztO44/7BSYL9xpRr79h9FB2zKIqb8mF9KkPnREmN +Ct6HWVdd2C9B0H/oZ+i0HafvxaHdONnpgaUY4feQlkV9iSRzknzi++lnAoGAXOuu +/X80oMpUKBFhEyaxqemREdHnJN6nC5NV+6pUHDWUimSvn6vFJH2m4BlbKUC/3V9+ +uExtXBjLM8FWmTyIIz8HRttyrvfuoEHV8ctrVG29R3ISS5FTCXMrZBR+bCgemB+c +N71NPVREaUGsjIBJN+G4XvTmxR2WTt81rfhqsokCgYEA1It9e9Ut2Krzf2FaPGLG +ozlKhWadMNntthg3uxze80Rx8WSvgJQdbVpdbni2B/9xdYBIIljW/LGivYBrCSSp +aXFpXL7ZGkvl3b0MkojfghIpXVGqu+8ISDtFgL0B1gZ5hq9xMBl94fLVfQgC9Cy6 +uvDHlz+fjWaWKYUPiouAtVs= +-----END PRIVATE KEY----- diff --git a/sijapi/data/ms365/.cert.pem b/sijapi/data/ms365/.cert.pem new file mode 100644 index 0000000..7cefa0c --- /dev/null +++ b/sijapi/data/ms365/.cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDAzCCAeugAwIBAgIUc+EtilZslnS7N6MAx0u9HeP83wAwDQYJKoZIhvcNAQEL +BQAwETEPMA0GA1UEAwwGcHl0aG9uMB4XDTI0MDYwODEyNTcxM1oXDTI1MDYwODEy +NTcxM1owETEPMA0GA1UEAwwGcHl0aG9uMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAopFuu1FtEaZ3CYeB2vR7pmf1s4BGVkVpIJLFSSDifLdT37XLYsy1 +Jc/sTFjZflaJaeHK8Zz6Z3M4rKTl1OIOuV6BRx6RfZ4Rf92A4MaOQQxDP5iCChDv +SxZCAUFQXRPOal5yFaLeTH7jYx3shLO3aETAcYpTkkfw5E/G01ru5ucnh5RefImF +TscbLBoT926OwnAHGmQ9SyQZzw3/lAas1kPWrPdN6eWxYsR8tpFc/EJHeZObciEP +7V6pYUXyyxxuAGB5gW0tOggINJxKSlHkMeJdSf8pAI0E8IR3+9Mb9c9KdvJh3K+Y +m+FL6WCvXzDOhSlnp8JagN6qys0FCUOxPwIDAQABo1MwUTAdBgNVHQ4EFgQUS74L +HD4Cdzh1ajatbvSHNQXIVvAwHwYDVR0jBBgwFoAUS74LHD4Cdzh1ajatbvSHNQXI +VvAwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAhpwtVubDjsyq +/LiTwpXKhjB/eFb6Yse782Iq+9rsiGGhsN88IA25fKgsJ2AIkR/KA7QSle3ds+1Q +EY9/vqpWnfBdpvOi7oV7ozBe+t/5JLu1GQBzg+cVa4iLAWYCiqg1d5NDdIcYMfsM +Yq2a3eQoP8Xbj3fFMXdNopXARa1d1zHB3ugXIJYinwMlS0EoGXVQVaHhemOh8GwW +keRaA6TDTBFsp0Gl4jv/NrisAt4qg+rlqr0mNcQK92vRX65mDWa/cQKwpUH8+Seq +Jl717NnsIGcqYWg8SSvVlkbFfxYhwYICXT824MAdSZtpHNCN/TegxsviYnlDyJKj +OJzn4fCxnQ== +-----END CERTIFICATE----- diff --git a/sijapi/data/ms365/.token.txt b/sijapi/data/ms365/.token.txt new file mode 100644 index 0000000..c15356b --- /dev/null +++ b/sijapi/data/ms365/.token.txt @@ -0,0 +1 @@ +{"token_type": "Bearer", "scope": "Calendars.Read Calendars.ReadWrite User.Read profile openid email", "expires_in": 3962, "ext_expires_in": 3962, "access_token": "eyJ0eXAiOiJKV1QiLCJub25jZSI6IldDeU91YXllN1RFX2FPM0F1alhlYmtvYTdVRHpUR1dVNWt5d3lJeDZ1MGciLCJhbGciOiJSUzI1NiIsIng1dCI6InE3UDFOdnh1R1F3RE4yVGFpTW92alo4YVp3cyIsImtpZCI6InE3UDFOdnh1R1F3RE4yVGFpTW92alo4YVp3cyJ9.eyJhdWQiOiIwMDAwMDAwMy0wMDAwLTAwMDAtYzAwMC0wMDAwMDAwMDAwMDAiLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC9iYWQ3ODA0OC1hNmUwLTQ3YjEtYTI0Yi00MDNjNDQ0YWEzNDkvIiwiaWF0IjoxNzE4Mzc0NzA5LCJuYmYiOjE3MTgzNzQ3MDksImV4cCI6MTcxODM3ODk3MiwiYWNjdCI6MCwiYWNyIjoiMSIsImFpbyI6IkFWUUFxLzhYQUFBQVRnWHJ0Q1pCVjlPa1M2WldldHVVSHNMSFN0LzErYVcxT1BSSjVOWjJEL1Bzd05mY1Fxb0JTNEFZRmhLR3UvaE5TNnNWOGtLQUpmcDNNTzdqRUlNMEZrY1VaZ0IyREh4cWdOK3lUQVBUYnRVPSIsImFtciI6WyJwd2QiLCJtZmEiXSwiYXBwX2Rpc3BsYXluYW1lIjoicHl0aG9uIiwiYXBwaWQiOiJjZThjYmQyNC1mMTQ2LTRkYzctOGVlNy01MWQ5YjY5ZGVjNTkiLCJhcHBpZGFjciI6IjEiLCJmYW1pbHlfbmFtZSI6IkluY2UtSm9oYW5uc2VuIiwiZ2l2ZW5fbmFtZSI6IlNhbmd5ZSIsImlkdHlwIjoidXNlciIsImlwYWRkciI6IjY4LjIzNS40NC4yMDIiLCJuYW1lIjoiU2FuZ3llIEluY2UtSm9oYW5uc2VuIiwib2lkIjoiMWNiMWQwNDAtZmM1OS00MjMxLTllMDUtOWRjNGI0MzJjY2MxIiwicGxhdGYiOiI1IiwicHVpZCI6IjEwMDMyMDAyQTNGQjU4RjIiLCJyaCI6IjAuQVgwQVNJRFh1dUNtc1VlaVMwQThSRXFqU1FNQUFBQUFBQUFBd0FBQUFBQUFBQUMxQUk4LiIsInNjcCI6IkNhbGVuZGFycy5SZWFkIENhbGVuZGFycy5SZWFkV3JpdGUgVXNlci5SZWFkIHByb2ZpbGUgb3BlbmlkIGVtYWlsIiwic2lnbmluX3N0YXRlIjpbImttc2kiXSwic3ViIjoiV0FYVFdIR0puVFhBTjlncmIyamlEU3U4ZENOMmc0dDFacERiVHlwM1k3USIsInRlbmFudF9yZWdpb25fc2NvcGUiOiJOQSIsInRpZCI6ImJhZDc4MDQ4LWE2ZTAtNDdiMS1hMjRiLTQwM2M0NDRhYTM0OSIsInVuaXF1ZV9uYW1lIjoic2FuZ3llaWpAd2VzdGVybmxhdy5vcmciLCJ1cG4iOiJzYW5neWVpakB3ZXN0ZXJubGF3Lm9yZyIsInV0aSI6InFHcVlEODRzaDBHMFBfSEdldlVXQUEiLCJ2ZXIiOiIxLjAiLCJ3aWRzIjpbImI3OWZiZjRkLTNlZjktNDY4OS04MTQzLTc2YjE5NGU4NTUwOSJdLCJ4bXNfaWRyZWwiOiIxIDIiLCJ4bXNfc3QiOnsic3ViIjoieXhjdzFhV1FiM2VrX0FvNFRuRy11SDN6ZndGbVRRUmMxVGpFaEdqZ2p2WSJ9LCJ4bXNfdGNkdCI6MTY0ODY4MTc1Mn0.ssgIrbYo1SPNusoB9bNIB7pLxCmwBKhox__KOnwRRtnE63vbfGWAl53ww1KpNWPdDfC3p94yuPybTRqjZnTPluv1oJgGINml4AleUnZJnJttRsFHvGflzKOLtXnzmhQGUBXxu7QucKTCMH4J36neeQAWthITMwCHbaGmSy0RLotaIsoEHIufxR9ZEYD4XP5e3sFX54eSnyf4P3GgHHC1y5xxWUlemG4G1BRas8i7oX9o-gqRFube6BMtCLir_HMTNPfrCG-lhd9msLhc6e_WJSmLMHQ7RVLo-GlTMY9UouE190GzBBVKUrTg462I3kP_GayO1kt6qopBrwnF6bDUsw", "refresh_token": "0.AX0ASIDXuuCmsUeiS0A8REqjSSS9jM5G8cdNjudR2bad7Fm1AI8.AgABAwEAAAApTwJmzXqdR4BN2miheQMYAgDs_wUA9P_rTWFRiXWkWxvihyyXonsZPLrulRvnKKRlZ9PxKUltEOQsxjlg86xvCYzAS6dYeDBQiQxRAS_WEuuXVmTqUWDVwqgwQOa3BCbLwxQhPwfG-O9uFY6D239Jo8rdXTrf8XOntGs6fCn3wuo5kvJr2D-FGRA_EepltvRxZgrWdHROKuqoL_ArjLDdoFP7zM95MKhVYTmCO7LCM7u6O9ItU4_6y2_lH864zUivT1LFG8-h9sx0Ln3wd8LBP3P5GSeXwtQlkbNpj1FNDl_Ex5SwGCTM7uDHj0dn5CdUMgLkOcAC__HJdzmlEryTquoXcjd1RAmkq1MqAGD7QQreI7NQTZXwTcjoMwiBg92-bk-_o2ajeIVqzgOVBQIu1W8gkN2F7PAqRc5lGB-2mAXchqKMoL31CLUPxgTMBjWgR4waAjfZXT4h2WqXAAdGFy2nzUJAjyEQa9ZW1J5B6asCf3cVJQwI6nWIN7OphrXkGHl0ffpfrC-skVG3N2vrelAutRvyvWi4bbMqAZNglRrkTn5G_kULmnyydZBcFSc5uPmKD7OkfBD5UpTa_KLTjYexWRVsBfG9czIVxOh3ojnnza9BjrN5cHwHhzPM1t67E5iqronvT2OR_r-4BerUfRNHXrxwrLvDUEZwQ8o5IRs2N5FH0y_QN049o_NTgqytCj6wrIB4T-ZBUK2AsFej7ipdHAMYtWLZdoAo1o4nMuPBb4syN0VYd1sLUP-RQ5iv7wIkMWmNjhjIErIktZ134pGK9TlWa904H6HUin0qNTXyTmX2feE0nBlm6xJbO1ISfFkaf8aEjcAMfeu9qiArKQqUgvY", "expires_at": 1718378971} \ No newline at end of file diff --git a/sijapi/helpers/cli.py b/sijapi/helpers/cli.py deleted file mode 100644 index 209d546..0000000 --- a/sijapi/helpers/cli.py +++ /dev/null @@ -1,57 +0,0 @@ -# cli.py -import click -import asyncio -from datetime import datetime as dt_datetime, timedelta - -# Import your async functions and dependencies -from sijapi import build_daily_note_range_endpoint, gis # broken! - -def async_command(f): - @click.command() - @click.pass_context - def wrapper(ctx, *args, **kwargs): - async def run(): - return await f(*args, **kwargs) - return asyncio.run(run()) - return wrapper - -@click.group() -def cli(): - """CLI for your application.""" - pass - -@cli.command() -@click.argument('dt_start') -@click.argument('dt_end') -@async_command -async def bulk_note_range(dt_start: str, dt_end: str): - """ - Build daily notes for a date range. - - DT_START and DT_END should be in YYYY-MM-DD format. - """ - try: - start_date = dt_datetime.strptime(dt_start, "%Y-%m-%d") - end_date = dt_datetime.strptime(dt_end, "%Y-%m-%d") - except ValueError: - click.echo("Error: Dates must be in YYYY-MM-DD format.") - return - - if start_date > end_date: - click.echo("Error: Start date must be before or equal to end date.") - return - - results = [] - current_date = start_date - while current_date <= end_date: - formatted_date = await gis.dt(current_date) - result = await build_daily_note(formatted_date) - results.append(result) - current_date += timedelta(days=1) - - click.echo("Generated notes for the following dates:") - for url in results: - click.echo(url) - -if __name__ == '__main__': - cli() \ No newline at end of file diff --git a/sijapi/routers/asr.py b/sijapi/routers/asr.py index f26cee3..4d51e9b 100644 --- a/sijapi/routers/asr.py +++ b/sijapi/routers/asr.py @@ -2,6 +2,7 @@ Uses whisper_cpp to create an OpenAI-compatible Whisper web service. ''' # routers/asr.py + import os import uuid import json diff --git a/sijapi/routers/cal.py b/sijapi/routers/cal.py index a4e83b5..607bdad 100644 --- a/sijapi/routers/cal.py +++ b/sijapi/routers/cal.py @@ -3,6 +3,8 @@ Calendar module using macOS Calendars and/or Microsoft 365 via its Graph API. Depends on: LOGGER, ICAL_TOGGLE, ICALENDARS, MS365_TOGGLE, MS365_CLIENT_ID, MS365_SECRET, MS365_AUTHORITY_URL, MS365_SCOPE, MS365_REDIRECT_PATH, MS365_TOKEN_PATH ''' +#routers/cal.py + from fastapi import APIRouter, Depends, HTTPException, status, Request from fastapi.responses import RedirectResponse, JSONResponse from fastapi.security import OAuth2PasswordBearer diff --git a/sijapi/routers/cf.py b/sijapi/routers/cf.py index b23fa25..2af253b 100644 --- a/sijapi/routers/cf.py +++ b/sijapi/routers/cf.py @@ -1,6 +1,8 @@ ''' IN DEVELOPMENT - Cloudflare + Caddy module. Based on a bash script that's able to rapidly deploy new Cloudflare subdomains on new Caddy reverse proxy configurations, managing everything including restarting Caddy. The Python version needs more testing before actual use. ''' +#routers/cf.py + from fastapi import APIRouter, HTTPException from pydantic import BaseModel from fastapi.responses import PlainTextResponse, JSONResponse diff --git a/sijapi/routers/dist.py b/sijapi/routers/dist.py deleted file mode 100644 index 05f09cb..0000000 --- a/sijapi/routers/dist.py +++ /dev/null @@ -1,65 +0,0 @@ -''' -WORK IN PROGRESS: This module will handle tasks related to multi-server environments. -''' -from fastapi import APIRouter, HTTPException -import asyncio -import logging -from sijapi.utilities import run_ssh_command -from sijapi import L, REBOOT_SCRIPT_PATH, HOST_CONFIG, API_CONFIG - -dist = APIRouter() -logger = L.get_module_logger("dist") -def debug(text: str): logger.debug(text) -def info(text: str): logger.info(text) -def warn(text: str): logger.warning(text) -def err(text: str): logger.error(text) -def crit(text: str): logger.critical(text) - -@dist.get("/update-restart-others") -async def update_and_restart_others(): - results = [] - for server in API_CONFIG.servers: - if HOST_CONFIG.id != server.id: - try: - output, error = await run_ssh_command(server, f"bash {server.scripts.update_and_restart}") - results.append({"server": server.id, "status": "success", "output": output, "error": error}) - except Exception as e: - results.append({"server": server.id, "status": "failed", "error": str(e)}) - return {"message": "Update and restart process initiated for other servers.", "results": results} - -@dist.get("/update-restart-self") -async def update_and_restart_self(safe: bool = True): - if safe and not await ensure_redundancy(): - raise HTTPException(status_code=400, detail="Cannot safely restart: no redundancy available") - try: - process = await asyncio.create_subprocess_exec( - "bash", API_CONFIG.servers[HOST_CONFIG.id].scripts.update_and_restart, - stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE - ) - stdout, stderr = await process.communicate() - info(f"Update and restart initiated for self. Stdout: {stdout.decode()}. Stderr: {stderr.decode()}") - return {"message": "Update and restart process initiated for this server."} - except Exception as e: - err(f"Failed to initiate update and restart for self: {str(e)}") - raise HTTPException(status_code=500, detail=f"Failed to initiate update and restart: {str(e)}") - -@dist.get("/update-and-restart-all") -async def update_and_restart_all(): - others_result = await update_and_restart_others() - self_result = await update_and_restart_self(safe=False) - return {"others": others_result, "self": self_result} - -async def ensure_redundancy(): - import aiohttp - redundancy = False - async with aiohttp.ClientSession() as session: - for server in API_CONFIG.servers: - if server.id != HOST_CONFIG.id: - try: - async with session.get(f"{server.protocol}://{server.ip}:{server.port}/health") as response: - if response.status // 100 == 2: - redundancy = True - break - except aiohttp.ClientError: - warn(f"Failed to check health of server {server.id}") - return redundancy diff --git a/sijapi/routers/email.py b/sijapi/routers/email.py index b2c0758..74fac13 100644 --- a/sijapi/routers/email.py +++ b/sijapi/routers/email.py @@ -1,6 +1,8 @@ ''' Uses IMAP and SMTP login credentials to monitor an inbox and summarize incoming emails that match certain criteria and save the Text-To-Speech converted summaries into a specified "podcast" folder. ''' +#routers/email.py + from fastapi import APIRouter import asyncio import aiofiles diff --git a/sijapi/routers/forward.py b/sijapi/routers/forward.py new file mode 100644 index 0000000..70530a8 --- /dev/null +++ b/sijapi/routers/forward.py @@ -0,0 +1,124 @@ +''' +Used to port-forwarding and reverse proxy configurations. +''' +#routers/forward.py + +import os +import io +import string +import json +import time +import base64 +import asyncpg +import asyncio +import subprocess +import requests +import random +import paramiko +import aiohttp +import httpx +from datetime import datetime +from hashlib import sha256 +from pathlib import Path +from typing import List, Optional +from pydantic import BaseModel +from PyPDF2 import PdfReader +from fastapi import APIRouter, Form, HTTPException, Request, Response, BackgroundTasks, status, Path as PathParam +from fastapi.responses import HTMLResponse, FileResponse, PlainTextResponse, JSONResponse, RedirectResponse +from fastapi.templating import Jinja2Templates +from selenium.webdriver.chrome.service import Service +from selenium.webdriver.chrome.options import Options +from webdriver_manager.chrome import ChromeDriverManager +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from sijapi import ( + L, API, Serve, LOGS_DIR, TS_ID, CASETABLE_PATH, COURTLISTENER_DOCKETS_URL, COURTLISTENER_API_KEY, + COURTLISTENER_BASE_URL, COURTLISTENER_DOCKETS_DIR, COURTLISTENER_SEARCH_DIR, ALERTS_DIR, + MAC_UN, MAC_PW, MAC_ID, TS_TAILNET, IMG_DIR, PUBLIC_KEY, OBSIDIAN_VAULT_DIR +) +from sijapi.classes import WidgetUpdate +from sijapi.utilities import bool_convert, sanitize_filename, assemble_journal_path +from sijapi.routers import gis + +forward = APIRouter() + +logger = L.get_module_logger("email") +def debug(text: str): logger.debug(text) +def info(text: str): logger.info(text) +def warn(text: str): logger.warning(text) +def err(text: str): logger.error(text) +def crit(text: str): logger.critical(text) + +async def forward_traffic(reader: asyncio.StreamReader, writer: asyncio.StreamWriter, destination: str): + try: + dest_host, dest_port = destination.split(':') + dest_port = int(dest_port) + except ValueError: + warn(f"Invalid destination format: {destination}. Expected 'host:port'.") + writer.close() + await writer.wait_closed() + return + + try: + dest_reader, dest_writer = await asyncio.open_connection(dest_host, dest_port) + except Exception as e: + warn(f"Failed to connect to destination {destination}: {str(e)}") + writer.close() + await writer.wait_closed() + return + + async def forward(src, dst): + try: + while True: + data = await src.read(8192) + if not data: + break + dst.write(data) + await dst.drain() + except Exception as e: + warn(f"Error in forwarding: {str(e)}") + finally: + dst.close() + await dst.wait_closed() + + await asyncio.gather( + forward(reader, dest_writer), + forward(dest_reader, writer) + ) + +async def start_server(source: str, destination: str): + if ':' in source: + host, port = source.split(':') + port = int(port) + else: + host = source + port = 80 + + server = await asyncio.start_server( + lambda r, w: forward_traffic(r, w, destination), + host, + port + ) + + async with server: + await server.serve_forever() + + +async def start_port_forwarding(): + if hasattr(Serve, 'forwarding_rules'): + for rule in Serve.forwarding_rules: + asyncio.create_task(start_server(rule.source, rule.destination)) + else: + warn("No forwarding rules found in the configuration.") + + +@forward.get("/forward_status") +async def get_forward_status(): + if hasattr(Serve, 'forwarding_rules'): + return {"status": "active", "rules": Serve.forwarding_rules} + else: + return {"status": "inactive", "message": "No forwarding rules configured"} + + +asyncio.create_task(start_port_forwarding()) \ No newline at end of file diff --git a/sijapi/routers/ghost.py b/sijapi/routers/ghost.py index f328302..cf60916 100644 --- a/sijapi/routers/ghost.py +++ b/sijapi/routers/ghost.py @@ -1,3 +1,19 @@ +''' +WIP. Will be used to manage and update Ghost blogs. +''' +#routers/ghost.py + +from fastapi import APIRouter +from datetime import date +import os +import requests +import json +import yaml +import jwt +from sijapi import GHOST_API_KEY, GHOST_API_URL + +ghost = APIRouter() + def generate_jwt_token(): key_id, key_secret = GHOST_API_KEY.split(':') iat = int(date.now().timestamp()) diff --git a/sijapi/routers/gis.py b/sijapi/routers/gis.py index d00ce14..9940fc6 100644 --- a/sijapi/routers/gis.py +++ b/sijapi/routers/gis.py @@ -1,6 +1,8 @@ ''' Uses Postgres/PostGIS for location tracking (data obtained via the companion mobile Pythonista scripts), and for geocoding purposes. ''' +#routers/gis.py + from fastapi import APIRouter, HTTPException, Query from fastapi.responses import HTMLResponse, JSONResponse import random diff --git a/sijapi/routers/health.py b/sijapi/routers/health.py index 41baa5e..67773c4 100644 --- a/sijapi/routers/health.py +++ b/sijapi/routers/health.py @@ -3,6 +3,8 @@ Health check module. /health returns `'status': 'ok'`, /id returns TS_ID, /route Depends on: TS_ID, LOGGER, SUBNET_BROADCAST ''' +#routers/health.py + import os import httpx import socket diff --git a/sijapi/routers/ig.py b/sijapi/routers/ig.py index 47825b5..8230471 100644 --- a/sijapi/routers/ig.py +++ b/sijapi/routers/ig.py @@ -1,6 +1,8 @@ ''' IN DEVELOPMENT: Instagram AI bot module. ''' +#routers/ig.py + from fastapi import APIRouter, UploadFile import os import io diff --git a/sijapi/routers/img.py b/sijapi/routers/img.py index f878271..80e374b 100644 --- a/sijapi/routers/img.py +++ b/sijapi/routers/img.py @@ -2,9 +2,10 @@ Image generation module using StableDiffusion and similar models by way of ComfyUI. DEPENDS ON: LLM module - COMFYUI_URL, COMFYUI_DIR, COMFYUI_OUTPUT_DIR, TS_SUBNET, TS_ADDRESS, DATA_DIR, IMG_CONFIG_DIR, IMG_DIR, IMG_WORKFLOWS_DIR, LOCAL_HOSTS, API.URL, PHOTOPRISM_USER*, PHOTOPRISM_URL*, PHOTOPRISM_PASS* + COMFYUI_URL, COMFYUI_DIR, COMFYUI_OUTPUT_DIR, TS_SUBNET, TS_ADDRESS, DATA_DIR, IMG_CONFIG_DIR, IMG_DIR, IMG_WORKFLOWS_DIR, LOCAL_HOSTS *unimplemented. ''' +#routers/img.py from fastapi import APIRouter, Request, Query from fastapi.responses import JSONResponse, RedirectResponse diff --git a/sijapi/routers/llm.py b/sijapi/routers/llm.py index 6329fc8..036f65a 100644 --- a/sijapi/routers/llm.py +++ b/sijapi/routers/llm.py @@ -2,6 +2,7 @@ Interfaces with Ollama and creates an OpenAI-compatible relay API. ''' # routers/llm.py + from fastapi import APIRouter, HTTPException, Request, Response, BackgroundTasks, File, Form, UploadFile from fastapi.responses import StreamingResponse, JSONResponse, FileResponse from datetime import datetime as dt_datetime diff --git a/sijapi/routers/news.py b/sijapi/routers/news.py index 24c6193..635cb8c 100644 --- a/sijapi/routers/news.py +++ b/sijapi/routers/news.py @@ -1,4 +1,8 @@ +''' +Used to scrape, process, summarize, markdownify, and speechify news articles. +''' # routers/news.py + import os import uuid import asyncio diff --git a/sijapi/routers/note.py b/sijapi/routers/note.py index 173c03c..2fb18ab 100644 --- a/sijapi/routers/note.py +++ b/sijapi/routers/note.py @@ -2,6 +2,7 @@ Manages an Obsidian vault, in particular daily notes, using information and functionality drawn from the other routers, primarily calendar, email, ig, llm, rag, img, serve, time, tts, and weather. ''' # routers/note.py + from fastapi import APIRouter, BackgroundTasks, File, UploadFile, Form, HTTPException, Response, Query, Path as FastAPIPath from fastapi.responses import JSONResponse, PlainTextResponse import os, re @@ -17,7 +18,7 @@ from fastapi import HTTPException, status from pathlib import Path from fastapi import APIRouter, Query, HTTPException from sijapi import API, L, OBSIDIAN_VAULT_DIR, OBSIDIAN_RESOURCES_DIR, OBSIDIAN_BANNER_SCENE, DEFAULT_11L_VOICE, DEFAULT_VOICE, GEO -from sijapi.routers import asr, cal, gis, img, llm, serve, time, tts, weather +from sijapi.routers import asr, cal, gis, img, llm, serve, timing, tts, weather from sijapi.utilities import assemble_journal_path, convert_to_12_hour_format, sanitize_filename, convert_degrees_to_cardinal, check_file_name, HOURLY_COLUMNS_MAPPING from sijapi.classes import Location @@ -388,7 +389,7 @@ async def build_daily_timeslips(date): ''' absolute_path, relative_path = assemble_journal_path(date, filename = "Timeslips", extension=".md", no_timestamp = True) - content = await time.process_timing_markdown(date, date) + content = await timing.process_timing_markdown(date, date) # document_content = await document.read() with open(absolute_path, 'wb') as f: f.write(content.encode()) diff --git a/sijapi/routers/rag.py b/sijapi/routers/rag.py index 1269b94..7acf1ca 100644 --- a/sijapi/routers/rag.py +++ b/sijapi/routers/rag.py @@ -2,6 +2,7 @@ IN DEVELOPMENT: Retrieval-Augmented Generation module. NOTES: Haven't yet decided if this should depend on the Obsidian and Chat modules, or if they should depend on it, or one of one the other the other. ''' +#routers/rag.py from fastapi import APIRouter from sijapi import L diff --git a/sijapi/routers/scrape.py b/sijapi/routers/scrape.py index 290ea4b..c160a82 100644 --- a/sijapi/routers/scrape.py +++ b/sijapi/routers/scrape.py @@ -1,3 +1,8 @@ +''' +Used to scrape data from websites, watch sites for changes, and make alerts based on specific changes. +''' +#routers/scrape.py + import asyncio import json import re diff --git a/sijapi/routers/serve.py b/sijapi/routers/serve.py index e87eb49..abbb896 100644 --- a/sijapi/routers/serve.py +++ b/sijapi/routers/serve.py @@ -1,6 +1,8 @@ ''' Web server module. Used by other modules when serving static content is required, e.g. the img image generation module. Also used to serve PUBLIC_KEY. ''' +#routers/serve.py + import os import io import string @@ -512,75 +514,3 @@ async def get_analytics(short_code: str): } -async def forward_traffic(reader: asyncio.StreamReader, writer: asyncio.StreamWriter, destination: str): - try: - dest_host, dest_port = destination.split(':') - dest_port = int(dest_port) - except ValueError: - warn(f"Invalid destination format: {destination}. Expected 'host:port'.") - writer.close() - await writer.wait_closed() - return - - try: - dest_reader, dest_writer = await asyncio.open_connection(dest_host, dest_port) - except Exception as e: - warn(f"Failed to connect to destination {destination}: {str(e)}") - writer.close() - await writer.wait_closed() - return - - async def forward(src, dst): - try: - while True: - data = await src.read(8192) - if not data: - break - dst.write(data) - await dst.drain() - except Exception as e: - warn(f"Error in forwarding: {str(e)}") - finally: - dst.close() - await dst.wait_closed() - - await asyncio.gather( - forward(reader, dest_writer), - forward(dest_reader, writer) - ) - -async def start_server(source: str, destination: str): - if ':' in source: - host, port = source.split(':') - port = int(port) - else: - host = source - port = 80 - - server = await asyncio.start_server( - lambda r, w: forward_traffic(r, w, destination), - host, - port - ) - - async with server: - await server.serve_forever() - - -async def start_port_forwarding(): - if hasattr(Serve, 'forwarding_rules'): - for rule in Serve.forwarding_rules: - asyncio.create_task(start_server(rule.source, rule.destination)) - else: - warn("No forwarding rules found in the configuration.") - - -@serve.get("/forward_status") -async def get_forward_status(): - if hasattr(Serve, 'forwarding_rules'): - return {"status": "active", "rules": Serve.forwarding_rules} - else: - return {"status": "inactive", "message": "No forwarding rules configured"} - - -asyncio.create_task(start_port_forwarding()) \ No newline at end of file diff --git a/sijapi/routers/signal.py b/sijapi/routers/signal.py deleted file mode 100644 index 1e21e41..0000000 --- a/sijapi/routers/signal.py +++ /dev/null @@ -1,33 +0,0 @@ -""" -Signal Bot example, repeats received messages. -""" -import os -from fastapi import APIRouter -from semaphore import Bot, ChatContext -from sijapi import L - -signal = APIRouter() - -logger = L.get_module_logger("signal") - -def debug(text: str): logger.debug(text) -def info(text: str): logger.info(text) -def warn(text: str): logger.warning(text) -def err(text: str): logger.error(text) -def crit(text: str): logger.critical(text) - -async def echo(ctx: ChatContext) -> None: - if not ctx.message.empty(): - await ctx.message.typing_started() - await ctx.message.reply(ctx.message.get_body()) - await ctx.message.typing_stopped() - -async def main() -> None: - """Start the bot.""" - async with Bot(os.environ["SIGNAL_PHONE_NUMBER"]) as bot: - bot.register_handler("", echo) - await bot.start() - -if __name__ == '__main__': - import anyio - anyio.run(main) \ No newline at end of file diff --git a/sijapi/routers/time.py b/sijapi/routers/timing.py similarity index 98% rename from sijapi/routers/time.py rename to sijapi/routers/timing.py index 3c20319..2854971 100644 --- a/sijapi/routers/time.py +++ b/sijapi/routers/timing.py @@ -1,6 +1,8 @@ ''' Uses the Timing.app API to get nicely formatted timeslip charts and spreadsheets. ''' +#routers/timing.py + import tempfile import os import json @@ -29,8 +31,8 @@ from sijapi import L, TIMING_API_KEY, TIMING_API_URL from sijapi.routers import gis -time = APIRouter(tags=["private"]) -logger = L.get_module_logger("time") +timing = APIRouter(tags=["private"]) +logger = L.get_module_logger("timing") def debug(text: str): logger.debug(text) def info(text: str): logger.info(text) def warn(text: str): logger.warning(text) @@ -56,7 +58,7 @@ class TimingRequest(BaseModel): #################### #### TIMING API #### #################### -@time.post("/time/post") +@timing.post("/time/post") async def post_time_entry_to_timing(entry: Dict): url = 'https://web.timingapp.com/api/v1/time-entries' headers = { @@ -195,7 +197,7 @@ def create_time_entry(original_entry, start_time, end_time, duration_seconds): # TIMELINE -@time.get("/time/line") +@timing.get("/time/line") async def get_timing_timeline( request: Request, start_date: str = Query(..., regex=r"\d{4}-\d{2}-\d{2}"), @@ -251,7 +253,7 @@ def process_timeline(timing_data, queried_start_date, queried_end_date): # CSV -@time.get("/time/csv") +@timing.get("/time/csv") async def get_timing_csv( request: Request, start_date: str = Query(..., regex=r"\d{4}-\d{2}-\d{2}"), @@ -311,7 +313,7 @@ def process_csv(timing_data, queried_start_date, queried_end_date): return output.getvalue() # MARKDOWN -@time.get("/time/markdown3") +@timing.get("/time/markdown3") async def get_timing_markdown3( request: Request, start_date: str = Query(..., regex=r"\d{4}-\d{2}-\d{2}"), @@ -373,7 +375,7 @@ def process_timing_markdown3(timing_data, queried_start_date, queried_end_date): return "\n".join(markdown_output) -@time.get("/time/markdown") +@timing.get("/time/markdown") async def get_timing_markdown( request: Request, start: str = Query(..., regex=r"\d{4}-\d{2}-\d{2}"), @@ -440,7 +442,7 @@ async def process_timing_markdown(start_date: datetime, end_date: datetime): # t #JSON -@time.get("/time/json") +@timing.get("/time/json") async def get_timing_json( request: Request, start_date: str = Query(..., regex=r"\d{4}-\d{2}-\d{2}"), @@ -563,13 +565,13 @@ async def post_time_entry_to_timing(entry): return response.status_code, response.json() -@time.get("/time/flagemoji/{country_code}") +@timing.get("/time/flagemoji/{country_code}") def flag_emoji(country_code: str): offset = 127397 flag = ''.join(chr(ord(char) + offset) for char in country_code.upper()) return {"emoji": flag} -@time.head("/time/") +@timing.head("/time/") async def read_root(): return {} \ No newline at end of file diff --git a/sijapi/routers/tts.py b/sijapi/routers/tts.py index d6a0821..7b59c0a 100644 --- a/sijapi/routers/tts.py +++ b/sijapi/routers/tts.py @@ -1,6 +1,8 @@ ''' Uses xtts-v2 and/or the Elevenlabs API for text to speech. ''' +#routers/tts.py + from fastapi import APIRouter, UploadFile, HTTPException, Response, Form, File, BackgroundTasks, Depends, Request from fastapi.responses import Response, StreamingResponse, FileResponse from fastapi.responses import StreamingResponse, PlainTextResponse diff --git a/sijapi/routers/weather.py b/sijapi/routers/weather.py index 27eb1c8..ec3ec44 100644 --- a/sijapi/routers/weather.py +++ b/sijapi/routers/weather.py @@ -1,6 +1,8 @@ ''' Uses the VisualCrossing API and Postgres/PostGIS to source local weather forecasts and history. ''' +#routers/weather.py + import asyncio import traceback from fastapi import APIRouter, HTTPException, Query