Major update - database sync methods finally work reliably

README.md

@@ -44,4 +44,4 @@ IN DRAFT
 IN DRAFT
 
 ## Extras
-[Apple Shortcut for companion Pythonista script](https://www.icloud.com/shortcuts/d63b179f9c664b0cbbacb1a607767ef7)
+[Apple Shortcut for location tracking Pythonista script](https://www.icloud.com/shortcuts/d63b179f9c664b0cbbacb1a607767ef7)

requirements.txt

@@ -51,4 +51,4 @@ urllib3
 anyio
 semaphore
 location
-SRTM.py
+SRTM.py

sijapi/config/tts.yaml-example

@@ -0,0 +1,10 @@
+default: xtts
+email: xtts
+webclip: 11L
+rss: xtts
+podcast_dir: '{{ DIR.HOME }}/Library/Mobile Documents/iCloud~co~supertop~castro/Documents/Sideloads'
+xtts:
+  voice: joanne
+elevenlabs:
+  voice: luna
+  api_key: '{{ SECRET.ELEVENLABS_API_KEY }}'

sijapi/data/ms365/.cert.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCikW67UW0RpncJ
+h4Ha9HumZ/WzgEZWRWkgksVJIOJ8t1PftctizLUlz+xMWNl+Volp4crxnPpnczis
+pOXU4g65XoFHHpF9nhF/3YDgxo5BDEM/mIIKEO9LFkIBQVBdE85qXnIVot5MfuNj
+HeyEs7doRMBxilOSR/DkT8bTWu7m5yeHlF58iYVOxxssGhP3bo7CcAcaZD1LJBnP
+Df+UBqzWQ9as903p5bFixHy2kVz8Qkd5k5tyIQ/tXqlhRfLLHG4AYHmBbS06CAg0
+nEpKUeQx4l1J/ykAjQTwhHf70xv1z0p28mHcr5ib4UvpYK9fMM6FKWenwlqA3qrK
+zQUJQ7E/AgMBAAECggEAQ5H/XIxzsSpnv+Y66y9DVd9QGNPwaFthXtCif8rTWNM6
+YXnGl8JOaPELXpBvljuR0hivqc19pxIVNG01uk5boGDPiygBgRz6WRNQRh1Bc3gN
+W5mgM17ml2cg+DSVmppo6X1oHeYcT99N1BzT+jRYv1YURx0fr2WHkt413hOlyQMR
+b8ir/TOBx3olg4KBPDuysRC5BCIr3Mkz4jsh+9wVIOReKVezsy7nxJVzipcxOyZO
+9VGgvlw4XLrRTOJEv4e3ldcg219j5KEGsJ4FFSziSmpj5fN4Vt+JmY7nueSHyL6P
+3hX52lRfOcTXTEeiEV2cXkm3h8uQ3zfiZRYi3P0DQQKBgQDXGBZc3WnfXim0u1EV
+JzZFwxBS7SHkyGgnd50ak6e9yDbdxOuYIOo4mBlc3ofd20EfT4TvR7Xyw+PD2fWJ
++isdwCEb9JZZ1H6RDGIzSDYXGNeGId4kMKBZdmKpEeLgStihsrYp/nxtwcE/8A7N
+jCEKZj1ld7QfbQlGT/NJ4Jj80wKBgQDBfBpth6vMyCETKMJVqYd2qhVnJKiFLfRn
+OD/Ck6xwUuedbfe9M34wNO3Pn2Xvu1xVsQGb2dmlT345Iq9Z1nbZCGXyY9yfLnTV
+fz7F2utjUjaJtuiSb52SgX7MWZ8E4nbqqKnC4SYSIlaeuL9KK3r/x6bcNLAYPcdk
+qKHexDkGZQKBgF0JGyshzhiChzGYUBMBOfVk0Ru9XAq0MHDZyQdk1Io/HpRAB9Nu
+cUD3cQj9a/EnU/yyDYLeFrIhztO44/7BSYL9xpRr79h9FB2zKIqb8mF9KkPnREmN
+Ct6HWVdd2C9B0H/oZ+i0HafvxaHdONnpgaUY4feQlkV9iSRzknzi++lnAoGAXOuu
+/X80oMpUKBFhEyaxqemREdHnJN6nC5NV+6pUHDWUimSvn6vFJH2m4BlbKUC/3V9+
+uExtXBjLM8FWmTyIIz8HRttyrvfuoEHV8ctrVG29R3ISS5FTCXMrZBR+bCgemB+c
+N71NPVREaUGsjIBJN+G4XvTmxR2WTt81rfhqsokCgYEA1It9e9Ut2Krzf2FaPGLG
+ozlKhWadMNntthg3uxze80Rx8WSvgJQdbVpdbni2B/9xdYBIIljW/LGivYBrCSSp
+aXFpXL7ZGkvl3b0MkojfghIpXVGqu+8ISDtFgL0B1gZ5hq9xMBl94fLVfQgC9Cy6
+uvDHlz+fjWaWKYUPiouAtVs=
+-----END PRIVATE KEY-----

sijapi/data/ms365/.cert.pem

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDAzCCAeugAwIBAgIUc+EtilZslnS7N6MAx0u9HeP83wAwDQYJKoZIhvcNAQEL
+BQAwETEPMA0GA1UEAwwGcHl0aG9uMB4XDTI0MDYwODEyNTcxM1oXDTI1MDYwODEy
+NTcxM1owETEPMA0GA1UEAwwGcHl0aG9uMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
+MIIBCgKCAQEAopFuu1FtEaZ3CYeB2vR7pmf1s4BGVkVpIJLFSSDifLdT37XLYsy1
+Jc/sTFjZflaJaeHK8Zz6Z3M4rKTl1OIOuV6BRx6RfZ4Rf92A4MaOQQxDP5iCChDv
+SxZCAUFQXRPOal5yFaLeTH7jYx3shLO3aETAcYpTkkfw5E/G01ru5ucnh5RefImF
+TscbLBoT926OwnAHGmQ9SyQZzw3/lAas1kPWrPdN6eWxYsR8tpFc/EJHeZObciEP
+7V6pYUXyyxxuAGB5gW0tOggINJxKSlHkMeJdSf8pAI0E8IR3+9Mb9c9KdvJh3K+Y
+m+FL6WCvXzDOhSlnp8JagN6qys0FCUOxPwIDAQABo1MwUTAdBgNVHQ4EFgQUS74L
+HD4Cdzh1ajatbvSHNQXIVvAwHwYDVR0jBBgwFoAUS74LHD4Cdzh1ajatbvSHNQXI
+VvAwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAhpwtVubDjsyq
+/LiTwpXKhjB/eFb6Yse782Iq+9rsiGGhsN88IA25fKgsJ2AIkR/KA7QSle3ds+1Q
+EY9/vqpWnfBdpvOi7oV7ozBe+t/5JLu1GQBzg+cVa4iLAWYCiqg1d5NDdIcYMfsM
+Yq2a3eQoP8Xbj3fFMXdNopXARa1d1zHB3ugXIJYinwMlS0EoGXVQVaHhemOh8GwW
+keRaA6TDTBFsp0Gl4jv/NrisAt4qg+rlqr0mNcQK92vRX65mDWa/cQKwpUH8+Seq
+Jl717NnsIGcqYWg8SSvVlkbFfxYhwYICXT824MAdSZtpHNCN/TegxsviYnlDyJKj
+OJzn4fCxnQ==
+-----END CERTIFICATE-----

sijapi/data/ms365/.token.txt

@@ -0,0 +1 @@
+{"token_type": "Bearer", "scope": "Calendars.Read Calendars.ReadWrite User.Read profile openid email", "expires_in": 3962, "ext_expires_in": 3962, "access_token": "eyJ0eXAiOiJKV1QiLCJub25jZSI6IldDeU91YXllN1RFX2FPM0F1alhlYmtvYTdVRHpUR1dVNWt5d3lJeDZ1MGciLCJhbGciOiJSUzI1NiIsIng1dCI6InE3UDFOdnh1R1F3RE4yVGFpTW92alo4YVp3cyIsImtpZCI6InE3UDFOdnh1R1F3RE4yVGFpTW92alo4YVp3cyJ9.eyJhdWQiOiIwMDAwMDAwMy0wMDAwLTAwMDAtYzAwMC0wMDAwMDAwMDAwMDAiLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC9iYWQ3ODA0OC1hNmUwLTQ3YjEtYTI0Yi00MDNjNDQ0YWEzNDkvIiwiaWF0IjoxNzE4Mzc0NzA5LCJuYmYiOjE3MTgzNzQ3MDksImV4cCI6MTcxODM3ODk3MiwiYWNjdCI6MCwiYWNyIjoiMSIsImFpbyI6IkFWUUFxLzhYQUFBQVRnWHJ0Q1pCVjlPa1M2WldldHVVSHNMSFN0LzErYVcxT1BSSjVOWjJEL1Bzd05mY1Fxb0JTNEFZRmhLR3UvaE5TNnNWOGtLQUpmcDNNTzdqRUlNMEZrY1VaZ0IyREh4cWdOK3lUQVBUYnRVPSIsImFtciI6WyJwd2QiLCJtZmEiXSwiYXBwX2Rpc3BsYXluYW1lIjoicHl0aG9uIiwiYXBwaWQiOiJjZThjYmQyNC1mMTQ2LTRkYzctOGVlNy01MWQ5YjY5ZGVjNTkiLCJhcHBpZGFjciI6IjEiLCJmYW1pbHlfbmFtZSI6IkluY2UtSm9oYW5uc2VuIiwiZ2l2ZW5fbmFtZSI6IlNhbmd5ZSIsImlkdHlwIjoidXNlciIsImlwYWRkciI6IjY4LjIzNS40NC4yMDIiLCJuYW1lIjoiU2FuZ3llIEluY2UtSm9oYW5uc2VuIiwib2lkIjoiMWNiMWQwNDAtZmM1OS00MjMxLTllMDUtOWRjNGI0MzJjY2MxIiwicGxhdGYiOiI1IiwicHVpZCI6IjEwMDMyMDAyQTNGQjU4RjIiLCJyaCI6IjAuQVgwQVNJRFh1dUNtc1VlaVMwQThSRXFqU1FNQUFBQUFBQUFBd0FBQUFBQUFBQUMxQUk4LiIsInNjcCI6IkNhbGVuZGFycy5SZWFkIENhbGVuZGFycy5SZWFkV3JpdGUgVXNlci5SZWFkIHByb2ZpbGUgb3BlbmlkIGVtYWlsIiwic2lnbmluX3N0YXRlIjpbImttc2kiXSwic3ViIjoiV0FYVFdIR0puVFhBTjlncmIyamlEU3U4ZENOMmc0dDFacERiVHlwM1k3USIsInRlbmFudF9yZWdpb25fc2NvcGUiOiJOQSIsInRpZCI6ImJhZDc4MDQ4LWE2ZTAtNDdiMS1hMjRiLTQwM2M0NDRhYTM0OSIsInVuaXF1ZV9uYW1lIjoic2FuZ3llaWpAd2VzdGVybmxhdy5vcmciLCJ1cG4iOiJzYW5neWVpakB3ZXN0ZXJubGF3Lm9yZyIsInV0aSI6InFHcVlEODRzaDBHMFBfSEdldlVXQUEiLCJ2ZXIiOiIxLjAiLCJ3aWRzIjpbImI3OWZiZjRkLTNlZjktNDY4OS04MTQzLTc2YjE5NGU4NTUwOSJdLCJ4bXNfaWRyZWwiOiIxIDIiLCJ4bXNfc3QiOnsic3ViIjoieXhjdzFhV1FiM2VrX0FvNFRuRy11SDN6ZndGbVRRUmMxVGpFaEdqZ2p2WSJ9LCJ4bXNfdGNkdCI6MTY0ODY4MTc1Mn0.ssgIrbYo1SPNusoB9bNIB7pLxCmwBKhox__KOnwRRtnE63vbfGWAl53ww1KpNWPdDfC3p94yuPybTRqjZnTPluv1oJgGINml4AleUnZJnJttRsFHvGflzKOLtXnzmhQGUBXxu7QucKTCMH4J36neeQAWthITMwCHbaGmSy0RLotaIsoEHIufxR9ZEYD4XP5e3sFX54eSnyf4P3GgHHC1y5xxWUlemG4G1BRas8i7oX9o-gqRFube6BMtCLir_HMTNPfrCG-lhd9msLhc6e_WJSmLMHQ7RVLo-GlTMY9UouE190GzBBVKUrTg462I3kP_GayO1kt6qopBrwnF6bDUsw", "refresh_token": "0.AX0ASIDXuuCmsUeiS0A8REqjSSS9jM5G8cdNjudR2bad7Fm1AI8.AgABAwEAAAApTwJmzXqdR4BN2miheQMYAgDs_wUA9P_rTWFRiXWkWxvihyyXonsZPLrulRvnKKRlZ9PxKUltEOQsxjlg86xvCYzAS6dYeDBQiQxRAS_WEuuXVmTqUWDVwqgwQOa3BCbLwxQhPwfG-O9uFY6D239Jo8rdXTrf8XOntGs6fCn3wuo5kvJr2D-FGRA_EepltvRxZgrWdHROKuqoL_ArjLDdoFP7zM95MKhVYTmCO7LCM7u6O9ItU4_6y2_lH864zUivT1LFG8-h9sx0Ln3wd8LBP3P5GSeXwtQlkbNpj1FNDl_Ex5SwGCTM7uDHj0dn5CdUMgLkOcAC__HJdzmlEryTquoXcjd1RAmkq1MqAGD7QQreI7NQTZXwTcjoMwiBg92-bk-_o2ajeIVqzgOVBQIu1W8gkN2F7PAqRc5lGB-2mAXchqKMoL31CLUPxgTMBjWgR4waAjfZXT4h2WqXAAdGFy2nzUJAjyEQa9ZW1J5B6asCf3cVJQwI6nWIN7OphrXkGHl0ffpfrC-skVG3N2vrelAutRvyvWi4bbMqAZNglRrkTn5G_kULmnyydZBcFSc5uPmKD7OkfBD5UpTa_KLTjYexWRVsBfG9czIVxOh3ojnnza9BjrN5cHwHhzPM1t67E5iqronvT2OR_r-4BerUfRNHXrxwrLvDUEZwQ8o5IRs2N5FH0y_QN049o_NTgqytCj6wrIB4T-ZBUK2AsFej7ipdHAMYtWLZdoAo1o4nMuPBb4syN0VYd1sLUP-RQ5iv7wIkMWmNjhjIErIktZ134pGK9TlWa904H6HUin0qNTXyTmX2feE0nBlm6xJbO1ISfFkaf8aEjcAMfeu9qiArKQqUgvY", "expires_at": 1718378971}

sijapi/helpers/cli.py

@@ -1,57 +0,0 @@
-# cli.py
-import click
-import asyncio
-from datetime import datetime as dt_datetime, timedelta
-
-# Import your async functions and dependencies
-from sijapi import build_daily_note_range_endpoint, gis # broken!
-
-def async_command(f):
-    @click.command()
-    @click.pass_context
-    def wrapper(ctx, *args, **kwargs):
-        async def run():
-            return await f(*args, **kwargs)
-        return asyncio.run(run())
-    return wrapper
-
-@click.group()
-def cli():
-    """CLI for your application."""
-    pass
-
-@cli.command()
-@click.argument('dt_start')
-@click.argument('dt_end')
-@async_command
-async def bulk_note_range(dt_start: str, dt_end: str):
-    """
-    Build daily notes for a date range.
-    
-    DT_START and DT_END should be in YYYY-MM-DD format.
-    """
-    try:
-        start_date = dt_datetime.strptime(dt_start, "%Y-%m-%d")
-        end_date = dt_datetime.strptime(dt_end, "%Y-%m-%d")
-    except ValueError:
-        click.echo("Error: Dates must be in YYYY-MM-DD format.")
-        return
-
-    if start_date > end_date:
-        click.echo("Error: Start date must be before or equal to end date.")
-        return
-
-    results = []
-    current_date = start_date
-    while current_date <= end_date:
-        formatted_date = await gis.dt(current_date)
-        result = await build_daily_note(formatted_date)
-        results.append(result)
-        current_date += timedelta(days=1)
-
-    click.echo("Generated notes for the following dates:")
-    for url in results:
-        click.echo(url)
-
-if __name__ == '__main__':
-    cli()

sijapi/routers/asr.py

@@ -2,6 +2,7 @@
 Uses whisper_cpp to create an OpenAI-compatible Whisper web service.
 '''
 # routers/asr.py
+
 import os
 import uuid
 import json

sijapi/routers/cal.py

@@ -3,6 +3,8 @@ Calendar module using macOS Calendars and/or Microsoft 365 via its Graph API.
 Depends on: 
   LOGGER, ICAL_TOGGLE, ICALENDARS, MS365_TOGGLE, MS365_CLIENT_ID, MS365_SECRET, MS365_AUTHORITY_URL, MS365_SCOPE, MS365_REDIRECT_PATH, MS365_TOKEN_PATH
 '''
+#routers/cal.py
+
 from fastapi import APIRouter, Depends, HTTPException, status, Request
 from fastapi.responses import RedirectResponse, JSONResponse
 from fastapi.security import OAuth2PasswordBearer

sijapi/routers/cf.py

@@ -1,6 +1,8 @@
 '''
 IN DEVELOPMENT - Cloudflare + Caddy module. Based on a bash script that's able to rapidly deploy new Cloudflare subdomains on new Caddy reverse proxy configurations, managing everything including restarting Caddy. The Python version needs more testing before actual use.
 '''
+#routers/cf.py
+
 from fastapi import APIRouter, HTTPException
 from pydantic import BaseModel
 from fastapi.responses import PlainTextResponse, JSONResponse

sijapi/routers/dist.py

@@ -1,65 +0,0 @@
-'''
-WORK IN PROGRESS: This module will handle tasks related to multi-server environments.
-'''
-from fastapi import APIRouter, HTTPException
-import asyncio
-import logging
-from sijapi.utilities import run_ssh_command
-from sijapi import L, REBOOT_SCRIPT_PATH, HOST_CONFIG, API_CONFIG
-
-dist = APIRouter()
-logger = L.get_module_logger("dist")
-def debug(text: str): logger.debug(text)
-def info(text: str): logger.info(text)
-def warn(text: str): logger.warning(text)
-def err(text: str): logger.error(text)
-def crit(text: str): logger.critical(text)
-
-@dist.get("/update-restart-others")
-async def update_and_restart_others():
-    results = []
-    for server in API_CONFIG.servers:
-        if HOST_CONFIG.id != server.id:
-            try:
-                output, error = await run_ssh_command(server, f"bash {server.scripts.update_and_restart}")
-                results.append({"server": server.id, "status": "success", "output": output, "error": error})
-            except Exception as e:
-                results.append({"server": server.id, "status": "failed", "error": str(e)})
-    return {"message": "Update and restart process initiated for other servers.", "results": results}
-
-@dist.get("/update-restart-self")
-async def update_and_restart_self(safe: bool = True):
-    if safe and not await ensure_redundancy():
-        raise HTTPException(status_code=400, detail="Cannot safely restart: no redundancy available")
-    try:
-        process = await asyncio.create_subprocess_exec(
-            "bash", API_CONFIG.servers[HOST_CONFIG.id].scripts.update_and_restart,
-            stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
-        )
-        stdout, stderr = await process.communicate()
-        info(f"Update and restart initiated for self. Stdout: {stdout.decode()}. Stderr: {stderr.decode()}")
-        return {"message": "Update and restart process initiated for this server."}
-    except Exception as e:
-        err(f"Failed to initiate update and restart for self: {str(e)}")
-        raise HTTPException(status_code=500, detail=f"Failed to initiate update and restart: {str(e)}")
-
-@dist.get("/update-and-restart-all")
-async def update_and_restart_all():
-    others_result = await update_and_restart_others()
-    self_result = await update_and_restart_self(safe=False)
-    return {"others": others_result, "self": self_result}
-
-async def ensure_redundancy():
-    import aiohttp
-    redundancy = False
-    async with aiohttp.ClientSession() as session:
-        for server in API_CONFIG.servers:
-            if server.id != HOST_CONFIG.id:
-                try:
-                    async with session.get(f"{server.protocol}://{server.ip}:{server.port}/health") as response:
-                        if response.status // 100 == 2:
-                            redundancy = True
-                            break
-                except aiohttp.ClientError:
-                    warn(f"Failed to check health of server {server.id}")
-    return redundancy

sijapi/routers/email.py

@@ -1,6 +1,8 @@
 '''
 Uses IMAP and SMTP login credentials to monitor an inbox and summarize incoming emails that match certain criteria and save the Text-To-Speech converted summaries into a specified "podcast" folder. 
 '''
+#routers/email.py
+
 from fastapi import APIRouter
 import asyncio
 import aiofiles

sijapi/routers/forward.py

@@ -0,0 +1,124 @@
+'''
+Used to port-forwarding and reverse proxy configurations.
+'''
+#routers/forward.py
+
+import os
+import io
+import string
+import json
+import time
+import base64
+import asyncpg
+import asyncio
+import subprocess
+import requests
+import random
+import paramiko
+import aiohttp
+import httpx
+from datetime import datetime
+from hashlib import sha256
+from pathlib import Path
+from typing import List, Optional
+from pydantic import BaseModel
+from PyPDF2 import PdfReader
+from fastapi import APIRouter, Form, HTTPException, Request, Response, BackgroundTasks, status, Path as PathParam
+from fastapi.responses import HTMLResponse, FileResponse, PlainTextResponse, JSONResponse, RedirectResponse
+from fastapi.templating import Jinja2Templates
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.chrome.options import Options
+from webdriver_manager.chrome import ChromeDriverManager
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from sijapi import (
+    L, API, Serve, LOGS_DIR, TS_ID, CASETABLE_PATH, COURTLISTENER_DOCKETS_URL, COURTLISTENER_API_KEY,
+    COURTLISTENER_BASE_URL, COURTLISTENER_DOCKETS_DIR, COURTLISTENER_SEARCH_DIR, ALERTS_DIR,
+    MAC_UN, MAC_PW, MAC_ID, TS_TAILNET, IMG_DIR, PUBLIC_KEY, OBSIDIAN_VAULT_DIR
+)
+from sijapi.classes import WidgetUpdate
+from sijapi.utilities import bool_convert, sanitize_filename, assemble_journal_path
+from sijapi.routers import gis
+
+forward = APIRouter()
+
+logger = L.get_module_logger("email")
+def debug(text: str): logger.debug(text)
+def info(text: str): logger.info(text)
+def warn(text: str): logger.warning(text)
+def err(text: str): logger.error(text)
+def crit(text: str): logger.critical(text)
+
+async def forward_traffic(reader: asyncio.StreamReader, writer: asyncio.StreamWriter, destination: str):
+    try:
+        dest_host, dest_port = destination.split(':')
+        dest_port = int(dest_port)
+    except ValueError:
+        warn(f"Invalid destination format: {destination}. Expected 'host:port'.")
+        writer.close()
+        await writer.wait_closed()
+        return
+    
+    try:
+        dest_reader, dest_writer = await asyncio.open_connection(dest_host, dest_port)
+    except Exception as e:
+        warn(f"Failed to connect to destination {destination}: {str(e)}")
+        writer.close()
+        await writer.wait_closed()
+        return
+
+    async def forward(src, dst):
+        try:
+            while True:
+                data = await src.read(8192)
+                if not data:
+                    break
+                dst.write(data)
+                await dst.drain()
+        except Exception as e:
+            warn(f"Error in forwarding: {str(e)}")
+        finally:
+            dst.close()
+            await dst.wait_closed()
+
+    await asyncio.gather(
+        forward(reader, dest_writer),
+        forward(dest_reader, writer)
+    )
+
+async def start_server(source: str, destination: str):
+    if ':' in source:
+        host, port = source.split(':')
+        port = int(port)
+    else:
+        host = source
+        port = 80
+
+    server = await asyncio.start_server(
+        lambda r, w: forward_traffic(r, w, destination),
+        host,
+        port
+    )
+
+    async with server:
+        await server.serve_forever()
+
+
+async def start_port_forwarding():
+    if hasattr(Serve, 'forwarding_rules'):
+        for rule in Serve.forwarding_rules:
+            asyncio.create_task(start_server(rule.source, rule.destination))
+    else:
+        warn("No forwarding rules found in the configuration.")
+
+
+@forward.get("/forward_status")
+async def get_forward_status():
+    if hasattr(Serve, 'forwarding_rules'):
+        return {"status": "active", "rules": Serve.forwarding_rules}
+    else:
+        return {"status": "inactive", "message": "No forwarding rules configured"}
+
+
+asyncio.create_task(start_port_forwarding())

sijapi/routers/ghost.py

@@ -1,3 +1,19 @@
+'''
+WIP. Will be used to manage and update Ghost blogs.
+'''
+#routers/ghost.py
+
+from fastapi import APIRouter
+from datetime import date
+import os
+import requests
+import json
+import yaml
+import jwt
+from sijapi import GHOST_API_KEY, GHOST_API_URL
+
+ghost = APIRouter()
+
 def generate_jwt_token():
     key_id, key_secret = GHOST_API_KEY.split(':')
     iat = int(date.now().timestamp())

sijapi/routers/gis.py

@@ -1,6 +1,8 @@
 '''
 Uses Postgres/PostGIS for location tracking (data obtained via the companion mobile Pythonista scripts), and for geocoding purposes.
 '''
+#routers/gis.py
+
 from fastapi import APIRouter, HTTPException, Query
 from fastapi.responses import HTMLResponse, JSONResponse
 import random

sijapi/routers/health.py

@@ -3,6 +3,8 @@ Health check module. /health returns `'status': 'ok'`, /id returns TS_ID, /route
 Depends on:
   TS_ID, LOGGER, SUBNET_BROADCAST
 '''
+#routers/health.py
+
 import os
 import httpx
 import socket

sijapi/routers/ig.py

@@ -1,6 +1,8 @@
 '''
 IN DEVELOPMENT: Instagram AI bot module.
 '''
+#routers/ig.py
+
 from fastapi import APIRouter, UploadFile
 import os
 import io

sijapi/routers/img.py

@@ -2,9 +2,10 @@
 Image generation module using StableDiffusion and similar models by way of ComfyUI.
 DEPENDS ON:
   LLM module
-  COMFYUI_URL, COMFYUI_DIR, COMFYUI_OUTPUT_DIR, TS_SUBNET, TS_ADDRESS, DATA_DIR, IMG_CONFIG_DIR, IMG_DIR, IMG_WORKFLOWS_DIR, LOCAL_HOSTS, API.URL, PHOTOPRISM_USER*, PHOTOPRISM_URL*, PHOTOPRISM_PASS*
+  COMFYUI_URL, COMFYUI_DIR, COMFYUI_OUTPUT_DIR, TS_SUBNET, TS_ADDRESS, DATA_DIR, IMG_CONFIG_DIR, IMG_DIR, IMG_WORKFLOWS_DIR, LOCAL_HOSTS
 *unimplemented.
 '''
+#routers/img.py
 
 from fastapi import APIRouter, Request, Query
 from fastapi.responses import JSONResponse, RedirectResponse

sijapi/routers/llm.py

@@ -2,6 +2,7 @@
 Interfaces with Ollama and creates an OpenAI-compatible relay API.
 '''
 # routers/llm.py
+
 from fastapi import APIRouter, HTTPException, Request, Response, BackgroundTasks, File, Form, UploadFile
 from fastapi.responses import StreamingResponse, JSONResponse, FileResponse
 from datetime import datetime as dt_datetime

sijapi/routers/news.py

@@ -1,4 +1,8 @@
+'''
+Used to scrape, process, summarize, markdownify, and speechify news articles.
+'''
 # routers/news.py
+
 import os
 import uuid
 import asyncio

sijapi/routers/note.py

@@ -2,6 +2,7 @@
 Manages an Obsidian vault, in particular daily notes, using information and functionality drawn from the other routers, primarily calendar, email, ig, llm, rag, img, serve, time, tts, and weather.
 '''
 # routers/note.py
+
 from fastapi import APIRouter, BackgroundTasks, File, UploadFile, Form, HTTPException, Response, Query, Path as FastAPIPath
 from fastapi.responses import JSONResponse, PlainTextResponse
 import os, re
@@ -17,7 +18,7 @@ from fastapi import HTTPException, status
 from pathlib import Path
 from fastapi import APIRouter, Query, HTTPException
 from sijapi import API, L, OBSIDIAN_VAULT_DIR, OBSIDIAN_RESOURCES_DIR, OBSIDIAN_BANNER_SCENE, DEFAULT_11L_VOICE, DEFAULT_VOICE, GEO
-from sijapi.routers import asr, cal, gis, img, llm, serve, time, tts, weather
+from sijapi.routers import asr, cal, gis, img, llm, serve, timing, tts, weather
 from sijapi.utilities import assemble_journal_path, convert_to_12_hour_format, sanitize_filename, convert_degrees_to_cardinal, check_file_name, HOURLY_COLUMNS_MAPPING
 from sijapi.classes import Location
 
@@ -388,7 +389,7 @@ async def build_daily_timeslips(date):
 
     '''
     absolute_path, relative_path = assemble_journal_path(date, filename = "Timeslips", extension=".md", no_timestamp = True)
-    content = await time.process_timing_markdown(date, date)
+    content = await timing.process_timing_markdown(date, date)
     # document_content = await document.read()
     with open(absolute_path, 'wb') as f:
         f.write(content.encode())

sijapi/routers/rag.py

@@ -2,6 +2,7 @@
 IN DEVELOPMENT: Retrieval-Augmented Generation module.
 NOTES: Haven't yet decided if this should depend on the Obsidian and Chat modules, or if they should depend on it, or one of one the other the other.
 '''
+#routers/rag.py
 
 from fastapi import APIRouter
 from sijapi import L

sijapi/routers/scrape.py

@@ -1,3 +1,8 @@
+'''
+Used to scrape data from websites, watch sites for changes, and make alerts based on specific changes.
+'''
+#routers/scrape.py
+
 import asyncio
 import json
 import re

sijapi/routers/serve.py

@@ -1,6 +1,8 @@
 '''
 Web server module. Used by other modules when serving static content is required, e.g. the img image generation module. Also used to serve PUBLIC_KEY.
 '''
+#routers/serve.py
+
 import os
 import io
 import string
@@ -512,75 +514,3 @@ async def get_analytics(short_code: str):
     }
 
 
-async def forward_traffic(reader: asyncio.StreamReader, writer: asyncio.StreamWriter, destination: str):
-    try:
-        dest_host, dest_port = destination.split(':')
-        dest_port = int(dest_port)
-    except ValueError:
-        warn(f"Invalid destination format: {destination}. Expected 'host:port'.")
-        writer.close()
-        await writer.wait_closed()
-        return
-    
-    try:
-        dest_reader, dest_writer = await asyncio.open_connection(dest_host, dest_port)
-    except Exception as e:
-        warn(f"Failed to connect to destination {destination}: {str(e)}")
-        writer.close()
-        await writer.wait_closed()
-        return
-
-    async def forward(src, dst):
-        try:
-            while True:
-                data = await src.read(8192)
-                if not data:
-                    break
-                dst.write(data)
-                await dst.drain()
-        except Exception as e:
-            warn(f"Error in forwarding: {str(e)}")
-        finally:
-            dst.close()
-            await dst.wait_closed()
-
-    await asyncio.gather(
-        forward(reader, dest_writer),
-        forward(dest_reader, writer)
-    )
-
-async def start_server(source: str, destination: str):
-    if ':' in source:
-        host, port = source.split(':')
-        port = int(port)
-    else:
-        host = source
-        port = 80
-
-    server = await asyncio.start_server(
-        lambda r, w: forward_traffic(r, w, destination),
-        host,
-        port
-    )
-
-    async with server:
-        await server.serve_forever()
-
-
-async def start_port_forwarding():
-    if hasattr(Serve, 'forwarding_rules'):
-        for rule in Serve.forwarding_rules:
-            asyncio.create_task(start_server(rule.source, rule.destination))
-    else:
-        warn("No forwarding rules found in the configuration.")
-
-
-@serve.get("/forward_status")
-async def get_forward_status():
-    if hasattr(Serve, 'forwarding_rules'):
-        return {"status": "active", "rules": Serve.forwarding_rules}
-    else:
-        return {"status": "inactive", "message": "No forwarding rules configured"}
-
-
-asyncio.create_task(start_port_forwarding())

sijapi/routers/signal.py

@@ -1,33 +0,0 @@
-"""
-Signal Bot example, repeats received messages.
-"""
-import os
-from fastapi import APIRouter
-from semaphore import Bot, ChatContext
-from sijapi import L
-
-signal = APIRouter()
-
-logger = L.get_module_logger("signal")
-
-def debug(text: str): logger.debug(text)
-def info(text: str): logger.info(text)
-def warn(text: str): logger.warning(text)
-def err(text: str): logger.error(text)
-def crit(text: str): logger.critical(text)
-
-async def echo(ctx: ChatContext) -> None:
-    if not ctx.message.empty():
-        await ctx.message.typing_started()
-        await ctx.message.reply(ctx.message.get_body())
-        await ctx.message.typing_stopped()
-
-async def main() -> None:
-    """Start the bot."""
-    async with Bot(os.environ["SIGNAL_PHONE_NUMBER"]) as bot:
-        bot.register_handler("", echo)
-        await bot.start()
-
-if __name__ == '__main__':
-    import anyio
-    anyio.run(main)

sijapi/routers/timing.py

@@ -1,6 +1,8 @@
 '''
 Uses the Timing.app API to get nicely formatted timeslip charts and spreadsheets.
 '''
+#routers/timing.py
+
 import tempfile
 import os
 import json
@@ -29,8 +31,8 @@ from sijapi import L, TIMING_API_KEY, TIMING_API_URL
 from sijapi.routers import gis
 
 
-time = APIRouter(tags=["private"])
-logger = L.get_module_logger("time")
+timing = APIRouter(tags=["private"])
+logger = L.get_module_logger("timing")
 def debug(text: str): logger.debug(text)
 def info(text: str): logger.info(text)
 def warn(text: str): logger.warning(text)
@@ -56,7 +58,7 @@ class TimingRequest(BaseModel):
 ####################
 #### TIMING API ####
 ####################
-@time.post("/time/post")
+@timing.post("/time/post")
 async def post_time_entry_to_timing(entry: Dict):
     url = 'https://web.timingapp.com/api/v1/time-entries'
     headers = {
@@ -195,7 +197,7 @@ def create_time_entry(original_entry, start_time, end_time, duration_seconds):
 
 
 # TIMELINE
-@time.get("/time/line")
+@timing.get("/time/line")
 async def get_timing_timeline(
     request: Request,
     start_date: str = Query(..., regex=r"\d{4}-\d{2}-\d{2}"),
@@ -251,7 +253,7 @@ def process_timeline(timing_data, queried_start_date, queried_end_date):
 
 
 # CSV
-@time.get("/time/csv")
+@timing.get("/time/csv")
 async def get_timing_csv(
     request: Request,
     start_date: str = Query(..., regex=r"\d{4}-\d{2}-\d{2}"),
@@ -311,7 +313,7 @@ def process_csv(timing_data, queried_start_date, queried_end_date):
     return output.getvalue()
 
 # MARKDOWN
-@time.get("/time/markdown3")
+@timing.get("/time/markdown3")
 async def get_timing_markdown3(
     request: Request,
     start_date: str = Query(..., regex=r"\d{4}-\d{2}-\d{2}"),
@@ -373,7 +375,7 @@ def process_timing_markdown3(timing_data, queried_start_date, queried_end_date):
     return "\n".join(markdown_output)
 
 
-@time.get("/time/markdown")
+@timing.get("/time/markdown")
 async def get_timing_markdown(
     request: Request,
     start: str = Query(..., regex=r"\d{4}-\d{2}-\d{2}"),
@@ -440,7 +442,7 @@ async def process_timing_markdown(start_date: datetime, end_date: datetime): # t
 
 
 #JSON
-@time.get("/time/json")
+@timing.get("/time/json")
 async def get_timing_json(
     request: Request,
     start_date: str = Query(..., regex=r"\d{4}-\d{2}-\d{2}"),
@@ -563,13 +565,13 @@ async def post_time_entry_to_timing(entry):
     return response.status_code, response.json()
 
 
-@time.get("/time/flagemoji/{country_code}")
+@timing.get("/time/flagemoji/{country_code}")
 def flag_emoji(country_code: str):
     offset = 127397
     flag = ''.join(chr(ord(char) + offset) for char in country_code.upper())
     return {"emoji": flag}
 
 
-@time.head("/time/")
+@timing.head("/time/")
 async def read_root():
     return {}

sijapi/routers/tts.py

@@ -1,6 +1,8 @@
 '''
 Uses xtts-v2 and/or the Elevenlabs API for text to speech.
 '''
+#routers/tts.py
+
 from fastapi import APIRouter, UploadFile, HTTPException, Response, Form, File, BackgroundTasks, Depends, Request
 from fastapi.responses import Response, StreamingResponse, FileResponse
 from fastapi.responses import StreamingResponse, PlainTextResponse

sijapi/routers/weather.py

@@ -1,6 +1,8 @@
 '''
 Uses the VisualCrossing API and Postgres/PostGIS to source local weather forecasts and history.
 '''
+#routers/weather.py
+
 import asyncio
 import traceback
 from fastapi import APIRouter, HTTPException, Query