Major update - database sync methods finally work reliably

This commit is contained in:
sanj 2024-08-02 11:15:52 -07:00
parent c5c426913e
commit 7e8f45fe64
28 changed files with 247 additions and 242 deletions

View file

@ -44,4 +44,4 @@ IN DRAFT
IN DRAFT
## Extras
[Apple Shortcut for companion Pythonista script](https://www.icloud.com/shortcuts/d63b179f9c664b0cbbacb1a607767ef7)
[Apple Shortcut for location tracking Pythonista script](https://www.icloud.com/shortcuts/d63b179f9c664b0cbbacb1a607767ef7)

View file

@ -0,0 +1,10 @@
default: xtts
email: xtts
webclip: 11L
rss: xtts
podcast_dir: '{{ DIR.HOME }}/Library/Mobile Documents/iCloud~co~supertop~castro/Documents/Sideloads'
xtts:
voice: joanne
elevenlabs:
voice: luna
api_key: '{{ SECRET.ELEVENLABS_API_KEY }}'

View file

@ -0,0 +1,28 @@
-----BEGIN PRIVATE KEY-----
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCikW67UW0RpncJ
h4Ha9HumZ/WzgEZWRWkgksVJIOJ8t1PftctizLUlz+xMWNl+Volp4crxnPpnczis
pOXU4g65XoFHHpF9nhF/3YDgxo5BDEM/mIIKEO9LFkIBQVBdE85qXnIVot5MfuNj
HeyEs7doRMBxilOSR/DkT8bTWu7m5yeHlF58iYVOxxssGhP3bo7CcAcaZD1LJBnP
Df+UBqzWQ9as903p5bFixHy2kVz8Qkd5k5tyIQ/tXqlhRfLLHG4AYHmBbS06CAg0
nEpKUeQx4l1J/ykAjQTwhHf70xv1z0p28mHcr5ib4UvpYK9fMM6FKWenwlqA3qrK
zQUJQ7E/AgMBAAECggEAQ5H/XIxzsSpnv+Y66y9DVd9QGNPwaFthXtCif8rTWNM6
YXnGl8JOaPELXpBvljuR0hivqc19pxIVNG01uk5boGDPiygBgRz6WRNQRh1Bc3gN
W5mgM17ml2cg+DSVmppo6X1oHeYcT99N1BzT+jRYv1YURx0fr2WHkt413hOlyQMR
b8ir/TOBx3olg4KBPDuysRC5BCIr3Mkz4jsh+9wVIOReKVezsy7nxJVzipcxOyZO
9VGgvlw4XLrRTOJEv4e3ldcg219j5KEGsJ4FFSziSmpj5fN4Vt+JmY7nueSHyL6P
3hX52lRfOcTXTEeiEV2cXkm3h8uQ3zfiZRYi3P0DQQKBgQDXGBZc3WnfXim0u1EV
JzZFwxBS7SHkyGgnd50ak6e9yDbdxOuYIOo4mBlc3ofd20EfT4TvR7Xyw+PD2fWJ
+isdwCEb9JZZ1H6RDGIzSDYXGNeGId4kMKBZdmKpEeLgStihsrYp/nxtwcE/8A7N
jCEKZj1ld7QfbQlGT/NJ4Jj80wKBgQDBfBpth6vMyCETKMJVqYd2qhVnJKiFLfRn
OD/Ck6xwUuedbfe9M34wNO3Pn2Xvu1xVsQGb2dmlT345Iq9Z1nbZCGXyY9yfLnTV
fz7F2utjUjaJtuiSb52SgX7MWZ8E4nbqqKnC4SYSIlaeuL9KK3r/x6bcNLAYPcdk
qKHexDkGZQKBgF0JGyshzhiChzGYUBMBOfVk0Ru9XAq0MHDZyQdk1Io/HpRAB9Nu
cUD3cQj9a/EnU/yyDYLeFrIhztO44/7BSYL9xpRr79h9FB2zKIqb8mF9KkPnREmN
Ct6HWVdd2C9B0H/oZ+i0HafvxaHdONnpgaUY4feQlkV9iSRzknzi++lnAoGAXOuu
/X80oMpUKBFhEyaxqemREdHnJN6nC5NV+6pUHDWUimSvn6vFJH2m4BlbKUC/3V9+
uExtXBjLM8FWmTyIIz8HRttyrvfuoEHV8ctrVG29R3ISS5FTCXMrZBR+bCgemB+c
N71NPVREaUGsjIBJN+G4XvTmxR2WTt81rfhqsokCgYEA1It9e9Ut2Krzf2FaPGLG
ozlKhWadMNntthg3uxze80Rx8WSvgJQdbVpdbni2B/9xdYBIIljW/LGivYBrCSSp
aXFpXL7ZGkvl3b0MkojfghIpXVGqu+8ISDtFgL0B1gZ5hq9xMBl94fLVfQgC9Cy6
uvDHlz+fjWaWKYUPiouAtVs=
-----END PRIVATE KEY-----

View file

@ -0,0 +1,19 @@
-----BEGIN CERTIFICATE-----
MIIDAzCCAeugAwIBAgIUc+EtilZslnS7N6MAx0u9HeP83wAwDQYJKoZIhvcNAQEL
BQAwETEPMA0GA1UEAwwGcHl0aG9uMB4XDTI0MDYwODEyNTcxM1oXDTI1MDYwODEy
NTcxM1owETEPMA0GA1UEAwwGcHl0aG9uMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
MIIBCgKCAQEAopFuu1FtEaZ3CYeB2vR7pmf1s4BGVkVpIJLFSSDifLdT37XLYsy1
Jc/sTFjZflaJaeHK8Zz6Z3M4rKTl1OIOuV6BRx6RfZ4Rf92A4MaOQQxDP5iCChDv
SxZCAUFQXRPOal5yFaLeTH7jYx3shLO3aETAcYpTkkfw5E/G01ru5ucnh5RefImF
TscbLBoT926OwnAHGmQ9SyQZzw3/lAas1kPWrPdN6eWxYsR8tpFc/EJHeZObciEP
7V6pYUXyyxxuAGB5gW0tOggINJxKSlHkMeJdSf8pAI0E8IR3+9Mb9c9KdvJh3K+Y
m+FL6WCvXzDOhSlnp8JagN6qys0FCUOxPwIDAQABo1MwUTAdBgNVHQ4EFgQUS74L
HD4Cdzh1ajatbvSHNQXIVvAwHwYDVR0jBBgwFoAUS74LHD4Cdzh1ajatbvSHNQXI
VvAwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAhpwtVubDjsyq
/LiTwpXKhjB/eFb6Yse782Iq+9rsiGGhsN88IA25fKgsJ2AIkR/KA7QSle3ds+1Q
EY9/vqpWnfBdpvOi7oV7ozBe+t/5JLu1GQBzg+cVa4iLAWYCiqg1d5NDdIcYMfsM
Yq2a3eQoP8Xbj3fFMXdNopXARa1d1zHB3ugXIJYinwMlS0EoGXVQVaHhemOh8GwW
keRaA6TDTBFsp0Gl4jv/NrisAt4qg+rlqr0mNcQK92vRX65mDWa/cQKwpUH8+Seq
Jl717NnsIGcqYWg8SSvVlkbFfxYhwYICXT824MAdSZtpHNCN/TegxsviYnlDyJKj
OJzn4fCxnQ==
-----END CERTIFICATE-----

View file

@ -0,0 +1 @@
{"token_type": "Bearer", "scope": "Calendars.Read Calendars.ReadWrite User.Read profile openid email", "expires_in": 3962, "ext_expires_in": 3962, "access_token": "eyJ0eXAiOiJKV1QiLCJub25jZSI6IldDeU91YXllN1RFX2FPM0F1alhlYmtvYTdVRHpUR1dVNWt5d3lJeDZ1MGciLCJhbGciOiJSUzI1NiIsIng1dCI6InE3UDFOdnh1R1F3RE4yVGFpTW92alo4YVp3cyIsImtpZCI6InE3UDFOdnh1R1F3RE4yVGFpTW92alo4YVp3cyJ9.eyJhdWQiOiIwMDAwMDAwMy0wMDAwLTAwMDAtYzAwMC0wMDAwMDAwMDAwMDAiLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC9iYWQ3ODA0OC1hNmUwLTQ3YjEtYTI0Yi00MDNjNDQ0YWEzNDkvIiwiaWF0IjoxNzE4Mzc0NzA5LCJuYmYiOjE3MTgzNzQ3MDksImV4cCI6MTcxODM3ODk3MiwiYWNjdCI6MCwiYWNyIjoiMSIsImFpbyI6IkFWUUFxLzhYQUFBQVRnWHJ0Q1pCVjlPa1M2WldldHVVSHNMSFN0LzErYVcxT1BSSjVOWjJEL1Bzd05mY1Fxb0JTNEFZRmhLR3UvaE5TNnNWOGtLQUpmcDNNTzdqRUlNMEZrY1VaZ0IyREh4cWdOK3lUQVBUYnRVPSIsImFtciI6WyJwd2QiLCJtZmEiXSwiYXBwX2Rpc3BsYXluYW1lIjoicHl0aG9uIiwiYXBwaWQiOiJjZThjYmQyNC1mMTQ2LTRkYzctOGVlNy01MWQ5YjY5ZGVjNTkiLCJhcHBpZGFjciI6IjEiLCJmYW1pbHlfbmFtZSI6IkluY2UtSm9oYW5uc2VuIiwiZ2l2ZW5fbmFtZSI6IlNhbmd5ZSIsImlkdHlwIjoidXNlciIsImlwYWRkciI6IjY4LjIzNS40NC4yMDIiLCJuYW1lIjoiU2FuZ3llIEluY2UtSm9oYW5uc2VuIiwib2lkIjoiMWNiMWQwNDAtZmM1OS00MjMxLTllMDUtOWRjNGI0MzJjY2MxIiwicGxhdGYiOiI1IiwicHVpZCI6IjEwMDMyMDAyQTNGQjU4RjIiLCJyaCI6IjAuQVgwQVNJRFh1dUNtc1VlaVMwQThSRXFqU1FNQUFBQUFBQUFBd0FBQUFBQUFBQUMxQUk4LiIsInNjcCI6IkNhbGVuZGFycy5SZWFkIENhbGVuZGFycy5SZWFkV3JpdGUgVXNlci5SZWFkIHByb2ZpbGUgb3BlbmlkIGVtYWlsIiwic2lnbmluX3N0YXRlIjpbImttc2kiXSwic3ViIjoiV0FYVFdIR0puVFhBTjlncmIyamlEU3U4ZENOMmc0dDFacERiVHlwM1k3USIsInRlbmFudF9yZWdpb25fc2NvcGUiOiJOQSIsInRpZCI6ImJhZDc4MDQ4LWE2ZTAtNDdiMS1hMjRiLTQwM2M0NDRhYTM0OSIsInVuaXF1ZV9uYW1lIjoic2FuZ3llaWpAd2VzdGVybmxhdy5vcmciLCJ1cG4iOiJzYW5neWVpakB3ZXN0ZXJubGF3Lm9yZyIsInV0aSI6InFHcVlEODRzaDBHMFBfSEdldlVXQUEiLCJ2ZXIiOiIxLjAiLCJ3aWRzIjpbImI3OWZiZjRkLTNlZjktNDY4OS04MTQzLTc2YjE5NGU4NTUwOSJdLCJ4bXNfaWRyZWwiOiIxIDIiLCJ4bXNfc3QiOnsic3ViIjoieXhjdzFhV1FiM2VrX0FvNFRuRy11SDN6ZndGbVRRUmMxVGpFaEdqZ2p2WSJ9LCJ4bXNfdGNkdCI6MTY0ODY4MTc1Mn0.ssgIrbYo1SPNusoB9bNIB7pLxCmwBKhox__KOnwRRtnE63vbfGWAl53ww1KpNWPdDfC3p94yuPybTRqjZnTPluv1oJgGINml4AleUnZJnJttRsFHvGflzKOLtXnzmhQGUBXxu7QucKTCMH4J36neeQAWthITMwCHbaGmSy0RLotaIsoEHIufxR9ZEYD4XP5e3sFX54eSnyf4P3GgHHC1y5xxWUlemG4G1BRas8i7oX9o-gqRFube6BMtCLir_HMTNPfrCG-lhd9msLhc6e_WJSmLMHQ7RVLo-GlTMY9UouE190GzBBVKUrTg462I3kP_GayO1kt6qopBrwnF6bDUsw", "refresh_token": "0.AX0ASIDXuuCmsUeiS0A8REqjSSS9jM5G8cdNjudR2bad7Fm1AI8.AgABAwEAAAApTwJmzXqdR4BN2miheQMYAgDs_wUA9P_rTWFRiXWkWxvihyyXonsZPLrulRvnKKRlZ9PxKUltEOQsxjlg86xvCYzAS6dYeDBQiQxRAS_WEuuXVmTqUWDVwqgwQOa3BCbLwxQhPwfG-O9uFY6D239Jo8rdXTrf8XOntGs6fCn3wuo5kvJr2D-FGRA_EepltvRxZgrWdHROKuqoL_ArjLDdoFP7zM95MKhVYTmCO7LCM7u6O9ItU4_6y2_lH864zUivT1LFG8-h9sx0Ln3wd8LBP3P5GSeXwtQlkbNpj1FNDl_Ex5SwGCTM7uDHj0dn5CdUMgLkOcAC__HJdzmlEryTquoXcjd1RAmkq1MqAGD7QQreI7NQTZXwTcjoMwiBg92-bk-_o2ajeIVqzgOVBQIu1W8gkN2F7PAqRc5lGB-2mAXchqKMoL31CLUPxgTMBjWgR4waAjfZXT4h2WqXAAdGFy2nzUJAjyEQa9ZW1J5B6asCf3cVJQwI6nWIN7OphrXkGHl0ffpfrC-skVG3N2vrelAutRvyvWi4bbMqAZNglRrkTn5G_kULmnyydZBcFSc5uPmKD7OkfBD5UpTa_KLTjYexWRVsBfG9czIVxOh3ojnnza9BjrN5cHwHhzPM1t67E5iqronvT2OR_r-4BerUfRNHXrxwrLvDUEZwQ8o5IRs2N5FH0y_QN049o_NTgqytCj6wrIB4T-ZBUK2AsFej7ipdHAMYtWLZdoAo1o4nMuPBb4syN0VYd1sLUP-RQ5iv7wIkMWmNjhjIErIktZ134pGK9TlWa904H6HUin0qNTXyTmX2feE0nBlm6xJbO1ISfFkaf8aEjcAMfeu9qiArKQqUgvY", "expires_at": 1718378971}

View file

@ -1,57 +0,0 @@
# cli.py
import click
import asyncio
from datetime import datetime as dt_datetime, timedelta
# Import your async functions and dependencies
from sijapi import build_daily_note_range_endpoint, gis # broken!
def async_command(f):
@click.command()
@click.pass_context
def wrapper(ctx, *args, **kwargs):
async def run():
return await f(*args, **kwargs)
return asyncio.run(run())
return wrapper
@click.group()
def cli():
"""CLI for your application."""
pass
@cli.command()
@click.argument('dt_start')
@click.argument('dt_end')
@async_command
async def bulk_note_range(dt_start: str, dt_end: str):
"""
Build daily notes for a date range.
DT_START and DT_END should be in YYYY-MM-DD format.
"""
try:
start_date = dt_datetime.strptime(dt_start, "%Y-%m-%d")
end_date = dt_datetime.strptime(dt_end, "%Y-%m-%d")
except ValueError:
click.echo("Error: Dates must be in YYYY-MM-DD format.")
return
if start_date > end_date:
click.echo("Error: Start date must be before or equal to end date.")
return
results = []
current_date = start_date
while current_date <= end_date:
formatted_date = await gis.dt(current_date)
result = await build_daily_note(formatted_date)
results.append(result)
current_date += timedelta(days=1)
click.echo("Generated notes for the following dates:")
for url in results:
click.echo(url)
if __name__ == '__main__':
cli()

View file

@ -2,6 +2,7 @@
Uses whisper_cpp to create an OpenAI-compatible Whisper web service.
'''
# routers/asr.py
import os
import uuid
import json

View file

@ -3,6 +3,8 @@ Calendar module using macOS Calendars and/or Microsoft 365 via its Graph API.
Depends on:
LOGGER, ICAL_TOGGLE, ICALENDARS, MS365_TOGGLE, MS365_CLIENT_ID, MS365_SECRET, MS365_AUTHORITY_URL, MS365_SCOPE, MS365_REDIRECT_PATH, MS365_TOKEN_PATH
'''
#routers/cal.py
from fastapi import APIRouter, Depends, HTTPException, status, Request
from fastapi.responses import RedirectResponse, JSONResponse
from fastapi.security import OAuth2PasswordBearer

View file

@ -1,6 +1,8 @@
'''
IN DEVELOPMENT - Cloudflare + Caddy module. Based on a bash script that's able to rapidly deploy new Cloudflare subdomains on new Caddy reverse proxy configurations, managing everything including restarting Caddy. The Python version needs more testing before actual use.
'''
#routers/cf.py
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from fastapi.responses import PlainTextResponse, JSONResponse

View file

@ -1,65 +0,0 @@
'''
WORK IN PROGRESS: This module will handle tasks related to multi-server environments.
'''
from fastapi import APIRouter, HTTPException
import asyncio
import logging
from sijapi.utilities import run_ssh_command
from sijapi import L, REBOOT_SCRIPT_PATH, HOST_CONFIG, API_CONFIG
dist = APIRouter()
logger = L.get_module_logger("dist")
def debug(text: str): logger.debug(text)
def info(text: str): logger.info(text)
def warn(text: str): logger.warning(text)
def err(text: str): logger.error(text)
def crit(text: str): logger.critical(text)
@dist.get("/update-restart-others")
async def update_and_restart_others():
results = []
for server in API_CONFIG.servers:
if HOST_CONFIG.id != server.id:
try:
output, error = await run_ssh_command(server, f"bash {server.scripts.update_and_restart}")
results.append({"server": server.id, "status": "success", "output": output, "error": error})
except Exception as e:
results.append({"server": server.id, "status": "failed", "error": str(e)})
return {"message": "Update and restart process initiated for other servers.", "results": results}
@dist.get("/update-restart-self")
async def update_and_restart_self(safe: bool = True):
if safe and not await ensure_redundancy():
raise HTTPException(status_code=400, detail="Cannot safely restart: no redundancy available")
try:
process = await asyncio.create_subprocess_exec(
"bash", API_CONFIG.servers[HOST_CONFIG.id].scripts.update_and_restart,
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
)
stdout, stderr = await process.communicate()
info(f"Update and restart initiated for self. Stdout: {stdout.decode()}. Stderr: {stderr.decode()}")
return {"message": "Update and restart process initiated for this server."}
except Exception as e:
err(f"Failed to initiate update and restart for self: {str(e)}")
raise HTTPException(status_code=500, detail=f"Failed to initiate update and restart: {str(e)}")
@dist.get("/update-and-restart-all")
async def update_and_restart_all():
others_result = await update_and_restart_others()
self_result = await update_and_restart_self(safe=False)
return {"others": others_result, "self": self_result}
async def ensure_redundancy():
import aiohttp
redundancy = False
async with aiohttp.ClientSession() as session:
for server in API_CONFIG.servers:
if server.id != HOST_CONFIG.id:
try:
async with session.get(f"{server.protocol}://{server.ip}:{server.port}/health") as response:
if response.status // 100 == 2:
redundancy = True
break
except aiohttp.ClientError:
warn(f"Failed to check health of server {server.id}")
return redundancy

View file

@ -1,6 +1,8 @@
'''
Uses IMAP and SMTP login credentials to monitor an inbox and summarize incoming emails that match certain criteria and save the Text-To-Speech converted summaries into a specified "podcast" folder.
'''
#routers/email.py
from fastapi import APIRouter
import asyncio
import aiofiles

124
sijapi/routers/forward.py Normal file
View file

@ -0,0 +1,124 @@
'''
Used to port-forwarding and reverse proxy configurations.
'''
#routers/forward.py
import os
import io
import string
import json
import time
import base64
import asyncpg
import asyncio
import subprocess
import requests
import random
import paramiko
import aiohttp
import httpx
from datetime import datetime
from hashlib import sha256
from pathlib import Path
from typing import List, Optional
from pydantic import BaseModel
from PyPDF2 import PdfReader
from fastapi import APIRouter, Form, HTTPException, Request, Response, BackgroundTasks, status, Path as PathParam
from fastapi.responses import HTMLResponse, FileResponse, PlainTextResponse, JSONResponse, RedirectResponse
from fastapi.templating import Jinja2Templates
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from sijapi import (
L, API, Serve, LOGS_DIR, TS_ID, CASETABLE_PATH, COURTLISTENER_DOCKETS_URL, COURTLISTENER_API_KEY,
COURTLISTENER_BASE_URL, COURTLISTENER_DOCKETS_DIR, COURTLISTENER_SEARCH_DIR, ALERTS_DIR,
MAC_UN, MAC_PW, MAC_ID, TS_TAILNET, IMG_DIR, PUBLIC_KEY, OBSIDIAN_VAULT_DIR
)
from sijapi.classes import WidgetUpdate
from sijapi.utilities import bool_convert, sanitize_filename, assemble_journal_path
from sijapi.routers import gis
forward = APIRouter()
logger = L.get_module_logger("email")
def debug(text: str): logger.debug(text)
def info(text: str): logger.info(text)
def warn(text: str): logger.warning(text)
def err(text: str): logger.error(text)
def crit(text: str): logger.critical(text)
async def forward_traffic(reader: asyncio.StreamReader, writer: asyncio.StreamWriter, destination: str):
try:
dest_host, dest_port = destination.split(':')
dest_port = int(dest_port)
except ValueError:
warn(f"Invalid destination format: {destination}. Expected 'host:port'.")
writer.close()
await writer.wait_closed()
return
try:
dest_reader, dest_writer = await asyncio.open_connection(dest_host, dest_port)
except Exception as e:
warn(f"Failed to connect to destination {destination}: {str(e)}")
writer.close()
await writer.wait_closed()
return
async def forward(src, dst):
try:
while True:
data = await src.read(8192)
if not data:
break
dst.write(data)
await dst.drain()
except Exception as e:
warn(f"Error in forwarding: {str(e)}")
finally:
dst.close()
await dst.wait_closed()
await asyncio.gather(
forward(reader, dest_writer),
forward(dest_reader, writer)
)
async def start_server(source: str, destination: str):
if ':' in source:
host, port = source.split(':')
port = int(port)
else:
host = source
port = 80
server = await asyncio.start_server(
lambda r, w: forward_traffic(r, w, destination),
host,
port
)
async with server:
await server.serve_forever()
async def start_port_forwarding():
if hasattr(Serve, 'forwarding_rules'):
for rule in Serve.forwarding_rules:
asyncio.create_task(start_server(rule.source, rule.destination))
else:
warn("No forwarding rules found in the configuration.")
@forward.get("/forward_status")
async def get_forward_status():
if hasattr(Serve, 'forwarding_rules'):
return {"status": "active", "rules": Serve.forwarding_rules}
else:
return {"status": "inactive", "message": "No forwarding rules configured"}
asyncio.create_task(start_port_forwarding())

View file

@ -1,3 +1,19 @@
'''
WIP. Will be used to manage and update Ghost blogs.
'''
#routers/ghost.py
from fastapi import APIRouter
from datetime import date
import os
import requests
import json
import yaml
import jwt
from sijapi import GHOST_API_KEY, GHOST_API_URL
ghost = APIRouter()
def generate_jwt_token():
key_id, key_secret = GHOST_API_KEY.split(':')
iat = int(date.now().timestamp())

View file

@ -1,6 +1,8 @@
'''
Uses Postgres/PostGIS for location tracking (data obtained via the companion mobile Pythonista scripts), and for geocoding purposes.
'''
#routers/gis.py
from fastapi import APIRouter, HTTPException, Query
from fastapi.responses import HTMLResponse, JSONResponse
import random

View file

@ -3,6 +3,8 @@ Health check module. /health returns `'status': 'ok'`, /id returns TS_ID, /route
Depends on:
TS_ID, LOGGER, SUBNET_BROADCAST
'''
#routers/health.py
import os
import httpx
import socket

View file

@ -1,6 +1,8 @@
'''
IN DEVELOPMENT: Instagram AI bot module.
'''
#routers/ig.py
from fastapi import APIRouter, UploadFile
import os
import io

View file

@ -2,9 +2,10 @@
Image generation module using StableDiffusion and similar models by way of ComfyUI.
DEPENDS ON:
LLM module
COMFYUI_URL, COMFYUI_DIR, COMFYUI_OUTPUT_DIR, TS_SUBNET, TS_ADDRESS, DATA_DIR, IMG_CONFIG_DIR, IMG_DIR, IMG_WORKFLOWS_DIR, LOCAL_HOSTS, API.URL, PHOTOPRISM_USER*, PHOTOPRISM_URL*, PHOTOPRISM_PASS*
COMFYUI_URL, COMFYUI_DIR, COMFYUI_OUTPUT_DIR, TS_SUBNET, TS_ADDRESS, DATA_DIR, IMG_CONFIG_DIR, IMG_DIR, IMG_WORKFLOWS_DIR, LOCAL_HOSTS
*unimplemented.
'''
#routers/img.py
from fastapi import APIRouter, Request, Query
from fastapi.responses import JSONResponse, RedirectResponse

View file

@ -2,6 +2,7 @@
Interfaces with Ollama and creates an OpenAI-compatible relay API.
'''
# routers/llm.py
from fastapi import APIRouter, HTTPException, Request, Response, BackgroundTasks, File, Form, UploadFile
from fastapi.responses import StreamingResponse, JSONResponse, FileResponse
from datetime import datetime as dt_datetime

View file

@ -1,4 +1,8 @@
'''
Used to scrape, process, summarize, markdownify, and speechify news articles.
'''
# routers/news.py
import os
import uuid
import asyncio

View file

@ -2,6 +2,7 @@
Manages an Obsidian vault, in particular daily notes, using information and functionality drawn from the other routers, primarily calendar, email, ig, llm, rag, img, serve, time, tts, and weather.
'''
# routers/note.py
from fastapi import APIRouter, BackgroundTasks, File, UploadFile, Form, HTTPException, Response, Query, Path as FastAPIPath
from fastapi.responses import JSONResponse, PlainTextResponse
import os, re
@ -17,7 +18,7 @@ from fastapi import HTTPException, status
from pathlib import Path
from fastapi import APIRouter, Query, HTTPException
from sijapi import API, L, OBSIDIAN_VAULT_DIR, OBSIDIAN_RESOURCES_DIR, OBSIDIAN_BANNER_SCENE, DEFAULT_11L_VOICE, DEFAULT_VOICE, GEO
from sijapi.routers import asr, cal, gis, img, llm, serve, time, tts, weather
from sijapi.routers import asr, cal, gis, img, llm, serve, timing, tts, weather
from sijapi.utilities import assemble_journal_path, convert_to_12_hour_format, sanitize_filename, convert_degrees_to_cardinal, check_file_name, HOURLY_COLUMNS_MAPPING
from sijapi.classes import Location
@ -388,7 +389,7 @@ async def build_daily_timeslips(date):
'''
absolute_path, relative_path = assemble_journal_path(date, filename = "Timeslips", extension=".md", no_timestamp = True)
content = await time.process_timing_markdown(date, date)
content = await timing.process_timing_markdown(date, date)
# document_content = await document.read()
with open(absolute_path, 'wb') as f:
f.write(content.encode())

View file

@ -2,6 +2,7 @@
IN DEVELOPMENT: Retrieval-Augmented Generation module.
NOTES: Haven't yet decided if this should depend on the Obsidian and Chat modules, or if they should depend on it, or one of one the other the other.
'''
#routers/rag.py
from fastapi import APIRouter
from sijapi import L

View file

@ -1,3 +1,8 @@
'''
Used to scrape data from websites, watch sites for changes, and make alerts based on specific changes.
'''
#routers/scrape.py
import asyncio
import json
import re

View file

@ -1,6 +1,8 @@
'''
Web server module. Used by other modules when serving static content is required, e.g. the img image generation module. Also used to serve PUBLIC_KEY.
'''
#routers/serve.py
import os
import io
import string
@ -512,75 +514,3 @@ async def get_analytics(short_code: str):
}
async def forward_traffic(reader: asyncio.StreamReader, writer: asyncio.StreamWriter, destination: str):
try:
dest_host, dest_port = destination.split(':')
dest_port = int(dest_port)
except ValueError:
warn(f"Invalid destination format: {destination}. Expected 'host:port'.")
writer.close()
await writer.wait_closed()
return
try:
dest_reader, dest_writer = await asyncio.open_connection(dest_host, dest_port)
except Exception as e:
warn(f"Failed to connect to destination {destination}: {str(e)}")
writer.close()
await writer.wait_closed()
return
async def forward(src, dst):
try:
while True:
data = await src.read(8192)
if not data:
break
dst.write(data)
await dst.drain()
except Exception as e:
warn(f"Error in forwarding: {str(e)}")
finally:
dst.close()
await dst.wait_closed()
await asyncio.gather(
forward(reader, dest_writer),
forward(dest_reader, writer)
)
async def start_server(source: str, destination: str):
if ':' in source:
host, port = source.split(':')
port = int(port)
else:
host = source
port = 80
server = await asyncio.start_server(
lambda r, w: forward_traffic(r, w, destination),
host,
port
)
async with server:
await server.serve_forever()
async def start_port_forwarding():
if hasattr(Serve, 'forwarding_rules'):
for rule in Serve.forwarding_rules:
asyncio.create_task(start_server(rule.source, rule.destination))
else:
warn("No forwarding rules found in the configuration.")
@serve.get("/forward_status")
async def get_forward_status():
if hasattr(Serve, 'forwarding_rules'):
return {"status": "active", "rules": Serve.forwarding_rules}
else:
return {"status": "inactive", "message": "No forwarding rules configured"}
asyncio.create_task(start_port_forwarding())

View file

@ -1,33 +0,0 @@
"""
Signal Bot example, repeats received messages.
"""
import os
from fastapi import APIRouter
from semaphore import Bot, ChatContext
from sijapi import L
signal = APIRouter()
logger = L.get_module_logger("signal")
def debug(text: str): logger.debug(text)
def info(text: str): logger.info(text)
def warn(text: str): logger.warning(text)
def err(text: str): logger.error(text)
def crit(text: str): logger.critical(text)
async def echo(ctx: ChatContext) -> None:
if not ctx.message.empty():
await ctx.message.typing_started()
await ctx.message.reply(ctx.message.get_body())
await ctx.message.typing_stopped()
async def main() -> None:
"""Start the bot."""
async with Bot(os.environ["SIGNAL_PHONE_NUMBER"]) as bot:
bot.register_handler("", echo)
await bot.start()
if __name__ == '__main__':
import anyio
anyio.run(main)

View file

@ -1,6 +1,8 @@
'''
Uses the Timing.app API to get nicely formatted timeslip charts and spreadsheets.
'''
#routers/timing.py
import tempfile
import os
import json
@ -29,8 +31,8 @@ from sijapi import L, TIMING_API_KEY, TIMING_API_URL
from sijapi.routers import gis
time = APIRouter(tags=["private"])
logger = L.get_module_logger("time")
timing = APIRouter(tags=["private"])
logger = L.get_module_logger("timing")
def debug(text: str): logger.debug(text)
def info(text: str): logger.info(text)
def warn(text: str): logger.warning(text)
@ -56,7 +58,7 @@ class TimingRequest(BaseModel):
####################
#### TIMING API ####
####################
@time.post("/time/post")
@timing.post("/time/post")
async def post_time_entry_to_timing(entry: Dict):
url = 'https://web.timingapp.com/api/v1/time-entries'
headers = {
@ -195,7 +197,7 @@ def create_time_entry(original_entry, start_time, end_time, duration_seconds):
# TIMELINE
@time.get("/time/line")
@timing.get("/time/line")
async def get_timing_timeline(
request: Request,
start_date: str = Query(..., regex=r"\d{4}-\d{2}-\d{2}"),
@ -251,7 +253,7 @@ def process_timeline(timing_data, queried_start_date, queried_end_date):
# CSV
@time.get("/time/csv")
@timing.get("/time/csv")
async def get_timing_csv(
request: Request,
start_date: str = Query(..., regex=r"\d{4}-\d{2}-\d{2}"),
@ -311,7 +313,7 @@ def process_csv(timing_data, queried_start_date, queried_end_date):
return output.getvalue()
# MARKDOWN
@time.get("/time/markdown3")
@timing.get("/time/markdown3")
async def get_timing_markdown3(
request: Request,
start_date: str = Query(..., regex=r"\d{4}-\d{2}-\d{2}"),
@ -373,7 +375,7 @@ def process_timing_markdown3(timing_data, queried_start_date, queried_end_date):
return "\n".join(markdown_output)
@time.get("/time/markdown")
@timing.get("/time/markdown")
async def get_timing_markdown(
request: Request,
start: str = Query(..., regex=r"\d{4}-\d{2}-\d{2}"),
@ -440,7 +442,7 @@ async def process_timing_markdown(start_date: datetime, end_date: datetime): # t
#JSON
@time.get("/time/json")
@timing.get("/time/json")
async def get_timing_json(
request: Request,
start_date: str = Query(..., regex=r"\d{4}-\d{2}-\d{2}"),
@ -563,13 +565,13 @@ async def post_time_entry_to_timing(entry):
return response.status_code, response.json()
@time.get("/time/flagemoji/{country_code}")
@timing.get("/time/flagemoji/{country_code}")
def flag_emoji(country_code: str):
offset = 127397
flag = ''.join(chr(ord(char) + offset) for char in country_code.upper())
return {"emoji": flag}
@time.head("/time/")
@timing.head("/time/")
async def read_root():
return {}

View file

@ -1,6 +1,8 @@
'''
Uses xtts-v2 and/or the Elevenlabs API for text to speech.
'''
#routers/tts.py
from fastapi import APIRouter, UploadFile, HTTPException, Response, Form, File, BackgroundTasks, Depends, Request
from fastapi.responses import Response, StreamingResponse, FileResponse
from fastapi.responses import StreamingResponse, PlainTextResponse

View file

@ -1,6 +1,8 @@
'''
Uses the VisualCrossing API and Postgres/PostGIS to source local weather forecasts and history.
'''
#routers/weather.py
import asyncio
import traceback
from fastapi import APIRouter, HTTPException, Query