sij/sijapi
1
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Auto-update: Sat Jun 29 13:04:54 PDT 2024

Browse source
This commit is contained in:
sanj 2024-06-29 13:04:54 -07:00
parent f019d4ede0
commit 727d89b749
12 changed files with 151 additions and 103 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
sijapi/__init__.py
View file

@ -17,30 +17,29 @@ BASE_DIR = Path(__file__).resolve().parent
CONFIG_DIR = BASE_DIR / "config"
ENV_PATH = CONFIG_DIR / ".env"
LOGS_DIR = BASE_DIR / "logs"
L = Logger("Central", LOGS_DIR)
os.makedirs(LOGS_DIR, exist_ok=True)
load_dotenv(ENV_PATH)
### API essentials
API_CONFIG_PATH = CONFIG_DIR / "api.yaml"
SECRETS_CONFIG_PATH = CONFIG_DIR / "secrets.yaml"
API = APIConfig.load_from_yaml(API_CONFIG_PATH, SECRETS_CONFIG_PATH)
SECRETS_PATH = CONFIG_DIR / "secrets.yaml"
API = APIConfig.load(API_CONFIG_PATH, SECRETS_PATH)
DIR_CONFIG_PATH = CONFIG_DIR / "dirs.yaml"
L.DEBUG(f"Loading DIR configuration from: {DIR_CONFIG_PATH}")
DIR = Configuration.load(DIR_CONFIG_PATH)
L.DEBUG(f"Loaded DIR configuration: {DIR.__dict__}")
DB = Database.from_env()
ROUTERS = os.getenv('ROUTERS', '').split(',')
PUBLIC_SERVICES = os.getenv('PUBLIC_SERVICES', '').split(',')
GLOBAL_API_KEY = os.getenv("GLOBAL_API_KEY")
# HOST_NET and HOST_PORT comprise HOST, which is what the server will bind to
HOST_NET = os.getenv("HOST_NET", "127.0.0.1")
HOST_PORT = int(os.getenv("HOST_PORT", 4444))
HOST = f"{HOST_NET}:{HOST_PORT}"
BASE_URL = os.getenv("BASE_URL", f"http://{HOST}")
HOST = f"{API.BIND}:{API.PORT}"
LOCAL_HOSTS = [ipaddress.ip_address(localhost.strip()) for localhost in os.getenv('LOCAL_HOSTS', '127.0.0.1').split(',')] + ['localhost']
SUBNET_BROADCAST = os.getenv("SUBNET_BROADCAST", '10.255.255.255')
TRUSTED_SUBNETS = [ipaddress.ip_network(subnet.strip()) for subnet in os.getenv('TRUSTED_SUBNETS', '127.0.0.1/32').split(',')]
MAX_CPU_CORES = min(int(os.getenv("MAX_CPU_CORES", int(multiprocessing.cpu_count()/2))), multiprocessing.cpu_count())
### Directories & general paths
HOME_DIR = Path.home()
ROUTER_DIR = BASE_DIR / "routers"
DATA_DIR = BASE_DIR / "data"
os.makedirs(DATA_DIR, exist_ok=True)
@ -50,7 +49,6 @@ REQUESTS_DIR = LOGS_DIR / "requests"
os.makedirs(REQUESTS_DIR, exist_ok=True)
REQUESTS_LOG_PATH = LOGS_DIR / "requests.log"
### LOCATE AND WEATHER LOCALIZATIONS
USER_FULLNAME = os.getenv('USER_FULLNAME')
USER_BIO = os.getenv('USER_BIO')
@ -68,7 +66,7 @@ GEO = Geocoder(NAMED_LOCATIONS, TZ_CACHE)
### Obsidian & notes
ALLOWED_FILENAME_CHARS = r'[^\w \.-]'
MAX_PATH_LENGTH = 254
OBSIDIAN_VAULT_DIR = Path(os.getenv("OBSIDIAN_BASE_DIR") or HOME_DIR / "Nextcloud" / "notes")
OBSIDIAN_VAULT_DIR = Path(os.getenv("OBSIDIAN_BASE_DIR") or Path(DIR.HOME) / "Nextcloud" / "notes")
OBSIDIAN_JOURNAL_DIR = OBSIDIAN_VAULT_DIR / "journal"
OBSIDIAN_RESOURCES_DIR = "obsidian/resources"
OBSIDIAN_BANNER_DIR = f"{OBSIDIAN_RESOURCES_DIR}/banners"
@ -122,7 +120,7 @@ SD_CONFIG_PATH = CONFIG_DIR / 'sd.yaml'
### ASR
ASR_DIR = DATA_DIR / "asr"
os.makedirs(ASR_DIR, exist_ok=True)
WHISPER_CPP_DIR = HOME_DIR / str(os.getenv("WHISPER_CPP_DIR"))
WHISPER_CPP_DIR = Path(DIR.HOME) / str(os.getenv("WHISPER_CPP_DIR"))
WHISPER_CPP_MODELS = os.getenv('WHISPER_CPP_MODELS', 'NULL,VOID').split(',')
### TTS
@ -185,7 +183,7 @@ CF_IP = DATA_DIR / "cf_ip.txt" # to be deprecated soon
CF_DOMAINS_PATH = DATA_DIR / "cf_domains.json" # to be deprecated soon
### Caddy - not fully implemented
BASE_URL = os.getenv("BASE_URL")
API.URL = os.getenv("API.URL")
CADDY_SERVER = os.getenv('CADDY_SERVER', None)
CADDYFILE_PATH = os.getenv("CADDYFILE_PATH", "") if CADDY_SERVER is not None else None
CADDY_API_KEY = os.getenv("CADDY_API_KEY")

13
sijapi/__main__.py
View file

@ -29,9 +29,7 @@ args = parser.parse_args()
from sijapi import L
L.setup_from_args(args)
from sijapi import HOST, ENV_PATH, GLOBAL_API_KEY, REQUESTS_DIR, ROUTER_DIR, REQUESTS_LOG_PATH, PUBLIC_SERVICES, TRUSTED_SUBNETS, ROUTERS
from sijapi import ROUTER_DIR
# Initialize a FastAPI application
api = FastAPI()
@ -52,13 +50,14 @@ class SimpleAPIKeyMiddleware(BaseHTTPMiddleware):
if request.method == "OPTIONS":
# Allow CORS preflight requests
return JSONResponse(status_code=200)
if request.url.path not in PUBLIC_SERVICES:
if not any(client_ip in subnet for subnet in TRUSTED_SUBNETS):
if request.url.path not in API.PUBLIC:
trusted_subnets = [ipaddress.ip_network(subnet) for subnet in API.TRUSTED_SUBNETS]
if not any(client_ip in subnet for subnet in trusted_subnets):
api_key_header = request.headers.get("Authorization")
api_key_query = request.query_params.get("api_key")
if api_key_header:
api_key_header = api_key_header.lower().split("bearer ")[-1]
if api_key_header != GLOBAL_API_KEY and api_key_query != GLOBAL_API_KEY:
if api_key_header not in API.KEYS and api_key_query not in API.KEYS:
L.ERR(f"Invalid API key provided by a requester.")
return JSONResponse(
status_code=401,
@ -68,8 +67,10 @@ class SimpleAPIKeyMiddleware(BaseHTTPMiddleware):
# L.DEBUG(f"Request from {client_ip} is complete")
return response
# Add the middleware to your FastAPI app
api.add_middleware(SimpleAPIKeyMiddleware)
canceled_middleware = """
@api.middleware("http")
async def log_requests(request: Request, call_next):

135
sijapi/classes.py
View file

@ -5,6 +5,13 @@ import asyncio
import json
import os
import re
from pathlib import Path
from typing import Union, Dict, Any, Optional
from pydantic import BaseModel, create_model
import yaml
from dotenv import load_dotenv
import os
import re
import yaml
import math
from timezonefinder import TimezoneFinder
@ -31,21 +38,10 @@ from dotenv import load_dotenv
T = TypeVar('T', bound='Configuration')
class ModulesConfig(BaseModel):
asr: bool = Field(alias="asr")
calendar: bool = Field(alias="calendar")
email: bool = Field(alias="email")
health: bool = Field(alias="health")
hooks: bool = Field(alias="hooks")
llm: bool = Field(alias="llm")
locate: bool = Field(alias="locate")
note: bool = Field(alias="note")
sd: bool = Field(alias="sd")
serve: bool = Field(alias="serve")
time: bool = Field(alias="time")
tts: bool = Field(alias="tts")
weather: bool = Field(alias="weather")
from pydantic import BaseModel, Field, create_model
from typing import List, Optional, Any, Dict
from pathlib import Path
import yaml
class APIConfig(BaseModel):
BIND: str
@ -53,12 +49,12 @@ class APIConfig(BaseModel):
URL: str
PUBLIC: List[str]
TRUSTED_SUBNETS: List[str]
MODULES: ModulesConfig
MODULES: Any # This will be replaced with a dynamic model
BaseTZ: Optional[str] = 'UTC'
KEYS: List[str]
@classmethod
def load_from_yaml(cls, config_path: Path, secrets_path: Path):
def load(cls, config_path: Path, secrets_path: Path):
# Load main configuration
with open(config_path, 'r') as file:
config_data = yaml.safe_load(file)
@ -93,66 +89,94 @@ class APIConfig(BaseModel):
else:
print(f"Invalid secret placeholder format: {placeholder}")
# Convert 'on'/'off' to boolean for MODULES if they are strings
for key, value in config_data['MODULES'].items():
# Create dynamic ModulesConfig
modules_data = config_data.get('MODULES', {})
modules_fields = {}
for key, value in modules_data.items():
if isinstance(value, str):
config_data['MODULES'][key] = value.lower() == 'on'
modules_fields[key] = (bool, value.lower() == 'on')
elif isinstance(value, bool):
config_data['MODULES'][key] = value
modules_fields[key] = (bool, value)
else:
raise ValueError(f"Invalid value for module {key}: {value}. Must be 'on', 'off', True, or False.")
DynamicModulesConfig = create_model('DynamicModulesConfig', **modules_fields)
config_data['MODULES'] = DynamicModulesConfig(**modules_data)
return cls(**config_data)
def __getattr__(self, name: str) -> Any:
if name == 'MODULES':
return self.__dict__['MODULES']
return super().__getattr__(name)
@property
def active_modules(self) -> List[str]:
return [module for module, is_active in self.MODULES.__dict__.items() if is_active]
class Configuration(BaseModel):
@classmethod
def load_config(cls: Type[T], yaml_path: Union[str, Path]) -> Union[T, List[T]]:
yaml_path = Path(yaml_path)
with yaml_path.open('r') as file:
config_data = yaml.safe_load(file)
# Load environment variables
load_dotenv()
# Resolve placeholders
config_data = cls.resolve_placeholders(config_data)
if isinstance(config_data, list):
return [cls.create_dynamic_model(**cfg) for cfg in config_data]
elif isinstance(config_data, dict):
return cls.create_dynamic_model(**config_data)
else:
raise ValueError(f"Unsupported YAML structure in {yaml_path}")
HOME: Path = Path.home()
_dir_config: Optional['Configuration'] = None
@classmethod
def resolve_placeholders(cls, data):
def load(cls, yaml_path: Union[str, Path], dir_config: Optional['Configuration'] = None) -> 'Configuration':
yaml_path = Path(yaml_path)
try:
with yaml_path.open('r') as file:
config_data = yaml.safe_load(file)
print(f"Loaded configuration data: {config_data}")
# Ensure HOME is set
if config_data.get('HOME') is None:
config_data['HOME'] = str(Path.home())
print(f"HOME was None in config, set to default: {config_data['HOME']}")
load_dotenv()
instance = cls.create_dynamic_model(**config_data)
instance._dir_config = dir_config or instance
resolved_data = instance.resolve_placeholders(config_data)
for key, value in resolved_data.items():
setattr(instance, key, value)
return instance
except Exception as e:
print(f"Error loading configuration from {yaml_path}: {str(e)}")
raise
def resolve_placeholders(self, data: Any) -> Any:
if isinstance(data, dict):
return {k: cls.resolve_placeholders(v) for k, v in data.items()}
return {k: self.resolve_placeholders(v) for k, v in data.items()}
elif isinstance(data, list):
return [cls.resolve_placeholders(v) for v in data]
return [self.resolve_placeholders(v) for v in data]
elif isinstance(data, str):
return cls.resolve_string_placeholders(data)
return self.resolve_string_placeholders(data)
else:
return data
@classmethod
def resolve_string_placeholders(cls, value):
def resolve_string_placeholders(self, value: str) -> Any:
pattern = r'\{\{\s*([^}]+)\s*\}\}'
matches = re.findall(pattern, value)
for match in matches:
parts = match.split('.')
if len(parts) == 2:
category, key = parts
if category == 'DIR':
replacement = str(Path(os.getenv(key, '')))
elif category == 'SECRET':
replacement = os.getenv(key, '')
else:
replacement = os.getenv(match, '')
value = value.replace('{{' + match + '}}', replacement)
if len(parts) == 1: # Internal reference
replacement = getattr(self._dir_config, parts[0], str(Path.home() / parts[0].lower()))
elif len(parts) == 2 and parts[0] == 'DIR':
replacement = getattr(self._dir_config, parts[1], str(Path.home() / parts[1].lower()))
elif len(parts) == 2 and parts[0] == 'ENV':
replacement = os.getenv(parts[1], '')
else:
replacement = value # Keep original if not recognized
value = value.replace('{{' + match + '}}', str(replacement))
# Convert to Path if it looks like a file path
if isinstance(value, str) and (value.startswith(('/', '~')) or (':' in value and value[1] == ':')):
return Path(value).expanduser()
return value
@classmethod
@ -172,6 +196,7 @@ class Configuration(BaseModel):
extra = "allow"
arbitrary_types_allowed = True
class Location(BaseModel):
latitude: float
longitude: float

8
sijapi/config/.env-example
View file

@ -52,14 +52,12 @@
# ─────────────────────────────────────────────────────────────────
#
#─── first, bind an ip address and port : ──────────────────────────────────────────
HOST_NET=0.0.0.0
HOST_PORT=4444
BASE_URL=http://localhost:4444 # <--- replace with base URL of reverse proxy, etc
# <--- replace with base URL of reverse proxy, etc
#─── notes: ──────────────────────────────────────────────────────────────────────
#
# HOST_NET† and HOST_PORT comprise HOST and determine the ip and port the server binds to.
# BASE_URL is used to assemble URLs, e.g. in the MS authentication flow and for serving images generated on the sd router.
# BASE_URL should match the base URL used to access sijapi sans endpoint, e.g. http://localhost:4444 or https://api.sij.ai
# API.URL is used to assemble URLs, e.g. in the MS authentication flow and for serving images generated on the sd router.
# API.URL should match the base URL used to access sijapi sans endpoint, e.g. http://localhost:4444 or https://api.sij.ai
#
# † Take care here! Please ensure you understand the implications of setting HOST_NET to anything besides 127.0.0.1, and configure your firewall and router appropriately if you do. Setting HOST_NET to 0.0.0.0, for instance, opens sijapi to any device the server running it is accessible to — including potentially frightening internet randos (depending how your firewall, router, and NAT are configured).
#

9
sijapi/config/api.yaml-example
View file

@ -15,13 +15,16 @@ TRUSTED_SUBNETS:
- 100.11.11.0/24 # optionally set to your tailscale subnet, or omit
MODULES:
asr: on
calendar: on
cal: on
cf: off
dist: off
email: on
health: on
hooks: on
ig: off
llm: on
locate: on
loc: on
note: on
rag: off
sd: on
serve: on
time: on

6
sijapi/config/dirs.yaml-example Normal file
View file

@ -0,0 +1,6 @@
HOME: ~
BASE: '{{ HOME }}/sijapi'
SIJAPI: '{{ BASE }}/sijapi'
CONFIG: '{{ SIJAPI }}/config'
DATA: '{{ SIJAPI }}/data'
LOGS: '{{ SIJAPI }}/logs'

8
sijapi/routers/health.py
View file

@ -1,14 +1,14 @@
'''
Health check module. /health returns `'status': 'ok'`, /id returns TS_ID, /routers responds with a list of the active routers, /ip responds with the device's local IP, /ts_ip responds with its tailnet IP, and /wan_ip responds with WAN IP.
Depends on:
TS_ID, ROUTERS, LOGGER, SUBNET_BROADCAST
TS_ID, LOGGER, SUBNET_BROADCAST
'''
import os
import httpx
import socket
from fastapi import APIRouter
from tailscale import Tailscale
from sijapi import L, TS_ID, ROUTERS, SUBNET_BROADCAST
from sijapi import L, API, TS_ID, SUBNET_BROADCAST
health = APIRouter(tags=["public", "trusted", "private"])
@ -22,8 +22,8 @@ def get_health() -> str:
@health.get("/routers")
def get_routers() -> str:
listrouters = ", ".join(ROUTERS)
return listrouters
active_modules = [module for module, is_active in API.MODULES.__dict__.items() if is_active]
return active_modules
@health.get("/ip")
def get_local_ip():

4
sijapi/routers/note.py
View file

@ -30,7 +30,7 @@ from dateutil.parser import parse as dateutil_parse
from fastapi import HTTPException, status
from pathlib import Path
from fastapi import APIRouter, Query, HTTPException
from sijapi import L, OBSIDIAN_VAULT_DIR, OBSIDIAN_RESOURCES_DIR, ARCHIVE_DIR, BASE_URL, OBSIDIAN_BANNER_SCENE, DEFAULT_11L_VOICE, DEFAULT_VOICE, GEO
from sijapi import API, L, OBSIDIAN_VAULT_DIR, OBSIDIAN_RESOURCES_DIR, OBSIDIAN_BANNER_SCENE, DEFAULT_11L_VOICE, DEFAULT_VOICE, GEO
from sijapi.routers import cal, loc, tts, llm, time, sd, weather, asr
from sijapi.utilities import assemble_journal_path, assemble_archive_path, convert_to_12_hour_format, sanitize_filename, convert_degrees_to_cardinal, check_file_name, HOURLY_COLUMNS_MAPPING
from sijapi.classes import Location
@ -399,7 +399,7 @@ async def post_update_daily_weather_and_calendar_and_timeslips(date: str) -> Pla
await update_dn_weather(date_time)
await update_daily_note_events(date_time)
await build_daily_timeslips(date_time)
return f"[Refresh]({BASE_URL}/update/note/{date_time.strftime('%Y-%m-%d')}"
return f"[Refresh]({API.URL}/update/note/{date_time.strftime('%Y-%m-%d')}"
async def update_dn_weather(date_time: dt_datetime, lat: float = None, lon: float = None):
L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our datetime in update_dn_weather.")

6
sijapi/routers/sd.py
View file

@ -2,7 +2,7 @@
Image generation module using StableDiffusion and similar models by way of ComfyUI.
DEPENDS ON:
LLM module
COMFYUI_URL, COMFYUI_DIR, COMFYUI_OUTPUT_DIR, HOST_PORT, TS_SUBNET, TS_ADDRESS, DATA_DIR, SD_CONFIG_DIR, SD_IMAGE_DIR, SD_WORKFLOWS_DIR, LOCAL_HOSTS, BASE_URL, PHOTOPRISM_USER*, PHOTOPRISM_URL*, PHOTOPRISM_PASS*
COMFYUI_URL, COMFYUI_DIR, COMFYUI_OUTPUT_DIR, TS_SUBNET, TS_ADDRESS, DATA_DIR, SD_CONFIG_DIR, SD_IMAGE_DIR, SD_WORKFLOWS_DIR, LOCAL_HOSTS, API.URL, PHOTOPRISM_USER*, PHOTOPRISM_URL*, PHOTOPRISM_PASS*
*unimplemented.
'''
@ -30,7 +30,7 @@ import shutil
# from photoprism.Photo import Photo
# from webdav3.client import Client
from sijapi.routers.llm import query_ollama
from sijapi import L, COMFYUI_URL, COMFYUI_LAUNCH_CMD, COMFYUI_DIR, COMFYUI_OUTPUT_DIR, HOST_PORT, TS_SUBNET, SD_CONFIG_PATH, SD_IMAGE_DIR, SD_WORKFLOWS_DIR, LOCAL_HOSTS, BASE_URL
from sijapi import API, L, COMFYUI_URL, COMFYUI_OUTPUT_DIR, SD_CONFIG_PATH, SD_IMAGE_DIR, SD_WORKFLOWS_DIR
sd = APIRouter()
@ -133,7 +133,7 @@ async def generate_and_save_image(prompt_id, saved_file_key, max_size, destinati
def get_web_path(file_path: Path) -> str:
uri = file_path.relative_to(SD_IMAGE_DIR)
web_path = f"{BASE_URL}/img/{uri}"
web_path = f"{API.URL}/img/{uri}"
return web_path

2
sijapi/routers/time.py
View file

@ -25,7 +25,7 @@ from typing import Optional, List, Dict, Union, Tuple
from collections import defaultdict
from dotenv import load_dotenv
from traceback import format_exc
from sijapi import L, HOME_DIR, TIMING_API_KEY, TIMING_API_URL
from sijapi import L, TIMING_API_KEY, TIMING_API_URL
from sijapi.routers import loc
### INITIALIZATIONS ###

2
sijapi/routers/tts.py
View file

@ -25,7 +25,7 @@ import tempfile
import random
import re
import os
from sijapi import L, HOME_DIR, DATA_DIR, DEFAULT_VOICE, TTS_DIR, TTS_SEGMENTS_DIR, VOICE_DIR, PODCAST_DIR, TTS_OUTPUT_DIR, ELEVENLABS_API_KEY
from sijapi import L, DEFAULT_VOICE, TTS_SEGMENTS_DIR, VOICE_DIR, PODCAST_DIR, TTS_OUTPUT_DIR, ELEVENLABS_API_KEY
from sijapi.utilities import sanitize_filename

31
sijapi/utilities.py
View file

@ -19,23 +19,40 @@ from typing import Optional, Union, Tuple
import asyncio
from PIL import Image
import pandas as pd
import ipaddress
from scipy.spatial import cKDTree
from dateutil.parser import parse as dateutil_parse
from docx import Document
from sshtunnel import SSHTunnelForwarder
from fastapi import Depends, HTTPException, Request, UploadFile
from fastapi.security.api_key import APIKeyHeader
from sijapi import L, GLOBAL_API_KEY, YEAR_FMT, MONTH_FMT, DAY_FMT, DAY_SHORT_FMT, OBSIDIAN_VAULT_DIR, ALLOWED_FILENAME_CHARS, MAX_PATH_LENGTH, ARCHIVE_DIR
api_key_header = APIKeyHeader(name="Authorization")
from sijapi import L, API, YEAR_FMT, MONTH_FMT, DAY_FMT, DAY_SHORT_FMT, OBSIDIAN_VAULT_DIR, ALLOWED_FILENAME_CHARS, MAX_PATH_LENGTH, ARCHIVE_DIR
api_key_header = APIKeyHeader(name="Authorization", auto_error=False)
def validate_api_key(request: Request, api_key: str = Depends(api_key_header)):
if request.url.path not in ["/health", "/ip", "/pgp"]:
api_key_query = request.query_params.get("api_key")
if api_key_header:
if request.url.path in API.PUBLIC:
return
client_ip = ipaddress.ip_address(request.client.host)
trusted_subnets = [ipaddress.ip_network(subnet) for subnet in API.TRUSTED_SUBNETS]
if any(client_ip in subnet for subnet in trusted_subnets):
return
# Check header-based API key
if api_key:
if api_key.lower().startswith("bearer "):
api_key = api_key.lower().split("bearer ")[-1]
if api_key != GLOBAL_API_KEY and api_key_query != GLOBAL_API_KEY:
raise HTTPException(status_code=401, detail="Invalid or missing API key")
if api_key in API.KEYS:
return
# Check query-based API key
api_key_query = request.query_params.get("api_key")
if api_key_query in API.KEYS:
return
raise HTTPException(status_code=401, detail="Invalid or missing API key")
def assemble_archive_path(filename: str, extension: str = ".md", date_time: datetime = datetime.now(), subdir: str = None) -> Tuple[Path, Path]: