sij/sijapi
1
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Auto-update: Fri Aug 2 13:02:52 PDT 2024

Browse source
This commit is contained in:
sanj 2024-08-02 13:02:52 -07:00
parent 7e8f45fe64
commit 89c88c9d25
5 changed files with 331 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.gitignore vendored
View file

@ -12,7 +12,8 @@ sijapi/data/*.json
sijapi/data/*.geojson
sijapi/data/img/images/
sijapi/config/*.yaml
sijapi/config/MS365/
sijapi/config/ms365/
sijapi/data/ms365/
sijapi/local_only/
sijapi/testbed/
khoj/
@ -36,6 +37,8 @@ podcast/sideloads/*
**/*.wav
**/*.pyc
**/.ipynb_checkpoints/
**/*.pem
**/*.key
venv/
env/
.venv/
@ -55,6 +58,8 @@ env/
*.gz
*.iso
*.jar
*.key
*.pem
*.rar
*.tar
*.zip

140
Extras/Caddyfile.example
View file

@ -1,19 +1,22 @@
{
log {
# Specify path and log level for Caddy logs
output file /var/log/caddy/logfile.log
level INFO
}
# replace `localhost` with an externally accessible IP address, e.g. a local LAN address or Tailscale IP. Take care not to use a publicly accessible IP address, as the Caddy API is not separately protected by API keys!
admin localhost:2019
servers {
metrics
}
email !{!{ YOUR EMAIL ADDRESS }!}!
# Replace with your email address for SSL certificate registration
email info@example.com
}
# This is an extremely permissive CORS config. Dial it back as your use case allows.
# This is a highly permissive CORS config. Dial it back as your use case allows.
(cors) {
@cors_preflight method OPTIONS
header {
@ -32,21 +35,29 @@
}
}
# Specify which endpoints are public, one or more methods of API key authentication, and your load balancing priority (if any)
!{!{ YOUR SIJAPI SUBDOMAIN }!}! {
# Replace with the subdomain you want to expose your API over
api.example.com {
import cors
# Specify which endpoints do not require an API key
@public {
path /img/* /oauth /oauth/* /MS365 /MS365/* /ip /health /health* /health/* /id /identity
}
# Accept your GLOBAL_API_KEY (specified via environment variable in Caddy's context) via `Authorization: Bearer` header
@apiKeyAuthHeader {
header Authorization "Bearer !{!{ YOUR GLOBAL_API_KEY }!}!"
header Authorization "Bearer {env.GLOBAL_API_KEY}"
}
# Optionally, accept your GLOBAL_API_KEY via query parameters
@apiKeyAuthQuery {
query api_key=!{!{ YOUR GLOBAL_API_KEY }!}!
query api_key={env.GLOBAL_API_KEY}
}
handle @public {
reverse_proxy {
to !{!{ YOUR IP(s) WHERE SIJAPI IS RUNNING, WITH PORTS, e.g. 100.64.64.20:4444 10.13.37.30:4444 localhost:4444 }!}!
# Specify the local (or Tailscale) IPs & ports where the API service is running
to 100.64.64.20:4444 100.64.64.11:4444 10.13.37.30:4444 localhost:4444
lb_policy first
health_uri /health
health_interval 10s
@ -56,9 +67,11 @@
header_up X-Forwarded-Proto {scheme}
}
}
handle @apiKeyAuthHeader {
reverse_proxy {
to !{!{ YOUR IP(s) WHERE SIJAPI IS RUNNING, WITH PORTS, e.g. 100.64.64.20:4444 10.13.37.30:4444 localhost:4444 }!}!
# Specify the local (or Tailscale) IPs & ports where the API service is running
to 100.64.64.20:4444 100.64.64.11:4444 10.13.37.30:4444 localhost:4444
lb_policy first
health_uri /health
health_interval 10s
@ -66,9 +79,11 @@
health_status 2xx
}
}
handle @apiKeyAuthQuery {
reverse_proxy {
to !{!{ YOUR IP(s) WHERE SIJAPI IS RUNNING, WITH PORTS, e.g. 100.64.64.20:4444 10.13.37.30:4444 localhost:4444 }!}!
# Specify the local (or Tailscale) IPs & ports where the API service is running
to 100.64.64.20:4444 100.64.64.11:4444 10.13.37.30:4444 localhost:4444
lb_policy first
health_uri /health
health_interval 10s
@ -76,12 +91,16 @@
health_status 2xx
}
}
handle {
respond "Unauthorized: Valid API key required" 401
}
# Assuming you use Cloudflare for DNS challenges and have configured a CLOUDFLARE_API_TOKEN environmental variable in Caddy's context
tls {
dns cloudflare {env.CLOUDFLARE_API_TOKEN}
}
log {
output file /var/log/caddy/sijapi.log {
roll_size 100mb
@ -94,3 +113,106 @@
}
}
}
# Everything below here is ancillary to the primary API functionality
# If you have another domain you want to expose a particular endpoint on, try something like this -- e.g., here, https://sij.law/pgp as a short URL to share my public PGP key via.
sij.law {
reverse_proxy /pgp 100.64.64.20:4444 100.64.64.30:4444 100.64.64.11:4444 localhost:4444 {
lb_policy first
health_uri /health
health_interval 10s
health_timeout 5s
health_status 2xx
}
# Because I maintain a seperate service on this domain (a Ghost blog), I need fall back handling for everything besides `/pgp`.
reverse_proxy localhost:2368
tls {
dns cloudflare {env.CLOUDFLARE_API_TOKEN}
}
}
# Another special use case example: this provides handling for my URL shortener.
sij.ai {
# Any three-character alphanumeric URI is construed as a shortened URL.
@shorturl {
path_regexp ^/[a-zA-Z0-9]{3}$
}
# https://sij.ai/s points to the WebUI for my URL shortener
@shortener_ui {
path /s
}
@apiKeyAuthHeader {
header Authorization "Bearer {env.GLOBAL_API_KEY}"
}
@apiKeyAuthQuery {
query api_key={env.GLOBAL_API_KEY}
}
@analytics {
path_regexp ^/analytics/[a-zA-Z0-9]{3}$
}
@pgp {
path /pgp
}
handle @shortener_ui {
reverse_proxy 100.64.64.20:4444 100.64.64.30:4444 100.64.64.11:4444 localhost:4444 {
lb_policy first
health_uri /health
health_interval 10s
health_timeout 5s
health_status 2xx
}
}
handle @shorturl {
reverse_proxy 100.64.64.20:4444 100.64.64.30:4444 100.64.64.11:4444 localhost:4444 {
lb_policy first
health_uri /health
health_interval 10s
health_timeout 5s
health_status 2xx
}
}
handle @analytics {
reverse_proxy 100.64.64.20:4444 100.64.64.30:4444 100.64.64.11:4444 localhost:4444 {
lb_policy first
health_uri /health
health_interval 10s
health_timeout 5s
health_status 2xx
}
}
# Handling for my public PGP key endpoint
handle @pgp {
reverse_proxy 100.64.64.20:4444 100.64.64.30:4444 100.64.64.11:4444 localhost:4444 {
lb_policy first
health_uri /health
health_interval 10s
health_timeout 5s
health_status 2xx
}
}
# Base domain redirects to my Ghost blog
handle / {
redir https://sij.law permanent
}
# All URIs that don't fit the patterns above redirect to the equivalent URI on my Ghost blog domain
handle /* {
redir https://sij.law{uri} permanent
}
tls {
dns cloudflare {env.CLOUDFLARE_API_TOKEN}
}
}

4
sijapi/__init__.py
View file

@ -180,8 +180,8 @@ CADDY_API_KEY = os.getenv("CADDY_API_KEY")
MS365_CLIENT_ID = os.getenv('MS365_CLIENT_ID')
MS365_SECRET = os.getenv('MS365_SECRET')
MS365_TENANT_ID = os.getenv('MS365_TENANT_ID')
MS365_CERT_PATH = CONFIG_DIR / 'MS365' / '.cert.pem' # deprecated
MS365_KEY_PATH = CONFIG_DIR / 'MS365' / '.cert.key' # deprecated
MS365_CERT_PATH = DATA_DIR / 'ms365' / '.cert.pem' # deprecated
MS365_KEY_PATH = DATA_DIR / 'ms365' / '.cert.key' # deprecated
MS365_KEY = MS365_KEY_PATH.read_text()
MS365_TOKEN_PATH = CONFIG_DIR / 'MS365' / '.token.txt'
MS365_THUMBPRINT = os.getenv('MS365_THUMBPRINT')

81
sijapi/config/api.yaml-example
View file

@ -1,7 +1,10 @@
HOST: 0.0.0.0
# Primary configuration file
HOST: '0.0.0.0'
PORT: 4444
BIND: '{{ HOST }}:{{ PORT }}'
URL: https://api.yourdomain.com
URL: 'https://api.example.com'
PUBLIC:
- /id
- /ip
@ -10,27 +13,85 @@ PUBLIC:
- /cl/dockets
- /cl/search
- /cd/alert
TRUSTED_SUBNETS:
- 127.0.0.1/32 # don't change this
- 192.168.50.0/24 # optionally set to your local subnet, or omit
- 100.11.11.0/24 # optionally set to your tailscale subnet, or omit
- 127.0.0.1/32
- 10.0.0.0/24
- 192.168.0.0/24
MODULES:
asr: on
cal: on
cf: off
dist: off
email: on
gis: on
health: on
ig: off
img: on
llm: on
loc: on
news: on
note: on
rag: off
img: on
scrape: on
serve: on
time: on
timing: on
tts: on
weather: on
TZ: 'America/Los_Angeles' # this is just for the initial config, and is dynamically updated based on location
KEYS: ['{{ SECRET.GLOBAL_API_KEYS }}'] # sourced from .env
POOL:
- ts_id: 'server1'
ts_ip: '192.168.0.10'
app_port: 4444
db_port: 5432
db_name: mydb
db_user: dbuser
db_pass: 'password123'
ssh_port: 22
ssh_user: sshuser
ssh_pass: 'password456'
path: '~/projects/myapi'
tmux: '/opt/homebrew/bin/tmux'
conda: '~/miniforge3/bin/mamba'
conda_env: 'myenv'
- ts_id: 'server2'
ts_ip: '192.168.0.11'
app_port: 4444
db_port: 5432
db_name: mydb
db_user: dbuser
db_pass: 'password123'
ssh_port: 22
ssh_user: sshuser
ssh_pass: 'password456'
path: '~/projects/myapi'
tmux: '/usr/bin/tmux'
conda: '~/miniforge3/bin/mamba'
conda_env: 'myenv'
- ts_id: 'server3'
ts_ip: '192.168.0.12'
app_port: 4444
db_port: 5432
db_name: mydb
db_user: dbuser
db_pass: 'password123'
ssh_port: 22
ssh_user: sshuser
ssh_pass: 'password456'
path: '~/projects/myapi'
tmux: '/usr/bin/tmux'
conda: '~/miniforge3/bin/mamba'
conda_env: 'myenv'
EXTENSIONS:
courtlistener: off
macnotify: on
shellfish: on
TZ: 'UTC'
KEYS: ['{{ SECRET.GLOBAL_API_KEYS }}']
GARBAGE:
COLLECTION_INTERVAL: 60 * 60
TTL: 60 * 60 * 24

121
sijapi/helpers/start.py Normal file
View file

@ -0,0 +1,121 @@
import yaml
import requests
import paramiko
import time
from pathlib import Path
import logging
import subprocess
import os
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
def load_config():
config_path = Path(__file__).parent.parent / 'config' / 'api.yaml'
with open(config_path, 'r') as file:
return yaml.safe_load(file)
def load_env():
env_path = Path(__file__).parent.parent / 'config' / '.env'
if env_path.exists():
with open(env_path, 'r') as file:
for line in file:
line = line.strip()
if line and not line.startswith('#'):
try:
key, value = line.split('=', 1)
os.environ[key.strip()] = value.strip()
except ValueError:
logging.warning(f"Skipping invalid line in .env file: {line}")
def check_server(ip, port, ts_id):
try:
response = requests.get(f"http://{ip}:{port}/ts_id", timeout=5)
return response.status_code == 200 and response.text.strip() == ts_id
except requests.RequestException as e:
logging.error(f"Error checking server {ts_id}: {str(e)}")
return False
def execute_ssh_command(ssh, command):
stdin, stdout, stderr = ssh.exec_command(command)
exit_status = stdout.channel.recv_exit_status()
output = stdout.read().decode().strip()
error = stderr.read().decode().strip()
return exit_status, output, error
def is_local_tmux_session_running(session_name):
try:
result = subprocess.run(['tmux', 'has-session', '-t', session_name], capture_output=True, text=True)
return result.returncode == 0
except subprocess.CalledProcessError:
return False
def start_local_server(server):
try:
if is_local_tmux_session_running('sijapi'):
logging.info("Local sijapi tmux session is already running.")
return
command = f"{server['tmux']} new-session -d -s sijapi 'cd {server['path']} && {server['conda_env']}/bin/python -m sijapi'"
logging.info(f"Executing local command: {command}")
result = subprocess.run(command, shell=True, check=True, capture_output=True, text=True)
logging.info(f"Successfully started sijapi session on local machine")
logging.debug(f"Command output: {result.stdout}")
except subprocess.CalledProcessError as e:
logging.error(f"Failed to start sijapi session on local machine. Error: {e}")
logging.error(f"Error output: {e.stderr}")
def start_remote_server(server):
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(
server['ts_ip'],
port=server['ssh_port'],
username=server['ssh_user'],
password=server['ssh_pass'],
timeout=10
)
# Check if tmux session already exists
status, output, error = execute_ssh_command(ssh, f"{server['tmux']} has-session -t sijapi 2>/dev/null && echo 'exists' || echo 'not exists'")
if output == 'exists':
logging.info(f"sijapi session already exists on {server['ts_id']}")
return
command = f"{server['tmux']} new-session -d -s sijapi 'cd {server['path']} && {server['conda_env']}/bin/python -m sijapi'"
status, output, error = execute_ssh_command(ssh, command)
if status == 0:
logging.info(f"Successfully started sijapi session on {server['ts_id']}")
else:
logging.error(f"Failed to start sijapi session on {server['ts_id']}. Error: {error}")
except paramiko.SSHException as e:
logging.error(f"Failed to connect to {server['ts_id']}: {str(e)}")
finally:
ssh.close()
def main():
load_env()
config = load_config()
pool = config['POOL']
local_ts_id = os.environ.get('TS_ID')
for server in pool:
logging.info(f"Checking {server['ts_id']}...")
if check_server(server['ts_ip'], server['app_port'], server['ts_id']):
logging.info(f"{server['ts_id']} is running and responding correctly.")
else:
logging.info(f"{server['ts_id']} is not responding. Attempting to start...")
if server['ts_id'] == local_ts_id:
start_local_server(server)
else:
start_remote_server(server)
logging.info("Waiting 5 seconds before next check...")
time.sleep(5)
if __name__ == "__main__":
main()