Auto-update: Fri Aug 2 13:02:52 PDT 2024

.gitignore

@@ -12,7 +12,8 @@ sijapi/data/*.json
 sijapi/data/*.geojson
 sijapi/data/img/images/
 sijapi/config/*.yaml
-sijapi/config/MS365/
+sijapi/config/ms365/
+sijapi/data/ms365/
 sijapi/local_only/
 sijapi/testbed/
 khoj/
@@ -36,6 +37,8 @@ podcast/sideloads/*
 **/*.wav
 **/*.pyc
 **/.ipynb_checkpoints/
+**/*.pem
+**/*.key
 venv/
 env/
 .venv/
@@ -55,6 +58,8 @@ env/
 *.gz
 *.iso
 *.jar
+*.key
+*.pem
 *.rar
 *.tar
 *.zip

Extras/Caddyfile.example

@@ -1,19 +1,22 @@
 {
 	log {
+		# Specify path and log level for Caddy logs
 		output file /var/log/caddy/logfile.log
 		level INFO
 	}
 
+	# replace `localhost` with an externally accessible IP address, e.g. a local LAN address or Tailscale IP. Take care not to use a publicly accessible IP address, as the Caddy API is not separately protected by API keys!
 	admin localhost:2019
 
 	servers {
 		metrics
 	}
 
-	email !{!{ YOUR EMAIL ADDRESS }!}!
+	# Replace with your email address for SSL certificate registration
+	email info@example.com
 }
 
-# This is an extremely permissive CORS config. Dial it back as your use case allows.
+# This is a highly permissive CORS config. Dial it back as your use case allows.
 (cors) {
 	@cors_preflight method OPTIONS
 	header {
@@ -32,21 +35,29 @@
 	}
 }
 
-# Specify which endpoints are public, one or more methods of API key authentication, and your load balancing priority (if any)
+# Replace with the subdomain you want to expose your API over
-!{!{ YOUR SIJAPI SUBDOMAIN }!}! {
+api.example.com {
 	import cors
+
+	# Specify which endpoints do not require an API key
 	@public {
 		path /img/* /oauth /oauth/* /MS365 /MS365/* /ip /health /health* /health/* /id /identity
 	}
+
+	# Accept your GLOBAL_API_KEY (specified via environment variable in Caddy's context) via `Authorization: Bearer` header
 	@apiKeyAuthHeader {
-		header Authorization "Bearer !{!{ YOUR GLOBAL_API_KEY }!}!"
+		header Authorization "Bearer {env.GLOBAL_API_KEY}"
 	}
+
+	# Optionally, accept your GLOBAL_API_KEY via query parameters
 	@apiKeyAuthQuery {
-		query api_key=!{!{ YOUR GLOBAL_API_KEY }!}!
+		query api_key={env.GLOBAL_API_KEY}
 	}
+
 	handle @public {
 		reverse_proxy {
-			to !{!{ YOUR IP(s) WHERE SIJAPI IS RUNNING, WITH PORTS, e.g. 100.64.64.20:4444 10.13.37.30:4444 localhost:4444 }!}!
+			# Specify the local (or Tailscale) IPs & ports where the API service is running
+			to  100.64.64.20:4444 100.64.64.11:4444 10.13.37.30:4444 localhost:4444
 			lb_policy first
 			health_uri /health
 			health_interval 10s
@@ -56,9 +67,11 @@
 			header_up X-Forwarded-Proto {scheme}
 		}
 	}
+
 	handle @apiKeyAuthHeader {
 		reverse_proxy {
-			to !{!{ YOUR IP(s) WHERE SIJAPI IS RUNNING, WITH PORTS, e.g. 100.64.64.20:4444 10.13.37.30:4444 localhost:4444 }!}!
+			# Specify the local (or Tailscale) IPs & ports where the API service is running
+			to  100.64.64.20:4444 100.64.64.11:4444 10.13.37.30:4444 localhost:4444
 			lb_policy first
 			health_uri /health
 			health_interval 10s
@@ -66,9 +79,11 @@
 			health_status 2xx
 		}
 	}
+
 	handle @apiKeyAuthQuery {
 		reverse_proxy {
-			to !{!{ YOUR IP(s) WHERE SIJAPI IS RUNNING, WITH PORTS, e.g. 100.64.64.20:4444 10.13.37.30:4444 localhost:4444 }!}!
+			# Specify the local (or Tailscale) IPs & ports where the API service is running
+			to  100.64.64.20:4444 100.64.64.11:4444 10.13.37.30:4444 localhost:4444
 			lb_policy first
 			health_uri /health
 			health_interval 10s
@@ -76,12 +91,16 @@
 			health_status 2xx
 		}
 	}
+
 	handle {
 		respond "Unauthorized: Valid API key required" 401
 	}
+
+	# Assuming you use Cloudflare for DNS challenges and have configured a CLOUDFLARE_API_TOKEN environmental variable in Caddy's context
 	tls {
 		dns cloudflare {env.CLOUDFLARE_API_TOKEN}
 	}
+
 	log {
 		output file /var/log/caddy/sijapi.log {
 			roll_size 100mb
@@ -94,3 +113,106 @@
 		}
 	}
 }
+
+# Everything below here is ancillary to the primary API functionality
+# If you have another domain you want to expose a particular endpoint on, try something like this -- e.g., here, https://sij.law/pgp as a short URL to share my public PGP key via.
+sij.law {
+	reverse_proxy /pgp 100.64.64.20:4444 100.64.64.30:4444 100.64.64.11:4444 localhost:4444 {
+		lb_policy first
+		health_uri /health
+		health_interval 10s
+		health_timeout 5s
+		health_status 2xx
+	}
+
+	# Because I maintain a seperate service on this domain (a Ghost blog), I need fall back handling for everything besides `/pgp`.
+	reverse_proxy localhost:2368
+	tls {
+		dns cloudflare {env.CLOUDFLARE_API_TOKEN}
+	}
+}
+
+# Another special use case example: this provides handling for my URL shortener. 
+sij.ai {
+	
+	# Any three-character alphanumeric URI is construed as a shortened URL.
+	@shorturl {
+		path_regexp ^/[a-zA-Z0-9]{3}$
+	}
+
+	# https://sij.ai/s points to the WebUI for my URL shortener
+	@shortener_ui {
+		path /s
+	}
+
+	@apiKeyAuthHeader {
+        header Authorization "Bearer {env.GLOBAL_API_KEY}"
+    }
+
+    @apiKeyAuthQuery {
+        query api_key={env.GLOBAL_API_KEY}
+    }
+
+	@analytics {
+		path_regexp ^/analytics/[a-zA-Z0-9]{3}$
+	}
+
+	@pgp {
+		path /pgp
+	}
+
+	handle @shortener_ui {
+		reverse_proxy 100.64.64.20:4444 100.64.64.30:4444 100.64.64.11:4444 localhost:4444 {
+			lb_policy first
+			health_uri /health
+			health_interval 10s
+			health_timeout 5s
+			health_status 2xx
+		}
+	}
+
+	handle @shorturl {
+		reverse_proxy 100.64.64.20:4444 100.64.64.30:4444 100.64.64.11:4444 localhost:4444 {
+			lb_policy first
+			health_uri /health
+			health_interval 10s
+			health_timeout 5s
+			health_status 2xx
+		}
+	}
+
+	handle @analytics {
+		reverse_proxy 100.64.64.20:4444 100.64.64.30:4444 100.64.64.11:4444 localhost:4444 {
+			lb_policy first
+			health_uri /health
+			health_interval 10s
+			health_timeout 5s
+			health_status 2xx
+		}
+	}
+
+	# Handling for my public PGP key endpoint
+	handle @pgp {
+		reverse_proxy 100.64.64.20:4444 100.64.64.30:4444 100.64.64.11:4444 localhost:4444 {
+			lb_policy first
+			health_uri /health
+			health_interval 10s
+			health_timeout 5s
+			health_status 2xx
+		}
+	}
+	
+	# Base domain redirects to my Ghost blog
+	handle / {
+		redir https://sij.law permanent
+	}
+
+	# All URIs that don't fit the patterns above redirect to the equivalent URI on my Ghost blog domain
+	handle /* {
+		redir https://sij.law{uri} permanent
+	}
+
+	tls {
+		dns cloudflare {env.CLOUDFLARE_API_TOKEN}
+	}
+}

sijapi/__init__.py

@@ -180,8 +180,8 @@ CADDY_API_KEY = os.getenv("CADDY_API_KEY")
 MS365_CLIENT_ID = os.getenv('MS365_CLIENT_ID')
 MS365_SECRET = os.getenv('MS365_SECRET')
 MS365_TENANT_ID = os.getenv('MS365_TENANT_ID')
-MS365_CERT_PATH = CONFIG_DIR / 'MS365' / '.cert.pem' # deprecated
+MS365_CERT_PATH = DATA_DIR / 'ms365' / '.cert.pem' # deprecated
-MS365_KEY_PATH = CONFIG_DIR / 'MS365' / '.cert.key' # deprecated
+MS365_KEY_PATH = DATA_DIR / 'ms365' / '.cert.key' # deprecated
 MS365_KEY = MS365_KEY_PATH.read_text()
 MS365_TOKEN_PATH = CONFIG_DIR / 'MS365' / '.token.txt'
 MS365_THUMBPRINT = os.getenv('MS365_THUMBPRINT')

sijapi/config/api.yaml-example

@@ -1,7 +1,10 @@
-HOST: 0.0.0.0
+# Primary configuration file
+
+HOST: '0.0.0.0'
 PORT: 4444
 BIND: '{{ HOST }}:{{ PORT }}'
-URL: https://api.yourdomain.com
+URL: 'https://api.example.com'
+
 PUBLIC: 
   - /id
   - /ip
@@ -10,27 +13,85 @@ PUBLIC:
   - /cl/dockets
   - /cl/search
   - /cd/alert
+
 TRUSTED_SUBNETS:
-  - 127.0.0.1/32 # don't change this
+  - 127.0.0.1/32
-  - 192.168.50.0/24 # optionally set to your local subnet, or omit
+  - 10.0.0.0/24
-  - 100.11.11.0/24 # optionally set to your tailscale subnet, or omit
+  - 192.168.0.0/24
+
 MODULES:
   asr: on
   cal: on
   cf: off
   dist: off
   email: on
+  gis: on
   health: on
   ig: off
+  img: on
   llm: on
-  loc: on
   news: on
   note: on
   rag: off
-  img: on
+  scrape: on
   serve: on
-  time: on
+  timing: on
   tts: on
   weather: on
-TZ: 'America/Los_Angeles' # this is just for the initial config, and is dynamically updated based on location
+
-KEYS: ['{{ SECRET.GLOBAL_API_KEYS }}'] # sourced from .env
+POOL:
+  - ts_id: 'server1'
+    ts_ip: '192.168.0.10'
+    app_port: 4444
+    db_port: 5432
+    db_name: mydb
+    db_user: dbuser
+    db_pass: 'password123'
+    ssh_port: 22
+    ssh_user: sshuser
+    ssh_pass: 'password456'
+    path: '~/projects/myapi'
+    tmux: '/opt/homebrew/bin/tmux'
+    conda: '~/miniforge3/bin/mamba'
+    conda_env: 'myenv'
+  - ts_id: 'server2'
+    ts_ip: '192.168.0.11'
+    app_port: 4444
+    db_port: 5432
+    db_name: mydb
+    db_user: dbuser
+    db_pass: 'password123'
+    ssh_port: 22
+    ssh_user: sshuser
+    ssh_pass: 'password456'
+    path: '~/projects/myapi'
+    tmux: '/usr/bin/tmux'
+    conda: '~/miniforge3/bin/mamba'
+    conda_env: 'myenv'
+  - ts_id: 'server3'
+    ts_ip: '192.168.0.12'
+    app_port: 4444
+    db_port: 5432
+    db_name: mydb
+    db_user: dbuser
+    db_pass: 'password123'
+    ssh_port: 22
+    ssh_user: sshuser
+    ssh_pass: 'password456'
+    path: '~/projects/myapi'
+    tmux: '/usr/bin/tmux'
+    conda: '~/miniforge3/bin/mamba'
+    conda_env: 'myenv'
+  
+EXTENSIONS:
+  courtlistener: off
+  macnotify: on
+  shellfish: on
+
+TZ: 'UTC'
+
+KEYS: ['{{ SECRET.GLOBAL_API_KEYS }}']
+
+GARBAGE:
+  COLLECTION_INTERVAL: 60 * 60
+  TTL: 60 * 60 * 24

sijapi/helpers/start.py

@@ -0,0 +1,121 @@
+import yaml
+import requests
+import paramiko
+import time
+from pathlib import Path
+import logging
+import subprocess
+import os
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+def load_config():
+    config_path = Path(__file__).parent.parent / 'config' / 'api.yaml'
+    with open(config_path, 'r') as file:
+        return yaml.safe_load(file)
+
+def load_env():
+    env_path = Path(__file__).parent.parent / 'config' / '.env'
+    if env_path.exists():
+        with open(env_path, 'r') as file:
+            for line in file:
+                line = line.strip()
+                if line and not line.startswith('#'):
+                    try:
+                        key, value = line.split('=', 1)
+                        os.environ[key.strip()] = value.strip()
+                    except ValueError:
+                        logging.warning(f"Skipping invalid line in .env file: {line}")
+
+
+def check_server(ip, port, ts_id):
+    try:
+        response = requests.get(f"http://{ip}:{port}/ts_id", timeout=5)
+        return response.status_code == 200 and response.text.strip() == ts_id
+    except requests.RequestException as e:
+        logging.error(f"Error checking server {ts_id}: {str(e)}")
+        return False
+
+def execute_ssh_command(ssh, command):
+    stdin, stdout, stderr = ssh.exec_command(command)
+    exit_status = stdout.channel.recv_exit_status()
+    output = stdout.read().decode().strip()
+    error = stderr.read().decode().strip()
+    return exit_status, output, error
+
+def is_local_tmux_session_running(session_name):
+    try:
+        result = subprocess.run(['tmux', 'has-session', '-t', session_name], capture_output=True, text=True)
+        return result.returncode == 0
+    except subprocess.CalledProcessError:
+        return False
+
+def start_local_server(server):
+    try:
+        if is_local_tmux_session_running('sijapi'):
+            logging.info("Local sijapi tmux session is already running.")
+            return
+
+        command = f"{server['tmux']} new-session -d -s sijapi 'cd {server['path']} && {server['conda_env']}/bin/python -m sijapi'"
+        logging.info(f"Executing local command: {command}")
+        result = subprocess.run(command, shell=True, check=True, capture_output=True, text=True)
+        logging.info(f"Successfully started sijapi session on local machine")
+        logging.debug(f"Command output: {result.stdout}")
+    except subprocess.CalledProcessError as e:
+        logging.error(f"Failed to start sijapi session on local machine. Error: {e}")
+        logging.error(f"Error output: {e.stderr}")
+
+def start_remote_server(server):
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    
+    try:
+        ssh.connect(
+            server['ts_ip'],
+            port=server['ssh_port'],
+            username=server['ssh_user'],
+            password=server['ssh_pass'],
+            timeout=10
+        )
+        
+        # Check if tmux session already exists
+        status, output, error = execute_ssh_command(ssh, f"{server['tmux']} has-session -t sijapi 2>/dev/null && echo 'exists' || echo 'not exists'")
+        if output == 'exists':
+            logging.info(f"sijapi session already exists on {server['ts_id']}")
+            return
+
+        command = f"{server['tmux']} new-session -d -s sijapi 'cd {server['path']} && {server['conda_env']}/bin/python -m sijapi'"
+        status, output, error = execute_ssh_command(ssh, command)
+        
+        if status == 0:
+            logging.info(f"Successfully started sijapi session on {server['ts_id']}")
+        else:
+            logging.error(f"Failed to start sijapi session on {server['ts_id']}. Error: {error}")
+        
+    except paramiko.SSHException as e:
+        logging.error(f"Failed to connect to {server['ts_id']}: {str(e)}")
+    finally:
+        ssh.close()
+
+def main():
+    load_env()
+    config = load_config()
+    pool = config['POOL']
+    local_ts_id = os.environ.get('TS_ID')
+    
+    for server in pool:
+        logging.info(f"Checking {server['ts_id']}...")
+        if check_server(server['ts_ip'], server['app_port'], server['ts_id']):
+            logging.info(f"{server['ts_id']} is running and responding correctly.")
+        else:
+            logging.info(f"{server['ts_id']} is not responding. Attempting to start...")
+            if server['ts_id'] == local_ts_id:
+                start_local_server(server)
+            else:
+                start_remote_server(server)
+        
+        logging.info("Waiting 5 seconds before next check...")
+        time.sleep(5)
+
+if __name__ == "__main__":
+    main()