sij/sijapi
1
1
Fork 0
Code Issues3 Pull requests Projects Releases Packages Wiki Activity Actions

Auto-update: Tue Aug 6 19:55:48 PDT 2024

Browse source
This commit is contained in:
sanj 2024-08-06 19:55:48 -07:00
parent 1c56bc5f47
commit 1a987996f6
8 changed files with 421 additions and 10 deletions
Extras
Archivist.js UserScript (web archiving)
README.mdarchivist.js
Caddyfile
Caddyfile.exampleREADME.md
GPS.py Pythonista Script (location tracking)
GPS.pyREADME.mduploadGPS.py
sijapi
__main__.py

3
Extras/Archivist.js UserScript (web archiving)/README.md Normal file
View file

@ -0,0 +1,3 @@
This is designed to work with UserScripts and similar browser extensions. Fill in the domain/URL where your sijapi instance is exposed (http://localhost:4444 is fine for the same device, but consider using a reverse proxy to extend to your mobile devices).
And fill in your GLOBAL_API_KEY that you chose when configuring sijapi.

119
Extras/Archivist.js UserScript (web archiving)/archivist.js Normal file
View file

@ -0,0 +1,119 @@
// ==UserScript==
// @name Archivist
// @version 0.5
// @description archivist userscript posts to sij.ai/archive
// @author sij
// @match *://*/*
// @grant GM_xmlhttpRequest
// ==/UserScript==
(function () {
"use strict";
// Function to check if the URL is likely an ad, tracker, or unwanted resource
function isUnwantedURL(url) {
const unwantedPatterns = [
/doubleclick\.net/,
/googlesyndication\.com/,
/adservice\./,
/analytics\./,
/tracker\./,
/pixel\./,
/ad\d*\./,
/\.ad\./,
/ads\./,
/\/ads\//,
/url=http/,
/safeframe/,
/adsystem/,
/adserver/,
/adnetwork/,
/sync\./,
/beacon\./,
/optimizely/,
/outbrain/,
/widgets\./,
/cdn\./,
/pixel\?/,
/recaptcha/,
/accounts\.google\.com\/gsi/,
/imasdk\.googleapis\.com/,
/amplify-imp/,
/zemanta/,
/monitor\.html/,
/widgetMonitor/,
/nanoWidget/,
/client_storage/,
];
return unwantedPatterns.some((pattern) => pattern.test(url));
}
// Function to archive the page
function archivePage() {
var currentUrl = window.location.href;
if (isUnwantedURL(currentUrl)) {
console.log("Skipping unwanted URL:", currentUrl);
return;
}
var data = new URLSearchParams({
title: document.title,
url: currentUrl,
referrer: document.referrer || "",
width: window.innerWidth ? window.innerWidth.toString() : "",
encoding: document.characterSet,
source: document.documentElement.outerHTML,
});
GM_xmlhttpRequest({
method: "POST",
url: "https://api.sij.ai/archive?api_key=sk-NhrtQwCHNdK5sRZC",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
Authorization: "bearer sk-NhrtQwCHNdK5sRZC",
},
data: data.toString(),
onload: function (response) {
console.log("Archive request sent for:", currentUrl);
},
onerror: function (error) {
console.error("Error sending archive request:", error);
},
});
}
// Debounce function to limit how often archivePage can be called
function debounce(func, wait) {
let timeout;
return function executedFunction(...args) {
const later = () => {
clearTimeout(timeout);
func(...args);
};
clearTimeout(timeout);
timeout = setTimeout(later, wait);
};
}
// Debounced version of archivePage
const debouncedArchivePage = debounce(archivePage, 2000);
// Listen for navigation events
window.addEventListener("popstate", debouncedArchivePage);
// Intercept link clicks
document.addEventListener(
"click",
function (e) {
var link = e.target.closest("a");
if (link && !isUnwantedURL(link.href)) {
setTimeout(debouncedArchivePage, 1000); // Delay to allow page to load
}
},
true
);
// Initial page load
setTimeout(archivePage, 5000);
})();

218
Extras/Caddyfile/Caddyfile.example Normal file
View file

@ -0,0 +1,218 @@
{
log {
# Specify path and log level for Caddy logs
output file /var/log/caddy/logfile.log
level INFO
}
# replace `localhost` with an externally accessible IP address, e.g. a local LAN address or Tailscale IP. Take care not to use a publicly accessible IP address, as the Caddy API is not separately protected by API keys!
admin localhost:2019
servers {
metrics
}
# Replace with your email address for SSL certificate registration
email info@example.com
}
# This is a highly permissive CORS config. Dial it back as your use case allows.
(cors) {
@cors_preflight method OPTIONS
header {
Access-Control-Allow-Origin "*"
Access-Control-Expose-Headers "Authorization"
Access-Control-Allow-Credentials "true"
Access-Control-Allow-Headers "Authorization, Content-Type"
}
handle @cors_preflight {
header {
Access-Control-Allow-Methods "GET, POST, PUT, PATCH, DELETE"
Access-Control-Max-Age "3600"
}
respond "" 204
}
}
# Replace with the subdomain you want to expose your API over
api.example.com {
import cors
# Specify which endpoints do not require an API key
@public {
path /img/* /oauth /oauth/* /MS365 /MS365/* /ip /health /health* /health/* /id /identity
}
# Accept your GLOBAL_API_KEY (specified via environment variable in Caddy's context) via `Authorization: Bearer` header
@apiKeyAuthHeader {
header Authorization "Bearer {env.GLOBAL_API_KEY}"
}
# Optionally, accept your GLOBAL_API_KEY via query parameters
@apiKeyAuthQuery {
query api_key={env.GLOBAL_API_KEY}
}
handle @public {
reverse_proxy {
# Specify the local (or Tailscale) IPs & ports where the API service is running
to 100.64.64.20:4444 100.64.64.11:4444 10.13.37.30:4444 localhost:4444
lb_policy first
health_uri /health
health_interval 10s
health_timeout 5s
health_status 2xx
header_up X-Forwarded-For {remote}
header_up X-Forwarded-Proto {scheme}
}
}
handle @apiKeyAuthHeader {
reverse_proxy {
# Specify the local (or Tailscale) IPs & ports where the API service is running
to 100.64.64.20:4444 100.64.64.11:4444 10.13.37.30:4444 localhost:4444
lb_policy first
health_uri /health
health_interval 10s
health_timeout 5s
health_status 2xx
}
}
handle @apiKeyAuthQuery {
reverse_proxy {
# Specify the local (or Tailscale) IPs & ports where the API service is running
to 100.64.64.20:4444 100.64.64.11:4444 10.13.37.30:4444 localhost:4444
lb_policy first
health_uri /health
health_interval 10s
health_timeout 5s
health_status 2xx
}
}
handle {
respond "Unauthorized: Valid API key required" 401
}
# Assuming you use Cloudflare for DNS challenges and have configured a CLOUDFLARE_API_TOKEN environmental variable in Caddy's context
tls {
dns cloudflare {env.CLOUDFLARE_API_TOKEN}
}
log {
output file /var/log/caddy/sijapi.log {
roll_size 100mb
roll_keep 5
roll_keep_for 720h
}
format json {
time_format "iso8601"
message_key "message"
}
}
}
# Everything below here is ancillary to the primary API functionality
# If you have another domain you want to expose a particular endpoint on, try something like this -- e.g., here, https://sij.law/pgp as a short URL to share my public PGP key via.
sij.law {
reverse_proxy /pgp 100.64.64.20:4444 100.64.64.30:4444 100.64.64.11:4444 localhost:4444 {
lb_policy first
health_uri /health
health_interval 10s
health_timeout 5s
health_status 2xx
}
# Because I maintain a seperate service on this domain (a Ghost blog), I need fall back handling for everything besides `/pgp`.
reverse_proxy localhost:2368
tls {
dns cloudflare {env.CLOUDFLARE_API_TOKEN}
}
}
# Another special use case example: this provides handling for my URL shortener.
sij.ai {
# Any three-character alphanumeric URI is construed as a shortened URL.
@shorturl {
path_regexp ^/[a-zA-Z0-9]{3}$
}
# https://sij.ai/s points to the WebUI for my URL shortener
@shortener_ui {
path /s
}
@apiKeyAuthHeader {
header Authorization "Bearer {env.GLOBAL_API_KEY}"
}
@apiKeyAuthQuery {
query api_key={env.GLOBAL_API_KEY}
}
@analytics {
path_regexp ^/analytics/[a-zA-Z0-9]{3}$
}
@pgp {
path /pgp
}
handle @shortener_ui {
reverse_proxy 100.64.64.20:4444 100.64.64.30:4444 100.64.64.11:4444 localhost:4444 {
lb_policy first
health_uri /health
health_interval 10s
health_timeout 5s
health_status 2xx
}
}
handle @shorturl {
reverse_proxy 100.64.64.20:4444 100.64.64.30:4444 100.64.64.11:4444 localhost:4444 {
lb_policy first
health_uri /health
health_interval 10s
health_timeout 5s
health_status 2xx
}
}
handle @analytics {
reverse_proxy 100.64.64.20:4444 100.64.64.30:4444 100.64.64.11:4444 localhost:4444 {
lb_policy first
health_uri /health
health_interval 10s
health_timeout 5s
health_status 2xx
}
}
# Handling for my public PGP key endpoint
handle @pgp {
reverse_proxy 100.64.64.20:4444 100.64.64.30:4444 100.64.64.11:4444 localhost:4444 {
lb_policy first
health_uri /health
health_interval 10s
health_timeout 5s
health_status 2xx
}
}
# Base domain redirects to my Ghost blog
handle / {
redir https://sij.law permanent
}
# All URIs that don't fit the patterns above redirect to the equivalent URI on my Ghost blog domain
handle /* {
redir https://sij.law{uri} permanent
}
tls {
dns cloudflare {env.CLOUDFLARE_API_TOKEN}
}
}

1
Extras/Caddyfile/README.md Normal file
View file

@ -0,0 +1 @@
This is a sample Caddyfile for a load-balancing reverse-proxy setup with HTTPS, Cloudflare DNS challenge handling, API key handling (and specified endpoints exempt from key requirement), and a second domain with special handling for certain endpoints (e.g. /s for the URL shortener, /pgp for public PGP key)

63
Extras/GPS.py Pythonista Script (location tracking)/GPS.py Normal file
View file

@ -0,0 +1,63 @@
import location
import time
import json
import os
import sys
from datetime import datetime, timezone
def get_current_location():
location.start_updates()
time.sleep(1) # Give it a moment to get an accurate fix
current_location = location.get_location()
location.stop_updates()
elevation = current_location['altitude']
latitude = current_location['latitude']
longitude = current_location['longitude']
current_time = datetime.now(timezone.utc)
timestamp = current_time.isoformat()
return {
'latitude': latitude,
'longitude': longitude,
'elevation': elevation,
'datetime': timestamp
}
def save_location_data(data, context, filename='location_log.json'):
if os.path.exists(filename):
with open(filename, 'r') as f:
existing_data = json.load(f)
else:
existing_data = []
data['context'] = context
existing_data.append(data)
with open(filename, 'w') as f:
json.dump(existing_data, f, indent=4)
if len(sys.argv) > 1:
try:
context = json.loads(sys.argv[1])
context.setdefault('action', 'manual')
context.setdefault('device_type', 'Pythonista')
context.setdefault('device_model', None)
context.setdefault('device_name', None)
context.setdefault('device_os', None)
except json.JSONDecodeError:
print("Error: The provided argument is not a valid JSON.")
sys.exit(1)
else:
context = {
'action': 'manual',
'device_type': 'Pythonista',
'device_model': None,
'device_name': None,
'device_os': None
}
location_data = get_current_location()
save_location_data(location_data, context)
print(f"Location data: {location_data} with context '{context}' saved locally.")
time.sleep(5)

1
Extras/GPS.py Pythonista Script (location tracking)/README.md Normal file
View file

@ -0,0 +1 @@
These two companion scripts are designed for use with Pythonista on iOS devices. GPS.py records and saves your latitude and longitude to a local file, and uploadGPS.py uploads it to your sijapi instance (assuming you've entered a valid URL and matching API key for your GLOBAL_API_KEY entered when configuring sijapi.

1
Extras/GPS.py Pythonista Script (location tracking)/uploadGPS.py Normal file
View file

@ -0,0 +1 @@
how

25
sijapi/__main__.py
View file

@ -117,28 +117,30 @@ async def http_exception_handler(request: Request, exc: HTTPException):
err(f"Request: {request.method} {request.url}")
return JSONResponse(status_code=exc.status_code, content={"detail": exc.detail})
@app.middleware("http")
async def handle_exception_middleware(request: Request, call_next):
try:
response = await call_next(request)
except RuntimeError as exc:
if str(exc) == "Response content longer than Content-Length":
# Update the Content-Length header to match the actual response content length
response.headers["Content-Length"] = str(len(response.body))
else:
raise
return response
return response
except Exception as exc:
err(f"Unhandled exception in request: {request.method} {request.url}")
err(f"Exception: {str(exc)}")
err(f"Traceback: {traceback.format_exc()}")
return JSONResponse(
status_code=500,
content={"detail": "Internal Server Error"}
)
@app.post("/sync/pull")
async def pull_changes():
info(f"Received request to /sync/pull")
try:
await API.add_primary_keys_to_local_tables()
await API.add_primary_keys_to_remote_tables()
try:
source = await API.get_most_recent_source()
if source:
# Pull changes from the source
total_changes = await API.pull_changes(source)
@ -156,9 +158,12 @@ async def pull_changes():
})
except Exception as e:
err(f"Error during pull: {str(e)}")
err(f"Error in /sync/pull: {str(e)}")
err(f"Traceback: {traceback.format_exc()}")
raise HTTPException(status_code=500, detail=f"Error during pull: {str(e)}")
finally:
info(f"Finished processing /sync/pull request")
except Exception as e:
err(f"Error while ensuring primary keys to tables: {str(e)}")