From 0272b9fa82ab8dfbcc1e81f1012a518fad83cb89 Mon Sep 17 00:00:00 2001 From: sanj <67624670+iodrift@users.noreply.github.com> Date: Tue, 6 Aug 2024 19:55:48 -0700 Subject: [PATCH] Auto-update: Tue Aug 6 19:55:48 PDT 2024 --- ...yUI Instagram Postgres GPSGIS Courtlis.md" | 11 -- .../README.md | 0 .../archivist.js | 119 ++++++++++++++++++ Extras/{ => Caddyfile}/Caddyfile.example | 0 Extras/Caddyfile/README.md | 1 + .../GPS.py | 0 .../README.md | 0 .../uploadGPS.py | 0 Extras/webclipper/archivist.js | 41 ------ sijapi/__main__.py | 25 ++-- 10 files changed, 135 insertions(+), 62 deletions(-) delete mode 100644 "'/Users/sij/.private/archive/'/\"2024\"/\"2024-08 August\"/\"2024-08-06 Tuesday\"/\"2024-08-06\" 170424 sijsijapi Multimodal API for integrating Obsidian Tailscale Cloudflare Ollama r2r ElevenLabs xtts_v2 whisper_cpp ComfyUI Instagram Postgres GPSGIS Courtlis.md" rename Extras/{webclipper => Archivist.js UserScript (web archiving)}/README.md (100%) create mode 100644 Extras/Archivist.js UserScript (web archiving)/archivist.js rename Extras/{ => Caddyfile}/Caddyfile.example (100%) create mode 100644 Extras/Caddyfile/README.md rename Extras/{Pythonista => GPS.py Pythonista Script (location tracking)}/GPS.py (100%) rename Extras/{Pythonista => GPS.py Pythonista Script (location tracking)}/README.md (100%) rename Extras/{Pythonista => GPS.py Pythonista Script (location tracking)}/uploadGPS.py (100%) delete mode 100644 Extras/webclipper/archivist.js diff --git "a/'/Users/sij/.private/archive/'/\"2024\"/\"2024-08 August\"/\"2024-08-06 Tuesday\"/\"2024-08-06\" 170424 sijsijapi Multimodal API for integrating Obsidian Tailscale Cloudflare Ollama r2r ElevenLabs xtts_v2 whisper_cpp ComfyUI Instagram Postgres GPSGIS Courtlis.md" "b/'/Users/sij/.private/archive/'/\"2024\"/\"2024-08 August\"/\"2024-08-06 Tuesday\"/\"2024-08-06\" 170424 sijsijapi Multimodal API for integrating Obsidian Tailscale Cloudflare Ollama r2r ElevenLabs xtts_v2 whisper_cpp ComfyUI Instagram Postgres GPSGIS Courtlis.md" deleted file mode 100644 index 0f70d7a..0000000 --- "a/'/Users/sij/.private/archive/'/\"2024\"/\"2024-08 August\"/\"2024-08-06 Tuesday\"/\"2024-08-06\" 170424 sijsijapi Multimodal API for integrating Obsidian Tailscale Cloudflare Ollama r2r ElevenLabs xtts_v2 whisper_cpp ComfyUI Instagram Postgres GPSGIS Courtlis.md" +++ /dev/null @@ -1,11 +0,0 @@ ---- -title: "sij/sijapi: Multimodal API for integrating Obsidian, Tailscale, Cloudflare, Ollama, r2r, ElevenLabs, xtts_v2, whisper_cpp, ComfyUI, Instagram, Postgres, GPS/GIS, Courtlistener, IMAP emails, MS365 & Apple Calendars, Timing, in a way that may not make sense to many except sij himself. - sijapi - sij's git" -added: Aug 06, 2024 at 18:33 -url: "https://git.sij.ai/sij/sijapi" -date: "2024-08-06" ---- - -# sij/sijapi: Multimodal API for integrating Obsidian, Tailscale, Cloudflare, Ollama, r2r, ElevenLabs, xtts_v2, whisper_cpp, ComfyUI, Instagram, Postgres, GPS/GIS, Courtlistener, IMAP emails, MS365 & Apple Calendars, Timing, in a way that may not make sense to many except sij himself. - sijapi - sij's git - -Clipped from [https://git.sij.ai/sij/sijapi](https://git.sij.ai/sij/sijapi) on Aug 06, 2024 at 18:33 - diff --git a/Extras/webclipper/README.md b/Extras/Archivist.js UserScript (web archiving)/README.md similarity index 100% rename from Extras/webclipper/README.md rename to Extras/Archivist.js UserScript (web archiving)/README.md diff --git a/Extras/Archivist.js UserScript (web archiving)/archivist.js b/Extras/Archivist.js UserScript (web archiving)/archivist.js new file mode 100644 index 0000000..0e91e5b --- /dev/null +++ b/Extras/Archivist.js UserScript (web archiving)/archivist.js @@ -0,0 +1,119 @@ +// ==UserScript== +// @name Archivist +// @version 0.5 +// @description archivist userscript posts to sij.ai/archive +// @author sij +// @match *://*/* +// @grant GM_xmlhttpRequest +// ==/UserScript== + +(function () { + "use strict"; + + // Function to check if the URL is likely an ad, tracker, or unwanted resource + function isUnwantedURL(url) { + const unwantedPatterns = [ + /doubleclick\.net/, + /googlesyndication\.com/, + /adservice\./, + /analytics\./, + /tracker\./, + /pixel\./, + /ad\d*\./, + /\.ad\./, + /ads\./, + /\/ads\//, + /url=http/, + /safeframe/, + /adsystem/, + /adserver/, + /adnetwork/, + /sync\./, + /beacon\./, + /optimizely/, + /outbrain/, + /widgets\./, + /cdn\./, + /pixel\?/, + /recaptcha/, + /accounts\.google\.com\/gsi/, + /imasdk\.googleapis\.com/, + /amplify-imp/, + /zemanta/, + /monitor\.html/, + /widgetMonitor/, + /nanoWidget/, + /client_storage/, + ]; + return unwantedPatterns.some((pattern) => pattern.test(url)); + } + + // Function to archive the page + function archivePage() { + var currentUrl = window.location.href; + + if (isUnwantedURL(currentUrl)) { + console.log("Skipping unwanted URL:", currentUrl); + return; + } + + var data = new URLSearchParams({ + title: document.title, + url: currentUrl, + referrer: document.referrer || "", + width: window.innerWidth ? window.innerWidth.toString() : "", + encoding: document.characterSet, + source: document.documentElement.outerHTML, + }); + + GM_xmlhttpRequest({ + method: "POST", + url: "https://api.sij.ai/archive?api_key=sk-NhrtQwCHNdK5sRZC", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + Authorization: "bearer sk-NhrtQwCHNdK5sRZC", + }, + data: data.toString(), + onload: function (response) { + console.log("Archive request sent for:", currentUrl); + }, + onerror: function (error) { + console.error("Error sending archive request:", error); + }, + }); + } + + // Debounce function to limit how often archivePage can be called + function debounce(func, wait) { + let timeout; + return function executedFunction(...args) { + const later = () => { + clearTimeout(timeout); + func(...args); + }; + clearTimeout(timeout); + timeout = setTimeout(later, wait); + }; + } + + // Debounced version of archivePage + const debouncedArchivePage = debounce(archivePage, 2000); + + // Listen for navigation events + window.addEventListener("popstate", debouncedArchivePage); + + // Intercept link clicks + document.addEventListener( + "click", + function (e) { + var link = e.target.closest("a"); + if (link && !isUnwantedURL(link.href)) { + setTimeout(debouncedArchivePage, 1000); // Delay to allow page to load + } + }, + true + ); + + // Initial page load + setTimeout(archivePage, 5000); +})(); diff --git a/Extras/Caddyfile.example b/Extras/Caddyfile/Caddyfile.example similarity index 100% rename from Extras/Caddyfile.example rename to Extras/Caddyfile/Caddyfile.example diff --git a/Extras/Caddyfile/README.md b/Extras/Caddyfile/README.md new file mode 100644 index 0000000..256b8a8 --- /dev/null +++ b/Extras/Caddyfile/README.md @@ -0,0 +1 @@ +This is a sample Caddyfile for a load-balancing reverse-proxy setup with HTTPS, Cloudflare DNS challenge handling, API key handling (and specified endpoints exempt from key requirement), and a second domain with special handling for certain endpoints (e.g. /s for the URL shortener, /pgp for public PGP key) diff --git a/Extras/Pythonista/GPS.py b/Extras/GPS.py Pythonista Script (location tracking)/GPS.py similarity index 100% rename from Extras/Pythonista/GPS.py rename to Extras/GPS.py Pythonista Script (location tracking)/GPS.py diff --git a/Extras/Pythonista/README.md b/Extras/GPS.py Pythonista Script (location tracking)/README.md similarity index 100% rename from Extras/Pythonista/README.md rename to Extras/GPS.py Pythonista Script (location tracking)/README.md diff --git a/Extras/Pythonista/uploadGPS.py b/Extras/GPS.py Pythonista Script (location tracking)/uploadGPS.py similarity index 100% rename from Extras/Pythonista/uploadGPS.py rename to Extras/GPS.py Pythonista Script (location tracking)/uploadGPS.py diff --git a/Extras/webclipper/archivist.js b/Extras/webclipper/archivist.js deleted file mode 100644 index 64ac8ca..0000000 --- a/Extras/webclipper/archivist.js +++ /dev/null @@ -1,41 +0,0 @@ -// ==UserScript== -// @name Archivist -// @version 0.1 -// @description archivist userscript posts to sij.ai/clip -// @author sij.ai -// @match *://*/* -// @grant GM_xmlhttpRequest -// ==/UserScript== - -(function() { - 'use strict'; - - window.addEventListener('load', function() { - setTimeout(function() { - var data = new URLSearchParams({ - title: document.title, - url: window.location.href, - referrer: document.referrer || '', - width: window.innerWidth ? window.innerWidth.toString() : '', - encoding: document.characterSet, - source: document.documentElement.outerHTML - }); - - GM_xmlhttpRequest({ - method: 'POST', - url: 'https://!{!{ YOUR DOMAIN HERE }!}!/clip?api_key=!{!{ YOUR API KEY HERE }!}!', - headers: { - 'Content-Type': 'application/x-www-form-urlencoded', - 'Authorization': 'bearer !{!{ GLOBAL_API_KEY HERE }!}!' - }, - data: data.toString(), - onload: function(response) { - console.log('Data sent to server'); - }, - onerror: function(error) { - console.error('Error sending data:', error); - } - }); - }, 5000); - }); -})(); diff --git a/sijapi/__main__.py b/sijapi/__main__.py index accac93..1eea4ec 100755 --- a/sijapi/__main__.py +++ b/sijapi/__main__.py @@ -117,28 +117,30 @@ async def http_exception_handler(request: Request, exc: HTTPException): err(f"Request: {request.method} {request.url}") return JSONResponse(status_code=exc.status_code, content={"detail": exc.detail}) + @app.middleware("http") async def handle_exception_middleware(request: Request, call_next): try: response = await call_next(request) - except RuntimeError as exc: - if str(exc) == "Response content longer than Content-Length": - # Update the Content-Length header to match the actual response content length - response.headers["Content-Length"] = str(len(response.body)) - else: - raise - return response + return response + except Exception as exc: + err(f"Unhandled exception in request: {request.method} {request.url}") + err(f"Exception: {str(exc)}") + err(f"Traceback: {traceback.format_exc()}") + return JSONResponse( + status_code=500, + content={"detail": "Internal Server Error"} + ) @app.post("/sync/pull") async def pull_changes(): + info(f"Received request to /sync/pull") try: await API.add_primary_keys_to_local_tables() await API.add_primary_keys_to_remote_tables() try: - source = await API.get_most_recent_source() - if source: # Pull changes from the source total_changes = await API.pull_changes(source) @@ -156,9 +158,12 @@ async def pull_changes(): }) except Exception as e: - err(f"Error during pull: {str(e)}") + err(f"Error in /sync/pull: {str(e)}") err(f"Traceback: {traceback.format_exc()}") raise HTTPException(status_code=500, detail=f"Error during pull: {str(e)}") + + finally: + info(f"Finished processing /sync/pull request") except Exception as e: err(f"Error while ensuring primary keys to tables: {str(e)}")