Auto-update: Tue Aug 6 19:55:48 PDT 2024

'/Users/sij/.private/archive/'/"2024"/"2024-08 August"/"2024-08-06 Tuesday"/"2024-08-06" 170424 sijsijapi Multimodal API for integrating Obsidian Tailscale Cloudflare Ollama r2r ElevenLabs xtts_v2 whisper_cpp ComfyUI Instagram Postgres GPSGIS Courtlis.md

@@ -1,11 +0,0 @@
----
-title: "sij/sijapi: Multimodal API for integrating Obsidian, Tailscale, Cloudflare, Ollama, r2r, ElevenLabs, xtts_v2, whisper_cpp, ComfyUI, Instagram, Postgres, GPS/GIS, Courtlistener, IMAP emails, MS365 & Apple Calendars, Timing, in a way that may not make sense to many except sij himself. - sijapi - sij's git"
-added: Aug 06, 2024 at 18:33
-url: "https://git.sij.ai/sij/sijapi"
-date: "2024-08-06"
----
-
-# sij/sijapi: Multimodal API for integrating Obsidian, Tailscale, Cloudflare, Ollama, r2r, ElevenLabs, xtts_v2, whisper_cpp, ComfyUI, Instagram, Postgres, GPS/GIS, Courtlistener, IMAP emails, MS365 & Apple Calendars, Timing, in a way that may not make sense to many except sij himself. - sijapi - sij's git
-
-Clipped from [https://git.sij.ai/sij/sijapi](https://git.sij.ai/sij/sijapi) on Aug 06, 2024 at 18:33
-

Extras/Archivist.js UserScript (web archiving)/archivist.js

@@ -0,0 +1,119 @@
+// ==UserScript==
+// @name         Archivist
+// @version      0.5
+// @description  archivist userscript posts to sij.ai/archive
+// @author       sij
+// @match        *://*/*
+// @grant        GM_xmlhttpRequest
+// ==/UserScript==
+
+(function () {
+  "use strict";
+
+  // Function to check if the URL is likely an ad, tracker, or unwanted resource
+  function isUnwantedURL(url) {
+    const unwantedPatterns = [
+      /doubleclick\.net/,
+      /googlesyndication\.com/,
+      /adservice\./,
+      /analytics\./,
+      /tracker\./,
+      /pixel\./,
+      /ad\d*\./,
+      /\.ad\./,
+      /ads\./,
+      /\/ads\//,
+      /url=http/,
+      /safeframe/,
+      /adsystem/,
+      /adserver/,
+      /adnetwork/,
+      /sync\./,
+      /beacon\./,
+      /optimizely/,
+      /outbrain/,
+      /widgets\./,
+      /cdn\./,
+      /pixel\?/,
+      /recaptcha/,
+      /accounts\.google\.com\/gsi/,
+      /imasdk\.googleapis\.com/,
+      /amplify-imp/,
+      /zemanta/,
+      /monitor\.html/,
+      /widgetMonitor/,
+      /nanoWidget/,
+      /client_storage/,
+    ];
+    return unwantedPatterns.some((pattern) => pattern.test(url));
+  }
+
+  // Function to archive the page
+  function archivePage() {
+    var currentUrl = window.location.href;
+
+    if (isUnwantedURL(currentUrl)) {
+      console.log("Skipping unwanted URL:", currentUrl);
+      return;
+    }
+
+    var data = new URLSearchParams({
+      title: document.title,
+      url: currentUrl,
+      referrer: document.referrer || "",
+      width: window.innerWidth ? window.innerWidth.toString() : "",
+      encoding: document.characterSet,
+      source: document.documentElement.outerHTML,
+    });
+
+    GM_xmlhttpRequest({
+      method: "POST",
+      url: "https://api.sij.ai/archive?api_key=sk-NhrtQwCHNdK5sRZC",
+      headers: {
+        "Content-Type": "application/x-www-form-urlencoded",
+        Authorization: "bearer sk-NhrtQwCHNdK5sRZC",
+      },
+      data: data.toString(),
+      onload: function (response) {
+        console.log("Archive request sent for:", currentUrl);
+      },
+      onerror: function (error) {
+        console.error("Error sending archive request:", error);
+      },
+    });
+  }
+
+  // Debounce function to limit how often archivePage can be called
+  function debounce(func, wait) {
+    let timeout;
+    return function executedFunction(...args) {
+      const later = () => {
+        clearTimeout(timeout);
+        func(...args);
+      };
+      clearTimeout(timeout);
+      timeout = setTimeout(later, wait);
+    };
+  }
+
+  // Debounced version of archivePage
+  const debouncedArchivePage = debounce(archivePage, 2000);
+
+  // Listen for navigation events
+  window.addEventListener("popstate", debouncedArchivePage);
+
+  // Intercept link clicks
+  document.addEventListener(
+    "click",
+    function (e) {
+      var link = e.target.closest("a");
+      if (link && !isUnwantedURL(link.href)) {
+        setTimeout(debouncedArchivePage, 1000); // Delay to allow page to load
+      }
+    },
+    true
+  );
+
+  // Initial page load
+  setTimeout(archivePage, 5000);
+})();

Extras/Caddyfile/README.md

@@ -0,0 +1 @@
+This is a sample Caddyfile for a load-balancing reverse-proxy setup with HTTPS, Cloudflare DNS challenge handling, API key handling (and specified endpoints exempt from key requirement), and a second domain with special handling for certain endpoints (e.g. /s for the URL shortener, /pgp for public PGP key)

Extras/webclipper/archivist.js

@@ -1,41 +0,0 @@
-// ==UserScript==
-// @name         Archivist
-// @version      0.1
-// @description  archivist userscript posts to sij.ai/clip
-// @author       sij.ai
-// @match        *://*/*
-// @grant        GM_xmlhttpRequest
-// ==/UserScript==
-
-(function() {
-    'use strict';
-    
-    window.addEventListener('load', function() {
-        setTimeout(function() {
-            var data = new URLSearchParams({
-                title: document.title,
-                url: window.location.href,
-                referrer: document.referrer || '',
-                width: window.innerWidth ? window.innerWidth.toString() : '',
-                encoding: document.characterSet,
-                source: document.documentElement.outerHTML
-            });
-
-            GM_xmlhttpRequest({
-                method: 'POST',
-                url: 'https://!{!{ YOUR DOMAIN HERE }!}!/clip?api_key=!{!{ YOUR API KEY HERE }!}!',
-                headers: {
-                    'Content-Type': 'application/x-www-form-urlencoded',
-                    'Authorization': 'bearer !{!{ GLOBAL_API_KEY HERE }!}!'
-                },
-                data: data.toString(),
-                onload: function(response) {
-                    console.log('Data sent to server');
-                },
-                onerror: function(error) {
-                    console.error('Error sending data:', error);
-                }
-            });
-        }, 5000);
-    });
-})();

sijapi/__main__.py

@@ -117,28 +117,30 @@ async def http_exception_handler(request: Request, exc: HTTPException):
     err(f"Request: {request.method} {request.url}")
     return JSONResponse(status_code=exc.status_code, content={"detail": exc.detail})
 
+
 @app.middleware("http")
 async def handle_exception_middleware(request: Request, call_next):
     try:
         response = await call_next(request)
-    except RuntimeError as exc:
+        return response
-        if str(exc) == "Response content longer than Content-Length":
+    except Exception as exc:
-            # Update the Content-Length header to match the actual response content length
+        err(f"Unhandled exception in request: {request.method} {request.url}")
-            response.headers["Content-Length"] = str(len(response.body))
+        err(f"Exception: {str(exc)}")
-        else:
+        err(f"Traceback: {traceback.format_exc()}")
-            raise
+        return JSONResponse(
-    return response
+            status_code=500,
+            content={"detail": "Internal Server Error"}
+        )
 
 
 @app.post("/sync/pull")
 async def pull_changes():
+    info(f"Received request to /sync/pull")
     try:
         await API.add_primary_keys_to_local_tables()
         await API.add_primary_keys_to_remote_tables()
         try:
-
             source = await API.get_most_recent_source()
-
             if source:
                 # Pull changes from the source
                 total_changes = await API.pull_changes(source)
@@ -156,10 +158,13 @@ async def pull_changes():
                 })
 
         except Exception as e:
-            err(f"Error during pull: {str(e)}")
+            err(f"Error in /sync/pull: {str(e)}")
             err(f"Traceback: {traceback.format_exc()}")
             raise HTTPException(status_code=500, detail=f"Error during pull: {str(e)}")
             
+        finally:
+            info(f"Finished processing /sync/pull request")
+
     except Exception as e:
             err(f"Error while ensuring primary keys to tables: {str(e)}")
             err(f"Traceback: {traceback.format_exc()}")