Auto-update: Sun Jun 30 15:51:51 PDT 2024

.gitignore

@@ -5,10 +5,10 @@ sijapi/data/geocoder/
 sijapi/data/courtlistener/
 sijapi/data/tts/
 sijapi/data/db/
-sijapi/data/sd/workflows/private
+sijapi/data/img/workflows/private
 sijapi/data/*.pbf
 sijapi/data/geonames.txt
-sijapi/data/sd/images/
+sijapi/data/img/images/
 sijapi/config/*.yaml
 sijapi/config/O365/
 sijapi/local_only/

sijapi/__init__.py

@@ -32,7 +32,7 @@ MAX_CPU_CORES = min(int(os.getenv("MAX_CPU_CORES", int(multiprocessing.cpu_count
 DB = Database.from_env()
 
 News = Configuration.load('news', 'secrets')
-SD = Configuration.load('sd', 'secrets')
+IMG = Configuration.load('img', 'secrets')
 
 ### Directories & general paths
 ROUTER_DIR = BASE_DIR / "routers"
@@ -100,15 +100,15 @@ SUMMARY_INSTRUCT_TTS = os.getenv('SUMMARY_INSTRUCT_TTS', "You are an AI assistan
 
 
 ### Stable diffusion
-SD_IMAGE_DIR = DATA_DIR / "sd" / "images"
-os.makedirs(SD_IMAGE_DIR, exist_ok=True)
-SD_WORKFLOWS_DIR = DATA_DIR / "sd" / "workflows"
-os.makedirs(SD_WORKFLOWS_DIR, exist_ok=True)
+IMG_DIR = DATA_DIR / "img" / "images"
+os.makedirs(IMG_DIR, exist_ok=True)
+IMG_WORKFLOWS_DIR = DATA_DIR / "img" / "workflows"
+os.makedirs(IMG_WORKFLOWS_DIR, exist_ok=True)
 COMFYUI_URL = os.getenv('COMFYUI_URL', "http://localhost:8188")
 COMFYUI_DIR = Path(os.getenv('COMFYUI_DIR'))
 COMFYUI_OUTPUT_DIR = COMFYUI_DIR / 'output'
 COMFYUI_LAUNCH_CMD = os.getenv('COMFYUI_LAUNCH_CMD', 'mamba activate comfyui && python main.py')
-SD_CONFIG_PATH = CONFIG_DIR / 'sd.yaml'
+IMG_CONFIG_PATH = CONFIG_DIR / 'img.yaml'
 
 ### ASR
 ASR_DIR = DATA_DIR / "asr"

sijapi/config/.env-example

@@ -56,7 +56,7 @@
 #─── notes: ──────────────────────────────────────────────────────────────────────
 #                                                                                   
 #   HOST_NET† and HOST_PORT comprise HOST and determine the ip and port the server binds to.
-#   API.URL is used to assemble URLs, e.g. in the MS authentication flow and for serving images generated on the sd router.
+#   API.URL is used to assemble URLs, e.g. in the MS authentication flow and for serving images generated on the img router.
 #   API.URL should match the base URL used to access sijapi sans endpoint, e.g. http://localhost:4444 or https://api.sij.ai 
 #                            
 #   † Take care here! Please ensure you understand the implications of setting HOST_NET to anything besides 127.0.0.1, and configure your firewall and router appropriately if you do. Setting HOST_NET to 0.0.0.0, for instance, opens sijapi to any device the server running it is accessible to — including potentially frightening internet randos (depending how your firewall, router, and NAT are configured). 
@@ -94,7 +94,7 @@ TRUSTED_SUBNETS=127.0.0.1/32,10.13.37.0/24,100.64.64.0/24
 #                                 ──────────                                       
 #                                                                                  
 #─── router selection: ────────────────────────────────────────────────────────────
-ROUTERS=asr,cal,cf,email,health,llm,loc,note,rag,sd,serve,time,tts,weather
+ROUTERS=asr,cal,cf,email,health,llm,loc,note,rag,img,serve,time,tts,weather
 UNLOADED=ig
 #─── notes: ──────────────────────────────────────────────────────────────────────
 #                                                                                  
@@ -134,7 +134,7 @@ UNLOADED=ig
 #                                                                                
 #         ig:  requires an Instagram account, with credentials and other settings
 #              configured separately in the ig_config.json file; relies heavily
-#              on the llm and sd routers which have their own dependencies.   
+#              on the llm and img routers which have their own dependencies.   
 #                                                                                  
 #        loc:  some endpoints work as is, but the core location tracking 
 #              functionality requires Postgresql + PostGIS extension and are
@@ -146,11 +146,11 @@ UNLOADED=ig
 #       note:  designed for use with Obsidian plus the Daily Notes and Tasks 
 #              core extensions; and the Admonitions, Banners, Icons (with the 
 #              Lucide pack), and Make.md community extensions. Moreover `notes`
-#              relies heavily on the cal, llm, loc, sd, summarize, time, loc, 
+#              relies heavily on the cal, llm, loc, img, summarize, time, loc, 
 #              and weather routers and accordingly on the external 
 #              dependencies of each.
 #                                                                                  
-#         sd:  requires ComfyUI plus any modules and StableDiffusion models 
+#         img:  requires ComfyUI plus any modules and StableDiffusion models 
 #              set in sd_config and individual workflow .json files.        
 #                                                                                 
 #  summarize:  relies on the llm router and thus requires ollama.
@@ -353,7 +353,7 @@ DAY_SHORT_FMT="%Y-%m-%d"
 #   
 #   DEFAULT_LLM is self-explanatory; DEFAULT_VISION is used for image recognition within
 #   a multimodal chat context, such as on the ig module for generating intelligible      
-#   comments to Instagram posts, or more realistic captions for sd-generated images.     
+#   comments to Instagram posts, or more realistic captions for img-generated images.     
 #   
 #   Note it's possible to specify a separate model for general purposes and for 
 #   summarization tasks. The other SUMMARY_ variables call for some explanation, 

sijapi/config/api.yaml-example

@@ -27,7 +27,7 @@ MODULES:
   news: on
   note: on
   rag: off
-  sd: on
+  img: on
   serve: on
   time: on
   tts: on

sijapi/helpers/elevenlabs_history_ids_20240630_131617.json

@@ -0,0 +1,104 @@
+{
+  "history_item_ids": [
+    "ncRYNd0Xef4LiUE74VjP",
+    "13pQLDAPYGIATwW1ySL5",
+    "dhsQNAYTWpcwo1X6rixf",
+    "V7wUip1NJuWAUw26sePF",
+    "mOYMa5lcI7wRHddIQTSa",
+    "mP97iOpA4oG7pwUBthq4",
+    "WTU5nsX6qZCYxLyoT5hq",
+    "15DPGnBgjr74KT3TMbK4",
+    "aCyBS1zoaweVjUoPf2TF",
+    "J8SUMQqZPtoy3Cgdhi3J",
+    "qKHaaJHfqh2je60Wmadb",
+    "2PaugQJ8c4rY44JGlaO5",
+    "TwzxcmYjo6XNebbMabcd",
+    "xdEK7rYq9UofOlkr565b",
+    "wik4jYd97aGMLgttTjC9",
+    "7oXn2yH7gdyhi6sEoWKd",
+    "jv8aZFiVe8gPMrAOBcNT",
+    "B2BctCDkCtLDxEMMBu9z",
+    "4KFO77NHDruNQvXIykwp",
+    "d033NizZaNZPc45fvxCO",
+    "yBKxOxfzsjpZYOFzoIM7",
+    "oEihKwMLWgvvoTLGx4yF",
+    "Q3guBm4hGml0KPAWKl7t",
+    "jaojY1gSafQmqshR48oT",
+    "yqGDMfcceaoceFEEurqa",
+    "oLdnyUp7plGrUMRVQ8Cf",
+    "FZAGCGosYEGMf8GCRFaA",
+    "TrWnXRdGkiH0K9kgwFiS",
+    "th16OEbg3u0XHslT9A33",
+    "856BAsn6dnzF7HeqGPfK",
+    "KjLoAfDXVBqR9s39T25j",
+    "uHQQJMMOfOxPAhEYQXLl",
+    "HO8WCIhkkI7AxwkU5MC6",
+    "9nxdesHWTRLCOd6YgWe9",
+    "tmx5tlIQ7hdSTgJt16P2",
+    "M9JN0YcBuCF6LhnqKN66",
+    "M9xkP4ecn0LIi7mQOfU6",
+    "CNtJgh52Ykh9ZqEppZeH",
+    "lgobcoiqmtWfbXkhEwbE",
+    "nr9jxnsE4DnwmTwCaHqC",
+    "Rnzo03tcyBqGPdmHemCb",
+    "X3YVGp7yf9GLgZ7WOuSU",
+    "wL3bkqxR9xqeFTvkJpSI",
+    "wNx3XDgFLTjVbMyGrIAO",
+    "rb0jj1ywBetmdvve5qIL",
+    "WdNnqvNswXeh6JFoaRSS",
+    "WT2ViyerKpodYmHDHhCw",
+    "OvhIRehXNwx7xMJHuTd7",
+    "EQb1iZtsADxJ0GxLJzEK",
+    "WXVfBJYoYGB7S61VyETD",
+    "q0q3Di1YJKF07dOhoa7E",
+    "a2XBIUPa68UiiKlzwFnG",
+    "YBuD7KsUpz8jxc5ItZcF",
+    "KdoucRVCVQGRVQ8Di9Ih",
+    "CkmDny98GEdfGuj2kaAx",
+    "R0R2p8luRZL7wwPtDilw",
+    "awvztgQnuaquK0dTpIuH",
+    "3ZPN0nJo8UQZYhFhoIOK",
+    "RJJeTkjYIgdv1ZoXXAax",
+    "ppxUNzWHAQafsM6OvEUE",
+    "f2VBm7yE7qmnjdS9CbYz",
+    "SZIMwz2T5ZAhTxTDBFol",
+    "YjC91PRgnQbAcdPhnWqU",
+    "fDTV7n8f6QK5yCwLkBwg",
+    "KbPpWUuiLPADj9H3OlvG",
+    "DIuqVoAg7lLxpvFBip84",
+    "pEwFAKMLGWUMHqfljJSq",
+    "9wwl7UbsgeKqrk8kNZin",
+    "2uLvjJgcZDiY9dqB8JlP",
+    "U5f1qZQM08t2YzJqEmxK",
+    "gnwn7QIhrCXRAGNddZ1H",
+    "g5nGEIHirFzKstdrGI1h",
+    "CQWH5dGSeS38VC4X4yg7",
+    "C5YGjhJPrTkVOpxIOHdj",
+    "YLbtnf1pSb9Ra7wgFHiF",
+    "qNLgNSvMr4VSoisKS9qj",
+    "Bq2ALvQVsj9L2wMpUvYO",
+    "gi0yTXLZLMhUKeKcalWc",
+    "3JQN9UbCsqj9ggi5sCkq",
+    "oPflJoA9kqBzjlmWY6zL",
+    "0kUZFgtZdqgdUBXFsXs9",
+    "aFTi7XdjR8W52ThmFpgc",
+    "pgIfjcy2UvKggfqJ1aNx",
+    "r0VguLaqnxTL9jza9H4y",
+    "444ehr4RtqgU1xjhhTLo",
+    "pEuzoznVDaQRBhIA9VTy",
+    "T9hdW9eJkEqDmOsSUoeY",
+    "wJjHbGzoWiKKOIGmf82T",
+    "kij4uMmkUlsSDu2zSH1k",
+    "oWt5rns196JsKIYPyrBS",
+    "SJ1m9mSOGOLIhkMgA8kq",
+    "kAaqe0ATrYtkifmZLOE5",
+    "O2Pvz7CP5rfyNvzFSDmy",
+    "w1rb8qN5nohVUovC0XAx",
+    "njFs4I4F7rtd9I6fEn6x",
+    "miFrp9GBm3MsHO03Z4eY",
+    "5DJywiPsfeVP9hFdqRhd",
+    "mUephoXhk5QdWrOfr9Xr",
+    "tDDiW3Yp0BptZ2wBv21A",
+    "YpX06liXWHquUVYFlKYa"
+  ]
+}

sijapi/helpers/upscaler.py

@@ -0,0 +1,22 @@
+from aura_sr import AuraSR
+from PIL import Image
+
+aura_sr = AuraSR.from_pretrained("fal-ai/AuraSR")
+
+def load_image_from_path(file_path):
+    # Open image file
+    return Image.open(file_path)
+
+def upscale_and_save(original_path):
+    # load the image from the path
+    original_image = load_image_from_path(original_path)
+    
+    # upscale the image using the pretrained model
+    upscaled_image = aura_sr.upscale_4x(original_image)
+    
+    # save the upscaled image back to the original path
+    # Overwrite the original image
+    upscaled_image.save(original_path)
+
+# Now to use the function, provide the image path
+upscale_and_save("/Users/sij/workshop/sijapi/sijapi/testbed/API__00482_ 2.png")

sijapi/routers/email.py

@@ -21,7 +21,7 @@ import yaml
 from typing import List, Dict, Optional, Set
 from datetime import datetime as dt_datetime
 from sijapi import L, PODCAST_DIR, DEFAULT_VOICE, EMAIL_CONFIG, EMAIL_LOGS
-from sijapi.routers import loc, tts, llm, sd
+from sijapi.routers import img, loc, tts, llm
 from sijapi.utilities import clean_text, assemble_journal_path, extract_text, prefix_lines
 from sijapi.classes import EmailAccount, IMAPConfig, SMTPConfig, IncomingEmail, EmailContact, AutoResponder
 from sijapi.classes import EmailAccount
@@ -302,7 +302,7 @@ async def autorespond_single_email(message, uid_str: str, account: EmailAccount,
         if response_body:
             subject = f"Re: {this_email.subject}"
             # add back scene=profile.image_scene,  to workflow call
-            jpg_path = await sd.workflow(profile.image_prompt, earlyout=False, downscale_to_fit=True) if profile.image_prompt else None
+            jpg_path = await img.workflow(profile.image_prompt, earlyout=False, downscale_to_fit=True) if profile.image_prompt else None
             success = await send_response(this_email.sender, subject, response_body, profile, jpg_path)
             if success:
                 L.WARN(f"Auto-responded to email: {this_email.subject}")

sijapi/routers/ig.py

@@ -32,7 +32,7 @@ from urllib.parse import urlparse
 from instagrapi.exceptions import LoginRequired as ClientLoginRequiredError
 import json
 from ollama import Client as oLlama
-from sd import sd
+from sijapi.routers.img import img
 from dotenv import load_dotenv
 from sijapi import L, COMFYUI_DIR
 

sijapi/routers/img.py

@@ -32,12 +32,12 @@ import shutil
 from sijapi.routers.llm import query_ollama
 from sijapi import API, L, COMFYUI_URL, COMFYUI_OUTPUT_DIR, SD_CONFIG_PATH, SD_IMAGE_DIR, SD_WORKFLOWS_DIR
 
-sd = APIRouter()
+img = APIRouter()
 
 CLIENT_ID = str(uuid.uuid4())
 
-@sd.post("/sd")
-@sd.post("/v1/images/generations")
+@img.post("/img")
+@img.post("/v1/images/generations")
 async def sd_endpoint(request: Request):
     request_data = await request.json()
     prompt = request_data.get("prompt")
@@ -54,8 +54,8 @@ async def sd_endpoint(request: Request):
     else:
         return JSONResponse({"image_url": image_path})
     
-@sd.get("/sd")
-@sd.get("/v1/images/generations")
+@img.get("/img")
+@img.get("/v1/images/generations")
 async def sd_endpoint(
     request: Request,
     prompt: str = Query(..., description="The prompt for image generation"),
@@ -252,8 +252,9 @@ def get_matching_scene(prompt):
         count = sum(1 for trigger in sc['triggers'] if trigger in prompt_lower)
         if count > max_count:
             max_count = count
-            L.DEBUG(f"Found better-matching scene: the prompt contains {max_count} words that match triggers for {scene_data.get('name')}!")
             scene_data = sc
+            if scene_data:
+                L.DEBUG(f"Found better-matching scene: the prompt contains {max_count} words that match triggers for {scene_data.get('name')}!")
     if scene_data:
         return scene_data
     else:
@@ -333,7 +334,7 @@ async def ensure_comfy(retries: int = 4, timeout: float = 6.0):
 
 
 
-@sd.get("/image/{prompt_id}")
+@img.get("/image/{prompt_id}")
 async def get_image_status(prompt_id: str):
     status_data = await poll_status(prompt_id)
     save_image_key = None
@@ -349,7 +350,7 @@ async def get_image_status(prompt_id: str):
     else:
         return JSONResponse(content={"status": "Processing", "details": status_data}, status_code=202)
 
-@sd.get("/image-status/{prompt_id}")
+@img.get("/image-status/{prompt_id}")
 async def get_image_processing_status(prompt_id: str):
     try:
         status_data = await poll_status(prompt_id)
@@ -359,7 +360,7 @@ async def get_image_processing_status(prompt_id: str):
 
 
 
-@sd.options("/v1/images/generations", tags=["generations"])
+@img.options("/v1/images/generations", tags=["generations"])
 async def get_generation_options():
     return {
         "model": {

sijapi/routers/note.py

@@ -1,5 +1,5 @@
 '''
-Manages an Obsidian vault, in particular daily notes, using information and functionality drawn from the other routers, primarily calendar, email, ig, llm, rag, sd, serve, time, tts, and weather.
+Manages an Obsidian vault, in particular daily notes, using information and functionality drawn from the other routers, primarily calendar, email, ig, llm, rag, img, serve, time, tts, and weather.
 '''
 from fastapi import APIRouter, BackgroundTasks, File, UploadFile, Form, HTTPException, Response, Query, Path as FastAPIPath
 from fastapi.responses import JSONResponse, PlainTextResponse
@@ -14,7 +14,7 @@ from fastapi import HTTPException, status
 from pathlib import Path
 from fastapi import APIRouter, Query, HTTPException
 from sijapi import API, L, OBSIDIAN_VAULT_DIR, OBSIDIAN_RESOURCES_DIR, OBSIDIAN_BANNER_SCENE, DEFAULT_11L_VOICE, DEFAULT_VOICE, GEO
-from sijapi.routers import cal, loc, tts, llm, time, sd, weather, asr
+from sijapi.routers import cal, img, loc, tts, llm, time, weather, asr
 from sijapi.utilities import assemble_journal_path, assemble_archive_path, convert_to_12_hour_format, sanitize_filename, convert_degrees_to_cardinal, check_file_name, HOURLY_COLUMNS_MAPPING
 from sijapi.classes import Location
 
@@ -270,7 +270,7 @@ async def generate_banner(dt, location: Location = None, forecast: str = None, m
 
     prompt = await generate_context(date_time, location, forecast, mood, other_context)
     L.DEBUG(f"Prompt: {prompt}")
-    final_path = await sd.workflow(prompt, scene=OBSIDIAN_BANNER_SCENE, destination_path=destination_path)
+    final_path = await img.workflow(prompt, scene=OBSIDIAN_BANNER_SCENE, destination_path=destination_path)
     if not str(local_path) in str(final_path):
         L.INFO(f"Apparent mismatch between local path, {local_path}, and final_path, {final_path}")
     jpg_embed = f"\"![[{local_path}]]\""

sijapi/routers/serve.py

@@ -1,5 +1,5 @@
 '''
-Web server module. Used by other modules when serving static content is required, e.g. the sd image generation module. Also used to serve PUBLIC_KEY.
+Web server module. Used by other modules when serving static content is required, e.g. the img image generation module. Also used to serve PUBLIC_KEY.
 '''
 import os
 import io