From 79d139312a3b5a64d4bc5af1677ce4b0894e2591 Mon Sep 17 00:00:00 2001 From: sanj <67624670+iodrift@users.noreply.github.com> Date: Sun, 30 Jun 2024 15:51:51 -0700 Subject: [PATCH] Auto-update: Sun Jun 30 15:51:51 PDT 2024 --- .gitignore | 4 +- sijapi/__init__.py | 12 +- sijapi/config/.env-example | 12 +- sijapi/config/api.yaml-example | 2 +- .../{sd.yaml-example => img.yaml-example} | 0 .../data/{sd => img}/workflows/default.json | 0 .../data/{sd => img}/workflows/landscape.json | 0 sijapi/data/{sd => img}/workflows/selfie.json | 0 .../data/{sd => img}/workflows/wallpaper.json | 0 ...levenlabs_history_ids_20240630_131617.json | 104 ++++++++++++++++++ sijapi/helpers/upscaler.py | 22 ++++ sijapi/routers/email.py | 4 +- sijapi/routers/ig.py | 2 +- sijapi/routers/{sd.py => img.py} | 19 ++-- sijapi/routers/note.py | 6 +- sijapi/routers/serve.py | 2 +- 16 files changed, 158 insertions(+), 31 deletions(-) rename sijapi/config/{sd.yaml-example => img.yaml-example} (100%) rename sijapi/data/{sd => img}/workflows/default.json (100%) rename sijapi/data/{sd => img}/workflows/landscape.json (100%) rename sijapi/data/{sd => img}/workflows/selfie.json (100%) rename sijapi/data/{sd => img}/workflows/wallpaper.json (100%) create mode 100644 sijapi/helpers/elevenlabs_history_ids_20240630_131617.json create mode 100644 sijapi/helpers/upscaler.py rename sijapi/routers/{sd.py => img.py} (97%) diff --git a/.gitignore b/.gitignore index 7b7ec4a..a0d77b1 100644 --- a/.gitignore +++ b/.gitignore @@ -5,10 +5,10 @@ sijapi/data/geocoder/ sijapi/data/courtlistener/ sijapi/data/tts/ sijapi/data/db/ -sijapi/data/sd/workflows/private +sijapi/data/img/workflows/private sijapi/data/*.pbf sijapi/data/geonames.txt -sijapi/data/sd/images/ +sijapi/data/img/images/ sijapi/config/*.yaml sijapi/config/O365/ sijapi/local_only/ diff --git a/sijapi/__init__.py b/sijapi/__init__.py index 6d5e91e..eeae5a9 100644 --- a/sijapi/__init__.py +++ b/sijapi/__init__.py @@ -32,7 +32,7 @@ MAX_CPU_CORES = min(int(os.getenv("MAX_CPU_CORES", int(multiprocessing.cpu_count DB = Database.from_env() News = Configuration.load('news', 'secrets') -SD = Configuration.load('sd', 'secrets') +IMG = Configuration.load('img', 'secrets') ### Directories & general paths ROUTER_DIR = BASE_DIR / "routers" @@ -100,15 +100,15 @@ SUMMARY_INSTRUCT_TTS = os.getenv('SUMMARY_INSTRUCT_TTS', "You are an AI assistan ### Stable diffusion -SD_IMAGE_DIR = DATA_DIR / "sd" / "images" -os.makedirs(SD_IMAGE_DIR, exist_ok=True) -SD_WORKFLOWS_DIR = DATA_DIR / "sd" / "workflows" -os.makedirs(SD_WORKFLOWS_DIR, exist_ok=True) +IMG_DIR = DATA_DIR / "img" / "images" +os.makedirs(IMG_DIR, exist_ok=True) +IMG_WORKFLOWS_DIR = DATA_DIR / "img" / "workflows" +os.makedirs(IMG_WORKFLOWS_DIR, exist_ok=True) COMFYUI_URL = os.getenv('COMFYUI_URL', "http://localhost:8188") COMFYUI_DIR = Path(os.getenv('COMFYUI_DIR')) COMFYUI_OUTPUT_DIR = COMFYUI_DIR / 'output' COMFYUI_LAUNCH_CMD = os.getenv('COMFYUI_LAUNCH_CMD', 'mamba activate comfyui && python main.py') -SD_CONFIG_PATH = CONFIG_DIR / 'sd.yaml' +IMG_CONFIG_PATH = CONFIG_DIR / 'img.yaml' ### ASR ASR_DIR = DATA_DIR / "asr" diff --git a/sijapi/config/.env-example b/sijapi/config/.env-example index 57e3e88..38a50b5 100644 --- a/sijapi/config/.env-example +++ b/sijapi/config/.env-example @@ -56,7 +56,7 @@ #─── notes: ────────────────────────────────────────────────────────────────────── # # HOST_NET† and HOST_PORT comprise HOST and determine the ip and port the server binds to. -# API.URL is used to assemble URLs, e.g. in the MS authentication flow and for serving images generated on the sd router. +# API.URL is used to assemble URLs, e.g. in the MS authentication flow and for serving images generated on the img router. # API.URL should match the base URL used to access sijapi sans endpoint, e.g. http://localhost:4444 or https://api.sij.ai # # † Take care here! Please ensure you understand the implications of setting HOST_NET to anything besides 127.0.0.1, and configure your firewall and router appropriately if you do. Setting HOST_NET to 0.0.0.0, for instance, opens sijapi to any device the server running it is accessible to — including potentially frightening internet randos (depending how your firewall, router, and NAT are configured). @@ -94,7 +94,7 @@ TRUSTED_SUBNETS=127.0.0.1/32,10.13.37.0/24,100.64.64.0/24 # ────────── # #─── router selection: ──────────────────────────────────────────────────────────── -ROUTERS=asr,cal,cf,email,health,llm,loc,note,rag,sd,serve,time,tts,weather +ROUTERS=asr,cal,cf,email,health,llm,loc,note,rag,img,serve,time,tts,weather UNLOADED=ig #─── notes: ────────────────────────────────────────────────────────────────────── # @@ -134,7 +134,7 @@ UNLOADED=ig # # ig: requires an Instagram account, with credentials and other settings # configured separately in the ig_config.json file; relies heavily -# on the llm and sd routers which have their own dependencies. +# on the llm and img routers which have their own dependencies. # # loc: some endpoints work as is, but the core location tracking # functionality requires Postgresql + PostGIS extension and are @@ -146,11 +146,11 @@ UNLOADED=ig # note: designed for use with Obsidian plus the Daily Notes and Tasks # core extensions; and the Admonitions, Banners, Icons (with the # Lucide pack), and Make.md community extensions. Moreover `notes` -# relies heavily on the cal, llm, loc, sd, summarize, time, loc, +# relies heavily on the cal, llm, loc, img, summarize, time, loc, # and weather routers and accordingly on the external # dependencies of each. # -# sd: requires ComfyUI plus any modules and StableDiffusion models +# img: requires ComfyUI plus any modules and StableDiffusion models # set in sd_config and individual workflow .json files. # # summarize: relies on the llm router and thus requires ollama. @@ -353,7 +353,7 @@ DAY_SHORT_FMT="%Y-%m-%d" # # DEFAULT_LLM is self-explanatory; DEFAULT_VISION is used for image recognition within # a multimodal chat context, such as on the ig module for generating intelligible -# comments to Instagram posts, or more realistic captions for sd-generated images. +# comments to Instagram posts, or more realistic captions for img-generated images. # # Note it's possible to specify a separate model for general purposes and for # summarization tasks. The other SUMMARY_ variables call for some explanation, diff --git a/sijapi/config/api.yaml-example b/sijapi/config/api.yaml-example index c48598a..62a9fea 100644 --- a/sijapi/config/api.yaml-example +++ b/sijapi/config/api.yaml-example @@ -27,7 +27,7 @@ MODULES: news: on note: on rag: off - sd: on + img: on serve: on time: on tts: on diff --git a/sijapi/config/sd.yaml-example b/sijapi/config/img.yaml-example similarity index 100% rename from sijapi/config/sd.yaml-example rename to sijapi/config/img.yaml-example diff --git a/sijapi/data/sd/workflows/default.json b/sijapi/data/img/workflows/default.json similarity index 100% rename from sijapi/data/sd/workflows/default.json rename to sijapi/data/img/workflows/default.json diff --git a/sijapi/data/sd/workflows/landscape.json b/sijapi/data/img/workflows/landscape.json similarity index 100% rename from sijapi/data/sd/workflows/landscape.json rename to sijapi/data/img/workflows/landscape.json diff --git a/sijapi/data/sd/workflows/selfie.json b/sijapi/data/img/workflows/selfie.json similarity index 100% rename from sijapi/data/sd/workflows/selfie.json rename to sijapi/data/img/workflows/selfie.json diff --git a/sijapi/data/sd/workflows/wallpaper.json b/sijapi/data/img/workflows/wallpaper.json similarity index 100% rename from sijapi/data/sd/workflows/wallpaper.json rename to sijapi/data/img/workflows/wallpaper.json diff --git a/sijapi/helpers/elevenlabs_history_ids_20240630_131617.json b/sijapi/helpers/elevenlabs_history_ids_20240630_131617.json new file mode 100644 index 0000000..d10f7da --- /dev/null +++ b/sijapi/helpers/elevenlabs_history_ids_20240630_131617.json @@ -0,0 +1,104 @@ +{ + "history_item_ids": [ + "ncRYNd0Xef4LiUE74VjP", + "13pQLDAPYGIATwW1ySL5", + "dhsQNAYTWpcwo1X6rixf", + "V7wUip1NJuWAUw26sePF", + "mOYMa5lcI7wRHddIQTSa", + "mP97iOpA4oG7pwUBthq4", + "WTU5nsX6qZCYxLyoT5hq", + "15DPGnBgjr74KT3TMbK4", + "aCyBS1zoaweVjUoPf2TF", + "J8SUMQqZPtoy3Cgdhi3J", + "qKHaaJHfqh2je60Wmadb", + "2PaugQJ8c4rY44JGlaO5", + "TwzxcmYjo6XNebbMabcd", + "xdEK7rYq9UofOlkr565b", + "wik4jYd97aGMLgttTjC9", + "7oXn2yH7gdyhi6sEoWKd", + "jv8aZFiVe8gPMrAOBcNT", + "B2BctCDkCtLDxEMMBu9z", + "4KFO77NHDruNQvXIykwp", + "d033NizZaNZPc45fvxCO", + "yBKxOxfzsjpZYOFzoIM7", + "oEihKwMLWgvvoTLGx4yF", + "Q3guBm4hGml0KPAWKl7t", + "jaojY1gSafQmqshR48oT", + "yqGDMfcceaoceFEEurqa", + "oLdnyUp7plGrUMRVQ8Cf", + "FZAGCGosYEGMf8GCRFaA", + "TrWnXRdGkiH0K9kgwFiS", + "th16OEbg3u0XHslT9A33", + "856BAsn6dnzF7HeqGPfK", + "KjLoAfDXVBqR9s39T25j", + "uHQQJMMOfOxPAhEYQXLl", + "HO8WCIhkkI7AxwkU5MC6", + "9nxdesHWTRLCOd6YgWe9", + "tmx5tlIQ7hdSTgJt16P2", + "M9JN0YcBuCF6LhnqKN66", + "M9xkP4ecn0LIi7mQOfU6", + "CNtJgh52Ykh9ZqEppZeH", + "lgobcoiqmtWfbXkhEwbE", + "nr9jxnsE4DnwmTwCaHqC", + "Rnzo03tcyBqGPdmHemCb", + "X3YVGp7yf9GLgZ7WOuSU", + "wL3bkqxR9xqeFTvkJpSI", + "wNx3XDgFLTjVbMyGrIAO", + "rb0jj1ywBetmdvve5qIL", + "WdNnqvNswXeh6JFoaRSS", + "WT2ViyerKpodYmHDHhCw", + "OvhIRehXNwx7xMJHuTd7", + "EQb1iZtsADxJ0GxLJzEK", + "WXVfBJYoYGB7S61VyETD", + "q0q3Di1YJKF07dOhoa7E", + "a2XBIUPa68UiiKlzwFnG", + "YBuD7KsUpz8jxc5ItZcF", + "KdoucRVCVQGRVQ8Di9Ih", + "CkmDny98GEdfGuj2kaAx", + "R0R2p8luRZL7wwPtDilw", + "awvztgQnuaquK0dTpIuH", + "3ZPN0nJo8UQZYhFhoIOK", + "RJJeTkjYIgdv1ZoXXAax", + "ppxUNzWHAQafsM6OvEUE", + "f2VBm7yE7qmnjdS9CbYz", + "SZIMwz2T5ZAhTxTDBFol", + "YjC91PRgnQbAcdPhnWqU", + "fDTV7n8f6QK5yCwLkBwg", + "KbPpWUuiLPADj9H3OlvG", + "DIuqVoAg7lLxpvFBip84", + "pEwFAKMLGWUMHqfljJSq", + "9wwl7UbsgeKqrk8kNZin", + "2uLvjJgcZDiY9dqB8JlP", + "U5f1qZQM08t2YzJqEmxK", + "gnwn7QIhrCXRAGNddZ1H", + "g5nGEIHirFzKstdrGI1h", + "CQWH5dGSeS38VC4X4yg7", + "C5YGjhJPrTkVOpxIOHdj", + "YLbtnf1pSb9Ra7wgFHiF", + "qNLgNSvMr4VSoisKS9qj", + "Bq2ALvQVsj9L2wMpUvYO", + "gi0yTXLZLMhUKeKcalWc", + "3JQN9UbCsqj9ggi5sCkq", + "oPflJoA9kqBzjlmWY6zL", + "0kUZFgtZdqgdUBXFsXs9", + "aFTi7XdjR8W52ThmFpgc", + "pgIfjcy2UvKggfqJ1aNx", + "r0VguLaqnxTL9jza9H4y", + "444ehr4RtqgU1xjhhTLo", + "pEuzoznVDaQRBhIA9VTy", + "T9hdW9eJkEqDmOsSUoeY", + "wJjHbGzoWiKKOIGmf82T", + "kij4uMmkUlsSDu2zSH1k", + "oWt5rns196JsKIYPyrBS", + "SJ1m9mSOGOLIhkMgA8kq", + "kAaqe0ATrYtkifmZLOE5", + "O2Pvz7CP5rfyNvzFSDmy", + "w1rb8qN5nohVUovC0XAx", + "njFs4I4F7rtd9I6fEn6x", + "miFrp9GBm3MsHO03Z4eY", + "5DJywiPsfeVP9hFdqRhd", + "mUephoXhk5QdWrOfr9Xr", + "tDDiW3Yp0BptZ2wBv21A", + "YpX06liXWHquUVYFlKYa" + ] +} \ No newline at end of file diff --git a/sijapi/helpers/upscaler.py b/sijapi/helpers/upscaler.py new file mode 100644 index 0000000..f6cb537 --- /dev/null +++ b/sijapi/helpers/upscaler.py @@ -0,0 +1,22 @@ +from aura_sr import AuraSR +from PIL import Image + +aura_sr = AuraSR.from_pretrained("fal-ai/AuraSR") + +def load_image_from_path(file_path): + # Open image file + return Image.open(file_path) + +def upscale_and_save(original_path): + # load the image from the path + original_image = load_image_from_path(original_path) + + # upscale the image using the pretrained model + upscaled_image = aura_sr.upscale_4x(original_image) + + # save the upscaled image back to the original path + # Overwrite the original image + upscaled_image.save(original_path) + +# Now to use the function, provide the image path +upscale_and_save("/Users/sij/workshop/sijapi/sijapi/testbed/API__00482_ 2.png") diff --git a/sijapi/routers/email.py b/sijapi/routers/email.py index 72ad9f9..6707e91 100644 --- a/sijapi/routers/email.py +++ b/sijapi/routers/email.py @@ -21,7 +21,7 @@ import yaml from typing import List, Dict, Optional, Set from datetime import datetime as dt_datetime from sijapi import L, PODCAST_DIR, DEFAULT_VOICE, EMAIL_CONFIG, EMAIL_LOGS -from sijapi.routers import loc, tts, llm, sd +from sijapi.routers import img, loc, tts, llm from sijapi.utilities import clean_text, assemble_journal_path, extract_text, prefix_lines from sijapi.classes import EmailAccount, IMAPConfig, SMTPConfig, IncomingEmail, EmailContact, AutoResponder from sijapi.classes import EmailAccount @@ -302,7 +302,7 @@ async def autorespond_single_email(message, uid_str: str, account: EmailAccount, if response_body: subject = f"Re: {this_email.subject}" # add back scene=profile.image_scene, to workflow call - jpg_path = await sd.workflow(profile.image_prompt, earlyout=False, downscale_to_fit=True) if profile.image_prompt else None + jpg_path = await img.workflow(profile.image_prompt, earlyout=False, downscale_to_fit=True) if profile.image_prompt else None success = await send_response(this_email.sender, subject, response_body, profile, jpg_path) if success: L.WARN(f"Auto-responded to email: {this_email.subject}") diff --git a/sijapi/routers/ig.py b/sijapi/routers/ig.py index 19fcc82..8dd1e48 100644 --- a/sijapi/routers/ig.py +++ b/sijapi/routers/ig.py @@ -32,7 +32,7 @@ from urllib.parse import urlparse from instagrapi.exceptions import LoginRequired as ClientLoginRequiredError import json from ollama import Client as oLlama -from sd import sd +from sijapi.routers.img import img from dotenv import load_dotenv from sijapi import L, COMFYUI_DIR diff --git a/sijapi/routers/sd.py b/sijapi/routers/img.py similarity index 97% rename from sijapi/routers/sd.py rename to sijapi/routers/img.py index b8fc1e8..e19ce14 100644 --- a/sijapi/routers/sd.py +++ b/sijapi/routers/img.py @@ -32,12 +32,12 @@ import shutil from sijapi.routers.llm import query_ollama from sijapi import API, L, COMFYUI_URL, COMFYUI_OUTPUT_DIR, SD_CONFIG_PATH, SD_IMAGE_DIR, SD_WORKFLOWS_DIR -sd = APIRouter() +img = APIRouter() CLIENT_ID = str(uuid.uuid4()) -@sd.post("/sd") -@sd.post("/v1/images/generations") +@img.post("/img") +@img.post("/v1/images/generations") async def sd_endpoint(request: Request): request_data = await request.json() prompt = request_data.get("prompt") @@ -54,8 +54,8 @@ async def sd_endpoint(request: Request): else: return JSONResponse({"image_url": image_path}) -@sd.get("/sd") -@sd.get("/v1/images/generations") +@img.get("/img") +@img.get("/v1/images/generations") async def sd_endpoint( request: Request, prompt: str = Query(..., description="The prompt for image generation"), @@ -252,8 +252,9 @@ def get_matching_scene(prompt): count = sum(1 for trigger in sc['triggers'] if trigger in prompt_lower) if count > max_count: max_count = count - L.DEBUG(f"Found better-matching scene: the prompt contains {max_count} words that match triggers for {scene_data.get('name')}!") scene_data = sc + if scene_data: + L.DEBUG(f"Found better-matching scene: the prompt contains {max_count} words that match triggers for {scene_data.get('name')}!") if scene_data: return scene_data else: @@ -333,7 +334,7 @@ async def ensure_comfy(retries: int = 4, timeout: float = 6.0): -@sd.get("/image/{prompt_id}") +@img.get("/image/{prompt_id}") async def get_image_status(prompt_id: str): status_data = await poll_status(prompt_id) save_image_key = None @@ -349,7 +350,7 @@ async def get_image_status(prompt_id: str): else: return JSONResponse(content={"status": "Processing", "details": status_data}, status_code=202) -@sd.get("/image-status/{prompt_id}") +@img.get("/image-status/{prompt_id}") async def get_image_processing_status(prompt_id: str): try: status_data = await poll_status(prompt_id) @@ -359,7 +360,7 @@ async def get_image_processing_status(prompt_id: str): -@sd.options("/v1/images/generations", tags=["generations"]) +@img.options("/v1/images/generations", tags=["generations"]) async def get_generation_options(): return { "model": { diff --git a/sijapi/routers/note.py b/sijapi/routers/note.py index 8c4ea3c..38ae5ad 100644 --- a/sijapi/routers/note.py +++ b/sijapi/routers/note.py @@ -1,5 +1,5 @@ ''' -Manages an Obsidian vault, in particular daily notes, using information and functionality drawn from the other routers, primarily calendar, email, ig, llm, rag, sd, serve, time, tts, and weather. +Manages an Obsidian vault, in particular daily notes, using information and functionality drawn from the other routers, primarily calendar, email, ig, llm, rag, img, serve, time, tts, and weather. ''' from fastapi import APIRouter, BackgroundTasks, File, UploadFile, Form, HTTPException, Response, Query, Path as FastAPIPath from fastapi.responses import JSONResponse, PlainTextResponse @@ -14,7 +14,7 @@ from fastapi import HTTPException, status from pathlib import Path from fastapi import APIRouter, Query, HTTPException from sijapi import API, L, OBSIDIAN_VAULT_DIR, OBSIDIAN_RESOURCES_DIR, OBSIDIAN_BANNER_SCENE, DEFAULT_11L_VOICE, DEFAULT_VOICE, GEO -from sijapi.routers import cal, loc, tts, llm, time, sd, weather, asr +from sijapi.routers import cal, img, loc, tts, llm, time, weather, asr from sijapi.utilities import assemble_journal_path, assemble_archive_path, convert_to_12_hour_format, sanitize_filename, convert_degrees_to_cardinal, check_file_name, HOURLY_COLUMNS_MAPPING from sijapi.classes import Location @@ -270,7 +270,7 @@ async def generate_banner(dt, location: Location = None, forecast: str = None, m prompt = await generate_context(date_time, location, forecast, mood, other_context) L.DEBUG(f"Prompt: {prompt}") - final_path = await sd.workflow(prompt, scene=OBSIDIAN_BANNER_SCENE, destination_path=destination_path) + final_path = await img.workflow(prompt, scene=OBSIDIAN_BANNER_SCENE, destination_path=destination_path) if not str(local_path) in str(final_path): L.INFO(f"Apparent mismatch between local path, {local_path}, and final_path, {final_path}") jpg_embed = f"\"![[{local_path}]]\"" diff --git a/sijapi/routers/serve.py b/sijapi/routers/serve.py index 700f5cb..67a617d 100644 --- a/sijapi/routers/serve.py +++ b/sijapi/routers/serve.py @@ -1,5 +1,5 @@ ''' -Web server module. Used by other modules when serving static content is required, e.g. the sd image generation module. Also used to serve PUBLIC_KEY. +Web server module. Used by other modules when serving static content is required, e.g. the img image generation module. Also used to serve PUBLIC_KEY. ''' import os import io