From 7bbe92ca9cccee02f02201d6486991f6e6e27d33 Mon Sep 17 00:00:00 2001
From: sanj <67624670+iodrift@users.noreply.github.com>
Date: Tue, 25 Jun 2024 03:12:07 -0700
Subject: [PATCH] updated exclusions, and added r2r
---
.gitignore | 1 +
.gitmodules | 3 +
sijapi/__init__.py | 34 ++---
sijapi/__main__.py | 4 +-
sijapi/classes.py | 45 ++++++
sijapi/routers/email.py | 286 +++++++++++++++++++++-----------------
sijapi/routers/note.py | 24 ++--
sijapi/routers/serve.py | 4 +-
sijapi/routers/tts.py | 7 +-
sijapi/routers/weather.py | 8 +-
sijapi/utilities.py | 15 +-
11 files changed, 244 insertions(+), 187 deletions(-)
create mode 100644 .gitmodules
create mode 100644 sijapi/classes.py
diff --git a/.gitignore b/.gitignore
index e512faf..692f9ed 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,6 +9,7 @@ sijapi/data/sd/workflows/private
sijapi/data/*.pbf
sijapi/data/geonames.txt
sijapi/data/sd/images/
+sijapi/config/*.yaml
sijapi/config/O365/
sijapi/local_only/
sijapi/testbed/
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..067558c
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "r2r"]
+ path = r2r
+ url = https://github.com/SciPhi-AI/R2R-Dashboard.git
diff --git a/sijapi/__init__.py b/sijapi/__init__.py
index ff9f9a2..6013a75 100644
--- a/sijapi/__init__.py
+++ b/sijapi/__init__.py
@@ -1,5 +1,6 @@
import os
import json
+import yaml
from pathlib import Path
import ipaddress
import multiprocessing
@@ -7,9 +8,11 @@ from dotenv import load_dotenv
from dateutil import tz
from pathlib import Path
from pydantic import BaseModel
+from typing import List, Optional
import traceback
import logging
from .logs import Logger
+from .classes import AutoResponder, IMAPConfig, SMTPConfig, EmailAccount, EmailContact, IncomingEmail
# from sijapi.config.config import load_config
# cfg = load_config()
@@ -116,6 +119,7 @@ SUMMARY_INSTRUCT_TTS = os.getenv('SUMMARY_INSTRUCT_TTS', "You are an AI assistan
DEFAULT_LLM = os.getenv("DEFAULT_LLM", "dolphin-mistral")
DEFAULT_VISION = os.getenv("DEFAULT_VISION", "llava")
DEFAULT_VOICE = os.getenv("DEFAULT_VOICE", "Luna")
+DEFAULT_11L_VOICE = os.getenv("DEFAULT_11L_VOICE", "Victoria")
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
### Stable diffusion
@@ -164,29 +168,15 @@ MS365_TOGGLE = True if os.getenv("MS365_TOGGLE") == "True" else False
ICAL_TOGGLE = True if os.getenv("ICAL_TOGGLE") == "True" else False
ICS_PATH = DATA_DIR / 'calendar.ics' # deprecated now, but maybe revive?
ICALENDARS = os.getenv('ICALENDARS', 'NULL,VOID').split(',')
-class IMAP_DETAILS(BaseModel):
- email: str
- password: str
- host: str
- imap_port: int
- smtp_port: int
- imap_encryption: str = None
- smtp_encryption: str = None
-IMAP = IMAP_DETAILS(
- email = os.getenv('IMAP_EMAIL'),
- password = os.getenv('IMAP_PASSWORD'),
- host = os.getenv('IMAP_HOST', '127.0.0.1'),
- imap_port = int(os.getenv('IMAP_PORT', 1143)),
- smtp_port = int(os.getenv('SMTP_PORT', 469)),
- imap_encryption = os.getenv('IMAP_ENCRYPTION', None),
- smtp_encryption = os.getenv('SMTP_ENCRYPTION', None)
-)
-AUTORESPONSE_WHITELIST = os.getenv('AUTORESPONSE_WHITELIST', '').split(',')
-AUTORESPONSE_BLACKLIST = os.getenv('AUTORESPONSE_BLACKLIST', '').split(',')
-AUTORESPONSE_BLACKLIST.extend(["no-reply@", "noreply@", "@uscourts.gov", "@doi.gov"])
-AUTORESPONSE_CONTEXT = os.getenv('AUTORESPONSE_CONTEXT', None)
-AUTORESPOND = AUTORESPONSE_CONTEXT != None
+def load_email_accounts(yaml_path: str) -> List[EmailAccount]:
+ with open(yaml_path, 'r') as file:
+ config = yaml.safe_load(file)
+ return [EmailAccount(**account) for account in config['accounts']]
+
+EMAIL_CONFIG = CONFIG_DIR / "email.yaml"
+EMAIL_ACCOUNTS = load_email_accounts(EMAIL_CONFIG)
+AUTORESPOND = True
### Courtlistener & other webhooks
COURTLISTENER_DOCKETS_DIR = DATA_DIR / "courtlistener" / "dockets"
diff --git a/sijapi/__main__.py b/sijapi/__main__.py
index 42d9eea..5f5b239 100755
--- a/sijapi/__main__.py
+++ b/sijapi/__main__.py
@@ -20,7 +20,7 @@ from datetime import datetime
import argparse
from . import LOGGER, LOGS_DIR, OBSIDIAN_VAULT_DIR
from .logs import Logger
-from .utilities import fix_nextcloud_filenames
+from .utilities import list_and_correct_impermissible_files
parser = argparse.ArgumentParser(description='Personal API.')
parser.add_argument('--debug', action='store_true', help='Set log level to INFO')
@@ -139,7 +139,7 @@ def main(argv):
load_router(router_name)
journal = OBSIDIAN_VAULT_DIR / "journal"
- fix_nextcloud_filenames(journal, rename=True)
+ list_and_correct_impermissible_files(journal, rename=True)
config = Config()
config.keep_alive_timeout = 1200
config.bind = [HOST]
diff --git a/sijapi/classes.py b/sijapi/classes.py
new file mode 100644
index 0000000..559bdb1
--- /dev/null
+++ b/sijapi/classes.py
@@ -0,0 +1,45 @@
+from pydantic import BaseModel
+from typing import List, Optional, Any
+from datetime import datetime
+
+class AutoResponder(BaseModel):
+ name: str
+ style: str
+ context: str
+ whitelist: List[str]
+ blacklist: List[str]
+ img_gen_prompt: Optional[str] = None
+
+class IMAPConfig(BaseModel):
+ username: str
+ password: str
+ host: str
+ port: int
+ encryption: str = None
+
+class SMTPConfig(BaseModel):
+ username: str
+ password: str
+ host: str
+ port: int
+ encryption: str = None
+
+class EmailAccount(BaseModel):
+ name: str
+ fullname: Optional[str]
+ bio: Optional[str]
+ imap: IMAPConfig
+ smtp: SMTPConfig
+ autoresponders: Optional[List[AutoResponder]]
+
+class EmailContact(BaseModel):
+ email: str
+ name: str
+
+class IncomingEmail(BaseModel):
+ sender: str
+ recipients: List[EmailContact]
+ datetime_received: datetime
+ subject: str
+ body: str
+ attachments: Optional[List[Any]] = None
\ No newline at end of file
diff --git a/sijapi/routers/email.py b/sijapi/routers/email.py
index 1ffebbc..d35e0ca 100644
--- a/sijapi/routers/email.py
+++ b/sijapi/routers/email.py
@@ -1,6 +1,5 @@
'''
-IN DEVELOPMENT Email module. Uses IMAP and SMTP login credentials to monitor an inbox and summarize incoming emails that match certain criteria and save the Text-To-Speech converted summaries into a specified "podcast" folder.
-UNIMPLEMENTED: AI auto-responder.
+Uses IMAP and SMTP login credentials to monitor an inbox and summarize incoming emails that match certain criteria and save the Text-To-Speech converted summaries into a specified "podcast" folder.
'''
from fastapi import APIRouter
import asyncio
@@ -15,36 +14,87 @@ import ssl
from smtplib import SMTP_SSL
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
+from email.mime.image import MIMEImage
from datetime import datetime as dt_datetime
from pydantic import BaseModel
from typing import List, Optional, Any
+import yaml
+from typing import List, Dict, Optional
+from pydantic import BaseModel
+
from sijapi import DEBUG, INFO, WARN, ERR, CRITICAL
-from sijapi import HOME_DIR, DATA_DIR, OBSIDIAN_VAULT_DIR, PODCAST_DIR, IMAP, OBSIDIAN_JOURNAL_DIR, DEFAULT_VOICE, AUTORESPONSE_BLACKLIST, AUTORESPONSE_WHITELIST, AUTORESPONSE_CONTEXT, USER_FULLNAME, USER_BIO, AUTORESPOND, TZ
-from sijapi.routers import summarize, tts, llm
-from sijapi.utilities import clean_text, assemble_journal_path, localize_dt, extract_text, prefix_lines
+from sijapi import PODCAST_DIR, DEFAULT_VOICE, TZ, EMAIL_ACCOUNTS, EmailAccount, IMAPConfig, SMTPConfig
+from sijapi.routers import summarize, tts, llm, sd
+from sijapi.utilities import clean_text, assemble_journal_path, localize_datetime, extract_text, prefix_lines
+from sijapi.classes import EmailAccount, IncomingEmail, EmailContact
email = APIRouter(tags=["private"])
+def get_account_by_email(email: str) -> Optional[EmailAccount]:
+ for account in EMAIL_ACCOUNTS:
+ if account.imap.username.lower() == email.lower():
+ return account
+ return None
-class Contact(BaseModel):
- email: str
- name: str
-class EmailModel(BaseModel):
- sender: str
- recipients: List[Contact]
- datetime_received: dt_datetime
- subject: str
- body: str
- attachments: Optional[List[Any]] = None
+def get_imap_details(email: str) -> Optional[IMAPConfig]:
+ account = get_account_by_email(email)
+ return account.imap if account else None
+
+def get_smtp_details(email: str) -> Optional[SMTPConfig]:
+ account = get_account_by_email(email)
+ return account.smtp if account else None
+
+
+def get_imap_connection(account: EmailAccount):
+ return Imbox(account.imap.host,
+ username=account.imap.username,
+ password=account.imap.password,
+ port=account.imap.port,
+ ssl=account.imap.encryption == 'SSL',
+ starttls=account.imap.encryption == 'STARTTLS')
+
+def get_matching_autoresponders(email: IncomingEmail, account: EmailAccount) -> List[Dict]:
+ matching_profiles = []
+
+ def matches_list(item: str, email: IncomingEmail) -> bool:
+ if '@' in item:
+ return item in email.sender
+ else:
+ return item.lower() in email.subject.lower() or item.lower() in email.body.lower()
+
+ for profile in account.autoresponders:
+ whitelist_match = not profile.whitelist or any(matches_list(item, email) for item in profile.whitelist)
+ blacklist_match = any(matches_list(item, email) for item in profile.blacklist)
+
+ if whitelist_match and not blacklist_match:
+ matching_profiles.append({
+ 'USER_FULLNAME': account.fullname,
+ 'RESPONSE_STYLE': profile.style,
+ 'AUTORESPONSE_CONTEXT': profile.context,
+ 'IMG_GEN_PROMPT': profile.img_gen_prompt,
+ 'USER_BIO': account.bio
+ })
+
+ return matching_profiles
+
+
+async def generate_auto_response_body(e: IncomingEmail, profile: Dict) -> str:
+ age = dt_datetime.now(TZ) - e.datetime_received
+ prompt = f'''
+Please generate a personalized auto-response to the following email. The email is from {e.sender} and was sent {age} ago with the subject line "{e.subject}." You are auto-responding on behalf of {profile['USER_FULLNAME']}, who is described by the following short bio (strictly for your context -- do not recite this in the response): "{profile['USER_BIO']}." {profile['USER_FULLNAME']} is unable to respond personally, because {profile['AUTORESPONSE_CONTEXT']}. Everything from here to ~~//END//~~ is the email body.
+{e.body}
+~~//END//~~
+Keep your auto-response {profile['RESPONSE_STYLE']} and to the point, but do aim to make it responsive specifically to the sender's inquiry.
+ '''
+
+ try:
+ response = await llm.query_ollama(prompt, 400)
+ return response
+ except Exception as e:
+ ERR(f"Error generating auto-response: {str(e)}")
+ return "Thank you for your email. Unfortunately, an error occurred while generating the auto-response. We apologize for any inconvenience."
-def imap_conn():
- return Imbox(IMAP.host,
- username=IMAP.email,
- password=IMAP.password,
- port=IMAP.imap_port,
- ssl=IMAP.imap_encryption == 'SSL',
- starttls=IMAP.imap_encryption == 'STARTTLS')
def clean_email_content(html_content):
@@ -73,68 +123,76 @@ async def extract_attachments(attachments) -> List[str]:
return attachment_texts
-async def process_unread_emails(auto_respond: bool = AUTORESPOND, summarize_emails: bool = True, podcast: bool = True):
+async def process_unread_emails(summarize_emails: bool = True, podcast: bool = True):
while True:
- try:
- with imap_conn() as inbox:
- unread_messages = inbox.messages(unread=True)
- for uid, message in unread_messages:
- recipients = [Contact(email=recipient['email'], name=recipient.get('name', '')) for recipient in message.sent_to]
- this_email = EmailModel(
- sender=message.sent_from[0]['email'],
- datetime_received=localize_dt(message.date),
- recipients=recipients,
- subject=message.subject,
- body=clean_email_content(message.body['html'][0]) if message.body['html'] else clean_email_content(message.body['plain'][0]) or "",
- attachments=message.attachments
- )
+ for account in EMAIL_ACCOUNTS:
+ DEBUG(f"Connecting to {account.name} to check for unread emails...")
+ try:
+ with get_imap_connection(account) as inbox:
+ DEBUG(f"Connected to {account.name}, checking for unread emails now...")
+ unread_messages = inbox.messages(unread=True)
+ for uid, message in unread_messages:
+ recipients = [EmailContact(email=recipient['email'], name=recipient.get('name', '')) for recipient in message.sent_to]
+ this_email = IncomingEmail(
+ sender=message.sent_from[0]['email'],
+ datetime_received=localize_datetime(message.date),
+ recipients=recipients,
+ subject=message.subject,
+ body=clean_email_content(message.body['html'][0]) if message.body['html'] else clean_email_content(message.body['plain'][0]) or "",
+ attachments=message.attachments
+ )
- DEBUG(f"\n\nProcessing email: {this_email.subject}\n\n")
- md_path, md_relative = assemble_journal_path(this_email.datetime_received, "Emails", this_email.subject, ".md")
- tts_path, tts_relative = assemble_journal_path(this_email.datetime_received, "Emails", this_email.subject, ".wav")
- if summarize_emails:
- email_content = f'At {this_email.datetime_received}, {this_email.sender} sent an email with the subject line "{this_email.subject}". The email in its entirety reads: \n\n{this_email.body}\n"'
- if this_email.attachments:
- attachment_texts = await extract_attachments(this_email.attachments)
- email_content += "\n—--\n" + "\n—--\n".join([f"Attachment: {text}" for text in attachment_texts])
+ DEBUG(f"\n\nProcessing email for account {account.name}: {this_email.subject}\n\n")
- summary = await summarize.summarize_text(email_content)
- await tts.local_tts(text_content = summary, speed = 1.1, voice = DEFAULT_VOICE, podcast = podcast, output_path = tts_path)
-
- if podcast:
- if PODCAST_DIR.exists():
- tts.copy_to_podcast_dir(tts_path)
- else:
- ERR(f"PODCAST_DIR does not exist: {PODCAST_DIR}")
+ md_path, md_relative = assemble_journal_path(this_email.datetime_received, "Emails", this_email.subject, ".md")
+ tts_path, tts_relative = assemble_journal_path(this_email.datetime_received, "Emails", this_email.subject, ".wav")
+ if summarize_emails:
+ email_content = f'At {this_email.datetime_received}, {this_email.sender} sent an email with the subject line "{this_email.subject}". The email in its entirety reads: \n\n{this_email.body}\n"'
+ if this_email.attachments:
+ attachment_texts = await extract_attachments(this_email.attachments)
+ email_content += "\n—--\n" + "\n—--\n".join([f"Attachment: {text}" for text in attachment_texts])
- save_email_as_markdown(this_email, summary, md_path, tts_relative)
- else:
- save_email_as_markdown(this_email, None, md_path, None)
+ summary = await summarize.summarize_text(email_content)
+ await tts.local_tts(text_content = summary, speed = 1.1, voice = DEFAULT_VOICE, podcast = podcast, output_path = tts_path)
+
+ if podcast:
+ if PODCAST_DIR.exists():
+ tts.copy_to_podcast_dir(tts_path)
+ else:
+ ERR(f"PODCAST_DIR does not exist: {PODCAST_DIR}")
- if auto_respond and should_auto_respond(this_email):
- DEBUG(f"Auto-responding to {this_email.subject}")
- auto_response_subject = 'Auto-Response Re:' + this_email.subject
- auto_response_body = await generate_auto_response_body(this_email)
- DEBUG(f"Auto-response: {auto_response_body}")
- await send_auto_response(this_email.sender, auto_response_subject, auto_response_body)
-
- inbox.mark_seen(uid)
+ save_email_as_markdown(this_email, summary, md_path, tts_relative)
+ DEBUG(f"Email '{this_email.subject}' saved to {md_relative}.")
+ else:
+ save_email_as_markdown(this_email, None, md_path, None)
- await asyncio.sleep(30)
- except Exception as e:
- ERR(f"An error occurred: {e}")
- await asyncio.sleep(30)
+ matching_profiles = get_matching_autoresponders(this_email, account)
+
+ for profile in matching_profiles:
+ DEBUG(f"Auto-responding to {this_email.subject} with profile: {profile['USER_FULLNAME']}")
+ auto_response_subject = f"Auto-Response Re: {this_email.subject}"
+ auto_response_body = await generate_auto_response_body(this_email, profile)
+ DEBUG(f"Auto-response: {auto_response_body}")
+ await send_auto_response(this_email.sender, auto_response_subject, auto_response_body, profile, account)
+
+ inbox.mark_seen(uid)
+
+ await asyncio.sleep(30)
+ except Exception as e:
+ ERR(f"An error occurred for account {account.name}: {e}")
+ await asyncio.sleep(30)
-def save_email_as_markdown(email: EmailModel, summary: str, md_path: Path, tts_path: Path):
+
+def save_email_as_markdown(email: IncomingEmail, summary: str, md_path: Path, tts_path: Path):
'''
Saves an email as a markdown file in the specified directory.
Args:
- email (EmailModel): The email object containing email details.
+ email (IncomingEmail): The email object containing email details.
summary (str): The summary of the email.
tts_path (str): The path to the text-to-speech audio file.
'''
-
+ DEBUG(f"Saving email to {md_path}...")
# Sanitize filename to avoid issues with filesystems
filename = f"{email.datetime_received.strftime('%Y%m%d%H%M%S')}_{email.subject.replace('/', '-')}.md".replace(':', '-').replace(' ', '_')
@@ -175,79 +233,45 @@ tags:
DEBUG(f"Saved markdown to {md_path}")
-AUTORESPONSE_SYS = "You are a helpful AI assistant that generates personalized auto-response messages to incoming emails."
-
-async def generate_auto_response_body(e: EmailModel, response_style: str = "professional") -> str:
- age = dt_datetime.now(TZ) - e.datetime_received
- prompt = f'''
-Please generate a personalized auto-response to the following email. The email is from {e.sender} and was sent {age} ago with the subject line "{e.subject}." You are auto-responding on behalf of {USER_FULLNAME}, who is described by the following short bio (strictly for your context -- do not recite this in the response): "{USER_BIO}." {USER_FULLNAME} is unable to respond himself, because {AUTORESPONSE_CONTEXT}. Everything from here to ~~//END//~~ is the email body.
-{e.body}
-~~//END//~~
-Keep your auto-response {response_style} and to the point, but do aim to make it responsive specifically to the sender's inquiry.
- '''
-
- try:
- response = await llm.query_ollama(prompt, AUTORESPONSE_SYS, 400)
- return response
- except Exception as e:
- ERR(f"Error generating auto-response: {str(e)}")
- return "Thank you for your email. Unfortunately, an error occurred while generating the auto-response. We apologize for any inconvenience."
-
-async def send_auto_response(to_email, subject, body):
+async def send_auto_response(to_email, subject, body, profile, account):
+ DEBUG(f"Sending auto response to {to_email}...")
try:
message = MIMEMultipart()
- message['From'] = IMAP.email # smtp_username
+ message['From'] = account.smtp.username
message['To'] = to_email
message['Subject'] = subject
message.attach(MIMEText(body, 'plain'))
- # DEBUG(f"Attempting to send auto_response to {to_email} concerning {subject}. We will use {IMAP.host}:{IMAP.smtp_port}, un: {IMAP.email}, pw: {IMAP.password}")
+ if profile['IMG_GEN_PROMPT']:
+ jpg_path = sd.workflow(profile['IMG_GEN_PROMPT'], earlyout=False, downscale_to_fit=True)
+ if jpg_path and os.path.exists(jpg_path):
+ with open(jpg_path, 'rb') as img_file:
+ img = MIMEImage(img_file.read(), name=os.path.basename(jpg_path))
+ message.attach(img)
- try:
- DEBUG(f"Initiating attempt to send auto-response via SMTP at {IMAP.host}:{IMAP.smtp_port}...")
- context = ssl._create_unverified_context()
+ context = ssl._create_unverified_context()
+ with SMTP_SSL(account.smtp.host, account.smtp.port, context=context) as server:
+ server.login(account.smtp.username, account.smtp.password)
+ server.send_message(message)
- with SMTP_SSL(IMAP.host, IMAP.smtp_port, context=context) as server:
- server.login(IMAP.email, IMAP.password)
- DEBUG(f"Successfully logged in to {IMAP.host} at {IMAP.smtp_port} as {IMAP.email}. Attempting to send email now.")
- server.send_message(message)
-
- INFO(f"Auto-response sent to {to_email} concerning {subject}")
-
- except Exception as e:
- ERR(f"Failed to send auto-response email to {to_email}: {e}")
- raise e
+ INFO(f"Auto-response sent to {to_email} concerning {subject} from account {account.name}")
except Exception as e:
- ERR(f"Error in preparing/sending auto-response: {e}")
+ ERR(f"Error in preparing/sending auto-response from account {account.name}: {e}")
raise e
-def should_auto_respond(email: EmailModel) -> bool:
- def matches_list(item: str, email: EmailModel) -> bool:
- if '@' in item:
- if item in email.sender:
- return True
- else:
- if item.lower() in email.subject.lower() or item.lower() in email.body.lower():
- return True
- return False
-
- if AUTORESPONSE_WHITELIST:
- for item in AUTORESPONSE_WHITELIST:
- if matches_list(item, email):
- if AUTORESPONSE_BLACKLIST:
- for blacklist_item in AUTORESPONSE_BLACKLIST:
- if matches_list(blacklist_item, email):
- return False
- return True
- return False
- else:
- if AUTORESPONSE_BLACKLIST:
- for item in AUTORESPONSE_BLACKLIST:
- if matches_list(item, email):
- return False
- return True
+
+
@email.on_event("startup")
async def startup_event():
- asyncio.create_task(process_unread_emails())
\ No newline at end of file
+ asyncio.create_task(process_unread_emails())
+
+
+
+
+
+ ####
+
+
+
diff --git a/sijapi/routers/note.py b/sijapi/routers/note.py
index c6589b0..996c603 100644
--- a/sijapi/routers/note.py
+++ b/sijapi/routers/note.py
@@ -17,12 +17,12 @@ from requests.adapters import HTTPAdapter
import re
import os
from datetime import timedelta, datetime, time as dt_time, date as dt_date
-from sijapi.utilities import localize_dt
+from sijapi.utilities import localize_datetime
from fastapi import HTTPException, status
from pathlib import Path
from fastapi import APIRouter, Query, HTTPException
from sijapi import DEBUG, INFO, WARN, ERR, CRITICAL, INFO
-from sijapi import YEAR_FMT, MONTH_FMT, DAY_FMT, DAY_SHORT_FMT, OBSIDIAN_VAULT_DIR, OBSIDIAN_RESOURCES_DIR, BASE_URL, OBSIDIAN_BANNER_SCENE, DEFAULT_VOICE, TZ
+from sijapi import YEAR_FMT, MONTH_FMT, DAY_FMT, DAY_SHORT_FMT, OBSIDIAN_VAULT_DIR, OBSIDIAN_RESOURCES_DIR, BASE_URL, OBSIDIAN_BANNER_SCENE, DEFAULT_11L_VOICE, DEFAULT_VOICE, TZ
from sijapi.routers import tts, time, sd, locate, weather, asr, calendar, summarize
from sijapi.routers.locate import Location
from sijapi.utilities import assemble_journal_path, convert_to_12_hour_format, sanitize_filename, convert_degrees_to_cardinal, HOURLY_COLUMNS_MAPPING
@@ -39,7 +39,7 @@ async def build_daily_note_range_endpoint(dt_start: str, dt_end: str):
results = []
current_date = start_date
while current_date <= end_date:
- formatted_date = localize_dt(current_date)
+ formatted_date = localize_datetime(current_date)
result = await build_daily_note(formatted_date)
results.append(result)
current_date += timedelta(days=1)
@@ -134,7 +134,7 @@ async def clip_post(
source: Optional[str] = Form(None),
title: Optional[str] = Form(None),
tts: str = Form('summary'),
- voice: str = Form('Luna'),
+ voice: str = Form(DEFAULT_VOICE),
encoding: str = Form('utf-8')
):
markdown_filename = await process_article(background_tasks, url, title, encoding, source, tts, voice)
@@ -159,7 +159,7 @@ async def clip_get(
title: Optional[str] = Query(None),
encoding: str = Query('utf-8'),
tts: str = Query('summary'),
- voice: str = Query('Luna')
+ voice: str = Query(DEFAULT_VOICE)
):
markdown_filename = await process_article(background_tasks, url, title, encoding, tts=tts, voice=voice)
return {"message": "Clip saved successfully", "markdown_filename": markdown_filename}
@@ -337,7 +337,7 @@ async def process_article(
encoding: str = 'utf-8',
source: Optional[str] = None,
tts_mode: str = "summary",
- voice: str = DEFAULT_VOICE
+ voice: str = DEFAULT_11L_VOICE
):
timestamp = datetime.now().strftime('%b %d, %Y at %H:%M')
@@ -442,7 +442,7 @@ def parse_article(url: str, source: Optional[str] = None):
title = np3k.title or traf.title
authors = np3k.authors or traf.author
authors = authors if isinstance(authors, List) else [authors]
- date = np3k.publish_date or localize_dt(traf.date)
+ date = np3k.publish_date or localize_datetime(traf.date)
excerpt = np3k.meta_description or traf.description
content = trafilatura.extract(source, output_format="markdown", include_comments=False) or np3k.text
image = np3k.top_image or traf.image
@@ -635,7 +635,7 @@ async def banner_endpoint(dt: str, location: str = None, mood: str = None, other
Endpoint (POST) that generates a new banner image for the Obsidian daily note for a specified date, taking into account optional additional information, then updates the frontmatter if necessary.
'''
DEBUG(f"banner_endpoint requested with date: {dt} ({type(dt)})")
- date_time = localize_dt(dt)
+ date_time = localize_datetime(dt)
DEBUG(f"date_time after localization: {date_time} ({type(date_time)})")
jpg_path = await generate_banner(date_time, location, mood=mood, other_context=other_context)
return jpg_path
@@ -643,7 +643,7 @@ async def banner_endpoint(dt: str, location: str = None, mood: str = None, other
async def generate_banner(dt, location: Location = None, forecast: str = None, mood: str = None, other_context: str = None):
DEBUG(f"Location: {location}, forecast: {forecast}, mood: {mood}, other_context: {other_context}")
- date_time = localize_dt(dt)
+ date_time = localize_datetime(dt)
DEBUG(f"generate_banner called with date_time: {date_time}")
destination_path, local_path = assemble_journal_path(date_time, filename="Banner", extension=".jpg", no_timestamp = True)
DEBUG(f"destination path generated: {destination_path}")
@@ -699,7 +699,7 @@ async def note_weather_get(
):
try:
- date_time = datetime.now() if date == "0" else localize_dt(date)
+ date_time = datetime.now() if date == "0" else localize_datetime(date)
DEBUG(f"date: {date} .. date_time: {date_time}")
content = await update_dn_weather(date_time) #, lat, lon)
return JSONResponse(content={"forecast": content}, status_code=200)
@@ -714,7 +714,7 @@ async def note_weather_get(
@note.post("/update/note/{date}")
async def post_update_daily_weather_and_calendar_and_timeslips(date: str) -> PlainTextResponse:
- date_time = localize_dt(date)
+ date_time = localize_datetime(date)
await update_dn_weather(date_time)
await update_daily_note_events(date_time)
await build_daily_timeslips(date_time)
@@ -1117,7 +1117,7 @@ async def format_events_as_markdown(event_data: Dict[str, Union[str, List[Dict[s
@note.get("/note/events", response_class=PlainTextResponse)
async def note_events_endpoint(date: str = Query(None)):
- date_time = localize_dt(date) if date else datetime.now(TZ)
+ date_time = localize_datetime(date) if date else datetime.now(TZ)
response = await update_daily_note_events(date_time)
return PlainTextResponse(content=response, status_code=200)
diff --git a/sijapi/routers/serve.py b/sijapi/routers/serve.py
index c362117..f9271c1 100644
--- a/sijapi/routers/serve.py
+++ b/sijapi/routers/serve.py
@@ -14,7 +14,7 @@ from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from pathlib import Path
from sijapi import DEBUG, INFO, WARN, ERR, CRITICAL
-from sijapi.utilities import bool_convert, sanitize_filename, assemble_journal_path, localize_dt
+from sijapi.utilities import bool_convert, sanitize_filename, assemble_journal_path, localize_datetime
from sijapi import DATA_DIR, SD_IMAGE_DIR, PUBLIC_KEY, OBSIDIAN_VAULT_DIR
serve = APIRouter(tags=["public"])
@@ -50,7 +50,7 @@ def is_valid_date(date_str: str) -> bool:
@serve.get("/notes/{file_path:path}")
async def get_file(file_path: str):
try:
- date_time = localize_dt(file_path);
+ date_time = localize_datetime(file_path);
absolute_path, local_path = assemble_journal_path(date_time, no_timestamp = True)
except ValueError as e:
DEBUG(f"Unable to parse {file_path} as a date, now trying to use it as a local path")
diff --git a/sijapi/routers/tts.py b/sijapi/routers/tts.py
index 29e3e94..3d89f6e 100644
--- a/sijapi/routers/tts.py
+++ b/sijapi/routers/tts.py
@@ -165,6 +165,8 @@ async def get_model(voice: str = None, voice_file: UploadFile = None):
raise HTTPException(status_code=400, detail="No model or voice specified")
async def determine_voice_id(voice_name: str) -> str:
+ DEBUG(f"Searching for voice id for {voice_name}")
+
hardcoded_voices = {
"alloy": "E3A1KVbKoWSIKSZwSUsW",
"echo": "b42GBisbu9r5m5n6pHF7",
@@ -172,7 +174,7 @@ async def determine_voice_id(voice_name: str) -> str:
"onyx": "clQb8NxY08xZ6mX6wCPE",
"nova": "6TayTBKLMOsghG7jYuMX",
"shimmer": "E7soeOyjpmuZFurvoxZ2",
- DEFAULT_VOICE: "6TayTBKLMOsghG7jYuMX",
+ "Luna": "6TayTBKLMOsghG7jYuMX",
"Sangye": "E7soeOyjpmuZFurvoxZ2",
"Herzog": "KAX2Y6tTs0oDWq7zZXW7",
"Attenborough": "b42GBisbu9r5m5n6pHF7"
@@ -198,7 +200,8 @@ async def determine_voice_id(voice_name: str) -> str:
except Exception as e:
ERR(f"Error determining voice ID: {str(e)}")
- return "6TayTBKLMOsghG7jYuMX"
+ # as a last fallback, rely on David Attenborough
+ return "b42GBisbu9r5m5n6pHF7"
async def elevenlabs_tts(model: str, input_text: str, voice: str, title: str = None, output_dir: str = None):
diff --git a/sijapi/routers/weather.py b/sijapi/routers/weather.py
index 38b3b8e..fd6b0f2 100644
--- a/sijapi/routers/weather.py
+++ b/sijapi/routers/weather.py
@@ -7,7 +7,7 @@ from typing import Dict
from datetime import datetime
from shapely.wkb import loads
from binascii import unhexlify
-from sijapi.utilities import localize_dt
+from sijapi.utilities import localize_datetime
from sijapi import DEBUG, INFO, WARN, ERR, CRITICAL
from sijapi import VISUALCROSSING_API_KEY, TZ
from sijapi.utilities import get_db_connection, haversine
@@ -25,7 +25,7 @@ async def get_weather(date_time: datetime, latitude: float, longitude: float):
try:
DEBUG(f"Daily weather data from db: {daily_weather_data}")
last_updated = str(daily_weather_data['DailyWeather'].get('last_updated'))
- last_updated = localize_dt(last_updated)
+ last_updated = localize_datetime(last_updated)
stored_loc_data = unhexlify(daily_weather_data['DailyWeather'].get('location'))
stored_loc = loads(stored_loc_data)
stored_lat = stored_loc.y
@@ -103,7 +103,7 @@ async def store_weather_to_db(date_time: datetime, weather_data: dict):
location_point = f"POINTZ({longitude} {latitude} {elevation})" if longitude and latitude and elevation else None
# Correct for the datetime objects
- day_data['datetime'] = localize_dt(day_data.get('datetime')) #day_data.get('datetime'))
+ day_data['datetime'] = localize_datetime(day_data.get('datetime')) #day_data.get('datetime'))
day_data['sunrise'] = day_data['datetime'].replace(hour=int(day_data.get('sunrise').split(':')[0]), minute=int(day_data.get('sunrise').split(':')[1]))
day_data['sunset'] = day_data['datetime'].replace(hour=int(day_data.get('sunset').split(':')[0]), minute=int(day_data.get('sunset').split(':')[1]))
@@ -160,7 +160,7 @@ async def store_weather_to_db(date_time: datetime, weather_data: dict):
await asyncio.sleep(0.1)
# hour_data['datetime'] = parse_date(hour_data.get('datetime'))
hour_timestamp = date_str + ' ' + hour_data['datetime']
- hour_data['datetime'] = localize_dt(hour_timestamp)
+ hour_data['datetime'] = localize_datetime(hour_timestamp)
DEBUG(f"Processing hours now...")
# DEBUG(f"Processing {hour_data['datetime']}")
diff --git a/sijapi/utilities.py b/sijapi/utilities.py
index b65d1ce..83b48ec 100644
--- a/sijapi/utilities.py
+++ b/sijapi/utilities.py
@@ -210,7 +210,7 @@ def list_and_correct_impermissible_files(root_dir, rename: bool = False):
if check_file_name(filename):
file_path = Path(dirpath) / filename
impermissible_files.append(file_path)
- print(f"Impermissible file found: {file_path}")
+ DEBUG(f"Impermissible file found: {file_path}")
# Sanitize the file name
new_filename = sanitize_filename(filename)
@@ -228,19 +228,10 @@ def list_and_correct_impermissible_files(root_dir, rename: bool = False):
# Rename the file
if rename:
os.rename(file_path, new_file_path)
- print(f"Renamed: {file_path} -> {new_file_path}")
+ DEBUG(f"Renamed: {file_path} -> {new_file_path}")
return impermissible_files
-def fix_nextcloud_filenames(dir_to_fix, rename: bool = False):
- impermissible_files = list_and_correct_impermissible_files(dir_to_fix, rename)
- if impermissible_files:
- print("\nList of impermissible files found and corrected:")
- for file in impermissible_files:
- print(file)
- else:
- print("No impermissible files found.")
-
def bool_convert(value: str = Form(None)):
return value.lower() in ["true", "1", "t", "y", "yes"]
@@ -454,7 +445,7 @@ def convert_degrees_to_cardinal(d):
return dirs[ix % len(dirs)]
-def localize_dt(dt):
+def localize_datetime(dt):
initial_dt = dt
try:
if isinstance(dt, str):