From bd26ea0b0e028fdba504587ea922b9476eac8620 Mon Sep 17 00:00:00 2001 From: sanj <67624670+iodrift@users.noreply.github.com> Date: Tue, 25 Jun 2024 03:12:07 -0700 Subject: [PATCH] updated exclusions, and added r2r --- .gitignore | 1 + .gitmodules | 3 + sijapi/__init__.py | 34 ++--- sijapi/__main__.py | 4 +- sijapi/classes.py | 45 ++++++ sijapi/routers/calendar.py | 18 +-- sijapi/routers/email.py | 286 ++++++++++++++++++++----------------- sijapi/routers/locate.py | 24 ++-- sijapi/routers/note.py | 24 ++-- sijapi/routers/serve.py | 4 +- sijapi/routers/time.py | 18 +-- sijapi/routers/tts.py | 7 +- sijapi/routers/weather.py | 8 +- sijapi/utilities.py | 15 +- 14 files changed, 274 insertions(+), 217 deletions(-) create mode 100644 .gitmodules create mode 100644 sijapi/classes.py diff --git a/.gitignore b/.gitignore index e512faf..692f9ed 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ sijapi/data/sd/workflows/private sijapi/data/*.pbf sijapi/data/geonames.txt sijapi/data/sd/images/ +sijapi/config/*.yaml sijapi/config/O365/ sijapi/local_only/ sijapi/testbed/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..067558c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "r2r"] + path = r2r + url = https://github.com/SciPhi-AI/R2R-Dashboard.git diff --git a/sijapi/__init__.py b/sijapi/__init__.py index ff9f9a2..6013a75 100644 --- a/sijapi/__init__.py +++ b/sijapi/__init__.py @@ -1,5 +1,6 @@ import os import json +import yaml from pathlib import Path import ipaddress import multiprocessing @@ -7,9 +8,11 @@ from dotenv import load_dotenv from dateutil import tz from pathlib import Path from pydantic import BaseModel +from typing import List, Optional import traceback import logging from .logs import Logger +from .classes import AutoResponder, IMAPConfig, SMTPConfig, EmailAccount, EmailContact, IncomingEmail # from sijapi.config.config import load_config # cfg = load_config() @@ -116,6 +119,7 @@ SUMMARY_INSTRUCT_TTS = os.getenv('SUMMARY_INSTRUCT_TTS', "You are an AI assistan DEFAULT_LLM = os.getenv("DEFAULT_LLM", "dolphin-mistral") DEFAULT_VISION = os.getenv("DEFAULT_VISION", "llava") DEFAULT_VOICE = os.getenv("DEFAULT_VOICE", "Luna") +DEFAULT_11L_VOICE = os.getenv("DEFAULT_11L_VOICE", "Victoria") OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") ### Stable diffusion @@ -164,29 +168,15 @@ MS365_TOGGLE = True if os.getenv("MS365_TOGGLE") == "True" else False ICAL_TOGGLE = True if os.getenv("ICAL_TOGGLE") == "True" else False ICS_PATH = DATA_DIR / 'calendar.ics' # deprecated now, but maybe revive? ICALENDARS = os.getenv('ICALENDARS', 'NULL,VOID').split(',') -class IMAP_DETAILS(BaseModel): - email: str - password: str - host: str - imap_port: int - smtp_port: int - imap_encryption: str = None - smtp_encryption: str = None -IMAP = IMAP_DETAILS( - email = os.getenv('IMAP_EMAIL'), - password = os.getenv('IMAP_PASSWORD'), - host = os.getenv('IMAP_HOST', '127.0.0.1'), - imap_port = int(os.getenv('IMAP_PORT', 1143)), - smtp_port = int(os.getenv('SMTP_PORT', 469)), - imap_encryption = os.getenv('IMAP_ENCRYPTION', None), - smtp_encryption = os.getenv('SMTP_ENCRYPTION', None) -) -AUTORESPONSE_WHITELIST = os.getenv('AUTORESPONSE_WHITELIST', '').split(',') -AUTORESPONSE_BLACKLIST = os.getenv('AUTORESPONSE_BLACKLIST', '').split(',') -AUTORESPONSE_BLACKLIST.extend(["no-reply@", "noreply@", "@uscourts.gov", "@doi.gov"]) -AUTORESPONSE_CONTEXT = os.getenv('AUTORESPONSE_CONTEXT', None) -AUTORESPOND = AUTORESPONSE_CONTEXT != None +def load_email_accounts(yaml_path: str) -> List[EmailAccount]: + with open(yaml_path, 'r') as file: + config = yaml.safe_load(file) + return [EmailAccount(**account) for account in config['accounts']] + +EMAIL_CONFIG = CONFIG_DIR / "email.yaml" +EMAIL_ACCOUNTS = load_email_accounts(EMAIL_CONFIG) +AUTORESPOND = True ### Courtlistener & other webhooks COURTLISTENER_DOCKETS_DIR = DATA_DIR / "courtlistener" / "dockets" diff --git a/sijapi/__main__.py b/sijapi/__main__.py index 42d9eea..5f5b239 100755 --- a/sijapi/__main__.py +++ b/sijapi/__main__.py @@ -20,7 +20,7 @@ from datetime import datetime import argparse from . import LOGGER, LOGS_DIR, OBSIDIAN_VAULT_DIR from .logs import Logger -from .utilities import fix_nextcloud_filenames +from .utilities import list_and_correct_impermissible_files parser = argparse.ArgumentParser(description='Personal API.') parser.add_argument('--debug', action='store_true', help='Set log level to INFO') @@ -139,7 +139,7 @@ def main(argv): load_router(router_name) journal = OBSIDIAN_VAULT_DIR / "journal" - fix_nextcloud_filenames(journal, rename=True) + list_and_correct_impermissible_files(journal, rename=True) config = Config() config.keep_alive_timeout = 1200 config.bind = [HOST] diff --git a/sijapi/classes.py b/sijapi/classes.py new file mode 100644 index 0000000..559bdb1 --- /dev/null +++ b/sijapi/classes.py @@ -0,0 +1,45 @@ +from pydantic import BaseModel +from typing import List, Optional, Any +from datetime import datetime + +class AutoResponder(BaseModel): + name: str + style: str + context: str + whitelist: List[str] + blacklist: List[str] + img_gen_prompt: Optional[str] = None + +class IMAPConfig(BaseModel): + username: str + password: str + host: str + port: int + encryption: str = None + +class SMTPConfig(BaseModel): + username: str + password: str + host: str + port: int + encryption: str = None + +class EmailAccount(BaseModel): + name: str + fullname: Optional[str] + bio: Optional[str] + imap: IMAPConfig + smtp: SMTPConfig + autoresponders: Optional[List[AutoResponder]] + +class EmailContact(BaseModel): + email: str + name: str + +class IncomingEmail(BaseModel): + sender: str + recipients: List[EmailContact] + datetime_received: datetime + subject: str + body: str + attachments: Optional[List[Any]] = None \ No newline at end of file diff --git a/sijapi/routers/calendar.py b/sijapi/routers/calendar.py index a2fd8fc..86bbe0d 100644 --- a/sijapi/routers/calendar.py +++ b/sijapi/routers/calendar.py @@ -17,7 +17,7 @@ from datetime import datetime, timedelta from Foundation import NSDate, NSRunLoop import EventKit as EK from sijapi import ICAL_TOGGLE, ICALENDARS, MS365_TOGGLE, MS365_CLIENT_ID, MS365_SECRET, MS365_AUTHORITY_URL, MS365_SCOPE, MS365_REDIRECT_PATH, MS365_TOKEN_PATH -from sijapi.utilities import localize_dt, localize_dt +from sijapi.utilities import localize_datetime, localize_datetime from sijapi import DEBUG, INFO, WARN, ERR, CRITICAL calendar = APIRouter() @@ -215,8 +215,8 @@ def datetime_to_nsdate(dt: datetime) -> NSDate: @calendar.get("/events") async def get_events_endpoint(start_date: str, end_date: str): - start_dt = localize_dt(start_date) - end_dt = localize_dt(end_date) + start_dt = localize_datetime(start_date) + end_dt = localize_datetime(end_date) datetime.strptime(start_date, "%Y-%m-%d") or datetime.now() end_dt = datetime.strptime(end_date, "%Y-%m-%d") or datetime.now() response = await get_events(start_dt, end_dt) @@ -342,8 +342,8 @@ async def get_ms365_events(start_date: datetime, end_date: datetime): async def parse_calendar_for_day(range_start: datetime, range_end: datetime, events: List[Dict[str, Any]]): - range_start = localize_dt(range_start) - range_end = localize_dt(range_end) + range_start = localize_datetime(range_start) + range_end = localize_datetime(range_end) event_list = [] for event in events: @@ -362,13 +362,13 @@ async def parse_calendar_for_day(range_start: datetime, range_end: datetime, eve INFO(f"End date string not a dict") try: - start_date = localize_dt(start_str) if start_str else None + start_date = localize_datetime(start_str) if start_str else None except (ValueError, TypeError) as e: ERR(f"Invalid start date format: {start_str}, error: {e}") continue try: - end_date = localize_dt(end_str) if end_str else None + end_date = localize_datetime(end_str) if end_str else None except (ValueError, TypeError) as e: ERR(f"Invalid end date format: {end_str}, error: {e}") continue @@ -377,13 +377,13 @@ async def parse_calendar_for_day(range_start: datetime, range_end: datetime, eve if start_date: # Ensure start_date is timezone-aware - start_date = localize_dt(start_date) + start_date = localize_datetime(start_date) # If end_date is not provided, assume it's the same as start_date if not end_date: end_date = start_date else: - end_date = localize_dt(end_date) + end_date = localize_datetime(end_date) # Check if the event overlaps with the given range if (start_date < range_end) and (end_date > range_start): diff --git a/sijapi/routers/email.py b/sijapi/routers/email.py index 1ffebbc..d35e0ca 100644 --- a/sijapi/routers/email.py +++ b/sijapi/routers/email.py @@ -1,6 +1,5 @@ ''' -IN DEVELOPMENT Email module. Uses IMAP and SMTP login credentials to monitor an inbox and summarize incoming emails that match certain criteria and save the Text-To-Speech converted summaries into a specified "podcast" folder. -UNIMPLEMENTED: AI auto-responder. +Uses IMAP and SMTP login credentials to monitor an inbox and summarize incoming emails that match certain criteria and save the Text-To-Speech converted summaries into a specified "podcast" folder. ''' from fastapi import APIRouter import asyncio @@ -15,36 +14,87 @@ import ssl from smtplib import SMTP_SSL from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText +from email.mime.image import MIMEImage from datetime import datetime as dt_datetime from pydantic import BaseModel from typing import List, Optional, Any +import yaml +from typing import List, Dict, Optional +from pydantic import BaseModel + from sijapi import DEBUG, INFO, WARN, ERR, CRITICAL -from sijapi import HOME_DIR, DATA_DIR, OBSIDIAN_VAULT_DIR, PODCAST_DIR, IMAP, OBSIDIAN_JOURNAL_DIR, DEFAULT_VOICE, AUTORESPONSE_BLACKLIST, AUTORESPONSE_WHITELIST, AUTORESPONSE_CONTEXT, USER_FULLNAME, USER_BIO, AUTORESPOND, TZ -from sijapi.routers import summarize, tts, llm -from sijapi.utilities import clean_text, assemble_journal_path, localize_dt, extract_text, prefix_lines +from sijapi import PODCAST_DIR, DEFAULT_VOICE, TZ, EMAIL_ACCOUNTS, EmailAccount, IMAPConfig, SMTPConfig +from sijapi.routers import summarize, tts, llm, sd +from sijapi.utilities import clean_text, assemble_journal_path, localize_datetime, extract_text, prefix_lines +from sijapi.classes import EmailAccount, IncomingEmail, EmailContact email = APIRouter(tags=["private"]) +def get_account_by_email(email: str) -> Optional[EmailAccount]: + for account in EMAIL_ACCOUNTS: + if account.imap.username.lower() == email.lower(): + return account + return None -class Contact(BaseModel): - email: str - name: str -class EmailModel(BaseModel): - sender: str - recipients: List[Contact] - datetime_received: dt_datetime - subject: str - body: str - attachments: Optional[List[Any]] = None +def get_imap_details(email: str) -> Optional[IMAPConfig]: + account = get_account_by_email(email) + return account.imap if account else None + +def get_smtp_details(email: str) -> Optional[SMTPConfig]: + account = get_account_by_email(email) + return account.smtp if account else None + + +def get_imap_connection(account: EmailAccount): + return Imbox(account.imap.host, + username=account.imap.username, + password=account.imap.password, + port=account.imap.port, + ssl=account.imap.encryption == 'SSL', + starttls=account.imap.encryption == 'STARTTLS') + +def get_matching_autoresponders(email: IncomingEmail, account: EmailAccount) -> List[Dict]: + matching_profiles = [] + + def matches_list(item: str, email: IncomingEmail) -> bool: + if '@' in item: + return item in email.sender + else: + return item.lower() in email.subject.lower() or item.lower() in email.body.lower() + + for profile in account.autoresponders: + whitelist_match = not profile.whitelist or any(matches_list(item, email) for item in profile.whitelist) + blacklist_match = any(matches_list(item, email) for item in profile.blacklist) + + if whitelist_match and not blacklist_match: + matching_profiles.append({ + 'USER_FULLNAME': account.fullname, + 'RESPONSE_STYLE': profile.style, + 'AUTORESPONSE_CONTEXT': profile.context, + 'IMG_GEN_PROMPT': profile.img_gen_prompt, + 'USER_BIO': account.bio + }) + + return matching_profiles + + +async def generate_auto_response_body(e: IncomingEmail, profile: Dict) -> str: + age = dt_datetime.now(TZ) - e.datetime_received + prompt = f''' +Please generate a personalized auto-response to the following email. The email is from {e.sender} and was sent {age} ago with the subject line "{e.subject}." You are auto-responding on behalf of {profile['USER_FULLNAME']}, who is described by the following short bio (strictly for your context -- do not recite this in the response): "{profile['USER_BIO']}." {profile['USER_FULLNAME']} is unable to respond personally, because {profile['AUTORESPONSE_CONTEXT']}. Everything from here to ~~//END//~~ is the email body. +{e.body} +~~//END//~~ +Keep your auto-response {profile['RESPONSE_STYLE']} and to the point, but do aim to make it responsive specifically to the sender's inquiry. + ''' + + try: + response = await llm.query_ollama(prompt, 400) + return response + except Exception as e: + ERR(f"Error generating auto-response: {str(e)}") + return "Thank you for your email. Unfortunately, an error occurred while generating the auto-response. We apologize for any inconvenience." -def imap_conn(): - return Imbox(IMAP.host, - username=IMAP.email, - password=IMAP.password, - port=IMAP.imap_port, - ssl=IMAP.imap_encryption == 'SSL', - starttls=IMAP.imap_encryption == 'STARTTLS') def clean_email_content(html_content): @@ -73,68 +123,76 @@ async def extract_attachments(attachments) -> List[str]: return attachment_texts -async def process_unread_emails(auto_respond: bool = AUTORESPOND, summarize_emails: bool = True, podcast: bool = True): +async def process_unread_emails(summarize_emails: bool = True, podcast: bool = True): while True: - try: - with imap_conn() as inbox: - unread_messages = inbox.messages(unread=True) - for uid, message in unread_messages: - recipients = [Contact(email=recipient['email'], name=recipient.get('name', '')) for recipient in message.sent_to] - this_email = EmailModel( - sender=message.sent_from[0]['email'], - datetime_received=localize_dt(message.date), - recipients=recipients, - subject=message.subject, - body=clean_email_content(message.body['html'][0]) if message.body['html'] else clean_email_content(message.body['plain'][0]) or "", - attachments=message.attachments - ) + for account in EMAIL_ACCOUNTS: + DEBUG(f"Connecting to {account.name} to check for unread emails...") + try: + with get_imap_connection(account) as inbox: + DEBUG(f"Connected to {account.name}, checking for unread emails now...") + unread_messages = inbox.messages(unread=True) + for uid, message in unread_messages: + recipients = [EmailContact(email=recipient['email'], name=recipient.get('name', '')) for recipient in message.sent_to] + this_email = IncomingEmail( + sender=message.sent_from[0]['email'], + datetime_received=localize_datetime(message.date), + recipients=recipients, + subject=message.subject, + body=clean_email_content(message.body['html'][0]) if message.body['html'] else clean_email_content(message.body['plain'][0]) or "", + attachments=message.attachments + ) - DEBUG(f"\n\nProcessing email: {this_email.subject}\n\n") - md_path, md_relative = assemble_journal_path(this_email.datetime_received, "Emails", this_email.subject, ".md") - tts_path, tts_relative = assemble_journal_path(this_email.datetime_received, "Emails", this_email.subject, ".wav") - if summarize_emails: - email_content = f'At {this_email.datetime_received}, {this_email.sender} sent an email with the subject line "{this_email.subject}". The email in its entirety reads: \n\n{this_email.body}\n"' - if this_email.attachments: - attachment_texts = await extract_attachments(this_email.attachments) - email_content += "\n—--\n" + "\n—--\n".join([f"Attachment: {text}" for text in attachment_texts]) + DEBUG(f"\n\nProcessing email for account {account.name}: {this_email.subject}\n\n") - summary = await summarize.summarize_text(email_content) - await tts.local_tts(text_content = summary, speed = 1.1, voice = DEFAULT_VOICE, podcast = podcast, output_path = tts_path) - - if podcast: - if PODCAST_DIR.exists(): - tts.copy_to_podcast_dir(tts_path) - else: - ERR(f"PODCAST_DIR does not exist: {PODCAST_DIR}") + md_path, md_relative = assemble_journal_path(this_email.datetime_received, "Emails", this_email.subject, ".md") + tts_path, tts_relative = assemble_journal_path(this_email.datetime_received, "Emails", this_email.subject, ".wav") + if summarize_emails: + email_content = f'At {this_email.datetime_received}, {this_email.sender} sent an email with the subject line "{this_email.subject}". The email in its entirety reads: \n\n{this_email.body}\n"' + if this_email.attachments: + attachment_texts = await extract_attachments(this_email.attachments) + email_content += "\n—--\n" + "\n—--\n".join([f"Attachment: {text}" for text in attachment_texts]) - save_email_as_markdown(this_email, summary, md_path, tts_relative) - else: - save_email_as_markdown(this_email, None, md_path, None) + summary = await summarize.summarize_text(email_content) + await tts.local_tts(text_content = summary, speed = 1.1, voice = DEFAULT_VOICE, podcast = podcast, output_path = tts_path) + + if podcast: + if PODCAST_DIR.exists(): + tts.copy_to_podcast_dir(tts_path) + else: + ERR(f"PODCAST_DIR does not exist: {PODCAST_DIR}") - if auto_respond and should_auto_respond(this_email): - DEBUG(f"Auto-responding to {this_email.subject}") - auto_response_subject = 'Auto-Response Re:' + this_email.subject - auto_response_body = await generate_auto_response_body(this_email) - DEBUG(f"Auto-response: {auto_response_body}") - await send_auto_response(this_email.sender, auto_response_subject, auto_response_body) - - inbox.mark_seen(uid) + save_email_as_markdown(this_email, summary, md_path, tts_relative) + DEBUG(f"Email '{this_email.subject}' saved to {md_relative}.") + else: + save_email_as_markdown(this_email, None, md_path, None) - await asyncio.sleep(30) - except Exception as e: - ERR(f"An error occurred: {e}") - await asyncio.sleep(30) + matching_profiles = get_matching_autoresponders(this_email, account) + + for profile in matching_profiles: + DEBUG(f"Auto-responding to {this_email.subject} with profile: {profile['USER_FULLNAME']}") + auto_response_subject = f"Auto-Response Re: {this_email.subject}" + auto_response_body = await generate_auto_response_body(this_email, profile) + DEBUG(f"Auto-response: {auto_response_body}") + await send_auto_response(this_email.sender, auto_response_subject, auto_response_body, profile, account) + + inbox.mark_seen(uid) + + await asyncio.sleep(30) + except Exception as e: + ERR(f"An error occurred for account {account.name}: {e}") + await asyncio.sleep(30) -def save_email_as_markdown(email: EmailModel, summary: str, md_path: Path, tts_path: Path): + +def save_email_as_markdown(email: IncomingEmail, summary: str, md_path: Path, tts_path: Path): ''' Saves an email as a markdown file in the specified directory. Args: - email (EmailModel): The email object containing email details. + email (IncomingEmail): The email object containing email details. summary (str): The summary of the email. tts_path (str): The path to the text-to-speech audio file. ''' - + DEBUG(f"Saving email to {md_path}...") # Sanitize filename to avoid issues with filesystems filename = f"{email.datetime_received.strftime('%Y%m%d%H%M%S')}_{email.subject.replace('/', '-')}.md".replace(':', '-').replace(' ', '_') @@ -175,79 +233,45 @@ tags: DEBUG(f"Saved markdown to {md_path}") -AUTORESPONSE_SYS = "You are a helpful AI assistant that generates personalized auto-response messages to incoming emails." - -async def generate_auto_response_body(e: EmailModel, response_style: str = "professional") -> str: - age = dt_datetime.now(TZ) - e.datetime_received - prompt = f''' -Please generate a personalized auto-response to the following email. The email is from {e.sender} and was sent {age} ago with the subject line "{e.subject}." You are auto-responding on behalf of {USER_FULLNAME}, who is described by the following short bio (strictly for your context -- do not recite this in the response): "{USER_BIO}." {USER_FULLNAME} is unable to respond himself, because {AUTORESPONSE_CONTEXT}. Everything from here to ~~//END//~~ is the email body. -{e.body} -~~//END//~~ -Keep your auto-response {response_style} and to the point, but do aim to make it responsive specifically to the sender's inquiry. - ''' - - try: - response = await llm.query_ollama(prompt, AUTORESPONSE_SYS, 400) - return response - except Exception as e: - ERR(f"Error generating auto-response: {str(e)}") - return "Thank you for your email. Unfortunately, an error occurred while generating the auto-response. We apologize for any inconvenience." - -async def send_auto_response(to_email, subject, body): +async def send_auto_response(to_email, subject, body, profile, account): + DEBUG(f"Sending auto response to {to_email}...") try: message = MIMEMultipart() - message['From'] = IMAP.email # smtp_username + message['From'] = account.smtp.username message['To'] = to_email message['Subject'] = subject message.attach(MIMEText(body, 'plain')) - # DEBUG(f"Attempting to send auto_response to {to_email} concerning {subject}. We will use {IMAP.host}:{IMAP.smtp_port}, un: {IMAP.email}, pw: {IMAP.password}") + if profile['IMG_GEN_PROMPT']: + jpg_path = sd.workflow(profile['IMG_GEN_PROMPT'], earlyout=False, downscale_to_fit=True) + if jpg_path and os.path.exists(jpg_path): + with open(jpg_path, 'rb') as img_file: + img = MIMEImage(img_file.read(), name=os.path.basename(jpg_path)) + message.attach(img) - try: - DEBUG(f"Initiating attempt to send auto-response via SMTP at {IMAP.host}:{IMAP.smtp_port}...") - context = ssl._create_unverified_context() + context = ssl._create_unverified_context() + with SMTP_SSL(account.smtp.host, account.smtp.port, context=context) as server: + server.login(account.smtp.username, account.smtp.password) + server.send_message(message) - with SMTP_SSL(IMAP.host, IMAP.smtp_port, context=context) as server: - server.login(IMAP.email, IMAP.password) - DEBUG(f"Successfully logged in to {IMAP.host} at {IMAP.smtp_port} as {IMAP.email}. Attempting to send email now.") - server.send_message(message) - - INFO(f"Auto-response sent to {to_email} concerning {subject}") - - except Exception as e: - ERR(f"Failed to send auto-response email to {to_email}: {e}") - raise e + INFO(f"Auto-response sent to {to_email} concerning {subject} from account {account.name}") except Exception as e: - ERR(f"Error in preparing/sending auto-response: {e}") + ERR(f"Error in preparing/sending auto-response from account {account.name}: {e}") raise e -def should_auto_respond(email: EmailModel) -> bool: - def matches_list(item: str, email: EmailModel) -> bool: - if '@' in item: - if item in email.sender: - return True - else: - if item.lower() in email.subject.lower() or item.lower() in email.body.lower(): - return True - return False - - if AUTORESPONSE_WHITELIST: - for item in AUTORESPONSE_WHITELIST: - if matches_list(item, email): - if AUTORESPONSE_BLACKLIST: - for blacklist_item in AUTORESPONSE_BLACKLIST: - if matches_list(blacklist_item, email): - return False - return True - return False - else: - if AUTORESPONSE_BLACKLIST: - for item in AUTORESPONSE_BLACKLIST: - if matches_list(item, email): - return False - return True + + @email.on_event("startup") async def startup_event(): - asyncio.create_task(process_unread_emails()) \ No newline at end of file + asyncio.create_task(process_unread_emails()) + + + + + + #### + + + diff --git a/sijapi/routers/locate.py b/sijapi/routers/locate.py index 01ba85d..3a734fe 100644 --- a/sijapi/routers/locate.py +++ b/sijapi/routers/locate.py @@ -17,7 +17,7 @@ from typing import Optional, Any, Dict, List, Union from datetime import datetime, timedelta, time from sijapi import LOCATION_OVERRIDES, TZ from sijapi import DEBUG, INFO, WARN, ERR, CRITICAL -from sijapi.utilities import get_db_connection, haversine, localize_dt +from sijapi.utilities import get_db_connection, haversine, localize_datetime # from osgeo import gdal # import elevation @@ -228,12 +228,12 @@ def get_elevation(latitude, longitude): async def fetch_locations(start: datetime, end: datetime = None) -> List[Location]: - start_datetime = localize_dt(start) + start_datetime = localize_datetime(start) if end is None: - end_datetime = localize_dt(start_datetime.replace(hour=23, minute=59, second=59)) + end_datetime = localize_datetime(start_datetime.replace(hour=23, minute=59, second=59)) else: - end_datetime = localize_dt(end) + end_datetime = localize_datetime(end) if start_datetime.time() == datetime.min.time() and end.time() == datetime.min.time(): end_datetime = end_datetime.replace(hour=23, minute=59, second=59) @@ -305,7 +305,7 @@ async def fetch_locations(start: datetime, end: datetime = None) -> List[Locatio # Function to fetch the last location before the specified datetime async def fetch_last_location_before(datetime: datetime) -> Optional[Location]: - datetime = localize_dt(datetime) + datetime = localize_datetime(datetime) DEBUG(f"Fetching last location before {datetime}") conn = await get_db_connection() @@ -337,8 +337,8 @@ async def fetch_last_location_before(datetime: datetime) -> Optional[Location]: @locate.get("/map/start_date={start_date_str}&end_date={end_date_str}", response_class=HTMLResponse) async def generate_map_endpoint(start_date_str: str, end_date_str: str): try: - start_date = localize_dt(start_date_str) - end_date = localize_dt(end_date_str) + start_date = localize_datetime(start_date_str) + end_date = localize_datetime(end_date_str) except ValueError: raise HTTPException(status_code=400, detail="Invalid date format") @@ -349,8 +349,8 @@ async def generate_map_endpoint(start_date_str: str, end_date_str: str): @locate.get("/map", response_class=HTMLResponse) async def generate_alltime_map_endpoint(): try: - start_date = localize_dt(datetime.fromisoformat("2022-01-01")) - end_date = localize_dt(datetime.now()) + start_date = localize_datetime(datetime.fromisoformat("2022-01-01")) + end_date = localize_datetime(datetime.now()) except ValueError: raise HTTPException(status_code=400, detail="Invalid date format") @@ -397,7 +397,7 @@ async def post_location(location: Location): device_os = context.get('device_os', 'Unknown') # Parse and localize the datetime - localized_datetime = localize_dt(location.datetime) + localized_datetime = localize_datetime(location.datetime) await conn.execute(''' INSERT INTO locations (datetime, location, city, state, zip, street, action, device_type, device_model, device_name, device_os) @@ -452,7 +452,7 @@ async def post_locate_endpoint(locations: Union[Location, List[Location]]): DEBUG(f"datetime before localization: {location.datetime}") # Convert datetime string to timezone-aware datetime object - location.datetime = localize_dt(location.datetime) + location.datetime = localize_datetime(location.datetime) DEBUG(f"datetime after localization: {location.datetime}") location_entry = await post_location(location) @@ -484,7 +484,7 @@ async def get_last_location() -> JSONResponse: @locate.get("/locate/{datetime_str}", response_model=List[Location]) async def get_locate(datetime_str: str, all: bool = False): try: - date_time = localize_dt(datetime_str) + date_time = localize_datetime(datetime_str) except ValueError as e: ERR(f"Invalid datetime string provided: {datetime_str}") return ["ERROR: INVALID DATETIME PROVIDED. USE YYYYMMDDHHmmss or YYYYMMDD format."] diff --git a/sijapi/routers/note.py b/sijapi/routers/note.py index c6589b0..996c603 100644 --- a/sijapi/routers/note.py +++ b/sijapi/routers/note.py @@ -17,12 +17,12 @@ from requests.adapters import HTTPAdapter import re import os from datetime import timedelta, datetime, time as dt_time, date as dt_date -from sijapi.utilities import localize_dt +from sijapi.utilities import localize_datetime from fastapi import HTTPException, status from pathlib import Path from fastapi import APIRouter, Query, HTTPException from sijapi import DEBUG, INFO, WARN, ERR, CRITICAL, INFO -from sijapi import YEAR_FMT, MONTH_FMT, DAY_FMT, DAY_SHORT_FMT, OBSIDIAN_VAULT_DIR, OBSIDIAN_RESOURCES_DIR, BASE_URL, OBSIDIAN_BANNER_SCENE, DEFAULT_VOICE, TZ +from sijapi import YEAR_FMT, MONTH_FMT, DAY_FMT, DAY_SHORT_FMT, OBSIDIAN_VAULT_DIR, OBSIDIAN_RESOURCES_DIR, BASE_URL, OBSIDIAN_BANNER_SCENE, DEFAULT_11L_VOICE, DEFAULT_VOICE, TZ from sijapi.routers import tts, time, sd, locate, weather, asr, calendar, summarize from sijapi.routers.locate import Location from sijapi.utilities import assemble_journal_path, convert_to_12_hour_format, sanitize_filename, convert_degrees_to_cardinal, HOURLY_COLUMNS_MAPPING @@ -39,7 +39,7 @@ async def build_daily_note_range_endpoint(dt_start: str, dt_end: str): results = [] current_date = start_date while current_date <= end_date: - formatted_date = localize_dt(current_date) + formatted_date = localize_datetime(current_date) result = await build_daily_note(formatted_date) results.append(result) current_date += timedelta(days=1) @@ -134,7 +134,7 @@ async def clip_post( source: Optional[str] = Form(None), title: Optional[str] = Form(None), tts: str = Form('summary'), - voice: str = Form('Luna'), + voice: str = Form(DEFAULT_VOICE), encoding: str = Form('utf-8') ): markdown_filename = await process_article(background_tasks, url, title, encoding, source, tts, voice) @@ -159,7 +159,7 @@ async def clip_get( title: Optional[str] = Query(None), encoding: str = Query('utf-8'), tts: str = Query('summary'), - voice: str = Query('Luna') + voice: str = Query(DEFAULT_VOICE) ): markdown_filename = await process_article(background_tasks, url, title, encoding, tts=tts, voice=voice) return {"message": "Clip saved successfully", "markdown_filename": markdown_filename} @@ -337,7 +337,7 @@ async def process_article( encoding: str = 'utf-8', source: Optional[str] = None, tts_mode: str = "summary", - voice: str = DEFAULT_VOICE + voice: str = DEFAULT_11L_VOICE ): timestamp = datetime.now().strftime('%b %d, %Y at %H:%M') @@ -442,7 +442,7 @@ def parse_article(url: str, source: Optional[str] = None): title = np3k.title or traf.title authors = np3k.authors or traf.author authors = authors if isinstance(authors, List) else [authors] - date = np3k.publish_date or localize_dt(traf.date) + date = np3k.publish_date or localize_datetime(traf.date) excerpt = np3k.meta_description or traf.description content = trafilatura.extract(source, output_format="markdown", include_comments=False) or np3k.text image = np3k.top_image or traf.image @@ -635,7 +635,7 @@ async def banner_endpoint(dt: str, location: str = None, mood: str = None, other Endpoint (POST) that generates a new banner image for the Obsidian daily note for a specified date, taking into account optional additional information, then updates the frontmatter if necessary. ''' DEBUG(f"banner_endpoint requested with date: {dt} ({type(dt)})") - date_time = localize_dt(dt) + date_time = localize_datetime(dt) DEBUG(f"date_time after localization: {date_time} ({type(date_time)})") jpg_path = await generate_banner(date_time, location, mood=mood, other_context=other_context) return jpg_path @@ -643,7 +643,7 @@ async def banner_endpoint(dt: str, location: str = None, mood: str = None, other async def generate_banner(dt, location: Location = None, forecast: str = None, mood: str = None, other_context: str = None): DEBUG(f"Location: {location}, forecast: {forecast}, mood: {mood}, other_context: {other_context}") - date_time = localize_dt(dt) + date_time = localize_datetime(dt) DEBUG(f"generate_banner called with date_time: {date_time}") destination_path, local_path = assemble_journal_path(date_time, filename="Banner", extension=".jpg", no_timestamp = True) DEBUG(f"destination path generated: {destination_path}") @@ -699,7 +699,7 @@ async def note_weather_get( ): try: - date_time = datetime.now() if date == "0" else localize_dt(date) + date_time = datetime.now() if date == "0" else localize_datetime(date) DEBUG(f"date: {date} .. date_time: {date_time}") content = await update_dn_weather(date_time) #, lat, lon) return JSONResponse(content={"forecast": content}, status_code=200) @@ -714,7 +714,7 @@ async def note_weather_get( @note.post("/update/note/{date}") async def post_update_daily_weather_and_calendar_and_timeslips(date: str) -> PlainTextResponse: - date_time = localize_dt(date) + date_time = localize_datetime(date) await update_dn_weather(date_time) await update_daily_note_events(date_time) await build_daily_timeslips(date_time) @@ -1117,7 +1117,7 @@ async def format_events_as_markdown(event_data: Dict[str, Union[str, List[Dict[s @note.get("/note/events", response_class=PlainTextResponse) async def note_events_endpoint(date: str = Query(None)): - date_time = localize_dt(date) if date else datetime.now(TZ) + date_time = localize_datetime(date) if date else datetime.now(TZ) response = await update_daily_note_events(date_time) return PlainTextResponse(content=response, status_code=200) diff --git a/sijapi/routers/serve.py b/sijapi/routers/serve.py index c362117..f9271c1 100644 --- a/sijapi/routers/serve.py +++ b/sijapi/routers/serve.py @@ -14,7 +14,7 @@ from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from pathlib import Path from sijapi import DEBUG, INFO, WARN, ERR, CRITICAL -from sijapi.utilities import bool_convert, sanitize_filename, assemble_journal_path, localize_dt +from sijapi.utilities import bool_convert, sanitize_filename, assemble_journal_path, localize_datetime from sijapi import DATA_DIR, SD_IMAGE_DIR, PUBLIC_KEY, OBSIDIAN_VAULT_DIR serve = APIRouter(tags=["public"]) @@ -50,7 +50,7 @@ def is_valid_date(date_str: str) -> bool: @serve.get("/notes/{file_path:path}") async def get_file(file_path: str): try: - date_time = localize_dt(file_path); + date_time = localize_datetime(file_path); absolute_path, local_path = assemble_journal_path(date_time, no_timestamp = True) except ValueError as e: DEBUG(f"Unable to parse {file_path} as a date, now trying to use it as a local path") diff --git a/sijapi/routers/time.py b/sijapi/routers/time.py index e52c703..ec2aa24 100644 --- a/sijapi/routers/time.py +++ b/sijapi/routers/time.py @@ -17,7 +17,7 @@ from fastapi import APIRouter, UploadFile, File, Response, Header, Query, Depend from fastapi.responses import FileResponse, JSONResponse from pydantic import BaseModel, Field from datetime import datetime, timedelta -from sijapi.utilities import localize_dt +from sijapi.utilities import localize_datetime from decimal import Decimal, ROUND_UP from typing import Optional, List, Dict, Union, Tuple from collections import defaultdict @@ -100,8 +100,8 @@ def truncate_project_title(title): async def fetch_and_prepare_timing_data(start: datetime, end: Optional[datetime] = None) -> List[Dict]: - # start_date = localize_dt(start) - # end_date = localize_dt(end) if end else None + # start_date = localize_datetime(start) + # end_date = localize_datetime(end) if end else None # Adjust the start date to include the day before and format the end date start_date_adjusted = (start - timedelta(days=1)).strftime("%Y-%m-%dT00:00:00") end_date_formatted = f"{datetime.strftime(end, '%Y-%m-%d')}T23:59:59" if end else f"{datetime.strftime(start, '%Y-%m-%d')}T23:59:59" @@ -315,8 +315,8 @@ async def get_timing_markdown3( ): # Fetch and process timing data - start = localize_dt(start_date) - end = localize_dt(end_date) if end_date else None + start = localize_datetime(start_date) + end = localize_datetime(end_date) if end_date else None timing_data = await fetch_and_prepare_timing_data(start, end) # Retain these for processing Markdown data with the correct timezone @@ -375,8 +375,8 @@ async def get_timing_markdown( start: str = Query(..., regex=r"\d{4}-\d{2}-\d{2}"), end: Optional[str] = Query(None, regex=r"\d{4}-\d{2}-\d{2}") ): - start_date = localize_dt(start) - end_date = localize_dt(end) + start_date = localize_datetime(start) + end_date = localize_datetime(end) markdown_formatted_data = await process_timing_markdown(start_date, end_date) return Response(content=markdown_formatted_data, media_type="text/markdown") @@ -444,8 +444,8 @@ async def get_timing_json( ): # Fetch and process timing data - start = localize_dt(start_date) - end = localize_dt(end_date) + start = localize_datetime(start_date) + end = localize_datetime(end_date) timing_data = await fetch_and_prepare_timing_data(start, end) # Convert processed data to the required JSON structure diff --git a/sijapi/routers/tts.py b/sijapi/routers/tts.py index 29e3e94..3d89f6e 100644 --- a/sijapi/routers/tts.py +++ b/sijapi/routers/tts.py @@ -165,6 +165,8 @@ async def get_model(voice: str = None, voice_file: UploadFile = None): raise HTTPException(status_code=400, detail="No model or voice specified") async def determine_voice_id(voice_name: str) -> str: + DEBUG(f"Searching for voice id for {voice_name}") + hardcoded_voices = { "alloy": "E3A1KVbKoWSIKSZwSUsW", "echo": "b42GBisbu9r5m5n6pHF7", @@ -172,7 +174,7 @@ async def determine_voice_id(voice_name: str) -> str: "onyx": "clQb8NxY08xZ6mX6wCPE", "nova": "6TayTBKLMOsghG7jYuMX", "shimmer": "E7soeOyjpmuZFurvoxZ2", - DEFAULT_VOICE: "6TayTBKLMOsghG7jYuMX", + "Luna": "6TayTBKLMOsghG7jYuMX", "Sangye": "E7soeOyjpmuZFurvoxZ2", "Herzog": "KAX2Y6tTs0oDWq7zZXW7", "Attenborough": "b42GBisbu9r5m5n6pHF7" @@ -198,7 +200,8 @@ async def determine_voice_id(voice_name: str) -> str: except Exception as e: ERR(f"Error determining voice ID: {str(e)}") - return "6TayTBKLMOsghG7jYuMX" + # as a last fallback, rely on David Attenborough + return "b42GBisbu9r5m5n6pHF7" async def elevenlabs_tts(model: str, input_text: str, voice: str, title: str = None, output_dir: str = None): diff --git a/sijapi/routers/weather.py b/sijapi/routers/weather.py index 38b3b8e..fd6b0f2 100644 --- a/sijapi/routers/weather.py +++ b/sijapi/routers/weather.py @@ -7,7 +7,7 @@ from typing import Dict from datetime import datetime from shapely.wkb import loads from binascii import unhexlify -from sijapi.utilities import localize_dt +from sijapi.utilities import localize_datetime from sijapi import DEBUG, INFO, WARN, ERR, CRITICAL from sijapi import VISUALCROSSING_API_KEY, TZ from sijapi.utilities import get_db_connection, haversine @@ -25,7 +25,7 @@ async def get_weather(date_time: datetime, latitude: float, longitude: float): try: DEBUG(f"Daily weather data from db: {daily_weather_data}") last_updated = str(daily_weather_data['DailyWeather'].get('last_updated')) - last_updated = localize_dt(last_updated) + last_updated = localize_datetime(last_updated) stored_loc_data = unhexlify(daily_weather_data['DailyWeather'].get('location')) stored_loc = loads(stored_loc_data) stored_lat = stored_loc.y @@ -103,7 +103,7 @@ async def store_weather_to_db(date_time: datetime, weather_data: dict): location_point = f"POINTZ({longitude} {latitude} {elevation})" if longitude and latitude and elevation else None # Correct for the datetime objects - day_data['datetime'] = localize_dt(day_data.get('datetime')) #day_data.get('datetime')) + day_data['datetime'] = localize_datetime(day_data.get('datetime')) #day_data.get('datetime')) day_data['sunrise'] = day_data['datetime'].replace(hour=int(day_data.get('sunrise').split(':')[0]), minute=int(day_data.get('sunrise').split(':')[1])) day_data['sunset'] = day_data['datetime'].replace(hour=int(day_data.get('sunset').split(':')[0]), minute=int(day_data.get('sunset').split(':')[1])) @@ -160,7 +160,7 @@ async def store_weather_to_db(date_time: datetime, weather_data: dict): await asyncio.sleep(0.1) # hour_data['datetime'] = parse_date(hour_data.get('datetime')) hour_timestamp = date_str + ' ' + hour_data['datetime'] - hour_data['datetime'] = localize_dt(hour_timestamp) + hour_data['datetime'] = localize_datetime(hour_timestamp) DEBUG(f"Processing hours now...") # DEBUG(f"Processing {hour_data['datetime']}") diff --git a/sijapi/utilities.py b/sijapi/utilities.py index b65d1ce..83b48ec 100644 --- a/sijapi/utilities.py +++ b/sijapi/utilities.py @@ -210,7 +210,7 @@ def list_and_correct_impermissible_files(root_dir, rename: bool = False): if check_file_name(filename): file_path = Path(dirpath) / filename impermissible_files.append(file_path) - print(f"Impermissible file found: {file_path}") + DEBUG(f"Impermissible file found: {file_path}") # Sanitize the file name new_filename = sanitize_filename(filename) @@ -228,19 +228,10 @@ def list_and_correct_impermissible_files(root_dir, rename: bool = False): # Rename the file if rename: os.rename(file_path, new_file_path) - print(f"Renamed: {file_path} -> {new_file_path}") + DEBUG(f"Renamed: {file_path} -> {new_file_path}") return impermissible_files -def fix_nextcloud_filenames(dir_to_fix, rename: bool = False): - impermissible_files = list_and_correct_impermissible_files(dir_to_fix, rename) - if impermissible_files: - print("\nList of impermissible files found and corrected:") - for file in impermissible_files: - print(file) - else: - print("No impermissible files found.") - def bool_convert(value: str = Form(None)): return value.lower() in ["true", "1", "t", "y", "yes"] @@ -454,7 +445,7 @@ def convert_degrees_to_cardinal(d): return dirs[ix % len(dirs)] -def localize_dt(dt): +def localize_datetime(dt): initial_dt = dt try: if isinstance(dt, str):