From c5e4cbbb4e65639423c5c4de6cef871b3ee234e4 Mon Sep 17 00:00:00 2001 From: sanj <67624670+iodrift@users.noreply.github.com> Date: Sun, 4 Aug 2024 22:12:18 -0700 Subject: [PATCH] Auto-update: Sun Aug 4 22:12:18 PDT 2024 --- sijapi/__main__.py | 14 +-- sijapi/classes.py | 109 +++++++++++----------- sijapi/cli.py | 58 ------------ sijapi/routers/email.py | 51 +++++++---- sijapi/routers/note.py | 197 +++++++++++++++++----------------------- 5 files changed, 176 insertions(+), 253 deletions(-) delete mode 100644 sijapi/cli.py diff --git a/sijapi/__main__.py b/sijapi/__main__.py index 5d70c40..accac93 100755 --- a/sijapi/__main__.py +++ b/sijapi/__main__.py @@ -11,7 +11,7 @@ from hypercorn.config import Config as HypercornConfig import sys import os import traceback -import asyncio +import asyncio import httpx import argparse import json @@ -56,7 +56,7 @@ async def lifespan(app: FastAPI): try: # Initialize sync structures on all databases await API.initialize_sync() - + except Exception as e: crit(f"Error during startup: {str(e)}") crit(f"Traceback: {traceback.format_exc()}") @@ -136,13 +136,13 @@ async def pull_changes(): await API.add_primary_keys_to_local_tables() await API.add_primary_keys_to_remote_tables() try: - + source = await API.get_most_recent_source() - + if source: # Pull changes from the source total_changes = await API.pull_changes(source) - + return JSONResponse(content={ "status": "success", "message": f"Pull complete. Total changes: {total_changes}", @@ -154,12 +154,12 @@ async def pull_changes(): "status": "info", "message": "No instances with more recent data found or all instances are offline." }) - + except Exception as e: err(f"Error during pull: {str(e)}") err(f"Traceback: {traceback.format_exc()}") raise HTTPException(status_code=500, detail=f"Error during pull: {str(e)}") - + except Exception as e: err(f"Error while ensuring primary keys to tables: {str(e)}") err(f"Traceback: {traceback.format_exc()}") diff --git a/sijapi/classes.py b/sijapi/classes.py index 313728c..efb39d7 100644 --- a/sijapi/classes.py +++ b/sijapi/classes.py @@ -29,6 +29,7 @@ from .logs import Logger L = Logger("classes", "classes") logger = L.get_module_logger("classes") +# Logging functions def debug(text: str): logger.debug(text) def info(text: str): logger.info(text) def warn(text: str): logger.warning(text) @@ -81,7 +82,7 @@ class Configuration(BaseModel): instance = cls.create_dynamic_model(**resolved_data) instance._dir_config = dir_config or instance return instance - + except Exception as e: err(f"Error loading configuration: {str(e)}") raise @@ -137,7 +138,7 @@ class Configuration(BaseModel): elif len(parts) == 2 and parts[0] == 'ENV': replacement = os.getenv(parts[1], '') else: - replacement = value + replacement = value value = value.replace('{{' + match + '}}', str(replacement)) @@ -314,7 +315,7 @@ class APIConfig(BaseModel): async def get_online_hosts(self) -> List[Dict[str, Any]]: current_time = time.time() cache_key = "online_hosts" - + if cache_key in self.online_hosts_cache: cached_hosts, cache_time = self.online_hosts_cache[cache_key] if current_time - cache_time < self.online_hosts_cache_ttl: @@ -322,7 +323,7 @@ class APIConfig(BaseModel): online_hosts = [] local_ts_id = os.environ.get('TS_ID') - + for pool_entry in self.POOL: if pool_entry['ts_id'] != local_ts_id: pool_key = f"{pool_entry['ts_ip']}:{pool_entry['db_port']}" @@ -331,7 +332,7 @@ class APIConfig(BaseModel): continue else: del self.offline_servers[pool_key] - + conn = await self.get_connection(pool_entry) if conn is not None: online_hosts.append(pool_entry) @@ -343,16 +344,16 @@ class APIConfig(BaseModel): async def get_connection(self, pool_entry: Dict[str, Any] = None): if pool_entry is None: pool_entry = self.local_db - + pool_key = f"{pool_entry['ts_ip']}:{pool_entry['db_port']}" - + # Check if the server is marked as offline if pool_key in self.offline_servers: if time.time() - self.offline_servers[pool_key] < self.offline_timeout: return None else: del self.offline_servers[pool_key] - + if pool_key not in self.db_pools: try: self.db_pools[pool_key] = await asyncpg.create_pool( @@ -385,7 +386,7 @@ class APIConfig(BaseModel): async def initialize_sync(self): local_ts_id = os.environ.get('TS_ID') online_hosts = await self.get_online_hosts() - + for pool_entry in online_hosts: if pool_entry['ts_id'] == local_ts_id: continue # Skip local database @@ -393,29 +394,29 @@ class APIConfig(BaseModel): conn = await self.get_connection(pool_entry) if conn is None: continue # Skip this database if connection failed - + debug(f"Starting sync initialization for {pool_entry['ts_ip']}...") - + # Check PostGIS installation postgis_installed = await self.check_postgis(conn) if not postgis_installed: warn(f"PostGIS is not installed on {pool_entry['ts_id']} ({pool_entry['ts_ip']}). Some spatial operations may fail.") - + tables = await conn.fetch(""" - SELECT tablename FROM pg_tables + SELECT tablename FROM pg_tables WHERE schemaname = 'public' """) - + for table in tables: table_name = table['tablename'] await self.ensure_sync_columns(conn, table_name) - + debug(f"Sync initialization complete for {pool_entry['ts_ip']}. All tables now have necessary sync columns and triggers.") - + except Exception as e: err(f"Error initializing sync for {pool_entry['ts_ip']}: {str(e)}") err(f"Traceback: {traceback.format_exc()}") - + def _schedule_sync_task(self, table_name: str, pk_value: Any, version: int, server_id: str): # Use a background task manager to handle syncing task_key = f"{table_name}:{pk_value}" if pk_value else table_name @@ -439,20 +440,20 @@ class APIConfig(BaseModel): if not primary_key: # Add an id column as primary key if it doesn't exist await conn.execute(f""" - ALTER TABLE "{table_name}" + ALTER TABLE "{table_name}" ADD COLUMN IF NOT EXISTS id SERIAL PRIMARY KEY; """) primary_key = 'id' # Ensure version column exists await conn.execute(f""" - ALTER TABLE "{table_name}" + ALTER TABLE "{table_name}" ADD COLUMN IF NOT EXISTS version INTEGER DEFAULT 1; """) # Ensure server_id column exists await conn.execute(f""" - ALTER TABLE "{table_name}" + ALTER TABLE "{table_name}" ADD COLUMN IF NOT EXISTS server_id TEXT DEFAULT '{os.environ.get('TS_ID')}'; """) @@ -487,7 +488,7 @@ class APIConfig(BaseModel): debug(f"Successfully ensured sync columns and trigger for table {table_name}") return primary_key - + except Exception as e: err(f"Error ensuring sync columns for table {table_name}: {str(e)}") err(f"Traceback: {traceback.format_exc()}") @@ -497,7 +498,7 @@ class APIConfig(BaseModel): if conn is None: debug(f"Skipping offline server...") return None - + try: result = await conn.fetchval("SELECT PostGIS_version();") if result: @@ -509,7 +510,7 @@ class APIConfig(BaseModel): except Exception as e: err(f"Error checking PostGIS: {str(e)}") return False - + async def pull_changes(self, source_pool_entry, batch_size=10000): if source_pool_entry['ts_id'] == os.environ.get('TS_ID'): @@ -538,10 +539,10 @@ class APIConfig(BaseModel): return 0 tables = await source_conn.fetch(""" - SELECT tablename FROM pg_tables + SELECT tablename FROM pg_tables WHERE schemaname = 'public' """) - + for table in tables: table_name = table['tablename'] try: @@ -550,7 +551,7 @@ class APIConfig(BaseModel): else: primary_key = await self.ensure_sync_columns(dest_conn, table_name) last_synced_version = await self.get_last_synced_version(dest_conn, table_name, source_id) - + while True: changes = await source_conn.fetch(f""" SELECT * FROM "{table_name}" @@ -558,16 +559,16 @@ class APIConfig(BaseModel): ORDER BY version ASC LIMIT $3 """, last_synced_version, source_id, batch_size) - + if not changes: break # No more changes for this table - + changes_count = await self.apply_batch_changes(dest_conn, table_name, changes, primary_key) total_changes += changes_count - + if changes_count > 0: info(f"Synced batch for {table_name}: {changes_count} changes. Total so far: {total_changes}") - + last_synced_version = changes[-1]['version'] # Update last synced version except Exception as e: @@ -599,7 +600,7 @@ class APIConfig(BaseModel): if conn is None: debug(f"Skipping offline server...") return 0 - + if table_name in self.SPECIAL_TABLES: debug(f"Skipping get_last_synced_version because {table_name} is special.") return 0 # Special handling for tables without version column @@ -623,15 +624,15 @@ class APIConfig(BaseModel): local_ts_id = os.environ.get('TS_ID') online_hosts = await self.get_online_hosts() num_online_hosts = len(online_hosts) - + if num_online_hosts > 0: online_ts_ids = [host['ts_id'] for host in online_hosts if host['ts_id'] != local_ts_id] crit(f"Online hosts: {', '.join(online_ts_ids)}") - + for pool_entry in online_hosts: if pool_entry['ts_id'] == local_ts_id: continue # Skip local database - + try: conn = await self.get_connection(pool_entry) if conn is None: @@ -639,14 +640,14 @@ class APIConfig(BaseModel): continue tables = await conn.fetch(""" - SELECT tablename FROM pg_tables + SELECT tablename FROM pg_tables WHERE schemaname = 'public' """) - + for table in tables: table_name = table['tablename'] if table_name in self.SPECIAL_TABLES: - continue + continue try: result = await conn.fetchrow(f""" SELECT MAX(version) as max_version, server_id @@ -684,7 +685,7 @@ class APIConfig(BaseModel): most_recent_source = next(host for host in online_hosts if host['ts_id'] != local_ts_id) else: crit("No other online hosts available for sync.") - + return most_recent_source @@ -737,7 +738,7 @@ class APIConfig(BaseModel): try: # Execute the query result = await conn.fetch(query, *args) - + if not result: continue @@ -846,9 +847,9 @@ class APIConfig(BaseModel): columns = updated_row.keys() placeholders = [f'${i+1}' for i in range(len(columns))] primary_key = self.get_primary_key(table_name) - + remote_version = await remote_conn.fetchval(f""" - SELECT version FROM "{table_name}" + SELECT version FROM "{table_name}" WHERE "{primary_key}" = $1 """, updated_row[primary_key]) @@ -920,17 +921,17 @@ class APIConfig(BaseModel): if all_rows: columns = list(all_rows[0].keys()) placeholders = [f'${i+1}' for i in range(len(columns))] - + insert_query = f""" INSERT INTO "{table_name}" ({', '.join(f'"{col}"' for col in columns)}) VALUES ({', '.join(placeholders)}) ON CONFLICT DO NOTHING """ - + async with remote_conn.transaction(): for row in all_rows: await remote_conn.execute(insert_query, *row.values()) - + return True else: debug(f"No rows to push for table {table_name}") @@ -976,7 +977,7 @@ class APIConfig(BaseModel): try: columns = list(changes[0].keys()) placeholders = [f'${i+1}' for i in range(len(columns))] - + if primary_key: insert_query = f""" INSERT INTO "{table_name}" ({', '.join(f'"{col}"' for col in columns)}) @@ -1057,7 +1058,7 @@ class APIConfig(BaseModel): proj4text = $4::text WHERE srid = $5::integer """ - await dest_conn.execute(update_query, + await dest_conn.execute(update_query, source_entry['auth_name'], source_entry['auth_srid'], source_entry['srtext'], @@ -1080,7 +1081,7 @@ class APIConfig(BaseModel): # You might want to cache this information for performance # For now, we'll assume it's always 'id', but you should implement proper logic here return 'id' - + async def add_primary_keys_to_local_tables(self): conn = await self.get_connection() @@ -1091,10 +1092,10 @@ class APIConfig(BaseModel): try: tables = await conn.fetch(""" - SELECT tablename FROM pg_tables + SELECT tablename FROM pg_tables WHERE schemaname = 'public' """) - + for table in tables: table_name = table['tablename'] if table_name not in self.SPECIAL_TABLES: @@ -1104,7 +1105,7 @@ class APIConfig(BaseModel): async def add_primary_keys_to_remote_tables(self): online_hosts = await self.get_online_hosts() - + for pool_entry in online_hosts: conn = await self.get_connection(pool_entry) if conn is None: @@ -1114,10 +1115,10 @@ class APIConfig(BaseModel): try: info(f"Adding primary keys to existing tables on {pool_entry['ts_id']}...") tables = await conn.fetch(""" - SELECT tablename FROM pg_tables + SELECT tablename FROM pg_tables WHERE schemaname = 'public' """) - + for table in tables: table_name = table['tablename'] if table_name not in self.SPECIAL_TABLES: @@ -1140,7 +1141,7 @@ class APIConfig(BaseModel): close_tasks = [] for pool_key, pool in self.db_pools.items(): close_tasks.append(self.close_pool_with_timeout(pool, pool_key)) - + await asyncio.gather(*close_tasks) self.db_pools.clear() info("All database connection pools closed.") @@ -1510,4 +1511,4 @@ class WidgetUpdate(BaseModel): color: Optional[str] = None url: Optional[str] = None shortcut: Optional[str] = None - graph: Optional[str] = None \ No newline at end of file + graph: Optional[str] = None diff --git a/sijapi/cli.py b/sijapi/cli.py deleted file mode 100644 index 0492540..0000000 --- a/sijapi/cli.py +++ /dev/null @@ -1,58 +0,0 @@ -# cli.py - -import click -import asyncio -from datetime import datetime as dt_datetime, timedelta - -# Import your async functions and dependencies -from your_main_app import build_daily_note_range_endpoint, loc - -def async_command(f): - @click.command() - @click.pass_context - def wrapper(ctx, *args, **kwargs): - async def run(): - return await f(*args, **kwargs) - return asyncio.run(run()) - return wrapper - -@click.group() -def cli(): - """CLI for your application.""" - pass - -@cli.command() -@click.argument('dt_start') -@click.argument('dt_end') -@async_command -async def bulk_note_range(dt_start: str, dt_end: str): - """ - Build daily notes for a date range. - - DT_START and DT_END should be in YYYY-MM-DD format. - """ - try: - start_date = dt_datetime.strptime(dt_start, "%Y-%m-%d") - end_date = dt_datetime.strptime(dt_end, "%Y-%m-%d") - except ValueError: - click.echo("Error: Dates must be in YYYY-MM-DD format.") - return - - if start_date > end_date: - click.echo("Error: Start date must be before or equal to end date.") - return - - results = [] - current_date = start_date - while current_date <= end_date: - formatted_date = await loc.dt(current_date) - result = await build_daily_note(formatted_date) - results.append(result) - current_date += timedelta(days=1) - - click.echo("Generated notes for the following dates:") - for url in results: - click.echo(url) - -if __name__ == '__main__': - cli() \ No newline at end of file diff --git a/sijapi/routers/email.py b/sijapi/routers/email.py index 74fac13..adfe790 100644 --- a/sijapi/routers/email.py +++ b/sijapi/routers/email.py @@ -1,5 +1,5 @@ ''' -Uses IMAP and SMTP login credentials to monitor an inbox and summarize incoming emails that match certain criteria and save the Text-To-Speech converted summaries into a specified "podcast" folder. +Uses IMAP and SMTP login credentials to monitor an inbox and summarize incoming emails that match certain criteria and save the Text-To-Speech converted summaries into a specified "podcast" folder. ''' #routers/email.py @@ -55,9 +55,9 @@ def get_smtp_connection(autoresponder: AutoResponder): context = ssl.create_default_context() context.check_hostname = False context.verify_mode = ssl.CERT_NONE - + smtp_config = autoresponder.smtp - + if smtp_config.encryption == 'SSL': try: debug(f"Attempting SSL connection to {smtp_config.host}:{smtp_config.port}") @@ -73,6 +73,7 @@ def get_smtp_connection(autoresponder: AutoResponder): except Exception as e: err(f"STARTTLS connection failed: {str(e)}") raise + elif smtp_config.encryption == 'STARTTLS': try: debug(f"Attempting STARTTLS connection to {smtp_config.host}:{smtp_config.port}") @@ -82,6 +83,7 @@ def get_smtp_connection(autoresponder: AutoResponder): except Exception as e: err(f"STARTTLS connection failed: {str(e)}") raise + else: try: debug(f"Attempting unencrypted connection to {smtp_config.host}:{smtp_config.port}") @@ -128,8 +130,6 @@ async def send_response(to_email: str, subject: str, body: str, profile: AutoRes err(f"Error closing SMTP connection: {str(e)}") - - def clean_email_content(html_content): soup = BeautifulSoup(html_content, "html.parser") return re.sub(r'[ \t\r\n]+', ' ', soup.get_text()).strip() @@ -141,11 +141,11 @@ async def extract_attachments(attachments) -> List[str]: attachment_name = attachment.get('filename', 'tempfile.txt') _, ext = os.path.splitext(attachment_name) ext = ext.lower() if ext else '.txt' - + with tempfile.NamedTemporaryFile(suffix=ext, delete=False) as tmp_file: tmp_file.write(attachment['content'].getvalue()) tmp_file_path = tmp_file.name - + try: attachment_text = await extract_text(tmp_file_path) attachment_texts.append(attachment_text) @@ -155,6 +155,7 @@ async def extract_attachments(attachments) -> List[str]: return attachment_texts + async def process_account_archival(account: EmailAccount): summarized_log = EMAIL_LOGS / account.name / "summarized.txt" os.makedirs(summarized_log.parent, exist_ok = True) @@ -177,7 +178,7 @@ async def process_account_archival(account: EmailAccount): recipients=recipients, subject=message.subject, body=clean_email_content(message.body['html'][0]) if message.body['html'] else clean_email_content(message.body['plain'][0]) or "", - attachments=message.attachments + attachments=message.attachments ) md_path, md_relative = assemble_journal_path(this_email.datetime_received, "Emails", this_email.subject, ".md") md_summary = await summarize_single_email(this_email, account.podcast) if account.summarize == True else None @@ -192,9 +193,10 @@ async def process_account_archival(account: EmailAccount): # debug(f"Skipping {uid_str} because it was already processed.") except Exception as e: err(f"An error occurred during summarization for account {account.name}: {e}") - + await asyncio.sleep(account.refresh) + async def summarize_single_email(this_email: IncomingEmail, podcast: bool = False): tts_path, tts_relative = assemble_journal_path(this_email.datetime_received, "Emails", this_email.subject, ".wav") summary = "" @@ -212,6 +214,7 @@ async def summarize_single_email(this_email: IncomingEmail, podcast: bool = Fals return md_summary + async def archive_single_email(this_email: IncomingEmail, summary: str = None): try: markdown_content = f'''--- @@ -219,14 +222,14 @@ date: {this_email.datetime_received.strftime('%Y-%m-%d')} tags: - email --- -| | | | -| --: | :--: | :--: | +| | | | +| --: | :--: | :--: | | *received* | **{this_email.datetime_received.strftime('%B %d, %Y at %H:%M:%S %Z')}** | | | *from* | **[[{this_email.sender}]]** | | | *to* | {', '.join([f'**[[{recipient.email}]]**' if not recipient.name else f'**[[{recipient.name}|{recipient.email}]]**' for recipient in this_email.recipients])} | | | *subject* | **{this_email.subject}** | | ''' - + if summary: markdown_content += summary @@ -235,11 +238,12 @@ tags: {this_email.body} ''' return markdown_content - + except Exception as e: err(f"Exception: {e}") return False - + + async def save_email(md_path, md_content): try: with open(md_path, 'w', encoding='utf-8') as md_file: @@ -251,6 +255,7 @@ async def save_email(md_path, md_content): err(f"Failed to save email: {e}") return False + def get_matching_autoresponders(this_email: IncomingEmail, account: EmailAccount) -> List[AutoResponder]: debug(f"Called get_matching_autoresponders for email \"{this_email.subject},\" account name \"{account.name}\"") def matches_list(item: str, this_email: IncomingEmail) -> bool: @@ -294,13 +299,14 @@ async def process_account_autoresponding(account: EmailAccount): except Exception as e: err(f"An error occurred during auto-responding for account {account.name}: {e}") - + await asyncio.sleep(account.refresh) + async def autorespond_single_email(message, uid_str: str, account: EmailAccount, log_file: Path): this_email = await create_incoming_email(message) debug(f"Evaluating {this_email.subject} for autoresponse-worthiness...") - + matching_profiles = get_matching_autoresponders(this_email, account) debug(f"Matching profiles: {matching_profiles}") @@ -319,6 +325,7 @@ async def autorespond_single_email(message, uid_str: str, account: EmailAccount, else: warn(f"Unable to generate auto-response for {this_email.subject}") + async def generate_response(this_email: IncomingEmail, profile: AutoResponder, account: EmailAccount) -> Optional[str]: info(f"Generating auto-response to {this_email.subject} with profile: {profile.name}") @@ -335,16 +342,16 @@ Body: {this_email.body} Respond on behalf of {account.fullname}, who is unable to respond personally because {profile.context}. Keep the response {profile.style} and to the point, but responsive to the sender's inquiry. Do not mention or recite this context information in your response. ''' sys_prompt = f"You are an AI assistant helping {account.fullname} with email responses. {account.fullname} is described as: {account.bio}" - + try: response = await llm.query_ollama(usr_prompt, sys_prompt, profile.ollama_model, 400) debug(f"query_ollama response: {response}") - + if isinstance(response, dict) and "message" in response and "content" in response["message"]: response = response["message"]["content"] - + return response + "\n\n" - + except Exception as e: err(f"Error generating auto-response: {str(e)}") return None @@ -363,22 +370,26 @@ async def create_incoming_email(message) -> IncomingEmail: attachments=message.attachments ) + async def load_processed_uids(filename: Path) -> Set[str]: if filename.exists(): async with aiofiles.open(filename, 'r') as f: return set(line.strip().split(':')[-1] for line in await f.readlines()) return set() + async def save_processed_uid(filename: Path, account_name: str, uid: str): async with aiofiles.open(filename, 'a') as f: await f.write(f"{account_name}:{uid}\n") + async def process_all_accounts(): email_accounts = load_email_accounts(EMAIL_CONFIG) summarization_tasks = [asyncio.create_task(process_account_archival(account)) for account in email_accounts] autoresponding_tasks = [asyncio.create_task(process_account_autoresponding(account)) for account in email_accounts] await asyncio.gather(*summarization_tasks, *autoresponding_tasks) + @email.on_event("startup") async def startup_event(): await asyncio.sleep(5) diff --git a/sijapi/routers/note.py b/sijapi/routers/note.py index 2fb18ab..9fdcc9e 100644 --- a/sijapi/routers/note.py +++ b/sijapi/routers/note.py @@ -22,7 +22,6 @@ from sijapi.routers import asr, cal, gis, img, llm, serve, timing, tts, weather from sijapi.utilities import assemble_journal_path, convert_to_12_hour_format, sanitize_filename, convert_degrees_to_cardinal, check_file_name, HOURLY_COLUMNS_MAPPING from sijapi.classes import Location - note = APIRouter() logger = L.get_module_logger("note") def debug(text: str): logger.debug(text) @@ -43,7 +42,6 @@ async def note_add_endpoint(file: Optional[UploadFile] = File(None), text: Optio return JSONResponse({"message": "Note added successfully", "entry": result}, status_code=201) - async def process_for_daily_note(file: Optional[UploadFile] = File(None), text: Optional[str] = None, source: Optional[str] = None, bg_tasks: BackgroundTasks = None): now = dt_datetime.now() transcription_entry = "" @@ -52,24 +50,24 @@ async def process_for_daily_note(file: Optional[UploadFile] = File(None), text: debug("File received...") file_content = await file.read() audio_io = BytesIO(file_content) - + # Improve error handling for file type guessing guessed_type = mimetypes.guess_type(file.filename) file_type = guessed_type[0] if guessed_type[0] else "application/octet-stream" - + debug(f"Processing as {file_type}...") - + # Extract the main type (e.g., 'audio', 'image', 'video') main_type = file_type.split('/')[0] subdir = main_type.title() if main_type else "Documents" - + absolute_path, relative_path = assemble_journal_path(now, subdir=subdir, filename=file.filename) debug(f"Destination path: {absolute_path}") - + with open(absolute_path, 'wb') as f: f.write(file_content) debug(f"Processing {f.name}...") - + if main_type == 'audio': transcription = await asr.transcribe_audio(file_path=absolute_path, params=asr.TranscribeParams(model="small-en", language="en", threads=6)) file_entry = f"![[{relative_path}]]" @@ -77,24 +75,22 @@ async def process_for_daily_note(file: Optional[UploadFile] = File(None), text: file_entry = f"![[{relative_path}]]" else: file_entry = f"[Source]({relative_path})" - + text_entry = text if text else "" debug(f"transcription: {transcription_entry}\nfile_entry: {file_entry}\ntext_entry: {text_entry}") return await add_to_daily_note(transcription_entry, file_entry, text_entry, now) - - async def add_to_daily_note(transcription: str = None, file_link: str = None, additional_text: str = None, date_time: dt_datetime = None): date_time = date_time or dt_datetime.now() note_path, _ = assemble_journal_path(date_time, filename='Notes', extension=".md", no_timestamp = True) time_str = date_time.strftime("%H:%M") - + entry_lines = [] if additional_text and additional_text.strip(): - entry_lines.append(f"\t* {additional_text.strip()}") + entry_lines.append(f"\t* {additional_text.strip()}") if transcription and transcription.strip(): - entry_lines.append(f"\t* {transcription.strip()}") + entry_lines.append(f"\t* {transcription.strip()}") if file_link and file_link.strip(): entry_lines.append(f"\t\t {file_link.strip()}") @@ -104,7 +100,7 @@ async def add_to_daily_note(transcription: str = None, file_link: str = None, ad if note_path.exists(): with open(note_path, 'a', encoding='utf-8') as note_file: note_file.write(entry) - else: + else: date_str = date_time.strftime("%Y-%m-%d") frontmatter = f"""--- date: {date_str} @@ -121,7 +117,6 @@ tags: return {"timestamp": time_str, "content": entry.strip()} - async def process_document( bg_tasks: BackgroundTasks, document: File, @@ -195,6 +190,7 @@ added: {timestamp} err(f"Failed to clip: {str(e)}") raise HTTPException(status_code=500, detail=str(e)) + def list_and_correct_impermissible_files(root_dir, rename: bool = False): """List and correct all files with impermissible names.""" impermissible_files = [] @@ -204,11 +200,11 @@ def list_and_correct_impermissible_files(root_dir, rename: bool = False): file_path = Path(dirpath) / filename impermissible_files.append(file_path) debug(f"Impermissible file found: {file_path}") - + # Sanitize the file name new_filename = sanitize_filename(filename) new_file_path = Path(dirpath) / new_filename - + # Ensure the new file name does not already exist if new_file_path.exists(): counter = 1 @@ -217,24 +213,23 @@ def list_and_correct_impermissible_files(root_dir, rename: bool = False): new_filename = f"{base_name}_{counter}{ext}" new_file_path = Path(dirpath) / new_filename counter += 1 - + # Rename the file if rename: os.rename(file_path, new_file_path) debug(f"Renamed: {file_path} -> {new_file_path}") - + return impermissible_files journal = OBSIDIAN_VAULT_DIR / "journal" list_and_correct_impermissible_files(journal, rename=True) -### Daily Note Builder ### @note.get("/note/bulk/{dt_start}/{dt_end}") async def build_daily_note_range_endpoint(dt_start: str, dt_end: str): start_date = dt_datetime.strptime(dt_start, "%Y-%m-%d") end_date = dt_datetime.strptime(dt_end, "%Y-%m-%d") - + results = [] current_date = start_date while current_date <= end_date: @@ -242,9 +237,8 @@ async def build_daily_note_range_endpoint(dt_start: str, dt_end: str): result = await build_daily_note(formatted_date) results.append(result) current_date += timedelta(days=1) - - return {"urls": results} + return {"urls": results} @note.get("/note/create") @@ -261,7 +255,7 @@ async def build_daily_note_getpoint(): date_time = dt_datetime.now(tz) path = await build_daily_note(date_time, loc.latitude, loc.longitude) path_str = str(path) - + info(f"Successfully created daily note at {path_str}") return JSONResponse(content={"path": path_str}, status_code=200) @@ -277,7 +271,6 @@ async def build_daily_note_getpoint(): raise HTTPException(status_code=500, detail="An unexpected error occurred") - @note.post("/note/create") async def build_daily_note_endpoint( date_str: Optional[str] = Form(dt_datetime.now().strftime("%Y-%m-%d")), @@ -305,7 +298,7 @@ async def build_daily_note_endpoint( path = await build_daily_note(date_time, lat, lon) path_str = str(path) # Convert PosixPath to string - + return JSONResponse(content={"path": path_str}, status_code=200) @@ -319,13 +312,13 @@ Obsidian helper. Takes a datetime and creates a new daily note. Note: it uses th day_before = (date_time - timedelta(days=1)).strftime("%Y-%m-%d %A") # 2024-05-26 Sunday formatting day_after = (date_time + timedelta(days=1)).strftime("%Y-%m-%d %A") # 2024-05-28 Tuesday formatting header = f"# [[{day_before}|← ]] {formatted_day} [[{day_after}| →]]\n\n" - + if not lat or not lon: places = await gis.fetch_locations(date_time) lat, lon = places[0].latitude, places[0].longitude location = await GEO.code((lat, lon)) - + timeslips = await build_daily_timeslips(date_time) fm_day = date_time.strftime("%Y-%m-%d") @@ -349,7 +342,7 @@ Obsidian helper. Takes a datetime and creates a new daily note. Note: it uses th map_embed = f"![[{map_path}]]" _, banner_path = assemble_journal_path(date_time, filename="Banner", extension=".jpg", no_timestamp = True) - + body = f"""--- date: "{fm_day}" banner: "![[{banner_path}]]" @@ -357,14 +350,14 @@ tags: - daily-note created: "{dt_datetime.now().strftime("%Y-%m-%d %H:%M:%S")}" --- - + {header} {weather_embed} {map_path} ## Events {event_embed} - + ## Tasks {task_embed} @@ -381,19 +374,15 @@ created: "{dt_datetime.now().strftime("%Y-%m-%d %H:%M:%S")}" banner = await generate_banner(formatted_day, location, weather_note) return absolute_path - async def build_daily_timeslips(date): - ''' - - ''' absolute_path, relative_path = assemble_journal_path(date, filename = "Timeslips", extension=".md", no_timestamp = True) content = await timing.process_timing_markdown(date, date) # document_content = await document.read() with open(absolute_path, 'wb') as f: f.write(content.encode()) - + return f"![[{relative_path}]]" @@ -402,21 +391,17 @@ async def update_frontmatter_endpoint(date: str, key: str, value: str): date_time = dt_datetime.strptime(date, "%Y-%m-%d") result = await update_frontmatter(date_time, key, value) return result - -async def update_frontmatter(date_time: dt_datetime, key: str, value: str): - # Parse the date and format paths - file_path, relative_path = assemble_journal_path(date_time) - # Check if the file exists + +async def update_frontmatter(date_time: dt_datetime, key: str, value: str): + file_path, relative_path = assemble_journal_path(date_time) if not file_path.exists(): crit(f"Markdown file not found at {file_path}") raise HTTPException(status_code=404, detail="Markdown file not found.") - # Read the file with open(file_path, "r", encoding="utf-8") as file: lines = file.readlines() - - # Extract the frontmatter + try: start_index = lines.index("---\n") + 1 end_index = lines[start_index:].index("---\n") + start_index @@ -424,33 +409,22 @@ async def update_frontmatter(date_time: dt_datetime, key: str, value: str): except ValueError: raise HTTPException(status_code=500, detail="Frontmatter not found.") - # Remove the existing key if present pattern = re.compile(f"^{key}:.*", re.IGNORECASE) frontmatter = [line for line in frontmatter if not pattern.match(line)] - - # Process value as a CSV string into a list values = value.split(',') - - # Determine insertion format if len(values) == 1: - # Single value, add as a simple key-value new_entry = f"{key}: {values[0]}\n" else: - # Multiple values, format as a list under the key new_entry = f"{key}:\n" + "\n".join([f" - {val}" for val in values]) + "\n" - # Insert the new key-value(s) frontmatter.append(new_entry) - - # Reassemble the file content = lines[:start_index] + frontmatter + ["---\n"] + lines[end_index + 1:] - - # Write changes back to the file with open(file_path, "w", encoding="utf-8") as file: file.writelines(content) return {"message": "Frontmatter updated successfully."} + @note.post("/note/banner") async def banner_endpoint(dt: str, location: str = None, forecast: str = None, mood: str = None, other_context: str = None): ''' @@ -483,6 +457,7 @@ async def generate_banner(dt, location: Location = None, forecast: str = None, m await update_frontmatter(date_time, "banner", jpg_embed) return local_path + async def generate_context(date_time, location: Location, forecast: str, mood: str, other_context: str): display_name = "Location: " if location and isinstance(location, Location): @@ -534,7 +509,7 @@ async def generate_context(date_time, location: Location, forecast: str, mood: s additional_info = ', '.join(formatted_events) if formatted_events else '' other_context = f"{other_context}, {additional_info}" if other_context else additional_info - other_context = f"Additional information: {other_context}" if other_context else "" + other_context = f"Additional information: {other_context}" if other_context else "" prompt = "Generate an aesthetically appealing banner image for a daily note that helps to visualize the following scene information: " prompt += "\n".join([display_name, forecast, mood, other_context]) @@ -542,7 +517,6 @@ async def generate_context(date_time, location: Location, forecast: str, mood: s return prompt - async def get_note(date_time: dt_datetime): date_time = await gis.dt(date_time); absolute_path, local_path = assemble_journal_path(date_time, filename = "Notes", extension = ".md", no_timestamp = True) @@ -552,6 +526,7 @@ async def get_note(date_time: dt_datetime): content = file.read() return content if content else None + async def sentiment_analysis(date_time: dt_datetime): most_recent_note = await get_note(date_time) most_recent_note = most_recent_note or await get_note(date_time - timedelta(days=1)) @@ -578,14 +553,14 @@ async def note_weather_get( debug(f"date: {date} .. date_time: {date_time}") content = await update_dn_weather(date_time, force_refresh_weather) #, lat, lon) return JSONResponse(content={"forecast": content}, status_code=200) - + except HTTPException as e: return JSONResponse(content={"detail": str(e.detail)}, status_code=e.status_code) except Exception as e: err(f"Error in note_weather_get: {str(e)}") raise HTTPException(status_code=500, detail=f"An error occurred: {str(e)}") - + @note.post("/update/note/{date}") async def post_update_daily_weather_and_calendar_and_timeslips(date: str, refresh: str="False") -> PlainTextResponse: @@ -597,6 +572,7 @@ async def post_update_daily_weather_and_calendar_and_timeslips(date: str, refres await build_daily_timeslips(date_time) return f"[Refresh]({API.URL}/update/note/{date_time.strftime('%Y-%m-%d')}" + async def update_dn_weather(date_time: dt_datetime, force_refresh: bool = False, lat: float = None, lon: float = None): warn(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our datetime in update_dn_weather.") try: @@ -609,7 +585,7 @@ async def update_dn_weather(date_time: dt_datetime, force_refresh: bool = False, place = places[0] lat = place.latitude lon = place.longitude - + debug(f"lat: {lat}, lon: {lon}, place: {place}") city = GEO.find_override_location(lat, lon) if city: @@ -626,7 +602,7 @@ async def update_dn_weather(date_time: dt_datetime, force_refresh: bool = False, city = location.name city = city if city else location.city city = city if city else location.house_number + ' ' + location.road - + debug(f"City geocoded: {city}") # Assemble journal path @@ -644,9 +620,9 @@ async def update_dn_weather(date_time: dt_datetime, force_refresh: bool = False, # debug(f"Day: {DailyWeather}") icon = DailyWeather.get('icon') debug(f"Icon: {icon}") - + weather_icon, admonition = get_icon_and_admonition(icon) if icon else (":LiSunMoon:", "ad-weather") - + temp = DailyWeather.get('feelslike') if DailyWeather.get('tempmax', 0) > 85: @@ -669,7 +645,7 @@ async def update_dn_weather(date_time: dt_datetime, force_refresh: bool = False, sunset = DailyWeather.get('sunset') srise_str = sunrise.time().strftime("%H:%M") sset_str = sunset.time().strftime("%H:%M") - + date_str = date_time.strftime("%Y-%m-%d") now = dt_datetime.now().strftime("%Y-%m-%d %H:%M:%S") @@ -701,17 +677,17 @@ async def update_dn_weather(date_time: dt_datetime, force_refresh: bool = False, for hour in HourlyWeather: if hour.get('datetime').strftime("%H:%M:%S") in HOURLY_COLUMNS_MAPPING.values(): - times.append(format_hourly_time(hour)) + times.append(format_hourly_time(hour)) condition_symbols.append(format_hourly_icon(hour, sunrise, sunset)) temps.append(format_hourly_temperature(hour)) winds.append(format_hourly_wind(hour)) - + detailed_forecast += assemble_hourly_data_table(times, condition_symbols, temps, winds) detailed_forecast += f"```\n\n" - + debug(f"Detailed forecast: {detailed_forecast}.") with open(absolute_path, 'w', encoding='utf-8') as note_file: @@ -725,12 +701,12 @@ async def update_dn_weather(date_time: dt_datetime, force_refresh: bool = False, else: err(f"Failed to get day") raise HTTPException(status_code=500, detail="Failed to retrieve weather data") - + except HTTPException as e: err(f"HTTP error: {e}") err(traceback.format_exc()) raise e - + except Exception as e: err(f"Error: {e}") err(traceback.format_exc()) @@ -740,12 +716,13 @@ async def update_dn_weather(date_time: dt_datetime, force_refresh: bool = False, err(f"Value error in update_dn_weather: {str(ve)}") err(traceback.format_exc()) raise HTTPException(status_code=400, detail=f"Value error: {str(ve)}") - + except Exception as e: err(f"Error in update_dn_weather: {str(e)}") err(traceback.format_exc()) raise HTTPException(status_code=500, detail=f"Error in update_dn_weather: {str(e)}") + def format_hourly_time(hour): try: hour_12 = convert_to_12_hour_format(hour.get("datetime")) @@ -754,21 +731,22 @@ def format_hourly_time(hour): err(f"Error in format_hourly_time: {str(e)}") err(traceback.format_exc()) return "" - + + def format_hourly_icon(hour, sunrise, sunset): try: icon_str = hour.get('icon', '') icon, _ = get_icon_and_admonition(icon_str) - + precip = hour.get('precip', float(0.0)) precip_prob = hour.get('precipprob', float(0.0)) debug(f"precip: {precip}, prob: {precip_prob}") - + sp_str = None if (precip > 0.05 and precip_prob > 25): precip_type = hour.get('preciptype', ['']) - sp_str = f"{str(precip)}mm" + sp_str = f"{str(precip)}mm" if abs(hour.get('datetime') - sunrise) < timedelta(minutes=60): icon = ":LiSunrise:" @@ -779,17 +757,18 @@ def format_hourly_icon(hour, sunrise, sunset): elif hour.get('uvindex') > 8: icon = ":LiRadiation:" sp_str = f"UV: {hour.get('uvindex', '')}" - + formatted = f"{icon}" if icon else "" formatted += f" {sp_str}" if sp_str else " " return formatted - + except Exception as e: err(f"Error in format_hourly_special: {str(e)}") err(traceback.format_exc()) return "" + def format_hourly_temperature(hour): try: temp_str = f"{hour.get('temp', '')}˚ F" @@ -798,7 +777,8 @@ def format_hourly_temperature(hour): err(f"Error in format_hourly_temperature: {str(e)}") err(traceback.format_exc()) return "" - + + def format_hourly_wind(hour): try: windspeed = hour.get('windspeed', '') @@ -823,7 +803,7 @@ def get_icon_and_admonition(icon_str) -> Tuple: debug(f"Received request for emoji {icon_str}") if icon_str.startswith(":") and icon_str.endswith(":"): return icon_str - + icon_str = icon_str.lower() if icon_str == "clear-day": @@ -876,34 +856,24 @@ def get_icon_and_admonition(icon_str) -> Tuple: ad = "ad-thunder" else: icon = ":LiHelpCircle:" - ad = "ad-weather" - + ad = "ad-weather" + return icon, ad def get_weather_emoji(weather_condition): condition = weather_condition.lower() - if 'clear' in condition or 'sunny' in condition: - return "☀️" - elif 'cloud' in condition or 'overcast' in condition: - return "☁️" - elif 'rain' in condition: - return "🌧️" - elif 'snow' in condition: - return "❄️" - elif 'thunder' in condition or 'storm' in condition: - return "⛈️" - elif 'fog' in condition or 'mist' in condition: - return "🌫️" - elif 'wind' in condition: - return "💨" - elif 'hail' in condition: - return "🌨️" - elif 'sleet' in condition: - return "🌧️" - elif 'partly' in condition: - return "⛅" - else: - return "🌡️" # Default emoji for unclassified weather + if 'clear' in condition or 'sunny' in condition: return "☀️" + elif 'cloud' in condition or 'overcast' in condition: return "☁️" + elif 'rain' in condition: return "🌧️" + elif 'snow' in condition: return "❄️" + elif 'thunder' in condition or 'storm' in condition: return "⛈️" + elif 'fog' in condition or 'mist' in condition: return "🌫️" + elif 'wind' in condition: return "💨" + elif 'hail' in condition: return "🌨️" + elif 'sleet' in condition: return "🌧️" + elif 'partly' in condition: return "⛅" + else: return "🌡️" # Default emoji for unclassified weather + async def format_events_as_markdown(event_data: Dict[str, Union[str, List[Dict[str, str]]]]) -> str: def remove_characters(s: str) -> str: @@ -911,7 +881,7 @@ async def format_events_as_markdown(event_data: Dict[str, Union[str, List[Dict[s s = s.strip('\n') s = re.sub(r'^_+|_+$', '', s) return s - + date_str = event_data["date"] now = dt_datetime.now().strftime("%Y-%m-%d %H:%M:%S") events_markdown = [] @@ -940,7 +910,7 @@ async def format_events_as_markdown(event_data: Dict[str, Union[str, List[Dict[s else: event_markdown += f"\n - [ ] **{event['start']}—{event['end']}** {markdown_name}" - + if event['attendees']: attendee_list = [] for att in event['attendees']: @@ -961,10 +931,10 @@ async def format_events_as_markdown(event_data: Dict[str, Union[str, List[Dict[s event_markdown += f"\n * {description}" event_markdown += f"\n " - + event_markdown += "\n```\n" events_markdown.append(event_markdown) - + header = ( "---\n" f"date: {date_str}\n" @@ -973,23 +943,25 @@ async def format_events_as_markdown(event_data: Dict[str, Union[str, List[Dict[s f"updated: {now}\n" "---\n" ) - + detailed_events = ( f"{header}" f"{''.join(events_markdown)}" ) return detailed_events + @note.get("/note/events", response_class=PlainTextResponse) async def note_events_endpoint(date: str = Query(None)): - + date_time = await gis.dt(date) if date else await gis.dt(dt_datetime.now()) response = await update_daily_note_events(date_time) return PlainTextResponse(content=response, status_code=200) + async def update_daily_note_events(date_time: dt_datetime): debug(f"Looking up events on date: {date_time.strftime('%Y-%m-%d')}") - try: + try: events = await cal.get_events(date_time, date_time) debug(f"Raw events: {events}") event_data = { @@ -1009,6 +981,3 @@ async def update_daily_note_events(date_time: dt_datetime): except Exception as e: err(f"Error processing events: {e}") raise HTTPException(status_code=500, detail=str(e)) - - -