From 14319edaa5d3333bb5c403a47dc0b275a3f9962f Mon Sep 17 00:00:00 2001 From: sanj <67624670+iodrift@users.noreply.github.com> Date: Mon, 12 Aug 2024 10:33:44 -0700 Subject: [PATCH] Auto-update: Mon Aug 12 10:33:44 PDT 2024 --- .gitignore | 1 + sijapi/helpers/fromvm/CaPLSS.py | 225 ++++ .../fromvm/CaPLSS_downloader_and_importer.py | 133 ++ sijapi/helpers/fromvm/CalFire_THP_scraper.py | 73 ++ sijapi/helpers/fromvm/article.py | 23 + sijapi/helpers/fromvm/db.py | 46 + sijapi/helpers/fromvm/db/docker-compose.yaml | 25 + sijapi/helpers/fromvm/db/init-db.sh | 11 + sijapi/helpers/fromvm/db_connection_test.py | 66 + sijapi/helpers/fromvm/db_get_schema.py | 89 ++ sijapi/helpers/fromvm/db_uuid_migrate.py | 46 + ...levenlabs_history_ids_20240630_131617.json | 104 ++ sijapi/helpers/fromvm/log_prior_emails.py | 63 + sijapi/helpers/fromvm/migrate_db_to_uuid.py | 191 +++ sijapi/helpers/fromvm/schema_info.yaml | 1103 +++++++++++++++++ sijapi/helpers/fromvm/start.py | 211 ++++ sijapi/helpers/fromvm/upscaler.py | 34 + 17 files changed, 2444 insertions(+) create mode 100644 sijapi/helpers/fromvm/CaPLSS.py create mode 100644 sijapi/helpers/fromvm/CaPLSS_downloader_and_importer.py create mode 100644 sijapi/helpers/fromvm/CalFire_THP_scraper.py create mode 100755 sijapi/helpers/fromvm/article.py create mode 100644 sijapi/helpers/fromvm/db.py create mode 100644 sijapi/helpers/fromvm/db/docker-compose.yaml create mode 100755 sijapi/helpers/fromvm/db/init-db.sh create mode 100644 sijapi/helpers/fromvm/db_connection_test.py create mode 100644 sijapi/helpers/fromvm/db_get_schema.py create mode 100644 sijapi/helpers/fromvm/db_uuid_migrate.py create mode 100644 sijapi/helpers/fromvm/elevenlabs_history_ids_20240630_131617.json create mode 100644 sijapi/helpers/fromvm/log_prior_emails.py create mode 100644 sijapi/helpers/fromvm/migrate_db_to_uuid.py create mode 100644 sijapi/helpers/fromvm/schema_info.yaml create mode 100644 sijapi/helpers/fromvm/start.py create mode 100644 sijapi/helpers/fromvm/upscaler.py diff --git a/.gitignore b/.gitignore index 6ae221a..e3d6881 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ podcast/sideloads/* **/.ipynb_checkpoints/ **/*.pem **/*.key +**/*.sql venv/ env/ .venv/ diff --git a/sijapi/helpers/fromvm/CaPLSS.py b/sijapi/helpers/fromvm/CaPLSS.py new file mode 100644 index 0000000..9d1457a --- /dev/null +++ b/sijapi/helpers/fromvm/CaPLSS.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 + +import requests +import json +import time +import os +import subprocess +import sys +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry +from datetime import datetime + +# Environment variables for database connection +DB_NAME = os.getenv('DB_NAME', 'sij') +DB_USER = os.getenv('DB_USER', 'sij') +DB_PASSWORD = os.getenv('DB_PASSWORD', 'Synchr0!') +DB_HOST = os.getenv('DB_HOST', 'localhost') +DB_PORT = os.getenv('DB_PORT', '5432') + +def get_feature_count(url): + params = { + 'where': '1=1', + 'returnCountOnly': 'true', + 'f': 'json' + } + retries = Retry(total=10, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504]) + with requests.Session() as session: + session.mount("https://", HTTPAdapter(max_retries=retries)) + response = session.get(url, params=params, timeout=30) + response.raise_for_status() + data = response.json() + return data.get('count', 0) + +def fetch_features(url, offset, num, max_retries=5): + params = { + 'where': '1=1', + 'outFields': '*', + 'geometryPrecision': 6, + 'outSR': 4326, + 'f': 'json', + 'resultOffset': offset, + 'resultRecordCount': num + } + for attempt in range(max_retries): + try: + retries = Retry(total=5, backoff_factor=1, status_forcelist=[500, 502, 503, 504]) + with requests.Session() as session: + session.mount("https://", HTTPAdapter(max_retries=retries)) + response = session.get(url, params=params, timeout=30) + response.raise_for_status() + return response.json() + except requests.exceptions.RequestException as e: + print(f"Error fetching features (attempt {attempt + 1}/{max_retries}): {e}") + if attempt == max_retries - 1: + raise + time.sleep(5 * (attempt + 1)) # Exponential backoff + +def download_layer(layer_num, layer_name): + base_dir = os.path.expanduser('~/data') + file_path = os.path.join(base_dir, f'PLSS_{layer_name}.geojson') + temp_file_path = os.path.join(base_dir, f'PLSS_{layer_name}_temp.json') + + url = f"https://gis.blm.gov/arcgis/rest/services/Cadastral/BLM_Natl_PLSS_CadNSDI/MapServer/{layer_num}/query" + + total_count = get_feature_count(url) + print(f"Total {layer_name} features: {total_count}") + + batch_size = 1000 + offset = 0 + all_features = [] + + # Check if temporary file exists and load its content + if os.path.exists(temp_file_path): + with open(temp_file_path, 'r') as f: + all_features = json.load(f) + offset = len(all_features) + print(f"Resuming download from offset {offset}") + + try: + while offset < total_count: + print(f"Fetching {layer_name} features {offset} to {offset + batch_size}...") + data = fetch_features(url, offset, batch_size) + + new_features = data.get('features', []) + if not new_features: + break + + all_features.extend(new_features) + offset += len(new_features) + + # Progress indicator + progress = len(all_features) / total_count + bar_length = 30 + filled_length = int(bar_length * progress) + bar = '=' * filled_length + '-' * (bar_length - filled_length) + print(f'\rProgress: [{bar}] {progress:.1%} ({len(all_features)}/{total_count} features)', end='', flush=True) + + # Save progress to temporary file + with open(temp_file_path, 'w') as f: + json.dump(all_features, f) + + time.sleep(1) + + print(f"\nTotal {layer_name} features fetched: {len(all_features)}") + + geojson_features = [ + { + "type": "Feature", + "properties": feature['attributes'], + "geometry": feature['geometry'] + } for feature in all_features + ] + + full_geojson = { + "type": "FeatureCollection", + "features": geojson_features + } + + os.makedirs(base_dir, exist_ok=True) + + with open(file_path, 'w') as f: + json.dump(full_geojson, f) + + print(f"GeoJSON file saved as '{file_path}'") + + # Remove temporary file + if os.path.exists(temp_file_path): + os.remove(temp_file_path) + + return file_path + except Exception as e: + print(f"\nError during download: {e}") + print(f"Partial data saved in {temp_file_path}") + return None + + +def check_postgres_connection(): + try: + subprocess.run(['psql', '-h', DB_HOST, '-p', DB_PORT, '-U', DB_USER, '-d', DB_NAME, '-c', 'SELECT 1;'], + check=True, capture_output=True, text=True) + return True + except subprocess.CalledProcessError: + return False + +def check_postgis_extension(): + try: + result = subprocess.run(['psql', '-h', DB_HOST, '-p', DB_PORT, '-U', DB_USER, '-d', DB_NAME, + '-c', "SELECT 1 FROM pg_extension WHERE extname = 'postgis';"], + check=True, capture_output=True, text=True) + return '1' in result.stdout + except subprocess.CalledProcessError: + return False + +def create_postgis_extension(): + try: + subprocess.run(['psql', '-h', DB_HOST, '-p', DB_PORT, '-U', DB_USER, '-d', DB_NAME, + '-c', "CREATE EXTENSION IF NOT EXISTS postgis;"], + check=True, capture_output=True, text=True) + print("PostGIS extension created successfully.") + except subprocess.CalledProcessError as e: + print(f"Error creating PostGIS extension: {e}") + sys.exit(1) + +def import_to_postgis(file_path, table_name): + if not check_postgres_connection(): + print("Error: Unable to connect to PostgreSQL. Please check your connection settings.") + sys.exit(1) + + if not check_postgis_extension(): + print("PostGIS extension not found. Attempting to create it...") + create_postgis_extension() + + ogr2ogr_command = [ + 'ogr2ogr', + '-f', 'PostgreSQL', + f'PG:dbname={DB_NAME} user={DB_USER} password={DB_PASSWORD} host={DB_HOST} port={DB_PORT}', + file_path, + '-nln', table_name, + '-overwrite' + ] + + try: + subprocess.run(ogr2ogr_command, check=True, capture_output=True, text=True) + print(f"Data successfully imported into PostGIS table: {table_name}") + except subprocess.CalledProcessError as e: + print(f"Error importing data into PostGIS: {e}") + print(f"Command that failed: {e.cmd}") + print(f"Error output: {e.stderr}") + +def check_ogr2ogr(): + try: + subprocess.run(['ogr2ogr', '--version'], check=True, capture_output=True, text=True) + return True + except subprocess.CalledProcessError: + return False + except FileNotFoundError: + return False + +def main(): + if not check_ogr2ogr(): + print("Error: ogr2ogr not found. Please install GDAL/OGR tools.") + print("On Debian: sudo apt-get install gdal-bin") + print("On macOS with Homebrew: brew install gdal") + sys.exit(1) + + try: + township_file = os.path.expanduser('~/data/PLSS_Townships.geojson') + if not os.path.exists(township_file): + township_file = download_layer(1, "Townships") + if township_file: + import_to_postgis(township_file, "public.plss_townships") + + section_file = os.path.expanduser('~/data/PLSS_Sections.geojson') + if not os.path.exists(section_file): + section_file = download_layer(2, "Sections") + if section_file: + import_to_postgis(section_file, "public.plss_sections") + + except requests.exceptions.RequestException as e: + print(f"Error fetching data: {e}") + except Exception as e: + print(f"An unexpected error occurred: {e}") + +if __name__ == "__main__": + main() diff --git a/sijapi/helpers/fromvm/CaPLSS_downloader_and_importer.py b/sijapi/helpers/fromvm/CaPLSS_downloader_and_importer.py new file mode 100644 index 0000000..5b3ea00 --- /dev/null +++ b/sijapi/helpers/fromvm/CaPLSS_downloader_and_importer.py @@ -0,0 +1,133 @@ +# CaPLSS_downloader_and_importer.py +import requests +import json +import time +import os +import subprocess +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +def get_feature_count(url): + params = { + 'where': '1=1', + 'returnCountOnly': 'true', + 'f': 'json' + } + retries = Retry(total=10, backoff_factor=0.5) + adapter = HTTPAdapter(max_retries=retries) + session = requests.Session() + session.mount("https://", adapter) + + response = session.get(url, params=params, timeout=15) # Add timeout parameter + response.raise_for_status() + data = response.json() + return data.get('count', 0) + + +def fetch_features(url, offset, num): + params = { + 'where': '1=1', + 'outFields': '*', + 'geometryPrecision': 6, + 'outSR': 4326, + 'f': 'json', + 'resultOffset': offset, + 'resultRecordCount': num + } + response = requests.get(url, params=params) + response.raise_for_status() + return response.json() + +def download_layer(layer_num, layer_name): + url = f"https://gis.blm.gov/arcgis/rest/services/Cadastral/BLM_Natl_PLSS_CadNSDI/MapServer/{layer_num}/query" + + total_count = get_feature_count(url) + print(f"Total {layer_name} features: {total_count}") + + batch_size = 1000 + offset = 0 + all_features = [] + + while offset < total_count: + print(f"Fetching {layer_name} features {offset} to {offset + batch_size}...") + data = fetch_features(url, offset, batch_size) + + new_features = data.get('features', []) + if not new_features: + break + + all_features.extend(new_features) + offset += len(new_features) + + print(f"Progress: {len(all_features)}/{total_count} features") + + time.sleep(1) # Be nice to the server + + print(f"Total {layer_name} features fetched: {len(all_features)}") + + # Convert to GeoJSON + geojson_features = [ + { + "type": "Feature", + "properties": feature['attributes'], + "geometry": feature['geometry'] + } for feature in all_features + ] + + full_geojson = { + "type": "FeatureCollection", + "features": geojson_features + } + + # Define a base directory that exists on both macOS and Debian + base_dir = os.path.expanduser('~/data') + os.makedirs(base_dir, exist_ok=True) # Create the directory if it doesn't exist + + # Use os.path.join to construct the file path + file_path = os.path.join(base_dir, f'PLSS_{layer_name}.geojson') + + # Save to file + with open(file_path, 'w') as f: + json.dump(full_geojson, f) + + print(f"GeoJSON file saved as '{file_path}'") + + return file_path + +def import_to_postgis(file_path, table_name): + db_name = 'sij' + db_user = 'sij' + db_password = 'Synchr0!' + + ogr2ogr_command = [ + 'ogr2ogr', + '-f', 'PostgreSQL', + f'PG:dbname={db_name} user={db_user} password={db_password}', + file_path, + '-nln', table_name, + '-overwrite' + ] + + subprocess.run(ogr2ogr_command, check=True) + print(f"Data successfully imported into PostGIS table: {table_name}") + +def main(): + try: + # Download and import Townships (Layer 1) + township_file = download_layer(1, "Townships") + import_to_postgis(township_file, "public.plss_townships") + + # Download and import Sections (Layer 2) + section_file = download_layer(2, "Sections") + import_to_postgis(section_file, "public.plss_sections") + + except requests.exceptions.RequestException as e: + print(f"Error fetching data: {e}") + except subprocess.CalledProcessError as e: + print(f"Error importing data into PostGIS: {e}") + except Exception as e: + print(f"An unexpected error occurred: {e}") + +if __name__ == "__main__": + main() diff --git a/sijapi/helpers/fromvm/CalFire_THP_scraper.py b/sijapi/helpers/fromvm/CalFire_THP_scraper.py new file mode 100644 index 0000000..92909a6 --- /dev/null +++ b/sijapi/helpers/fromvm/CalFire_THP_scraper.py @@ -0,0 +1,73 @@ +import requests +import PyPDF2 +import io +import re + +def scrape_data_from_pdf(url): + response = requests.get(url) + pdf_file = io.BytesIO(response.content) + + pdf_reader = PyPDF2.PdfReader(pdf_file) + + all_text = "" + for page in pdf_reader.pages: + all_text += page.extract_text() + "\n" + + return all_text + +def parse_data(raw_data): + lines = raw_data.split('\n') + data = [] + current_entry = None + + for line in lines: + line = line.strip() + if re.match(r'\d+-\d+-\d+-\w+', line): + if current_entry: + data.append(current_entry) + current_entry = {'Harvest Document': line, 'Raw Data': []} + elif current_entry: + current_entry['Raw Data'].append(line) + + if current_entry: + data.append(current_entry) + + return data + +def filter_data(data): + return [entry for entry in data if any(owner.lower() in ' '.join(entry['Raw Data']).lower() for owner in ["Sierra Pacific", "SPI", "Land & Timber"])] + +def extract_location(raw_data): + location = [] + for line in raw_data: + if 'MDBM:' in line or 'HBM:' in line: + location.append(line) + return ' '.join(location) + +def extract_plss_coordinates(text): + pattern = r'(\w+): T(\d+)([NSEW]) R(\d+)([NSEW]) S(\d+)' + return re.findall(pattern, text) + +# Main execution +url = "https://caltreesplans.resources.ca.gov/Caltrees/Report/ShowReport.aspx?module=TH_Document&reportID=492&reportType=LINK_REPORT_LIST" +raw_data = scrape_data_from_pdf(url) + +parsed_data = parse_data(raw_data) +print(f"Total timber plans parsed: {len(parsed_data)}") + +filtered_data = filter_data(parsed_data) +print(f"Found {len(filtered_data)} matching entries.") + +for plan in filtered_data: + print("\nHarvest Document:", plan['Harvest Document']) + + location = extract_location(plan['Raw Data']) + print("Location:", location) + + plss_coordinates = extract_plss_coordinates(location) + print("PLSS Coordinates:") + for coord in plss_coordinates: + meridian, township, township_dir, range_, range_dir, section = coord + print(f" {meridian}: T{township}{township_dir} R{range_}{range_dir} S{section}") + + print("-" * 50) diff --git a/sijapi/helpers/fromvm/article.py b/sijapi/helpers/fromvm/article.py new file mode 100755 index 0000000..c76a6e1 --- /dev/null +++ b/sijapi/helpers/fromvm/article.py @@ -0,0 +1,23 @@ +#!/Users/sij/miniforge3/envs/sijapi/bin/python +import sys +import asyncio +from fastapi import BackgroundTasks +from sijapi.routers.news import process_and_save_article + +async def main(): + if len(sys.argv) != 2: + print("Usage: python script.py ") + sys.exit(1) + + url = sys.argv[1] + bg_tasks = BackgroundTasks() + + try: + result = await process_and_save_article(bg_tasks, url) + print(result) + except Exception as e: + print(f"Error processing article: {str(e)}") + sys.exit(1) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/sijapi/helpers/fromvm/db.py b/sijapi/helpers/fromvm/db.py new file mode 100644 index 0000000..b7bb5d1 --- /dev/null +++ b/sijapi/helpers/fromvm/db.py @@ -0,0 +1,46 @@ +import asyncio +import asyncpg + +# Database connection information +DB_INFO = { + 'host': '100.64.64.20', + 'port': 5432, + 'database': 'sij', + 'user': 'sij', + 'password': 'Synchr0!' +} + +async def update_click_logs(): + # Connect to the database + conn = await asyncpg.connect(**DB_INFO) + + try: + # Drop existing 'id' and 'new_id' columns if they exist + await conn.execute(""" + ALTER TABLE click_logs + DROP COLUMN IF EXISTS id, + DROP COLUMN IF EXISTS new_id; + """) + print("Dropped existing id and new_id columns (if they existed)") + + # Add new UUID column as primary key + await conn.execute(""" + ALTER TABLE click_logs + ADD COLUMN id UUID PRIMARY KEY DEFAULT gen_random_uuid(); + """) + print("Added new UUID column as primary key") + + # Get the number of rows in the table + row_count = await conn.fetchval("SELECT COUNT(*) FROM click_logs") + print(f"Number of rows in click_logs: {row_count}") + + except Exception as e: + print(f"An error occurred: {str(e)}") + import traceback + traceback.print_exc() + finally: + # Close the database connection + await conn.close() + +# Run the update +asyncio.run(update_click_logs()) diff --git a/sijapi/helpers/fromvm/db/docker-compose.yaml b/sijapi/helpers/fromvm/db/docker-compose.yaml new file mode 100644 index 0000000..de8fa37 --- /dev/null +++ b/sijapi/helpers/fromvm/db/docker-compose.yaml @@ -0,0 +1,25 @@ +version: '3.8' + +services: + db: + image: postgis/postgis:16-3.4 + container_name: sij_postgres + environment: + POSTGRES_DB: sij + POSTGRES_USER: sij + POSTGRES_PASSWORD: Synchr0! + volumes: + - postgres_data:/var/lib/postgresql/data + - ./init-db.sh:/docker-entrypoint-initdb.d/init-db.sh + ports: + - "5432:5432" + networks: + - sij_network + +networks: + sij_network: + driver: bridge + +volumes: + postgres_data: + diff --git a/sijapi/helpers/fromvm/db/init-db.sh b/sijapi/helpers/fromvm/db/init-db.sh new file mode 100755 index 0000000..d354e40 --- /dev/null +++ b/sijapi/helpers/fromvm/db/init-db.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + CREATE EXTENSION IF NOT EXISTS postgis; + CREATE EXTENSION IF NOT EXISTS postgis_topology; +EOSQL + +# Modify pg_hba.conf to allow connections from Tailscale network +echo "host all all 100.64.64.0/24 trust" >> /var/lib/postgresql/data/pg_hba.conf + diff --git a/sijapi/helpers/fromvm/db_connection_test.py b/sijapi/helpers/fromvm/db_connection_test.py new file mode 100644 index 0000000..0d94404 --- /dev/null +++ b/sijapi/helpers/fromvm/db_connection_test.py @@ -0,0 +1,66 @@ +import asyncio +import asyncpg +import psycopg2 +import sys + +async def try_async_connect(host, port, user, password, database): + try: + conn = await asyncpg.connect( + host=host, + port=port, + user=user, + password=password, + database=database + ) + version = await conn.fetchval('SELECT version()') + print(f"Async connection successful to {host}:{port}") + print(f"PostgreSQL version: {version}") + await conn.close() + return True + except Exception as e: + print(f"Async connection failed to {host}:{port}") + print(f"Error: {str(e)}") + return False + +def try_sync_connect(host, port, user, password, database): + try: + conn = psycopg2.connect( + host=host, + port=port, + user=user, + password=password, + database=database + ) + cur = conn.cursor() + cur.execute('SELECT version()') + version = cur.fetchone()[0] + print(f"Sync connection successful to {host}:{port}") + print(f"PostgreSQL version: {version}") + conn.close() + return True + except Exception as e: + print(f"Sync connection failed to {host}:{port}") + print(f"Error: {str(e)}") + return False + +async def main(): + # Database connection parameters + port = 5432 + user = 'sij' + password = 'Synchr0!' + database = 'sij' + + hosts = ['100.64.64.20', '127.0.0.1', 'localhost'] + + print("Attempting asynchronous connections:") + for host in hosts: + await try_async_connect(host, port, user, password, database) + print() + + print("Attempting synchronous connections:") + for host in hosts: + try_sync_connect(host, port, user, password, database) + print() + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/sijapi/helpers/fromvm/db_get_schema.py b/sijapi/helpers/fromvm/db_get_schema.py new file mode 100644 index 0000000..c672c6e --- /dev/null +++ b/sijapi/helpers/fromvm/db_get_schema.py @@ -0,0 +1,89 @@ +import psycopg2 +from psycopg2 import sql + +def connect_to_db(): + return psycopg2.connect( + dbname='sij', + user='sij', + password='Synchr0!', + host='localhost' # Adjust if your database is not on localhost + ) + +def get_table_info(conn): + with conn.cursor() as cur: + # Get all tables in the public schema + cur.execute(""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + """) + tables = cur.fetchall() + + table_info = {} + for (table_name,) in tables: + table_info[table_name] = { + 'primary_keys': get_primary_keys(cur, table_name), + 'foreign_keys': get_foreign_keys(cur, table_name) + } + + return table_info + +def get_primary_keys(cur, table_name): + cur.execute(""" + SELECT a.attname + FROM pg_index i + JOIN pg_attribute a ON a.attrelid = i.indrelid + AND a.attnum = ANY(i.indkey) + WHERE i.indrelid = %s::regclass + AND i.indisprimary + """, (table_name,)) + return [row[0] for row in cur.fetchall()] + +def get_foreign_keys(cur, table_name): + cur.execute(""" + SELECT + tc.constraint_name, + kcu.column_name, + ccu.table_name AS foreign_table_name, + ccu.column_name AS foreign_column_name + FROM + information_schema.table_constraints AS tc + JOIN information_schema.key_column_usage AS kcu + ON tc.constraint_name = kcu.constraint_name + AND tc.table_schema = kcu.table_schema + JOIN information_schema.constraint_column_usage AS ccu + ON ccu.constraint_name = tc.constraint_name + AND ccu.table_schema = tc.table_schema + WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name=%s + """, (table_name,)) + return cur.fetchall() + +def main(): + try: + with connect_to_db() as conn: + table_info = get_table_info(conn) + + for table_name, info in table_info.items(): + print(f"\n## Table: {table_name}") + + print("\nPrimary Keys:") + if info['primary_keys']: + for pk in info['primary_keys']: + print(f"- {pk}") + else: + print("- No primary keys found") + + print("\nForeign Keys:") + if info['foreign_keys']: + for fk in info['foreign_keys']: + print(f"- {fk[1]} -> {fk[2]}.{fk[3]} (Constraint: {fk[0]})") + else: + print("- No foreign keys found") + + except psycopg2.Error as e: + print(f"Database error: {e}") + except Exception as e: + print(f"An unexpected error occurred: {e}") + +if __name__ == "__main__": + main() diff --git a/sijapi/helpers/fromvm/db_uuid_migrate.py b/sijapi/helpers/fromvm/db_uuid_migrate.py new file mode 100644 index 0000000..b7bb5d1 --- /dev/null +++ b/sijapi/helpers/fromvm/db_uuid_migrate.py @@ -0,0 +1,46 @@ +import asyncio +import asyncpg + +# Database connection information +DB_INFO = { + 'host': '100.64.64.20', + 'port': 5432, + 'database': 'sij', + 'user': 'sij', + 'password': 'Synchr0!' +} + +async def update_click_logs(): + # Connect to the database + conn = await asyncpg.connect(**DB_INFO) + + try: + # Drop existing 'id' and 'new_id' columns if they exist + await conn.execute(""" + ALTER TABLE click_logs + DROP COLUMN IF EXISTS id, + DROP COLUMN IF EXISTS new_id; + """) + print("Dropped existing id and new_id columns (if they existed)") + + # Add new UUID column as primary key + await conn.execute(""" + ALTER TABLE click_logs + ADD COLUMN id UUID PRIMARY KEY DEFAULT gen_random_uuid(); + """) + print("Added new UUID column as primary key") + + # Get the number of rows in the table + row_count = await conn.fetchval("SELECT COUNT(*) FROM click_logs") + print(f"Number of rows in click_logs: {row_count}") + + except Exception as e: + print(f"An error occurred: {str(e)}") + import traceback + traceback.print_exc() + finally: + # Close the database connection + await conn.close() + +# Run the update +asyncio.run(update_click_logs()) diff --git a/sijapi/helpers/fromvm/elevenlabs_history_ids_20240630_131617.json b/sijapi/helpers/fromvm/elevenlabs_history_ids_20240630_131617.json new file mode 100644 index 0000000..d10f7da --- /dev/null +++ b/sijapi/helpers/fromvm/elevenlabs_history_ids_20240630_131617.json @@ -0,0 +1,104 @@ +{ + "history_item_ids": [ + "ncRYNd0Xef4LiUE74VjP", + "13pQLDAPYGIATwW1ySL5", + "dhsQNAYTWpcwo1X6rixf", + "V7wUip1NJuWAUw26sePF", + "mOYMa5lcI7wRHddIQTSa", + "mP97iOpA4oG7pwUBthq4", + "WTU5nsX6qZCYxLyoT5hq", + "15DPGnBgjr74KT3TMbK4", + "aCyBS1zoaweVjUoPf2TF", + "J8SUMQqZPtoy3Cgdhi3J", + "qKHaaJHfqh2je60Wmadb", + "2PaugQJ8c4rY44JGlaO5", + "TwzxcmYjo6XNebbMabcd", + "xdEK7rYq9UofOlkr565b", + "wik4jYd97aGMLgttTjC9", + "7oXn2yH7gdyhi6sEoWKd", + "jv8aZFiVe8gPMrAOBcNT", + "B2BctCDkCtLDxEMMBu9z", + "4KFO77NHDruNQvXIykwp", + "d033NizZaNZPc45fvxCO", + "yBKxOxfzsjpZYOFzoIM7", + "oEihKwMLWgvvoTLGx4yF", + "Q3guBm4hGml0KPAWKl7t", + "jaojY1gSafQmqshR48oT", + "yqGDMfcceaoceFEEurqa", + "oLdnyUp7plGrUMRVQ8Cf", + "FZAGCGosYEGMf8GCRFaA", + "TrWnXRdGkiH0K9kgwFiS", + "th16OEbg3u0XHslT9A33", + "856BAsn6dnzF7HeqGPfK", + "KjLoAfDXVBqR9s39T25j", + "uHQQJMMOfOxPAhEYQXLl", + "HO8WCIhkkI7AxwkU5MC6", + "9nxdesHWTRLCOd6YgWe9", + "tmx5tlIQ7hdSTgJt16P2", + "M9JN0YcBuCF6LhnqKN66", + "M9xkP4ecn0LIi7mQOfU6", + "CNtJgh52Ykh9ZqEppZeH", + "lgobcoiqmtWfbXkhEwbE", + "nr9jxnsE4DnwmTwCaHqC", + "Rnzo03tcyBqGPdmHemCb", + "X3YVGp7yf9GLgZ7WOuSU", + "wL3bkqxR9xqeFTvkJpSI", + "wNx3XDgFLTjVbMyGrIAO", + "rb0jj1ywBetmdvve5qIL", + "WdNnqvNswXeh6JFoaRSS", + "WT2ViyerKpodYmHDHhCw", + "OvhIRehXNwx7xMJHuTd7", + "EQb1iZtsADxJ0GxLJzEK", + "WXVfBJYoYGB7S61VyETD", + "q0q3Di1YJKF07dOhoa7E", + "a2XBIUPa68UiiKlzwFnG", + "YBuD7KsUpz8jxc5ItZcF", + "KdoucRVCVQGRVQ8Di9Ih", + "CkmDny98GEdfGuj2kaAx", + "R0R2p8luRZL7wwPtDilw", + "awvztgQnuaquK0dTpIuH", + "3ZPN0nJo8UQZYhFhoIOK", + "RJJeTkjYIgdv1ZoXXAax", + "ppxUNzWHAQafsM6OvEUE", + "f2VBm7yE7qmnjdS9CbYz", + "SZIMwz2T5ZAhTxTDBFol", + "YjC91PRgnQbAcdPhnWqU", + "fDTV7n8f6QK5yCwLkBwg", + "KbPpWUuiLPADj9H3OlvG", + "DIuqVoAg7lLxpvFBip84", + "pEwFAKMLGWUMHqfljJSq", + "9wwl7UbsgeKqrk8kNZin", + "2uLvjJgcZDiY9dqB8JlP", + "U5f1qZQM08t2YzJqEmxK", + "gnwn7QIhrCXRAGNddZ1H", + "g5nGEIHirFzKstdrGI1h", + "CQWH5dGSeS38VC4X4yg7", + "C5YGjhJPrTkVOpxIOHdj", + "YLbtnf1pSb9Ra7wgFHiF", + "qNLgNSvMr4VSoisKS9qj", + "Bq2ALvQVsj9L2wMpUvYO", + "gi0yTXLZLMhUKeKcalWc", + "3JQN9UbCsqj9ggi5sCkq", + "oPflJoA9kqBzjlmWY6zL", + "0kUZFgtZdqgdUBXFsXs9", + "aFTi7XdjR8W52ThmFpgc", + "pgIfjcy2UvKggfqJ1aNx", + "r0VguLaqnxTL9jza9H4y", + "444ehr4RtqgU1xjhhTLo", + "pEuzoznVDaQRBhIA9VTy", + "T9hdW9eJkEqDmOsSUoeY", + "wJjHbGzoWiKKOIGmf82T", + "kij4uMmkUlsSDu2zSH1k", + "oWt5rns196JsKIYPyrBS", + "SJ1m9mSOGOLIhkMgA8kq", + "kAaqe0ATrYtkifmZLOE5", + "O2Pvz7CP5rfyNvzFSDmy", + "w1rb8qN5nohVUovC0XAx", + "njFs4I4F7rtd9I6fEn6x", + "miFrp9GBm3MsHO03Z4eY", + "5DJywiPsfeVP9hFdqRhd", + "mUephoXhk5QdWrOfr9Xr", + "tDDiW3Yp0BptZ2wBv21A", + "YpX06liXWHquUVYFlKYa" + ] +} \ No newline at end of file diff --git a/sijapi/helpers/fromvm/log_prior_emails.py b/sijapi/helpers/fromvm/log_prior_emails.py new file mode 100644 index 0000000..10148cf --- /dev/null +++ b/sijapi/helpers/fromvm/log_prior_emails.py @@ -0,0 +1,63 @@ +import asyncio +from pathlib import Path +from sijapi import L, EMAIL_CONFIG, EMAIL_LOGS +from sijapi.classes import EmailAccount +from sijapi.routers import email + +async def initialize_log_files(): + summarized_log = EMAIL_LOGS / "summarized.txt" + autoresponded_log = EMAIL_LOGS / "autoresponded.txt" + diagnostic_log = EMAIL_LOGS / "diagnostic.txt" + for log_file in [summarized_log, autoresponded_log, diagnostic_log]: + log_file.parent.mkdir(parents=True, exist_ok=True) + log_file.write_text("") + L.DEBUG(f"Log files initialized: {summarized_log}, {autoresponded_log}, {diagnostic_log}") + return summarized_log, autoresponded_log, diagnostic_log + +async def process_all_emails(account: EmailAccount, summarized_log: Path, autoresponded_log: Path, diagnostic_log: Path): + try: + with email.get_imap_connection(account) as inbox: + L.DEBUG(f"Connected to {account.name}, processing all emails...") + all_messages = inbox.messages() + unread_messages = set(uid for uid, _ in inbox.messages(unread=True)) + + processed_count = 0 + for identifier, message in all_messages: + # Log diagnostic information + with open(diagnostic_log, 'a') as f: + f.write(f"Account: {account.name}, Raw Identifier: {identifier}, Type: {type(identifier)}\n") + + # Attempt to get a string representation of the identifier + if isinstance(identifier, bytes): + id_str = identifier.decode() + elif isinstance(identifier, (int, str)): + id_str = str(identifier) + else: + id_str = repr(identifier) + + if identifier not in unread_messages: + processed_count += 1 + for log_file in [summarized_log, autoresponded_log]: + with open(log_file, 'a') as f: + f.write(f"{id_str}\n") + + L.INFO(f"Processed {processed_count} non-unread emails for account {account.name}") + except Exception as e: + L.logger.error(f"An error occurred while processing emails for account {account.name}: {e}") + +async def main(): + email_accounts = email.load_email_accounts(EMAIL_CONFIG) + summarized_log, autoresponded_log, diagnostic_log = await initialize_log_files() + + L.DEBUG(f"Processing {len(email_accounts)} email accounts") + + tasks = [process_all_emails(account, summarized_log, autoresponded_log, diagnostic_log) for account in email_accounts] + await asyncio.gather(*tasks) + + # Final verification + with open(summarized_log, 'r') as f: + final_count = len(f.readlines()) + L.INFO(f"Final non-unread email count: {final_count}") + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/sijapi/helpers/fromvm/migrate_db_to_uuid.py b/sijapi/helpers/fromvm/migrate_db_to_uuid.py new file mode 100644 index 0000000..60034fb --- /dev/null +++ b/sijapi/helpers/fromvm/migrate_db_to_uuid.py @@ -0,0 +1,191 @@ +import psycopg2 +from psycopg2 import sql +import sys + +def connect_to_db(): + return psycopg2.connect( + dbname='sij', + user='sij', + password='Synchr0!', + host='localhost' + ) + +def get_tables(cur): + cur.execute(""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' AND table_type = 'BASE TABLE' + AND table_name NOT LIKE '%_uuid' AND table_name NOT LIKE '%_orig' + AND table_name != 'spatial_ref_sys' + """) + return [row[0] for row in cur.fetchall()] + +def get_columns(cur, table_name): + cur.execute(""" + SELECT column_name, udt_name, + is_nullable, column_default, + character_maximum_length, numeric_precision, numeric_scale + FROM information_schema.columns + WHERE table_name = %s + ORDER BY ordinal_position + """, (table_name,)) + return cur.fetchall() + +def get_constraints(cur, table_name): + cur.execute(""" + SELECT conname, contype, pg_get_constraintdef(c.oid) + FROM pg_constraint c + JOIN pg_namespace n ON n.oid = c.connamespace + WHERE conrelid = %s::regclass + AND n.nspname = 'public' + """, (table_name,)) + return cur.fetchall() + +def drop_table_if_exists(cur, table_name): + cur.execute(sql.SQL("DROP TABLE IF EXISTS {} CASCADE").format(sql.Identifier(table_name))) + +def create_uuid_table(cur, old_table, new_table): + drop_table_if_exists(cur, new_table) + columns = get_columns(cur, old_table) + constraints = get_constraints(cur, old_table) + + column_defs = [] + has_id_column = any(col[0] == 'id' for col in columns) + + for col in columns: + col_name, udt_name, is_nullable, default, max_length, precision, scale = col + if col_name == 'id' and has_id_column: + column_defs.append(sql.SQL("{} UUID PRIMARY KEY DEFAULT gen_random_uuid()").format(sql.Identifier(col_name))) + else: + type_sql = sql.SQL("{}").format(sql.Identifier(udt_name)) + if max_length: + type_sql = sql.SQL("{}({})").format(type_sql, sql.Literal(max_length)) + elif precision and scale: + type_sql = sql.SQL("{}({},{})").format(type_sql, sql.Literal(precision), sql.Literal(scale)) + + column_def = sql.SQL("{} {}").format(sql.Identifier(col_name), type_sql) + if is_nullable == 'NO': + column_def = sql.SQL("{} NOT NULL").format(column_def) + if default and 'nextval' not in default: # Skip auto-increment defaults + column_def = sql.SQL("{} DEFAULT {}").format(column_def, sql.SQL(default)) + column_defs.append(column_def) + + constraint_defs = [] + for constraint in constraints: + conname, contype, condef = constraint + if contype != 'p' or not has_id_column: # Keep primary key if there's no id column + constraint_defs.append(sql.SQL(condef)) + + if not has_id_column: + column_defs.append(sql.SQL("uuid UUID DEFAULT gen_random_uuid()")) + + query = sql.SQL("CREATE TABLE {} ({})").format( + sql.Identifier(new_table), + sql.SQL(", ").join(column_defs + constraint_defs) + ) + cur.execute(query) + +def migrate_data(cur, old_table, new_table): + columns = get_columns(cur, old_table) + column_names = [col[0] for col in columns] + has_id_column = 'id' in column_names + + if has_id_column: + column_names.remove('id') + old_cols = sql.SQL(", ").join(map(sql.Identifier, column_names)) + new_cols = sql.SQL(", ").join(map(sql.Identifier, ['id'] + column_names)) + query = sql.SQL("INSERT INTO {} ({}) SELECT gen_random_uuid(), {} FROM {}").format( + sql.Identifier(new_table), + new_cols, + old_cols, + sql.Identifier(old_table) + ) + else: + old_cols = sql.SQL(", ").join(map(sql.Identifier, column_names)) + new_cols = sql.SQL(", ").join(map(sql.Identifier, column_names + ['uuid'])) + query = sql.SQL("INSERT INTO {} ({}) SELECT {}, gen_random_uuid() FROM {}").format( + sql.Identifier(new_table), + new_cols, + old_cols, + sql.Identifier(old_table) + ) + cur.execute(query) + +def update_foreign_keys(cur, tables): + for table in tables: + constraints = get_constraints(cur, table) + for constraint in constraints: + conname, contype, condef = constraint + if contype == 'f': # Foreign key constraint + referenced_table = condef.split('REFERENCES ')[1].split('(')[0].strip() + referenced_column = condef.split('(')[2].split(')')[0].strip() + local_column = condef.split('(')[1].split(')')[0].strip() + + cur.execute(sql.SQL(""" + UPDATE {table_uuid} + SET {local_column} = subquery.new_id::text::{local_column_type} + FROM ( + SELECT old.{ref_column} AS old_id, new_table.id AS new_id + FROM {ref_table} old + JOIN public.{ref_table_uuid} new_table ON new_table.{ref_column}::text = old.{ref_column}::text + ) AS subquery + WHERE {local_column}::text = subquery.old_id::text + """).format( + table_uuid=sql.Identifier(f"{table}_uuid"), + local_column=sql.Identifier(local_column), + local_column_type=sql.SQL(get_column_type(cur, f"{table}_uuid", local_column)), + ref_column=sql.Identifier(referenced_column), + ref_table=sql.Identifier(referenced_table), + ref_table_uuid=sql.Identifier(f"{referenced_table}_uuid") + )) + +def get_column_type(cur, table_name, column_name): + cur.execute(""" + SELECT data_type + FROM information_schema.columns + WHERE table_name = %s AND column_name = %s + """, (table_name, column_name)) + return cur.fetchone()[0] + +def rename_tables(cur, tables): + for table in tables: + drop_table_if_exists(cur, f"{table}_orig") + cur.execute(sql.SQL("ALTER TABLE IF EXISTS {} RENAME TO {}").format( + sql.Identifier(table), sql.Identifier(f"{table}_orig") + )) + cur.execute(sql.SQL("ALTER TABLE IF EXISTS {} RENAME TO {}").format( + sql.Identifier(f"{table}_uuid"), sql.Identifier(table) + )) + +def main(): + try: + with connect_to_db() as conn: + with conn.cursor() as cur: + tables = get_tables(cur) + + # Create new UUID tables + for table in tables: + print(f"Creating UUID table for {table}...") + create_uuid_table(cur, table, f"{table}_uuid") + + # Migrate data + for table in tables: + print(f"Migrating data for {table}...") + migrate_data(cur, table, f"{table}_uuid") + + # Update foreign keys + print("Updating foreign key references...") + update_foreign_keys(cur, tables) + + # Rename tables + print("Renaming tables...") + rename_tables(cur, tables) + + conn.commit() + print("Migration completed successfully.") + except Exception as e: + print(f"An error occurred: {e}") + conn.rollback() + +if __name__ == "__main__": + main() diff --git a/sijapi/helpers/fromvm/schema_info.yaml b/sijapi/helpers/fromvm/schema_info.yaml new file mode 100644 index 0000000..4899924 --- /dev/null +++ b/sijapi/helpers/fromvm/schema_info.yaml @@ -0,0 +1,1103 @@ +click_logs: + clicked_at: + data_type: timestamp without time zone + is_nullable: 'YES' + max_length: null + id: + data_type: uuid + is_nullable: 'NO' + max_length: null + ip_address: + data_type: text + is_nullable: 'YES' + max_length: null + short_code: + data_type: character varying + is_nullable: 'YES' + max_length: 3 + user_agent: + data_type: text + is_nullable: 'YES' + max_length: null +dailyweather: + cloudcover: + data_type: double precision + is_nullable: 'YES' + max_length: null + conditions: + data_type: character varying + is_nullable: 'YES' + max_length: null + datetime: + data_type: timestamp with time zone + is_nullable: 'YES' + max_length: null + datetimeepoch: + data_type: bigint + is_nullable: 'YES' + max_length: null + description: + data_type: character varying + is_nullable: 'YES' + max_length: null + dew: + data_type: double precision + is_nullable: 'YES' + max_length: null + feelslike: + data_type: double precision + is_nullable: 'YES' + max_length: null + feelslikemax: + data_type: double precision + is_nullable: 'YES' + max_length: null + feelslikemin: + data_type: double precision + is_nullable: 'YES' + max_length: null + humidity: + data_type: double precision + is_nullable: 'YES' + max_length: null + icon: + data_type: character varying + is_nullable: 'YES' + max_length: null + id: + data_type: integer + is_nullable: 'NO' + max_length: null + last_updated: + data_type: timestamp with time zone + is_nullable: 'YES' + max_length: null + location: + data_type: USER-DEFINED + is_nullable: 'YES' + max_length: null + moonphase: + data_type: double precision + is_nullable: 'YES' + max_length: null + precip: + data_type: double precision + is_nullable: 'YES' + max_length: null + precipcover: + data_type: double precision + is_nullable: 'YES' + max_length: null + precipprob: + data_type: double precision + is_nullable: 'YES' + max_length: null + preciptype: + data_type: ARRAY + is_nullable: 'YES' + max_length: null + pressure: + data_type: double precision + is_nullable: 'YES' + max_length: null + severerisk: + data_type: double precision + is_nullable: 'YES' + max_length: null + snow: + data_type: double precision + is_nullable: 'YES' + max_length: null + snowdepth: + data_type: double precision + is_nullable: 'YES' + max_length: null + solarenergy: + data_type: double precision + is_nullable: 'YES' + max_length: null + solarradiation: + data_type: double precision + is_nullable: 'YES' + max_length: null + source: + data_type: character varying + is_nullable: 'YES' + max_length: null + stations: + data_type: ARRAY + is_nullable: 'YES' + max_length: null + sunrise: + data_type: timestamp with time zone + is_nullable: 'YES' + max_length: null + sunriseepoch: + data_type: bigint + is_nullable: 'YES' + max_length: null + sunset: + data_type: timestamp with time zone + is_nullable: 'YES' + max_length: null + sunsetepoch: + data_type: bigint + is_nullable: 'YES' + max_length: null + temp: + data_type: double precision + is_nullable: 'YES' + max_length: null + tempmax: + data_type: double precision + is_nullable: 'YES' + max_length: null + tempmin: + data_type: double precision + is_nullable: 'YES' + max_length: null + uvindex: + data_type: integer + is_nullable: 'YES' + max_length: null + visibility: + data_type: double precision + is_nullable: 'YES' + max_length: null + winddir: + data_type: integer + is_nullable: 'YES' + max_length: null + windgust: + data_type: double precision + is_nullable: 'YES' + max_length: null + windspeed: + data_type: double precision + is_nullable: 'YES' + max_length: null +dailyweather_uuid: + cloudcover: + data_type: double precision + is_nullable: 'YES' + max_length: null + conditions: + data_type: character varying + is_nullable: 'YES' + max_length: null + datetime: + data_type: timestamp with time zone + is_nullable: 'YES' + max_length: null + datetimeepoch: + data_type: bigint + is_nullable: 'YES' + max_length: null + description: + data_type: character varying + is_nullable: 'YES' + max_length: null + dew: + data_type: double precision + is_nullable: 'YES' + max_length: null + feelslike: + data_type: double precision + is_nullable: 'YES' + max_length: null + feelslikemax: + data_type: double precision + is_nullable: 'YES' + max_length: null + feelslikemin: + data_type: double precision + is_nullable: 'YES' + max_length: null + humidity: + data_type: double precision + is_nullable: 'YES' + max_length: null + icon: + data_type: character varying + is_nullable: 'YES' + max_length: null + id: + data_type: uuid + is_nullable: 'NO' + max_length: null + last_updated: + data_type: timestamp with time zone + is_nullable: 'YES' + max_length: null + location: + data_type: USER-DEFINED + is_nullable: 'YES' + max_length: null + moonphase: + data_type: double precision + is_nullable: 'YES' + max_length: null + precip: + data_type: double precision + is_nullable: 'YES' + max_length: null + precipcover: + data_type: double precision + is_nullable: 'YES' + max_length: null + precipprob: + data_type: double precision + is_nullable: 'YES' + max_length: null + preciptype: + data_type: ARRAY + is_nullable: 'YES' + max_length: null + pressure: + data_type: double precision + is_nullable: 'YES' + max_length: null + severerisk: + data_type: double precision + is_nullable: 'YES' + max_length: null + snow: + data_type: double precision + is_nullable: 'YES' + max_length: null + snowdepth: + data_type: double precision + is_nullable: 'YES' + max_length: null + solarenergy: + data_type: double precision + is_nullable: 'YES' + max_length: null + solarradiation: + data_type: double precision + is_nullable: 'YES' + max_length: null + source: + data_type: character varying + is_nullable: 'YES' + max_length: null + stations: + data_type: ARRAY + is_nullable: 'YES' + max_length: null + sunrise: + data_type: timestamp with time zone + is_nullable: 'YES' + max_length: null + sunriseepoch: + data_type: bigint + is_nullable: 'YES' + max_length: null + sunset: + data_type: timestamp with time zone + is_nullable: 'YES' + max_length: null + sunsetepoch: + data_type: bigint + is_nullable: 'YES' + max_length: null + temp: + data_type: double precision + is_nullable: 'YES' + max_length: null + tempmax: + data_type: double precision + is_nullable: 'YES' + max_length: null + tempmin: + data_type: double precision + is_nullable: 'YES' + max_length: null + uvindex: + data_type: integer + is_nullable: 'YES' + max_length: null + visibility: + data_type: double precision + is_nullable: 'YES' + max_length: null + winddir: + data_type: integer + is_nullable: 'YES' + max_length: null + windgust: + data_type: double precision + is_nullable: 'YES' + max_length: null + windspeed: + data_type: double precision + is_nullable: 'YES' + max_length: null +geography_columns: + coord_dimension: + data_type: integer + is_nullable: 'YES' + max_length: null + f_geography_column: + data_type: name + is_nullable: 'YES' + max_length: null + f_table_catalog: + data_type: name + is_nullable: 'YES' + max_length: null + f_table_name: + data_type: name + is_nullable: 'YES' + max_length: null + f_table_schema: + data_type: name + is_nullable: 'YES' + max_length: null + srid: + data_type: integer + is_nullable: 'YES' + max_length: null + type: + data_type: text + is_nullable: 'YES' + max_length: null +geometry_columns: + coord_dimension: + data_type: integer + is_nullable: 'YES' + max_length: null + f_geometry_column: + data_type: name + is_nullable: 'YES' + max_length: null + f_table_catalog: + data_type: character varying + is_nullable: 'YES' + max_length: 256 + f_table_name: + data_type: name + is_nullable: 'YES' + max_length: null + f_table_schema: + data_type: name + is_nullable: 'YES' + max_length: null + srid: + data_type: integer + is_nullable: 'YES' + max_length: null + type: + data_type: character varying + is_nullable: 'YES' + max_length: 30 +hourlyweather: + cloudcover: + data_type: double precision + is_nullable: 'YES' + max_length: null + conditions: + data_type: character varying + is_nullable: 'YES' + max_length: null + daily_weather_id: + data_type: integer + is_nullable: 'YES' + max_length: null + datetime: + data_type: timestamp with time zone + is_nullable: 'YES' + max_length: null + datetimeepoch: + data_type: bigint + is_nullable: 'YES' + max_length: null + dew: + data_type: double precision + is_nullable: 'YES' + max_length: null + feelslike: + data_type: double precision + is_nullable: 'YES' + max_length: null + humidity: + data_type: double precision + is_nullable: 'YES' + max_length: null + icon: + data_type: character varying + is_nullable: 'YES' + max_length: null + id: + data_type: integer + is_nullable: 'NO' + max_length: null + precip: + data_type: double precision + is_nullable: 'YES' + max_length: null + precipprob: + data_type: double precision + is_nullable: 'YES' + max_length: null + preciptype: + data_type: ARRAY + is_nullable: 'YES' + max_length: null + pressure: + data_type: double precision + is_nullable: 'YES' + max_length: null + severerisk: + data_type: double precision + is_nullable: 'YES' + max_length: null + snow: + data_type: double precision + is_nullable: 'YES' + max_length: null + snowdepth: + data_type: double precision + is_nullable: 'YES' + max_length: null + solarenergy: + data_type: double precision + is_nullable: 'YES' + max_length: null + solarradiation: + data_type: double precision + is_nullable: 'YES' + max_length: null + source: + data_type: character varying + is_nullable: 'YES' + max_length: null + stations: + data_type: ARRAY + is_nullable: 'YES' + max_length: null + temp: + data_type: double precision + is_nullable: 'YES' + max_length: null + uvindex: + data_type: integer + is_nullable: 'YES' + max_length: null + visibility: + data_type: double precision + is_nullable: 'YES' + max_length: null + winddir: + data_type: integer + is_nullable: 'YES' + max_length: null + windgust: + data_type: double precision + is_nullable: 'YES' + max_length: null + windspeed: + data_type: double precision + is_nullable: 'YES' + max_length: null +hourlyweather_uuid: + cloudcover: + data_type: double precision + is_nullable: 'YES' + max_length: null + conditions: + data_type: character varying + is_nullable: 'YES' + max_length: null + daily_weather_id: + data_type: integer + is_nullable: 'YES' + max_length: null + datetime: + data_type: timestamp with time zone + is_nullable: 'YES' + max_length: null + datetimeepoch: + data_type: bigint + is_nullable: 'YES' + max_length: null + dew: + data_type: double precision + is_nullable: 'YES' + max_length: null + feelslike: + data_type: double precision + is_nullable: 'YES' + max_length: null + humidity: + data_type: double precision + is_nullable: 'YES' + max_length: null + icon: + data_type: character varying + is_nullable: 'YES' + max_length: null + id: + data_type: uuid + is_nullable: 'NO' + max_length: null + precip: + data_type: double precision + is_nullable: 'YES' + max_length: null + precipprob: + data_type: double precision + is_nullable: 'YES' + max_length: null + preciptype: + data_type: ARRAY + is_nullable: 'YES' + max_length: null + pressure: + data_type: double precision + is_nullable: 'YES' + max_length: null + severerisk: + data_type: double precision + is_nullable: 'YES' + max_length: null + snow: + data_type: double precision + is_nullable: 'YES' + max_length: null + snowdepth: + data_type: double precision + is_nullable: 'YES' + max_length: null + solarenergy: + data_type: double precision + is_nullable: 'YES' + max_length: null + solarradiation: + data_type: double precision + is_nullable: 'YES' + max_length: null + source: + data_type: character varying + is_nullable: 'YES' + max_length: null + stations: + data_type: ARRAY + is_nullable: 'YES' + max_length: null + temp: + data_type: double precision + is_nullable: 'YES' + max_length: null + uvindex: + data_type: integer + is_nullable: 'YES' + max_length: null + visibility: + data_type: double precision + is_nullable: 'YES' + max_length: null + winddir: + data_type: integer + is_nullable: 'YES' + max_length: null + windgust: + data_type: double precision + is_nullable: 'YES' + max_length: null + windspeed: + data_type: double precision + is_nullable: 'YES' + max_length: null +locations: + action: + data_type: character varying + is_nullable: 'YES' + max_length: null + amenity: + data_type: character varying + is_nullable: 'YES' + max_length: null + boundingbox: + data_type: ARRAY + is_nullable: 'YES' + max_length: null + city: + data_type: character varying + is_nullable: 'YES' + max_length: 255 + class: + data_type: character varying + is_nullable: 'YES' + max_length: null + class_: + data_type: text + is_nullable: 'YES' + max_length: null + context: + data_type: jsonb + is_nullable: 'YES' + max_length: null + country: + data_type: character varying + is_nullable: 'YES' + max_length: null + country_code: + data_type: character varying + is_nullable: 'YES' + max_length: null + county: + data_type: character varying + is_nullable: 'YES' + max_length: null + daily_weather_id: + data_type: integer + is_nullable: 'YES' + max_length: null + datetime: + data_type: timestamp with time zone + is_nullable: 'YES' + max_length: null + device_model: + data_type: character varying + is_nullable: 'YES' + max_length: null + device_name: + data_type: character varying + is_nullable: 'YES' + max_length: null + device_os: + data_type: character varying + is_nullable: 'YES' + max_length: null + device_type: + data_type: character varying + is_nullable: 'YES' + max_length: null + display_name: + data_type: character varying + is_nullable: 'YES' + max_length: null + house_number: + data_type: character varying + is_nullable: 'YES' + max_length: null + id: + data_type: integer + is_nullable: 'NO' + max_length: null + location: + data_type: USER-DEFINED + is_nullable: 'YES' + max_length: null + name: + data_type: character varying + is_nullable: 'YES' + max_length: null + neighbourhood: + data_type: character varying + is_nullable: 'YES' + max_length: null + quarter: + data_type: character varying + is_nullable: 'YES' + max_length: null + road: + data_type: character varying + is_nullable: 'YES' + max_length: null + state: + data_type: character varying + is_nullable: 'YES' + max_length: 255 + street: + data_type: character varying + is_nullable: 'YES' + max_length: 255 + suburb: + data_type: character varying + is_nullable: 'YES' + max_length: null + type: + data_type: character varying + is_nullable: 'YES' + max_length: null + zip: + data_type: character varying + is_nullable: 'YES' + max_length: 255 +locations_uuid: + action: + data_type: character varying + is_nullable: 'YES' + max_length: null + amenity: + data_type: character varying + is_nullable: 'YES' + max_length: null + boundingbox: + data_type: ARRAY + is_nullable: 'YES' + max_length: null + city: + data_type: character varying + is_nullable: 'YES' + max_length: 255 + class: + data_type: character varying + is_nullable: 'YES' + max_length: null + class_: + data_type: text + is_nullable: 'YES' + max_length: null + context: + data_type: jsonb + is_nullable: 'YES' + max_length: null + country: + data_type: character varying + is_nullable: 'YES' + max_length: null + country_code: + data_type: character varying + is_nullable: 'YES' + max_length: null + county: + data_type: character varying + is_nullable: 'YES' + max_length: null + daily_weather_id: + data_type: integer + is_nullable: 'YES' + max_length: null + datetime: + data_type: timestamp with time zone + is_nullable: 'YES' + max_length: null + device_model: + data_type: character varying + is_nullable: 'YES' + max_length: null + device_name: + data_type: character varying + is_nullable: 'YES' + max_length: null + device_os: + data_type: character varying + is_nullable: 'YES' + max_length: null + device_type: + data_type: character varying + is_nullable: 'YES' + max_length: null + display_name: + data_type: character varying + is_nullable: 'YES' + max_length: null + house_number: + data_type: character varying + is_nullable: 'YES' + max_length: null + id: + data_type: uuid + is_nullable: 'NO' + max_length: null + location: + data_type: USER-DEFINED + is_nullable: 'YES' + max_length: null + name: + data_type: character varying + is_nullable: 'YES' + max_length: null + neighbourhood: + data_type: character varying + is_nullable: 'YES' + max_length: null + quarter: + data_type: character varying + is_nullable: 'YES' + max_length: null + road: + data_type: character varying + is_nullable: 'YES' + max_length: null + state: + data_type: character varying + is_nullable: 'YES' + max_length: 255 + street: + data_type: character varying + is_nullable: 'YES' + max_length: 255 + suburb: + data_type: character varying + is_nullable: 'YES' + max_length: null + type: + data_type: character varying + is_nullable: 'YES' + max_length: null + zip: + data_type: character varying + is_nullable: 'YES' + max_length: 255 +plss_sections: + frstdivdup: + data_type: character varying + is_nullable: 'YES' + max_length: null + frstdivid: + data_type: character varying + is_nullable: 'YES' + max_length: null + frstdivlab: + data_type: character varying + is_nullable: 'YES' + max_length: null + frstdivno: + data_type: character varying + is_nullable: 'YES' + max_length: null + frstdivtxt: + data_type: character varying + is_nullable: 'YES' + max_length: null + frstdivtyp: + data_type: character varying + is_nullable: 'YES' + max_length: null + objectid: + data_type: integer + is_nullable: 'YES' + max_length: null + ogc_fid: + data_type: integer + is_nullable: 'NO' + max_length: null + plssid: + data_type: character varying + is_nullable: 'YES' + max_length: null + shape.starea(): + data_type: double precision + is_nullable: 'YES' + max_length: null + shape.stlength(): + data_type: double precision + is_nullable: 'YES' + max_length: null + sourcedate: + data_type: bigint + is_nullable: 'YES' + max_length: null + sourceref: + data_type: character varying + is_nullable: 'YES' + max_length: null + survtyp: + data_type: character varying + is_nullable: 'YES' + max_length: null + survtyptxt: + data_type: character varying + is_nullable: 'YES' + max_length: null + wkb_geometry: + data_type: USER-DEFINED + is_nullable: 'YES' + max_length: null +plss_townships: + created_date: + data_type: bigint + is_nullable: 'YES' + max_length: null + created_user: + data_type: character varying + is_nullable: 'YES' + max_length: null + globalid: + data_type: character varying + is_nullable: 'YES' + max_length: null + last_edited_date: + data_type: bigint + is_nullable: 'YES' + max_length: null + last_edited_user: + data_type: character varying + is_nullable: 'YES' + max_length: null + objectid: + data_type: integer + is_nullable: 'YES' + max_length: null + ogc_fid: + data_type: integer + is_nullable: 'NO' + max_length: null + origid: + data_type: character varying + is_nullable: 'YES' + max_length: null + partcount: + data_type: character varying + is_nullable: 'YES' + max_length: null + plssid: + data_type: character varying + is_nullable: 'YES' + max_length: null + prinmer: + data_type: character varying + is_nullable: 'YES' + max_length: null + prinmercd: + data_type: character varying + is_nullable: 'YES' + max_length: null + rangedir: + data_type: character varying + is_nullable: 'YES' + max_length: null + rangefrac: + data_type: character varying + is_nullable: 'YES' + max_length: null + rangeno: + data_type: character varying + is_nullable: 'YES' + max_length: null + secsrvname: + data_type: character varying + is_nullable: 'YES' + max_length: null + shape.starea(): + data_type: double precision + is_nullable: 'YES' + max_length: null + shape.stlength(): + data_type: double precision + is_nullable: 'YES' + max_length: null + sourcedate: + data_type: bigint + is_nullable: 'YES' + max_length: null + sourceref: + data_type: character varying + is_nullable: 'YES' + max_length: null + srvname: + data_type: character varying + is_nullable: 'YES' + max_length: null + stateabbr: + data_type: character varying + is_nullable: 'YES' + max_length: null + steward: + data_type: character varying + is_nullable: 'YES' + max_length: null + survtyp: + data_type: character varying + is_nullable: 'YES' + max_length: null + survtyptxt: + data_type: character varying + is_nullable: 'YES' + max_length: null + twnshpdir: + data_type: character varying + is_nullable: 'YES' + max_length: null + twnshpdpcd: + data_type: character varying + is_nullable: 'YES' + max_length: null + twnshpfrac: + data_type: character varying + is_nullable: 'YES' + max_length: null + twnshplab: + data_type: character varying + is_nullable: 'YES' + max_length: null + twnshpno: + data_type: character varying + is_nullable: 'YES' + max_length: null + wkb_geometry: + data_type: USER-DEFINED + is_nullable: 'YES' + max_length: null +relations: + id: + data_type: bigint + is_nullable: 'NO' + max_length: null + members: + data_type: jsonb + is_nullable: 'YES' + max_length: null + tags: + data_type: jsonb + is_nullable: 'YES' + max_length: null +short_urls: + created_at: + data_type: timestamp without time zone + is_nullable: 'YES' + max_length: null + id: + data_type: integer + is_nullable: 'NO' + max_length: null + long_url: + data_type: text + is_nullable: 'NO' + max_length: null + short_code: + data_type: character varying + is_nullable: 'NO' + max_length: 3 +short_urls_uuid: + created_at: + data_type: timestamp without time zone + is_nullable: 'YES' + max_length: null + id: + data_type: uuid + is_nullable: 'NO' + max_length: null + long_url: + data_type: text + is_nullable: 'NO' + max_length: null + short_code: + data_type: character varying + is_nullable: 'NO' + max_length: 3 +spatial_ref_sys: + auth_name: + data_type: character varying + is_nullable: 'YES' + max_length: 256 + auth_srid: + data_type: integer + is_nullable: 'YES' + max_length: null + proj4text: + data_type: character varying + is_nullable: 'YES' + max_length: 2048 + srid: + data_type: integer + is_nullable: 'NO' + max_length: null + srtext: + data_type: character varying + is_nullable: 'YES' + max_length: 2048 diff --git a/sijapi/helpers/fromvm/start.py b/sijapi/helpers/fromvm/start.py new file mode 100644 index 0000000..031e4fb --- /dev/null +++ b/sijapi/helpers/fromvm/start.py @@ -0,0 +1,211 @@ +import yaml +import requests +import paramiko +import time +from pathlib import Path +import logging +import subprocess +import os +import argparse +import sys + +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') + +def load_config(): + config_path = Path(__file__).parent.parent / 'config' / 'sys.yaml' + with open(config_path, 'r') as file: + return yaml.safe_load(file) + +def load_env(): + env_path = Path(__file__).parent.parent / 'config' / '.env' + if env_path.exists(): + with open(env_path, 'r') as file: + for line in file: + line = line.strip() + if line and not line.startswith('#'): + try: + key, value = line.split('=', 1) + os.environ[key.strip()] = value.strip() + except ValueError: + logging.warning(f"Skipping invalid line in .env file: {line}") + +def check_server(ip, port, ts_id): + address = f"http://{ip}:{port}/id" + try: + response = requests.get(address, timeout=5) + response_text = response.text.strip().strip('"') + return response.status_code == 200 and response_text == ts_id + except requests.RequestException as e: + logging.error(f"Error checking server {ts_id}: {str(e)}") + return False + +def execute_ssh_command(ssh, command): + stdin, stdout, stderr = ssh.exec_command(command) + exit_status = stdout.channel.recv_exit_status() + output = stdout.read().decode().strip() + error = stderr.read().decode().strip() + return exit_status, output, error + +def is_local_tmux_session_running(session_name): + try: + result = subprocess.run(['tmux', 'has-session', '-t', session_name], capture_output=True, text=True) + return result.returncode == 0 + except subprocess.CalledProcessError: + return False + +def start_local_server(server, pull=False, push=False): + try: + if is_local_tmux_session_running('sijapi'): + logging.info("Local sijapi tmux session is already running.") + return + + git_command = "" + if pull: + git_command = "git pull &&" + elif push: + git_command = "git add -A . && git commit -m \"auto-update\" && git push origin --force &&" + + command = f"{server['tmux']} new-session -d -s sijapi 'cd {server['path']} && {git_command} {server['conda_env']}/bin/python -m sijapi'" + logging.info(f"Executing local command: {command}") + result = subprocess.run(command, shell=True, check=True, capture_output=True, text=True) + logging.info(f"Successfully started sijapi session on local machine") + logging.debug(f"Command output: {result.stdout}") + except subprocess.CalledProcessError as e: + logging.error(f"Failed to start sijapi session on local machine. Error: {e}") + logging.error(f"Error output: {e.stderr}") + +def start_remote_server(server, pull=False, push=False): + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + try: + ssh.connect( + server['ts_ip'], + port=server['ssh_port'], + username=server['ssh_user'], + password=server['ssh_pass'], + timeout=10 + ) + + status, output, error = execute_ssh_command(ssh, f"{server['tmux']} has-session -t sijapi 2>/dev/null && echo 'exists' || echo 'not exists'") + if output == 'exists': + logging.info(f"sijapi session already exists on {server['ts_id']}") + return + + git_command = "" + if pull: + git_command = "git pull &&" + elif push: + git_command = "git add -A . && git commit -m \"auto-update\" && git push origin --force &&" + + command = f"{server['tmux']} new-session -d -s sijapi 'cd {server['path']} && {git_command} {server['conda_env']}/bin/python -m sijapi'" + status, output, error = execute_ssh_command(ssh, command) + + if status == 0: + logging.info(f"Successfully started sijapi session on {server['ts_id']}") + else: + logging.error(f"Failed to start sijapi session on {server['ts_id']}. Error: {error}") + + except paramiko.SSHException as e: + logging.error(f"Failed to connect to {server['ts_id']}: {str(e)}") + finally: + ssh.close() + +def kill_local_server(): + try: + if is_local_tmux_session_running('sijapi'): + subprocess.run(['tmux', 'kill-session', '-t', 'sijapi'], check=True) + logging.info("Killed local sijapi tmux session.") + else: + logging.info("No local sijapi tmux session to kill.") + except subprocess.CalledProcessError as e: + logging.error(f"Failed to kill local sijapi tmux session. Error: {e}") + +def kill_remote_server(server): + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + try: + ssh.connect( + server['ts_ip'], + port=server['ssh_port'], + username=server['ssh_user'], + password=server['ssh_pass'], + timeout=10 + ) + + command = f"{server['tmux']} kill-session -t sijapi" + status, output, error = execute_ssh_command(ssh, command) + + if status == 0: + logging.info(f"Successfully killed sijapi session on {server['ts_id']}") + else: + logging.error(f"Failed to kill sijapi session on {server['ts_id']}. Error: {error}") + + except paramiko.SSHException as e: + logging.error(f"Failed to connect to {server['ts_id']}: {str(e)}") + finally: + ssh.close() + +def main(): + load_env() + config = load_config() + pool = config['POOL'] + local_ts_id = os.environ.get('TS_ID') + + parser = argparse.ArgumentParser(description='Manage sijapi servers') + parser.add_argument('--kill', action='store_true', help='Kill the local sijapi tmux session') + parser.add_argument('--restart', action='store_true', help='Restart the local sijapi tmux session') + parser.add_argument('--all', action='store_true', help='Apply the action to all servers') + parser.add_argument('--pull', action='store_true', help='Pull latest changes before starting the server') + parser.add_argument('--push', action='store_true', help='Push changes before starting the server') + + args = parser.parse_args() + + if args.kill: + if args.all: + for server in pool: + if server['ts_id'] == local_ts_id: + kill_local_server() + else: + kill_remote_server(server) + else: + kill_local_server() + sys.exit(0) + + if args.restart or args.pull or args.push: + if args.all: + for server in pool: + if server['ts_id'] == local_ts_id: + kill_local_server() + start_local_server(server, pull=args.pull, push=args.push) + else: + kill_remote_server(server) + start_remote_server(server, pull=args.pull, push=args.push) + else: + kill_local_server() + local_server = next(server for server in pool if server['ts_id'] == local_ts_id) + start_local_server(local_server, pull=args.pull, push=args.push) + sys.exit(0) + + # If no specific arguments, run the default behavior + local_server = next(server for server in pool if server['ts_id'] == local_ts_id) + if not check_server(local_server['ts_ip'], local_server['app_port'], local_server['ts_id']): + logging.info(f"Local server {local_server['ts_id']} is not responding correctly. Attempting to start...") + kill_local_server() + start_local_server(local_server, push=True) + + for server in pool: + if server['ts_id'] != local_ts_id: + if not check_server(server['ts_ip'], server['app_port'], server['ts_id']): + logging.info(f"{server['ts_id']} is not responding correctly. Attempting to start...") + kill_remote_server(server) + start_remote_server(server, pull=True) + else: + logging.info(f"{server['ts_id']} is running and responding correctly.") + + time.sleep(1) + +if __name__ == "__main__": + main() + diff --git a/sijapi/helpers/fromvm/upscaler.py b/sijapi/helpers/fromvm/upscaler.py new file mode 100644 index 0000000..4e7ea5d --- /dev/null +++ b/sijapi/helpers/fromvm/upscaler.py @@ -0,0 +1,34 @@ +from aura_sr import AuraSR +from PIL import Image +import torch +import os + +# Set environment variables for MPS +os.environ['PYTORCH_MPS_HIGH_WATERMARK_RATIO'] = '0.0' +os.environ['PYTORCH_ENABLE_MPS_FALLBACK'] = '1' + +# Initialize device as CPU for default +device = torch.device('cpu') + +# Check if MPS is available +if torch.backends.mps.is_available(): + if not torch.backends.mps.is_built(): + print("MPS not available because the current PyTorch install was not built with MPS enabled.") + else: + device = torch.device('mps:0') + +# Overwrite the default CUDA device with MPS +torch.cuda.default_stream = device + +aura_sr = AuraSR.from_pretrained("fal-ai/AuraSR").to(device) + +def load_image_from_path(file_path): + return Image.open(file_path) + +def upscale_and_save(original_path): + original_image = load_image_from_path(original_path) + upscaled_image = aura_sr.upscale_4x(original_image) + upscaled_image.save(original_path) + +# Insert your image path +upscale_and_save("/Users/sij/workshop/sijapi/sijapi/testbed/API__00482_ 2.png")