From 124b18406eeaaea7b14b314bf5012c1a713159cf Mon Sep 17 00:00:00 2001
From: sanj <67624670+iodrift@users.noreply.github.com>
Date: Mon, 12 Aug 2024 10:33:44 -0700
Subject: [PATCH] Auto-update: Mon Aug 12 10:33:44 PDT 2024
---
.gitignore | 1 +
sijapi/helpers/fromvm/CaPLSS.py | 225 ++++
.../fromvm/CaPLSS_downloader_and_importer.py | 133 ++
sijapi/helpers/fromvm/CalFire_THP_scraper.py | 73 ++
sijapi/helpers/fromvm/article.py | 23 +
sijapi/helpers/fromvm/db/docker-compose.yaml | 25 +
sijapi/helpers/fromvm/db/init-db.sh | 11 +
...levenlabs_history_ids_20240630_131617.json | 104 ++
sijapi/helpers/fromvm/log_prior_emails.py | 63 +
sijapi/helpers/fromvm/migrate_db_to_uuid.py | 191 +++
sijapi/helpers/fromvm/schema_info.yaml | 1103 +++++++++++++++++
sijapi/helpers/fromvm/start.py | 211 ++++
sijapi/helpers/fromvm/upscaler.py | 34 +
13 files changed, 2197 insertions(+)
create mode 100644 sijapi/helpers/fromvm/CaPLSS.py
create mode 100644 sijapi/helpers/fromvm/CaPLSS_downloader_and_importer.py
create mode 100644 sijapi/helpers/fromvm/CalFire_THP_scraper.py
create mode 100755 sijapi/helpers/fromvm/article.py
create mode 100644 sijapi/helpers/fromvm/db/docker-compose.yaml
create mode 100755 sijapi/helpers/fromvm/db/init-db.sh
create mode 100644 sijapi/helpers/fromvm/elevenlabs_history_ids_20240630_131617.json
create mode 100644 sijapi/helpers/fromvm/log_prior_emails.py
create mode 100644 sijapi/helpers/fromvm/migrate_db_to_uuid.py
create mode 100644 sijapi/helpers/fromvm/schema_info.yaml
create mode 100644 sijapi/helpers/fromvm/start.py
create mode 100644 sijapi/helpers/fromvm/upscaler.py
diff --git a/.gitignore b/.gitignore
index 6ae221a..e3d6881 100644
--- a/.gitignore
+++ b/.gitignore
@@ -39,6 +39,7 @@ podcast/sideloads/*
**/.ipynb_checkpoints/
**/*.pem
**/*.key
+**/*.sql
venv/
env/
.venv/
diff --git a/sijapi/helpers/fromvm/CaPLSS.py b/sijapi/helpers/fromvm/CaPLSS.py
new file mode 100644
index 0000000..9d1457a
--- /dev/null
+++ b/sijapi/helpers/fromvm/CaPLSS.py
@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+
+import requests
+import json
+import time
+import os
+import subprocess
+import sys
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+from datetime import datetime
+
+# Environment variables for database connection
+DB_NAME = os.getenv('DB_NAME', 'sij')
+DB_USER = os.getenv('DB_USER', 'sij')
+DB_PASSWORD = os.getenv('DB_PASSWORD', 'Synchr0!')
+DB_HOST = os.getenv('DB_HOST', 'localhost')
+DB_PORT = os.getenv('DB_PORT', '5432')
+
+def get_feature_count(url):
+ params = {
+ 'where': '1=1',
+ 'returnCountOnly': 'true',
+ 'f': 'json'
+ }
+ retries = Retry(total=10, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504])
+ with requests.Session() as session:
+ session.mount("https://", HTTPAdapter(max_retries=retries))
+ response = session.get(url, params=params, timeout=30)
+ response.raise_for_status()
+ data = response.json()
+ return data.get('count', 0)
+
+def fetch_features(url, offset, num, max_retries=5):
+ params = {
+ 'where': '1=1',
+ 'outFields': '*',
+ 'geometryPrecision': 6,
+ 'outSR': 4326,
+ 'f': 'json',
+ 'resultOffset': offset,
+ 'resultRecordCount': num
+ }
+ for attempt in range(max_retries):
+ try:
+ retries = Retry(total=5, backoff_factor=1, status_forcelist=[500, 502, 503, 504])
+ with requests.Session() as session:
+ session.mount("https://", HTTPAdapter(max_retries=retries))
+ response = session.get(url, params=params, timeout=30)
+ response.raise_for_status()
+ return response.json()
+ except requests.exceptions.RequestException as e:
+ print(f"Error fetching features (attempt {attempt + 1}/{max_retries}): {e}")
+ if attempt == max_retries - 1:
+ raise
+ time.sleep(5 * (attempt + 1)) # Exponential backoff
+
+def download_layer(layer_num, layer_name):
+ base_dir = os.path.expanduser('~/data')
+ file_path = os.path.join(base_dir, f'PLSS_{layer_name}.geojson')
+ temp_file_path = os.path.join(base_dir, f'PLSS_{layer_name}_temp.json')
+
+ url = f"https://gis.blm.gov/arcgis/rest/services/Cadastral/BLM_Natl_PLSS_CadNSDI/MapServer/{layer_num}/query"
+
+ total_count = get_feature_count(url)
+ print(f"Total {layer_name} features: {total_count}")
+
+ batch_size = 1000
+ offset = 0
+ all_features = []
+
+ # Check if temporary file exists and load its content
+ if os.path.exists(temp_file_path):
+ with open(temp_file_path, 'r') as f:
+ all_features = json.load(f)
+ offset = len(all_features)
+ print(f"Resuming download from offset {offset}")
+
+ try:
+ while offset < total_count:
+ print(f"Fetching {layer_name} features {offset} to {offset + batch_size}...")
+ data = fetch_features(url, offset, batch_size)
+
+ new_features = data.get('features', [])
+ if not new_features:
+ break
+
+ all_features.extend(new_features)
+ offset += len(new_features)
+
+ # Progress indicator
+ progress = len(all_features) / total_count
+ bar_length = 30
+ filled_length = int(bar_length * progress)
+ bar = '=' * filled_length + '-' * (bar_length - filled_length)
+ print(f'\rProgress: [{bar}] {progress:.1%} ({len(all_features)}/{total_count} features)', end='', flush=True)
+
+ # Save progress to temporary file
+ with open(temp_file_path, 'w') as f:
+ json.dump(all_features, f)
+
+ time.sleep(1)
+
+ print(f"\nTotal {layer_name} features fetched: {len(all_features)}")
+
+ geojson_features = [
+ {
+ "type": "Feature",
+ "properties": feature['attributes'],
+ "geometry": feature['geometry']
+ } for feature in all_features
+ ]
+
+ full_geojson = {
+ "type": "FeatureCollection",
+ "features": geojson_features
+ }
+
+ os.makedirs(base_dir, exist_ok=True)
+
+ with open(file_path, 'w') as f:
+ json.dump(full_geojson, f)
+
+ print(f"GeoJSON file saved as '{file_path}'")
+
+ # Remove temporary file
+ if os.path.exists(temp_file_path):
+ os.remove(temp_file_path)
+
+ return file_path
+ except Exception as e:
+ print(f"\nError during download: {e}")
+ print(f"Partial data saved in {temp_file_path}")
+ return None
+
+
+def check_postgres_connection():
+ try:
+ subprocess.run(['psql', '-h', DB_HOST, '-p', DB_PORT, '-U', DB_USER, '-d', DB_NAME, '-c', 'SELECT 1;'],
+ check=True, capture_output=True, text=True)
+ return True
+ except subprocess.CalledProcessError:
+ return False
+
+def check_postgis_extension():
+ try:
+ result = subprocess.run(['psql', '-h', DB_HOST, '-p', DB_PORT, '-U', DB_USER, '-d', DB_NAME,
+ '-c', "SELECT 1 FROM pg_extension WHERE extname = 'postgis';"],
+ check=True, capture_output=True, text=True)
+ return '1' in result.stdout
+ except subprocess.CalledProcessError:
+ return False
+
+def create_postgis_extension():
+ try:
+ subprocess.run(['psql', '-h', DB_HOST, '-p', DB_PORT, '-U', DB_USER, '-d', DB_NAME,
+ '-c', "CREATE EXTENSION IF NOT EXISTS postgis;"],
+ check=True, capture_output=True, text=True)
+ print("PostGIS extension created successfully.")
+ except subprocess.CalledProcessError as e:
+ print(f"Error creating PostGIS extension: {e}")
+ sys.exit(1)
+
+def import_to_postgis(file_path, table_name):
+ if not check_postgres_connection():
+ print("Error: Unable to connect to PostgreSQL. Please check your connection settings.")
+ sys.exit(1)
+
+ if not check_postgis_extension():
+ print("PostGIS extension not found. Attempting to create it...")
+ create_postgis_extension()
+
+ ogr2ogr_command = [
+ 'ogr2ogr',
+ '-f', 'PostgreSQL',
+ f'PG:dbname={DB_NAME} user={DB_USER} password={DB_PASSWORD} host={DB_HOST} port={DB_PORT}',
+ file_path,
+ '-nln', table_name,
+ '-overwrite'
+ ]
+
+ try:
+ subprocess.run(ogr2ogr_command, check=True, capture_output=True, text=True)
+ print(f"Data successfully imported into PostGIS table: {table_name}")
+ except subprocess.CalledProcessError as e:
+ print(f"Error importing data into PostGIS: {e}")
+ print(f"Command that failed: {e.cmd}")
+ print(f"Error output: {e.stderr}")
+
+def check_ogr2ogr():
+ try:
+ subprocess.run(['ogr2ogr', '--version'], check=True, capture_output=True, text=True)
+ return True
+ except subprocess.CalledProcessError:
+ return False
+ except FileNotFoundError:
+ return False
+
+def main():
+ if not check_ogr2ogr():
+ print("Error: ogr2ogr not found. Please install GDAL/OGR tools.")
+ print("On Debian: sudo apt-get install gdal-bin")
+ print("On macOS with Homebrew: brew install gdal")
+ sys.exit(1)
+
+ try:
+ township_file = os.path.expanduser('~/data/PLSS_Townships.geojson')
+ if not os.path.exists(township_file):
+ township_file = download_layer(1, "Townships")
+ if township_file:
+ import_to_postgis(township_file, "public.plss_townships")
+
+ section_file = os.path.expanduser('~/data/PLSS_Sections.geojson')
+ if not os.path.exists(section_file):
+ section_file = download_layer(2, "Sections")
+ if section_file:
+ import_to_postgis(section_file, "public.plss_sections")
+
+ except requests.exceptions.RequestException as e:
+ print(f"Error fetching data: {e}")
+ except Exception as e:
+ print(f"An unexpected error occurred: {e}")
+
+if __name__ == "__main__":
+ main()
diff --git a/sijapi/helpers/fromvm/CaPLSS_downloader_and_importer.py b/sijapi/helpers/fromvm/CaPLSS_downloader_and_importer.py
new file mode 100644
index 0000000..5b3ea00
--- /dev/null
+++ b/sijapi/helpers/fromvm/CaPLSS_downloader_and_importer.py
@@ -0,0 +1,133 @@
+# CaPLSS_downloader_and_importer.py
+import requests
+import json
+import time
+import os
+import subprocess
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+def get_feature_count(url):
+ params = {
+ 'where': '1=1',
+ 'returnCountOnly': 'true',
+ 'f': 'json'
+ }
+ retries = Retry(total=10, backoff_factor=0.5)
+ adapter = HTTPAdapter(max_retries=retries)
+ session = requests.Session()
+ session.mount("https://", adapter)
+
+ response = session.get(url, params=params, timeout=15) # Add timeout parameter
+ response.raise_for_status()
+ data = response.json()
+ return data.get('count', 0)
+
+
+def fetch_features(url, offset, num):
+ params = {
+ 'where': '1=1',
+ 'outFields': '*',
+ 'geometryPrecision': 6,
+ 'outSR': 4326,
+ 'f': 'json',
+ 'resultOffset': offset,
+ 'resultRecordCount': num
+ }
+ response = requests.get(url, params=params)
+ response.raise_for_status()
+ return response.json()
+
+def download_layer(layer_num, layer_name):
+ url = f"https://gis.blm.gov/arcgis/rest/services/Cadastral/BLM_Natl_PLSS_CadNSDI/MapServer/{layer_num}/query"
+
+ total_count = get_feature_count(url)
+ print(f"Total {layer_name} features: {total_count}")
+
+ batch_size = 1000
+ offset = 0
+ all_features = []
+
+ while offset < total_count:
+ print(f"Fetching {layer_name} features {offset} to {offset + batch_size}...")
+ data = fetch_features(url, offset, batch_size)
+
+ new_features = data.get('features', [])
+ if not new_features:
+ break
+
+ all_features.extend(new_features)
+ offset += len(new_features)
+
+ print(f"Progress: {len(all_features)}/{total_count} features")
+
+ time.sleep(1) # Be nice to the server
+
+ print(f"Total {layer_name} features fetched: {len(all_features)}")
+
+ # Convert to GeoJSON
+ geojson_features = [
+ {
+ "type": "Feature",
+ "properties": feature['attributes'],
+ "geometry": feature['geometry']
+ } for feature in all_features
+ ]
+
+ full_geojson = {
+ "type": "FeatureCollection",
+ "features": geojson_features
+ }
+
+ # Define a base directory that exists on both macOS and Debian
+ base_dir = os.path.expanduser('~/data')
+ os.makedirs(base_dir, exist_ok=True) # Create the directory if it doesn't exist
+
+ # Use os.path.join to construct the file path
+ file_path = os.path.join(base_dir, f'PLSS_{layer_name}.geojson')
+
+ # Save to file
+ with open(file_path, 'w') as f:
+ json.dump(full_geojson, f)
+
+ print(f"GeoJSON file saved as '{file_path}'")
+
+ return file_path
+
+def import_to_postgis(file_path, table_name):
+ db_name = 'sij'
+ db_user = 'sij'
+ db_password = 'Synchr0!'
+
+ ogr2ogr_command = [
+ 'ogr2ogr',
+ '-f', 'PostgreSQL',
+ f'PG:dbname={db_name} user={db_user} password={db_password}',
+ file_path,
+ '-nln', table_name,
+ '-overwrite'
+ ]
+
+ subprocess.run(ogr2ogr_command, check=True)
+ print(f"Data successfully imported into PostGIS table: {table_name}")
+
+def main():
+ try:
+ # Download and import Townships (Layer 1)
+ township_file = download_layer(1, "Townships")
+ import_to_postgis(township_file, "public.plss_townships")
+
+ # Download and import Sections (Layer 2)
+ section_file = download_layer(2, "Sections")
+ import_to_postgis(section_file, "public.plss_sections")
+
+ except requests.exceptions.RequestException as e:
+ print(f"Error fetching data: {e}")
+ except subprocess.CalledProcessError as e:
+ print(f"Error importing data into PostGIS: {e}")
+ except Exception as e:
+ print(f"An unexpected error occurred: {e}")
+
+if __name__ == "__main__":
+ main()
diff --git a/sijapi/helpers/fromvm/CalFire_THP_scraper.py b/sijapi/helpers/fromvm/CalFire_THP_scraper.py
new file mode 100644
index 0000000..92909a6
--- /dev/null
+++ b/sijapi/helpers/fromvm/CalFire_THP_scraper.py
@@ -0,0 +1,73 @@
+import requests
+import PyPDF2
+import io
+import re
+
+def scrape_data_from_pdf(url):
+ response = requests.get(url)
+ pdf_file = io.BytesIO(response.content)
+
+ pdf_reader = PyPDF2.PdfReader(pdf_file)
+
+ all_text = ""
+ for page in pdf_reader.pages:
+ all_text += page.extract_text() + "\n"
+
+ return all_text
+
+def parse_data(raw_data):
+ lines = raw_data.split('\n')
+ data = []
+ current_entry = None
+
+ for line in lines:
+ line = line.strip()
+ if re.match(r'\d+-\d+-\d+-\w+', line):
+ if current_entry:
+ data.append(current_entry)
+ current_entry = {'Harvest Document': line, 'Raw Data': []}
+ elif current_entry:
+ current_entry['Raw Data'].append(line)
+
+ if current_entry:
+ data.append(current_entry)
+
+ return data
+
+def filter_data(data):
+ return [entry for entry in data if any(owner.lower() in ' '.join(entry['Raw Data']).lower() for owner in ["Sierra Pacific", "SPI", "Land & Timber"])]
+
+def extract_location(raw_data):
+ location = []
+ for line in raw_data:
+ if 'MDBM:' in line or 'HBM:' in line:
+ location.append(line)
+ return ' '.join(location)
+
+def extract_plss_coordinates(text):
+ pattern = r'(\w+): T(\d+)([NSEW]) R(\d+)([NSEW]) S(\d+)'
+ return re.findall(pattern, text)
+
+# Main execution
+url = "https://caltreesplans.resources.ca.gov/Caltrees/Report/ShowReport.aspx?module=TH_Document&reportID=492&reportType=LINK_REPORT_LIST"
+raw_data = scrape_data_from_pdf(url)
+
+parsed_data = parse_data(raw_data)
+print(f"Total timber plans parsed: {len(parsed_data)}")
+
+filtered_data = filter_data(parsed_data)
+print(f"Found {len(filtered_data)} matching entries.")
+
+for plan in filtered_data:
+ print("\nHarvest Document:", plan['Harvest Document'])
+
+ location = extract_location(plan['Raw Data'])
+ print("Location:", location)
+
+ plss_coordinates = extract_plss_coordinates(location)
+ print("PLSS Coordinates:")
+ for coord in plss_coordinates:
+ meridian, township, township_dir, range_, range_dir, section = coord
+ print(f" {meridian}: T{township}{township_dir} R{range_}{range_dir} S{section}")
+
+ print("-" * 50)
diff --git a/sijapi/helpers/fromvm/article.py b/sijapi/helpers/fromvm/article.py
new file mode 100755
index 0000000..c76a6e1
--- /dev/null
+++ b/sijapi/helpers/fromvm/article.py
@@ -0,0 +1,23 @@
+#!/Users/sij/miniforge3/envs/sijapi/bin/python
+import sys
+import asyncio
+from fastapi import BackgroundTasks
+from sijapi.routers.news import process_and_save_article
+
+async def main():
+ if len(sys.argv) != 2:
+ print("Usage: python script.py <article_url>")
+ sys.exit(1)
+
+ url = sys.argv[1]
+ bg_tasks = BackgroundTasks()
+
+ try:
+ result = await process_and_save_article(bg_tasks, url)
+ print(result)
+ except Exception as e:
+ print(f"Error processing article: {str(e)}")
+ sys.exit(1)
+
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/sijapi/helpers/fromvm/db/docker-compose.yaml b/sijapi/helpers/fromvm/db/docker-compose.yaml
new file mode 100644
index 0000000..de8fa37
--- /dev/null
+++ b/sijapi/helpers/fromvm/db/docker-compose.yaml
@@ -0,0 +1,25 @@
+version: '3.8'
+
+services:
+ db:
+ image: postgis/postgis:16-3.4
+ container_name: sij_postgres
+ environment:
+ POSTGRES_DB: sij
+ POSTGRES_USER: sij
+ POSTGRES_PASSWORD: Synchr0!
+ volumes:
+ - postgres_data:/var/lib/postgresql/data
+ - ./init-db.sh:/docker-entrypoint-initdb.d/init-db.sh
+ ports:
+ - "5432:5432"
+ networks:
+ - sij_network
+
+networks:
+ sij_network:
+ driver: bridge
+
+volumes:
+ postgres_data:
+
diff --git a/sijapi/helpers/fromvm/db/init-db.sh b/sijapi/helpers/fromvm/db/init-db.sh
new file mode 100755
index 0000000..d354e40
--- /dev/null
+++ b/sijapi/helpers/fromvm/db/init-db.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+ CREATE EXTENSION IF NOT EXISTS postgis;
+ CREATE EXTENSION IF NOT EXISTS postgis_topology;
+EOSQL
+
+# Modify pg_hba.conf to allow connections from Tailscale network
+echo "host all all 100.64.64.0/24 trust" >> /var/lib/postgresql/data/pg_hba.conf
+
diff --git a/sijapi/helpers/fromvm/elevenlabs_history_ids_20240630_131617.json b/sijapi/helpers/fromvm/elevenlabs_history_ids_20240630_131617.json
new file mode 100644
index 0000000..d10f7da
--- /dev/null
+++ b/sijapi/helpers/fromvm/elevenlabs_history_ids_20240630_131617.json
@@ -0,0 +1,104 @@
+{
+ "history_item_ids": [
+ "ncRYNd0Xef4LiUE74VjP",
+ "13pQLDAPYGIATwW1ySL5",
+ "dhsQNAYTWpcwo1X6rixf",
+ "V7wUip1NJuWAUw26sePF",
+ "mOYMa5lcI7wRHddIQTSa",
+ "mP97iOpA4oG7pwUBthq4",
+ "WTU5nsX6qZCYxLyoT5hq",
+ "15DPGnBgjr74KT3TMbK4",
+ "aCyBS1zoaweVjUoPf2TF",
+ "J8SUMQqZPtoy3Cgdhi3J",
+ "qKHaaJHfqh2je60Wmadb",
+ "2PaugQJ8c4rY44JGlaO5",
+ "TwzxcmYjo6XNebbMabcd",
+ "xdEK7rYq9UofOlkr565b",
+ "wik4jYd97aGMLgttTjC9",
+ "7oXn2yH7gdyhi6sEoWKd",
+ "jv8aZFiVe8gPMrAOBcNT",
+ "B2BctCDkCtLDxEMMBu9z",
+ "4KFO77NHDruNQvXIykwp",
+ "d033NizZaNZPc45fvxCO",
+ "yBKxOxfzsjpZYOFzoIM7",
+ "oEihKwMLWgvvoTLGx4yF",
+ "Q3guBm4hGml0KPAWKl7t",
+ "jaojY1gSafQmqshR48oT",
+ "yqGDMfcceaoceFEEurqa",
+ "oLdnyUp7plGrUMRVQ8Cf",
+ "FZAGCGosYEGMf8GCRFaA",
+ "TrWnXRdGkiH0K9kgwFiS",
+ "th16OEbg3u0XHslT9A33",
+ "856BAsn6dnzF7HeqGPfK",
+ "KjLoAfDXVBqR9s39T25j",
+ "uHQQJMMOfOxPAhEYQXLl",
+ "HO8WCIhkkI7AxwkU5MC6",
+ "9nxdesHWTRLCOd6YgWe9",
+ "tmx5tlIQ7hdSTgJt16P2",
+ "M9JN0YcBuCF6LhnqKN66",
+ "M9xkP4ecn0LIi7mQOfU6",
+ "CNtJgh52Ykh9ZqEppZeH",
+ "lgobcoiqmtWfbXkhEwbE",
+ "nr9jxnsE4DnwmTwCaHqC",
+ "Rnzo03tcyBqGPdmHemCb",
+ "X3YVGp7yf9GLgZ7WOuSU",
+ "wL3bkqxR9xqeFTvkJpSI",
+ "wNx3XDgFLTjVbMyGrIAO",
+ "rb0jj1ywBetmdvve5qIL",
+ "WdNnqvNswXeh6JFoaRSS",
+ "WT2ViyerKpodYmHDHhCw",
+ "OvhIRehXNwx7xMJHuTd7",
+ "EQb1iZtsADxJ0GxLJzEK",
+ "WXVfBJYoYGB7S61VyETD",
+ "q0q3Di1YJKF07dOhoa7E",
+ "a2XBIUPa68UiiKlzwFnG",
+ "YBuD7KsUpz8jxc5ItZcF",
+ "KdoucRVCVQGRVQ8Di9Ih",
+ "CkmDny98GEdfGuj2kaAx",
+ "R0R2p8luRZL7wwPtDilw",
+ "awvztgQnuaquK0dTpIuH",
+ "3ZPN0nJo8UQZYhFhoIOK",
+ "RJJeTkjYIgdv1ZoXXAax",
+ "ppxUNzWHAQafsM6OvEUE",
+ "f2VBm7yE7qmnjdS9CbYz",
+ "SZIMwz2T5ZAhTxTDBFol",
+ "YjC91PRgnQbAcdPhnWqU",
+ "fDTV7n8f6QK5yCwLkBwg",
+ "KbPpWUuiLPADj9H3OlvG",
+ "DIuqVoAg7lLxpvFBip84",
+ "pEwFAKMLGWUMHqfljJSq",
+ "9wwl7UbsgeKqrk8kNZin",
+ "2uLvjJgcZDiY9dqB8JlP",
+ "U5f1qZQM08t2YzJqEmxK",
+ "gnwn7QIhrCXRAGNddZ1H",
+ "g5nGEIHirFzKstdrGI1h",
+ "CQWH5dGSeS38VC4X4yg7",
+ "C5YGjhJPrTkVOpxIOHdj",
+ "YLbtnf1pSb9Ra7wgFHiF",
+ "qNLgNSvMr4VSoisKS9qj",
+ "Bq2ALvQVsj9L2wMpUvYO",
+ "gi0yTXLZLMhUKeKcalWc",
+ "3JQN9UbCsqj9ggi5sCkq",
+ "oPflJoA9kqBzjlmWY6zL",
+ "0kUZFgtZdqgdUBXFsXs9",
+ "aFTi7XdjR8W52ThmFpgc",
+ "pgIfjcy2UvKggfqJ1aNx",
+ "r0VguLaqnxTL9jza9H4y",
+ "444ehr4RtqgU1xjhhTLo",
+ "pEuzoznVDaQRBhIA9VTy",
+ "T9hdW9eJkEqDmOsSUoeY",
+ "wJjHbGzoWiKKOIGmf82T",
+ "kij4uMmkUlsSDu2zSH1k",
+ "oWt5rns196JsKIYPyrBS",
+ "SJ1m9mSOGOLIhkMgA8kq",
+ "kAaqe0ATrYtkifmZLOE5",
+ "O2Pvz7CP5rfyNvzFSDmy",
+ "w1rb8qN5nohVUovC0XAx",
+ "njFs4I4F7rtd9I6fEn6x",
+ "miFrp9GBm3MsHO03Z4eY",
+ "5DJywiPsfeVP9hFdqRhd",
+ "mUephoXhk5QdWrOfr9Xr",
+ "tDDiW3Yp0BptZ2wBv21A",
+ "YpX06liXWHquUVYFlKYa"
+ ]
+}
\ No newline at end of file
diff --git a/sijapi/helpers/fromvm/log_prior_emails.py b/sijapi/helpers/fromvm/log_prior_emails.py
new file mode 100644
index 0000000..10148cf
--- /dev/null
+++ b/sijapi/helpers/fromvm/log_prior_emails.py
@@ -0,0 +1,63 @@
+import asyncio
+from pathlib import Path
+from sijapi import L, EMAIL_CONFIG, EMAIL_LOGS
+from sijapi.classes import EmailAccount
+from sijapi.routers import email
+
+async def initialize_log_files():
+ summarized_log = EMAIL_LOGS / "summarized.txt"
+ autoresponded_log = EMAIL_LOGS / "autoresponded.txt"
+ diagnostic_log = EMAIL_LOGS / "diagnostic.txt"
+ for log_file in [summarized_log, autoresponded_log, diagnostic_log]:
+ log_file.parent.mkdir(parents=True, exist_ok=True)
+ log_file.write_text("")
+ L.DEBUG(f"Log files initialized: {summarized_log}, {autoresponded_log}, {diagnostic_log}")
+ return summarized_log, autoresponded_log, diagnostic_log
+
+async def process_all_emails(account: EmailAccount, summarized_log: Path, autoresponded_log: Path, diagnostic_log: Path):
+ try:
+ with email.get_imap_connection(account) as inbox:
+ L.DEBUG(f"Connected to {account.name}, processing all emails...")
+ all_messages = inbox.messages()
+ unread_messages = set(uid for uid, _ in inbox.messages(unread=True))
+
+ processed_count = 0
+ for identifier, message in all_messages:
+ # Log diagnostic information
+ with open(diagnostic_log, 'a') as f:
+ f.write(f"Account: {account.name}, Raw Identifier: {identifier}, Type: {type(identifier)}\n")
+
+ # Attempt to get a string representation of the identifier
+ if isinstance(identifier, bytes):
+ id_str = identifier.decode()
+ elif isinstance(identifier, (int, str)):
+ id_str = str(identifier)
+ else:
+ id_str = repr(identifier)
+
+ if identifier not in unread_messages:
+ processed_count += 1
+ for log_file in [summarized_log, autoresponded_log]:
+ with open(log_file, 'a') as f:
+ f.write(f"{id_str}\n")
+
+ L.INFO(f"Processed {processed_count} non-unread emails for account {account.name}")
+ except Exception as e:
+ L.logger.error(f"An error occurred while processing emails for account {account.name}: {e}")
+
+async def main():
+ email_accounts = email.load_email_accounts(EMAIL_CONFIG)
+ summarized_log, autoresponded_log, diagnostic_log = await initialize_log_files()
+
+ L.DEBUG(f"Processing {len(email_accounts)} email accounts")
+
+ tasks = [process_all_emails(account, summarized_log, autoresponded_log, diagnostic_log) for account in email_accounts]
+ await asyncio.gather(*tasks)
+
+ # Final verification
+ with open(summarized_log, 'r') as f:
+ final_count = len(f.readlines())
+ L.INFO(f"Final non-unread email count: {final_count}")
+
+if __name__ == "__main__":
+ asyncio.run(main())
\ No newline at end of file
diff --git a/sijapi/helpers/fromvm/migrate_db_to_uuid.py b/sijapi/helpers/fromvm/migrate_db_to_uuid.py
new file mode 100644
index 0000000..60034fb
--- /dev/null
+++ b/sijapi/helpers/fromvm/migrate_db_to_uuid.py
@@ -0,0 +1,191 @@
+import psycopg2
+from psycopg2 import sql
+import sys
+
+def connect_to_db():
+ return psycopg2.connect(
+ dbname='sij',
+ user='sij',
+ password='Synchr0!',
+ host='localhost'
+ )
+
+def get_tables(cur):
+ cur.execute("""
+ SELECT table_name
+ FROM information_schema.tables
+ WHERE table_schema = 'public' AND table_type = 'BASE TABLE'
+ AND table_name NOT LIKE '%_uuid' AND table_name NOT LIKE '%_orig'
+ AND table_name != 'spatial_ref_sys'
+ """)
+ return [row[0] for row in cur.fetchall()]
+
+def get_columns(cur, table_name):
+ cur.execute("""
+ SELECT column_name, udt_name,
+ is_nullable, column_default,
+ character_maximum_length, numeric_precision, numeric_scale
+ FROM information_schema.columns
+ WHERE table_name = %s
+ ORDER BY ordinal_position
+ """, (table_name,))
+ return cur.fetchall()
+
+def get_constraints(cur, table_name):
+ cur.execute("""
+ SELECT conname, contype, pg_get_constraintdef(c.oid)
+ FROM pg_constraint c
+ JOIN pg_namespace n ON n.oid = c.connamespace
+ WHERE conrelid = %s::regclass
+ AND n.nspname = 'public'
+ """, (table_name,))
+ return cur.fetchall()
+
+def drop_table_if_exists(cur, table_name):
+ cur.execute(sql.SQL("DROP TABLE IF EXISTS {} CASCADE").format(sql.Identifier(table_name)))
+
+def create_uuid_table(cur, old_table, new_table):
+ drop_table_if_exists(cur, new_table)
+ columns = get_columns(cur, old_table)
+ constraints = get_constraints(cur, old_table)
+
+ column_defs = []
+ has_id_column = any(col[0] == 'id' for col in columns)
+
+ for col in columns:
+ col_name, udt_name, is_nullable, default, max_length, precision, scale = col
+ if col_name == 'id' and has_id_column:
+ column_defs.append(sql.SQL("{} UUID PRIMARY KEY DEFAULT gen_random_uuid()").format(sql.Identifier(col_name)))
+ else:
+ type_sql = sql.SQL("{}").format(sql.Identifier(udt_name))
+ if max_length:
+ type_sql = sql.SQL("{}({})").format(type_sql, sql.Literal(max_length))
+ elif precision and scale:
+ type_sql = sql.SQL("{}({},{})").format(type_sql, sql.Literal(precision), sql.Literal(scale))
+
+ column_def = sql.SQL("{} {}").format(sql.Identifier(col_name), type_sql)
+ if is_nullable == 'NO':
+ column_def = sql.SQL("{} NOT NULL").format(column_def)
+ if default and 'nextval' not in default: # Skip auto-increment defaults
+ column_def = sql.SQL("{} DEFAULT {}").format(column_def, sql.SQL(default))
+ column_defs.append(column_def)
+
+ constraint_defs = []
+ for constraint in constraints:
+ conname, contype, condef = constraint
+ if contype != 'p' or not has_id_column: # Keep primary key if there's no id column
+ constraint_defs.append(sql.SQL(condef))
+
+ if not has_id_column:
+ column_defs.append(sql.SQL("uuid UUID DEFAULT gen_random_uuid()"))
+
+ query = sql.SQL("CREATE TABLE {} ({})").format(
+ sql.Identifier(new_table),
+ sql.SQL(", ").join(column_defs + constraint_defs)
+ )
+ cur.execute(query)
+
+def migrate_data(cur, old_table, new_table):
+ columns = get_columns(cur, old_table)
+ column_names = [col[0] for col in columns]
+ has_id_column = 'id' in column_names
+
+ if has_id_column:
+ column_names.remove('id')
+ old_cols = sql.SQL(", ").join(map(sql.Identifier, column_names))
+ new_cols = sql.SQL(", ").join(map(sql.Identifier, ['id'] + column_names))
+ query = sql.SQL("INSERT INTO {} ({}) SELECT gen_random_uuid(), {} FROM {}").format(
+ sql.Identifier(new_table),
+ new_cols,
+ old_cols,
+ sql.Identifier(old_table)
+ )
+ else:
+ old_cols = sql.SQL(", ").join(map(sql.Identifier, column_names))
+ new_cols = sql.SQL(", ").join(map(sql.Identifier, column_names + ['uuid']))
+ query = sql.SQL("INSERT INTO {} ({}) SELECT {}, gen_random_uuid() FROM {}").format(
+ sql.Identifier(new_table),
+ new_cols,
+ old_cols,
+ sql.Identifier(old_table)
+ )
+ cur.execute(query)
+
+def update_foreign_keys(cur, tables):
+ for table in tables:
+ constraints = get_constraints(cur, table)
+ for constraint in constraints:
+ conname, contype, condef = constraint
+ if contype == 'f': # Foreign key constraint
+ referenced_table = condef.split('REFERENCES ')[1].split('(')[0].strip()
+ referenced_column = condef.split('(')[2].split(')')[0].strip()
+ local_column = condef.split('(')[1].split(')')[0].strip()
+
+ cur.execute(sql.SQL("""
+ UPDATE {table_uuid}
+ SET {local_column} = subquery.new_id::text::{local_column_type}
+ FROM (
+ SELECT old.{ref_column} AS old_id, new_table.id AS new_id
+ FROM {ref_table} old
+ JOIN public.{ref_table_uuid} new_table ON new_table.{ref_column}::text = old.{ref_column}::text
+ ) AS subquery
+ WHERE {local_column}::text = subquery.old_id::text
+ """).format(
+ table_uuid=sql.Identifier(f"{table}_uuid"),
+ local_column=sql.Identifier(local_column),
+ local_column_type=sql.SQL(get_column_type(cur, f"{table}_uuid", local_column)),
+ ref_column=sql.Identifier(referenced_column),
+ ref_table=sql.Identifier(referenced_table),
+ ref_table_uuid=sql.Identifier(f"{referenced_table}_uuid")
+ ))
+
+def get_column_type(cur, table_name, column_name):
+ cur.execute("""
+ SELECT data_type
+ FROM information_schema.columns
+ WHERE table_name = %s AND column_name = %s
+ """, (table_name, column_name))
+ return cur.fetchone()[0]
+
+def rename_tables(cur, tables):
+ for table in tables:
+ drop_table_if_exists(cur, f"{table}_orig")
+ cur.execute(sql.SQL("ALTER TABLE IF EXISTS {} RENAME TO {}").format(
+ sql.Identifier(table), sql.Identifier(f"{table}_orig")
+ ))
+ cur.execute(sql.SQL("ALTER TABLE IF EXISTS {} RENAME TO {}").format(
+ sql.Identifier(f"{table}_uuid"), sql.Identifier(table)
+ ))
+
+def main():
+ try:
+ with connect_to_db() as conn:
+ with conn.cursor() as cur:
+ tables = get_tables(cur)
+
+ # Create new UUID tables
+ for table in tables:
+ print(f"Creating UUID table for {table}...")
+ create_uuid_table(cur, table, f"{table}_uuid")
+
+ # Migrate data
+ for table in tables:
+ print(f"Migrating data for {table}...")
+ migrate_data(cur, table, f"{table}_uuid")
+
+ # Update foreign keys
+ print("Updating foreign key references...")
+ update_foreign_keys(cur, tables)
+
+ # Rename tables
+ print("Renaming tables...")
+ rename_tables(cur, tables)
+
+ conn.commit()
+ print("Migration completed successfully.")
+ except Exception as e:
+ print(f"An error occurred: {e}")
+ conn.rollback()
+
+if __name__ == "__main__":
+ main()
diff --git a/sijapi/helpers/fromvm/schema_info.yaml b/sijapi/helpers/fromvm/schema_info.yaml
new file mode 100644
index 0000000..4899924
--- /dev/null
+++ b/sijapi/helpers/fromvm/schema_info.yaml
@@ -0,0 +1,1103 @@
+click_logs:
+ clicked_at:
+ data_type: timestamp without time zone
+ is_nullable: 'YES'
+ max_length: null
+ id:
+ data_type: uuid
+ is_nullable: 'NO'
+ max_length: null
+ ip_address:
+ data_type: text
+ is_nullable: 'YES'
+ max_length: null
+ short_code:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: 3
+ user_agent:
+ data_type: text
+ is_nullable: 'YES'
+ max_length: null
+dailyweather:
+ cloudcover:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ conditions:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ datetime:
+ data_type: timestamp with time zone
+ is_nullable: 'YES'
+ max_length: null
+ datetimeepoch:
+ data_type: bigint
+ is_nullable: 'YES'
+ max_length: null
+ description:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ dew:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ feelslike:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ feelslikemax:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ feelslikemin:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ humidity:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ icon:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ id:
+ data_type: integer
+ is_nullable: 'NO'
+ max_length: null
+ last_updated:
+ data_type: timestamp with time zone
+ is_nullable: 'YES'
+ max_length: null
+ location:
+ data_type: USER-DEFINED
+ is_nullable: 'YES'
+ max_length: null
+ moonphase:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ precip:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ precipcover:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ precipprob:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ preciptype:
+ data_type: ARRAY
+ is_nullable: 'YES'
+ max_length: null
+ pressure:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ severerisk:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ snow:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ snowdepth:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ solarenergy:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ solarradiation:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ source:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ stations:
+ data_type: ARRAY
+ is_nullable: 'YES'
+ max_length: null
+ sunrise:
+ data_type: timestamp with time zone
+ is_nullable: 'YES'
+ max_length: null
+ sunriseepoch:
+ data_type: bigint
+ is_nullable: 'YES'
+ max_length: null
+ sunset:
+ data_type: timestamp with time zone
+ is_nullable: 'YES'
+ max_length: null
+ sunsetepoch:
+ data_type: bigint
+ is_nullable: 'YES'
+ max_length: null
+ temp:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ tempmax:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ tempmin:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ uvindex:
+ data_type: integer
+ is_nullable: 'YES'
+ max_length: null
+ visibility:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ winddir:
+ data_type: integer
+ is_nullable: 'YES'
+ max_length: null
+ windgust:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ windspeed:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+dailyweather_uuid:
+ cloudcover:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ conditions:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ datetime:
+ data_type: timestamp with time zone
+ is_nullable: 'YES'
+ max_length: null
+ datetimeepoch:
+ data_type: bigint
+ is_nullable: 'YES'
+ max_length: null
+ description:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ dew:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ feelslike:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ feelslikemax:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ feelslikemin:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ humidity:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ icon:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ id:
+ data_type: uuid
+ is_nullable: 'NO'
+ max_length: null
+ last_updated:
+ data_type: timestamp with time zone
+ is_nullable: 'YES'
+ max_length: null
+ location:
+ data_type: USER-DEFINED
+ is_nullable: 'YES'
+ max_length: null
+ moonphase:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ precip:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ precipcover:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ precipprob:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ preciptype:
+ data_type: ARRAY
+ is_nullable: 'YES'
+ max_length: null
+ pressure:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ severerisk:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ snow:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ snowdepth:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ solarenergy:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ solarradiation:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ source:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ stations:
+ data_type: ARRAY
+ is_nullable: 'YES'
+ max_length: null
+ sunrise:
+ data_type: timestamp with time zone
+ is_nullable: 'YES'
+ max_length: null
+ sunriseepoch:
+ data_type: bigint
+ is_nullable: 'YES'
+ max_length: null
+ sunset:
+ data_type: timestamp with time zone
+ is_nullable: 'YES'
+ max_length: null
+ sunsetepoch:
+ data_type: bigint
+ is_nullable: 'YES'
+ max_length: null
+ temp:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ tempmax:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ tempmin:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ uvindex:
+ data_type: integer
+ is_nullable: 'YES'
+ max_length: null
+ visibility:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ winddir:
+ data_type: integer
+ is_nullable: 'YES'
+ max_length: null
+ windgust:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ windspeed:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+geography_columns:
+ coord_dimension:
+ data_type: integer
+ is_nullable: 'YES'
+ max_length: null
+ f_geography_column:
+ data_type: name
+ is_nullable: 'YES'
+ max_length: null
+ f_table_catalog:
+ data_type: name
+ is_nullable: 'YES'
+ max_length: null
+ f_table_name:
+ data_type: name
+ is_nullable: 'YES'
+ max_length: null
+ f_table_schema:
+ data_type: name
+ is_nullable: 'YES'
+ max_length: null
+ srid:
+ data_type: integer
+ is_nullable: 'YES'
+ max_length: null
+ type:
+ data_type: text
+ is_nullable: 'YES'
+ max_length: null
+geometry_columns:
+ coord_dimension:
+ data_type: integer
+ is_nullable: 'YES'
+ max_length: null
+ f_geometry_column:
+ data_type: name
+ is_nullable: 'YES'
+ max_length: null
+ f_table_catalog:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: 256
+ f_table_name:
+ data_type: name
+ is_nullable: 'YES'
+ max_length: null
+ f_table_schema:
+ data_type: name
+ is_nullable: 'YES'
+ max_length: null
+ srid:
+ data_type: integer
+ is_nullable: 'YES'
+ max_length: null
+ type:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: 30
+hourlyweather:
+ cloudcover:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ conditions:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ daily_weather_id:
+ data_type: integer
+ is_nullable: 'YES'
+ max_length: null
+ datetime:
+ data_type: timestamp with time zone
+ is_nullable: 'YES'
+ max_length: null
+ datetimeepoch:
+ data_type: bigint
+ is_nullable: 'YES'
+ max_length: null
+ dew:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ feelslike:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ humidity:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ icon:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ id:
+ data_type: integer
+ is_nullable: 'NO'
+ max_length: null
+ precip:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ precipprob:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ preciptype:
+ data_type: ARRAY
+ is_nullable: 'YES'
+ max_length: null
+ pressure:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ severerisk:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ snow:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ snowdepth:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ solarenergy:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ solarradiation:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ source:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ stations:
+ data_type: ARRAY
+ is_nullable: 'YES'
+ max_length: null
+ temp:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ uvindex:
+ data_type: integer
+ is_nullable: 'YES'
+ max_length: null
+ visibility:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ winddir:
+ data_type: integer
+ is_nullable: 'YES'
+ max_length: null
+ windgust:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ windspeed:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+hourlyweather_uuid:
+ cloudcover:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ conditions:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ daily_weather_id:
+ data_type: integer
+ is_nullable: 'YES'
+ max_length: null
+ datetime:
+ data_type: timestamp with time zone
+ is_nullable: 'YES'
+ max_length: null
+ datetimeepoch:
+ data_type: bigint
+ is_nullable: 'YES'
+ max_length: null
+ dew:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ feelslike:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ humidity:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ icon:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ id:
+ data_type: uuid
+ is_nullable: 'NO'
+ max_length: null
+ precip:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ precipprob:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ preciptype:
+ data_type: ARRAY
+ is_nullable: 'YES'
+ max_length: null
+ pressure:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ severerisk:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ snow:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ snowdepth:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ solarenergy:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ solarradiation:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ source:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ stations:
+ data_type: ARRAY
+ is_nullable: 'YES'
+ max_length: null
+ temp:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ uvindex:
+ data_type: integer
+ is_nullable: 'YES'
+ max_length: null
+ visibility:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ winddir:
+ data_type: integer
+ is_nullable: 'YES'
+ max_length: null
+ windgust:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ windspeed:
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+locations:
+ action:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ amenity:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ boundingbox:
+ data_type: ARRAY
+ is_nullable: 'YES'
+ max_length: null
+ city:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: 255
+ class:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ class_:
+ data_type: text
+ is_nullable: 'YES'
+ max_length: null
+ context:
+ data_type: jsonb
+ is_nullable: 'YES'
+ max_length: null
+ country:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ country_code:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ county:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ daily_weather_id:
+ data_type: integer
+ is_nullable: 'YES'
+ max_length: null
+ datetime:
+ data_type: timestamp with time zone
+ is_nullable: 'YES'
+ max_length: null
+ device_model:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ device_name:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ device_os:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ device_type:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ display_name:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ house_number:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ id:
+ data_type: integer
+ is_nullable: 'NO'
+ max_length: null
+ location:
+ data_type: USER-DEFINED
+ is_nullable: 'YES'
+ max_length: null
+ name:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ neighbourhood:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ quarter:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ road:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ state:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: 255
+ street:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: 255
+ suburb:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ type:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ zip:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: 255
+locations_uuid:
+ action:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ amenity:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ boundingbox:
+ data_type: ARRAY
+ is_nullable: 'YES'
+ max_length: null
+ city:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: 255
+ class:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ class_:
+ data_type: text
+ is_nullable: 'YES'
+ max_length: null
+ context:
+ data_type: jsonb
+ is_nullable: 'YES'
+ max_length: null
+ country:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ country_code:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ county:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ daily_weather_id:
+ data_type: integer
+ is_nullable: 'YES'
+ max_length: null
+ datetime:
+ data_type: timestamp with time zone
+ is_nullable: 'YES'
+ max_length: null
+ device_model:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ device_name:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ device_os:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ device_type:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ display_name:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ house_number:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ id:
+ data_type: uuid
+ is_nullable: 'NO'
+ max_length: null
+ location:
+ data_type: USER-DEFINED
+ is_nullable: 'YES'
+ max_length: null
+ name:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ neighbourhood:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ quarter:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ road:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ state:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: 255
+ street:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: 255
+ suburb:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ type:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ zip:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: 255
+plss_sections:
+ frstdivdup:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ frstdivid:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ frstdivlab:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ frstdivno:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ frstdivtxt:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ frstdivtyp:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ objectid:
+ data_type: integer
+ is_nullable: 'YES'
+ max_length: null
+ ogc_fid:
+ data_type: integer
+ is_nullable: 'NO'
+ max_length: null
+ plssid:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ shape.starea():
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ shape.stlength():
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ sourcedate:
+ data_type: bigint
+ is_nullable: 'YES'
+ max_length: null
+ sourceref:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ survtyp:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ survtyptxt:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ wkb_geometry:
+ data_type: USER-DEFINED
+ is_nullable: 'YES'
+ max_length: null
+plss_townships:
+ created_date:
+ data_type: bigint
+ is_nullable: 'YES'
+ max_length: null
+ created_user:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ globalid:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ last_edited_date:
+ data_type: bigint
+ is_nullable: 'YES'
+ max_length: null
+ last_edited_user:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ objectid:
+ data_type: integer
+ is_nullable: 'YES'
+ max_length: null
+ ogc_fid:
+ data_type: integer
+ is_nullable: 'NO'
+ max_length: null
+ origid:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ partcount:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ plssid:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ prinmer:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ prinmercd:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ rangedir:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ rangefrac:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ rangeno:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ secsrvname:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ shape.starea():
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ shape.stlength():
+ data_type: double precision
+ is_nullable: 'YES'
+ max_length: null
+ sourcedate:
+ data_type: bigint
+ is_nullable: 'YES'
+ max_length: null
+ sourceref:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ srvname:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ stateabbr:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ steward:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ survtyp:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ survtyptxt:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ twnshpdir:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ twnshpdpcd:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ twnshpfrac:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ twnshplab:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ twnshpno:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: null
+ wkb_geometry:
+ data_type: USER-DEFINED
+ is_nullable: 'YES'
+ max_length: null
+relations:
+ id:
+ data_type: bigint
+ is_nullable: 'NO'
+ max_length: null
+ members:
+ data_type: jsonb
+ is_nullable: 'YES'
+ max_length: null
+ tags:
+ data_type: jsonb
+ is_nullable: 'YES'
+ max_length: null
+short_urls:
+ created_at:
+ data_type: timestamp without time zone
+ is_nullable: 'YES'
+ max_length: null
+ id:
+ data_type: integer
+ is_nullable: 'NO'
+ max_length: null
+ long_url:
+ data_type: text
+ is_nullable: 'NO'
+ max_length: null
+ short_code:
+ data_type: character varying
+ is_nullable: 'NO'
+ max_length: 3
+short_urls_uuid:
+ created_at:
+ data_type: timestamp without time zone
+ is_nullable: 'YES'
+ max_length: null
+ id:
+ data_type: uuid
+ is_nullable: 'NO'
+ max_length: null
+ long_url:
+ data_type: text
+ is_nullable: 'NO'
+ max_length: null
+ short_code:
+ data_type: character varying
+ is_nullable: 'NO'
+ max_length: 3
+spatial_ref_sys:
+ auth_name:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: 256
+ auth_srid:
+ data_type: integer
+ is_nullable: 'YES'
+ max_length: null
+ proj4text:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: 2048
+ srid:
+ data_type: integer
+ is_nullable: 'NO'
+ max_length: null
+ srtext:
+ data_type: character varying
+ is_nullable: 'YES'
+ max_length: 2048
diff --git a/sijapi/helpers/fromvm/start.py b/sijapi/helpers/fromvm/start.py
new file mode 100644
index 0000000..031e4fb
--- /dev/null
+++ b/sijapi/helpers/fromvm/start.py
@@ -0,0 +1,211 @@
+import yaml
+import requests
+import paramiko
+import time
+from pathlib import Path
+import logging
+import subprocess
+import os
+import argparse
+import sys
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+def load_config():
+ config_path = Path(__file__).parent.parent / 'config' / 'sys.yaml'
+ with open(config_path, 'r') as file:
+ return yaml.safe_load(file)
+
+def load_env():
+ env_path = Path(__file__).parent.parent / 'config' / '.env'
+ if env_path.exists():
+ with open(env_path, 'r') as file:
+ for line in file:
+ line = line.strip()
+ if line and not line.startswith('#'):
+ try:
+ key, value = line.split('=', 1)
+ os.environ[key.strip()] = value.strip()
+ except ValueError:
+ logging.warning(f"Skipping invalid line in .env file: {line}")
+
+def check_server(ip, port, ts_id):
+ address = f"http://{ip}:{port}/id"
+ try:
+ response = requests.get(address, timeout=5)
+ response_text = response.text.strip().strip('"')
+ return response.status_code == 200 and response_text == ts_id
+ except requests.RequestException as e:
+ logging.error(f"Error checking server {ts_id}: {str(e)}")
+ return False
+
+def execute_ssh_command(ssh, command):
+ stdin, stdout, stderr = ssh.exec_command(command)
+ exit_status = stdout.channel.recv_exit_status()
+ output = stdout.read().decode().strip()
+ error = stderr.read().decode().strip()
+ return exit_status, output, error
+
+def is_local_tmux_session_running(session_name):
+ try:
+ result = subprocess.run(['tmux', 'has-session', '-t', session_name], capture_output=True, text=True)
+ return result.returncode == 0
+ except subprocess.CalledProcessError:
+ return False
+
+def start_local_server(server, pull=False, push=False):
+ try:
+ if is_local_tmux_session_running('sijapi'):
+ logging.info("Local sijapi tmux session is already running.")
+ return
+
+ git_command = ""
+ if pull:
+ git_command = "git pull &&"
+ elif push:
+ git_command = "git add -A . && git commit -m \"auto-update\" && git push origin --force &&"
+
+ command = f"{server['tmux']} new-session -d -s sijapi 'cd {server['path']} && {git_command} {server['conda_env']}/bin/python -m sijapi'"
+ logging.info(f"Executing local command: {command}")
+ result = subprocess.run(command, shell=True, check=True, capture_output=True, text=True)
+ logging.info(f"Successfully started sijapi session on local machine")
+ logging.debug(f"Command output: {result.stdout}")
+ except subprocess.CalledProcessError as e:
+ logging.error(f"Failed to start sijapi session on local machine. Error: {e}")
+ logging.error(f"Error output: {e.stderr}")
+
+def start_remote_server(server, pull=False, push=False):
+ ssh = paramiko.SSHClient()
+ ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+ try:
+ ssh.connect(
+ server['ts_ip'],
+ port=server['ssh_port'],
+ username=server['ssh_user'],
+ password=server['ssh_pass'],
+ timeout=10
+ )
+
+ status, output, error = execute_ssh_command(ssh, f"{server['tmux']} has-session -t sijapi 2>/dev/null && echo 'exists' || echo 'not exists'")
+ if output == 'exists':
+ logging.info(f"sijapi session already exists on {server['ts_id']}")
+ return
+
+ git_command = ""
+ if pull:
+ git_command = "git pull &&"
+ elif push:
+ git_command = "git add -A . && git commit -m \"auto-update\" && git push origin --force &&"
+
+ command = f"{server['tmux']} new-session -d -s sijapi 'cd {server['path']} && {git_command} {server['conda_env']}/bin/python -m sijapi'"
+ status, output, error = execute_ssh_command(ssh, command)
+
+ if status == 0:
+ logging.info(f"Successfully started sijapi session on {server['ts_id']}")
+ else:
+ logging.error(f"Failed to start sijapi session on {server['ts_id']}. Error: {error}")
+
+ except paramiko.SSHException as e:
+ logging.error(f"Failed to connect to {server['ts_id']}: {str(e)}")
+ finally:
+ ssh.close()
+
+def kill_local_server():
+ try:
+ if is_local_tmux_session_running('sijapi'):
+ subprocess.run(['tmux', 'kill-session', '-t', 'sijapi'], check=True)
+ logging.info("Killed local sijapi tmux session.")
+ else:
+ logging.info("No local sijapi tmux session to kill.")
+ except subprocess.CalledProcessError as e:
+ logging.error(f"Failed to kill local sijapi tmux session. Error: {e}")
+
+def kill_remote_server(server):
+ ssh = paramiko.SSHClient()
+ ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+ try:
+ ssh.connect(
+ server['ts_ip'],
+ port=server['ssh_port'],
+ username=server['ssh_user'],
+ password=server['ssh_pass'],
+ timeout=10
+ )
+
+ command = f"{server['tmux']} kill-session -t sijapi"
+ status, output, error = execute_ssh_command(ssh, command)
+
+ if status == 0:
+ logging.info(f"Successfully killed sijapi session on {server['ts_id']}")
+ else:
+ logging.error(f"Failed to kill sijapi session on {server['ts_id']}. Error: {error}")
+
+ except paramiko.SSHException as e:
+ logging.error(f"Failed to connect to {server['ts_id']}: {str(e)}")
+ finally:
+ ssh.close()
+
+def main():
+ load_env()
+ config = load_config()
+ pool = config['POOL']
+ local_ts_id = os.environ.get('TS_ID')
+
+ parser = argparse.ArgumentParser(description='Manage sijapi servers')
+ parser.add_argument('--kill', action='store_true', help='Kill the local sijapi tmux session')
+ parser.add_argument('--restart', action='store_true', help='Restart the local sijapi tmux session')
+ parser.add_argument('--all', action='store_true', help='Apply the action to all servers')
+ parser.add_argument('--pull', action='store_true', help='Pull latest changes before starting the server')
+ parser.add_argument('--push', action='store_true', help='Push changes before starting the server')
+
+ args = parser.parse_args()
+
+ if args.kill:
+ if args.all:
+ for server in pool:
+ if server['ts_id'] == local_ts_id:
+ kill_local_server()
+ else:
+ kill_remote_server(server)
+ else:
+ kill_local_server()
+ sys.exit(0)
+
+ if args.restart or args.pull or args.push:
+ if args.all:
+ for server in pool:
+ if server['ts_id'] == local_ts_id:
+ kill_local_server()
+ start_local_server(server, pull=args.pull, push=args.push)
+ else:
+ kill_remote_server(server)
+ start_remote_server(server, pull=args.pull, push=args.push)
+ else:
+ kill_local_server()
+ local_server = next(server for server in pool if server['ts_id'] == local_ts_id)
+ start_local_server(local_server, pull=args.pull, push=args.push)
+ sys.exit(0)
+
+ # If no specific arguments, run the default behavior
+ local_server = next(server for server in pool if server['ts_id'] == local_ts_id)
+ if not check_server(local_server['ts_ip'], local_server['app_port'], local_server['ts_id']):
+ logging.info(f"Local server {local_server['ts_id']} is not responding correctly. Attempting to start...")
+ kill_local_server()
+ start_local_server(local_server, push=True)
+
+ for server in pool:
+ if server['ts_id'] != local_ts_id:
+ if not check_server(server['ts_ip'], server['app_port'], server['ts_id']):
+ logging.info(f"{server['ts_id']} is not responding correctly. Attempting to start...")
+ kill_remote_server(server)
+ start_remote_server(server, pull=True)
+ else:
+ logging.info(f"{server['ts_id']} is running and responding correctly.")
+
+ time.sleep(1)
+
+if __name__ == "__main__":
+ main()
+
diff --git a/sijapi/helpers/fromvm/upscaler.py b/sijapi/helpers/fromvm/upscaler.py
new file mode 100644
index 0000000..4e7ea5d
--- /dev/null
+++ b/sijapi/helpers/fromvm/upscaler.py
@@ -0,0 +1,34 @@
+from aura_sr import AuraSR
+from PIL import Image
+import torch
+import os
+
+# Set environment variables for MPS
+os.environ['PYTORCH_MPS_HIGH_WATERMARK_RATIO'] = '0.0'
+os.environ['PYTORCH_ENABLE_MPS_FALLBACK'] = '1'
+
+# Initialize device as CPU for default
+device = torch.device('cpu')
+
+# Check if MPS is available
+if torch.backends.mps.is_available():
+ if not torch.backends.mps.is_built():
+ print("MPS not available because the current PyTorch install was not built with MPS enabled.")
+ else:
+ device = torch.device('mps:0')
+
+# Overwrite the default CUDA device with MPS
+torch.cuda.default_stream = device
+
+aura_sr = AuraSR.from_pretrained("fal-ai/AuraSR").to(device)
+
+def load_image_from_path(file_path):
+ return Image.open(file_path)
+
+def upscale_and_save(original_path):
+ original_image = load_image_from_path(original_path)
+ upscaled_image = aura_sr.upscale_4x(original_image)
+ upscaled_image.save(original_path)
+
+# Insert your image path
+upscale_and_save("/Users/sij/workshop/sijapi/sijapi/testbed/API__00482_ 2.png")