Auto-update: Mon Aug 12 10:33:44 PDT 2024

This commit is contained in:
sanj 2024-08-12 10:33:44 -07:00
parent b60e60ec1e
commit 14319edaa5
17 changed files with 2444 additions and 0 deletions

1
.gitignore vendored
View file

@ -39,6 +39,7 @@ podcast/sideloads/*
**/.ipynb_checkpoints/
**/*.pem
**/*.key
**/*.sql
venv/
env/
.venv/

View file

@ -0,0 +1,225 @@
#!/usr/bin/env python3
import requests
import json
import time
import os
import subprocess
import sys
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from datetime import datetime
# Environment variables for database connection
DB_NAME = os.getenv('DB_NAME', 'sij')
DB_USER = os.getenv('DB_USER', 'sij')
DB_PASSWORD = os.getenv('DB_PASSWORD', 'Synchr0!')
DB_HOST = os.getenv('DB_HOST', 'localhost')
DB_PORT = os.getenv('DB_PORT', '5432')
def get_feature_count(url):
params = {
'where': '1=1',
'returnCountOnly': 'true',
'f': 'json'
}
retries = Retry(total=10, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504])
with requests.Session() as session:
session.mount("https://", HTTPAdapter(max_retries=retries))
response = session.get(url, params=params, timeout=30)
response.raise_for_status()
data = response.json()
return data.get('count', 0)
def fetch_features(url, offset, num, max_retries=5):
params = {
'where': '1=1',
'outFields': '*',
'geometryPrecision': 6,
'outSR': 4326,
'f': 'json',
'resultOffset': offset,
'resultRecordCount': num
}
for attempt in range(max_retries):
try:
retries = Retry(total=5, backoff_factor=1, status_forcelist=[500, 502, 503, 504])
with requests.Session() as session:
session.mount("https://", HTTPAdapter(max_retries=retries))
response = session.get(url, params=params, timeout=30)
response.raise_for_status()
return response.json()
except requests.exceptions.RequestException as e:
print(f"Error fetching features (attempt {attempt + 1}/{max_retries}): {e}")
if attempt == max_retries - 1:
raise
time.sleep(5 * (attempt + 1)) # Exponential backoff
def download_layer(layer_num, layer_name):
base_dir = os.path.expanduser('~/data')
file_path = os.path.join(base_dir, f'PLSS_{layer_name}.geojson')
temp_file_path = os.path.join(base_dir, f'PLSS_{layer_name}_temp.json')
url = f"https://gis.blm.gov/arcgis/rest/services/Cadastral/BLM_Natl_PLSS_CadNSDI/MapServer/{layer_num}/query"
total_count = get_feature_count(url)
print(f"Total {layer_name} features: {total_count}")
batch_size = 1000
offset = 0
all_features = []
# Check if temporary file exists and load its content
if os.path.exists(temp_file_path):
with open(temp_file_path, 'r') as f:
all_features = json.load(f)
offset = len(all_features)
print(f"Resuming download from offset {offset}")
try:
while offset < total_count:
print(f"Fetching {layer_name} features {offset} to {offset + batch_size}...")
data = fetch_features(url, offset, batch_size)
new_features = data.get('features', [])
if not new_features:
break
all_features.extend(new_features)
offset += len(new_features)
# Progress indicator
progress = len(all_features) / total_count
bar_length = 30
filled_length = int(bar_length * progress)
bar = '=' * filled_length + '-' * (bar_length - filled_length)
print(f'\rProgress: [{bar}] {progress:.1%} ({len(all_features)}/{total_count} features)', end='', flush=True)
# Save progress to temporary file
with open(temp_file_path, 'w') as f:
json.dump(all_features, f)
time.sleep(1)
print(f"\nTotal {layer_name} features fetched: {len(all_features)}")
geojson_features = [
{
"type": "Feature",
"properties": feature['attributes'],
"geometry": feature['geometry']
} for feature in all_features
]
full_geojson = {
"type": "FeatureCollection",
"features": geojson_features
}
os.makedirs(base_dir, exist_ok=True)
with open(file_path, 'w') as f:
json.dump(full_geojson, f)
print(f"GeoJSON file saved as '{file_path}'")
# Remove temporary file
if os.path.exists(temp_file_path):
os.remove(temp_file_path)
return file_path
except Exception as e:
print(f"\nError during download: {e}")
print(f"Partial data saved in {temp_file_path}")
return None
def check_postgres_connection():
try:
subprocess.run(['psql', '-h', DB_HOST, '-p', DB_PORT, '-U', DB_USER, '-d', DB_NAME, '-c', 'SELECT 1;'],
check=True, capture_output=True, text=True)
return True
except subprocess.CalledProcessError:
return False
def check_postgis_extension():
try:
result = subprocess.run(['psql', '-h', DB_HOST, '-p', DB_PORT, '-U', DB_USER, '-d', DB_NAME,
'-c', "SELECT 1 FROM pg_extension WHERE extname = 'postgis';"],
check=True, capture_output=True, text=True)
return '1' in result.stdout
except subprocess.CalledProcessError:
return False
def create_postgis_extension():
try:
subprocess.run(['psql', '-h', DB_HOST, '-p', DB_PORT, '-U', DB_USER, '-d', DB_NAME,
'-c', "CREATE EXTENSION IF NOT EXISTS postgis;"],
check=True, capture_output=True, text=True)
print("PostGIS extension created successfully.")
except subprocess.CalledProcessError as e:
print(f"Error creating PostGIS extension: {e}")
sys.exit(1)
def import_to_postgis(file_path, table_name):
if not check_postgres_connection():
print("Error: Unable to connect to PostgreSQL. Please check your connection settings.")
sys.exit(1)
if not check_postgis_extension():
print("PostGIS extension not found. Attempting to create it...")
create_postgis_extension()
ogr2ogr_command = [
'ogr2ogr',
'-f', 'PostgreSQL',
f'PG:dbname={DB_NAME} user={DB_USER} password={DB_PASSWORD} host={DB_HOST} port={DB_PORT}',
file_path,
'-nln', table_name,
'-overwrite'
]
try:
subprocess.run(ogr2ogr_command, check=True, capture_output=True, text=True)
print(f"Data successfully imported into PostGIS table: {table_name}")
except subprocess.CalledProcessError as e:
print(f"Error importing data into PostGIS: {e}")
print(f"Command that failed: {e.cmd}")
print(f"Error output: {e.stderr}")
def check_ogr2ogr():
try:
subprocess.run(['ogr2ogr', '--version'], check=True, capture_output=True, text=True)
return True
except subprocess.CalledProcessError:
return False
except FileNotFoundError:
return False
def main():
if not check_ogr2ogr():
print("Error: ogr2ogr not found. Please install GDAL/OGR tools.")
print("On Debian: sudo apt-get install gdal-bin")
print("On macOS with Homebrew: brew install gdal")
sys.exit(1)
try:
township_file = os.path.expanduser('~/data/PLSS_Townships.geojson')
if not os.path.exists(township_file):
township_file = download_layer(1, "Townships")
if township_file:
import_to_postgis(township_file, "public.plss_townships")
section_file = os.path.expanduser('~/data/PLSS_Sections.geojson')
if not os.path.exists(section_file):
section_file = download_layer(2, "Sections")
if section_file:
import_to_postgis(section_file, "public.plss_sections")
except requests.exceptions.RequestException as e:
print(f"Error fetching data: {e}")
except Exception as e:
print(f"An unexpected error occurred: {e}")
if __name__ == "__main__":
main()

View file

@ -0,0 +1,133 @@
# CaPLSS_downloader_and_importer.py
import requests
import json
import time
import os
import subprocess
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
def get_feature_count(url):
params = {
'where': '1=1',
'returnCountOnly': 'true',
'f': 'json'
}
retries = Retry(total=10, backoff_factor=0.5)
adapter = HTTPAdapter(max_retries=retries)
session = requests.Session()
session.mount("https://", adapter)
response = session.get(url, params=params, timeout=15) # Add timeout parameter
response.raise_for_status()
data = response.json()
return data.get('count', 0)
def fetch_features(url, offset, num):
params = {
'where': '1=1',
'outFields': '*',
'geometryPrecision': 6,
'outSR': 4326,
'f': 'json',
'resultOffset': offset,
'resultRecordCount': num
}
response = requests.get(url, params=params)
response.raise_for_status()
return response.json()
def download_layer(layer_num, layer_name):
url = f"https://gis.blm.gov/arcgis/rest/services/Cadastral/BLM_Natl_PLSS_CadNSDI/MapServer/{layer_num}/query"
total_count = get_feature_count(url)
print(f"Total {layer_name} features: {total_count}")
batch_size = 1000
offset = 0
all_features = []
while offset < total_count:
print(f"Fetching {layer_name} features {offset} to {offset + batch_size}...")
data = fetch_features(url, offset, batch_size)
new_features = data.get('features', [])
if not new_features:
break
all_features.extend(new_features)
offset += len(new_features)
print(f"Progress: {len(all_features)}/{total_count} features")
time.sleep(1) # Be nice to the server
print(f"Total {layer_name} features fetched: {len(all_features)}")
# Convert to GeoJSON
geojson_features = [
{
"type": "Feature",
"properties": feature['attributes'],
"geometry": feature['geometry']
} for feature in all_features
]
full_geojson = {
"type": "FeatureCollection",
"features": geojson_features
}
# Define a base directory that exists on both macOS and Debian
base_dir = os.path.expanduser('~/data')
os.makedirs(base_dir, exist_ok=True) # Create the directory if it doesn't exist
# Use os.path.join to construct the file path
file_path = os.path.join(base_dir, f'PLSS_{layer_name}.geojson')
# Save to file
with open(file_path, 'w') as f:
json.dump(full_geojson, f)
print(f"GeoJSON file saved as '{file_path}'")
return file_path
def import_to_postgis(file_path, table_name):
db_name = 'sij'
db_user = 'sij'
db_password = 'Synchr0!'
ogr2ogr_command = [
'ogr2ogr',
'-f', 'PostgreSQL',
f'PG:dbname={db_name} user={db_user} password={db_password}',
file_path,
'-nln', table_name,
'-overwrite'
]
subprocess.run(ogr2ogr_command, check=True)
print(f"Data successfully imported into PostGIS table: {table_name}")
def main():
try:
# Download and import Townships (Layer 1)
township_file = download_layer(1, "Townships")
import_to_postgis(township_file, "public.plss_townships")
# Download and import Sections (Layer 2)
section_file = download_layer(2, "Sections")
import_to_postgis(section_file, "public.plss_sections")
except requests.exceptions.RequestException as e:
print(f"Error fetching data: {e}")
except subprocess.CalledProcessError as e:
print(f"Error importing data into PostGIS: {e}")
except Exception as e:
print(f"An unexpected error occurred: {e}")
if __name__ == "__main__":
main()

View file

@ -0,0 +1,73 @@
import requests
import PyPDF2
import io
import re
def scrape_data_from_pdf(url):
response = requests.get(url)
pdf_file = io.BytesIO(response.content)
pdf_reader = PyPDF2.PdfReader(pdf_file)
all_text = ""
for page in pdf_reader.pages:
all_text += page.extract_text() + "\n"
return all_text
def parse_data(raw_data):
lines = raw_data.split('\n')
data = []
current_entry = None
for line in lines:
line = line.strip()
if re.match(r'\d+-\d+-\d+-\w+', line):
if current_entry:
data.append(current_entry)
current_entry = {'Harvest Document': line, 'Raw Data': []}
elif current_entry:
current_entry['Raw Data'].append(line)
if current_entry:
data.append(current_entry)
return data
def filter_data(data):
return [entry for entry in data if any(owner.lower() in ' '.join(entry['Raw Data']).lower() for owner in ["Sierra Pacific", "SPI", "Land & Timber"])]
def extract_location(raw_data):
location = []
for line in raw_data:
if 'MDBM:' in line or 'HBM:' in line:
location.append(line)
return ' '.join(location)
def extract_plss_coordinates(text):
pattern = r'(\w+): T(\d+)([NSEW]) R(\d+)([NSEW]) S(\d+)'
return re.findall(pattern, text)
# Main execution
url = "https://caltreesplans.resources.ca.gov/Caltrees/Report/ShowReport.aspx?module=TH_Document&reportID=492&reportType=LINK_REPORT_LIST"
raw_data = scrape_data_from_pdf(url)
parsed_data = parse_data(raw_data)
print(f"Total timber plans parsed: {len(parsed_data)}")
filtered_data = filter_data(parsed_data)
print(f"Found {len(filtered_data)} matching entries.")
for plan in filtered_data:
print("\nHarvest Document:", plan['Harvest Document'])
location = extract_location(plan['Raw Data'])
print("Location:", location)
plss_coordinates = extract_plss_coordinates(location)
print("PLSS Coordinates:")
for coord in plss_coordinates:
meridian, township, township_dir, range_, range_dir, section = coord
print(f" {meridian}: T{township}{township_dir} R{range_}{range_dir} S{section}")
print("-" * 50)

View file

@ -0,0 +1,23 @@
#!/Users/sij/miniforge3/envs/sijapi/bin/python
import sys
import asyncio
from fastapi import BackgroundTasks
from sijapi.routers.news import process_and_save_article
async def main():
if len(sys.argv) != 2:
print("Usage: python script.py <article_url>")
sys.exit(1)
url = sys.argv[1]
bg_tasks = BackgroundTasks()
try:
result = await process_and_save_article(bg_tasks, url)
print(result)
except Exception as e:
print(f"Error processing article: {str(e)}")
sys.exit(1)
if __name__ == "__main__":
asyncio.run(main())

View file

@ -0,0 +1,46 @@
import asyncio
import asyncpg
# Database connection information
DB_INFO = {
'host': '100.64.64.20',
'port': 5432,
'database': 'sij',
'user': 'sij',
'password': 'Synchr0!'
}
async def update_click_logs():
# Connect to the database
conn = await asyncpg.connect(**DB_INFO)
try:
# Drop existing 'id' and 'new_id' columns if they exist
await conn.execute("""
ALTER TABLE click_logs
DROP COLUMN IF EXISTS id,
DROP COLUMN IF EXISTS new_id;
""")
print("Dropped existing id and new_id columns (if they existed)")
# Add new UUID column as primary key
await conn.execute("""
ALTER TABLE click_logs
ADD COLUMN id UUID PRIMARY KEY DEFAULT gen_random_uuid();
""")
print("Added new UUID column as primary key")
# Get the number of rows in the table
row_count = await conn.fetchval("SELECT COUNT(*) FROM click_logs")
print(f"Number of rows in click_logs: {row_count}")
except Exception as e:
print(f"An error occurred: {str(e)}")
import traceback
traceback.print_exc()
finally:
# Close the database connection
await conn.close()
# Run the update
asyncio.run(update_click_logs())

View file

@ -0,0 +1,25 @@
version: '3.8'
services:
db:
image: postgis/postgis:16-3.4
container_name: sij_postgres
environment:
POSTGRES_DB: sij
POSTGRES_USER: sij
POSTGRES_PASSWORD: Synchr0!
volumes:
- postgres_data:/var/lib/postgresql/data
- ./init-db.sh:/docker-entrypoint-initdb.d/init-db.sh
ports:
- "5432:5432"
networks:
- sij_network
networks:
sij_network:
driver: bridge
volumes:
postgres_data:

View file

@ -0,0 +1,11 @@
#!/bin/bash
set -e
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
CREATE EXTENSION IF NOT EXISTS postgis;
CREATE EXTENSION IF NOT EXISTS postgis_topology;
EOSQL
# Modify pg_hba.conf to allow connections from Tailscale network
echo "host all all 100.64.64.0/24 trust" >> /var/lib/postgresql/data/pg_hba.conf

View file

@ -0,0 +1,66 @@
import asyncio
import asyncpg
import psycopg2
import sys
async def try_async_connect(host, port, user, password, database):
try:
conn = await asyncpg.connect(
host=host,
port=port,
user=user,
password=password,
database=database
)
version = await conn.fetchval('SELECT version()')
print(f"Async connection successful to {host}:{port}")
print(f"PostgreSQL version: {version}")
await conn.close()
return True
except Exception as e:
print(f"Async connection failed to {host}:{port}")
print(f"Error: {str(e)}")
return False
def try_sync_connect(host, port, user, password, database):
try:
conn = psycopg2.connect(
host=host,
port=port,
user=user,
password=password,
database=database
)
cur = conn.cursor()
cur.execute('SELECT version()')
version = cur.fetchone()[0]
print(f"Sync connection successful to {host}:{port}")
print(f"PostgreSQL version: {version}")
conn.close()
return True
except Exception as e:
print(f"Sync connection failed to {host}:{port}")
print(f"Error: {str(e)}")
return False
async def main():
# Database connection parameters
port = 5432
user = 'sij'
password = 'Synchr0!'
database = 'sij'
hosts = ['100.64.64.20', '127.0.0.1', 'localhost']
print("Attempting asynchronous connections:")
for host in hosts:
await try_async_connect(host, port, user, password, database)
print()
print("Attempting synchronous connections:")
for host in hosts:
try_sync_connect(host, port, user, password, database)
print()
if __name__ == "__main__":
asyncio.run(main())

View file

@ -0,0 +1,89 @@
import psycopg2
from psycopg2 import sql
def connect_to_db():
return psycopg2.connect(
dbname='sij',
user='sij',
password='Synchr0!',
host='localhost' # Adjust if your database is not on localhost
)
def get_table_info(conn):
with conn.cursor() as cur:
# Get all tables in the public schema
cur.execute("""
SELECT table_name
FROM information_schema.tables
WHERE table_schema = 'public'
""")
tables = cur.fetchall()
table_info = {}
for (table_name,) in tables:
table_info[table_name] = {
'primary_keys': get_primary_keys(cur, table_name),
'foreign_keys': get_foreign_keys(cur, table_name)
}
return table_info
def get_primary_keys(cur, table_name):
cur.execute("""
SELECT a.attname
FROM pg_index i
JOIN pg_attribute a ON a.attrelid = i.indrelid
AND a.attnum = ANY(i.indkey)
WHERE i.indrelid = %s::regclass
AND i.indisprimary
""", (table_name,))
return [row[0] for row in cur.fetchall()]
def get_foreign_keys(cur, table_name):
cur.execute("""
SELECT
tc.constraint_name,
kcu.column_name,
ccu.table_name AS foreign_table_name,
ccu.column_name AS foreign_column_name
FROM
information_schema.table_constraints AS tc
JOIN information_schema.key_column_usage AS kcu
ON tc.constraint_name = kcu.constraint_name
AND tc.table_schema = kcu.table_schema
JOIN information_schema.constraint_column_usage AS ccu
ON ccu.constraint_name = tc.constraint_name
AND ccu.table_schema = tc.table_schema
WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name=%s
""", (table_name,))
return cur.fetchall()
def main():
try:
with connect_to_db() as conn:
table_info = get_table_info(conn)
for table_name, info in table_info.items():
print(f"\n## Table: {table_name}")
print("\nPrimary Keys:")
if info['primary_keys']:
for pk in info['primary_keys']:
print(f"- {pk}")
else:
print("- No primary keys found")
print("\nForeign Keys:")
if info['foreign_keys']:
for fk in info['foreign_keys']:
print(f"- {fk[1]} -> {fk[2]}.{fk[3]} (Constraint: {fk[0]})")
else:
print("- No foreign keys found")
except psycopg2.Error as e:
print(f"Database error: {e}")
except Exception as e:
print(f"An unexpected error occurred: {e}")
if __name__ == "__main__":
main()

View file

@ -0,0 +1,46 @@
import asyncio
import asyncpg
# Database connection information
DB_INFO = {
'host': '100.64.64.20',
'port': 5432,
'database': 'sij',
'user': 'sij',
'password': 'Synchr0!'
}
async def update_click_logs():
# Connect to the database
conn = await asyncpg.connect(**DB_INFO)
try:
# Drop existing 'id' and 'new_id' columns if they exist
await conn.execute("""
ALTER TABLE click_logs
DROP COLUMN IF EXISTS id,
DROP COLUMN IF EXISTS new_id;
""")
print("Dropped existing id and new_id columns (if they existed)")
# Add new UUID column as primary key
await conn.execute("""
ALTER TABLE click_logs
ADD COLUMN id UUID PRIMARY KEY DEFAULT gen_random_uuid();
""")
print("Added new UUID column as primary key")
# Get the number of rows in the table
row_count = await conn.fetchval("SELECT COUNT(*) FROM click_logs")
print(f"Number of rows in click_logs: {row_count}")
except Exception as e:
print(f"An error occurred: {str(e)}")
import traceback
traceback.print_exc()
finally:
# Close the database connection
await conn.close()
# Run the update
asyncio.run(update_click_logs())

View file

@ -0,0 +1,104 @@
{
"history_item_ids": [
"ncRYNd0Xef4LiUE74VjP",
"13pQLDAPYGIATwW1ySL5",
"dhsQNAYTWpcwo1X6rixf",
"V7wUip1NJuWAUw26sePF",
"mOYMa5lcI7wRHddIQTSa",
"mP97iOpA4oG7pwUBthq4",
"WTU5nsX6qZCYxLyoT5hq",
"15DPGnBgjr74KT3TMbK4",
"aCyBS1zoaweVjUoPf2TF",
"J8SUMQqZPtoy3Cgdhi3J",
"qKHaaJHfqh2je60Wmadb",
"2PaugQJ8c4rY44JGlaO5",
"TwzxcmYjo6XNebbMabcd",
"xdEK7rYq9UofOlkr565b",
"wik4jYd97aGMLgttTjC9",
"7oXn2yH7gdyhi6sEoWKd",
"jv8aZFiVe8gPMrAOBcNT",
"B2BctCDkCtLDxEMMBu9z",
"4KFO77NHDruNQvXIykwp",
"d033NizZaNZPc45fvxCO",
"yBKxOxfzsjpZYOFzoIM7",
"oEihKwMLWgvvoTLGx4yF",
"Q3guBm4hGml0KPAWKl7t",
"jaojY1gSafQmqshR48oT",
"yqGDMfcceaoceFEEurqa",
"oLdnyUp7plGrUMRVQ8Cf",
"FZAGCGosYEGMf8GCRFaA",
"TrWnXRdGkiH0K9kgwFiS",
"th16OEbg3u0XHslT9A33",
"856BAsn6dnzF7HeqGPfK",
"KjLoAfDXVBqR9s39T25j",
"uHQQJMMOfOxPAhEYQXLl",
"HO8WCIhkkI7AxwkU5MC6",
"9nxdesHWTRLCOd6YgWe9",
"tmx5tlIQ7hdSTgJt16P2",
"M9JN0YcBuCF6LhnqKN66",
"M9xkP4ecn0LIi7mQOfU6",
"CNtJgh52Ykh9ZqEppZeH",
"lgobcoiqmtWfbXkhEwbE",
"nr9jxnsE4DnwmTwCaHqC",
"Rnzo03tcyBqGPdmHemCb",
"X3YVGp7yf9GLgZ7WOuSU",
"wL3bkqxR9xqeFTvkJpSI",
"wNx3XDgFLTjVbMyGrIAO",
"rb0jj1ywBetmdvve5qIL",
"WdNnqvNswXeh6JFoaRSS",
"WT2ViyerKpodYmHDHhCw",
"OvhIRehXNwx7xMJHuTd7",
"EQb1iZtsADxJ0GxLJzEK",
"WXVfBJYoYGB7S61VyETD",
"q0q3Di1YJKF07dOhoa7E",
"a2XBIUPa68UiiKlzwFnG",
"YBuD7KsUpz8jxc5ItZcF",
"KdoucRVCVQGRVQ8Di9Ih",
"CkmDny98GEdfGuj2kaAx",
"R0R2p8luRZL7wwPtDilw",
"awvztgQnuaquK0dTpIuH",
"3ZPN0nJo8UQZYhFhoIOK",
"RJJeTkjYIgdv1ZoXXAax",
"ppxUNzWHAQafsM6OvEUE",
"f2VBm7yE7qmnjdS9CbYz",
"SZIMwz2T5ZAhTxTDBFol",
"YjC91PRgnQbAcdPhnWqU",
"fDTV7n8f6QK5yCwLkBwg",
"KbPpWUuiLPADj9H3OlvG",
"DIuqVoAg7lLxpvFBip84",
"pEwFAKMLGWUMHqfljJSq",
"9wwl7UbsgeKqrk8kNZin",
"2uLvjJgcZDiY9dqB8JlP",
"U5f1qZQM08t2YzJqEmxK",
"gnwn7QIhrCXRAGNddZ1H",
"g5nGEIHirFzKstdrGI1h",
"CQWH5dGSeS38VC4X4yg7",
"C5YGjhJPrTkVOpxIOHdj",
"YLbtnf1pSb9Ra7wgFHiF",
"qNLgNSvMr4VSoisKS9qj",
"Bq2ALvQVsj9L2wMpUvYO",
"gi0yTXLZLMhUKeKcalWc",
"3JQN9UbCsqj9ggi5sCkq",
"oPflJoA9kqBzjlmWY6zL",
"0kUZFgtZdqgdUBXFsXs9",
"aFTi7XdjR8W52ThmFpgc",
"pgIfjcy2UvKggfqJ1aNx",
"r0VguLaqnxTL9jza9H4y",
"444ehr4RtqgU1xjhhTLo",
"pEuzoznVDaQRBhIA9VTy",
"T9hdW9eJkEqDmOsSUoeY",
"wJjHbGzoWiKKOIGmf82T",
"kij4uMmkUlsSDu2zSH1k",
"oWt5rns196JsKIYPyrBS",
"SJ1m9mSOGOLIhkMgA8kq",
"kAaqe0ATrYtkifmZLOE5",
"O2Pvz7CP5rfyNvzFSDmy",
"w1rb8qN5nohVUovC0XAx",
"njFs4I4F7rtd9I6fEn6x",
"miFrp9GBm3MsHO03Z4eY",
"5DJywiPsfeVP9hFdqRhd",
"mUephoXhk5QdWrOfr9Xr",
"tDDiW3Yp0BptZ2wBv21A",
"YpX06liXWHquUVYFlKYa"
]
}

View file

@ -0,0 +1,63 @@
import asyncio
from pathlib import Path
from sijapi import L, EMAIL_CONFIG, EMAIL_LOGS
from sijapi.classes import EmailAccount
from sijapi.routers import email
async def initialize_log_files():
summarized_log = EMAIL_LOGS / "summarized.txt"
autoresponded_log = EMAIL_LOGS / "autoresponded.txt"
diagnostic_log = EMAIL_LOGS / "diagnostic.txt"
for log_file in [summarized_log, autoresponded_log, diagnostic_log]:
log_file.parent.mkdir(parents=True, exist_ok=True)
log_file.write_text("")
L.DEBUG(f"Log files initialized: {summarized_log}, {autoresponded_log}, {diagnostic_log}")
return summarized_log, autoresponded_log, diagnostic_log
async def process_all_emails(account: EmailAccount, summarized_log: Path, autoresponded_log: Path, diagnostic_log: Path):
try:
with email.get_imap_connection(account) as inbox:
L.DEBUG(f"Connected to {account.name}, processing all emails...")
all_messages = inbox.messages()
unread_messages = set(uid for uid, _ in inbox.messages(unread=True))
processed_count = 0
for identifier, message in all_messages:
# Log diagnostic information
with open(diagnostic_log, 'a') as f:
f.write(f"Account: {account.name}, Raw Identifier: {identifier}, Type: {type(identifier)}\n")
# Attempt to get a string representation of the identifier
if isinstance(identifier, bytes):
id_str = identifier.decode()
elif isinstance(identifier, (int, str)):
id_str = str(identifier)
else:
id_str = repr(identifier)
if identifier not in unread_messages:
processed_count += 1
for log_file in [summarized_log, autoresponded_log]:
with open(log_file, 'a') as f:
f.write(f"{id_str}\n")
L.INFO(f"Processed {processed_count} non-unread emails for account {account.name}")
except Exception as e:
L.logger.error(f"An error occurred while processing emails for account {account.name}: {e}")
async def main():
email_accounts = email.load_email_accounts(EMAIL_CONFIG)
summarized_log, autoresponded_log, diagnostic_log = await initialize_log_files()
L.DEBUG(f"Processing {len(email_accounts)} email accounts")
tasks = [process_all_emails(account, summarized_log, autoresponded_log, diagnostic_log) for account in email_accounts]
await asyncio.gather(*tasks)
# Final verification
with open(summarized_log, 'r') as f:
final_count = len(f.readlines())
L.INFO(f"Final non-unread email count: {final_count}")
if __name__ == "__main__":
asyncio.run(main())

View file

@ -0,0 +1,191 @@
import psycopg2
from psycopg2 import sql
import sys
def connect_to_db():
return psycopg2.connect(
dbname='sij',
user='sij',
password='Synchr0!',
host='localhost'
)
def get_tables(cur):
cur.execute("""
SELECT table_name
FROM information_schema.tables
WHERE table_schema = 'public' AND table_type = 'BASE TABLE'
AND table_name NOT LIKE '%_uuid' AND table_name NOT LIKE '%_orig'
AND table_name != 'spatial_ref_sys'
""")
return [row[0] for row in cur.fetchall()]
def get_columns(cur, table_name):
cur.execute("""
SELECT column_name, udt_name,
is_nullable, column_default,
character_maximum_length, numeric_precision, numeric_scale
FROM information_schema.columns
WHERE table_name = %s
ORDER BY ordinal_position
""", (table_name,))
return cur.fetchall()
def get_constraints(cur, table_name):
cur.execute("""
SELECT conname, contype, pg_get_constraintdef(c.oid)
FROM pg_constraint c
JOIN pg_namespace n ON n.oid = c.connamespace
WHERE conrelid = %s::regclass
AND n.nspname = 'public'
""", (table_name,))
return cur.fetchall()
def drop_table_if_exists(cur, table_name):
cur.execute(sql.SQL("DROP TABLE IF EXISTS {} CASCADE").format(sql.Identifier(table_name)))
def create_uuid_table(cur, old_table, new_table):
drop_table_if_exists(cur, new_table)
columns = get_columns(cur, old_table)
constraints = get_constraints(cur, old_table)
column_defs = []
has_id_column = any(col[0] == 'id' for col in columns)
for col in columns:
col_name, udt_name, is_nullable, default, max_length, precision, scale = col
if col_name == 'id' and has_id_column:
column_defs.append(sql.SQL("{} UUID PRIMARY KEY DEFAULT gen_random_uuid()").format(sql.Identifier(col_name)))
else:
type_sql = sql.SQL("{}").format(sql.Identifier(udt_name))
if max_length:
type_sql = sql.SQL("{}({})").format(type_sql, sql.Literal(max_length))
elif precision and scale:
type_sql = sql.SQL("{}({},{})").format(type_sql, sql.Literal(precision), sql.Literal(scale))
column_def = sql.SQL("{} {}").format(sql.Identifier(col_name), type_sql)
if is_nullable == 'NO':
column_def = sql.SQL("{} NOT NULL").format(column_def)
if default and 'nextval' not in default: # Skip auto-increment defaults
column_def = sql.SQL("{} DEFAULT {}").format(column_def, sql.SQL(default))
column_defs.append(column_def)
constraint_defs = []
for constraint in constraints:
conname, contype, condef = constraint
if contype != 'p' or not has_id_column: # Keep primary key if there's no id column
constraint_defs.append(sql.SQL(condef))
if not has_id_column:
column_defs.append(sql.SQL("uuid UUID DEFAULT gen_random_uuid()"))
query = sql.SQL("CREATE TABLE {} ({})").format(
sql.Identifier(new_table),
sql.SQL(", ").join(column_defs + constraint_defs)
)
cur.execute(query)
def migrate_data(cur, old_table, new_table):
columns = get_columns(cur, old_table)
column_names = [col[0] for col in columns]
has_id_column = 'id' in column_names
if has_id_column:
column_names.remove('id')
old_cols = sql.SQL(", ").join(map(sql.Identifier, column_names))
new_cols = sql.SQL(", ").join(map(sql.Identifier, ['id'] + column_names))
query = sql.SQL("INSERT INTO {} ({}) SELECT gen_random_uuid(), {} FROM {}").format(
sql.Identifier(new_table),
new_cols,
old_cols,
sql.Identifier(old_table)
)
else:
old_cols = sql.SQL(", ").join(map(sql.Identifier, column_names))
new_cols = sql.SQL(", ").join(map(sql.Identifier, column_names + ['uuid']))
query = sql.SQL("INSERT INTO {} ({}) SELECT {}, gen_random_uuid() FROM {}").format(
sql.Identifier(new_table),
new_cols,
old_cols,
sql.Identifier(old_table)
)
cur.execute(query)
def update_foreign_keys(cur, tables):
for table in tables:
constraints = get_constraints(cur, table)
for constraint in constraints:
conname, contype, condef = constraint
if contype == 'f': # Foreign key constraint
referenced_table = condef.split('REFERENCES ')[1].split('(')[0].strip()
referenced_column = condef.split('(')[2].split(')')[0].strip()
local_column = condef.split('(')[1].split(')')[0].strip()
cur.execute(sql.SQL("""
UPDATE {table_uuid}
SET {local_column} = subquery.new_id::text::{local_column_type}
FROM (
SELECT old.{ref_column} AS old_id, new_table.id AS new_id
FROM {ref_table} old
JOIN public.{ref_table_uuid} new_table ON new_table.{ref_column}::text = old.{ref_column}::text
) AS subquery
WHERE {local_column}::text = subquery.old_id::text
""").format(
table_uuid=sql.Identifier(f"{table}_uuid"),
local_column=sql.Identifier(local_column),
local_column_type=sql.SQL(get_column_type(cur, f"{table}_uuid", local_column)),
ref_column=sql.Identifier(referenced_column),
ref_table=sql.Identifier(referenced_table),
ref_table_uuid=sql.Identifier(f"{referenced_table}_uuid")
))
def get_column_type(cur, table_name, column_name):
cur.execute("""
SELECT data_type
FROM information_schema.columns
WHERE table_name = %s AND column_name = %s
""", (table_name, column_name))
return cur.fetchone()[0]
def rename_tables(cur, tables):
for table in tables:
drop_table_if_exists(cur, f"{table}_orig")
cur.execute(sql.SQL("ALTER TABLE IF EXISTS {} RENAME TO {}").format(
sql.Identifier(table), sql.Identifier(f"{table}_orig")
))
cur.execute(sql.SQL("ALTER TABLE IF EXISTS {} RENAME TO {}").format(
sql.Identifier(f"{table}_uuid"), sql.Identifier(table)
))
def main():
try:
with connect_to_db() as conn:
with conn.cursor() as cur:
tables = get_tables(cur)
# Create new UUID tables
for table in tables:
print(f"Creating UUID table for {table}...")
create_uuid_table(cur, table, f"{table}_uuid")
# Migrate data
for table in tables:
print(f"Migrating data for {table}...")
migrate_data(cur, table, f"{table}_uuid")
# Update foreign keys
print("Updating foreign key references...")
update_foreign_keys(cur, tables)
# Rename tables
print("Renaming tables...")
rename_tables(cur, tables)
conn.commit()
print("Migration completed successfully.")
except Exception as e:
print(f"An error occurred: {e}")
conn.rollback()
if __name__ == "__main__":
main()

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,211 @@
import yaml
import requests
import paramiko
import time
from pathlib import Path
import logging
import subprocess
import os
import argparse
import sys
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
def load_config():
config_path = Path(__file__).parent.parent / 'config' / 'sys.yaml'
with open(config_path, 'r') as file:
return yaml.safe_load(file)
def load_env():
env_path = Path(__file__).parent.parent / 'config' / '.env'
if env_path.exists():
with open(env_path, 'r') as file:
for line in file:
line = line.strip()
if line and not line.startswith('#'):
try:
key, value = line.split('=', 1)
os.environ[key.strip()] = value.strip()
except ValueError:
logging.warning(f"Skipping invalid line in .env file: {line}")
def check_server(ip, port, ts_id):
address = f"http://{ip}:{port}/id"
try:
response = requests.get(address, timeout=5)
response_text = response.text.strip().strip('"')
return response.status_code == 200 and response_text == ts_id
except requests.RequestException as e:
logging.error(f"Error checking server {ts_id}: {str(e)}")
return False
def execute_ssh_command(ssh, command):
stdin, stdout, stderr = ssh.exec_command(command)
exit_status = stdout.channel.recv_exit_status()
output = stdout.read().decode().strip()
error = stderr.read().decode().strip()
return exit_status, output, error
def is_local_tmux_session_running(session_name):
try:
result = subprocess.run(['tmux', 'has-session', '-t', session_name], capture_output=True, text=True)
return result.returncode == 0
except subprocess.CalledProcessError:
return False
def start_local_server(server, pull=False, push=False):
try:
if is_local_tmux_session_running('sijapi'):
logging.info("Local sijapi tmux session is already running.")
return
git_command = ""
if pull:
git_command = "git pull &&"
elif push:
git_command = "git add -A . && git commit -m \"auto-update\" && git push origin --force &&"
command = f"{server['tmux']} new-session -d -s sijapi 'cd {server['path']} && {git_command} {server['conda_env']}/bin/python -m sijapi'"
logging.info(f"Executing local command: {command}")
result = subprocess.run(command, shell=True, check=True, capture_output=True, text=True)
logging.info(f"Successfully started sijapi session on local machine")
logging.debug(f"Command output: {result.stdout}")
except subprocess.CalledProcessError as e:
logging.error(f"Failed to start sijapi session on local machine. Error: {e}")
logging.error(f"Error output: {e.stderr}")
def start_remote_server(server, pull=False, push=False):
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(
server['ts_ip'],
port=server['ssh_port'],
username=server['ssh_user'],
password=server['ssh_pass'],
timeout=10
)
status, output, error = execute_ssh_command(ssh, f"{server['tmux']} has-session -t sijapi 2>/dev/null && echo 'exists' || echo 'not exists'")
if output == 'exists':
logging.info(f"sijapi session already exists on {server['ts_id']}")
return
git_command = ""
if pull:
git_command = "git pull &&"
elif push:
git_command = "git add -A . && git commit -m \"auto-update\" && git push origin --force &&"
command = f"{server['tmux']} new-session -d -s sijapi 'cd {server['path']} && {git_command} {server['conda_env']}/bin/python -m sijapi'"
status, output, error = execute_ssh_command(ssh, command)
if status == 0:
logging.info(f"Successfully started sijapi session on {server['ts_id']}")
else:
logging.error(f"Failed to start sijapi session on {server['ts_id']}. Error: {error}")
except paramiko.SSHException as e:
logging.error(f"Failed to connect to {server['ts_id']}: {str(e)}")
finally:
ssh.close()
def kill_local_server():
try:
if is_local_tmux_session_running('sijapi'):
subprocess.run(['tmux', 'kill-session', '-t', 'sijapi'], check=True)
logging.info("Killed local sijapi tmux session.")
else:
logging.info("No local sijapi tmux session to kill.")
except subprocess.CalledProcessError as e:
logging.error(f"Failed to kill local sijapi tmux session. Error: {e}")
def kill_remote_server(server):
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(
server['ts_ip'],
port=server['ssh_port'],
username=server['ssh_user'],
password=server['ssh_pass'],
timeout=10
)
command = f"{server['tmux']} kill-session -t sijapi"
status, output, error = execute_ssh_command(ssh, command)
if status == 0:
logging.info(f"Successfully killed sijapi session on {server['ts_id']}")
else:
logging.error(f"Failed to kill sijapi session on {server['ts_id']}. Error: {error}")
except paramiko.SSHException as e:
logging.error(f"Failed to connect to {server['ts_id']}: {str(e)}")
finally:
ssh.close()
def main():
load_env()
config = load_config()
pool = config['POOL']
local_ts_id = os.environ.get('TS_ID')
parser = argparse.ArgumentParser(description='Manage sijapi servers')
parser.add_argument('--kill', action='store_true', help='Kill the local sijapi tmux session')
parser.add_argument('--restart', action='store_true', help='Restart the local sijapi tmux session')
parser.add_argument('--all', action='store_true', help='Apply the action to all servers')
parser.add_argument('--pull', action='store_true', help='Pull latest changes before starting the server')
parser.add_argument('--push', action='store_true', help='Push changes before starting the server')
args = parser.parse_args()
if args.kill:
if args.all:
for server in pool:
if server['ts_id'] == local_ts_id:
kill_local_server()
else:
kill_remote_server(server)
else:
kill_local_server()
sys.exit(0)
if args.restart or args.pull or args.push:
if args.all:
for server in pool:
if server['ts_id'] == local_ts_id:
kill_local_server()
start_local_server(server, pull=args.pull, push=args.push)
else:
kill_remote_server(server)
start_remote_server(server, pull=args.pull, push=args.push)
else:
kill_local_server()
local_server = next(server for server in pool if server['ts_id'] == local_ts_id)
start_local_server(local_server, pull=args.pull, push=args.push)
sys.exit(0)
# If no specific arguments, run the default behavior
local_server = next(server for server in pool if server['ts_id'] == local_ts_id)
if not check_server(local_server['ts_ip'], local_server['app_port'], local_server['ts_id']):
logging.info(f"Local server {local_server['ts_id']} is not responding correctly. Attempting to start...")
kill_local_server()
start_local_server(local_server, push=True)
for server in pool:
if server['ts_id'] != local_ts_id:
if not check_server(server['ts_ip'], server['app_port'], server['ts_id']):
logging.info(f"{server['ts_id']} is not responding correctly. Attempting to start...")
kill_remote_server(server)
start_remote_server(server, pull=True)
else:
logging.info(f"{server['ts_id']} is running and responding correctly.")
time.sleep(1)
if __name__ == "__main__":
main()

View file

@ -0,0 +1,34 @@
from aura_sr import AuraSR
from PIL import Image
import torch
import os
# Set environment variables for MPS
os.environ['PYTORCH_MPS_HIGH_WATERMARK_RATIO'] = '0.0'
os.environ['PYTORCH_ENABLE_MPS_FALLBACK'] = '1'
# Initialize device as CPU for default
device = torch.device('cpu')
# Check if MPS is available
if torch.backends.mps.is_available():
if not torch.backends.mps.is_built():
print("MPS not available because the current PyTorch install was not built with MPS enabled.")
else:
device = torch.device('mps:0')
# Overwrite the default CUDA device with MPS
torch.cuda.default_stream = device
aura_sr = AuraSR.from_pretrained("fal-ai/AuraSR").to(device)
def load_image_from_path(file_path):
return Image.open(file_path)
def upscale_and_save(original_path):
original_image = load_image_from_path(original_path)
upscaled_image = aura_sr.upscale_4x(original_image)
upscaled_image.save(original_path)
# Insert your image path
upscale_and_save("/Users/sij/workshop/sijapi/sijapi/testbed/API__00482_ 2.png")