Auto-update: Tue Jul 30 12:12:04 PDT 2024
This commit is contained in:
sanj
parent
2f1384bc23
commit
c4be5cbd7c
1 changed files with 85 additions and 69 deletions
|
|
@ -332,45 +332,56 @@ class APIConfig(BaseModel):
|
||||||
if pool_entry is None:
|
if pool_entry is None:
|
||||||
pool_entry = self.local_db
|
pool_entry = self.local_db
|
||||||
|
|
||||||
info(f"Attempting to connect to database: {pool_entry}")
|
for attempt in range(3): # Retry up to 3 times
|
||||||
try:
|
|
||||||
conn = await self.db_pool.get_connection(pool_entry)
|
|
||||||
try:
|
try:
|
||||||
yield conn
|
conn = await asyncpg.connect(
|
||||||
finally:
|
host=pool_entry['ts_ip'],
|
||||||
await self.db_pool.release_connection(pool_entry, conn)
|
port=pool_entry['db_port'],
|
||||||
except asyncpg.exceptions.ConnectionDoesNotExistError:
|
user=pool_entry['db_user'],
|
||||||
err(f"Connection to database {pool_entry['ts_ip']}:{pool_entry['db_port']} does not exist or has been closed")
|
password=pool_entry['db_pass'],
|
||||||
raise
|
database=pool_entry['db_name']
|
||||||
except asyncpg.exceptions.ConnectionFailureError as e:
|
)
|
||||||
err(f"Failed to connect to database: {pool_entry['ts_ip']}:{pool_entry['db_port']}")
|
try:
|
||||||
err(f"Connection error: {str(e)}")
|
yield conn
|
||||||
raise
|
finally:
|
||||||
except Exception as e:
|
await conn.close()
|
||||||
err(f"Unexpected error connecting to database: {pool_entry['ts_ip']}:{pool_entry['db_port']}")
|
return
|
||||||
err(f"Error: {str(e)}")
|
except asyncpg.exceptions.CannotConnectNowError:
|
||||||
raise
|
if attempt < 2: # Don't sleep on the last attempt
|
||||||
|
await asyncio.sleep(1) # Wait before retrying
|
||||||
|
except Exception as e:
|
||||||
|
err(f"Failed to connect to database: {pool_entry['ts_ip']}:{pool_entry['db_port']}")
|
||||||
|
err(f"Error: {str(e)}")
|
||||||
|
if attempt == 2: # Raise the exception on the last attempt
|
||||||
|
raise
|
||||||
|
|
||||||
|
raise Exception(f"Failed to connect to database after 3 attempts: {pool_entry['ts_ip']}:{pool_entry['db_port']}")
|
||||||
|
|
||||||
async def initialize_sync(self):
|
async def initialize_sync(self):
|
||||||
for pool_entry in self.POOL:
|
for pool_entry in self.POOL:
|
||||||
try:
|
for attempt in range(3): # Retry up to 3 times
|
||||||
async with self.get_connection(pool_entry) as conn:
|
try:
|
||||||
tables = await conn.fetch("""
|
async with self.get_connection(pool_entry) as conn:
|
||||||
SELECT tablename FROM pg_tables
|
tables = await conn.fetch("""
|
||||||
WHERE schemaname = 'public'
|
SELECT tablename FROM pg_tables
|
||||||
""")
|
WHERE schemaname = 'public'
|
||||||
|
""")
|
||||||
for table in tables:
|
|
||||||
table_name = table['tablename']
|
for table in tables:
|
||||||
await self.ensure_sync_columns(conn, table_name)
|
table_name = table['tablename']
|
||||||
await self.create_sync_trigger(conn, table_name)
|
await self.ensure_sync_columns(conn, table_name)
|
||||||
|
await self.create_sync_trigger(conn, table_name)
|
||||||
|
|
||||||
info(f"Sync initialization complete for {pool_entry['ts_ip']}. All tables now have version and server_id columns with appropriate triggers.")
|
info(f"Sync initialization complete for {pool_entry['ts_ip']}. All tables now have version and server_id columns with appropriate triggers.")
|
||||||
except asyncpg.exceptions.ConnectionFailureError:
|
break # If successful, break the retry loop
|
||||||
err(f"Failed to connect to database during initialization: {pool_entry['ts_ip']}:{pool_entry['db_port']}")
|
except asyncpg.exceptions.ConnectionFailureError:
|
||||||
except Exception as e:
|
err(f"Failed to connect to database during initialization: {pool_entry['ts_ip']}:{pool_entry['db_port']}")
|
||||||
err(f"Error initializing sync for {pool_entry['ts_ip']}: {str(e)}")
|
if attempt < 2: # Don't sleep on the last attempt
|
||||||
err(f"Traceback: {traceback.format_exc()}")
|
await asyncio.sleep(1) # Wait before retrying
|
||||||
|
except Exception as e:
|
||||||
|
err(f"Error initializing sync for {pool_entry['ts_ip']}: {str(e)}")
|
||||||
|
err(f"Traceback: {traceback.format_exc()}")
|
||||||
|
break # Don't retry for unexpected errors
|
||||||
|
|
||||||
async def ensure_sync_columns(self, conn, table_name):
|
async def ensure_sync_columns(self, conn, table_name):
|
||||||
try:
|
try:
|
||||||
|
|
@ -383,7 +394,7 @@ class APIConfig(BaseModel):
|
||||||
ADD COLUMN IF NOT EXISTS server_id TEXT DEFAULT '{os.environ.get('TS_ID')}';
|
ADD COLUMN IF NOT EXISTS server_id TEXT DEFAULT '{os.environ.get('TS_ID')}';
|
||||||
EXCEPTION
|
EXCEPTION
|
||||||
WHEN duplicate_column THEN
|
WHEN duplicate_column THEN
|
||||||
RAISE NOTICE 'column version or server_id already exists in {table_name}.';
|
NULL; -- Silently handle duplicate column
|
||||||
END;
|
END;
|
||||||
END $$;
|
END $$;
|
||||||
""")
|
""")
|
||||||
|
|
@ -392,6 +403,7 @@ class APIConfig(BaseModel):
|
||||||
err(f"Error ensuring sync columns for table {table_name}: {str(e)}")
|
err(f"Error ensuring sync columns for table {table_name}: {str(e)}")
|
||||||
err(f"Traceback: {traceback.format_exc()}")
|
err(f"Traceback: {traceback.format_exc()}")
|
||||||
|
|
||||||
|
|
||||||
async def create_sync_trigger(self, conn, table_name):
|
async def create_sync_trigger(self, conn, table_name):
|
||||||
await conn.execute(f"""
|
await conn.execute(f"""
|
||||||
CREATE OR REPLACE FUNCTION update_version_and_server_id()
|
CREATE OR REPLACE FUNCTION update_version_and_server_id()
|
||||||
|
|
@ -421,40 +433,44 @@ class APIConfig(BaseModel):
|
||||||
if pool_entry['ts_id'] == os.environ.get('TS_ID'):
|
if pool_entry['ts_id'] == os.environ.get('TS_ID'):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
try:
|
for _ in range(3): # Retry up to 3 times
|
||||||
async with self.get_connection(pool_entry) as conn:
|
try:
|
||||||
# Check if the version column exists in any table
|
async with self.get_connection(pool_entry) as conn:
|
||||||
version_exists = await conn.fetchval("""
|
# Check if the version column exists in any table
|
||||||
SELECT EXISTS (
|
version_exists = await conn.fetchval("""
|
||||||
SELECT 1
|
SELECT EXISTS (
|
||||||
FROM information_schema.columns
|
SELECT 1
|
||||||
WHERE table_schema = 'public'
|
FROM information_schema.columns
|
||||||
AND column_name = 'version'
|
WHERE table_schema = 'public'
|
||||||
)
|
AND column_name = 'version'
|
||||||
""")
|
)
|
||||||
|
""")
|
||||||
if not version_exists:
|
|
||||||
info(f"Version column does not exist in any table for {pool_entry['ts_id']}")
|
if not version_exists:
|
||||||
continue
|
info(f"Version column does not exist in any table for {pool_entry['ts_id']}")
|
||||||
|
break # Move to the next pool entry
|
||||||
version = await conn.fetchval("""
|
|
||||||
SELECT COALESCE(MAX(version), -1)
|
version = await conn.fetchval("""
|
||||||
FROM (
|
SELECT COALESCE(MAX(version), -1)
|
||||||
SELECT MAX(version) as version
|
FROM (
|
||||||
FROM information_schema.columns
|
SELECT MAX(version) as version
|
||||||
WHERE table_schema = 'public'
|
FROM information_schema.columns
|
||||||
AND column_name = 'version'
|
WHERE table_schema = 'public'
|
||||||
AND is_updatable = 'YES'
|
AND column_name = 'version'
|
||||||
) as subquery
|
AND is_updatable = 'YES'
|
||||||
""")
|
) as subquery
|
||||||
info(f"Max version for {pool_entry['ts_id']}: {version}")
|
""")
|
||||||
if version > max_version:
|
info(f"Max version for {pool_entry['ts_id']}: {version}")
|
||||||
max_version = version
|
if version > max_version:
|
||||||
most_recent_source = pool_entry
|
max_version = version
|
||||||
except asyncpg.exceptions.ConnectionFailureError:
|
most_recent_source = pool_entry
|
||||||
err(f"Failed to connect to database: {pool_entry['ts_ip']}:{pool_entry['db_port']}")
|
break # If successful, break the retry loop
|
||||||
except Exception as e:
|
except asyncpg.exceptions.PostgresError as e:
|
||||||
err(f"Error checking version for {pool_entry['ts_id']}: {str(e)}")
|
err(f"Error checking version for {pool_entry['ts_id']}: {str(e)}")
|
||||||
|
await asyncio.sleep(1) # Wait before retrying
|
||||||
|
except Exception as e:
|
||||||
|
err(f"Unexpected error for {pool_entry['ts_id']}: {str(e)}")
|
||||||
|
break # Don't retry for unexpected errors
|
||||||
|
|
||||||
if most_recent_source:
|
if most_recent_source:
|
||||||
info(f"Most recent source: {most_recent_source['ts_id']} with version {max_version}")
|
info(f"Most recent source: {most_recent_source['ts_id']} with version {max_version}")
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue