Auto-update: Tue Jul 30 15:31:26 PDT 2024

sijapi/classes.py

@@ -300,38 +300,37 @@ class APIConfig(BaseModel):
         if pool_entry is None:
             pool_entry = self.local_db
         
-        pool_key = f"{pool_entry['ts_ip']}:{pool_entry['db_port']}"
-        
-        if pool_key not in self._db_pools:
-            try:
-                self._db_pools[pool_key] = await asyncpg.create_pool(
-                    host=pool_entry['ts_ip'],
-                    port=pool_entry['db_port'],
-                    user=pool_entry['db_user'],
-                    password=pool_entry['db_pass'],
-                    database=pool_entry['db_name'],
-                    min_size=1,
-                    max_size=10,  # adjust as needed
-                    timeout=5  # connection timeout in seconds
-                )
-            except Exception as e:
-                err(f"Failed to create connection pool for {pool_key}: {str(e)}")
-                raise
-
+        info(f"Attempting to connect to database: {pool_entry['ts_ip']}:{pool_entry['db_port']}")
         try:
-            async with self._db_pools[pool_key].acquire() as conn:
+            conn = await asyncpg.connect(
+                host=pool_entry['ts_ip'],
+                port=pool_entry['db_port'],
+                user=pool_entry['db_user'],
+                password=pool_entry['db_pass'],
+                database=pool_entry['db_name'],
+                timeout=5  # Add a timeout to prevent hanging
+            )
+            info(f"Successfully connected to {pool_entry['ts_ip']}:{pool_entry['db_port']}")
+            try:
                 yield conn
+            finally:
+                await conn.close()
+                info(f"Closed connection to {pool_entry['ts_ip']}:{pool_entry['db_port']}")
         except asyncpg.exceptions.ConnectionDoesNotExistError:
-            err(f"Failed to acquire connection from pool for {pool_key}: Connection does not exist")
+            err(f"Failed to connect to database: {pool_entry['ts_ip']}:{pool_entry['db_port']} - Connection does not exist")
             raise
         except asyncpg.exceptions.ConnectionFailureError:
-            err(f"Failed to acquire connection from pool for {pool_key}: Connection failure")
+            err(f"Failed to connect to database: {pool_entry['ts_ip']}:{pool_entry['db_port']} - Connection failure")
+            raise
+        except asyncpg.exceptions.PostgresError as e:
+            err(f"PostgreSQL error when connecting to {pool_entry['ts_ip']}:{pool_entry['db_port']}: {str(e)}")
             raise
         except Exception as e:
-            err(f"Unexpected error when acquiring connection from pool for {pool_key}: {str(e)}")
+            err(f"Unexpected error when connecting to {pool_entry['ts_ip']}:{pool_entry['db_port']}: {str(e)}")
             raise
 
 
+
     async def close_db_pools(self):
         info("Closing database connection pools...")
         for pool_key, pool in self._db_pools.items():
@@ -603,9 +602,9 @@ class APIConfig(BaseModel):
                     
                     for table in tables:
                         table_name = table['tablename']
-                        last_synced_version = await self.get_last_synced_version(dest_conn, table_name, source_id)
-                        
-                        while True:
+                        try:
+                            last_synced_version = await self.get_last_synced_version(dest_conn, table_name, source_id)
+                            
                             changes = await source_conn.fetch(f"""
                                 SELECT * FROM "{table_name}"
                                 WHERE version > $1 AND server_id = $2
@@ -613,16 +612,22 @@ class APIConfig(BaseModel):
                                 LIMIT $3
                             """, last_synced_version, source_id, batch_size)
                             
-                            if not changes:
-                                break
+                            if changes:
+                                changes_count = await self.apply_batch_changes(dest_conn, table_name, changes)
+                                total_changes += changes_count
+                                
+                                if changes_count > 0:
+                                    last_synced_version = changes[-1]['version']
+                                    await self.update_sync_status(dest_conn, table_name, source_id, last_synced_version)
+                                
+                                info(f"Synced batch for {table_name}: {changes_count} changes. Total so far: {total_changes}")
+                            else:
+                                info(f"No changes to sync for {table_name}")
 
-                            changes_count = await self.apply_batch_changes(dest_conn, table_name, changes)
-                            total_changes += changes_count
-                            
-                            last_synced_version = changes[-1]['version']
-                            await self.update_sync_status(dest_conn, table_name, source_id, last_synced_version)
-                            
-                            info(f"Synced batch for {table_name}: {changes_count} changes. Total so far: {total_changes}")
+                        except Exception as e:
+                            err(f"Error syncing table {table_name}: {str(e)}")
+                            err(f"Traceback: {traceback.format_exc()}")
+                            # Continue with the next table
 
             info(f"Sync complete from {source_id} ({source_ip}) to {dest_id} ({dest_ip}). Total changes: {total_changes}")
 
@@ -633,48 +638,35 @@ class APIConfig(BaseModel):
         return total_changes
 
 
+
     async def apply_batch_changes(self, conn, table_name, changes):
         if not changes:
             return 0
 
-        temp_table_name = f"temp_{table_name}_{uuid.uuid4().hex[:8]}"
-        
         try:
-            # Create temporary table
-            await conn.execute(f"""
-                CREATE TEMPORARY TABLE {temp_table_name} (LIKE "{table_name}" INCLUDING ALL)
-                ON COMMIT DROP
-            """)
-
-            # Bulk insert changes into temporary table
-            columns = changes[0].keys()
-            records = [tuple(change[col] for col in columns) for change in changes]
-            
-            # Use copy_records_to_table instead of copy_records
-            await conn.copy_records_to_table(temp_table_name, records=records)
-
-            # Perform upsert
-            result = await conn.execute(f"""
-                INSERT INTO "{table_name}" 
-                SELECT * FROM {temp_table_name}
+            # Convert the keys to a list
+            columns = list(changes[0].keys())
+            placeholders = [f'${i+1}' for i in range(len(columns))]
+            insert_query = f"""
+                INSERT INTO "{table_name}" ({', '.join(columns)})
+                VALUES ({', '.join(placeholders)})
                 ON CONFLICT (id) DO UPDATE SET
                 {', '.join(f"{col} = EXCLUDED.{col}" for col in columns if col != 'id')}
-            """)
+            """
+
+            # Execute the insert for each change
+            affected_rows = 0
+            for change in changes:
+                values = [change[col] for col in columns]
+                result = await conn.execute(insert_query, *values)
+                affected_rows += int(result.split()[-1])
 
-            # Parse the result to get the number of affected rows
-            affected_rows = int(result.split()[-1])
             return affected_rows
 
         except Exception as e:
             err(f"Error applying batch changes to {table_name}: {str(e)}")
             err(f"Traceback: {traceback.format_exc()}")
             return 0
-        finally:
-            # Ensure temporary table is dropped
-            try:
-                await conn.execute(f"DROP TABLE IF EXISTS {temp_table_name}")
-            except Exception as e:
-                warn(f"Failed to drop temporary table {temp_table_name}: {str(e)}")
 
 
 
@@ -727,6 +719,7 @@ class APIConfig(BaseModel):
             err(f"Error pushing changes to {pool_entry['ts_id']}: {str(e)}")
             err(f"Traceback: {traceback.format_exc()}")
 
+
     async def update_sync_status(self, conn, table_name, server_id, version):
         await conn.execute("""
             INSERT INTO sync_status (table_name, server_id, last_synced_version, last_sync_time)
@@ -736,6 +729,7 @@ class APIConfig(BaseModel):
                 last_sync_time = EXCLUDED.last_sync_time
         """, table_name, server_id, version)
 
+
     async def get_last_synced_version(self, conn, table_name, server_id):
         return await conn.fetchval(f"""
             SELECT COALESCE(MAX(version), 0)