Cleanup

sijapi/helpers/elevenlabs_history_ids_20240630_131617.json

@@ -1,104 +0,0 @@
-{
-  "history_item_ids": [
-    "ncRYNd0Xef4LiUE74VjP",
-    "13pQLDAPYGIATwW1ySL5",
-    "dhsQNAYTWpcwo1X6rixf",
-    "V7wUip1NJuWAUw26sePF",
-    "mOYMa5lcI7wRHddIQTSa",
-    "mP97iOpA4oG7pwUBthq4",
-    "WTU5nsX6qZCYxLyoT5hq",
-    "15DPGnBgjr74KT3TMbK4",
-    "aCyBS1zoaweVjUoPf2TF",
-    "J8SUMQqZPtoy3Cgdhi3J",
-    "qKHaaJHfqh2je60Wmadb",
-    "2PaugQJ8c4rY44JGlaO5",
-    "TwzxcmYjo6XNebbMabcd",
-    "xdEK7rYq9UofOlkr565b",
-    "wik4jYd97aGMLgttTjC9",
-    "7oXn2yH7gdyhi6sEoWKd",
-    "jv8aZFiVe8gPMrAOBcNT",
-    "B2BctCDkCtLDxEMMBu9z",
-    "4KFO77NHDruNQvXIykwp",
-    "d033NizZaNZPc45fvxCO",
-    "yBKxOxfzsjpZYOFzoIM7",
-    "oEihKwMLWgvvoTLGx4yF",
-    "Q3guBm4hGml0KPAWKl7t",
-    "jaojY1gSafQmqshR48oT",
-    "yqGDMfcceaoceFEEurqa",
-    "oLdnyUp7plGrUMRVQ8Cf",
-    "FZAGCGosYEGMf8GCRFaA",
-    "TrWnXRdGkiH0K9kgwFiS",
-    "th16OEbg3u0XHslT9A33",
-    "856BAsn6dnzF7HeqGPfK",
-    "KjLoAfDXVBqR9s39T25j",
-    "uHQQJMMOfOxPAhEYQXLl",
-    "HO8WCIhkkI7AxwkU5MC6",
-    "9nxdesHWTRLCOd6YgWe9",
-    "tmx5tlIQ7hdSTgJt16P2",
-    "M9JN0YcBuCF6LhnqKN66",
-    "M9xkP4ecn0LIi7mQOfU6",
-    "CNtJgh52Ykh9ZqEppZeH",
-    "lgobcoiqmtWfbXkhEwbE",
-    "nr9jxnsE4DnwmTwCaHqC",
-    "Rnzo03tcyBqGPdmHemCb",
-    "X3YVGp7yf9GLgZ7WOuSU",
-    "wL3bkqxR9xqeFTvkJpSI",
-    "wNx3XDgFLTjVbMyGrIAO",
-    "rb0jj1ywBetmdvve5qIL",
-    "WdNnqvNswXeh6JFoaRSS",
-    "WT2ViyerKpodYmHDHhCw",
-    "OvhIRehXNwx7xMJHuTd7",
-    "EQb1iZtsADxJ0GxLJzEK",
-    "WXVfBJYoYGB7S61VyETD",
-    "q0q3Di1YJKF07dOhoa7E",
-    "a2XBIUPa68UiiKlzwFnG",
-    "YBuD7KsUpz8jxc5ItZcF",
-    "KdoucRVCVQGRVQ8Di9Ih",
-    "CkmDny98GEdfGuj2kaAx",
-    "R0R2p8luRZL7wwPtDilw",
-    "awvztgQnuaquK0dTpIuH",
-    "3ZPN0nJo8UQZYhFhoIOK",
-    "RJJeTkjYIgdv1ZoXXAax",
-    "ppxUNzWHAQafsM6OvEUE",
-    "f2VBm7yE7qmnjdS9CbYz",
-    "SZIMwz2T5ZAhTxTDBFol",
-    "YjC91PRgnQbAcdPhnWqU",
-    "fDTV7n8f6QK5yCwLkBwg",
-    "KbPpWUuiLPADj9H3OlvG",
-    "DIuqVoAg7lLxpvFBip84",
-    "pEwFAKMLGWUMHqfljJSq",
-    "9wwl7UbsgeKqrk8kNZin",
-    "2uLvjJgcZDiY9dqB8JlP",
-    "U5f1qZQM08t2YzJqEmxK",
-    "gnwn7QIhrCXRAGNddZ1H",
-    "g5nGEIHirFzKstdrGI1h",
-    "CQWH5dGSeS38VC4X4yg7",
-    "C5YGjhJPrTkVOpxIOHdj",
-    "YLbtnf1pSb9Ra7wgFHiF",
-    "qNLgNSvMr4VSoisKS9qj",
-    "Bq2ALvQVsj9L2wMpUvYO",
-    "gi0yTXLZLMhUKeKcalWc",
-    "3JQN9UbCsqj9ggi5sCkq",
-    "oPflJoA9kqBzjlmWY6zL",
-    "0kUZFgtZdqgdUBXFsXs9",
-    "aFTi7XdjR8W52ThmFpgc",
-    "pgIfjcy2UvKggfqJ1aNx",
-    "r0VguLaqnxTL9jza9H4y",
-    "444ehr4RtqgU1xjhhTLo",
-    "pEuzoznVDaQRBhIA9VTy",
-    "T9hdW9eJkEqDmOsSUoeY",
-    "wJjHbGzoWiKKOIGmf82T",
-    "kij4uMmkUlsSDu2zSH1k",
-    "oWt5rns196JsKIYPyrBS",
-    "SJ1m9mSOGOLIhkMgA8kq",
-    "kAaqe0ATrYtkifmZLOE5",
-    "O2Pvz7CP5rfyNvzFSDmy",
-    "w1rb8qN5nohVUovC0XAx",
-    "njFs4I4F7rtd9I6fEn6x",
-    "miFrp9GBm3MsHO03Z4eY",
-    "5DJywiPsfeVP9hFdqRhd",
-    "mUephoXhk5QdWrOfr9Xr",
-    "tDDiW3Yp0BptZ2wBv21A",
-    "YpX06liXWHquUVYFlKYa"
-  ]
-}

sijapi/helpers/fromvm/db_connection_test.py

@@ -1,66 +0,0 @@
-import asyncio
-import asyncpg
-import psycopg2
-import sys
-
-async def try_async_connect(host, port, user, password, database):
-    try:
-        conn = await asyncpg.connect(
-            host=host,
-            port=port,
-            user=user,
-            password=password,
-            database=database
-        )
-        version = await conn.fetchval('SELECT version()')
-        print(f"Async connection successful to {host}:{port}")
-        print(f"PostgreSQL version: {version}")
-        await conn.close()
-        return True
-    except Exception as e:
-        print(f"Async connection failed to {host}:{port}")
-        print(f"Error: {str(e)}")
-        return False
-
-def try_sync_connect(host, port, user, password, database):
-    try:
-        conn = psycopg2.connect(
-            host=host,
-            port=port,
-            user=user,
-            password=password,
-            database=database
-        )
-        cur = conn.cursor()
-        cur.execute('SELECT version()')
-        version = cur.fetchone()[0]
-        print(f"Sync connection successful to {host}:{port}")
-        print(f"PostgreSQL version: {version}")
-        conn.close()
-        return True
-    except Exception as e:
-        print(f"Sync connection failed to {host}:{port}")
-        print(f"Error: {str(e)}")
-        return False
-
-async def main():
-    # Database connection parameters
-    port = 5432
-    user = 'sij'
-    password = 'Synchr0!'
-    database = 'sij'
-
-    hosts = ['100.64.64.20', '127.0.0.1', 'localhost']
-
-    print("Attempting asynchronous connections:")
-    for host in hosts:
-        await try_async_connect(host, port, user, password, database)
-        print()
-
-    print("Attempting synchronous connections:")
-    for host in hosts:
-        try_sync_connect(host, port, user, password, database)
-        print()
-
-if __name__ == "__main__":
-    asyncio.run(main())

sijapi/helpers/upscaler.py

@@ -1,34 +0,0 @@
-from aura_sr import AuraSR
-from PIL import Image
-import torch
-import os
-
-# Set environment variables for MPS
-os.environ['PYTORCH_MPS_HIGH_WATERMARK_RATIO'] = '0.0'
-os.environ['PYTORCH_ENABLE_MPS_FALLBACK'] = '1'
-
-# Initialize device as CPU for default
-device = torch.device('cpu')
-
-# Check if MPS is available
-if torch.backends.mps.is_available():
-    if not torch.backends.mps.is_built():
-        print("MPS not available because the current PyTorch install was not built with MPS enabled.")
-    else:
-        device = torch.device('mps:0')
-
-# Overwrite the default CUDA device with MPS
-torch.cuda.default_stream = device
-
-aura_sr = AuraSR.from_pretrained("fal-ai/AuraSR").to(device)
-
-def load_image_from_path(file_path):
-    return Image.open(file_path)
-
-def upscale_and_save(original_path):
-    original_image = load_image_from_path(original_path)
-    upscaled_image = aura_sr.upscale_4x(original_image)
-    upscaled_image.save(original_path)
-
-# Insert your image path
-upscale_and_save("/Users/sij/workshop/sijapi/sijapi/testbed/API__00482_ 2.png")

sijapi/routers/news.py

@@ -195,24 +195,56 @@ async def process_and_save_article(
         raise HTTPException(status_code=500, detail=str(e))
 
 
+
+
+from newspaper import Article as NewspaperArticle
+
 async def fetch_and_parse_article(url: str) -> Article:
+    # Try trafilatura first
     source = trafilatura.fetch_url(url)
-    traf = trafilatura.extract_metadata(filecontent=source, default_url=url)
-
-    article = Article(url)
-    article.set_html(source)
-    article.parse()
-
-    # Update article properties with trafilatura data
-    article.title = article.title or traf.title or url
-    article.authors = article.authors or (traf.author if isinstance(traf.author, list) else [traf.author])
-    article.publish_date = await gis.dt(article.publish_date or traf.date or dt_datetime.now(), "UTC")
-    article.text = trafilatura.extract(source, output_format="markdown", include_comments=False) or article.text
-    article.top_image = article.top_image or traf.image
-    article.source_url = traf.sitename or urlparse(url).netloc.replace('www.', '').title()
-    article.meta_keywords = list(set(article.meta_keywords or traf.categories or traf.tags or []))
-
-    return article
+    
+    if source:
+        try:
+            traf = trafilatura.extract_metadata(filecontent=source, default_url=url)
+            
+            article = Article(url)
+            article.set_html(source)
+            article.parse()
+            
+            # Update article properties with trafilatura data
+            article.title = article.title or traf.title or url
+            article.authors = article.authors or (traf.author if isinstance(traf.author, list) else [traf.author])
+            article.publish_date = await gis.dt(article.publish_date or traf.date or dt_datetime.now(), "UTC")
+            article.text = trafilatura.extract(source, output_format="markdown", include_comments=False) or article.text
+            article.top_image = article.top_image or traf.image
+            article.source_url = traf.sitename or urlparse(url).netloc.replace('www.', '').title()
+            article.meta_keywords = list(set(article.meta_keywords or traf.categories or traf.tags or []))
+            
+            return article
+        except Exception as e:
+            l.warning(f"Trafilatura failed to parse {url}: {str(e)}. Falling back to newspaper3k.")
+    else:
+        l.warning(f"Trafilatura failed to fetch {url}. Falling back to newspaper3k.")
+    
+    # If trafilatura fails, use newspaper3k
+    try:
+        newspaper_article = NewspaperArticle(url)
+        newspaper_article.download()
+        newspaper_article.parse()
+        
+        article = Article(url)
+        article.title = newspaper_article.title
+        article.authors = newspaper_article.authors
+        article.publish_date = await gis.dt(newspaper_article.publish_date or dt_datetime.now(), "UTC")
+        article.text = newspaper_article.text
+        article.top_image = newspaper_article.top_image
+        article.source_url = urlparse(url).netloc.replace('www.', '').title()
+        article.meta_keywords = newspaper_article.keywords
+        
+        return article
+    except Exception as e:
+        l.error(f"Both trafilatura and newspaper3k failed to fetch and parse {url}: {str(e)}")
+        raise HTTPException(status_code=500, detail="Failed to fetch and parse article content")
 
 
 def generate_markdown_content(article, title: str, summary: str, audio_link: Optional[str], site_name: Optional[str] = None) -> str:
@@ -258,4 +290,4 @@ tags:
     body += f"> [!summary]+\n> {summary}\n\n"
     body += article.text
 
-    return frontmatter + body
+    return frontmatter + body