sij/sijapi
1
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Auto-update: Sat Jun 29 21:33:48 PDT 2024

Browse source
This commit is contained in:
sanj 2024-06-29 21:33:48 -07:00
parent ad0ae30575
commit e4d6ec6412
4 changed files with 147 additions and 139 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
sijapi/classes.py
View file

@ -8,7 +8,7 @@ from contextlib import asynccontextmanager
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from pathlib import Path from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, Union, TypeVar, Type from typing import Any, Dict, List, Optional, Tuple, Union, TypeVar, Type
from zoneinfo import ZoneInfo
import aiofiles import aiofiles
import aiohttp import aiohttp
import asyncpg import asyncpg
@ -311,11 +311,11 @@ class Geocoder:
else: else:
raise ValueError(f"Unsupported unit: {unit}") raise ValueError(f"Unsupported unit: {unit}")
async def timezone(self, lat: float, lon: float) -> Optional[ZoneInfo]:
async def timezone(self, lat: float, lon: float):
loop = asyncio.get_running_loop() loop = asyncio.get_running_loop()
timezone = await loop.run_in_executor(self.executor, lambda: self.tf.timezone_at(lat=lat, lng=lon)) timezone_str = await loop.run_in_executor(self.executor, lambda: self.tf.timezone_at(lat=lat, lng=lon))
return timezone if timezone else 'Unknown' return ZoneInfo(timezone_str) if timezone_str else None
async def lookup(self, lat: float, lon: float): async def lookup(self, lat: float, lon: float):
@ -443,7 +443,7 @@ class Geocoder:
def coords_equal(self, coord1: Tuple[float, float], coord2: Tuple[float, float], tolerance: float = 1e-5) -> bool: def coords_equal(self, coord1: Tuple[float, float], coord2: Tuple[float, float], tolerance: float = 1e-5) -> bool:
return math.isclose(coord1[0], coord2[0], abs_tol=tolerance) and math.isclose(coord1[1], coord2[1], abs_tol=tolerance) return math.isclose(coord1[0], coord2[0], abs_tol=tolerance) and math.isclose(coord1[1], coord2[1], abs_tol=tolerance)
async def refresh_timezone(self, location: Union[Location, Tuple[float, float]], force: bool = False) -> str: async def refresh_timezone(self, location: Union[Location, Tuple[float, float]], force: bool = False) -> Optional[ZoneInfo]:
if isinstance(location, Location): if isinstance(location, Location):
lat, lon = location.latitude, location.longitude lat, lon = location.latitude, location.longitude
else: else:
@ -457,6 +457,7 @@ class Geocoder:
current_time - self.last_update > timedelta(hours=1) or current_time - self.last_update > timedelta(hours=1) or
not self.coords_equal(rounded_location, self.round_coords(*self.last_location) if self.last_location else (None, None))): not self.coords_equal(rounded_location, self.round_coords(*self.last_location) if self.last_location else (None, None))):
new_timezone = await self.timezone(lat, lon) new_timezone = await self.timezone(lat, lon)
self.last_timezone = new_timezone self.last_timezone = new_timezone
self.last_update = current_time self.last_update = current_time
@ -465,9 +466,10 @@ class Geocoder:
return self.last_timezone return self.last_timezone
async def tz_save(self): async def tz_save(self):
cache_data = { cache_data = {
'last_timezone': self.last_timezone, 'last_timezone': str(self.last_timezone) if self.last_timezone else None,
'last_update': self.last_update.isoformat() if self.last_update else None, 'last_update': self.last_update.isoformat() if self.last_update else None,
'last_location': self.last_location 'last_location': self.last_location
} }
@ -478,29 +480,31 @@ class Geocoder:
try: try:
async with aiofiles.open(self.cache_file, 'r') as f: async with aiofiles.open(self.cache_file, 'r') as f:
cache_data = json.loads(await f.read()) cache_data = json.loads(await f.read())
self.last_timezone = cache_data.get('last_timezone') self.last_timezone = ZoneInfo(cache_data['last_timezone']) if cache_data.get('last_timezone') else None
self.last_update = datetime.fromisoformat(cache_data['last_update']) if cache_data.get('last_update') else None self.last_update = datetime.fromisoformat(cache_data['last_update']) if cache_data.get('last_update') else None
self.last_location = tuple(cache_data['last_location']) if cache_data.get('last_location') else None self.last_location = tuple(cache_data['last_location']) if cache_data.get('last_location') else None
except (FileNotFoundError, json.JSONDecodeError): except (FileNotFoundError, json.JSONDecodeError):
# If file doesn't exist or is invalid, we'll start fresh # If file doesn't exist or is invalid, we'll start fresh
pass self.last_timezone = None
self.last_update = None
self.last_location = None
async def tz_current(self, location: Union[Location, Tuple[float, float]]) -> str: async def tz_current(self, location: Union[Location, Tuple[float, float]]) -> Optional[ZoneInfo]:
await self.tz_cached() await self.tz_cached()
return await self.refresh_timezone(location) return await self.refresh_timezone(location)
async def tz_last(self) -> Optional[str]: async def tz_last(self) -> Optional[ZoneInfo]:
await self.tz_cached() await self.tz_cached()
return self.last_timezone return self.last_timezone
async def tz_at(self, lat: float, lon: float) -> Optional[ZoneInfo]:
async def tz_at(self, lat: float, lon: float) -> str:
""" """
Get the timezone at a specific latitude and longitude without affecting the cache. Get the timezone at a specific latitude and longitude without affecting the cache.
:param lat: Latitude :param lat: Latitude
:param lon: Longitude :param lon: Longitude
:return: Timezone string :return: ZoneInfo object representing the timezone
""" """
return await self.timezone(lat, lon) return await self.timezone(lat, lon)

56
sijapi/routers/loc.py
View file

@ -19,63 +19,50 @@ from sijapi.utilities import haversine
loc = APIRouter() loc = APIRouter()
async def dt( async def dt(
date_time: Union[str, datetime], date_time: Union[str, int, datetime],
tz: Union[str, ZoneInfo, None] = None tz: Union[str, ZoneInfo, None] = None
) -> datetime: ) -> datetime:
try: try:
# Convert integer (epoch time) to UTC datetime
if isinstance(date_time, int):
date_time = datetime.utcfromtimestamp(date_time).replace(tzinfo=timezone.utc)
L.DEBUG(f"Converted epoch time {date_time} to UTC datetime object.")
# Convert string to datetime if necessary # Convert string to datetime if necessary
if isinstance(date_time, str): elif isinstance(date_time, str):
date_time = dateutil_parse(date_time) date_time = dateutil_parse(date_time)
L.DEBUG(f"Converted string '{date_time}' to datetime object.") L.DEBUG(f"Converted string '{date_time}' to datetime object.")
if not isinstance(date_time, datetime): if not isinstance(date_time, datetime):
raise ValueError("Input must be a string or datetime object.") raise ValueError(f"Input must be a string, integer (epoch time), or datetime object. What we received: {date_time}, type {type(date_time)}")
# Ensure the datetime is timezone-aware (UTC if not specified)
if date_time.tzinfo is None:
date_time = date_time.replace(tzinfo=timezone.utc)
L.DEBUG("Added UTC timezone to naive datetime.")
# Handle provided timezone # Handle provided timezone
if tz is not None: if tz is not None:
if isinstance(tz, str):
if tz == "local": if tz == "local":
last_loc = await get_timezone_without_timezone(date_time) last_loc = await get_timezone_without_timezone(date_time)
tz_str = GEO.tz(last_loc.latitude, last_loc.longitude) tz = await GEO.tz_at(last_loc.latitude, last_loc.longitude)
try:
tz = ZoneInfo(tz_str)
except Exception as e:
L.WARN(f"Invalid timezone string '{tz_str}' from DynamicTZ. Falling back to UTC. Error: {e}")
tz = ZoneInfo('UTC')
L.DEBUG(f"Using local timezone: {tz}") L.DEBUG(f"Using local timezone: {tz}")
elif isinstance(tz, str): else:
try: try:
tz = ZoneInfo(tz) tz = ZoneInfo(tz)
except Exception as e: except Exception as e:
L.ERR(f"Invalid timezone string '{tz}'. Error: {e}") L.ERR(f"Invalid timezone string '{tz}'. Error: {e}")
raise ValueError(f"Invalid timezone string: {tz}") raise ValueError(f"Invalid timezone string: {tz}")
elif isinstance(tz, ZoneInfo): elif isinstance(tz, ZoneInfo):
tz = tz # tz is already a ZoneInfo object pass # tz is already a ZoneInfo object
else: else:
raise ValueError("tz must be 'local', a string, or a ZoneInfo object.") raise ValueError(f"What we needed: tz == 'local', a string, or a ZoneInfo object. What we got: tz, a {type(tz)}, == {tz})")
# Localize to the provided or determined timezone # Convert to the provided or determined timezone
if date_time.tzinfo is None:
date_time = date_time.replace(tzinfo=tz)
L.DEBUG(f"Localized naive datetime to timezone: {tz}")
else:
date_time = date_time.astimezone(tz) date_time = date_time.astimezone(tz)
L.DEBUG(f"Converted datetime from {date_time.tzinfo} to timezone: {tz}") L.DEBUG(f"Converted datetime to timezone: {tz}")
# If no timezone provided, only fill in missing timezone info
elif date_time.tzinfo is None:
#tz_str = await get_ti`mezone_without_timezone(date_time)
#try:
# tz = ZoneInfo(tz_str)
#except Exception as e:
# L.WARN(f"Invalid timezone string '{tz_str}' from Geocoder. Falling back to UTC. Error: {e}")
# tz = ZoneInfo('UTC')
tz = ZoneInfo('UTC') # force use of UTC when we can't find real timezone
date_time = date_time.replace(tzinfo=tz)
L.DEBUG(f"Filled in missing timezone info: {tz}")
else:
L.DEBUG(f"Datetime already has timezone {date_time.tzinfo}. No changes made.")
return date_time return date_time
except ValueError as e: except ValueError as e:
@ -83,7 +70,8 @@ async def dt(
raise raise
except Exception as e: except Exception as e:
L.ERR(f"Unexpected error in dt: {e}") L.ERR(f"Unexpected error in dt: {e}")
raise ValueError(f"Failed to localize datetime: {e}") raise ValueError(f"Failed to process datetime: {e}")
async def get_timezone_without_timezone(date_time): async def get_timezone_without_timezone(date_time):
# This is a bit convoluted because we're trying to solve the paradox of needing to know the location in order to determine the timezone, but needing the timezone to be certain we've got the right location if this datetime coincided with inter-timezone travel. Our imperfect solution is to use UTC for an initial location query to determine roughly where we were at the time, get that timezone, then check the location again using that timezone, and if this location is different from the one using UTC, get the timezone again usng it, otherwise use the one we already sourced using UTC. # This is a bit convoluted because we're trying to solve the paradox of needing to know the location in order to determine the timezone, but needing the timezone to be certain we've got the right location if this datetime coincided with inter-timezone travel. Our imperfect solution is to use UTC for an initial location query to determine roughly where we were at the time, get that timezone, then check the location again using that timezone, and if this location is different from the one using UTC, get the timezone again usng it, otherwise use the one we already sourced using UTC.

23
sijapi/routers/note.py
View file

@ -84,14 +84,14 @@ async def build_daily_note_endpoint(
date_str = dt_datetime.now().strftime("%Y-%m-%d") date_str = dt_datetime.now().strftime("%Y-%m-%d")
if location: if location:
lat, lon = map(float, location.split(',')) lat, lon = map(float, location.split(','))
tz = GEO.tz_at(lat, lon) tz = await GEO.tz_at(lat, lon)
date_time = dateutil_parse(date_str).replace(tzinfo=tz) date_time = dateutil_parse(date_str).replace(tzinfo=tz)
else: else:
raise ValueError("Location is not provided or invalid.") raise ValueError("Location is not provided or invalid.")
except (ValueError, AttributeError, TypeError) as e: except (ValueError, AttributeError, TypeError) as e:
L.WARN(f"Falling back to localized datetime due to error: {e}") L.WARN(f"Falling back to localized datetime due to error: {e}")
try: try:
date_time = loc.dt(date_str) date_time = await loc.dt(date_str)
places = await loc.fetch_locations(date_time) places = await loc.fetch_locations(date_time)
lat, lon = places[0].latitude, places[0].longitude lat, lon = places[0].latitude, places[0].longitude
except Exception as e: except Exception as e:
@ -358,14 +358,14 @@ async def generate_banner(dt, location: Location = None, forecast: str = None, m
async def note_weather_get( async def note_weather_get(
date: str = Query(default="0", description="Enter a date in YYYY-MM-DD format, otherwise it will default to today."), date: str = Query(default="0", description="Enter a date in YYYY-MM-DD format, otherwise it will default to today."),
latlon: str = Query(default="45,-125"), latlon: str = Query(default="45,-125"),
refresh: bool = Query(default=False, description="Set to true to refresh the weather data") refresh: str = Query(default="False", description="Set to True to force refresh the weather data")
): ):
force_refresh_weather = refresh == "True"
try: try:
date_time = dt_datetime.now() if date == "0" else await loc.dt(date) date_time = dt_datetime.now() if date == "0" else await loc.dt(date)
L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our dt_datetime in note_weather_get.") L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our dt_datetime in note_weather_get.")
L.DEBUG(f"date: {date} .. date_time: {date_time}") L.DEBUG(f"date: {date} .. date_time: {date_time}")
content = await update_dn_weather(date_time) #, lat, lon) content = await update_dn_weather(date_time, force_refresh_weather) #, lat, lon)
return JSONResponse(content={"forecast": content}, status_code=200) return JSONResponse(content={"forecast": content}, status_code=200)
except HTTPException as e: except HTTPException as e:
@ -377,19 +377,20 @@ async def note_weather_get(
@note.post("/update/note/{date}") @note.post("/update/note/{date}")
async def post_update_daily_weather_and_calendar_and_timeslips(date: str) -> PlainTextResponse: async def post_update_daily_weather_and_calendar_and_timeslips(date: str, refresh: str="False") -> PlainTextResponse:
date_time = await loc.dt(date) date_time = await loc.dt(date)
L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our dt_datetime in post_update_daily_weather_and_calendar_and_timeslips.") L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our dt_datetime in post_update_daily_weather_and_calendar_and_timeslips.")
await update_dn_weather(date_time) force_refresh_weather = refresh == "True"
await update_dn_weather(date_time, force_refresh_weather)
await update_daily_note_events(date_time) await update_daily_note_events(date_time)
await build_daily_timeslips(date_time) await build_daily_timeslips(date_time)
return f"[Refresh]({API.URL}/update/note/{date_time.strftime('%Y-%m-%d')}" return f"[Refresh]({API.URL}/update/note/{date_time.strftime('%Y-%m-%d')}"
async def update_dn_weather(date_time: dt_datetime, lat: float = None, lon: float = None): async def update_dn_weather(date_time: dt_datetime, force_refresh: bool = False, lat: float = None, lon: float = None):
L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our datetime in update_dn_weather.") L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our datetime in update_dn_weather.")
try: try:
if lat and lon: if lat and lon:
place = GEO.code((lat, lon)) place = await GEO.code((lat, lon))
else: else:
L.DEBUG(f"Updating weather for {date_time}") L.DEBUG(f"Updating weather for {date_time}")
@ -422,8 +423,8 @@ async def update_dn_weather(date_time: dt_datetime, lat: float = None, lon: floa
L.DEBUG(f"Journal path: absolute_path={absolute_path}, relative_path={relative_path}") L.DEBUG(f"Journal path: absolute_path={absolute_path}, relative_path={relative_path}")
try: try:
L.DEBUG(f"passing date_time {date_time.strftime('%Y-%m-%d %H:%M:%S')}, {lat}/{lon} into fetch_and_store") L.DEBUG(f"passing date_time {date_time.strftime('%Y-%m-%d %H:%M:%S')}, {lat}/{lon} into get_weather")
day = await weather.get_weather(date_time, lat, lon) day = await weather.get_weather(date_time, lat, lon, force_refresh)
L.DEBUG(f"day information obtained from get_weather: {day}") L.DEBUG(f"day information obtained from get_weather: {day}")
if day: if day:
DailyWeather = day.get('DailyWeather') DailyWeather = day.get('DailyWeather')

109
sijapi/routers/weather.py
View file

@ -2,12 +2,13 @@
Uses the VisualCrossing API and Postgres/PostGIS to source local weather forecasts and history. Uses the VisualCrossing API and Postgres/PostGIS to source local weather forecasts and history.
''' '''
import asyncio import asyncio
from fastapi import APIRouter, HTTPException from fastapi import APIRouter, HTTPException, Query
from fastapi import HTTPException from fastapi import HTTPException
from fastapi.responses import JSONResponse
from asyncpg.cursor import Cursor from asyncpg.cursor import Cursor
from httpx import AsyncClient from httpx import AsyncClient
from typing import Dict from typing import Dict
from datetime import datetime from datetime import datetime as dt_datetime
from shapely.wkb import loads from shapely.wkb import loads
from binascii import unhexlify from binascii import unhexlify
from sijapi import L, VISUALCROSSING_API_KEY, TZ, DB, GEO from sijapi import L, VISUALCROSSING_API_KEY, TZ, DB, GEO
@ -16,13 +17,41 @@ from sijapi.routers import loc
weather = APIRouter() weather = APIRouter()
@weather.get("/weather/refresh", response_class=JSONResponse)
async def get_refreshed_weather(
date: str = Query(default=dt_datetime.now().strftime("%Y-%m-%d"), description="Enter a date in YYYY-MM-DD format, otherwise it will default to today."),
latlon: str = Query(default="None", description="Optionally enter latitude and longitude in the format 45.8411,-123.1765; if not provided it will use your recorded location."),
):
# date = await date
try:
if latlon == "None":
date_time = await loc.dt(date)
place = await loc.fetch_last_location_before(date_time)
lat = place.latitude
lon = place.longitude
else:
lat, lon = latlon.split(',')
tz = await GEO.tz_at(lat, lon)
date_time = await loc.dt(date, tz)
async def get_weather(date_time: datetime, latitude: float, longitude: float): L.DEBUG(f"passing date_time {date_time.strftime('%Y-%m-%d %H:%M:%S')}, {lat}/{lon} into get_weather")
# request_date_str = date_time.strftime("%Y-%m-%d") day = await get_weather(date_time, lat, lon, force_refresh=True)
day_str = str(day)
return JSONResponse(content={"weather": day_str}, status_code=200)
except HTTPException as e:
return JSONResponse(content={"detail": str(e.detail)}, status_code=e.status_code)
except Exception as e:
L.ERR(f"Error in note_weather_get: {str(e)}")
raise HTTPException(status_code=500, detail=f"An error occurred: {str(e)}")
async def get_weather(date_time: dt_datetime, latitude: float, longitude: float, force_refresh: bool = False):
L.DEBUG(f"Called get_weather with lat: {latitude}, lon: {longitude}, date_time: {date_time}") L.DEBUG(f"Called get_weather with lat: {latitude}, lon: {longitude}, date_time: {date_time}")
L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our datetime in get_weather.") L.WARN(f"Using {date_time} as our datetime in get_weather.")
daily_weather_data = await get_weather_from_db(date_time, latitude, longitude)
fetch_new_data = True fetch_new_data = True
if force_refresh == False:
daily_weather_data = await get_weather_from_db(date_time, latitude, longitude)
if daily_weather_data: if daily_weather_data:
try: try:
L.DEBUG(f"Daily weather data from db: {daily_weather_data}") L.DEBUG(f"Daily weather data from db: {daily_weather_data}")
@ -43,7 +72,7 @@ async def get_weather(date_time: datetime, latitude: float, longitude: float):
request_haversine = haversine(latitude, longitude, stored_lat, stored_lon) request_haversine = haversine(latitude, longitude, stored_lat, stored_lon)
L.DEBUG(f"\nINFO:\nlast updated {last_updated}\nstored lat: {stored_lat} - requested lat: {latitude}\nstored lon: {stored_lon} - requested lon: {longitude}\nHaversine: {request_haversine}") L.DEBUG(f"\nINFO:\nlast updated {last_updated}\nstored lat: {stored_lat} - requested lat: {latitude}\nstored lon: {stored_lon} - requested lon: {longitude}\nHaversine: {request_haversine}")
if last_updated and (date_time <= datetime.now(TZ) and last_updated > date_time and request_haversine < 8) and hourly_weather and len(hourly_weather) > 0: if last_updated and (date_time <= dt_datetime.now(TZ) and last_updated > date_time and request_haversine < 8) and hourly_weather and len(hourly_weather) > 0:
L.DEBUG(f"We can use existing data... :')") L.DEBUG(f"We can use existing data... :')")
fetch_new_data = False fetch_new_data = False
@ -85,13 +114,12 @@ async def get_weather(date_time: datetime, latitude: float, longitude: float):
return daily_weather_data return daily_weather_data
async def store_weather_to_db(date_time: datetime, weather_data: dict): async def store_weather_to_db(date_time: dt_datetime, weather_data: dict):
L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our datetime in store_weather_to_db") L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our datetime in store_weather_to_db")
async with DB.get_connection() as conn: async with DB.get_connection() as conn:
try: try:
day_data = weather_data.get('days')[0] day_data = weather_data.get('days')[0]
L.DEBUG(f"day_data.get('sunrise'): {day_data.get('sunrise')}") L.DEBUG(f"RAW DAY_DATA: {day_data}")
# Handle preciptype and stations as PostgreSQL arrays # Handle preciptype and stations as PostgreSQL arrays
preciptype_array = day_data.get('preciptype', []) or [] preciptype_array = day_data.get('preciptype', []) or []
stations_array = day_data.get('stations', []) or [] stations_array = day_data.get('stations', []) or []
@ -102,17 +130,15 @@ async def store_weather_to_db(date_time: datetime, weather_data: dict):
# Get location details from weather data if available # Get location details from weather data if available
longitude = weather_data.get('longitude') longitude = weather_data.get('longitude')
latitude = weather_data.get('latitude') latitude = weather_data.get('latitude')
tz = await GEO.tz_at(latitude, longitude)
elevation = await GEO.elevation(latitude, longitude) elevation = await GEO.elevation(latitude, longitude)
location_point = f"POINTZ({longitude} {latitude} {elevation})" if longitude and latitude and elevation else None location_point = f"POINTZ({longitude} {latitude} {elevation})" if longitude and latitude and elevation else None
# Correct for the datetime objects L.WARN(f"Uncorrected datetimes in store_weather_to_db: {day_data['datetime']}, sunrise: {day_data['sunrise']}, sunset: {day_data['sunset']}")
L.WARN(f"Uncorrected datetime in store_weather_to_db: {day_data['datetime']}") day_data['datetime'] = await loc.dt(day_data.get('datetimeEpoch'))
day_data['datetime'] = await loc.dt(day_data.get('datetime')) #day_data.get('datetime')) day_data['sunrise'] = await loc.dt(day_data.get('sunriseEpoch'))
L.WARN(f"Corrected datetime in store_weather_to_db with localized datetime: {day_data['datetime']}") day_data['sunset'] = await loc.dt(day_data.get('sunsetEpoch'))
L.WARN(f"Uncorrected sunrise time in store_weather_to_db: {day_data['sunrise']}") L.WARN(f"Corrected datetimes in store_weather_to_db: {day_data['datetime']}, sunrise: {day_data['sunrise']}, sunset: {day_data['sunset']}")
day_data['sunrise'] = day_data['datetime'].replace(hour=int(day_data.get('sunrise').split(':')[0]), minute=int(day_data.get('sunrise').split(':')[1]))
L.WARN(f"Corrected sunrise time in store_weather_to_db with localized datetime: {day_data['sunrise']}")
day_data['sunset'] = day_data['datetime'].replace(hour=int(day_data.get('sunset').split(':')[0]), minute=int(day_data.get('sunset').split(':')[1]))
daily_weather_params = ( daily_weather_params = (
day_data.get('sunrise'), day_data.get('sunriseEpoch'), day_data.get('sunrise'), day_data.get('sunriseEpoch'),
@ -120,7 +146,7 @@ async def store_weather_to_db(date_time: datetime, weather_data: dict):
day_data.get('description'), day_data.get('tempmax'), day_data.get('description'), day_data.get('tempmax'),
day_data.get('tempmin'), day_data.get('uvindex'), day_data.get('tempmin'), day_data.get('uvindex'),
day_data.get('winddir'), day_data.get('windspeed'), day_data.get('winddir'), day_data.get('windspeed'),
day_data.get('icon'), datetime.now(), day_data.get('icon'), dt_datetime.now(tz),
day_data.get('datetime'), day_data.get('datetimeEpoch'), day_data.get('datetime'), day_data.get('datetimeEpoch'),
day_data.get('temp'), day_data.get('feelslikemax'), day_data.get('temp'), day_data.get('feelslikemax'),
day_data.get('feelslikemin'), day_data.get('feelslike'), day_data.get('feelslikemin'), day_data.get('feelslike'),
@ -141,9 +167,9 @@ async def store_weather_to_db(date_time: datetime, weather_data: dict):
try: try:
daily_weather_query = ''' daily_weather_query = '''
INSERT INTO DailyWeather ( INSERT INTO DailyWeather (
sunrise, sunriseEpoch, sunset, sunsetEpoch, description, sunrise, sunriseepoch, sunset, sunsetepoch, description,
tempmax, tempmin, uvindex, winddir, windspeed, icon, last_updated, tempmax, tempmin, uvindex, winddir, windspeed, icon, last_updated,
datetime, datetimeEpoch, temp, feelslikemax, feelslikemin, feelslike, datetime, datetimeepoch, temp, feelslikemax, feelslikemin, feelslike,
dew, humidity, precip, precipprob, precipcover, preciptype, dew, humidity, precip, precipprob, precipcover, preciptype,
snow, snowdepth, windgust, pressure, cloudcover, visibility, snow, snowdepth, windgust, pressure, cloudcover, visibility,
solarradiation, solarenergy, severerisk, moonphase, conditions, solarradiation, solarenergy, severerisk, moonphase, conditions,
@ -152,25 +178,15 @@ async def store_weather_to_db(date_time: datetime, weather_data: dict):
RETURNING id RETURNING id
''' '''
# Debug logs for better insights
# L.DEBUG("Executing query: %s", daily_weather_query)
# L.DEBUG("With parameters: %s", daily_weather_params)
# Execute the query to insert daily weather data
async with conn.transaction(): async with conn.transaction():
daily_weather_id = await conn.fetchval(daily_weather_query, *daily_weather_params) daily_weather_id = await conn.fetchval(daily_weather_query, *daily_weather_params)
if 'hours' in day_data: if 'hours' in day_data:
L.DEBUG(f"Processing hours now...")
for hour_data in day_data['hours']: for hour_data in day_data['hours']:
try: try:
await asyncio.sleep(0.1) await asyncio.sleep(0.01)
# hour_data['datetime'] = parse_date(hour_data.get('datetime')) hour_data['datetime'] = await loc.dt(hour_data.get('datetimeEpoch'))
hour_timestamp = date_str + ' ' + hour_data['datetime']
hour_data['datetime'] = await loc.dt(hour_timestamp)
L.DEBUG(f"Processing hours now...")
# L.DEBUG(f"Processing {hour_data['datetime']}")
hour_preciptype_array = hour_data.get('preciptype', []) or [] hour_preciptype_array = hour_data.get('preciptype', []) or []
hour_stations_array = hour_data.get('stations', []) or [] hour_stations_array = hour_data.get('stations', []) or []
hourly_weather_params = ( hourly_weather_params = (
@ -204,21 +220,15 @@ async def store_weather_to_db(date_time: datetime, weather_data: dict):
try: try:
hourly_weather_query = ''' hourly_weather_query = '''
INSERT INTO HourlyWeather (daily_weather_id, datetime, datetimeEpoch, temp, feelslike, humidity, dew, precip, precipprob, INSERT INTO HourlyWeather (daily_weather_id, datetime, datetimeepoch, temp, feelslike, humidity, dew, precip, precipprob,
preciptype, snow, snowdepth, windgust, windspeed, winddir, pressure, cloudcover, visibility, solarradiation, solarenergy, preciptype, snow, snowdepth, windgust, windspeed, winddir, pressure, cloudcover, visibility, solarradiation, solarenergy,
uvindex, severerisk, conditions, icon, stations, source) uvindex, severerisk, conditions, icon, stations, source)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26)
RETURNING id RETURNING id
''' '''
# Debug logs for better insights
# L.DEBUG("Executing query: %s", hourly_weather_query)
# L.DEBUG("With parameters: %s", hourly_weather_params)
# Execute the query to insert hourly weather data
async with conn.transaction(): async with conn.transaction():
hourly_weather_id = await conn.fetchval(hourly_weather_query, *hourly_weather_params) hourly_weather_id = await conn.fetchval(hourly_weather_query, *hourly_weather_params)
# L.ERR(f"\n{hourly_weather_id}") L.DEBUG(f"Done processing hourly_weather_id {hourly_weather_id}")
except Exception as e: except Exception as e:
L.ERR(f"EXCEPTION: {e}") L.ERR(f"EXCEPTION: {e}")
@ -232,7 +242,7 @@ async def store_weather_to_db(date_time: datetime, weather_data: dict):
async def get_weather_from_db(date_time: datetime, latitude: float, longitude: float): async def get_weather_from_db(date_time: dt_datetime, latitude: float, longitude: float):
L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our datetime in get_weather_from_db.") L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our datetime in get_weather_from_db.")
async with DB.get_connection() as conn: async with DB.get_connection() as conn:
query_date = date_time.date() query_date = date_time.date()
@ -246,7 +256,6 @@ async def get_weather_from_db(date_time: datetime, latitude: float, longitude: f
LIMIT 1 LIMIT 1
''' '''
daily_weather_record = await conn.fetchrow(query, query_date, longitude, latitude, longitude, latitude) daily_weather_record = await conn.fetchrow(query, query_date, longitude, latitude, longitude, latitude)
if daily_weather_record is None: if daily_weather_record is None:
@ -255,9 +264,14 @@ async def get_weather_from_db(date_time: datetime, latitude: float, longitude: f
# Convert asyncpg.Record to a mutable dictionary # Convert asyncpg.Record to a mutable dictionary
daily_weather_data = dict(daily_weather_record) daily_weather_data = dict(daily_weather_record)
# L.DEBUG(f"Daily weather data prior to tz corrections: {daily_weather_data}")
# Now we can modify the dictionary # Now we can modify the dictionary
daily_weather_data['datetime'] = await loc.dt(daily_weather_data.get('datetime')) # tz = await GEO.tz_at(latitude, longitude)
# daily_weather_data['datetime'] = await loc.dt(daily_weather_data.get('datetime'), tz)
# daily_weather_data['sunrise'] = await loc.dt(daily_weather_data.get('sunrise'), tz)
# daily_weather_data['sunset'] = await loc.dt(daily_weather_data.get('sunset'), tz)
# L.DEBUG(f"Daily weather data after tz corrections: {daily_weather_data}")
# Query to get hourly weather data # Query to get hourly weather data
query = ''' query = '''
@ -270,9 +284,10 @@ async def get_weather_from_db(date_time: datetime, latitude: float, longitude: f
hourly_weather_data = [] hourly_weather_data = []
for record in hourly_weather_records: for record in hourly_weather_records:
hour_data = dict(record) hour_data = dict(record)
hour_data['datetime'] = await loc.dt(hour_data.get('datetime')) # hour_data['datetime'] = await loc.dt(hour_data.get('datetime'), tz)
hourly_weather_data.append(hour_data) hourly_weather_data.append(hour_data)
L.DEBUG(f"Hourly weather data after tz corrections: {hourly_weather_data}")
day = { day = {
'DailyWeather': daily_weather_data, 'DailyWeather': daily_weather_data,
'HourlyWeather': hourly_weather_data, 'HourlyWeather': hourly_weather_data,