diff --git a/sijapi/classes.py b/sijapi/classes.py
index 1ea470c..e2aed2a 100644
--- a/sijapi/classes.py
+++ b/sijapi/classes.py
@@ -8,7 +8,7 @@ from contextlib import asynccontextmanager
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, Union, TypeVar, Type
-
+from zoneinfo import ZoneInfo
import aiofiles
import aiohttp
import asyncpg
@@ -311,13 +311,13 @@ class Geocoder:
else:
raise ValueError(f"Unsupported unit: {unit}")
-
- async def timezone(self, lat: float, lon: float):
+ async def timezone(self, lat: float, lon: float) -> Optional[ZoneInfo]:
loop = asyncio.get_running_loop()
- timezone = await loop.run_in_executor(self.executor, lambda: self.tf.timezone_at(lat=lat, lng=lon))
- return timezone if timezone else 'Unknown'
+ timezone_str = await loop.run_in_executor(self.executor, lambda: self.tf.timezone_at(lat=lat, lng=lon))
+ return ZoneInfo(timezone_str) if timezone_str else None
+
async def lookup(self, lat: float, lon: float):
city, state, country = (await self.location(lat, lon))[0]['name'], (await self.location(lat, lon))[0]['admin1'], (await self.location(lat, lon))[0]['cc']
elevation = await self.elevation(lat, lon)
@@ -443,7 +443,7 @@ class Geocoder:
def coords_equal(self, coord1: Tuple[float, float], coord2: Tuple[float, float], tolerance: float = 1e-5) -> bool:
return math.isclose(coord1[0], coord2[0], abs_tol=tolerance) and math.isclose(coord1[1], coord2[1], abs_tol=tolerance)
- async def refresh_timezone(self, location: Union[Location, Tuple[float, float]], force: bool = False) -> str:
+ async def refresh_timezone(self, location: Union[Location, Tuple[float, float]], force: bool = False) -> Optional[ZoneInfo]:
if isinstance(location, Location):
lat, lon = location.latitude, location.longitude
else:
@@ -457,6 +457,7 @@ class Geocoder:
current_time - self.last_update > timedelta(hours=1) or
not self.coords_equal(rounded_location, self.round_coords(*self.last_location) if self.last_location else (None, None))):
+
new_timezone = await self.timezone(lat, lon)
self.last_timezone = new_timezone
self.last_update = current_time
@@ -465,9 +466,10 @@ class Geocoder:
return self.last_timezone
+
async def tz_save(self):
cache_data = {
- 'last_timezone': self.last_timezone,
+ 'last_timezone': str(self.last_timezone) if self.last_timezone else None,
'last_update': self.last_update.isoformat() if self.last_update else None,
'last_location': self.last_location
}
@@ -478,29 +480,31 @@ class Geocoder:
try:
async with aiofiles.open(self.cache_file, 'r') as f:
cache_data = json.loads(await f.read())
- self.last_timezone = cache_data.get('last_timezone')
+ self.last_timezone = ZoneInfo(cache_data['last_timezone']) if cache_data.get('last_timezone') else None
self.last_update = datetime.fromisoformat(cache_data['last_update']) if cache_data.get('last_update') else None
self.last_location = tuple(cache_data['last_location']) if cache_data.get('last_location') else None
+
except (FileNotFoundError, json.JSONDecodeError):
# If file doesn't exist or is invalid, we'll start fresh
- pass
+ self.last_timezone = None
+ self.last_update = None
+ self.last_location = None
- async def tz_current(self, location: Union[Location, Tuple[float, float]]) -> str:
+ async def tz_current(self, location: Union[Location, Tuple[float, float]]) -> Optional[ZoneInfo]:
await self.tz_cached()
return await self.refresh_timezone(location)
- async def tz_last(self) -> Optional[str]:
+ async def tz_last(self) -> Optional[ZoneInfo]:
await self.tz_cached()
return self.last_timezone
-
- async def tz_at(self, lat: float, lon: float) -> str:
+ async def tz_at(self, lat: float, lon: float) -> Optional[ZoneInfo]:
"""
Get the timezone at a specific latitude and longitude without affecting the cache.
:param lat: Latitude
:param lon: Longitude
- :return: Timezone string
+ :return: ZoneInfo object representing the timezone
"""
return await self.timezone(lat, lon)
diff --git a/sijapi/routers/note.py b/sijapi/routers/note.py
index 887c5a8..1498f22 100644
--- a/sijapi/routers/note.py
+++ b/sijapi/routers/note.py
@@ -84,14 +84,14 @@ async def build_daily_note_endpoint(
date_str = dt_datetime.now().strftime("%Y-%m-%d")
if location:
lat, lon = map(float, location.split(','))
- tz = GEO.tz_at(lat, lon)
+ tz = await GEO.tz_at(lat, lon)
date_time = dateutil_parse(date_str).replace(tzinfo=tz)
else:
raise ValueError("Location is not provided or invalid.")
except (ValueError, AttributeError, TypeError) as e:
L.WARN(f"Falling back to localized datetime due to error: {e}")
try:
- date_time = loc.dt(date_str)
+ date_time = await loc.dt(date_str)
places = await loc.fetch_locations(date_time)
lat, lon = places[0].latitude, places[0].longitude
except Exception as e:
@@ -358,14 +358,14 @@ async def generate_banner(dt, location: Location = None, forecast: str = None, m
async def note_weather_get(
date: str = Query(default="0", description="Enter a date in YYYY-MM-DD format, otherwise it will default to today."),
latlon: str = Query(default="45,-125"),
- refresh: bool = Query(default=False, description="Set to true to refresh the weather data")
+ refresh: str = Query(default="False", description="Set to True to force refresh the weather data")
):
-
+ force_refresh_weather = refresh == "True"
try:
date_time = dt_datetime.now() if date == "0" else await loc.dt(date)
L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our dt_datetime in note_weather_get.")
L.DEBUG(f"date: {date} .. date_time: {date_time}")
- content = await update_dn_weather(date_time) #, lat, lon)
+ content = await update_dn_weather(date_time, force_refresh_weather) #, lat, lon)
return JSONResponse(content={"forecast": content}, status_code=200)
except HTTPException as e:
@@ -377,19 +377,20 @@ async def note_weather_get(
@note.post("/update/note/{date}")
-async def post_update_daily_weather_and_calendar_and_timeslips(date: str) -> PlainTextResponse:
+async def post_update_daily_weather_and_calendar_and_timeslips(date: str, refresh: str="False") -> PlainTextResponse:
date_time = await loc.dt(date)
L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our dt_datetime in post_update_daily_weather_and_calendar_and_timeslips.")
- await update_dn_weather(date_time)
+ force_refresh_weather = refresh == "True"
+ await update_dn_weather(date_time, force_refresh_weather)
await update_daily_note_events(date_time)
await build_daily_timeslips(date_time)
return f"[Refresh]({API.URL}/update/note/{date_time.strftime('%Y-%m-%d')}"
-async def update_dn_weather(date_time: dt_datetime, lat: float = None, lon: float = None):
+async def update_dn_weather(date_time: dt_datetime, force_refresh: bool = False, lat: float = None, lon: float = None):
L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our datetime in update_dn_weather.")
try:
if lat and lon:
- place = GEO.code((lat, lon))
+ place = await GEO.code((lat, lon))
else:
L.DEBUG(f"Updating weather for {date_time}")
@@ -422,8 +423,8 @@ async def update_dn_weather(date_time: dt_datetime, lat: float = None, lon: floa
L.DEBUG(f"Journal path: absolute_path={absolute_path}, relative_path={relative_path}")
try:
- L.DEBUG(f"passing date_time {date_time.strftime('%Y-%m-%d %H:%M:%S')}, {lat}/{lon} into fetch_and_store")
- day = await weather.get_weather(date_time, lat, lon)
+ L.DEBUG(f"passing date_time {date_time.strftime('%Y-%m-%d %H:%M:%S')}, {lat}/{lon} into get_weather")
+ day = await weather.get_weather(date_time, lat, lon, force_refresh)
L.DEBUG(f"day information obtained from get_weather: {day}")
if day:
DailyWeather = day.get('DailyWeather')
diff --git a/sijapi/routers/weather.py b/sijapi/routers/weather.py
index a9068ca..324d294 100644
--- a/sijapi/routers/weather.py
+++ b/sijapi/routers/weather.py
@@ -2,12 +2,13 @@
Uses the VisualCrossing API and Postgres/PostGIS to source local weather forecasts and history.
'''
import asyncio
-from fastapi import APIRouter, HTTPException
+from fastapi import APIRouter, HTTPException, Query
from fastapi import HTTPException
+from fastapi.responses import JSONResponse
from asyncpg.cursor import Cursor
from httpx import AsyncClient
from typing import Dict
-from datetime import datetime
+from datetime import datetime as dt_datetime
from shapely.wkb import loads
from binascii import unhexlify
from sijapi import L, VISUALCROSSING_API_KEY, TZ, DB, GEO
@@ -16,39 +17,67 @@ from sijapi.routers import loc
weather = APIRouter()
+@weather.get("/weather/refresh", response_class=JSONResponse)
+async def get_refreshed_weather(
+ date: str = Query(default=dt_datetime.now().strftime("%Y-%m-%d"), description="Enter a date in YYYY-MM-DD format, otherwise it will default to today."),
+ latlon: str = Query(default="None", description="Optionally enter latitude and longitude in the format 45.8411,-123.1765; if not provided it will use your recorded location."),
+):
+ # date = await date
+ try:
+ if latlon == "None":
+ date_time = await loc.dt(date)
+ place = await loc.fetch_last_location_before(date_time)
+ lat = place.latitude
+ lon = place.longitude
+ else:
+ lat, lon = latlon.split(',')
+ tz = await GEO.tz_at(lat, lon)
+ date_time = await loc.dt(date, tz)
-async def get_weather(date_time: datetime, latitude: float, longitude: float):
- # request_date_str = date_time.strftime("%Y-%m-%d")
+ L.DEBUG(f"passing date_time {date_time.strftime('%Y-%m-%d %H:%M:%S')}, {lat}/{lon} into get_weather")
+ day = await get_weather(date_time, lat, lon, force_refresh=True)
+ day_str = str(day)
+ return JSONResponse(content={"weather": day_str}, status_code=200)
+
+ except HTTPException as e:
+ return JSONResponse(content={"detail": str(e.detail)}, status_code=e.status_code)
+
+ except Exception as e:
+ L.ERR(f"Error in note_weather_get: {str(e)}")
+ raise HTTPException(status_code=500, detail=f"An error occurred: {str(e)}")
+
+async def get_weather(date_time: dt_datetime, latitude: float, longitude: float, force_refresh: bool = False):
L.DEBUG(f"Called get_weather with lat: {latitude}, lon: {longitude}, date_time: {date_time}")
- L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our datetime in get_weather.")
- daily_weather_data = await get_weather_from_db(date_time, latitude, longitude)
+ L.WARN(f"Using {date_time} as our datetime in get_weather.")
fetch_new_data = True
- if daily_weather_data:
- try:
- L.DEBUG(f"Daily weather data from db: {daily_weather_data}")
- last_updated = str(daily_weather_data['DailyWeather'].get('last_updated'))
- last_updated = await loc.dt(last_updated)
- stored_loc_data = unhexlify(daily_weather_data['DailyWeather'].get('location'))
- stored_loc = loads(stored_loc_data)
- stored_lat = stored_loc.y
- stored_lon = stored_loc.x
- stored_ele = stored_loc.z
-
- hourly_weather = daily_weather_data.get('HourlyWeather')
-
- L.DEBUG(f"Hourly: {hourly_weather}")
-
- L.DEBUG(f"\nINFO:\nlast updated {last_updated}\nstored lat: {stored_lat} - requested lat: {latitude}\nstored lon: {stored_lon} - requested lon: {longitude}\n")
-
- request_haversine = haversine(latitude, longitude, stored_lat, stored_lon)
- L.DEBUG(f"\nINFO:\nlast updated {last_updated}\nstored lat: {stored_lat} - requested lat: {latitude}\nstored lon: {stored_lon} - requested lon: {longitude}\nHaversine: {request_haversine}")
-
- if last_updated and (date_time <= datetime.now(TZ) and last_updated > date_time and request_haversine < 8) and hourly_weather and len(hourly_weather) > 0:
- L.DEBUG(f"We can use existing data... :')")
- fetch_new_data = False
+ if force_refresh == False:
+ daily_weather_data = await get_weather_from_db(date_time, latitude, longitude)
+ if daily_weather_data:
+ try:
+ L.DEBUG(f"Daily weather data from db: {daily_weather_data}")
+ last_updated = str(daily_weather_data['DailyWeather'].get('last_updated'))
+ last_updated = await loc.dt(last_updated)
+ stored_loc_data = unhexlify(daily_weather_data['DailyWeather'].get('location'))
+ stored_loc = loads(stored_loc_data)
+ stored_lat = stored_loc.y
+ stored_lon = stored_loc.x
+ stored_ele = stored_loc.z
- except Exception as e:
- L.ERR(f"Error in get_weather: {e}")
+ hourly_weather = daily_weather_data.get('HourlyWeather')
+
+ L.DEBUG(f"Hourly: {hourly_weather}")
+
+ L.DEBUG(f"\nINFO:\nlast updated {last_updated}\nstored lat: {stored_lat} - requested lat: {latitude}\nstored lon: {stored_lon} - requested lon: {longitude}\n")
+
+ request_haversine = haversine(latitude, longitude, stored_lat, stored_lon)
+ L.DEBUG(f"\nINFO:\nlast updated {last_updated}\nstored lat: {stored_lat} - requested lat: {latitude}\nstored lon: {stored_lon} - requested lon: {longitude}\nHaversine: {request_haversine}")
+
+ if last_updated and (date_time <= dt_datetime.now(TZ) and last_updated > date_time and request_haversine < 8) and hourly_weather and len(hourly_weather) > 0:
+ L.DEBUG(f"We can use existing data... :')")
+ fetch_new_data = False
+
+ except Exception as e:
+ L.ERR(f"Error in get_weather: {e}")
if fetch_new_data:
L.DEBUG(f"We require new data!")
@@ -85,13 +114,12 @@ async def get_weather(date_time: datetime, latitude: float, longitude: float):
return daily_weather_data
-async def store_weather_to_db(date_time: datetime, weather_data: dict):
+async def store_weather_to_db(date_time: dt_datetime, weather_data: dict):
L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our datetime in store_weather_to_db")
async with DB.get_connection() as conn:
try:
day_data = weather_data.get('days')[0]
- L.DEBUG(f"day_data.get('sunrise'): {day_data.get('sunrise')}")
-
+ L.DEBUG(f"RAW DAY_DATA: {day_data}")
# Handle preciptype and stations as PostgreSQL arrays
preciptype_array = day_data.get('preciptype', []) or []
stations_array = day_data.get('stations', []) or []
@@ -102,17 +130,15 @@ async def store_weather_to_db(date_time: datetime, weather_data: dict):
# Get location details from weather data if available
longitude = weather_data.get('longitude')
latitude = weather_data.get('latitude')
+ tz = await GEO.tz_at(latitude, longitude)
elevation = await GEO.elevation(latitude, longitude)
location_point = f"POINTZ({longitude} {latitude} {elevation})" if longitude and latitude and elevation else None
- # Correct for the datetime objects
- L.WARN(f"Uncorrected datetime in store_weather_to_db: {day_data['datetime']}")
- day_data['datetime'] = await loc.dt(day_data.get('datetime')) #day_data.get('datetime'))
- L.WARN(f"Corrected datetime in store_weather_to_db with localized datetime: {day_data['datetime']}")
- L.WARN(f"Uncorrected sunrise time in store_weather_to_db: {day_data['sunrise']}")
- day_data['sunrise'] = day_data['datetime'].replace(hour=int(day_data.get('sunrise').split(':')[0]), minute=int(day_data.get('sunrise').split(':')[1]))
- L.WARN(f"Corrected sunrise time in store_weather_to_db with localized datetime: {day_data['sunrise']}")
- day_data['sunset'] = day_data['datetime'].replace(hour=int(day_data.get('sunset').split(':')[0]), minute=int(day_data.get('sunset').split(':')[1]))
+ L.WARN(f"Uncorrected datetimes in store_weather_to_db: {day_data['datetime']}, sunrise: {day_data['sunrise']}, sunset: {day_data['sunset']}")
+ day_data['datetime'] = await loc.dt(day_data.get('datetimeEpoch'))
+ day_data['sunrise'] = await loc.dt(day_data.get('sunriseEpoch'))
+ day_data['sunset'] = await loc.dt(day_data.get('sunsetEpoch'))
+ L.WARN(f"Corrected datetimes in store_weather_to_db: {day_data['datetime']}, sunrise: {day_data['sunrise']}, sunset: {day_data['sunset']}")
daily_weather_params = (
day_data.get('sunrise'), day_data.get('sunriseEpoch'),
@@ -120,7 +146,7 @@ async def store_weather_to_db(date_time: datetime, weather_data: dict):
day_data.get('description'), day_data.get('tempmax'),
day_data.get('tempmin'), day_data.get('uvindex'),
day_data.get('winddir'), day_data.get('windspeed'),
- day_data.get('icon'), datetime.now(),
+ day_data.get('icon'), dt_datetime.now(tz),
day_data.get('datetime'), day_data.get('datetimeEpoch'),
day_data.get('temp'), day_data.get('feelslikemax'),
day_data.get('feelslikemin'), day_data.get('feelslike'),
@@ -141,9 +167,9 @@ async def store_weather_to_db(date_time: datetime, weather_data: dict):
try:
daily_weather_query = '''
INSERT INTO DailyWeather (
- sunrise, sunriseEpoch, sunset, sunsetEpoch, description,
+ sunrise, sunriseepoch, sunset, sunsetepoch, description,
tempmax, tempmin, uvindex, winddir, windspeed, icon, last_updated,
- datetime, datetimeEpoch, temp, feelslikemax, feelslikemin, feelslike,
+ datetime, datetimeepoch, temp, feelslikemax, feelslikemin, feelslike,
dew, humidity, precip, precipprob, precipcover, preciptype,
snow, snowdepth, windgust, pressure, cloudcover, visibility,
solarradiation, solarenergy, severerisk, moonphase, conditions,
@@ -151,26 +177,16 @@ async def store_weather_to_db(date_time: datetime, weather_data: dict):
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38)
RETURNING id
'''
-
- # Debug logs for better insights
- # L.DEBUG("Executing query: %s", daily_weather_query)
- # L.DEBUG("With parameters: %s", daily_weather_params)
- # Execute the query to insert daily weather data
async with conn.transaction():
daily_weather_id = await conn.fetchval(daily_weather_query, *daily_weather_params)
-
if 'hours' in day_data:
+ L.DEBUG(f"Processing hours now...")
for hour_data in day_data['hours']:
try:
- await asyncio.sleep(0.1)
- # hour_data['datetime'] = parse_date(hour_data.get('datetime'))
- hour_timestamp = date_str + ' ' + hour_data['datetime']
- hour_data['datetime'] = await loc.dt(hour_timestamp)
- L.DEBUG(f"Processing hours now...")
- # L.DEBUG(f"Processing {hour_data['datetime']}")
-
+ await asyncio.sleep(0.01)
+ hour_data['datetime'] = await loc.dt(hour_data.get('datetimeEpoch'))
hour_preciptype_array = hour_data.get('preciptype', []) or []
hour_stations_array = hour_data.get('stations', []) or []
hourly_weather_params = (
@@ -204,21 +220,15 @@ async def store_weather_to_db(date_time: datetime, weather_data: dict):
try:
hourly_weather_query = '''
- INSERT INTO HourlyWeather (daily_weather_id, datetime, datetimeEpoch, temp, feelslike, humidity, dew, precip, precipprob,
+ INSERT INTO HourlyWeather (daily_weather_id, datetime, datetimeepoch, temp, feelslike, humidity, dew, precip, precipprob,
preciptype, snow, snowdepth, windgust, windspeed, winddir, pressure, cloudcover, visibility, solarradiation, solarenergy,
uvindex, severerisk, conditions, icon, stations, source)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26)
RETURNING id
'''
- # Debug logs for better insights
- # L.DEBUG("Executing query: %s", hourly_weather_query)
- # L.DEBUG("With parameters: %s", hourly_weather_params)
-
- # Execute the query to insert hourly weather data
async with conn.transaction():
hourly_weather_id = await conn.fetchval(hourly_weather_query, *hourly_weather_params)
- # L.ERR(f"\n{hourly_weather_id}")
-
+ L.DEBUG(f"Done processing hourly_weather_id {hourly_weather_id}")
except Exception as e:
L.ERR(f"EXCEPTION: {e}")
@@ -232,7 +242,7 @@ async def store_weather_to_db(date_time: datetime, weather_data: dict):
-async def get_weather_from_db(date_time: datetime, latitude: float, longitude: float):
+async def get_weather_from_db(date_time: dt_datetime, latitude: float, longitude: float):
L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our datetime in get_weather_from_db.")
async with DB.get_connection() as conn:
query_date = date_time.date()
@@ -246,7 +256,6 @@ async def get_weather_from_db(date_time: datetime, latitude: float, longitude: f
LIMIT 1
'''
-
daily_weather_record = await conn.fetchrow(query, query_date, longitude, latitude, longitude, latitude)
if daily_weather_record is None:
@@ -255,9 +264,14 @@ async def get_weather_from_db(date_time: datetime, latitude: float, longitude: f
# Convert asyncpg.Record to a mutable dictionary
daily_weather_data = dict(daily_weather_record)
-
+ # L.DEBUG(f"Daily weather data prior to tz corrections: {daily_weather_data}")
# Now we can modify the dictionary
- daily_weather_data['datetime'] = await loc.dt(daily_weather_data.get('datetime'))
+ # tz = await GEO.tz_at(latitude, longitude)
+ # daily_weather_data['datetime'] = await loc.dt(daily_weather_data.get('datetime'), tz)
+ # daily_weather_data['sunrise'] = await loc.dt(daily_weather_data.get('sunrise'), tz)
+ # daily_weather_data['sunset'] = await loc.dt(daily_weather_data.get('sunset'), tz)
+
+ # L.DEBUG(f"Daily weather data after tz corrections: {daily_weather_data}")
# Query to get hourly weather data
query = '''
@@ -270,9 +284,10 @@ async def get_weather_from_db(date_time: datetime, latitude: float, longitude: f
hourly_weather_data = []
for record in hourly_weather_records:
hour_data = dict(record)
- hour_data['datetime'] = await loc.dt(hour_data.get('datetime'))
+ # hour_data['datetime'] = await loc.dt(hour_data.get('datetime'), tz)
hourly_weather_data.append(hour_data)
+ L.DEBUG(f"Hourly weather data after tz corrections: {hourly_weather_data}")
day = {
'DailyWeather': daily_weather_data,
'HourlyWeather': hourly_weather_data,