Auto-update: Sat Jun 29 21:33:48 PDT 2024

This commit is contained in:
sanj 2024-06-29 21:33:48 -07:00
parent ad0ae30575
commit e4d6ec6412
4 changed files with 147 additions and 139 deletions

View file

@ -8,7 +8,7 @@ from contextlib import asynccontextmanager
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, Union, TypeVar, Type
from zoneinfo import ZoneInfo
import aiofiles
import aiohttp
import asyncpg
@ -311,11 +311,11 @@ class Geocoder:
else:
raise ValueError(f"Unsupported unit: {unit}")
async def timezone(self, lat: float, lon: float):
async def timezone(self, lat: float, lon: float) -> Optional[ZoneInfo]:
loop = asyncio.get_running_loop()
timezone = await loop.run_in_executor(self.executor, lambda: self.tf.timezone_at(lat=lat, lng=lon))
return timezone if timezone else 'Unknown'
timezone_str = await loop.run_in_executor(self.executor, lambda: self.tf.timezone_at(lat=lat, lng=lon))
return ZoneInfo(timezone_str) if timezone_str else None
async def lookup(self, lat: float, lon: float):
@ -443,7 +443,7 @@ class Geocoder:
def coords_equal(self, coord1: Tuple[float, float], coord2: Tuple[float, float], tolerance: float = 1e-5) -> bool:
return math.isclose(coord1[0], coord2[0], abs_tol=tolerance) and math.isclose(coord1[1], coord2[1], abs_tol=tolerance)
async def refresh_timezone(self, location: Union[Location, Tuple[float, float]], force: bool = False) -> str:
async def refresh_timezone(self, location: Union[Location, Tuple[float, float]], force: bool = False) -> Optional[ZoneInfo]:
if isinstance(location, Location):
lat, lon = location.latitude, location.longitude
else:
@ -457,6 +457,7 @@ class Geocoder:
current_time - self.last_update > timedelta(hours=1) or
not self.coords_equal(rounded_location, self.round_coords(*self.last_location) if self.last_location else (None, None))):
new_timezone = await self.timezone(lat, lon)
self.last_timezone = new_timezone
self.last_update = current_time
@ -465,9 +466,10 @@ class Geocoder:
return self.last_timezone
async def tz_save(self):
cache_data = {
'last_timezone': self.last_timezone,
'last_timezone': str(self.last_timezone) if self.last_timezone else None,
'last_update': self.last_update.isoformat() if self.last_update else None,
'last_location': self.last_location
}
@ -478,29 +480,31 @@ class Geocoder:
try:
async with aiofiles.open(self.cache_file, 'r') as f:
cache_data = json.loads(await f.read())
self.last_timezone = cache_data.get('last_timezone')
self.last_timezone = ZoneInfo(cache_data['last_timezone']) if cache_data.get('last_timezone') else None
self.last_update = datetime.fromisoformat(cache_data['last_update']) if cache_data.get('last_update') else None
self.last_location = tuple(cache_data['last_location']) if cache_data.get('last_location') else None
except (FileNotFoundError, json.JSONDecodeError):
# If file doesn't exist or is invalid, we'll start fresh
pass
self.last_timezone = None
self.last_update = None
self.last_location = None
async def tz_current(self, location: Union[Location, Tuple[float, float]]) -> str:
async def tz_current(self, location: Union[Location, Tuple[float, float]]) -> Optional[ZoneInfo]:
await self.tz_cached()
return await self.refresh_timezone(location)
async def tz_last(self) -> Optional[str]:
async def tz_last(self) -> Optional[ZoneInfo]:
await self.tz_cached()
return self.last_timezone
async def tz_at(self, lat: float, lon: float) -> str:
async def tz_at(self, lat: float, lon: float) -> Optional[ZoneInfo]:
"""
Get the timezone at a specific latitude and longitude without affecting the cache.
:param lat: Latitude
:param lon: Longitude
:return: Timezone string
:return: ZoneInfo object representing the timezone
"""
return await self.timezone(lat, lon)

View file

@ -19,63 +19,50 @@ from sijapi.utilities import haversine
loc = APIRouter()
async def dt(
date_time: Union[str, datetime],
date_time: Union[str, int, datetime],
tz: Union[str, ZoneInfo, None] = None
) -> datetime:
try:
# Convert integer (epoch time) to UTC datetime
if isinstance(date_time, int):
date_time = datetime.utcfromtimestamp(date_time).replace(tzinfo=timezone.utc)
L.DEBUG(f"Converted epoch time {date_time} to UTC datetime object.")
# Convert string to datetime if necessary
if isinstance(date_time, str):
elif isinstance(date_time, str):
date_time = dateutil_parse(date_time)
L.DEBUG(f"Converted string '{date_time}' to datetime object.")
if not isinstance(date_time, datetime):
raise ValueError("Input must be a string or datetime object.")
raise ValueError(f"Input must be a string, integer (epoch time), or datetime object. What we received: {date_time}, type {type(date_time)}")
# Ensure the datetime is timezone-aware (UTC if not specified)
if date_time.tzinfo is None:
date_time = date_time.replace(tzinfo=timezone.utc)
L.DEBUG("Added UTC timezone to naive datetime.")
# Handle provided timezone
if tz is not None:
if isinstance(tz, str):
if tz == "local":
last_loc = await get_timezone_without_timezone(date_time)
tz_str = GEO.tz(last_loc.latitude, last_loc.longitude)
try:
tz = ZoneInfo(tz_str)
except Exception as e:
L.WARN(f"Invalid timezone string '{tz_str}' from DynamicTZ. Falling back to UTC. Error: {e}")
tz = ZoneInfo('UTC')
tz = await GEO.tz_at(last_loc.latitude, last_loc.longitude)
L.DEBUG(f"Using local timezone: {tz}")
elif isinstance(tz, str):
else:
try:
tz = ZoneInfo(tz)
except Exception as e:
L.ERR(f"Invalid timezone string '{tz}'. Error: {e}")
raise ValueError(f"Invalid timezone string: {tz}")
elif isinstance(tz, ZoneInfo):
tz = tz # tz is already a ZoneInfo object
pass # tz is already a ZoneInfo object
else:
raise ValueError("tz must be 'local', a string, or a ZoneInfo object.")
raise ValueError(f"What we needed: tz == 'local', a string, or a ZoneInfo object. What we got: tz, a {type(tz)}, == {tz})")
# Localize to the provided or determined timezone
if date_time.tzinfo is None:
date_time = date_time.replace(tzinfo=tz)
L.DEBUG(f"Localized naive datetime to timezone: {tz}")
else:
# Convert to the provided or determined timezone
date_time = date_time.astimezone(tz)
L.DEBUG(f"Converted datetime from {date_time.tzinfo} to timezone: {tz}")
# If no timezone provided, only fill in missing timezone info
elif date_time.tzinfo is None:
#tz_str = await get_ti`mezone_without_timezone(date_time)
#try:
# tz = ZoneInfo(tz_str)
#except Exception as e:
# L.WARN(f"Invalid timezone string '{tz_str}' from Geocoder. Falling back to UTC. Error: {e}")
# tz = ZoneInfo('UTC')
tz = ZoneInfo('UTC') # force use of UTC when we can't find real timezone
date_time = date_time.replace(tzinfo=tz)
L.DEBUG(f"Filled in missing timezone info: {tz}")
else:
L.DEBUG(f"Datetime already has timezone {date_time.tzinfo}. No changes made.")
L.DEBUG(f"Converted datetime to timezone: {tz}")
return date_time
except ValueError as e:
@ -83,7 +70,8 @@ async def dt(
raise
except Exception as e:
L.ERR(f"Unexpected error in dt: {e}")
raise ValueError(f"Failed to localize datetime: {e}")
raise ValueError(f"Failed to process datetime: {e}")
async def get_timezone_without_timezone(date_time):
# This is a bit convoluted because we're trying to solve the paradox of needing to know the location in order to determine the timezone, but needing the timezone to be certain we've got the right location if this datetime coincided with inter-timezone travel. Our imperfect solution is to use UTC for an initial location query to determine roughly where we were at the time, get that timezone, then check the location again using that timezone, and if this location is different from the one using UTC, get the timezone again usng it, otherwise use the one we already sourced using UTC.

View file

@ -84,14 +84,14 @@ async def build_daily_note_endpoint(
date_str = dt_datetime.now().strftime("%Y-%m-%d")
if location:
lat, lon = map(float, location.split(','))
tz = GEO.tz_at(lat, lon)
tz = await GEO.tz_at(lat, lon)
date_time = dateutil_parse(date_str).replace(tzinfo=tz)
else:
raise ValueError("Location is not provided or invalid.")
except (ValueError, AttributeError, TypeError) as e:
L.WARN(f"Falling back to localized datetime due to error: {e}")
try:
date_time = loc.dt(date_str)
date_time = await loc.dt(date_str)
places = await loc.fetch_locations(date_time)
lat, lon = places[0].latitude, places[0].longitude
except Exception as e:
@ -358,14 +358,14 @@ async def generate_banner(dt, location: Location = None, forecast: str = None, m
async def note_weather_get(
date: str = Query(default="0", description="Enter a date in YYYY-MM-DD format, otherwise it will default to today."),
latlon: str = Query(default="45,-125"),
refresh: bool = Query(default=False, description="Set to true to refresh the weather data")
refresh: str = Query(default="False", description="Set to True to force refresh the weather data")
):
force_refresh_weather = refresh == "True"
try:
date_time = dt_datetime.now() if date == "0" else await loc.dt(date)
L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our dt_datetime in note_weather_get.")
L.DEBUG(f"date: {date} .. date_time: {date_time}")
content = await update_dn_weather(date_time) #, lat, lon)
content = await update_dn_weather(date_time, force_refresh_weather) #, lat, lon)
return JSONResponse(content={"forecast": content}, status_code=200)
except HTTPException as e:
@ -377,19 +377,20 @@ async def note_weather_get(
@note.post("/update/note/{date}")
async def post_update_daily_weather_and_calendar_and_timeslips(date: str) -> PlainTextResponse:
async def post_update_daily_weather_and_calendar_and_timeslips(date: str, refresh: str="False") -> PlainTextResponse:
date_time = await loc.dt(date)
L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our dt_datetime in post_update_daily_weather_and_calendar_and_timeslips.")
await update_dn_weather(date_time)
force_refresh_weather = refresh == "True"
await update_dn_weather(date_time, force_refresh_weather)
await update_daily_note_events(date_time)
await build_daily_timeslips(date_time)
return f"[Refresh]({API.URL}/update/note/{date_time.strftime('%Y-%m-%d')}"
async def update_dn_weather(date_time: dt_datetime, lat: float = None, lon: float = None):
async def update_dn_weather(date_time: dt_datetime, force_refresh: bool = False, lat: float = None, lon: float = None):
L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our datetime in update_dn_weather.")
try:
if lat and lon:
place = GEO.code((lat, lon))
place = await GEO.code((lat, lon))
else:
L.DEBUG(f"Updating weather for {date_time}")
@ -422,8 +423,8 @@ async def update_dn_weather(date_time: dt_datetime, lat: float = None, lon: floa
L.DEBUG(f"Journal path: absolute_path={absolute_path}, relative_path={relative_path}")
try:
L.DEBUG(f"passing date_time {date_time.strftime('%Y-%m-%d %H:%M:%S')}, {lat}/{lon} into fetch_and_store")
day = await weather.get_weather(date_time, lat, lon)
L.DEBUG(f"passing date_time {date_time.strftime('%Y-%m-%d %H:%M:%S')}, {lat}/{lon} into get_weather")
day = await weather.get_weather(date_time, lat, lon, force_refresh)
L.DEBUG(f"day information obtained from get_weather: {day}")
if day:
DailyWeather = day.get('DailyWeather')

View file

@ -2,12 +2,13 @@
Uses the VisualCrossing API and Postgres/PostGIS to source local weather forecasts and history.
'''
import asyncio
from fastapi import APIRouter, HTTPException
from fastapi import APIRouter, HTTPException, Query
from fastapi import HTTPException
from fastapi.responses import JSONResponse
from asyncpg.cursor import Cursor
from httpx import AsyncClient
from typing import Dict
from datetime import datetime
from datetime import datetime as dt_datetime
from shapely.wkb import loads
from binascii import unhexlify
from sijapi import L, VISUALCROSSING_API_KEY, TZ, DB, GEO
@ -16,13 +17,41 @@ from sijapi.routers import loc
weather = APIRouter()
@weather.get("/weather/refresh", response_class=JSONResponse)
async def get_refreshed_weather(
date: str = Query(default=dt_datetime.now().strftime("%Y-%m-%d"), description="Enter a date in YYYY-MM-DD format, otherwise it will default to today."),
latlon: str = Query(default="None", description="Optionally enter latitude and longitude in the format 45.8411,-123.1765; if not provided it will use your recorded location."),
):
# date = await date
try:
if latlon == "None":
date_time = await loc.dt(date)
place = await loc.fetch_last_location_before(date_time)
lat = place.latitude
lon = place.longitude
else:
lat, lon = latlon.split(',')
tz = await GEO.tz_at(lat, lon)
date_time = await loc.dt(date, tz)
async def get_weather(date_time: datetime, latitude: float, longitude: float):
# request_date_str = date_time.strftime("%Y-%m-%d")
L.DEBUG(f"passing date_time {date_time.strftime('%Y-%m-%d %H:%M:%S')}, {lat}/{lon} into get_weather")
day = await get_weather(date_time, lat, lon, force_refresh=True)
day_str = str(day)
return JSONResponse(content={"weather": day_str}, status_code=200)
except HTTPException as e:
return JSONResponse(content={"detail": str(e.detail)}, status_code=e.status_code)
except Exception as e:
L.ERR(f"Error in note_weather_get: {str(e)}")
raise HTTPException(status_code=500, detail=f"An error occurred: {str(e)}")
async def get_weather(date_time: dt_datetime, latitude: float, longitude: float, force_refresh: bool = False):
L.DEBUG(f"Called get_weather with lat: {latitude}, lon: {longitude}, date_time: {date_time}")
L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our datetime in get_weather.")
daily_weather_data = await get_weather_from_db(date_time, latitude, longitude)
L.WARN(f"Using {date_time} as our datetime in get_weather.")
fetch_new_data = True
if force_refresh == False:
daily_weather_data = await get_weather_from_db(date_time, latitude, longitude)
if daily_weather_data:
try:
L.DEBUG(f"Daily weather data from db: {daily_weather_data}")
@ -43,7 +72,7 @@ async def get_weather(date_time: datetime, latitude: float, longitude: float):
request_haversine = haversine(latitude, longitude, stored_lat, stored_lon)
L.DEBUG(f"\nINFO:\nlast updated {last_updated}\nstored lat: {stored_lat} - requested lat: {latitude}\nstored lon: {stored_lon} - requested lon: {longitude}\nHaversine: {request_haversine}")
if last_updated and (date_time <= datetime.now(TZ) and last_updated > date_time and request_haversine < 8) and hourly_weather and len(hourly_weather) > 0:
if last_updated and (date_time <= dt_datetime.now(TZ) and last_updated > date_time and request_haversine < 8) and hourly_weather and len(hourly_weather) > 0:
L.DEBUG(f"We can use existing data... :')")
fetch_new_data = False
@ -85,13 +114,12 @@ async def get_weather(date_time: datetime, latitude: float, longitude: float):
return daily_weather_data
async def store_weather_to_db(date_time: datetime, weather_data: dict):
async def store_weather_to_db(date_time: dt_datetime, weather_data: dict):
L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our datetime in store_weather_to_db")
async with DB.get_connection() as conn:
try:
day_data = weather_data.get('days')[0]
L.DEBUG(f"day_data.get('sunrise'): {day_data.get('sunrise')}")
L.DEBUG(f"RAW DAY_DATA: {day_data}")
# Handle preciptype and stations as PostgreSQL arrays
preciptype_array = day_data.get('preciptype', []) or []
stations_array = day_data.get('stations', []) or []
@ -102,17 +130,15 @@ async def store_weather_to_db(date_time: datetime, weather_data: dict):
# Get location details from weather data if available
longitude = weather_data.get('longitude')
latitude = weather_data.get('latitude')
tz = await GEO.tz_at(latitude, longitude)
elevation = await GEO.elevation(latitude, longitude)
location_point = f"POINTZ({longitude} {latitude} {elevation})" if longitude and latitude and elevation else None
# Correct for the datetime objects
L.WARN(f"Uncorrected datetime in store_weather_to_db: {day_data['datetime']}")
day_data['datetime'] = await loc.dt(day_data.get('datetime')) #day_data.get('datetime'))
L.WARN(f"Corrected datetime in store_weather_to_db with localized datetime: {day_data['datetime']}")
L.WARN(f"Uncorrected sunrise time in store_weather_to_db: {day_data['sunrise']}")
day_data['sunrise'] = day_data['datetime'].replace(hour=int(day_data.get('sunrise').split(':')[0]), minute=int(day_data.get('sunrise').split(':')[1]))
L.WARN(f"Corrected sunrise time in store_weather_to_db with localized datetime: {day_data['sunrise']}")
day_data['sunset'] = day_data['datetime'].replace(hour=int(day_data.get('sunset').split(':')[0]), minute=int(day_data.get('sunset').split(':')[1]))
L.WARN(f"Uncorrected datetimes in store_weather_to_db: {day_data['datetime']}, sunrise: {day_data['sunrise']}, sunset: {day_data['sunset']}")
day_data['datetime'] = await loc.dt(day_data.get('datetimeEpoch'))
day_data['sunrise'] = await loc.dt(day_data.get('sunriseEpoch'))
day_data['sunset'] = await loc.dt(day_data.get('sunsetEpoch'))
L.WARN(f"Corrected datetimes in store_weather_to_db: {day_data['datetime']}, sunrise: {day_data['sunrise']}, sunset: {day_data['sunset']}")
daily_weather_params = (
day_data.get('sunrise'), day_data.get('sunriseEpoch'),
@ -120,7 +146,7 @@ async def store_weather_to_db(date_time: datetime, weather_data: dict):
day_data.get('description'), day_data.get('tempmax'),
day_data.get('tempmin'), day_data.get('uvindex'),
day_data.get('winddir'), day_data.get('windspeed'),
day_data.get('icon'), datetime.now(),
day_data.get('icon'), dt_datetime.now(tz),
day_data.get('datetime'), day_data.get('datetimeEpoch'),
day_data.get('temp'), day_data.get('feelslikemax'),
day_data.get('feelslikemin'), day_data.get('feelslike'),
@ -141,9 +167,9 @@ async def store_weather_to_db(date_time: datetime, weather_data: dict):
try:
daily_weather_query = '''
INSERT INTO DailyWeather (
sunrise, sunriseEpoch, sunset, sunsetEpoch, description,
sunrise, sunriseepoch, sunset, sunsetepoch, description,
tempmax, tempmin, uvindex, winddir, windspeed, icon, last_updated,
datetime, datetimeEpoch, temp, feelslikemax, feelslikemin, feelslike,
datetime, datetimeepoch, temp, feelslikemax, feelslikemin, feelslike,
dew, humidity, precip, precipprob, precipcover, preciptype,
snow, snowdepth, windgust, pressure, cloudcover, visibility,
solarradiation, solarenergy, severerisk, moonphase, conditions,
@ -152,25 +178,15 @@ async def store_weather_to_db(date_time: datetime, weather_data: dict):
RETURNING id
'''
# Debug logs for better insights
# L.DEBUG("Executing query: %s", daily_weather_query)
# L.DEBUG("With parameters: %s", daily_weather_params)
# Execute the query to insert daily weather data
async with conn.transaction():
daily_weather_id = await conn.fetchval(daily_weather_query, *daily_weather_params)
if 'hours' in day_data:
L.DEBUG(f"Processing hours now...")
for hour_data in day_data['hours']:
try:
await asyncio.sleep(0.1)
# hour_data['datetime'] = parse_date(hour_data.get('datetime'))
hour_timestamp = date_str + ' ' + hour_data['datetime']
hour_data['datetime'] = await loc.dt(hour_timestamp)
L.DEBUG(f"Processing hours now...")
# L.DEBUG(f"Processing {hour_data['datetime']}")
await asyncio.sleep(0.01)
hour_data['datetime'] = await loc.dt(hour_data.get('datetimeEpoch'))
hour_preciptype_array = hour_data.get('preciptype', []) or []
hour_stations_array = hour_data.get('stations', []) or []
hourly_weather_params = (
@ -204,21 +220,15 @@ async def store_weather_to_db(date_time: datetime, weather_data: dict):
try:
hourly_weather_query = '''
INSERT INTO HourlyWeather (daily_weather_id, datetime, datetimeEpoch, temp, feelslike, humidity, dew, precip, precipprob,
INSERT INTO HourlyWeather (daily_weather_id, datetime, datetimeepoch, temp, feelslike, humidity, dew, precip, precipprob,
preciptype, snow, snowdepth, windgust, windspeed, winddir, pressure, cloudcover, visibility, solarradiation, solarenergy,
uvindex, severerisk, conditions, icon, stations, source)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26)
RETURNING id
'''
# Debug logs for better insights
# L.DEBUG("Executing query: %s", hourly_weather_query)
# L.DEBUG("With parameters: %s", hourly_weather_params)
# Execute the query to insert hourly weather data
async with conn.transaction():
hourly_weather_id = await conn.fetchval(hourly_weather_query, *hourly_weather_params)
# L.ERR(f"\n{hourly_weather_id}")
L.DEBUG(f"Done processing hourly_weather_id {hourly_weather_id}")
except Exception as e:
L.ERR(f"EXCEPTION: {e}")
@ -232,7 +242,7 @@ async def store_weather_to_db(date_time: datetime, weather_data: dict):
async def get_weather_from_db(date_time: datetime, latitude: float, longitude: float):
async def get_weather_from_db(date_time: dt_datetime, latitude: float, longitude: float):
L.WARN(f"Using {date_time.strftime('%Y-%m-%d %H:%M:%S')} as our datetime in get_weather_from_db.")
async with DB.get_connection() as conn:
query_date = date_time.date()
@ -246,7 +256,6 @@ async def get_weather_from_db(date_time: datetime, latitude: float, longitude: f
LIMIT 1
'''
daily_weather_record = await conn.fetchrow(query, query_date, longitude, latitude, longitude, latitude)
if daily_weather_record is None:
@ -255,9 +264,14 @@ async def get_weather_from_db(date_time: datetime, latitude: float, longitude: f
# Convert asyncpg.Record to a mutable dictionary
daily_weather_data = dict(daily_weather_record)
# L.DEBUG(f"Daily weather data prior to tz corrections: {daily_weather_data}")
# Now we can modify the dictionary
daily_weather_data['datetime'] = await loc.dt(daily_weather_data.get('datetime'))
# tz = await GEO.tz_at(latitude, longitude)
# daily_weather_data['datetime'] = await loc.dt(daily_weather_data.get('datetime'), tz)
# daily_weather_data['sunrise'] = await loc.dt(daily_weather_data.get('sunrise'), tz)
# daily_weather_data['sunset'] = await loc.dt(daily_weather_data.get('sunset'), tz)
# L.DEBUG(f"Daily weather data after tz corrections: {daily_weather_data}")
# Query to get hourly weather data
query = '''
@ -270,9 +284,10 @@ async def get_weather_from_db(date_time: datetime, latitude: float, longitude: f
hourly_weather_data = []
for record in hourly_weather_records:
hour_data = dict(record)
hour_data['datetime'] = await loc.dt(hour_data.get('datetime'))
# hour_data['datetime'] = await loc.dt(hour_data.get('datetime'), tz)
hourly_weather_data.append(hour_data)
L.DEBUG(f"Hourly weather data after tz corrections: {hourly_weather_data}")
day = {
'DailyWeather': daily_weather_data,
'HourlyWeather': hourly_weather_data,