update to a right mess for fetching data and doing some sanity checks

weather/ingest.py

@@ -0,0 +1,116 @@
+import os
+import json
+import requests
+from datetime import datetime
+
+from weather.db import get_db
+
+def fetchHourlyForecasts():
+    try:
+        remote_url = 'https://api.weather.gov/gridpoints/SLC/106,146/forecast/hourly'
+        headers = {
+            "User-Agent": "Brian's Python API test (captbrogers@gmail.com)"
+        }
+        response = requests.get(remote_url, headers=headers)
+        response.raise_for_status()
+
+        current_date = datetime.now()
+        if not os.path.exists(f"../data/{current_date.strftime('%Y-%m-%d')}"):
+            os.makedirs(f"../data/{current_date.strftime('%Y-%m-%d')}", exist_ok=True)
+
+        hourly_filename = f"../data/{current_date.strftime('%Y-%m-%d')}/hourly_{current_date.strftime('%H')}.json"
+        with open(hourly_filename, 'w') as json_file:
+            json.dump(response.json(), json_file, indent=4)
+
+
+        db = get_db()
+        insert_sql = """
+            INSERT INTO `current_forecasts`
+            (
+                `start_time`,
+                `end_time`,
+                `is_daytime`,
+                `temperature`,
+                `precipitation_probability`,
+                `relative_humidity`,
+                `wind_speed`,
+                `wind_direction`,
+                `icon_url`,
+                `short_forecast`,
+                `detailed_forecast`,
+                `created_at`
+            ) VALUES (
+                ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
+            )
+        """
+
+        for period_report in response.json()['properties']['periods']:
+            # TODO: this should be a transaction with rollback, pretty hacky right now
+            db.execute(insert_sql, (
+                datetime.strptime(period_report['startTime'], "%Y-%m-%dT%H:%M:%S%z").strftime("%Y-%m-%d %H:%M:%S"),
+                datetime.strptime(period_report['endTime'], "%Y-%m-%dT%H:%M:%S%z").strftime("%Y-%m-%d %H:%M:%S"),
+                period_report['isDaytime'],
+                period_report['temperature'],
+                period_report['probabilityOfPrecipitation']['value'],
+                period_report['relativeHumidity']['value'],
+                period_report['windSpeed'],
+                period_report['windDirection'],
+                period_report['icon'],
+                period_report['shortForecast'],
+                period_report['detailedForecast'],
+                datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+            ))
+
+
+    except requests.exceptions.RequestException as e:
+        return str(e)
+
+
+
+def fetchDailyForecasts():
+    try:
+        remote_url = 'https://api.weather.gov/gridpoints/SLC/106,146/forecast'
+        headers = {
+            "User-Agent": "Brian's Python API test (captbrogers@gmail.com)"
+        }
+        response = requests.get(remote_url, headers=headers)
+        response.raise_for_status()
+
+        current_date = datetime.now()
+        if not os.path.exists(f"../data/{current_date.strftime('%Y-%m-%d')}"):
+            os.makedirs(f"../data/{current_date.strftime('%Y-%m-%d')}", exist_ok=True)
+        
+        daily_filename = f"../data/{current_date.strftime('%Y-%m-%d')}/daily_{current_date.strftime('%H')}.json"
+        with open(daily_filename, 'w') as json_file:
+            json.dump(response.json(), json_file, indent=4)
+
+        db = get_db()
+        insert_sql = """
+            INSERT INTO `daily_forecasts`
+            (
+                `forecasted_date`,
+                `temperature_high`,
+                `precipitation_probability`,
+                `icon_url`,
+                `short_forecast`,
+                `created_at`
+            ) VALUES (
+                ?, ?, ?, ?, ?, ?
+            )
+        """
+
+        for period_report in response.json()['properties']['periods']:
+            if period_report['isDaytime']:
+                # TODO: this should be a transaction with rollback
+                db.execute(insert_sql, (
+                    datetime.strptime(period_report['startTime'], "%Y-%m-%dT%H:%M:%S%z").strftime("%Y-%m-%d %H:%M:%S"),
+                    period_report['temperature'],
+                    period_report['probabilityOfPrecipitation']['value'],
+                    period_report['icon'],
+                    period_report['shortForecast'],
+                    datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+                ))
+
+
+    except requests.exceptions.RequestException as e:
+        return str(e)

weather/schema.sql

@@ -1,17 +1,13 @@
-DROP TABLE IF EXISTS periods;
-DROP TABLE IF EXISTS reports;
+DROP TABLE IF EXISTS current_forecasts;
+DROP TABLE IF EXISTS daily_forecasts;
 
-CREATE TABLE "periods" (
+CREATE TABLE "current_forecasts" (
     "id" Integer NOT NULL PRIMARY KEY AUTOINCREMENT,
-    "report_id" Integer NOT NULL,
-    "period_number" Integer NOT NULL,
     "start_time" DateTime NOT NULL,
     "end_time" DateTime NOT NULL,
     "is_daytime" Integer NOT NULL,
     "temperature" Integer NOT NULL,
-    "temperature_unit" Text NOT NULL DEFAULT 'F',
     "precipitation_probability" Integer,
-    "dewpoint_celsius" Numeric,
     "relative_humidity" Integer,
     "wind_speed" Text,
     "wind_direction" Text,
@@ -19,24 +15,17 @@ CREATE TABLE "periods" (
     "short_forecast" Text,
     "detailed_forecast" Text,
     "created_at" DateTime,
-    "updated_at" DateTime,
-    CONSTRAINT "periods_reports_CASCADE_NO ACTION_report_id_id_0" FOREIGN KEY ( "report_id" ) REFERENCES "reports"( "id" )
-        ON DELETE Cascade
- );
+    "updated_at" DateTime
+);
 
-CREATE INDEX "periods_report_id_start_time_index" ON "periods"( "report_id", "start_time" );
-
-CREATE TABLE "reports" (
+CREATE TABLE "daily_forecasts" (
     "id" Integer NOT NULL PRIMARY KEY AUTOINCREMENT,
-    "type" Text NOT NULL,
-    "reported_at" DateTime NOT NULL,
-    "generated_at" DateTime NOT NULL,
-    "latitude" Numeric NOT NULL,
-    "longitude" Numeric NOT NULL,
-    "elevation_meters" Numeric,
+    "forecasted_date" DateTime NOT NULL,
+    "temperature_high" Integer NOT NULL,
+    "precipitation_probability" Integer,
+    "icon_url" Text,
+    "short_forecast" Text,
     "created_at" DateTime,
-    "updated_at" DateTime,
-CONSTRAINT "check ""type"" in ('hourly', 'weekly')" CHECK ("type" in ('hourly', 'weekly')) );
+    "updated_at" DateTime
+);
 
-CREATE INDEX "reports_type_reported_at_index" ON "reports"( "type", "reported_at" );
-CREATE UNIQUE INDEX "reports_type_reported_at_unique" ON "reports"( "type", "reported_at" );

weather/weather.py

@@ -5,41 +5,58 @@ from werkzeug.exceptions import abort
 from datetime import datetime
 
 from weather.db import get_db
+from weather.ingest import (
+    fetchHourlyForecasts, fetchDailyForecasts
+)
 
 bp = Blueprint('weather', __name__)
 
 @bp.route('/')
 def index():
     db = get_db()
-    current_conditions = dict(db.execute(
-        f"SELECT * FROM `reports` WHERE `type` = 'hourly' ORDER BY `end_time` DESC LIMIT 1"
-    ).fetchone())
+    current_conditions = db.execute(
+        "SELECT * FROM `current_forecasts` ORDER BY `end_time` DESC LIMIT 1"
+    ).fetchone()
+
+    if current_conditions is None:
+        fetchHourlyForecasts()
+        current_conditions = db.execute(
+            "SELECT * FROM `current_forecasts` ORDER BY `end_time` DESC LIMIT 1"
+        ).fetchone()
 
-    stale_datetime = datetime.strptime(current_conditions['end_time'], "%Y-%m-%d %H:%M:%S")
-    if datetime.now() > stale_datetime:
-        # fetch new data
-        # save new data
 
-    # TODO: add conditions to check for day/night
     day_or_night = 'day'
     if current_conditions['is_daytime']:
         day_or_night = 'night'
 
     condition = mapForecastToImage(current_conditions['short_forecast'])
-    condition_image = f"images/{time_of_day}_{condition}.jpg"
+    condition_image = f"images/{day_or_night}_{condition}.jpg"
 
-    hourly_reports = db.execute(
-        'SELECT *'
-        '  FROM `reports`'
-        '  ORDER BY `id` DESC'
-        '  LIMIT 7'
+    hourly_conditions = db.execute(
+        'SELECT * FROM `current_forecasts` WHERE `start_time` > DATETIME("now", "+1 hour") LIMIT 6'
     ).fetchall()
+    if len(hourly_conditions) < 6:
+        fetchHourlyForecasts()
+        hourly_conditions = db.execute(
+            'SELECT * FROM `current_forecasts` WHERE `start_time` > DATETIME("now", "+1 hour") LIMIT 6'
+        ).fetchall()
+
+
+    week_forcasts = db.execute(
+        'SELECT * FROM `daily_forecasts` WHERE `forecasted_date` > DATETIME("now") LIMIT 7'
+    ).fetchall()
+    if len(week_forcasts) < 7:
+        fetchDailyForecasts()
+        week_forcasts = db.execute(
+            'SELECT * FROM `current_forecasts` WHERE `start_time` > DATETIME("now", "+1 hour") LIMIT 7'
+        ).fetchall()
 
     return render_template(
         'weather/index.html',
-        current_conditions=current_conditions,
         condition_image=condition_image,
-        periods=periods
+        current_conditions=current_conditions,
+        hourly_conditions=hourly_conditions,
+        week_forcasts=week_forcasts
     )
 
 def mapForecastToImage(condition: str):
@@ -56,7 +73,7 @@ def mapForecastToImage(condition: str):
     elif 'wind' in condition:
         return 'windy'
     elif 'sunny' in condition or 'clear' in condition:
-        return 'sunny'
+        return 'clear'
     elif 'cloud' in condition or 'overcast' in condition:
         return 'cloudy'
     elif 'fog' in condition or 'mist' in condition:
@@ -64,3 +81,4 @@ def mapForecastToImage(condition: str):
     else:
         return 'cloudy'
 
+