ingest data from national weather service

2026-01-12 14:10:30 -07:00
parent f9bcdf6ef9
commit 331d2757bf
1 changed files with 116 additions and 0 deletions
--- a/weather/ingest.py
+++ b/weather/ingest.py
@@ -0,0 +1,116 @@
+import os
+import json
+import requests
+from datetime import datetime
+
+from weather.db import get_db
+
+def fetchHourlyForecasts():
+    try:
+        remote_url = 'https://api.weather.gov/gridpoints/SLC/106,146/forecast/hourly'
+        headers = {
+            "User-Agent": "Brian's Python API test (captbrogers@gmail.com)"
+        }
+        response = requests.get(remote_url, headers=headers)
+        response.raise_for_status()
+
+        current_date = datetime.now()
+        if not os.path.exists(f"../data/{current_date.strftime('%Y-%m-%d')}"):
+            os.makedirs(f"../data/{current_date.strftime('%Y-%m-%d')}", exist_ok=True)
+
+        hourly_filename = f"../data/{current_date.strftime('%Y-%m-%d')}/hourly_{current_date.strftime('%H')}.json"
+        with open(hourly_filename, 'w') as json_file:
+            json.dump(response.json(), json_file, indent=4)
+
+
+        db = get_db()
+        insert_sql = """
+            INSERT INTO `current_forecasts`
+            (
+                `start_time`,
+                `end_time`,
+                `is_daytime`,
+                `temperature`,
+                `precipitation_probability`,
+                `relative_humidity`,
+                `wind_speed`,
+                `wind_direction`,
+                `icon_url`,
+                `short_forecast`,
+                `detailed_forecast`,
+                `created_at`
+            ) VALUES (
+                ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
+            )
+        """
+
+        for period_report in response.json()['properties']['periods']:
+            # TODO: this should be a transaction with rollback, pretty hacky right now
+            db.execute(insert_sql, (
+                datetime.strptime(period_report['startTime'], "%Y-%m-%dT%H:%M:%S%z").strftime("%Y-%m-%d %H:%M:%S"),
+                datetime.strptime(period_report['endTime'], "%Y-%m-%dT%H:%M:%S%z").strftime("%Y-%m-%d %H:%M:%S"),
+                period_report['isDaytime'],
+                period_report['temperature'],
+                period_report['probabilityOfPrecipitation']['value'],
+                period_report['relativeHumidity']['value'],
+                period_report['windSpeed'],
+                period_report['windDirection'],
+                period_report['icon'],
+                period_report['shortForecast'],
+                period_report['detailedForecast'],
+                datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+            ))
+
+
+    except requests.exceptions.RequestException as e:
+        return str(e)
+
+
+
+def fetchDailyForecasts():
+    try:
+        remote_url = 'https://api.weather.gov/gridpoints/SLC/106,146/forecast'
+        headers = {
+            "User-Agent": "Brian's Python API test (captbrogers@gmail.com)"
+        }
+        response = requests.get(remote_url, headers=headers)
+        response.raise_for_status()
+
+        current_date = datetime.now()
+        if not os.path.exists(f"../data/{current_date.strftime('%Y-%m-%d')}"):
+            os.makedirs(f"../data/{current_date.strftime('%Y-%m-%d')}", exist_ok=True)
+        
+        daily_filename = f"../data/{current_date.strftime('%Y-%m-%d')}/daily_{current_date.strftime('%H')}.json"
+        with open(daily_filename, 'w') as json_file:
+            json.dump(response.json(), json_file, indent=4)
+
+        db = get_db()
+        insert_sql = """
+            INSERT INTO `daily_forecasts`
+            (
+                `forecasted_date`,
+                `temperature_high`,
+                `precipitation_probability`,
+                `icon_url`,
+                `short_forecast`,
+                `created_at`
+            ) VALUES (
+                ?, ?, ?, ?, ?, ?
+            )
+        """
+
+        for period_report in response.json()['properties']['periods']:
+            if period_report['isDaytime']:
+                # TODO: this should be a transaction with rollback
+                db.execute(insert_sql, (
+                    datetime.strptime(period_report['startTime'], "%Y-%m-%dT%H:%M:%S%z").strftime("%Y-%m-%d %H:%M:%S"),
+                    period_report['temperature'],
+                    period_report['probabilityOfPrecipitation']['value'],
+                    period_report['icon'],
+                    period_report['shortForecast'],
+                    datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+                ))
+
+
+    except requests.exceptions.RequestException as e:
+        return str(e)