diff --git a/weather/ingest.py b/weather/ingest.py new file mode 100644 index 0000000..ee5929f --- /dev/null +++ b/weather/ingest.py @@ -0,0 +1,116 @@ +import os +import json +import requests +from datetime import datetime + +from weather.db import get_db + +def fetchHourlyForecasts(): + try: + remote_url = 'https://api.weather.gov/gridpoints/SLC/106,146/forecast/hourly' + headers = { + "User-Agent": "Brian's Python API test (captbrogers@gmail.com)" + } + response = requests.get(remote_url, headers=headers) + response.raise_for_status() + + current_date = datetime.now() + if not os.path.exists(f"../data/{current_date.strftime('%Y-%m-%d')}"): + os.makedirs(f"../data/{current_date.strftime('%Y-%m-%d')}", exist_ok=True) + + hourly_filename = f"../data/{current_date.strftime('%Y-%m-%d')}/hourly_{current_date.strftime('%H')}.json" + with open(hourly_filename, 'w') as json_file: + json.dump(response.json(), json_file, indent=4) + + + db = get_db() + insert_sql = """ + INSERT INTO `current_forecasts` + ( + `start_time`, + `end_time`, + `is_daytime`, + `temperature`, + `precipitation_probability`, + `relative_humidity`, + `wind_speed`, + `wind_direction`, + `icon_url`, + `short_forecast`, + `detailed_forecast`, + `created_at` + ) VALUES ( + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ) + """ + + for period_report in response.json()['properties']['periods']: + # TODO: this should be a transaction with rollback, pretty hacky right now + db.execute(insert_sql, ( + datetime.strptime(period_report['startTime'], "%Y-%m-%dT%H:%M:%S%z").strftime("%Y-%m-%d %H:%M:%S"), + datetime.strptime(period_report['endTime'], "%Y-%m-%dT%H:%M:%S%z").strftime("%Y-%m-%d %H:%M:%S"), + period_report['isDaytime'], + period_report['temperature'], + period_report['probabilityOfPrecipitation']['value'], + period_report['relativeHumidity']['value'], + period_report['windSpeed'], + period_report['windDirection'], + period_report['icon'], + period_report['shortForecast'], + period_report['detailedForecast'], + datetime.now().strftime("%Y-%m-%d %H:%M:%S") + )) + + + except requests.exceptions.RequestException as e: + return str(e) + + + +def fetchDailyForecasts(): + try: + remote_url = 'https://api.weather.gov/gridpoints/SLC/106,146/forecast' + headers = { + "User-Agent": "Brian's Python API test (captbrogers@gmail.com)" + } + response = requests.get(remote_url, headers=headers) + response.raise_for_status() + + current_date = datetime.now() + if not os.path.exists(f"../data/{current_date.strftime('%Y-%m-%d')}"): + os.makedirs(f"../data/{current_date.strftime('%Y-%m-%d')}", exist_ok=True) + + daily_filename = f"../data/{current_date.strftime('%Y-%m-%d')}/daily_{current_date.strftime('%H')}.json" + with open(daily_filename, 'w') as json_file: + json.dump(response.json(), json_file, indent=4) + + db = get_db() + insert_sql = """ + INSERT INTO `daily_forecasts` + ( + `forecasted_date`, + `temperature_high`, + `precipitation_probability`, + `icon_url`, + `short_forecast`, + `created_at` + ) VALUES ( + ?, ?, ?, ?, ?, ? + ) + """ + + for period_report in response.json()['properties']['periods']: + if period_report['isDaytime']: + # TODO: this should be a transaction with rollback + db.execute(insert_sql, ( + datetime.strptime(period_report['startTime'], "%Y-%m-%dT%H:%M:%S%z").strftime("%Y-%m-%d %H:%M:%S"), + period_report['temperature'], + period_report['probabilityOfPrecipitation']['value'], + period_report['icon'], + period_report['shortForecast'], + datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + )) + + + except requests.exceptions.RequestException as e: + return str(e)