Files
flask-weather/weather/ingest.py

117 lines
4.3 KiB
Python

import os
import json
import requests
from datetime import datetime
from weather.db import get_db
def fetchHourlyForecasts():
try:
remote_url = 'https://api.weather.gov/gridpoints/SLC/106,146/forecast/hourly'
headers = {
"User-Agent": "Brian's Python API test (captbrogers@gmail.com)"
}
response = requests.get(remote_url, headers=headers)
response.raise_for_status()
current_date = datetime.now()
if not os.path.exists(f"../data/{current_date.strftime('%Y-%m-%d')}"):
os.makedirs(f"../data/{current_date.strftime('%Y-%m-%d')}", exist_ok=True)
hourly_filename = f"../data/{current_date.strftime('%Y-%m-%d')}/hourly_{current_date.strftime('%H')}.json"
with open(hourly_filename, 'w') as json_file:
json.dump(response.json(), json_file, indent=4)
db = get_db()
insert_sql = """
INSERT INTO `current_forecasts`
(
`start_time`,
`end_time`,
`is_daytime`,
`temperature`,
`precipitation_probability`,
`relative_humidity`,
`wind_speed`,
`wind_direction`,
`icon_url`,
`short_forecast`,
`detailed_forecast`,
`created_at`
) VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)
"""
for period_report in response.json()['properties']['periods']:
# TODO: this should be a transaction with rollback, pretty hacky right now
db.execute(insert_sql, (
datetime.strptime(period_report['startTime'], "%Y-%m-%dT%H:%M:%S%z").strftime("%Y-%m-%d %H:%M:%S"),
datetime.strptime(period_report['endTime'], "%Y-%m-%dT%H:%M:%S%z").strftime("%Y-%m-%d %H:%M:%S"),
period_report['isDaytime'],
period_report['temperature'],
period_report['probabilityOfPrecipitation']['value'],
period_report['relativeHumidity']['value'],
period_report['windSpeed'],
period_report['windDirection'],
period_report['icon'],
period_report['shortForecast'],
period_report['detailedForecast'],
datetime.now().strftime("%Y-%m-%d %H:%M:%S")
))
except requests.exceptions.RequestException as e:
return str(e)
def fetchDailyForecasts():
try:
remote_url = 'https://api.weather.gov/gridpoints/SLC/106,146/forecast'
headers = {
"User-Agent": "Brian's Python API test (captbrogers@gmail.com)"
}
response = requests.get(remote_url, headers=headers)
response.raise_for_status()
current_date = datetime.now()
if not os.path.exists(f"../data/{current_date.strftime('%Y-%m-%d')}"):
os.makedirs(f"../data/{current_date.strftime('%Y-%m-%d')}", exist_ok=True)
daily_filename = f"../data/{current_date.strftime('%Y-%m-%d')}/daily_{current_date.strftime('%H')}.json"
with open(daily_filename, 'w') as json_file:
json.dump(response.json(), json_file, indent=4)
db = get_db()
insert_sql = """
INSERT INTO `daily_forecasts`
(
`forecasted_date`,
`temperature_high`,
`precipitation_probability`,
`icon_url`,
`short_forecast`,
`created_at`
) VALUES (
?, ?, ?, ?, ?, ?
)
"""
for period_report in response.json()['properties']['periods']:
if period_report['isDaytime']:
# TODO: this should be a transaction with rollback
db.execute(insert_sql, (
datetime.strptime(period_report['startTime'], "%Y-%m-%dT%H:%M:%S%z").strftime("%Y-%m-%d %H:%M:%S"),
period_report['temperature'],
period_report['probabilityOfPrecipitation']['value'],
period_report['icon'],
period_report['shortForecast'],
datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
))
except requests.exceptions.RequestException as e:
return str(e)