diff --git a/weather/ingest.py b/weather/ingest.py index ee5929f..32f75ab 100644 --- a/weather/ingest.py +++ b/weather/ingest.py @@ -15,15 +15,16 @@ def fetchHourlyForecasts(): response.raise_for_status() current_date = datetime.now() - if not os.path.exists(f"../data/{current_date.strftime('%Y-%m-%d')}"): - os.makedirs(f"../data/{current_date.strftime('%Y-%m-%d')}", exist_ok=True) + if not os.path.exists(f"data/{current_date.strftime('%Y-%m-%d')}"): + os.makedirs(f"data/{current_date.strftime('%Y-%m-%d')}", exist_ok=True) - hourly_filename = f"../data/{current_date.strftime('%Y-%m-%d')}/hourly_{current_date.strftime('%H')}.json" + hourly_filename = f"data/{current_date.strftime('%Y-%m-%d')}/hourly_{current_date.strftime('%H')}.json" with open(hourly_filename, 'w') as json_file: json.dump(response.json(), json_file, indent=4) db = get_db() + cursor = db.cursor() insert_sql = """ INSERT INTO `current_forecasts` ( @@ -44,24 +45,25 @@ def fetchHourlyForecasts(): ) """ - for period_report in response.json()['properties']['periods']: + for forecast in response.json()['properties']['periods']: # TODO: this should be a transaction with rollback, pretty hacky right now - db.execute(insert_sql, ( - datetime.strptime(period_report['startTime'], "%Y-%m-%dT%H:%M:%S%z").strftime("%Y-%m-%d %H:%M:%S"), - datetime.strptime(period_report['endTime'], "%Y-%m-%dT%H:%M:%S%z").strftime("%Y-%m-%d %H:%M:%S"), - period_report['isDaytime'], - period_report['temperature'], - period_report['probabilityOfPrecipitation']['value'], - period_report['relativeHumidity']['value'], - period_report['windSpeed'], - period_report['windDirection'], - period_report['icon'], - period_report['shortForecast'], - period_report['detailedForecast'], + cursor.execute(insert_sql, ( + datetime.strptime(forecast['startTime'], "%Y-%m-%dT%H:%M:%S%z").strftime("%Y-%m-%d %H:%M:%S"), + datetime.strptime(forecast['endTime'], "%Y-%m-%dT%H:%M:%S%z").strftime("%Y-%m-%d %H:%M:%S"), + forecast['isDaytime'], + forecast['temperature'], + forecast['probabilityOfPrecipitation']['value'], + forecast['relativeHumidity']['value'], + forecast['windSpeed'], + forecast['windDirection'], + forecast['icon'], + forecast['shortForecast'], + forecast['detailedForecast'], datetime.now().strftime("%Y-%m-%d %H:%M:%S") )) + db.commit() - + close_db() except requests.exceptions.RequestException as e: return str(e) @@ -77,14 +79,15 @@ def fetchDailyForecasts(): response.raise_for_status() current_date = datetime.now() - if not os.path.exists(f"../data/{current_date.strftime('%Y-%m-%d')}"): - os.makedirs(f"../data/{current_date.strftime('%Y-%m-%d')}", exist_ok=True) - - daily_filename = f"../data/{current_date.strftime('%Y-%m-%d')}/daily_{current_date.strftime('%H')}.json" + if not os.path.exists(f"data/{current_date.strftime('%Y-%m-%d')}"): + os.makedirs(f"data/{current_date.strftime('%Y-%m-%d')}", exist_ok=True) + + daily_filename = f"data/{current_date.strftime('%Y-%m-%d')}/daily_{current_date.strftime('%H')}.json" with open(daily_filename, 'w') as json_file: json.dump(response.json(), json_file, indent=4) db = get_db() + cursor = db.cursor() insert_sql = """ INSERT INTO `daily_forecasts` ( @@ -99,18 +102,19 @@ def fetchDailyForecasts(): ) """ - for period_report in response.json()['properties']['periods']: - if period_report['isDaytime']: + for forecast in response.json()['properties']['periods']: + if forecast['isDaytime']: # TODO: this should be a transaction with rollback - db.execute(insert_sql, ( - datetime.strptime(period_report['startTime'], "%Y-%m-%dT%H:%M:%S%z").strftime("%Y-%m-%d %H:%M:%S"), - period_report['temperature'], - period_report['probabilityOfPrecipitation']['value'], - period_report['icon'], - period_report['shortForecast'], + cursor.execute(insert_sql, ( + datetime.strptime(forecast['startTime'], "%Y-%m-%dT%H:%M:%S%z").strftime("%Y-%m-%d %H:%M:%S"), + forecast['temperature'], + forecast['probabilityOfPrecipitation']['value'], + forecast['icon'], + forecast['shortForecast'], datetime.now().strftime("%Y-%m-%d %H:%M:%S"), )) + db.commit() - + close_db() except requests.exceptions.RequestException as e: return str(e)