diff --git a/.gitignore b/.gitignore index c32beb1..35eaf1c 100644 --- a/.gitignore +++ b/.gitignore @@ -169,3 +169,6 @@ data/silver/weather_forecasts/*.nc data/geo/*.nc .vscode/settings.json data/bronze/observations/*.csv.gz + +#Docker volumes +postgres/postgres_volume/* \ No newline at end of file diff --git a/data/gold/our_tempo_prediction.csv b/data/gold/our_tempo_prediction.csv index 29b71a3..aa6df61 100644 --- a/data/gold/our_tempo_prediction.csv +++ b/data/gold/our_tempo_prediction.csv @@ -1,49 +1,84 @@ ,Type_de_jour_TEMPO,our_tempo_J-1,our_tempo_J-2,our_tempo_J-3 -2024-09-01,BLUE,prediction_bleu,, -2024-09-02,BLUE,prediction_bleu,, -2024-09-03,BLUE,prediction_bleu,, -2024-09-04,BLUE,prediction_bleu,, -2024-09-05,BLUE,prediction_bleu,, -2024-09-06,BLUE,prediction_bleu,, -2024-09-07,BLUE,prediction_bleu,, -2024-09-08,BLUE,prediction_bleu,, -2024-09-09,BLUE,prediction_bleu,, -2024-09-10,BLUE,prediction_bleu,, -2024-09-11,BLUE,prediction_bleu,, -2024-09-12,BLUE,prediction_bleu,, -2024-09-13,BLUE,prediction_bleu,, -2024-09-14,BLUE,prediction_bleu,, -2024-09-15,BLUE,prediction_bleu,, -2024-09-16,BLUE,prediction_bleu,, -2024-09-17,BLUE,prediction_bleu,, -2024-09-18,BLUE,prediction_bleu,, -2024-09-19,BLUE,prediction_bleu,, -2024-09-20,BLUE,prediction_bleu,, -2024-09-21,BLUE,prediction_bleu,, -2024-09-22,BLUE,prediction_bleu,, -2024-09-23,BLUE,prediction_bleu,, -2024-09-24,BLUE,prediction_bleu,, -2024-09-25,BLUE,prediction_bleu,, -2024-09-26,BLUE,prediction_bleu,, -2024-09-27,BLUE,prediction_bleu,, -2024-09-28,BLUE,prediction_bleu,, -2024-09-29,BLUE,prediction_bleu,, -2024-09-30,BLUE,prediction_bleu,, -2024-10-01,BLUE,prediction_bleu,, -2024-10-02,BLUE,prediction_bleu,, -2024-10-03,BLUE,prediction_bleu,, -2024-10-04,BLUE,prediction_bleu,, -2024-10-05,BLUE,prediction_bleu,, -2024-10-06,BLUE,prediction_bleu,, -2024-10-07,BLUE,prediction_bleu,, -2024-10-08,BLUE,prediction_bleu,, -2024-10-09,BLUE,prediction_bleu,prediction_bleu,prediction_bleu -2024-10-10,BLUE,prediction_bleu,prediction_bleu,prediction_bleu -2024-10-11,BLUE,prediction_bleu,prediction_bleu,prediction_bleu -2024-10-12,BLUE,prediction_bleu,prediction_bleu,prediction_bleu -2024-10-13,BLUE,prediction_bleu,prediction_bleu,prediction_bleu -2024-10-14,BLUE,prediction_bleu,, -2024-10-15,BLUE,prediction_bleu,, -2024-10-16,BLUE,prediction_bleu,prediction_bleu, -2024-10-17,,,prediction_bleu,prediction_bleu -2024-10-18,,,,prediction_bleu +2024-09-01,BLUE,BLUE,, +2024-09-02,BLUE,BLUE,, +2024-09-03,BLUE,BLUE,, +2024-09-04,BLUE,BLUE,, +2024-09-05,BLUE,BLUE,, +2024-09-06,BLUE,BLUE,, +2024-09-07,BLUE,BLUE,, +2024-09-08,BLUE,BLUE,, +2024-09-09,BLUE,BLUE,, +2024-09-10,BLUE,BLUE,, +2024-09-11,BLUE,BLUE,, +2024-09-12,BLUE,BLUE,, +2024-09-13,BLUE,BLUE,, +2024-09-14,BLUE,BLUE,, +2024-09-15,BLUE,BLUE,, +2024-09-16,BLUE,BLUE,, +2024-09-17,BLUE,BLUE,, +2024-09-18,BLUE,BLUE,, +2024-09-19,BLUE,BLUE,, +2024-09-20,BLUE,BLUE,, +2024-09-21,BLUE,BLUE,, +2024-09-22,BLUE,BLUE,, +2024-09-23,BLUE,BLUE,, +2024-09-24,BLUE,BLUE,, +2024-09-25,BLUE,BLUE,, +2024-09-26,BLUE,BLUE,, +2024-09-27,BLUE,BLUE,, +2024-09-28,BLUE,BLUE,, +2024-09-29,BLUE,BLUE,, +2024-09-30,BLUE,BLUE,, +2024-10-01,BLUE,BLUE,, +2024-10-02,BLUE,BLUE,, +2024-10-03,BLUE,BLUE,, +2024-10-04,BLUE,BLUE,, +2024-10-05,BLUE,BLUE,, +2024-10-06,BLUE,BLUE,, +2024-10-07,BLUE,BLUE,, +2024-10-08,BLUE,BLUE,, +2024-10-09,BLUE,BLUE,BLUE,BLUE +2024-10-10,BLUE,BLUE,BLUE,BLUE +2024-10-11,BLUE,BLUE,BLUE,BLUE +2024-10-12,BLUE,BLUE,BLUE,BLUE +2024-10-13,BLUE,BLUE,BLUE,BLUE +2024-10-14,BLUE,BLUE,, +2024-10-15,BLUE,BLUE,, +2024-10-16,BLUE,BLUE,BLUE, +2024-10-17,BLUE,BLUE,BLUE,BLUE +2024-10-18,BLUE,BLUE,,BLUE +2024-10-19,BLUE,BLUE,, +2024-10-20,BLUE,BLUE,, +2024-10-21,BLUE,BLUE,, +2024-10-22,BLUE,BLUE,, +2024-10-23,BLUE,BLUE,, +2024-10-24,BLUE,BLUE,, +2024-10-25,BLUE,BLUE,, +2024-10-26,BLUE,BLUE,, +2024-10-27,BLUE,BLUE,, +2024-10-28,BLUE,BLUE,BLUE,BLUE +2024-10-29,BLUE,BLUE,BLUE,BLUE +2024-10-30,BLUE,BLUE,BLUE,BLUE +2024-10-31,BLUE,BLUE,,BLUE +2024-11-01,BLUE,BLUE,BLUE, +2024-11-02,BLUE,BLUE,BLUE,BLUE +2024-11-03,BLUE,BLUE,BLUE,BLUE +2024-11-04,BLUE,BLUE,BLUE,BLUE +2024-11-05,BLUE,BLUE,BLUE,WHITE +2024-11-06,BLUE,BLUE,BLUE,BLUE +2024-11-07,BLUE,BLUE,BLUE,BLUE +2024-11-08,BLUE,BLUE,BLUE,BLUE +2024-11-09,BLUE,BLUE,BLUE,BLUE +2024-11-10,BLUE,BLUE,BLUE,BLUE +2024-11-11,BLUE,BLUE,BLUE,BLUE +2024-11-12,BLUE,BLUE,BLUE,BLUE +2024-11-13,BLUE,WHITE,BLUE,WHITE +2024-11-14,WHITE,WHITE,WHITE,WHITE +2024-11-15,WHITE,WHITE,WHITE,WHITE +2024-11-16,BLUE,BLUE,BLUE,BLUE +2024-11-17,BLUE,BLUE,BLUE,BLUE +2024-11-18,BLUE,BLUE,BLUE,BLUE +2024-11-19,BLUE,BLUE,,BLUE +2024-11-20,,WHITE,WHITE, +2024-11-21,,,WHITE,WHITE +2024-11-22,,,,WHITE diff --git a/postgres/compose-postgres.yaml b/postgres/compose-postgres.yaml new file mode 100644 index 0000000..c68a031 --- /dev/null +++ b/postgres/compose-postgres.yaml @@ -0,0 +1,29 @@ +version: "3.8" +services: + postgres: + image: postgres:17-alpine + restart: unless-stopped + env_file: + - postgres.env + - secrets.env + ports: + - "5434:5432" + volumes: + - ./postgres_volume:/var/lib/postgresql/data + - ./init_scripts:/docker-entrypoint-initdb.d + # pgadmin: + # image: dpage/pgadmin4 + # restart: always + # environment: + # PGADMIN_DEFAULT_EMAIL: stress.energetic@d4g.com + # PGADMIN_DEFAULT_PASSWORD_FILE: /run/secrets/pgadmin_default_password + # secrets: + # - pgadmin_default_password + # ports: + # - "8880:80" + # depends_on: + # - postgres + + +volumes: + db: diff --git a/postgres/init_scripts/01_tables.sql b/postgres/init_scripts/01_tables.sql new file mode 100644 index 0000000..b6a3194 --- /dev/null +++ b/postgres/init_scripts/01_tables.sql @@ -0,0 +1,61 @@ +\c stress_energetic; + +CREATE TABLE IF NOT EXISTS eco2mix_raw ( + date_heure TIMESTAMP, + consommation FLOAT, + eolien FLOAT, + solaire FLOAT +); + +CREATE TABLE IF NOT EXISTS pred_conso_rte ( + date_heure TIMESTAMP, + consommation FLOAT +); + +CREATE TABLE IF NOT EXISTS forecast_flux_solaire ( + date_heure TIMESTAMP, + departement CHAR, + delta_hours_pred FLOAT, + flux_solaire FLOAT +); + +CREATE TABLE IF NOT EXISTS forecast_vent ( + date_heure TIMESTAMP, + departement CHAR, + delta_hours_pred FLOAT, + vent FLOAT +); + +CREATE TABLE IF NOT EXISTS forecast_ENR ( + date_heure TIMESTAMP, + delta_hours_pred FLOAT, + eolien FLOAT, + solaire FLOAT +); + +-- Je ne pense pas que ça soit nécessaire, l'information est gardée indirectement +-- dans les données pour prédire le jour tempo +-- CREATE TABLE IF NOT EXISTS temperature ( +-- date_heure TIMESTAMP, +-- temperature FLOAT +-- ); + +CREATE TABLE IF NOT EXISTS prep_prediction_tempo ( + date DATE, + consommation FLOAT, + eolien FLOAT, + solaire FLOAT, + temperature FLOAT, + production_nette FLOAT, + production_nette_q40 FLOAT, + production_nette_q80 FLOAT, + mean_temp_q30 FLOAT, + jour_tempo CHAR, +); + +CREATE TABLE IF NOT EXISTS prediction_tempo ( + date DATE, + delta_hours_pred FLOAT, + eolien FLOAT, + solaire FLOAT +); diff --git a/postgres/init_scripts/02_setup_users.sql b/postgres/init_scripts/02_setup_users.sql new file mode 100644 index 0000000..8ed3736 --- /dev/null +++ b/postgres/init_scripts/02_setup_users.sql @@ -0,0 +1,13 @@ +-- Création de l'utilisateur super +CREATE ROLE super_user WITH LOGIN PASSWORD '$$POSTGRES_SUPER_USER_PASSWORD$$' SUPERUSER; + +-- Création de l'utilisateur read-only +CREATE ROLE readonly_user WITH LOGIN PASSWORD '$$POSTGRES_READONLY_USER_PASSWORD$$'; + +-- Accorder des privilèges de lecture +GRANT CONNECT ON DATABASE stress_energetic TO readonly_user; +GRANT USAGE ON SCHEMA public TO readonly_user; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO readonly_user; + +-- Pour accorder des privilèges de lecture sur les nouvelles tables créées ultérieurement +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO readonly_user; \ No newline at end of file diff --git a/postgres/postgres.env b/postgres/postgres.env new file mode 100644 index 0000000..139e7eb --- /dev/null +++ b/postgres/postgres.env @@ -0,0 +1,3 @@ +POSTGRES_USER=root +POSTGRES_HOST=postgres +POSTGRES_DB=stress_energetic diff --git a/pyproject.toml b/pyproject.toml index 45a57a5..29002f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ dependencies = [ "bokeh>=3.4", "joblib", "python-dotenv", + "sqlalchemy>=2.0.36", ] [project.urls] diff --git a/src/energy_forecast/database_core.py b/src/energy_forecast/database_core.py new file mode 100644 index 0000000..f303eb9 --- /dev/null +++ b/src/energy_forecast/database_core.py @@ -0,0 +1,153 @@ +import sqlalchemy as sa +import pandas as pd + +# change the URL_DB to the url of the postgres database +URL_DB = 'sqlite:///energy_forecast.db' + +# tempo_classification table +tempo_classes = sa.Enum("BLUE", "WHITE", "RED", "Not Available", name="tempo_classification_enum") + +metadata = sa.MetaData() +tempo_classification = sa.Table( + 'tempo_classification', + metadata, + sa.Column('date', sa.Date, nullable=False, primary_key=True), + sa.Column('truth_by_RTE', tempo_classes, nullable=False), + sa.Column('our_J_1', tempo_classes, nullable=False), + sa.Column('our_J_2', tempo_classes, nullable=False), + sa.Column('our_J_3', tempo_classes, nullable=False), +) + +eco2mix_raw = sa.Table( + "eco2mix_raw", + metadata, + sa.Column("date_heure", sa.DateTime, nullable=False, primary_key=True), + sa.Column("consommation", sa.Float), + sa.Column("eolien", sa.Float), + sa.Column("solaire", sa.Float), +) + +pred_conso_rte = sa.Table( + "pred_conso_rte", + metadata, + sa.Column("date_heure", sa.DateTime, nullable=False, primary_key=True), + sa.Column("consommation", sa.Float), +) + +forecast_flux_solaire = sa.Table( + "forecast_flux_solaire", + metadata, + sa.Column("date_heure", sa.DateTime, nullable=False, primary_key=True), + sa.Column("departement", sa.String), + sa.Column("delta_hours_pred", sa.Float), + sa.Column("flux_solaire", sa.Float), +) + +forecast_vent = sa.Table( + "forecast_vent", + metadata, + sa.Column("date_heure", sa.DateTime, nullable=False, primary_key=True), + sa.Column("departement", sa.String), + sa.Column("delta_hours_pred", sa.Float), + sa.Column("vent", sa.Float), +) + +forecast_temperature = sa.Table( + "forecast_temperature", + metadata, + sa.Column("date_heure", sa.DateTime, nullable=False, primary_key=True), + sa.Column("departement", sa.String), + sa.Column("delta_hours_pred", sa.Float), + sa.Column("temperature", sa.Float), +) + +forecast_ENR = sa.Table( + "forecast_ENR", + metadata, + sa.Column("date_heure", sa.DateTime, nullable=False, primary_key=True), + sa.Column("departement", sa.String), + sa.Column("delta_hours_pred", sa.Float), + sa.Column("eolien", sa.Float), + sa.Column("solaire", sa.Float), +) + +prep_prediction_tempo = sa.Table( + "prep_prediction_tempo", + metadata, + sa.Column("date", sa.Date, nullable=False, primary_key=True), + sa.Column("temperature", sa.Float), + sa.Column("eolien", sa.Float), + sa.Column("solaire", sa.Float), + sa.Column("consommation", sa.Float), + sa.Column("production_nette", sa.Float), + sa.Column("production_nette_q40", sa.Float), + sa.Column("production_nette_q80", sa.Float), + sa.Column("mean_temp_q30", sa.Float), + sa.Column("jour_tempo", sa.String), +) + +def init_db(): + engine = sa.create_engine(URL_DB) + metadata.create_all(engine) + +def get_all_tempo_classification(): + engine = sa.create_engine(URL_DB) + with engine.connect() as connection: + query = sa.select([tempo_classification]) + result = connection.execute(query) + return result.fetchall() + +def insert_tempo_classification(date, truth_by_RTE, our_J_1, our_J_2, our_J_3): + engine = sa.create_engine(URL_DB) + with engine.connect() as connection: + query = tempo_classification.insert().values( + date=date, + truth_by_RTE=truth_by_RTE, + our_J_1=our_J_1, + our_J_2=our_J_2, + our_J_3=our_J_3, + ) + connection.execute(query) + connection.commit() # Ensure the transaction is committed + + +def add_the_csv_predictions(path_to_csv): + mapping_cal_names = { + "Type_de_jour_TEMPO": "truth_by_RTE", + "our_tempo_J-1": "our_J_1", + "our_tempo_J-2": "our_J_2", + "our_tempo_J-3": "our_J_3", + "index": "date", + } + df = pd.read_csv(path_to_csv, index_col=0) + print(df.columns) + + df.index = pd.to_datetime(df.index).date + for index, row in df.iterrows(): + row = row.fillna("Not Available") + + insert_tempo_classification( + date=row.name, + truth_by_RTE=row["Type_de_jour_TEMPO"], + our_J_1=row["our_tempo_J-1"], + our_J_2=row["our_tempo_J-2"], + our_J_3=row["our_tempo_J-3"], + ) + +def select_last_n_days(n_days): + engine = sa.create_engine(URL_DB) + with engine.connect() as connection: + query = sa.select([tempo_classification]).order_by(sa.desc(tempo_classification.c.date)).limit(n_days). + print(query) + result = connection.execute(query) + results = result.fetchall() + return pd.DataFrame(results, columns=result.keys()) + +if __name__ == '__main__': + init_db() + from energy_forecast import ROOT_DIR + add_the_csv_predictions(ROOT_DIR / 'data' /"gold" / 'our_tempo_prediction.csv') + + selection = select_last_n_days(2) + print(selection) +