diff --git a/LocalFeeder/requirements.txt b/LocalFeeder/requirements.txt index cd911b4..1256674 100644 --- a/LocalFeeder/requirements.txt +++ b/LocalFeeder/requirements.txt @@ -11,5 +11,5 @@ boto3 xarray fastapi uvicorn -oedisi==1.1.1 +git+https://github.com/openEDI/oedisi@al/config python-multipart \ No newline at end of file diff --git a/LocalFeeder/server.py b/LocalFeeder/server.py index 2a0eac4..a77cf97 100644 --- a/LocalFeeder/server.py +++ b/LocalFeeder/server.py @@ -51,7 +51,7 @@ def read_root(): return JSONResponse(response, 200) -@app.get("/sensor/") +@app.get("/sensor") async def sensor(): logging.info(os.getcwd()) sensor_path = os.path.join(base_path, "sensors", "sensors.json") @@ -63,7 +63,7 @@ async def sensor(): return data -@app.post("/profiles/") +@app.post("/profiles") async def upload_profiles(file: UploadFile): try: data = file.file.read() @@ -96,7 +96,7 @@ async def upload_profiles(file: UploadFile): ) -@app.post("/model/") +@app.post("/model") async def upload_model(file: UploadFile): try: data = file.file.read() @@ -125,7 +125,7 @@ async def upload_model(file: UploadFile): HTTPException(500, "Unknown error while uploading userdefined opendss model.") -@app.post("/run/") +@app.post("/run") async def run_feeder( broker_config: BrokerConfig, background_tasks: BackgroundTasks ): # :BrokerConfig @@ -140,7 +140,7 @@ async def run_feeder( HTTPException(500, str(err)) -@app.post("/configure/") +@app.post("/configure") async def configure(component_struct:ComponentStruct): component = component_struct.component params = component.parameters diff --git a/broker/__pycache__/server.cpython-312.pyc b/broker/__pycache__/server.cpython-312.pyc new file mode 100644 index 0000000..8d3cf75 Binary files /dev/null and b/broker/__pycache__/server.cpython-312.pyc differ diff --git a/broker/docker-compose.yml b/broker/docker-compose.yml new file mode 100644 index 0000000..791e0bb --- /dev/null +++ b/broker/docker-compose.yml @@ -0,0 +1,19 @@ +networks: + custom-network: + driver: bridge + ipam: + config: + - gateway: 10.5.0.1 + subnet: 10.5.0.0/16 +services: + oedisi_broker: + build: + context: ./broker/. + environment: + PORT: '8766' + hostname: broker + image: aadillatif/oedisi_broker + networks: + custom-network: {} + ports: + - 8766:8766 diff --git a/broker/requirements.txt b/broker/requirements.txt index af0d7fb..8b07bec 100644 --- a/broker/requirements.txt +++ b/broker/requirements.txt @@ -3,6 +3,6 @@ helics-apps==3.4.0 pyyaml fastapi uvicorn -oedisi==1.1.1 +git+https://github.com/openEDI/oedisi@al/config grequests python-multipart diff --git a/broker/server.py b/broker/server.py index 2709f2e..08ec9c0 100644 --- a/broker/server.py +++ b/broker/server.py @@ -1,6 +1,7 @@ from fastapi import FastAPI, BackgroundTasks, UploadFile from fastapi.responses import FileResponse, JSONResponse from fastapi.exceptions import HTTPException +from pathlib import Path import helics as h import grequests import traceback @@ -12,15 +13,27 @@ import shutil import time import yaml +import json import sys import os from oedisi.componentframework.system_configuration import WiringDiagram, ComponentStruct -from oedisi.types.common import ServerReply, HeathCheck +from oedisi.types.common import ServerReply, HeathCheck, DefaultFileNames app = FastAPI() +is_kubernetes_env = os.environ['SERVICE_NAME'] if 'SERVICE_NAME' in os.environ else None + +def build_url(host:str, port:int, enpoint:list): + if is_kubernetes_env: + SERVICE_NAME = os.environ['SERVICE_NAME'] + url = f"http://{host}.{SERVICE_NAME}:{port}/" + else: + url = f"http://{host}:{port}/" + url = url + "/".join(enpoint) + "/" + return url + def find_filenames(path_to_dir=os.getcwd(), suffix=".feather"): filenames = os.listdir(path_to_dir) return [filename for filename in filenames if filename.endswith(suffix)] @@ -33,13 +46,15 @@ def read_settings(): services = config["services"] print(services) broker = services.pop("oedisi_broker") - broker_ip = broker["networks"]["custom-network"]["ipv4_address"] + broker_host = broker["hostname"] + + broker_ip = socket.gethostbyname(broker_host) api_port = int(broker["ports"][0].split(":")[0]) for service in services: - ip = services[service]["networks"]["custom-network"]["ipv4_address"] + host = services[service]["hostname"] port = int(services[service]["ports"][0].split(":")[0]) - component_map[ip] = port + component_map[host] = port return services, component_map, broker_ip, api_port @@ -54,13 +69,13 @@ def read_root(): return JSONResponse(response, 200) -@app.post("/profiles/") +@app.post("/profiles") async def upload_profiles(file: UploadFile): try: services, _, _, _ = read_settings() for service in services: if "feeder" in service.lower(): - ip = services[service]["networks"]["custom-network"]["ipv4_address"] + ip = services[service]["host"] port = int(services[service]["ports"][0].split(":")[0]) data = file.file.read() if not file.filename.endswith(".zip"): @@ -69,7 +84,10 @@ async def upload_profiles(file: UploadFile): ) with open(file.filename, "wb") as f: f.write(data) - url = f"http://{ip}:{port}/profiles/" + + url = build_url(ip, port, ["profiles"]) + logging.info(f"making a request to url - {url}") + files = {"file": open(file.filename, "rb")} r = requests.post(url, files=files) response = ServerReply(detail=r.text).dict() @@ -80,13 +98,13 @@ async def upload_profiles(file: UploadFile): raise HTTPException(status_code=500, detail=str(err)) -@app.post("/model/") +@app.post("/model") async def upload_model(file: UploadFile): try: services, _, _, _ = read_settings() for service in services: if "feeder" in service.lower(): - ip = services[service]["networks"]["custom-network"]["ipv4_address"] + ip = services[service]["host"] port = int(services[service]["ports"][0].split(":")[0]) data = file.file.read() if not file.filename.endswith(".zip"): @@ -95,7 +113,10 @@ async def upload_model(file: UploadFile): ) with open(file.filename, "wb") as f: f.write(data) - url = f"http://{ip}:{port}/model/" + + url = build_url(ip, port, ["model"]) + logging.info(f"making a request to url - {url}") + files = {"file": open(file.filename, "rb")} r = requests.post(url, files=files) response = ServerReply(detail=r.text).dict() @@ -105,15 +126,17 @@ async def upload_model(file: UploadFile): err = traceback.format_exc() raise HTTPException(status_code=500, detail=str(err)) - -@app.get("/results/") +@app.get("/results") def download_results(): services, _, _, _ = read_settings() for service in services: if "recorder" in service.lower(): - ip = services[service]["networks"]["custom-network"]["ipv4_address"] + host = services[service]["hostname"] port = int(services[service]["ports"][0].split(":")[0]) - url = f"http://{ip}:{port}/download/" + + url = build_url(host, port, ["download"]) + logging.info(f"making a request to url - {url}") + response = requests.get(url) with open(f"{service}.feather", "wb") as out_file: shutil.copyfileobj(response.raw, out_file) @@ -130,7 +153,7 @@ def download_results(): raise HTTPException(status_code=404, detail="Failed download") -@app.get("/terminate/") +@app.get("/terminate") def terminate_simulation(): try: h.helicsCloseLibrary() @@ -141,6 +164,7 @@ def terminate_simulation(): def run_simulation(): services, component_map, broker_ip, api_port = read_settings() + logging.info(f"{broker_ip}, {api_port}") initstring = f"-f {len(component_map)} --name=mainbroker --loglevel=trace --local_interface={broker_ip} --localport=23404" logging.info(f"Broker initaialization string: {initstring}") broker = h.helicsCreateBroker("zmq", "", initstring) @@ -150,8 +174,10 @@ def run_simulation(): logging.info(str(component_map)) replies = [] for service_ip, service_port in component_map.items(): - url = f"http://{service_ip}:{service_port}/run/" - print(url) + + url = build_url(service_ip, service_port, ["run"]) + logging.info(f"making a request to url - {url}") + myobj = { "broker_port": 23404, "broker_ip": broker_ip, @@ -167,7 +193,7 @@ def run_simulation(): return -@app.post("/run/") +@app.post("/run") async def run_feeder(background_tasks: BackgroundTasks): try: background_tasks.add_task(run_simulation) @@ -178,8 +204,9 @@ async def run_feeder(background_tasks: BackgroundTasks): raise HTTPException(status_code=404, detail=str(err)) -@app.post("/configure/") +@app.post("/configure") async def configure(wiring_diagram:WiringDiagram): + json.dump(wiring_diagram.dict(), open(DefaultFileNames.WIRING_DIAGRAM.value, "w")) for component in wiring_diagram.components: component_model = ComponentStruct( component = component, @@ -188,11 +215,13 @@ async def configure(wiring_diagram:WiringDiagram): for link in wiring_diagram.links: if link.target == component.name: component_model.links.append(link) + + url = build_url(component.host, component.container_port, ["configure"]) + logging.info(f"making a request to url - {url}") - url = f'http://{component.host}:{component.container_port}/configure/' - logging.info(f"making post request to: {url}") r = requests.post(url, json=component_model.dict()) assert r.status_code==200, f"POST request to update configuration failed for url - {url}" + return JSONResponse(ServerReply(detail="Sucessfully updated config files for all containers").dict(), 200) if __name__ == "__main__": port = int(sys.argv[2]) diff --git a/measuring_federate/requirements.txt b/measuring_federate/requirements.txt index 1cd6fd2..1348db6 100644 --- a/measuring_federate/requirements.txt +++ b/measuring_federate/requirements.txt @@ -8,4 +8,4 @@ fastapi uvicorn requests grequests -oedisi==1.1.1 +git+https://github.com/openEDI/oedisi@al/config diff --git a/measuring_federate/server.py b/measuring_federate/server.py index 015156e..601587b 100644 --- a/measuring_federate/server.py +++ b/measuring_federate/server.py @@ -5,6 +5,7 @@ import requests import uvicorn import socket +import logging import sys import json import os @@ -15,6 +16,17 @@ app = FastAPI() +is_kubernetes_env = os.environ['SERVICE_NAME'] if 'SERVICE_NAME' in os.environ else None + +def build_url(host:str, port:int, enpoint:list): + if is_kubernetes_env: + SERVICE_NAME = os.environ['SERVICE_NAME'] + url = f"http://{host}.{SERVICE_NAME}:{port}/" + else: + url = f"http://{host}:{port}/" + url = url + "/".join(enpoint) + return url + @app.get("/") async def read_root(): hostname = socket.gethostname() @@ -25,18 +37,20 @@ async def read_root(): ).dict() return JSONResponse(response, 200) -@app.post("/run/") +@app.post("/run") async def run_model(broker_config:BrokerConfig, background_tasks: BackgroundTasks): - try: - print(broker_config) - feeder_ip = broker_config.services['oedisi_feeder']['networks']['custom-network']['ipv4_address'] - feeder_port = int(broker_config.services['oedisi_feeder']['ports'][0].split(":")[0]) - - url =f"http://{feeder_ip}:{feeder_port}/sensor/" - print(url) + logging.info(broker_config) + feeder_host = broker_config.services['oedisi_feeder']['hostname'] + feeder_port = int(broker_config.services['oedisi_feeder']['ports'][0].split(":")[0]) + url = build_url(feeder_host, feeder_port, 'sensor') + logging.info(url) + try: reply = requests.get(url) sensor_data = reply.json() - print(sensor_data) + if not sensor_data: + msg = "empty sensor list" + raise HTTPException(404, msg) + logging.info(sensor_data) with open("sensors.json", "w") as outfile: json.dump(sensor_data, outfile) @@ -47,9 +61,9 @@ async def run_model(broker_config:BrokerConfig, background_tasks: BackgroundTask return JSONResponse(response, 200) except Exception as e: err = traceback.format_exc() - HTTPException(500,str(err)) + raise HTTPException(500,str(err)) -@app.post("/configure/") +@app.post("/configure") async def configure(component_struct:ComponentStruct): component = component_struct.component params = component.parameters diff --git a/recorder/requirements.txt b/recorder/requirements.txt index 51c0e94..aa469b6 100644 --- a/recorder/requirements.txt +++ b/recorder/requirements.txt @@ -6,4 +6,4 @@ numpy pandas fastapi uvicorn -oedisi==1.1.1 +git+https://github.com/openEDI/oedisi@al/config diff --git a/recorder/server.py b/recorder/server.py index e2ad116..dba2df7 100644 --- a/recorder/server.py +++ b/recorder/server.py @@ -35,7 +35,7 @@ def find_filenames(path_to_dir=os.getcwd(), suffix=".feather"): return [filename for filename in filenames if filename.endswith(suffix)] -@app.get("/download/") +@app.get("/download") def download_results(): file_list = find_filenames() if file_list: @@ -46,7 +46,7 @@ def download_results(): raise HTTPException(status_code=404, detail="No feather file found") -@app.post("/run/") +@app.post("/run") async def run_model(broker_config: BrokerConfig, background_tasks: BackgroundTasks): logging.info(broker_config) try: @@ -58,7 +58,7 @@ async def run_model(broker_config: BrokerConfig, background_tasks: BackgroundTas HTTPException(500, str(err)) -@app.post("/configure/") +@app.post("/configure") async def configure(component_struct:ComponentStruct): component = component_struct.component params = component.parameters diff --git a/system.json b/system.json new file mode 100644 index 0000000..cc5f25d --- /dev/null +++ b/system.json @@ -0,0 +1,203 @@ +{ + "name": "docker_test", + "components": [ + { + "name": "feeder", + "type": "LocalFeeder", + "host": "feeder", + "container_port": 5678, + "parameters": { + "use_smartds": false, + "user_uploads_model": false, + "profile_location": "gadal_ieee123/profiles", + "opendss_location": "gadal_ieee123/qsts", + "sensor_location": "gadal_ieee123/sensors.json", + "start_date": "2017-01-01 00:00:00", + "number_of_timesteps": 96, + "run_freq_sec": 900, + "topology_output": "topology.json" + } + }, + { + "name": "recorder_voltage_real", + "type": "Recorder", + "host": "recorder-voltage-real", + "container_port": 5679, + "parameters": {"feather_filename": "voltage_real.feather", + "csv_filename": "voltage_real.csv" + } + }, + { + "name": "recorder_voltage_imag", + "type": "Recorder", + "host": "recorder-voltage-imag", + "container_port": 5680, + "parameters": {"feather_filename": "voltage_imag.feather", + "csv_filename": "voltage_imag.csv" + } + }, + { + "name": "recorder_voltage_mag", + "type": "Recorder", + "host": "recorder-voltage-mag", + "container_port": 5681, + "parameters": {"feather_filename": "voltage_mag.feather", + "csv_filename": "voltage_mag.csv" + } + }, + { + "name": "recorder_voltage_angle", + "type": "Recorder", + "host": "recorder-voltage-angle", + "container_port": 5682, + "parameters": {"feather_filename": "voltage_angle.feather", + "csv_filename": "voltage_angle.csv" + } + }, + { + "name": "state_estimator", + "type": "StateEstimatorComponent", + "host": "state-estimator", + "container_port": 5683, + "parameters": { + "algorithm_parameters": {"tol": 1e-5} + } + }, + { + "name": "sensor_voltage_real", + "type": "MeasurementComponent", + "host": "sensor-voltage-real", + "container_port": 5684, + "parameters": { + "gaussian_variance": 0.0, + "random_percent": 0.0, + "measurement_file": "sensors.json" + } + }, + { + "name": "sensor_voltage_magnitude", + "type": "MeasurementComponent", + "host": "sensor-voltage-magnitude", + "container_port": 5685, + "parameters": { + "gaussian_variance": 0.0, + "random_percent": 0.0, + "measurement_file": "sensors.json" + } + }, + { + "name": "sensor_voltage_imaginary", + "type": "MeasurementComponent", + "host": "sensor-voltage-imaginary", + "container_port": 5686, + "parameters": { + "gaussian_variance": 0.0, + "random_percent": 0.0, + "measurement_file": "sensors.json" + } + }, + { + "name": "sensor_power_real", + "type": "MeasurementComponent", + "host": "sensor-power-real", + "container_port": 5687, + "parameters": { + "gaussian_variance": 0.0, + "random_percent": 0.0, + "measurement_file": "sensors.json" + } + }, + { + "name": "sensor_power_imaginary", + "type": "MeasurementComponent", + "host": "sensor-power-imaginary", + "container_port": 5688, + "parameters": { + "gaussian_variance": 0.0, + "random_percent": 0.0, + "measurement_file": "sensors.json" + } + } + + ], + "links": [ + { + "source": "feeder", + "source_port": "voltages_magnitude", + "target": "sensor_voltage_magnitude", + "target_port": "subscription" + }, + { + "source": "feeder", + "source_port": "voltages_real", + "target": "sensor_voltage_real", + "target_port": "subscription" + }, + { + "source": "feeder", + "source_port": "voltages_imag", + "target": "sensor_voltage_imaginary", + "target_port": "subscription" + }, + { + "source": "feeder", + "source_port": "powers_real", + "target": "sensor_power_real", + "target_port": "subscription" + }, + { + "source": "feeder", + "source_port": "powers_imag", + "target": "sensor_power_imaginary", + "target_port": "subscription" + }, + { + "source": "feeder", + "source_port": "topology", + "target": "state_estimator", + "target_port": "topology" + }, + { + "source": "sensor_voltage_magnitude", + "source_port": "publication", + "target": "state_estimator", + "target_port": "voltages_magnitude" + }, + { + "source": "sensor_power_real", + "source_port": "publication", + "target": "state_estimator", + "target_port": "powers_real" + }, + { + "source": "sensor_power_imaginary", + "source_port": "publication", + "target": "state_estimator", + "target_port": "powers_imaginary" + }, + { + "source": "feeder", + "source_port": "voltages_real", + "target": "recorder_voltage_real", + "target_port": "subscription" + }, + { + "source": "feeder", + "source_port": "voltages_imag", + "target": "recorder_voltage_imag", + "target_port": "subscription" + }, + { + "source": "state_estimator", + "source_port": "voltage_angle", + "target": "recorder_voltage_angle", + "target_port": "subscription" + }, + { + "source": "state_estimator", + "source_port": "voltage_mag", + "target": "recorder_voltage_mag", + "target_port": "subscription" + } + ] +} diff --git a/wls_federate/requirements.txt b/wls_federate/requirements.txt index 3a4d410..cae4e56 100644 --- a/wls_federate/requirements.txt +++ b/wls_federate/requirements.txt @@ -5,4 +5,4 @@ scipy numpy fastapi uvicorn -oedisi==1.1.1 +git+https://github.com/openEDI/oedisi@al/config diff --git a/wls_federate/server.py b/wls_federate/server.py index 320f927..d7cf70b 100644 --- a/wls_federate/server.py +++ b/wls_federate/server.py @@ -26,7 +26,7 @@ def read_root(): return JSONResponse(response, 200) -@app.post("/run/") +@app.post("/run") async def run_model(broker_config: BrokerConfig, background_tasks: BackgroundTasks): print(broker_config) try: @@ -40,7 +40,7 @@ async def run_model(broker_config: BrokerConfig, background_tasks: BackgroundTas HTTPException(500, str(err)) -@app.post("/configure/") +@app.post("/configure") async def configure(component_struct:ComponentStruct): component = component_struct.component params = component.parameters