Merge pull request #1 from hydroserver2/dev

v0.2.0

README.md

@@ -1,6 +1,6 @@
-# HydroServer Hydroloader
+# HydroServer REST Client
 
-A Python package for loading streaming data to a HydroServer instance using the SensorThings API.
+A Python package for interacting with the HydroServer data management and SensorThings REST APIs.
 
 ## Funding and Acknowledgements
 

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
-name = hydroloader
-version = 0.1.15
+name = hydroserverpy
+version = 0.2.0
 
 [options]
 package_dir=
@@ -14,6 +14,8 @@ install_requires =
     simplejson >= 3
     crontab >= 1
     python-dateutil >= 2.8.2
+    croniter >= 2.0.1
+    frost-sta-client >= 1.1.44
 
 [options.extras_require]
 docs =

src/hydroserverpy/__init__.py

@@ -0,0 +1,13 @@
+from hydroserverpy.main import HydroServer
+from hydroserverpy.schemas.things import ThingPostBody
+from hydroserverpy.schemas.observed_properties import ObservedPropertyPostBody
+from hydroserverpy.schemas.units import UnitPostBody
+from hydroserverpy.schemas.datastreams import DatastreamPostBody
+
+__all__ = [
+    "HydroServer",
+    "ThingPostBody",
+    "ObservedPropertyPostBody",
+    "UnitPostBody",
+    "DatastreamPostBody"
+]

src/hydroserverpy/components/__init__.py

@@ -0,0 +1,10 @@
+from .data_loaders import DataLoader
+from .data_sources import DataSource
+from .datastreams import Datastream
+from .observed_properties import ObservedProperty
+from .processing_levels import ProcessingLevel
+from .result_qualifiers import ResultQualifier
+from .sensors import Sensor
+from .things import Thing
+from .units import Unit
+from .users import User

src/hydroserverpy/components/data_loaders.py

@@ -0,0 +1,67 @@
+from typing import List
+from uuid import UUID
+from hydroserverpy.schemas.data_loaders import DataLoaderGetResponse, DataLoaderPostBody, DataLoaderPatchBody
+from hydroserverpy.schemas.data_sources import DataSourceGetResponse
+
+
+class DataLoader:
+
+    def __init__(self, service):
+        self._service = service
+
+    def list(self):
+
+        return self._service.get(
+            'data-loaders',
+            response_schema=List[DataLoaderGetResponse]
+        )
+
+    def list_data_sources(self, data_loader_id: UUID):
+
+        return self._service.get(
+            f'data-loaders/{data_loader_id}/data-sources',
+            response_schema=List[DataSourceGetResponse]
+        )
+
+    def get(self, data_loader_id: UUID):
+
+        return self._service.get(
+            f'data-loaders/{str(data_loader_id)}',
+            response_schema=DataLoaderGetResponse
+        )
+
+    def create(self, data_loader_body: DataLoaderPostBody):
+
+        return self._service.post(
+            f'data-loaders',
+            headers={'Content-type': 'application/json'},
+            data=data_loader_body.json(by_alias=True),
+            response_schema=DataLoaderGetResponse
+        )
+
+    def update(self, data_loader_id: UUID, data_loader_body: DataLoaderPatchBody):
+
+        return self._service.patch(
+            f'data-loaders/{str(data_loader_id)}',
+            headers={'Content-type': 'application/json'},
+            data=data_loader_body.json(exclude_unset=True, by_alias=True),
+            response_schema=DataLoaderGetResponse
+        )
+
+    def delete(self, data_loader_id: UUID):
+
+        return self._service.delete(
+            f'data-loaders/{str(data_loader_id)}'
+        )
+
+    def load_data(self, data_loader_id: str):
+
+        data_sources_response = self.list_data_sources(data_loader_id=UUID(data_loader_id))
+
+        if data_sources_response.data:
+            data_sources = data_sources_response.data
+        else:
+            return None
+
+        for data_source in data_sources:
+            self._service.data_sources.load_data(data_source_id=data_source.id)

src/hydroserverpy/components/data_sources.py

@@ -0,0 +1,98 @@
+import tempfile
+from urllib.request import urlopen
+from uuid import UUID
+from typing import List
+from hydroserverpy.schemas.data_sources import DataSourceGetResponse, DataSourcePostBody, DataSourcePatchBody
+from hydroserverpy.schemas.datastreams import DatastreamGetResponse
+from hydroserverpy.etl import HydroServerETL
+
+
+class DataSource:
+
+    def __init__(self, service):
+        self._service = service
+
+    def list(self):
+
+        return self._service.get(
+            'data-sources',
+            response_schema=List[DataSourceGetResponse]
+        )
+
+    def list_datastreams(self, data_source_id: UUID):
+
+        return self._service.get(
+            f'data-sources/{data_source_id}/datastreams',
+            response_schema=List[DatastreamGetResponse]
+        )
+
+    def get(self, data_source_id: UUID):
+
+        return self._service.get(
+            f'data-sources/{data_source_id}',
+            response_schema=DataSourceGetResponse
+        )
+
+    def create(self, data_source_body: DataSourcePostBody):
+
+        return self._service.post(
+            f'data-sources',
+            headers={'Content-type': 'application/json'},
+            data=data_source_body.json(by_alias=True),
+            response_schema=DataSourceGetResponse
+        )
+
+    def update(self, data_source_id: UUID, data_source_body: DataSourcePatchBody):
+        return self._service.patch(
+            f'data-sources/{str(data_source_id)}',
+            headers={'Content-type': 'application/json'},
+            data=data_source_body.json(exclude_unset=True, by_alias=True),
+            response_schema=DataSourceGetResponse
+        )
+
+    def delete(self, data_source_id: UUID):
+
+        return self._service.delete(
+            f'data-sources/{str(data_source_id)}'
+        )
+
+    def load_data(self, data_source_id: UUID):
+
+        data_source_response = self.get(data_source_id=data_source_id)
+
+        if data_source_response.data:
+            data_source = data_source_response.data
+        else:
+            return None
+
+        datastreams_response = self.list_datastreams(data_source_id=data_source_id)
+        datastreams = datastreams_response.data
+
+        if data_source.path:
+            with open(data_source.path) as data_file:
+                hs_etl = HydroServerETL(
+                    service=self._service,
+                    data_file=data_file,
+                    data_source=data_source,
+                    datastreams=datastreams
+                )
+                hs_etl.run()
+        elif data_source.url:
+            with tempfile.NamedTemporaryFile(mode='w') as temp_file:
+                with urlopen(data_source.url) as response:
+                    chunk_size = 1024 * 1024 * 10  # Use a 10mb chunk size.
+                    while True:
+                        chunk = response.read(chunk_size)
+                        if not chunk:
+                            break
+                        temp_file.write(chunk)
+                temp_file.seek(0)
+                hs_etl = HydroServerETL(
+                    service=self._service,
+                    data_file=temp_file,
+                    data_source=data_source,
+                    datastreams=datastreams
+                )
+                hs_etl.run()
+        else:
+            return None

src/hydroserverpy/components/datastreams.py

@@ -0,0 +1,47 @@
+from uuid import UUID
+from typing import List
+from hydroserverpy.schemas.datastreams import DatastreamGetResponse, DatastreamPostBody, DatastreamPatchBody
+
+
+class Datastream:
+
+    def __init__(self, service):
+        self._service = service
+
+    def list(self):
+
+        return self._service.get(
+            'datastreams',
+            response_schema=List[DatastreamGetResponse]
+        )
+
+    def get(self, datastream_id: str):
+
+        return self._service.get(
+            f'datastreams/{datastream_id}',
+            response_schema=DatastreamGetResponse
+        )
+
+    def create(self, datastream_body: DatastreamPostBody):
+
+        return self._service.post(
+            f'datastreams',
+            headers={'Content-type': 'application/json'},
+            data=datastream_body.json(by_alias=True),
+            response_schema=DatastreamGetResponse
+        )
+
+    def update(self, datastream_id: UUID, datastream_body: DatastreamPatchBody):
+
+        return self._service.patch(
+            f'datastreams/{str(datastream_id)}',
+            headers={'Content-type': 'application/json'},
+            data=datastream_body.json(exclude_unset=True, by_alias=True),
+            response_schema=DatastreamGetResponse
+        )
+
+    def delete(self, datastream_id: UUID):
+
+        return self._service.delete(
+            f'datastreams/{str(datastream_id)}'
+        )

src/hydroserverpy/components/observed_properties.py

@@ -0,0 +1,48 @@
+from typing import List
+from uuid import UUID
+from hydroserverpy.schemas.observed_properties import ObservedPropertyGetResponse, ObservedPropertyPostBody, \
+     ObservedPropertyPatchBody
+
+
+class ObservedProperty:
+
+    def __init__(self, service):
+        self._service = service
+
+    def list(self):
+
+        return self._service.get(
+            'observed-properties',
+            response_schema=List[ObservedPropertyGetResponse]
+        )
+
+    def get(self, observed_property_id: UUID):
+
+        return self._service.get(
+            f'observed-properties/{observed_property_id}',
+            response_schema=ObservedPropertyGetResponse
+        )
+
+    def create(self, observed_property_body: ObservedPropertyPostBody):
+
+        return self._service.post(
+            f'observed-properties',
+            headers={'Content-type': 'application/json'},
+            data=observed_property_body.json(by_alias=True),
+            response_schema=ObservedPropertyGetResponse
+        )
+
+    def update(self, observed_property_id: UUID, observed_property_body: ObservedPropertyPatchBody):
+
+        return self._service.patch(
+            f'observed-properties/{str(observed_property_id)}',
+            headers={'Content-type': 'application/json'},
+            data=observed_property_body.json(exclude_unset=True, by_alias=True),
+            response_schema=ObservedPropertyGetResponse
+        )
+
+    def delete(self, observed_property_id: UUID):
+
+        return self._service.delete(
+            f'observed-properties/{str(observed_property_id)}'
+        )

src/hydroserverpy/components/processing_levels.py

@@ -0,0 +1,48 @@
+from typing import List
+from uuid import UUID
+from hydroserverpy.schemas.processing_levels import ProcessingLevelGetResponse, ProcessingLevelPostBody, \
+     ProcessingLevelPatchBody
+
+
+class ProcessingLevel:
+
+    def __init__(self, service):
+        self._service = service
+
+    def list(self):
+
+        return self._service.get(
+            'processing-levels',
+            response_schema=List[ProcessingLevelGetResponse]
+        )
+
+    def get(self, processing_level_id: str):
+
+        return self._service.get(
+            f'processing-levels/{processing_level_id}',
+            response_schema=ProcessingLevelGetResponse
+        )
+
+    def create(self, processing_level_body: ProcessingLevelPostBody):
+
+        return self._service.post(
+            f'processing-levels',
+            headers={'Content-type': 'application/json'},
+            data=processing_level_body.json(by_alias=True),
+            response_schema=ProcessingLevelGetResponse
+        )
+
+    def update(self, processing_level_id: UUID, processing_level_body: ProcessingLevelPatchBody):
+
+        return self._service.patch(
+            f'processing-levels/{str(processing_level_id)}',
+            headers={'Content-type': 'application/json'},
+            data=processing_level_body.json(exclude_unset=True, by_alias=True),
+            response_schema=ProcessingLevelGetResponse
+        )
+
+    def delete(self, processing_level_id: UUID):
+
+        return self._service.delete(
+            f'processing-levels/{str(processing_level_id)}'
+        )