Fixed imports and logging issues. Fixed header parsing issue.

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = hydroserverpy
-version = 0.2.0
+version = 0.2.1
 
 [options]
 package_dir=

src/hydroserverpy/__init__.py

@@ -1,13 +1,19 @@
-from hydroserverpy.main import HydroServer
-from hydroserverpy.schemas.things import ThingPostBody
-from hydroserverpy.schemas.observed_properties import ObservedPropertyPostBody
-from hydroserverpy.schemas.units import UnitPostBody
-from hydroserverpy.schemas.datastreams import DatastreamPostBody
+from .main import HydroServer
+from .schemas.things import ThingPostBody
+from .schemas.sensors import SensorPostBody
+from .schemas.observed_properties import ObservedPropertyPostBody
+from .schemas.units import UnitPostBody
+from .schemas.processing_levels import ProcessingLevelPostBody
+from .schemas.result_qualifiers import ResultQualifierPostBody
+from .schemas.datastreams import DatastreamPostBody
 
 __all__ = [
     "HydroServer",
     "ThingPostBody",
+    "SensorPostBody",
     "ObservedPropertyPostBody",
     "UnitPostBody",
+    "ProcessingLevelPostBody",
+    "ResultQualifierPostBody",
     "DatastreamPostBody"
 ]

src/hydroserverpy/etl.py

@@ -5,10 +5,10 @@
 from typing import IO, List
 from datetime import datetime, timezone, timedelta
 from dateutil.parser import isoparse
-from hydroserverpy.schemas.data_sources import DataSourceGetResponse
-from hydroserverpy.schemas.datastreams import DatastreamGetResponse
-from hydroserverpy.exceptions import HeaderParsingError, TimestampParsingError
-from hydroserverpy.schemas.data_sources import DataSourcePatchBody
+from .schemas.data_sources import DataSourceGetResponse
+from .schemas.datastreams import DatastreamGetResponse
+from .exceptions import HeaderParsingError, TimestampParsingError
+from .schemas.data_sources import DataSourcePatchBody
 
 logger = logging.getLogger('hydroserver_etl')
 logger.addHandler(logging.NullHandler())
@@ -137,7 +137,7 @@ def _parse_file_header(self, row: List[str]) -> None:
 
         try:
             self._timestamp_column_index = row.index(self._data_source.timestamp_column) \
-                if not self._data_source.timestamp_column.isdigit() \
+                if isinstance(self._data_source.timestamp_column, str) \
                 else int(self._data_source.timestamp_column) - 1
             if self._timestamp_column_index > len(row):
                 raise ValueError
@@ -151,6 +151,7 @@ def _parse_file_header(self, row: List[str]) -> None:
                     max(self._datastream_column_indexes.values()) > len(row):
                 raise ValueError
         except ValueError as e:
+            logger.error(f'Failed to load data from data source: "{self._data_source.name}"')
             raise HeaderParsingError(str(e)) from e
 
     def _parse_row_timestamp(self, row: List[str]) -> datetime:
@@ -163,7 +164,7 @@ def _parse_row_timestamp(self, row: List[str]) -> datetime:
         """
 
         try:
-            if self._data_source.timestamp_format == 'iso':
+            if self._data_source.timestamp_format == 'iso' or self._data_source.timestamp_format is None:
                 timestamp = isoparse(
                     row[self._timestamp_column_index]
                 )
@@ -188,6 +189,7 @@ def _parse_row_timestamp(self, row: List[str]) -> datetime:
                         ).tzinfo
                     )
                 except ValueError as e:
+                    logger.error(f'Failed to load data from data source: "{self._data_source.name}"')
                     raise TimestampParsingError(str(e)) from e
 
         return timestamp
@@ -214,7 +216,7 @@ def _post_observations(self) -> List[str]:
                     f'Loading observations from ' +
                     f'{observations[0]["phenomenon_time"].strftime("%Y-%m-%dT%H:%M:%S%z")} to ' +
                     f'{observations[-1]["phenomenon_time"].strftime("%Y-%m-%dT%H:%M:%S%z")} for datastream: ' +
-                    f'{str(datastream_id)}.'
+                    f'{str(datastream_id)} in data source "{self._data_source.name}".'
                 )
 
                 data_array_value = getattr(fsc.model, 'ext').data_array_value.DataArrayValue()
@@ -250,7 +252,8 @@ def _post_observations(self) -> List[str]:
                     f'Skipping observations POST request from ' +
                     f'{observations[0]["phenomenon_time"].strftime("%Y-%m-%dT%H:%M:%S%z")} to ' +
                     f'{observations[-1]["phenomenon_time"].strftime("%Y-%m-%dT%H:%M:%S%z")} for datastream: ' +
-                    f'{str(datastream_id)}, due to previous failed POST request.'
+                    f'{str(datastream_id)} in data source "{self._data_source.name}",' +
+                    f'due to previous failed POST request.'
                 )
 
         self._observations = {}

src/hydroserverpy/schemas/datastreams.py

@@ -44,7 +44,9 @@ class Config:
 
 
 class DatastreamPostBody(DatastreamFields):
-    pass
+
+    class Config:
+        allow_population_by_field_name = True
 
 
 @allow_partial

src/hydroserverpy/schemas/observed_properties.py

@@ -18,7 +18,7 @@ class ObservedPropertyFields(BaseModel):
 
 
 class ObservedPropertyGetResponse(ObservedPropertyFields, ObservedPropertyID):
-    owner: Optional[UserFields]
+    owner: Optional[str]
 
     class Config:
         allow_population_by_field_name = True

src/hydroserverpy/schemas/processing_levels.py

@@ -1,8 +1,8 @@
 from pydantic import BaseModel
 from uuid import UUID
 from typing import Optional
-from hydroserverpy.utils import allow_partial
-from hydroserverpy.schemas.users import UserFields
+from ..utils import allow_partial
+from ..schemas.users import UserFields
 
 
 class ProcessingLevelID(BaseModel):
@@ -16,14 +16,16 @@ class ProcessingLevelFields(BaseModel):
 
 
 class ProcessingLevelGetResponse(ProcessingLevelFields, ProcessingLevelID):
-    owner: Optional[UserFields]
+    owner: Optional[str]
 
     class Config:
         allow_population_by_field_name = True
 
 
 class ProcessingLevelPostBody(ProcessingLevelFields):
-    pass
+
+    class Config:
+        allow_population_by_field_name = True
 
 
 @allow_partial

src/hydroserverpy/schemas/result_qualifiers.py

@@ -1,8 +1,8 @@
 from pydantic import BaseModel
 from uuid import UUID
 from typing import Optional
-from hydroserverpy.utils import allow_partial
-from hydroserverpy.schemas.users import UserFields
+from ..utils import allow_partial
+from ..schemas.users import UserFields
 
 
 class ResultQualifierID(BaseModel):
@@ -15,14 +15,16 @@ class ResultQualifierFields(BaseModel):
 
 
 class ResultQualifierGetResponse(ResultQualifierFields, ResultQualifierID):
-    owner: Optional[UserFields]
+    owner: Optional[str]
 
     class Config:
         allow_population_by_field_name = True
 
 
 class ResultQualifierPostBody(ResultQualifierFields):
-    pass
+
+    class Config:
+        allow_population_by_field_name = True
 
 
 @allow_partial

src/hydroserverpy/schemas/sensors.py

@@ -22,14 +22,16 @@ class SensorFields(BaseModel):
 
 
 class SensorGetResponse(SensorFields, SensorID):
-    owner: Optional[UserFields]
+    owner: Optional[str]
 
     class Config:
         allow_population_by_field_name = True
 
 
 class SensorPostBody(SensorFields):
-    pass
+
+    class Config:
+        allow_population_by_field_name = True
 
 
 @allow_partial

src/hydroserverpy/schemas/units.py

@@ -17,7 +17,7 @@ class UnitFields(BaseModel):
 
 
 class UnitGetResponse(UnitFields, UnitID):
-    owner: Optional[UserFields]
+    owner: Optional[str]
 
     class Config:
         allow_population_by_field_name = True