From a40093f8af6c81af185d457661d39689d9c9e276 Mon Sep 17 00:00:00 2001 From: John Kyrre Hansen Date: Thu, 6 Jul 2023 15:24:20 +0200 Subject: [PATCH] awt#218 --- tagreader/cache.py | 9 +++- tagreader/clients.py | 2 + tests/test_AspenHandlerODBC_connect.py | 2 +- tests/test_AspenHandlerREST_connect.py | 7 ++- tests/test_PIHandlerODBC_connect.py | 2 +- tests/test_PIHandlerREST_connect.py | 7 ++- tests/test_cache.py | 71 +++++++++++++++++++------- tests/test_clients.py | 49 +++++++++++++----- tests/test_data_integrity.py | 20 ++++++-- 9 files changed, 126 insertions(+), 43 deletions(-) diff --git a/tagreader/cache.py b/tagreader/cache.py index 3747c999..d8ae08cd 100644 --- a/tagreader/cache.py +++ b/tagreader/cache.py @@ -313,6 +313,7 @@ def key_path( df: Union[str, pd.DataFrame], read_type: ReaderType, ts: Optional[Union[int, timedelta]] = None, + get_status: bool = False, ) -> str: """Return a string on the form XXX$sYY$ZZZ where XXX is the ReadType, YY is the interval between samples @@ -340,8 +341,9 @@ def store( df: pd.DataFrame, read_type: ReaderType, ts: Optional[Union[int, timedelta]] = None, + get_status: bool = False, ) -> None: - key = self.key_path(df=df, read_type=read_type, ts=ts) + key = self.key_path(df=df, read_type=read_type, ts=ts, get_status=get_status) if df.empty: return # Weirdness ensues when using empty df in select statement below if key in self: @@ -363,8 +365,11 @@ def fetch( ts: Optional[Union[int, timedelta]] = None, start: Optional[datetime] = None, end: Optional[datetime] = None, + get_status: bool = False, ) -> pd.DataFrame: - key = self.key_path(df=tagname, read_type=read_type, ts=ts) + key = self.key_path( + df=tagname, read_type=read_type, ts=ts, get_status=get_status + ) df = cast(Optional[pd.DataFrame], self.get(key=key)) if df is None: return pd.DataFrame() diff --git a/tagreader/clients.py b/tagreader/clients.py index aaceeb9c..82f97349 100644 --- a/tagreader/clients.py +++ b/tagreader/clients.py @@ -311,6 +311,7 @@ def __init__( verifySSL: bool = True, auth: Optional[Any] = None, cache: Optional[Union[SmartCache, BucketCache]] = None, + get_status: bool = False, ): if isinstance(imstype, str): try: @@ -402,6 +403,7 @@ def _read_single_tag( ts=ts, start=time_slice[0], end=time_slice[1], + get_status=get_status, ) missing_intervals = get_missing_intervals( df=df, diff --git a/tests/test_AspenHandlerODBC_connect.py b/tests/test_AspenHandlerODBC_connect.py index f72da0cb..d1dbb7de 100644 --- a/tests/test_AspenHandlerODBC_connect.py +++ b/tests/test_AspenHandlerODBC_connect.py @@ -30,7 +30,7 @@ @pytest.fixture # type: ignore[misc] def client() -> Generator[IMSClient, None, None]: - c = IMSClient(datasource=SOURCE, imstype="ip21") + c = IMSClient(datasource=SOURCE, imstype="ip21", get_status=False) c.cache = None # type: ignore[assignment] c.connect() yield c diff --git a/tests/test_AspenHandlerREST_connect.py b/tests/test_AspenHandlerREST_connect.py index ed7f8607..c50c04d6 100644 --- a/tests/test_AspenHandlerREST_connect.py +++ b/tests/test_AspenHandlerREST_connect.py @@ -33,7 +33,12 @@ @pytest.fixture # type: ignore[misc] def client() -> Generator[IMSClient, None, None]: - c = IMSClient(datasource=SOURCE, imstype="aspenone", verifySSL=bool(VERIFY_SSL)) + c = IMSClient( + datasource=SOURCE, + imstype="aspenone", + verifySSL=bool(VERIFY_SSL), + get_status=False, + ) c.cache = None # type: ignore[assignment] c.connect() yield c diff --git a/tests/test_PIHandlerODBC_connect.py b/tests/test_PIHandlerODBC_connect.py index d47feb47..62a8bed2 100644 --- a/tests/test_PIHandlerODBC_connect.py +++ b/tests/test_PIHandlerODBC_connect.py @@ -37,7 +37,7 @@ @pytest.fixture # type: ignore[misc] def client() -> Generator[IMSClient, None, None]: - c = IMSClient(datasource=SOURCE, imstype="pi") + c = IMSClient(datasource=SOURCE, imstype="pi", get_status=False) c.cache = None # type: ignore[assignment] c.connect() yield c diff --git a/tests/test_PIHandlerREST_connect.py b/tests/test_PIHandlerREST_connect.py index 2423baa8..647e384e 100644 --- a/tests/test_PIHandlerREST_connect.py +++ b/tests/test_PIHandlerREST_connect.py @@ -32,7 +32,12 @@ @pytest.fixture # type: ignore[misc] def client() -> Generator[IMSClient, None, None]: - c = IMSClient(datasource=SOURCE, imstype="piwebapi", verifySSL=bool(verifySSL)) + c = IMSClient( + datasource=SOURCE, + imstype="piwebapi", + verifySSL=bool(verifySSL), + get_status=False, + ) c.cache = None # type: ignore[assignment] c.connect() c.handler._max_rows = 1000 # For the long raw test diff --git a/tests/test_cache.py b/tests/test_cache.py index a12d409f..a4fdc997 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -52,14 +52,18 @@ def test_key_path(cache: SmartCache) -> None: pass -def test_cache_single_store_and_fetch(cache: SmartCache, data: pd.DataFrame) -> None: - cache.store(df=data, read_type=ReaderType.INT) - df_read = cache.fetch(tagname="tag1", read_type=ReaderType.INT, ts=60) +def test_cache_single_store_and_fetch( + cache: SmartCache, data: pd.DataFrame, get_status: bool = False +) -> None: + cache.store(df=data, read_type=ReaderType.INT, get_status=get_status) + df_read = cache.fetch( + tagname="tag1", read_type=ReaderType.INT, ts=60, get_status=get_status + ) pd.testing.assert_frame_equal(data, df_read) def test_cache_multiple_store_single_fetch( - cache: SmartCache, data: pd.DataFrame + cache: SmartCache, data: pd.DataFrame, get_status: bool = False ) -> None: df1 = data[0:3] df2 = data[2:10] @@ -69,22 +73,42 @@ def test_cache_multiple_store_single_fetch( pd.testing.assert_frame_equal(df_read, data) -def test_interval_reads(cache: SmartCache, data: pd.DataFrame) -> None: - cache.store(df=data, read_type=ReaderType.INT) +def test_interval_reads( + cache: SmartCache, data: pd.DataFrame, get_status: bool = False +) -> None: + cache.store(df=data, read_type=ReaderType.INT, get_status=get_status) start_oob = pd.to_datetime("2018-01-18 04:55:00") start = pd.to_datetime("2018-01-18 05:05:00") end = pd.to_datetime("2018-01-18 05:08:00") end_oob = pd.to_datetime("2018-01-18 06:00:00") - df_read = cache.fetch(tagname="tag1", read_type=ReaderType.INT, ts=60, start=start) + df_read = cache.fetch( + tagname="tag1", + read_type=ReaderType.INT, + ts=60, + start=start, + get_status=get_status, + ) pd.testing.assert_frame_equal(data[start:], df_read) # type: ignore[misc] - df_read = cache.fetch(tagname="tag1", read_type=ReaderType.INT, ts=60, end=end) + df_read = cache.fetch( + tagname="tag1", read_type=ReaderType.INT, ts=60, end=end, get_status=get_status + ) pd.testing.assert_frame_equal(data[:end], df_read) # type: ignore[misc] df_read = cache.fetch( - tagname="tag1", read_type=ReaderType.INT, ts=60, start=start_oob + tagname="tag1", + read_type=ReaderType.INT, + ts=60, + start=start_oob, + get_status=get_status, ) pd.testing.assert_frame_equal(data, df_read) - df_read = cache.fetch(tagname="tag1", read_type=ReaderType.INT, ts=60, end=end_oob) + df_read = cache.fetch( + tagname="tag1", + read_type=ReaderType.INT, + ts=60, + end=end_oob, + get_status=get_status, + ) pd.testing.assert_frame_equal(data, df_read) df_read = cache.fetch( tagname="tag1", @@ -92,18 +116,23 @@ def test_interval_reads(cache: SmartCache, data: pd.DataFrame) -> None: ts=60, start=start, end=end, + get_status=get_status, ) pd.testing.assert_frame_equal(data[start:end], df_read) # type: ignore[misc] -def test_store_empty_df(cache: SmartCache, data: pd.DataFrame) -> None: +def test_store_empty_df( + cache: SmartCache, data: pd.DataFrame, get_status: bool = False +) -> None: # Empty dataframes should not be stored (note: df full of NaN is not empty!) - cache.store(df=data, read_type=ReaderType.INT) + cache.store(df=data, read_type=ReaderType.INT, get_status=get_status) df = pd.DataFrame({"tag1": []}) cache.store( df=df, read_type=ReaderType.INT, ts=60 ) # Specify ts to ensure correct key /if/ stored - df_read = cache.fetch(tagname="tag1", read_type=ReaderType.INT, ts=60) + df_read = cache.fetch( + tagname="tag1", read_type=ReaderType.INT, ts=60, get_status=get_status + ) pd.testing.assert_frame_equal(data, df_read) @@ -120,7 +149,7 @@ def test_store_metadata(cache: SmartCache) -> None: assert "noworky" not in r -def test_to_dst_skips_time(cache: SmartCache) -> None: +def test_to_dst_skips_time(cache: SmartCache, get_status: bool = False) -> None: index = pd.date_range( start="2018-03-25 01:50:00", end="2018-03-25 03:30:00", @@ -133,12 +162,14 @@ def test_to_dst_skips_time(cache: SmartCache) -> None: assert ( df.loc["2018-03-25 01:50:00":"2018-03-25 03:10:00"].size == (2 + 1 * 6 + 1) - 6 # type: ignore[misc] ) - cache.store(df=df, read_type=ReaderType.INT) - df_read = cache.fetch(tagname="tag1", read_type=ReaderType.INT, ts=600) + cache.store(df=df, read_type=ReaderType.INT, get_status=get_status) + df_read = cache.fetch( + tagname="tag1", read_type=ReaderType.INT, ts=600, get_status=get_status + ) pd.testing.assert_frame_equal(df_read, df) -def test_from_dst_folds_time(cache: SmartCache) -> None: +def test_from_dst_folds_time(cache: SmartCache, get_status: bool = False) -> None: index = pd.date_range( start="2017-10-29 00:30:00", end="2017-10-29 04:30:00", @@ -158,6 +189,8 @@ def test_from_dst_folds_time(cache: SmartCache) -> None: assert ( df.loc["2017-10-29 01:50:00":"2017-10-29 03:10:00"].size == 2 + (1 + 1) * 6 + 1 # type: ignore[misc] ) - cache.store(df=df, read_type=ReaderType.INT) - df_read = cache.fetch(tagname="tag1", read_type=ReaderType.INT, ts=600) + cache.store(df=df, read_type=ReaderType.INT, get_status=get_status) + df_read = cache.fetch( + tagname="tag1", read_type=ReaderType.INT, ts=600, get_status=get_status + ) pd.testing.assert_frame_equal(df_read, df) diff --git a/tests/test_clients.py b/tests/test_clients.py index 46d5c69b..47c50cac 100644 --- a/tests/test_clients.py +++ b/tests/test_clients.py @@ -58,55 +58,78 @@ class TestODBC: def test_pi_init_odbc_client_with_host_port(self) -> None: host = "thehostname" port = 999 - c = IMSClient(datasource="whatever", imstype="pi", host=host) + get_status = False + c = IMSClient( + datasource="whatever", imstype="pi", host=host, get_status=get_status + ) assert c.handler.host == host assert c.handler.port == 5450 - c = IMSClient(datasource="whatever", imstype="pi", host=host, port=port) + c = IMSClient( + datasource="whatever", + imstype="pi", + host=host, + port=port, + get_status=get_status, + ) assert c.handler.host == host assert c.handler.port == port def test_ip21_init_odbc_client_with_host_port(self) -> None: host = "thehostname" port = 999 - c = IMSClient(datasource="whatever", imstype="ip21", host=host) + get_status = False + c = IMSClient( + datasource="whatever", imstype="ip21", host=host, get_status=get_status + ) assert c.handler.host == host assert c.handler.port == 10014 - c = IMSClient(datasource="whatever", imstype="ip21", host=host, port=port) + c = IMSClient( + datasource="whatever", + imstype="ip21", + host=host, + port=port, + get_status=get_status, + ) assert c.handler.host == host assert c.handler.port == port def test_pi_connection_string_override(self) -> None: connstr = "someuserspecifiedconnectionstring" + get_status = False c = IMSClient( datasource="whatever", - host="host", imstype="pi", + host="host", handler_options={"connection_string": connstr}, + get_status=get_status, ) assert c.handler.generate_connection_string() == connstr def test_ip21_connection_string_override(self) -> None: connstr = "someuserspecifiedconnectionstring" + get_status = False c = IMSClient( datasource="whatever", - host="host", imstype="ip21", + host="host", handler_options={"connection_string": connstr}, + get_status=get_status, ) assert c.handler.generate_connection_string() == connstr def test_init_odbc_clients(self) -> None: + get_status = False with pytest.raises(ValueError): - _ = IMSClient(datasource="xyz") + _ = IMSClient(datasource="xyz", get_status=get_status) with pytest.raises(ValueError): - _ = IMSClient(datasource="sNa", imstype="pi") + _ = IMSClient(datasource="sNa", imstype="pi", get_status=get_status) with pytest.raises(ValueError): - _ = IMSClient(datasource="Ono-imS", imstype="aspen") + _ = IMSClient(datasource="Ono-imS", imstype="aspen", get_status=get_status) with pytest.raises(ValueError): - _ = IMSClient(datasource="ono-ims", imstype="aspen") + _ = IMSClient(datasource="ono-ims", imstype="aspen", get_status=get_status) with pytest.raises(ValueError): - _ = IMSClient(datasource="sna", imstype="pi") - c = IMSClient(datasource="onO-iMs", imstype="pi") + _ = IMSClient(datasource="sna", imstype="pi", get_status=get_status) + c = IMSClient(datasource="onO-iMs", imstype="pi", get_status=get_status) assert isinstance(c.handler, PIHandlerODBC) - c = IMSClient(datasource="snA", imstype="aspen") + c = IMSClient(datasource="snA", imstype="aspen", get_status=get_status) assert isinstance(c.handler, AspenHandlerODBC) diff --git a/tests/test_data_integrity.py b/tests/test_data_integrity.py index 5ff854f1..59431c74 100644 --- a/tests/test_data_integrity.py +++ b/tests/test_data_integrity.py @@ -40,7 +40,7 @@ @pytest.fixture # type: ignore[misc] def pi_client_odbc() -> Generator[IMSClient, None, None]: - c = IMSClient(datasource=PI_DS, imstype="pi") + c = IMSClient(datasource=PI_DS, imstype="pi", get_status=False) if os.path.exists(PI_DS + ".h5"): os.remove(PI_DS + ".h5") c.cache = None # type: ignore[assignment] @@ -52,7 +52,9 @@ def pi_client_odbc() -> Generator[IMSClient, None, None]: @pytest.fixture # type: ignore[misc] def pi_client_web() -> Generator[IMSClient, None, None]: - c = IMSClient(datasource=PI_DS, imstype="piwebapi", verifySSL=verifySSL) + c = IMSClient( + datasource=PI_DS, imstype="piwebapi", verifySSL=verifySSL, get_status=False + ) if os.path.exists(PI_DS + ".h5"): os.remove(PI_DS + ".h5") c.cache = None # type: ignore[assignment] @@ -64,7 +66,7 @@ def pi_client_web() -> Generator[IMSClient, None, None]: @pytest.fixture # type: ignore[misc] def aspen_client_odbc() -> Generator[IMSClient, None, None]: - c = IMSClient(datasource=ASPEN_DS, imstype="ip21") + c = IMSClient(datasource=ASPEN_DS, imstype="ip21", get_status=False) if os.path.exists(ASPEN_DS + ".h5"): os.remove(ASPEN_DS + ".h5") c.cache = None # type: ignore[assignment] @@ -76,7 +78,12 @@ def aspen_client_odbc() -> Generator[IMSClient, None, None]: @pytest.fixture # type: ignore[misc] def aspen_client_web() -> Generator[IMSClient, None, None]: - c = IMSClient(datasource=ASPEN_DS, imstype="aspenone", verifySSL=bool(verifySSL)) + c = IMSClient( + datasource=ASPEN_DS, + imstype="aspenone", + verifySSL=bool(verifySSL), + get_status=False, + ) if os.path.exists(ASPEN_DS + ".h5"): os.remove(ASPEN_DS + ".h5") c.cache = None # type: ignore[assignment] @@ -234,7 +241,9 @@ def test_concat_proper_fill_up(pi_client_web: IMSClient) -> None: pi_client_web.handler._max_rows = max_rows_backup -def test_cache_proper_fill_up(pi_client_web: IMSClient, tmp_path: Path) -> None: +def test_cache_proper_fill_up( + pi_client_web: IMSClient, tmp_path: Path, get_status: bool = False +) -> None: pi_client_web.cache = SmartCache(directory=tmp_path) df_int_1 = pi_client_web.read( tags=PI_TAG, @@ -258,5 +267,6 @@ def test_cache_proper_fill_up(pi_client_web: IMSClient, tmp_path: Path) -> None: ts=TS, start=ensure_datetime_with_tz(PI_START_TIME), end=ensure_datetime_with_tz(PI_END_TIME_2), + get_status=get_status, ) assert len(df_cached) == 32