Moved out tests to "test" directory

.gitignore

@@ -266,4 +266,7 @@ fabric.properties
 # Ignore all local history of files
 .history
 
-# End of https://www.gitignore.io/api/python,django,pycharm,visualstudiocode
+# End of https://www.gitignore.io/api/python,django,pycharm,visualstudiocode
+#
+cdm_interface/try.sh
+cdm_interface/outputs

test/test-bbox.py

@@ -0,0 +1,91 @@
+import requests
+import pandas as pd
+import io
+import math
+
+
+TMPL = 'http://glamod2.ceda.ac.uk/select/?domain=land&frequency=monthly&variable=accumulated_precipitation,air_temperature&intended_use=non-commercial&data_quality=quality_controlled&column_selection=detailed_metadata&year=1974&month=03&bbox={w}.0,{s}.0,{e}.0,{n}.0&compress=false'
+
+
+def _assert_in_range(df, w, s, e, n, to_nearest_degree=False):
+
+    if len(df) == 0:
+        print('Empty df')
+        return
+
+    lats, lons = df.latitude, df.longitude
+    min_lat, max_lat = lats.min(), lats.max()
+    min_lon, max_lon = lons.min(), lons.max()
+
+    print(f'Wanted lons: {w} to {e};      lats: {s} to {n}')
+    print(f'Actual lons: {min_lon} to {max_lon}; lats: {min_lat} to {max_lat}')
+
+    def fix(n):
+        if n < 0: 
+            return math.ceil(n)
+        else:
+            return math.floor(n)
+
+    if to_nearest_degree:
+        min_lat, max_lat, min_lon, max_lon = [fix(_) for _ in [min_lat, max_lat, min_lon, max_lon]]
+
+#    print(lats, lats.max(), lats.min())
+
+    assert(min_lat >= s), 'min_lat >= s'
+    assert(max_lat <= n), 'max_lat <= n'
+
+    if min_lat == max_lat and min_lat == -90 or min_lat == 90:
+        print('Longitude checks are meaningless at the north/south pole')
+        return
+
+    if 90 in list(lats) or -90 in list(lats):
+        print('Some lats are north/south pole - so ignore longitude checks')
+
+    assert(min_lon >= w), 'min_lon >= w'
+    assert(max_lon <= e), 'max_lon <= e'
+
+
+def _fetch_as_df(w, s, e, n):
+    url = TMPL.format(**vars())
+    print(f'{url}')
+
+    content = requests.get(url).text
+
+    if content.startswith('Exception raised'):
+        print(f'[ERROR] Fetch error: {content}')
+        return content
+
+    return pd.read_csv(io.StringIO(content)) 
+
+
+def test_bbox_in_range():
+
+    for w in range(-180, 160, 30):
+        e = w + 30
+
+        for s in range(-90, 61, 30):
+            n = s + 30
+
+            df = _fetch_as_df(w, s, e, n)
+            _assert_in_range(df, w, s, e, n, True)
+
+
+def test_bbox_full_range():
+    bboxes = ['-180,-90,180,90']  #, '-90,90,-180,180', '-90,-180,90,180']
+
+    for bbox in bboxes:
+        w, s, e, n = [int(_) for _ in bbox.split(',')]
+
+        df = _fetch_as_df(w, s, e, n)
+
+        if type(df) == str:
+            continue
+
+        _assert_in_range(df, w, s, e, n, True) 
+
+
+
+if __name__ == '__main__':
+
+    test_bbox_full_range()    
+    test_bbox_in_range()

test/test-read.py

@@ -0,0 +1,10 @@
+fpath = '/gws/nopw/j04/c3s311a_lot2/data/level2/land/r202005/station_configuration/station_configuration.psv'
+
+with open(fpath) as reader:
+
+ try:
+   for _, i in enumerate(reader):
+     pass
+
+ except Exception as exc:
+   print(f'Failed at: {_}')

test/test-time.py

@@ -0,0 +1,32 @@
+from utils import decompose_datetime
+import datetime
+UTC = datetime.timezone.utc
+
+
+def n(time_range):
+
+#        import pdb; pdb.set_trace()
+        if '/' not in time_range:
+             raise Exception(f'Time range must be provided as "<start_time>/<end_time>", not "{time_range}".')
+
+        start, end = time_range.split('/')
+        start = decompose_datetime(start, 'start')
+        end = decompose_datetime(end, 'end')
+
+        if start.year != end.year:
+            raise Exception('Time range selections must be a maximum of 1 year. Please modify your request.')
+
+        start_time, end_time = [_.astimezone(UTC) for _ in (start, end)]
+        start_time2, end_time2 = [f'{_}+00:00' for _ in (start, end)]
+
+        print( str(start_time), start_time2)
+        assert str(start_time) == start_time2
+        assert str(end_time) == end_time2
+        print(start_time)
+        time_condition = f"date_time BETWEEN '{start_time}'::timestamptz AND '{end_time}'::timestamptz;"
+        return time_condition
+
+
+n('1900-01-01/1900-02-01')
+
+n('1900-01-01T12:23:31/1900-02-01T14')

test/test_db_query.py

@@ -0,0 +1,67 @@
+import os
+
+import pandas as pd
+import psycopg2
+
+
+SCHEMA = 'lite_2_0'
+
+pw = os.environ.get('PGPW_READER')
+if not pw:
+    raise Exception('No password!')
+
+# Make connection
+conn_str = f'dbname=cdm user=glamod_read host=localhost password={pw}'
+conn = psycopg2.connect(conn_str)
+
+# Open a cursor to perform database operations
+cur = conn.cursor()
+
+# Query the database and obtain data as Python objects
+query = f"SELECT * FROM {SCHEMA}.observations WHERE date_time < '1763-01-01';"
+query2 = f"SELECT * FROM {SCHEMA}.observations_1763_land_2 WHERE date_time < '1763-01-01';"
+
+# select ST_AsText(location) from lite.observations_2011_land_2 where date_time > '2011-11-01' AND
+# observed_variable = 85 AND
+# ST_Intersects(ST_Polygon('LINESTRING(50 10, 60 10, 60 26, 50 26, 50 10)'::geometry, 4326), location);
+# bbox: lon lat, lon lat, ...
+
+def _bbox_to_linestring(w, s, e, n, srid='4326'):
+    return f"ST_Polygon('LINESTRING({w} {s}, {w} {n}, {e} {n}, {e} {s}, {w} {s})'::geometry, {srid})"
+
+
+bbox = (-1, 50, 10, 60)
+ls = _bbox_to_linestring(*bbox)
+
+fields = 'ST_AsText(location)'
+query3 = f"SELECT {fields} FROM {SCHEMA}.observations_2011_land_2 WHERE date_time > '2011-11-01' AND ST_Intersects({ls}, location);" 
+print(query3)
+cur.execute(query3)
+columns = [_.name for _ in cur.description]
+
+fields = '*'
+query4 = f"SELECT {fields} FROM {SCHEMA}.observations_2011_land_2 WHERE date_time > '2011-11-01' AND ST_Intersects({ls}, location);"
+
+print(columns)
+print(cur.fetchone())
+#(1, 100, "abc'def")
+
+print("""
+pd.read_sql(sql, con, index_col=None, coerce_float=True, params=None, parse_dates=None, columns=None, chunksize=None)
+""")
+print(conn_str)
+
+df = pd.read_sql(query4, conn)
+df.drop(columns=['location'])
+print(df.loc[0])
+
+df.to_csv('out.csv', sep=',', index=False, float_format='%.3f',
+          date_format='%Y-%m-%d %H:%M:%S%z')
+
+print('Latitude selected:')
+print(df.latitude.min(), df.latitude.max())
+
+print('Longitude selected:')
+print(df.longitude.min(), df.longitude.max())
+
+print(f'BBOX: {bbox}')