From e46a7ac1fbbb1d87e4d833452c6a8af96ec516f1 Mon Sep 17 00:00:00 2001 From: aanersc Date: Wed, 3 Mar 2021 12:46:10 +0100 Subject: [PATCH 01/13] #126 removing the geospaas.url.py file --- geospaas/urls.py | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 geospaas/urls.py diff --git a/geospaas/urls.py b/geospaas/urls.py deleted file mode 100644 index 0c346f22..00000000 --- a/geospaas/urls.py +++ /dev/null @@ -1,12 +0,0 @@ -from django.conf.urls import include, url - -from django.contrib import admin -admin.autodiscover() - -app_name = 'geospaas' -urlpatterns = [ - # Examples: - # - #url(r'adas/', include('geospaas.adas_viewer.urls')), - url(r'^', include('geospaas.base_viewer.urls')), -] From 0a18a98c18e9c4e4a3495eb11bee680e2b344d43 Mon Sep 17 00:00:00 2001 From: aanersc Date: Wed, 3 Mar 2021 14:31:29 +0100 Subject: [PATCH 02/13] #126 correcting one test with mocking --- geospaas/tests.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/geospaas/tests.py b/geospaas/tests.py index 07978fc5..bbbf3274 100644 --- a/geospaas/tests.py +++ b/geospaas/tests.py @@ -6,11 +6,10 @@ from geospaas.utils import utils class TestUtils(TestCase): - - def test_validate_uri_opendap_does_not_exist(self): - uri = 'http://www.ifremer.fr/opendap/cerdap1/globcurrent/' \ - 'v2.0/global_012_deg/geostrophic/2014/001/' \ - '20140101000000-GLOBCURRENT-L4-CURgeo_0m-ALT_OI-v02.0-fv01.0.nc.tull' + @patch('urllib3.PoolManager') + def test_validate_uri_opendap_does_not_exist(self, mock_PoolManager): + uri = 'http://www.ifremer.fr' + mock_PoolManager.status=1 with self.assertRaises(OSError) as cm: utils.validate_uri(uri) self.assertEqual('NetCDF: file not found', cm.exception.args[1]) From 2e2f7ef6d2f9e7554fc1b0862b3b01e44795ea4e Mon Sep 17 00:00:00 2001 From: aanersc Date: Wed, 3 Mar 2021 14:43:43 +0100 Subject: [PATCH 03/13] #126 correcting one test --- geospaas/catalog/tests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/geospaas/catalog/tests.py b/geospaas/catalog/tests.py index aaea4e26..be2256d1 100644 --- a/geospaas/catalog/tests.py +++ b/geospaas/catalog/tests.py @@ -175,8 +175,7 @@ def test_unique_constraint_geographic_location(self): def test__reproduce__not_null_constraint_failed(self): geom_wkt = 'POLYGON((1.9952502916543926 43.079137301616434,1.8083614437225106 43.110281163194,1.6280391132319614 43.13999933989308,1.4543047771860391 43.168322409488,1.287178802518546 43.19527992537942,1.126680477150093 43.22090040126175,0.9728280404789855 43.24521129666272,0.8256387132257121 43.26823900332679,0.6851287265540695 43.2900088324148,0.5513133503959514 43.31054500249216,0.4177032107533156 43.3308546554019,0.4177032107533156 43.3308546554019,0.31209072545607186 42.966172534807384,0.2072770834059167 42.60138984322352,0.10324224766647609 42.23651548835664,-3.327518831779395e-05 41.87155835974214,-0.10256845388361147 41.50652732885059,-0.20438175048840848 41.14143124914533,-0.305491137731497 40.776278956093606,-0.4059141156224018 40.4110792671334,-0.5056677274094622 40.045840981598744,-0.6188735003262834 39.62838980058282,-0.6188735003262834 39.62838980058282,-0.4938090620192412 39.60834071737128,-0.36846516623385345 39.58812662484392,-0.2367679070115216 39.566753658618175,-0.09872965092928164 39.54419980869909,0.045636463325510676 39.520442055142105,0.19631650129236156 39.49545637806485,0.35329570832956364 39.469217768317954,0.516558484513175 39.441700238839005,0.686088358898322 39.412876836714965,0.8618679632011015 39.382719655976956,0.8618679632011015 39.382719655976956,0.985334472893415 39.799577386800905,1.0941941665822539 40.164279112775866,1.2038450353475123 40.52892574316672,1.3143064728956748 40.89350812553239,1.425598388091744 41.25801708090206,1.5377412226553768 41.622443403572326,1.6507559695776892 41.98677786085107,1.764664192292795 42.351011192745254,1.8794880446399878 42.71513411159017,1.9952502916543926 43.079137301616434))' - with self.assertRaises(django.db.utils.IntegrityError): - gg, created = GeographicLocation.objects.get_or_create(geometry=WKTReader().read(geom_wkt)) + gg, created = GeographicLocation.objects.get_or_create(geometry=WKTReader().read(geom_wkt)) geom_wkt = 'POLYGON((1.995 43.079,1.808 43.1102,1.628 43.139,1.454 43.168,1.287 43.195,1.126 43.220,0.972 43.245,0.825 43.268,0.685 43.290,0.551 43.310,0.417 43.330,0.417 43.330,0.312 42.966,0.207 42.601,0.103 42.236,0.0 41.871,-0.102 41.506,-0.204 41.141,-0.305 40.776,-0.405 40.411,-0.505 40.045,-0.618 39.628,-0.618 39.628,-0.493 39.608,-0.368 39.588,-0.236 39.566,-0.098 39.544,0.0456 39.520,0.196 39.495,0.353 39.469,0.516 39.441,0.686 39.412,0.861 39.382,0.861 39.382,0.985 39.799,1.094 40.164,1.203 40.528,1.314 40.893,1.425 41.258,1.537 41.622,1.650 41.986,1.764 42.351,1.879 42.715,1.995 43.079))' gg, created = GeographicLocation.objects.get_or_create(geometry=WKTReader().read(geom_wkt)) self.assertTrue(created) From b90668a46cb0b9abcd4e1397aca73192deb97713 Mon Sep 17 00:00:00 2001 From: aanersc Date: Wed, 3 Mar 2021 15:00:10 +0100 Subject: [PATCH 04/13] #126 delete the whole nansat_ingestor app --- geospaas/nansat_ingestor/__init__.py | 0 geospaas/nansat_ingestor/admin.py | 3 - .../nansat_ingestor/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../management/commands/ingest.py | 57 --- .../management/commands/ingest_hyrax.py | 46 -- .../commands/ingest_thredds_crawl.py | 97 ----- geospaas/nansat_ingestor/managers.py | 173 -------- .../migrations/0001_initial.py | 26 -- .../nansat_ingestor/migrations/__init__.py | 0 geospaas/nansat_ingestor/models.py | 10 - geospaas/nansat_ingestor/tests.py | 397 ------------------ geospaas/nansat_ingestor/views.py | 3 - 13 files changed, 812 deletions(-) delete mode 100644 geospaas/nansat_ingestor/__init__.py delete mode 100644 geospaas/nansat_ingestor/admin.py delete mode 100644 geospaas/nansat_ingestor/management/__init__.py delete mode 100644 geospaas/nansat_ingestor/management/commands/__init__.py delete mode 100644 geospaas/nansat_ingestor/management/commands/ingest.py delete mode 100644 geospaas/nansat_ingestor/management/commands/ingest_hyrax.py delete mode 100644 geospaas/nansat_ingestor/management/commands/ingest_thredds_crawl.py delete mode 100644 geospaas/nansat_ingestor/managers.py delete mode 100644 geospaas/nansat_ingestor/migrations/0001_initial.py delete mode 100644 geospaas/nansat_ingestor/migrations/__init__.py delete mode 100644 geospaas/nansat_ingestor/models.py delete mode 100644 geospaas/nansat_ingestor/tests.py delete mode 100644 geospaas/nansat_ingestor/views.py diff --git a/geospaas/nansat_ingestor/__init__.py b/geospaas/nansat_ingestor/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/geospaas/nansat_ingestor/admin.py b/geospaas/nansat_ingestor/admin.py deleted file mode 100644 index 8c38f3f3..00000000 --- a/geospaas/nansat_ingestor/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/geospaas/nansat_ingestor/management/__init__.py b/geospaas/nansat_ingestor/management/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/geospaas/nansat_ingestor/management/commands/__init__.py b/geospaas/nansat_ingestor/management/commands/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/geospaas/nansat_ingestor/management/commands/ingest.py b/geospaas/nansat_ingestor/management/commands/ingest.py deleted file mode 100644 index b1732881..00000000 --- a/geospaas/nansat_ingestor/management/commands/ingest.py +++ /dev/null @@ -1,57 +0,0 @@ -from django.core.management.base import BaseCommand, CommandError - -from geospaas.utils.utils import uris_from_args -from geospaas.catalog.models import DatasetURI -from geospaas.nansat_ingestor.models import Dataset - -class Command(BaseCommand): - args = '' - help = 'Add file to catalog archive' - n_points = 10 - - def add_arguments(self, parser): - parser.add_argument('files', nargs='*', type=str) - parser.add_argument('--nansat-option', - action='append', - help='''Option for Nansat() e.g. - mapperName="sentinel1a_l1" - (can be repated)''') - - parser.add_argument('--n_points', - action='store', - default=self.n_points, - help='''Number of points in the border''') - - def _get_args(self, *args, **options): - """ Get arguments needed to create the Dataset - """ - if len(options['files'])==0: - raise IOError('Please provide at least one filename') - n_points = int(options.get('n_points', self.n_points)) - - non_ingested_uris = DatasetURI.objects.all().get_non_ingested_uris( - uris_from_args(options['files']) - ) - - nansat_options = {} - if options['nansat_option']: - for opt in options['nansat_option']: - var, val = opt.split('=') - nansat_options[var] = val - - return non_ingested_uris, n_points, nansat_options - - def handle(self, *args, **options): - """ Ingest one Dataset per file that has not previously been ingested - """ - non_ingested_uris, n_points, nansat_options = self._get_args(*args, **options) - - for non_ingested_uri in non_ingested_uris: - self.stdout.write('Ingesting %s ...\n' % non_ingested_uri) - ds, cr = Dataset.objects.get_or_create(non_ingested_uri, n_points=n_points, **nansat_options) - if cr: - self.stdout.write('Successfully added: %s\n' % non_ingested_uri) - elif type(ds) == Dataset: - self.stdout.write('%s has been added before.\n' % non_ingested_uri) - else: - self.stdout.write('Could not add %s.\n' % non_ingested_uri) diff --git a/geospaas/nansat_ingestor/management/commands/ingest_hyrax.py b/geospaas/nansat_ingestor/management/commands/ingest_hyrax.py deleted file mode 100644 index 8edb27dc..00000000 --- a/geospaas/nansat_ingestor/management/commands/ingest_hyrax.py +++ /dev/null @@ -1,46 +0,0 @@ -import os -import time -import re -import urllib2 - -from django.core.management.base import BaseCommand, CommandError - -from geospaas.nansat_ingestor.management.commands.ingest import Command as IngestCommand - -# extend Ingest -class Command(IngestCommand): - args = 'HYRAX_URL' - help = 'Add file to catalog from HYRAX server' - - def handle(self, *args, **options): - print('Searching netcdf files. May take some time...\n\n\n') - nc_uris = find_netcdf_uris(args[0]) - num_nc_uris = len(nc_uris) - print('\nTrying to ingest %d datasets\n'%num_nc_uris) - super(Command, self).handle(*nc_uris, **options) - -def find_netcdf_uris(uri0, sleep=1.0): - uri0_base = os.path.split(uri0)[0] - print('Search in ', uri0_base) - # get HTML from the URL - time.sleep(sleep) - response = urllib2.urlopen(uri0) - html = response.read() - - # find all links to netDCF - nc_uris = [os.path.join(uri0_base, uri.replace('href="', '').replace('.nc.dds', '.nc')) - for uri in re.findall('href=.*nc\.dds', html)] - - # find all links to sub-pages - uris = [os.path.join(uri0_base, uri.replace('href="', '')) - for uri in re.findall('href=.*/contents.html', html)] - # get links to netcDF also from sub-pages - try: - for uri in uris: - nc_uris += find_netcdf_uris(uri) - except (urllib2.HTTPError, urllib2.URLError) as e: - print('Server error %s'%e.message) - # return all links to netCDF - return nc_uris - - diff --git a/geospaas/nansat_ingestor/management/commands/ingest_thredds_crawl.py b/geospaas/nansat_ingestor/management/commands/ingest_thredds_crawl.py deleted file mode 100644 index cb7069e5..00000000 --- a/geospaas/nansat_ingestor/management/commands/ingest_thredds_crawl.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Note: This is tested on Sentinel-1 and Sentinel-2 data from the Norwegian ground segment, and -Arome forecasts from thredds.met.no. Other repositories may require slight changes in the code. This -must be developed gradually.. -""" -import warnings - -from django.core.management.base import BaseCommand, CommandError -from django.db.utils import IntegrityError -from thredds_crawler.crawl import Crawl - -from geospaas.catalog.models import DatasetURI -from geospaas.nansat_ingestor.models import Dataset as NansatDataset -from geospaas.utils.utils import validate_uri - - -def crawl_and_ingest(url, **options): - validate_uri(url) - - date = options.get('date', None) - filename = options.get('filename', None) - if date: - select = ['(.*%s.*\.nc)' % date] - elif filename: - select = ['(.*%s)' % filename] - else: - select = None - - skips = Crawl.SKIPS + ['.*ncml'] - c = Crawl(url, select=select, skip=skips, debug=True) - added = 0 - for ds in c.datasets: - for s in ds.services: - if s.get('service').lower() == 'opendap': - url = s.get('url') - name = s.get('name') - service = s.get('service') - try: - # Create Dataset from OPeNDAP url - this is necessary to get all metadata - gds, cr = NansatDataset.objects.get_or_create(url, uri_service_name=name, - uri_service_type=service) - except (IOError, AttributeError) as e: - # warnings.warn(e.message) - continue - if cr: - added += 1 - print('Added %s, no. %d/%d' % (url, added, len(c.datasets))) - # Connect all service uris to the dataset - for s in ds.services: - ds_uri, _ = DatasetURI.objects.get_or_create( - name=s.get('name'), - service=s.get('service'), - uri=s.get('url'), - dataset=gds) - print('Added %s, no. %d/%d' % (url, added, len(c.datasets))) - return added - - -class Command(BaseCommand): - args = ' ' + help = """ + Add metadata of datasets available on thredds/opendap to archive. + + Args: + : the url to the thredds catalog + : Select datasets by date (yyyy/mm/dd) + : Select datasets by filename + + Example: + (1) Find all Sentinel-2A datasets in 2017 + + url = 'http://nbstds.met.no/thredds/catalog/NBS/S2A/catalog.html' + date = '2017/01/10' + + (2) Find a specific file + + url = 'http://nbstds.met.no/thredds/catalog/NBS/S2A/catalog.html' + filename = 'S2A_MSIL1C_20170201T111301_N0204_R137_T32WNS_20170201T111255.nc' + """ + + def add_arguments(self, parser): + parser.add_argument('url', nargs='*', type=str) + parser.add_argument('--date', + action='store', + default='', + help='''Date of coverage (yyyy/mm/dd)''') + parser.add_argument('--filename', + action='store', + default='', + help='''Filename of a specific dataset''') + + def handle(self, *args, **options): + if not len(options['url']) == 1: + raise IOError('Please provide a url to the data') + url = options.pop('url')[0] + added = crawl_and_ingest(url, **options) + self.stdout.write( + 'Successfully added metadata of %s datasets' % added) diff --git a/geospaas/nansat_ingestor/managers.py b/geospaas/nansat_ingestor/managers.py new file mode 100644 index 00000000..c7d00cc7 --- /dev/null +++ b/geospaas/nansat_ingestor/managers.py @@ -0,0 +1,173 @@ +import json +import uuid +import warnings + +import pythesint as pti +from django.contrib.gis.geos import WKTReader +from django.db import models + +from geospaas.catalog.managers import FILE_SERVICE_NAME, LOCAL_FILE_SERVICE +from geospaas.catalog.models import (Dataset, DatasetURI, + GeographicLocation, Source) +from geospaas.utils.utils import nansat_filename, validate_uri +from geospaas.vocabularies.models import (DataCenter, Instrument, + ISOTopicCategory, Location, + Parameter, Platform) +from nansat.nansat import Nansat + + +class DatasetManager(models.Manager): + + def get_or_create(self, + uri, + n_points=10, + uri_filter_args=None, + uri_service_name=FILE_SERVICE_NAME, + uri_service_type=LOCAL_FILE_SERVICE, + *args, **kwargs): + """ Create dataset and corresponding metadata + + Parameters: + ---------- + uri : str + URI to file or stream openable by Nansat + n_points : int + Number of border points (default is 10) + uri_filter_args : dict + Extra DatasetURI filter arguments if several datasets can refer to the same URI + uri_service_name : str + name of the service which is used ('dapService', 'fileService', 'http' or 'wms') + uri_service_type : str + type of the service which is used ('OPENDAP', 'local', 'HTTPServer' or 'WMS') + + Returns: + ------- + dataset and flag + """ + if not uri_filter_args: + uri_filter_args = {} + + # Validate uri - this should raise an exception if the uri doesn't point to a valid + # file or stream + validate_uri(uri) + + # Several datasets can refer to the same uri (e.g., scatterometers and svp drifters), so we + # need to pass uri_filter_args + uris = DatasetURI.objects.filter(uri=uri, **uri_filter_args) + if len(uris) > 0: + return uris[0].dataset, False + + # Open file with Nansat + n = Nansat(nansat_filename(uri), **kwargs) + + # get metadata from Nansat and get objects from vocabularies + n_metadata = n.get_metadata() + + entry_id = n_metadata.get('entry_id', None) + # set compulsory metadata (source) + platform, _ = Platform.objects.get_or_create( + json.loads(n_metadata['platform'])) + instrument, _ = Instrument.objects.get_or_create( + json.loads(n_metadata['instrument'])) + specs = n_metadata.get('specs', '') + source, _ = Source.objects.get_or_create(platform=platform, + instrument=instrument, + specs=specs) + + default_char_fields = { + # Adding NERSC_ in front of the id violates the string representation of the uuid + #'entry_id': lambda: 'NERSC_' + str(uuid.uuid4()), + 'entry_id': lambda: str(uuid.uuid4()), + 'entry_title': lambda: 'NONE', + 'summary': lambda: 'NONE', + } + + # set optional CharField metadata from Nansat or from default_char_fields + options = {} + try: + existing_ds = Dataset.objects.get(entry_id=entry_id) + except Dataset.DoesNotExist: + existing_ds = None + for name in default_char_fields: + if name not in n_metadata: + warnings.warn('%s is not provided in Nansat metadata!' % name) + # prevent overwriting of existing values by defaults + if existing_ds: + options[name] = existing_ds.__getattribute__(name) + else: + options[name] = default_char_fields[name]() + else: + options[name] = n_metadata[name] + + default_foreign_keys = { + 'gcmd_location': {'model': Location, + 'value': pti.get_gcmd_location('SEA SURFACE')}, + 'data_center': {'model': DataCenter, + 'value': pti.get_gcmd_provider('NERSC')}, + 'ISO_topic_category': {'model': ISOTopicCategory, + 'value': pti.get_iso19115_topic_category('Oceans')}, + } + + # set optional ForeignKey metadata from Nansat or from default_foreign_keys + for name in default_foreign_keys: + value = default_foreign_keys[name]['value'] + model = default_foreign_keys[name]['model'] + if name not in n_metadata: + warnings.warn('%s is not provided in Nansat metadata!' % name) + else: + try: + value = json.loads(n_metadata[name]) + except: + warnings.warn('%s value of %s metadata provided in Nansat is wrong!' % + (n_metadata[name], name)) + if existing_ds: + options[name] = existing_ds.__getattribute__(name) + else: + options[name], _ = model.objects.get_or_create(value) + + # Find coverage to set number of points in the geolocation + if len(n.vrt.dataset.GetGCPs()) > 0: + n.reproject_gcps() + geolocation = GeographicLocation.objects.get_or_create( + geometry=WKTReader().read(n.get_border_wkt(nPoints=n_points)))[0] + + # create dataset + # - the get_or_create method should use get_or_create here as well, + # or its name should be changed - see issue #127 + ds, created = Dataset.objects.update_or_create(entry_id=options['entry_id'], defaults={ + 'time_coverage_start': n.get_metadata('time_coverage_start'), + 'time_coverage_end': n.get_metadata('time_coverage_end'), + 'source': source, + 'geographic_location': geolocation, + 'gcmd_location': options["gcmd_location"], + 'ISO_topic_category': options["ISO_topic_category"], + "data_center": options["data_center"], + 'entry_title': options["entry_title"], + 'summary': options["summary"]} + ) + + # create parameter + all_band_meta = n.bands() + for band_id in range(1, len(all_band_meta)+1): + band_meta = all_band_meta[band_id] + standard_name = band_meta.get('standard_name', None) + short_name = band_meta.get('short_name', None) + units = band_meta.get('units', None) + if standard_name in ['latitude', 'longitude', None]: + continue + params = Parameter.objects.filter(standard_name=standard_name) + if params.count() > 1 and short_name is not None: + params = params.filter(short_name=short_name) + if params.count() > 1 and units is not None: + params = params.filter(units=units) + if params.count() >= 1: + ds.parameters.add(params[0]) + + # create dataset URI + DatasetURI.objects.get_or_create( + name=uri_service_name, + service=uri_service_type, + uri=uri, + dataset=ds) + + return ds, created diff --git a/geospaas/nansat_ingestor/migrations/0001_initial.py b/geospaas/nansat_ingestor/migrations/0001_initial.py new file mode 100644 index 00000000..f20224d7 --- /dev/null +++ b/geospaas/nansat_ingestor/migrations/0001_initial.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.2 on 2016-03-22 12:10 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('catalog', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='Dataset', + fields=[ + ], + options={ + 'proxy': True, + }, + bases=('catalog.dataset',), + ), + ] diff --git a/geospaas/nansat_ingestor/migrations/__init__.py b/geospaas/nansat_ingestor/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/geospaas/nansat_ingestor/models.py b/geospaas/nansat_ingestor/models.py new file mode 100644 index 00000000..2b7a3dbc --- /dev/null +++ b/geospaas/nansat_ingestor/models.py @@ -0,0 +1,10 @@ +from django.db import models + +from geospaas.catalog.models import Dataset as CatalogDataset +from geospaas.nansat_ingestor.managers import DatasetManager + +class Dataset(CatalogDataset): + class Meta: + proxy = True + objects = DatasetManager() + diff --git a/geospaas/nansat_ingestor/tests.py b/geospaas/nansat_ingestor/tests.py new file mode 100644 index 00000000..616ad1c2 --- /dev/null +++ b/geospaas/nansat_ingestor/tests.py @@ -0,0 +1,397 @@ +"""Tests for nansat_ingestor app""" +import sys +from contextlib import contextmanager +from io import StringIO + +from django.core.management import call_command +from django.test import TestCase +from mock import PropertyMock, patch + +from geospaas.catalog.models import DatasetURI +from geospaas.nansat_ingestor.models import Dataset +from geospaas.nansat_ingestor.management.commands.ingest_thredds_crawl import crawl_and_ingest + + +@contextmanager +def captured_output(): + new_out, new_err = StringIO(), StringIO() + old_out, old_err = sys.stdout, sys.stderr + try: + sys.stdout, sys.stderr = new_out, new_err + yield sys.stdout, sys.stderr + finally: + sys.stdout, sys.stderr = old_out, old_err + +# See also: +# https://docs.python.org/3.5/library/unittest.mock-examples.html#applying-the-same-patch-to-every-test-method + + +class BasetForTests(TestCase): + """Base class for creating the testing environment""" + fixtures = ['vocabularies', 'catalog'] + predefined_metadata_dict = { + 'platform': '{"Category": "Earth Observation Satellites", "Series_Entity": "", "Short_Name": "ENVISAT", "Long_Name": "Environmental Satellite"}', + 'instrument': '{"Category": "Earth Remote Sensing Instruments", "Class": "Passive Remote Sensing", "Type": "Spectrometers/Radiometers", "Subtype": "Imaging Spectrometers/Radiometers", "Short_Name": "MERIS", "Long_Name": "Medium Resolution Imaging Spectrometer"}', + 'time_coverage_start': '2011-05-03T10:56:38.995099', + 'time_coverage_end': '2011-05-03T10:56:38.995099', + 'data_center': '{"Bucket_Level0": "MULTINATIONAL ORGANIZATIONS", "Bucket_Level1": "", "Bucket_Level2": "", "Bucket_Level3": "", "Short_Name": "ESA/EO", "Long_Name": "Observing the Earth, European Space Agency", "Data_Center_URL": "http://www.esa.int/esaEO/"}', + 'gcmd_location': '{"Location_Category": "VERTICAL LOCATION", "Location_Type": "SEA SURFACE", "Location_Subregion1": "", "Location_Subregion2": "", "Location_Subregion3": ""}', + 'ISO_topic_category': '{"iso_topic_category": "Oceans_XXXXX"}', + } + predefined_band_metadata_dict = { + 1: {'dataType': '2', + 'name': 'DN_HH', + 'SourceBand': '1', + 'SourceFilename': '/some/folder/filename.ext'}, + 2: {'colormap': 'gray', + 'dataType': '6', + 'long_name': 'Normalized Radar Cross Section', + 'minmax': '0 0.1', + 'name': 'sigma0_HH', + 'PixelFunctionType': 'Sentinel1Calibration', + 'polarization': 'HH', + 'short_name': 'sigma0', + 'SourceBand': '1', + 'SourceFilename': '/vsimem/0BSD1QSPFL.vrt', + 'standard_name': 'surface_backwards_scattering_coefficient_of_radar_wave', + 'suffix': 'HH', + 'units': 'm/m', + 'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave'}, + 3: {'colormap': 'testing', + 'dataType': '6', + 'long_name': 'testing', + 'minmax': '0 0.1', + 'name': 'testing', + 'PixelFunctionType': 'testing', + 'polarization': 'testing', + 'short_name': 'testing', + 'SourceBand': '1', + 'SourceFilename': 'testing', + 'standard_name': 'longitude', # <=== notice the difference between testing records + 'suffix': 'testing', + 'units': 'testing', + 'wkv': 'testing'}, + 4: {'colormap': 'testing', + 'dataType': '10', + 'long_name': 'testing', + 'minmax': '0 0.1', + 'name': 'testing', + 'PixelFunctionType': 'testing', + 'polarization': 'testing', + 'short_name': 'testing', + 'SourceBand': '1', + 'SourceFilename': 'testing', + 'standard_name': 'latitude', # <=== notice the difference between testing records + 'suffix': 'testing', + 'units': 'testing', + 'wkv': 'testing'}, + 5: {'colormap': 'gray', + 'dataType': '6', + 'long_name': 'Normalized Radar Cross Section', + 'minmax': '0 0.1', + 'name': 'gamma0_HH', + 'PixelFunctionType': 'Sentinel1Calibration', + 'polarization': 'HH', + 'short_name': 'gamma0', + 'SourceBand': '1', + 'SourceFilename': '/vsimem/0BSD1QSPFL.vrt', + 'standard_name': 'surface_backwards_scattering_coefficient_of_radar_wave', + 'suffix': 'HH', + 'units': 'm/m', + 'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave'}, + } + + def setUp(self): + self.patcher = patch('geospaas.nansat_ingestor.managers.Nansat') + self.mock_Nansat = self.patcher.start() + self.mock_Nansat.return_value.get_metadata.side_effect = self.mock_get_metadata + self.mock_Nansat.return_value.get_border_wkt.return_value = 'POLYGON((24.88 68.08,22.46 68.71,19.96 69.31,17.39 69.87,24.88 68.08))' + self.mock_Nansat.return_value.bands.side_effect = self.mock_bands + + + self.patcher2 = patch('geospaas.nansat_ingestor.managers.validate_uri') + self.mock_validate_uri = self.patcher2.start() + self.mock_validate_uri.return_value = None + # in order to prevent "mock leak" in the tests + self.addCleanup(patch.stopall) + + def mock_get_metadata(self, *args): + """ Mock behaviour of Nansat.get_metadata method """ + if len(args) == 0: + return self.predefined_metadata_dict + if args[0] not in self.predefined_metadata_dict: + raise + return self.predefined_metadata_dict[args[0]] + + def mock_bands(self): + return self.predefined_band_metadata_dict + + +class TestDatasetManager(BasetForTests): + """Class for containing all the tests of creating the datasets and related situations""" + + def test__get_or_create__with__entry_id__in__metadata(self): + self.predefined_metadata_dict['entry_id'] = 'UNIQUE_ID_1000' + uri = 'file://localhost/some/folder/filename.ext' + _, cr0 = Dataset.objects.get_or_create(uri) + self.assertTrue(cr0) + + def test_getorcreate_localfile_only_created_for_the_very_first_time(self): + '''shall return the creation flag (the second returned value) + equals to True for the first time and + equals to False for the second time''' + uri = 'file://localhost/some/folder/filename.ext' + _, cr0 = Dataset.objects.get_or_create(uri) + _, cr1 = Dataset.objects.get_or_create(uri) + self.assertTrue(cr0) + self.assertFalse(cr1) + + def test_getorcreate_localfile_is_matched_in_metadata(self): + '''shall return the correct specification of dataset created based on + predefined metadata declared in the test''' + uri = 'file://localhost/some/folder/filename.ext' + ds0, _ = Dataset.objects.get_or_create(uri) + self.assertEqual( + ds0.entry_id, self.predefined_metadata_dict['entry_id']) + self.assertEqual(ds0.entry_title, 'NONE') + self.assertEqual(ds0.summary, 'NONE') + + def test_getorcreate_localfile_matched_parameter(self): + uri = 'file://localhost/some/folder/filename.ext' + ds0, _ = Dataset.objects.get_or_create(uri) + self.assertEqual(ds0.parameters.values()[0]['short_name'], + self.predefined_band_metadata_dict[2]['short_name']) + + def test_getorcreate_localfile_filtering_base_on_parameter(self): + '''shall return standard name of + an specified parameter of the correct filtered dataset + based on parameter filtering''' + uri = 'file://localhost/some/folder/filename.ext' + Dataset.objects.get_or_create(uri) + testingDataset = Dataset.objects.filter( + parameters__standard_name='surface_backwards_scattering_coefficient_of_radar_wave') + self.assertEqual(str(testingDataset.first().parameters.first()), + self.predefined_band_metadata_dict[2]['standard_name']) + + def test_dont_add_longitude_latitude(self): + """ shall not add latitude and longitude into Parameters of Dataset table """ + uri = 'file://localhost/some/folder/filename.ext' + ds0, _ = Dataset.objects.get_or_create(uri) + ds_params_standard_names = ds0.parameters.values_list('standard_name', flat=True) + # longitude should not be one of the parameters + self.assertNotIn('longitude', ds_params_standard_names) + # latitude should not be one of the parameters + self.assertNotIn('latidtude', ds_params_standard_names) + + def test_add_sigma0_gamma0(self): + """ shall add both sigma0 and gamma0 with same standard name into Parameters of Dataset table """ + uri = 'file://localhost/some/folder/filename.ext' + ds0, _ = Dataset.objects.get_or_create(uri) + ds_params_standard_names = ds0.parameters.values_list('standard_name', flat=True) + ds_params_short_names = ds0.parameters.values_list('short_name', flat=True) + self.assertEqual(len(ds_params_standard_names), 2) + self.assertEqual(len(ds_params_short_names), 2) + self.assertIn('surface_backwards_scattering_coefficient_of_radar_wave', + ds_params_standard_names) + self.assertIn('sigma0', ds_params_short_names) + self.assertIn('gamma0', ds_params_short_names) + + +class TestDatasetURI(BasetForTests): + """Class for containing the tests of creation of dataseturi after creation of dataset""" + def test_get_non_ingested_uris(self): + ''' Shall return list with only non existing files ''' + testfile = 'file://localhost/vagrant/shared/test_data/meris_l1/MER_FRS_1PNPDK20120303_093810_000000333112_00180_52349_3561.N1' + Dataset.objects.get_or_create(testfile) + new_uris = ['file://fake/path/file1.ext', 'file://fake/path/file2.ext'] + all_uris = new_uris + [testfile] + uris = DatasetURI.objects.all().get_non_ingested_uris(all_uris) + self.assertEqual(uris, new_uris) + + +class TestIngestNansatCommand(BasetForTests): + def test_add_asar(self): + out = StringIO() + f = 'file://localhost/vagrant/shared/test_data/asar/ASA_WSM_1PNPDK20081110_205618_000000922073_00401_35024_0844.N1' + call_command('ingest', f, stdout=out) + self.assertIn('Successfully added:', out.getvalue()) + + def test_add_asar_with_nansat_options(self): + out = StringIO() + f = 'file://localhost/vagrant/shared/test_data/asar/ASA_WSM_1PNPDK20081110_205618_000000922073_00401_35024_0844.N1' + call_command('ingest', f, nansat_option=['mapperName=asar'], stdout=out) + self.assertIn('Successfully added:', out.getvalue()) + + +class TestIngestThreddsCrawl(TestCase): + def setUp(self): + self.uri = 'http://nbstds.met.no/TEST/thredds/dodsC/NBS/S2A/2019/01/24/' + self.patch_crawl = patch( + 'geospaas.nansat_ingestor.management.commands.ingest_thredds_crawl.crawl_and_ingest') + self.mock_crawl = self.patch_crawl.start() + self.mock_crawl.return_value = 10 + + def tearDown(self): + self.patch_crawl.stop() + + def test_ingest_no_args(self): + with captured_output() as (out, err): + call_command('ingest_thredds_crawl', self.uri) + output = out.getvalue().strip() + self.assertEqual(output, 'Successfully added metadata of 10 datasets') + + def test_ingest_with_year_arg(self): + with captured_output() as (out, err): + call_command('ingest_thredds_crawl', self.uri, date=['2019/01/24']) + output = out.getvalue().strip() + self.assertEqual(output, 'Successfully added metadata of 10 datasets') + + def test_ingest_with_filename_arg(self): + with captured_output() as (out, err): + call_command('ingest_thredds_crawl', self.uri, + filename='S2A_MSIL1C_20190124T115401_N0207_R023_T30VWP_20190124T120414.nc') + output = out.getvalue().strip() + self.assertEqual(output, 'Successfully added metadata of 10 datasets') + + +class TestIngestThreddsCrawl__crawl__function(TestCase): + def setUp(self): + self.uri = 'http://nbstds.met.no/TEST/thredds/dodsC/NBS/S2A/2019/01/24/' + self.patch_LeafDataset = patch('thredds_crawler.crawl.LeafDataset') + self.mock_LeafDataset = self.patch_LeafDataset.start() + self.patch_validate_uri = patch( + 'geospaas.nansat_ingestor.management.commands.ingest_thredds_crawl.validate_uri') + self.mock_validate_uri = self.patch_validate_uri.start() + self.mock_validate_uri.return_value = True + self.patch_Crawl = patch( + 'geospaas.nansat_ingestor.management.commands.ingest_thredds_crawl.Crawl') + self.mock_Crawl = self.patch_Crawl.start() + self.mock_Crawl.SKIPS = [] + self.patch_Dataset = patch( + 'geospaas.nansat_ingestor.management.commands.ingest_thredds_crawl.NansatDataset') + self.mock_ds = self.patch_Dataset.start() + pmod = 'geospaas.nansat_ingestor.management.commands.ingest_thredds_crawl.DatasetURI' + self.patch_DatasetURI = patch(pmod) + self.mock_dsuri = self.patch_DatasetURI.start() + from thredds_crawler.crawl import LeafDataset + test_LeafDataset = LeafDataset() + test_LeafDataset.services = [ + { + 'name': 'odap', + 'service': 'OPENDAP', + 'url': 'http://nbstds.met.no/TEST/thredds/dodsC/NBS/S2A/2019/01/24/' + 'S2A_MSIL1C_20190124T115401_N0207_R023_T30VWP_20190124T120414.nc' + }, { + 'name': 'httpService', + 'service': 'HTTPServer', + 'url': 'http://nbstds.met.no/TEST/fileServer/dodsC/NBS/S2A/2019/01/24/' + 'S2A_MSIL1C_20190124T115401_N0207_R023_T30VWP_20190124T120414.nc' + }, { + 'name': 'wms', + 'service': 'WMS', + 'url': 'http://nbstds.met.no/TEST/wms/dodsC/NBS/S2A/2019/01/24/' + 'S2A_MSIL1C_20190124T115401_N0207_R023_T30VWP_20190124T120414.nc' + }, { + 'name': 'wcs', + 'service': 'WCS', + 'url': 'http://nbstds.met.no/TEST/wcs/dodsC/NBS/S2A/2019/01/24/' + 'S2A_MSIL1C_20190124T115401_N0207_R023_T30VWP_20190124T120414.nc' + } + ] + self.mock_Crawl.return_value = PropertyMock( + datasets=[test_LeafDataset]) + self.addCleanup(patch.stopall) + + def test_ds_created(self): + '''shall assert that NansatDataset.objects.get_or_create + is called for only once with correct input calls (called with opendap)''' + self.mock_ds.objects.get_or_create.return_value = (Dataset(), True) + self.mock_dsuri.objects.get_or_create.return_value = ( + DatasetURI(), True) + added = crawl_and_ingest(self.uri) + self.mock_validate_uri.assert_called_once_with(self.uri) + self.mock_Crawl.assert_called_once_with( + self.uri, debug=True, select=None, skip=['.*ncml']) + self.mock_ds.objects.get_or_create.assert_called_once_with( + 'http://nbstds.met.no/TEST/thredds/dodsC/NBS/S2A/2019/01/24/' + 'S2A_MSIL1C_20190124T115401_N0207_R023_T30VWP_20190124T120414.nc', + uri_service_name='odap', + uri_service_type='OPENDAP') + self.assertEqual(added, 1) + + def test_dsuri_ds_created(self): + '''shall assert that DatasetURI.objects.get_or_create + is called several times with correct input calls''' + self.mock_ds.objects.get_or_create.return_value = (Dataset(), True) + self.mock_dsuri.objects.get_or_create.return_value = ( + DatasetURI(), True) + crawl_and_ingest(self.uri) + self.mock_validate_uri.assert_called_once_with(self.uri) + self.mock_Crawl.assert_called_once_with( + self.uri, debug=True, select=None, skip=['.*ncml']) + self.assertEqual( + self.mock_dsuri.objects.get_or_create.call_args_list[0].kwargs['name'], 'odap') + self.assertEqual( + self.mock_dsuri.objects.get_or_create.call_args_list[1].kwargs['service'], 'HTTPServer') + self.assertEqual( + self.mock_dsuri.objects.get_or_create.call_args_list[2].kwargs['name'], 'wms') + self.assertEqual( + self.mock_dsuri.objects.get_or_create.call_args_list[3].kwargs['service'], 'WCS') + + def test_ds_created_with_date_arg(self): + self.mock_ds.objects.get_or_create.return_value = (Dataset(), True) + self.mock_dsuri.objects.get_or_create.return_value = ( + DatasetURI(), True) + + added = crawl_and_ingest(self.uri, date='2019/01/01') + self.mock_validate_uri.assert_called_once_with(self.uri) + self.mock_Crawl.assert_called_once_with(self.uri, debug=True, + select=['(.*2019/01/01.*\\.nc)'], skip=['.*ncml']) + self.assertEqual(added, 1) + + def test_ds_created_with_filename_arg(self): + self.mock_ds.objects.get_or_create.return_value = (Dataset(), True) + self.mock_dsuri.objects.get_or_create.return_value = ( + DatasetURI(), True) + fn = 'S2A_MSIL1C_20190124T115401_N0207_R023_T30VWP_20190124T120414.nc' + added = crawl_and_ingest(self.uri, filename=fn) + self.mock_validate_uri.assert_called_once_with(self.uri) + self.mock_Crawl.assert_called_once_with(self.uri, debug=True, + select=[ + '(.*S2A_MSIL1C_20190124T115401_N0207_R023_T30VWP_20190124T120414.nc)'], + skip=['.*ncml']) + self.assertEqual(added, 1) + + def test_get_or_create_raises_IOError(self): + # I am not sure which situations caused IOError, so this is not tested now (on 2019-02-15, + # the S2 data access from the Norwegian ground segment was failing) + pass + + def test_get_or_create_raises_AttributeError(self): + # I am not sure which situations caused AttributeError, so this is not tested now (on 2019-02-15, + # the S2 data access from the Norwegian ground segment was failing) + pass + + +class TestsForUpdateAbility(BasetForTests): + predefined_metadata_dict = { + 'entry_id': 'NERSC_test_dataset_titusen', + 'entry_title': 'new title from nansat mapper', + 'platform': '{"Category": "Earth Observation Satellites", "Series_Entity": "", "Short_Name": "ENVISAT", "Long_Name": "Environmental Satellite"}', + 'instrument': '{"Category": "Earth Remote Sensing Instruments", "Class": "Passive Remote Sensing", "Type": "Spectrometers/Radiometers", "Subtype": "Imaging Spectrometers/Radiometers", "Short_Name": "MERIS", "Long_Name": "Medium Resolution Imaging Spectrometer"}', + 'time_coverage_start': '2011-05-03T10:56:38.995099', + 'time_coverage_end': '2012-05-03T10:56:38.995099' + } + + def test_for_examining_the_updating_purpose_of_ingestor_code(self): + '''shall update the previous record (existing dataset) in the database without creating a new one''' + uri = 'file://localhost/some/folder/filename.ext' + d0, _ = Dataset.objects.get_or_create(uri) + # assertion of updating ability + self.assertEqual(d0.entry_title, 'new title from nansat mapper') + # assertion of presence of both online link and offline link are present in the set of uri + self.assertEqual(2, d0.dataseturi_set.all().count()) + self.assertIn('file://localhost/some/test/file1.ext', [d0.dataseturi_set.first( + ).uri, d0.dataseturi_set.last().uri]) # assertion of online link + self.assertIn('file://localhost/some/folder/filename.ext', [d0.dataseturi_set.first( + ).uri, d0.dataseturi_set.last().uri]) # assertion of offline link diff --git a/geospaas/nansat_ingestor/views.py b/geospaas/nansat_ingestor/views.py new file mode 100644 index 00000000..91ea44a2 --- /dev/null +++ b/geospaas/nansat_ingestor/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. From 42ffcf6b8f3960c9f76e450fae8d049dc19e1381 Mon Sep 17 00:00:00 2001 From: aanersc Date: Thu, 4 Mar 2021 08:35:44 +0100 Subject: [PATCH 09/13] Revert "#126 travisfile modification for deletion of nansat_ingestor" This reverts commit 132720869f10ba7cdc9d98eec5953c5ec3dcd302. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7cf61b0f..c381c14d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,7 @@ jobs: -e TRAVIS_PULL_REQUEST -v "$(pwd):/src" ${TEST_IMAGE} - bash -c "source /opt/conda/bin/activate && coverage run --omit=geospaas/catalog/tests/*,geospaas/vocabularies/tests/* runtests.py && coveralls" + bash -c "source /opt/conda/bin/activate && coverage run --omit=geospaas/nansat_ingestor/tests/*,geospaas/catalog/tests/*,geospaas/vocabularies/tests/* runtests.py && coveralls" deploy: - provider: pypi edge: true From 97eb2dc025faca9fc8a68e2830e0d58cdfe2ab42 Mon Sep 17 00:00:00 2001 From: aanersc Date: Thu, 4 Mar 2021 08:36:09 +0100 Subject: [PATCH 10/13] Revert "#126 removing nansat_ingestor from settings.py" This reverts commit 1fa62ebda5ea4cbe610a7fe9a9d9d08edc0ce80f. --- tests/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/settings.py b/tests/settings.py index f229c7e6..bcf8f14c 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -31,6 +31,7 @@ 'leaflet', 'django_forms_bootstrap', 'geospaas.base_viewer', + 'geospaas.nansat_ingestor', 'geospaas.catalog', 'geospaas.vocabularies', 'django.contrib.admin', @@ -137,3 +138,4 @@ LEAFLET_CONFIG = { 'NO_GLOBALS': False, } + From ceb530867a03d43171bcb09540d83b322bf71e1f Mon Sep 17 00:00:00 2001 From: aanersc Date: Thu, 4 Mar 2021 08:37:51 +0100 Subject: [PATCH 11/13] Revert "#126 remove one none-usefull test of provisioning vagrant" This reverts commit db913f26a29e2c72661997c46f7c9c371c0f9970. --- geospaas/tests.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/geospaas/tests.py b/geospaas/tests.py index 3fc42b62..bbbf3274 100644 --- a/geospaas/tests.py +++ b/geospaas/tests.py @@ -34,6 +34,14 @@ def test__validate_uri__opendap_exists(self, mock_PoolManager): uri = 'http://nbstds.met.no/thredds/catalog/NBS/S2A/test_catalog.html' self.assertEqual(utils.validate_uri(uri), None) + @patch('geospaas.utils.utils.os.mkdir') + def test_module_path(self, mock_mkdir): + mock_mkdir.return_value = None + module = 'geospaas.catalog' + root = '/home/vagrant/site_media/media' + mpath = utils.module_path(module, root) + self.assertEqual(mpath, os.path.join(root, 'geospaas/catalog')) + def test_fail_invalid_uri(self): uri = '/this/is/some/file/but/not/an/uri' with self.assertRaises(ValueError): From 20dff046ef2a5780a86033be0e0e873406940b04 Mon Sep 17 00:00:00 2001 From: aanersc Date: Thu, 4 Mar 2021 09:02:00 +0100 Subject: [PATCH 12/13] #126 remove geospaas.url from rst and site.yml file --- docs/source/geospaas.rst | 6 ------ provisioning/site.yml | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/docs/source/geospaas.rst b/docs/source/geospaas.rst index 0d94ee4a..8d69c49e 100644 --- a/docs/source/geospaas.rst +++ b/docs/source/geospaas.rst @@ -32,13 +32,7 @@ geospaas.models module :undoc-members: :show-inheritance: -geospaas.urls module --------------------- -.. automodule:: geospaas.urls - :members: - :undoc-members: - :show-inheritance: Module contents diff --git a/provisioning/site.yml b/provisioning/site.yml index 84564c29..36a45c41 100644 --- a/provisioning/site.yml +++ b/provisioning/site.yml @@ -20,7 +20,7 @@ - django_forms_bootstrap - leaflet django_urlpatterns: - - "path('', include('geospaas.urls'))," + - "path('', include('geospaas.base_viewer.urls'))," - role: geospaas geospaas_project_home: '{{ project_home }}' From 04899cf89061413aa1aa016a52e7c80f66e80abf Mon Sep 17 00:00:00 2001 From: azamifard Date: Tue, 9 Mar 2021 14:31:14 +0100 Subject: [PATCH 13/13] #126 correcting one test --- geospaas/catalog/tests.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/geospaas/catalog/tests.py b/geospaas/catalog/tests.py index be2256d1..4d184058 100644 --- a/geospaas/catalog/tests.py +++ b/geospaas/catalog/tests.py @@ -173,7 +173,10 @@ def test_unique_constraint_geographic_location(self): with self.assertRaises(django.db.utils.IntegrityError): geolocation2.save() - def test__reproduce__not_null_constraint_failed(self): + def test_GeographicLocation_object_creation_and_uniqueness(self): + """Test that GeographicLocation object can be created with high and low precision of + numbers (db can handle both cases). In addition, the same GeographicLocation can't + be created twice""" geom_wkt = 'POLYGON((1.9952502916543926 43.079137301616434,1.8083614437225106 43.110281163194,1.6280391132319614 43.13999933989308,1.4543047771860391 43.168322409488,1.287178802518546 43.19527992537942,1.126680477150093 43.22090040126175,0.9728280404789855 43.24521129666272,0.8256387132257121 43.26823900332679,0.6851287265540695 43.2900088324148,0.5513133503959514 43.31054500249216,0.4177032107533156 43.3308546554019,0.4177032107533156 43.3308546554019,0.31209072545607186 42.966172534807384,0.2072770834059167 42.60138984322352,0.10324224766647609 42.23651548835664,-3.327518831779395e-05 41.87155835974214,-0.10256845388361147 41.50652732885059,-0.20438175048840848 41.14143124914533,-0.305491137731497 40.776278956093606,-0.4059141156224018 40.4110792671334,-0.5056677274094622 40.045840981598744,-0.6188735003262834 39.62838980058282,-0.6188735003262834 39.62838980058282,-0.4938090620192412 39.60834071737128,-0.36846516623385345 39.58812662484392,-0.2367679070115216 39.566753658618175,-0.09872965092928164 39.54419980869909,0.045636463325510676 39.520442055142105,0.19631650129236156 39.49545637806485,0.35329570832956364 39.469217768317954,0.516558484513175 39.441700238839005,0.686088358898322 39.412876836714965,0.8618679632011015 39.382719655976956,0.8618679632011015 39.382719655976956,0.985334472893415 39.799577386800905,1.0941941665822539 40.164279112775866,1.2038450353475123 40.52892574316672,1.3143064728956748 40.89350812553239,1.425598388091744 41.25801708090206,1.5377412226553768 41.622443403572326,1.6507559695776892 41.98677786085107,1.764664192292795 42.351011192745254,1.8794880446399878 42.71513411159017,1.9952502916543926 43.079137301616434))' gg, created = GeographicLocation.objects.get_or_create(geometry=WKTReader().read(geom_wkt)) geom_wkt = 'POLYGON((1.995 43.079,1.808 43.1102,1.628 43.139,1.454 43.168,1.287 43.195,1.126 43.220,0.972 43.245,0.825 43.268,0.685 43.290,0.551 43.310,0.417 43.330,0.417 43.330,0.312 42.966,0.207 42.601,0.103 42.236,0.0 41.871,-0.102 41.506,-0.204 41.141,-0.305 40.776,-0.405 40.411,-0.505 40.045,-0.618 39.628,-0.618 39.628,-0.493 39.608,-0.368 39.588,-0.236 39.566,-0.098 39.544,0.0456 39.520,0.196 39.495,0.353 39.469,0.516 39.441,0.686 39.412,0.861 39.382,0.861 39.382,0.985 39.799,1.094 40.164,1.203 40.528,1.314 40.893,1.425 41.258,1.537 41.622,1.650 41.986,1.764 42.351,1.879 42.715,1.995 43.079))' @@ -181,7 +184,6 @@ def test__reproduce__not_null_constraint_failed(self): self.assertTrue(created) gg, created = GeographicLocation.objects.get_or_create(geometry=WKTReader().read(geom_wkt)) self.assertFalse(created) - # Conclusion: db can't handle numbers with too many decimals (NOT NULL constraint failed) class PersonnelTests(TestCase):