diff --git a/.gitignore b/.gitignore index d0944ee66b..e6b53ea4ee 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ *.tar *.zip !regional_areas.zip +!qaqc_well_data.zip # Logs and databases # ###################### diff --git a/app/backend/aquifers/fixtures/aquifers.json b/app/backend/aquifers/fixtures/aquifers.json index 09fc45a1f6..470c1cee40 100644 --- a/app/backend/aquifers/fixtures/aquifers.json +++ b/app/backend/aquifers/fixtures/aquifers.json @@ -984,7 +984,13 @@ "hydro_fracturing_yield_increase": null, "decommission_sealant_material": "CONCRETE", "decommission_backfill_material": "test", - "geom": "POINT(-125.376319 52.528545)" + "geom": "POINT(-125.376319 52.528545)", + "distance_to_pid": "123.45", + "geocode_distance": "543.21", + "score_address": "78.90", + "score_city": "56.78", + "cross_referenced": false, + "natural_resource_region": "Northeast" } }, { @@ -1109,7 +1115,13 @@ "hydro_fracturing_yield_increase": null, "decommission_sealant_material": "CONCRETE", "decommission_backfill_material": "test", - "geom": "POINT(-125.351850 52.461419)" + "geom": "POINT(-125.351850 52.461419)", + "distance_to_pid": "234.56", + "geocode_distance": "654.32", + "score_address": "89.01", + "score_city": "67.89", + "cross_referenced": true, + "natural_resource_region": "Northeast" } }, { @@ -1234,7 +1246,13 @@ "hydro_fracturing_yield_increase": null, "decommission_sealant_material": "CONCRETE", "decommission_backfill_material": "test", - "geom": "POINT(-125.361842 52.476454)" + "geom": "POINT(-125.361842 52.476454)", + "distance_to_pid": "345.67", + "geocode_distance": "765.43", + "score_address": "90.12", + "score_city": "78.90", + "cross_referenced": false, + "natural_resource_region": "Northeast" } }, { @@ -1358,7 +1376,13 @@ "hydro_fracturing_yield_increase": null, "decommission_sealant_material": "CONCRETE", "decommission_backfill_material": "test", - "geom": "POINT(-125.381245 52.466459)" + "geom": "POINT(-125.381245 52.466459)", + "distance_to_pid": "456.78", + "geocode_distance": "876.54", + "score_address": "12.34", + "score_city": "89.01", + "cross_referenced": true, + "natural_resource_region": "Northeast" } }, { diff --git a/app/backend/gwells/fixtures/wellsearch.json b/app/backend/gwells/fixtures/wellsearch.json index b493fb2f6c..93590be5aa 100644 --- a/app/backend/gwells/fixtures/wellsearch.json +++ b/app/backend/gwells/fixtures/wellsearch.json @@ -845,7 +845,13 @@ "decommission_method": null, "decommission_details": null, "water_quality_characteristics": [], - "geom": "POINT(-122.540000 49.260000)" + "geom": "POINT(-122.540000 49.260000)", + "distance_to_pid": "123.45", + "geocode_distance": "543.21", + "score_address": "78.90", + "score_city": "56.78", + "cross_referenced": false, + "natural_resource_region": "Northeast" } }, { @@ -966,7 +972,13 @@ "drawdown": "190.00", "hydro_fracturing_performed": false, "hydro_fracturing_yield_increase": null, - "geom": "POINT(-122.540000 49.200000)" + "geom": "POINT(-122.540000 49.200000)", + "distance_to_pid": "234.56", + "geocode_distance": "654.32", + "score_address": "89.01", + "score_city": "67.89", + "cross_referenced": true, + "natural_resource_region": "Northeast" } }, { @@ -1087,7 +1099,13 @@ "drawdown": null, "hydro_fracturing_performed": false, "hydro_fracturing_yield_increase": null, - "geom": "POINT(-122.580000 49.230000)" + "geom": "POINT(-122.580000 49.230000)", + "distance_to_pid": "345.67", + "geocode_distance": "765.43", + "score_address": "90.12", + "score_city": "78.90", + "cross_referenced": false, + "natural_resource_region": "Northeast" } }, { @@ -1208,7 +1226,13 @@ "drawdown": null, "hydro_fracturing_performed": false, "hydro_fracturing_yield_increase": null, - "geom": "POINT(-122.590000 49.250000)" + "geom": "POINT(-122.590000 49.250000)", + "distance_to_pid": "456.78", + "geocode_distance": "876.54", + "score_address": "12.34", + "score_city": "89.01", + "cross_referenced": true, + "natural_resource_region": "Northeast" } }, { @@ -1332,7 +1356,13 @@ "hydro_fracturing_yield_increase": null, "decommission_sealant_material": "CONCRETE", "decommission_backfill_material": "test", - "geom": "POINT(-125.360830 52.456449)" + "geom": "POINT(-125.360830 52.456449)", + "distance_to_pid": "567.89", + "geocode_distance": "987.65", + "score_address": "23.45", + "score_city": "90.12", + "cross_referenced": false, + "natural_resource_region": "South Coast" } }, { @@ -1479,7 +1509,13 @@ ], "decommission_backfill_material": null, "decommission_sealant_material": null, - "geom": null + "geom": null, + "distance_to_pid": "678.90", + "geocode_distance": "109.87", + "score_address": "34.56", + "score_city": "12.34", + "cross_referenced": true, + "natural_resource_region": "Kootenay" } }, { diff --git a/app/backend/gwells/utils.py b/app/backend/gwells/utils.py index 69753110ab..937977a0c3 100644 --- a/app/backend/gwells/utils.py +++ b/app/backend/gwells/utils.py @@ -20,56 +20,45 @@ def isPointInsideBC(latitude, longitude): return False -def geocode_bc_location(options={}): - """ - Makes an HTTP call to the BC Physical Address Geocoder API - (https://www2.gov.bc.ca/gov/content/data/geographic-data-services/location-services/geocoder) - using any options provided as query string parameters. (the 'options' - parameter supports any query string parameter supported by the "addresses.json" - endpoint. - If the address is successfully geocoded then this method returns a - django.contrib.gis.geos.Point object corresponding to the first result. - If a HTTP error occurs during - communication with the remote API then an HTTPError exception is - raised. If the API call succeeds but does not find a coordinate - matching the given address_string, then a ValueError is raised. - :param options: typical options are: - { - "addressString": "101 main st.", - "localityName": "Kelowna" - } - """ +def setup_parameters(options): default_options = { - "provinceCode": "BC", - "outputSRS": 4326, - "maxResults": 1, - "minScore": 65 + "provinceCode": "BC", + "outputSRS": 4326, + "maxResults": 1, + "minScore": 65 } - params = {} - params.update(default_options) - params.update(options) + return {**default_options, **options} - url = "https://geocoder.api.gov.bc.ca/addresses.json" - +def perform_api_request(url, params): try: resp = requests.get(url, params=params, timeout=10) resp.raise_for_status() - except HTTPError as e: - #caught and re-raised to be clear ane explicit which exceptions - #this method may cause + return resp + except requests.HTTPError as e: raise e - features = [] - - try: - features = resp.json().get('features') +def process_response(response): + try: + features = response.json().get('features') + if not features: + raise ValueError("Unable to geocode address") + return features[0] except AttributeError as e: raise ValueError("Unable to geocode address") - if not len(features): - raise ValueError("Unable to geocode address") - - first_feature = features[0] +def geocode_bc_location(options={}): + """ + Performs an HTTP request to the BC Physical Address Geocoder API, + returning a django.contrib.gis.geos.Point for the first result. Supports query + string parameters via the 'options' argument. Raises HTTPError for + communication issues and ValueError if no matching coordinate is found. + Example 'options': {"addressString": "101 main st.", "localityName": "Kelowna"}. + """ + params = setup_parameters(options) + url = "https://geocoder.api.gov.bc.ca/addresses.json" + response = perform_api_request(url, params) + first_feature = process_response(response) + try: point = GEOSGeometry(json.dumps(first_feature.get("geometry", {}))) except TypeError as e: diff --git a/app/backend/requirements.txt b/app/backend/requirements.txt index daba280b3d..ce4ed3232c 100644 --- a/app/backend/requirements.txt +++ b/app/backend/requirements.txt @@ -28,3 +28,5 @@ geojson==2.4.1 MarkupSafe>=2.0.1 djangorestframework-simplejwt==4.4.0 pyjwt>=2.0,<=2.4.0 +thefuzz==0.19.0 +geopandas==0.9.0 \ No newline at end of file diff --git a/app/backend/wells/constants.py b/app/backend/wells/constants.py index 1a6321629b..fefc90d808 100644 --- a/app/backend/wells/constants.py +++ b/app/backend/wells/constants.py @@ -28,3 +28,36 @@ WELL_TAGS = [] WELL_TAGS.extend(WELL_TAGS_PUBLIC.copy()) WELL_TAGS.extend(WELL_TAGS_PRIVATE.copy()) + +# bc geocoder endpoint of interest +GEOCODER_ENDPOINT = "https://geocoder.api.gov.bc.ca/sites/nearest.json" +ADDRESS_COLUMNS = [ + "fullAddress", + "siteName", + "unitDesignator", + "unitNumber", + "unitNumberSuffix", + "civicNumber", + "civicNumberSuffix", + "streetName", + "streetType", + "isStreetTypePrefix", + "streetDirection", + "isStreetDirectionPrefix", + "streetQualifier", + "localityName", + "localityType", + "electoralArea", + "provinceCode", + "locationPositionalAccuracy", + "locationDescriptor", + "siteID", + "blockID", + "fullSiteDescriptor", + "accessNotes", + "siteStatus", + "siteRetireDate", + "changeDate", + "isOfficial", + "distance", +] diff --git a/app/backend/wells/fixtures/qaqc_well_data.zip b/app/backend/wells/fixtures/qaqc_well_data.zip new file mode 100644 index 0000000000..7a53c0ac7a Binary files /dev/null and b/app/backend/wells/fixtures/qaqc_well_data.zip differ diff --git a/app/backend/wells/fixtures/well_detail_fixture.json b/app/backend/wells/fixtures/well_detail_fixture.json index 751e2ee4ae..e39286f9d1 100644 --- a/app/backend/wells/fixtures/well_detail_fixture.json +++ b/app/backend/wells/fixtures/well_detail_fixture.json @@ -114,5 +114,12 @@ "construction_start_date": null, "well_status": null, "filter_pack_thickness": null, - "well_yield": null}} + "well_yield": null, + "distance_to_pid": "123.45", + "geocode_distance": "543.21", + "score_address": "78.90", + "score_city": "56.78", + "cross_referenced": false, + "natural_resource_region": "Northeast" + }} ] diff --git a/app/backend/wells/migrations/0146_auto_20240105_qaqc.py b/app/backend/wells/migrations/0146_auto_20240105_qaqc.py new file mode 100644 index 0000000000..79258bd370 --- /dev/null +++ b/app/backend/wells/migrations/0146_auto_20240105_qaqc.py @@ -0,0 +1,46 @@ +# Generated by Django 2.2.28 on 2024-01-05 02:20 + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('wells', '0145_auto_20231127_2105'), + ] + + operations = [ + migrations.AddField( + model_name='well', + name='distance_to_pid', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=7, null=True, verbose_name='Distance to PID'), + ), + migrations.AddField( + model_name='well', + name='geocode_distance', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=7, null=True, verbose_name='Geocode Distance'), + ), + migrations.AddField( + model_name='well', + name='score_address', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=7, null=True, verbose_name='Score for Address'), + ), + migrations.AddField( + model_name='well', + name='score_city', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=7, null=True, verbose_name='Score for City'), + ), + migrations.AddField( + model_name='well', + name='cross_referenced', + field=models.BooleanField(default=False, verbose_name='Cross Referenced'), + ), + migrations.AddField( + model_name='well', + name='natural_resource_region', + field=models.CharField(default='', max_length=200), + ), + ] diff --git a/app/backend/wells/migrations/0147_auto_20240105_qaqc_data.py b/app/backend/wells/migrations/0147_auto_20240105_qaqc_data.py new file mode 100644 index 0000000000..0defc456e2 --- /dev/null +++ b/app/backend/wells/migrations/0147_auto_20240105_qaqc_data.py @@ -0,0 +1,67 @@ +from django.db import migrations +import csv +import zipfile +import os + +def import_well_data(apps, schema_editor): + WellModel = apps.get_model('wells', 'Well') + well_count = WellModel.objects.count() + dev_threshold = 100 + + if well_count < dev_threshold: + print("Skipping qaqc data migration as it seems to be a non-production environment.") + return + + process_wells(WellModel, get_well_data()) + +def get_well_data(): + migration_dir = os.path.dirname(__file__) + with zipfile.ZipFile(os.path.join(migration_dir, '../fixtures/qaqc_well_data.zip'), 'r') as zip_file: + csv_filename = zip_file.namelist()[0] + with zip_file.open(csv_filename, 'r') as csvfile: + return csv.DictReader(csvfile.read().decode('utf-8').splitlines()) + +def process_wells(WellModel, reader): + batch_size = 1000 # Adjust batch size if there are memory issues + wells_to_update = [] + count = 0 + + for row in reader: + try: + well_instance = WellModel.objects.get(well_tag_number=row['well_tag_number']) + update_well_attributes(well_instance, row) + wells_to_update.append(well_instance) + count += 1 + + # Process in batches of batch_size + if count % batch_size == 0: + WellModel.objects.bulk_update(wells_to_update, ['geocode_distance', 'distance_to_pid', 'score_address', 'score_city', 'cross_referenced', 'natural_resource_region']) + wells_to_update = [] # Reset the list after updating + + except WellModel.DoesNotExist: + print(f"Well with tag number {row['well_tag_number']} not found.") + except ValueError as e: + print(f"Error processing well {row['well_tag_number']}: {e}") + + # Update any remaining wells in the list + if wells_to_update: + WellModel.objects.bulk_update(wells_to_update, ['geocode_distance', 'distance_to_pid', 'score_address', 'score_city', 'cross_referenced', 'natural_resource_region']) + +def update_well_attributes(well, row): + fields_to_update = ['distance_geocode', 'distance_to_matching_pid', 'score_address', 'score_city'] + for field in fields_to_update: + setattr(well, 'geocode_distance' if field == 'distance_geocode' else field, + float(row[field]) if row[field] else None) + + well.cross_referenced = row['xref_ind'] == 'True' + well.natural_resource_region = row['nr_region_name'] if row['nr_region_name'] else None + +class Migration(migrations.Migration): + + dependencies = [ + ('wells', '0146_auto_20240105_qaqc'), + ] + + operations = [ + migrations.RunPython(import_well_data, reverse_code=migrations.RunPython.noop), + ] diff --git a/app/backend/wells/models.py b/app/backend/wells/models.py index ee00e7e598..292e85789c 100644 --- a/app/backend/wells/models.py +++ b/app/backend/wells/models.py @@ -33,6 +33,8 @@ LithologyMaterialCode, BedrockMaterialCode, BedrockMaterialDescriptorCode, LithologyStructureCode, LithologyMoistureCode, SurficialMaterialCode) from gwells.db_comments.patch_fields import patch_fields +from wells.utils import calculate_geocode_distance, calculate_pid_distance_for_well, \ + calculate_score_address, calculate_score_city, calculate_natural_resource_region_for_well # from aquifers.models import Aquifer @@ -1148,6 +1150,25 @@ class Well(AuditModelStructure): recommended_pump_rate = models.DecimalField(max_digits=7, decimal_places=2, blank=True, null=True, verbose_name='Recommended pump rate', validators=[MinValueValidator(Decimal('0.00'))]) + # QaQc Fields for internal use + geocode_distance = models.DecimalField( + null=True, blank=True, max_digits=7, decimal_places=2, verbose_name='Geocode Distance', + db_comment='Distance calculated during geocoding process.') + distance_to_pid = models.DecimalField( + null=True, blank=True, max_digits=7, decimal_places=2, verbose_name='Distance to PID', + db_comment='Distance to the Property Identification Description.') + score_address = models.DecimalField( + null=True, blank=True, max_digits=7, decimal_places=2, verbose_name='Score for Address', + db_comment='Score representing the accuracy or confidence of the address geocoding.') + score_city = models.DecimalField( + null=True, blank=True, max_digits=7, decimal_places=2, verbose_name='Score for City', + db_comment='Score representing the accuracy or confidence of the city geocoding.') + cross_referenced = models.BooleanField( + default=False, verbose_name='Cross Referenced', + db_comment='Indicates if the record has been cross-referenced by an internal team member.') + natural_resource_region = models.CharField( + max_length=250, blank=True, null=True, verbose_name="Natural Resource Region", + db_comment='The Natural Resource Region the well is located within.') class Meta: db_table = 'well' @@ -1294,6 +1315,50 @@ def update_utm(sender, instance, **kwargs): instance.utm_northing = round(utm_point.y) +@receiver(pre_save, sender=Well) +def update_well(sender, instance, **kwargs): + """ + Signal receiver that triggers before a Well instance is saved. + + For new Well instances, it calculates and sets various geographical and scoring fields. + For existing Well instances, it recalculates these fields if the geographical location (geom) has changed. + + Parameters: + sender (Model Class): The model class that sent the signal. Should always be the Well model. + instance (Well instance): The instance of Well being saved. + kwargs: Additional keyword arguments. Not used in this function. + """ + + def is_valid_geom(geom): + """ + Helper function to check if the geom attribute is valid. + A valid geom should be non-null and must have both latitude and longitude. + """ + return geom and hasattr(geom, 'latitude') and hasattr(geom, 'longitude') + + try: + if instance._state.adding and not instance.pk: + # Handling new instance creation + if is_valid_geom(instance.geom): + instance.geocode_distance = calculate_geocode_distance(instance) + instance.distance_to_pid = calculate_pid_distance_for_well(instance) + instance.score_address = calculate_score_address(instance) + instance.score_city = calculate_score_city(instance) + instance.natural_resource_region = calculate_natural_resource_region_for_well(instance) + else: + # Handling updates to existing instances + original_instance = sender.objects.get(pk=instance.pk) + if original_instance.geom != instance.geom: + if is_valid_geom(instance.geom): + instance.geocode_distance = calculate_geocode_distance(instance) + instance.distance_to_pid = calculate_pid_distance_for_well(instance) + instance.score_address = calculate_score_address(instance) + instance.score_city = calculate_score_city(instance) + instance.natural_resource_region = calculate_natural_resource_region_for_well(instance) + except Exception as e: + print(f"Error in update_well for Well ID {instance.pk}: {str(e)}") + + class CasingMaterialCode(CodeTableModel): """ The material used for casing a well, e.g., Cement, Plastic, Steel. diff --git a/app/backend/wells/serializers_v2.py b/app/backend/wells/serializers_v2.py index b432502683..fc003164ce 100644 --- a/app/backend/wells/serializers_v2.py +++ b/app/backend/wells/serializers_v2.py @@ -475,3 +475,76 @@ class WellDetailSerializer(WellDetailSerializerV1): class Meta(WellDetailSerializerV1.Meta): ref_name = "well_detail_v2" + + +class MislocatedWellsSerializer(serializers.ModelSerializer): + work_start_date = serializers.DateField(read_only=True) + work_end_date = serializers.DateField(read_only=True) + + class Meta: + model = Well + fields = [ + 'well_tag_number', + 'geocode_distance', + 'distance_to_pid', + 'score_address', + 'score_city', + 'well_status', + 'work_start_date', + 'work_end_date', + 'company_of_person_responsible', + 'natural_resource_region', + 'create_date', + 'create_user', + 'internal_comments' + ] + + +class CrossReferencingSerializer(serializers.ModelSerializer): + work_start_date = serializers.DateField(read_only=True) + work_end_date = serializers.DateField(read_only=True) + + class Meta: + model = Well + fields = [ + 'well_tag_number', + 'well_status', + 'work_start_date', + 'work_end_date', + 'create_user', + 'create_date', + 'update_date', + 'update_user', + 'natural_resource_region', + 'internal_comments', + 'cross_referenced' + ] + + +class RecordComplianceSerializer(serializers.ModelSerializer): + work_start_date = serializers.DateField(read_only=True) + work_end_date = serializers.DateField(read_only=True) + + class Meta: + model = Well + fields = [ + 'well_tag_number', + 'identification_plate_number', + 'well_class', + 'latitude', + 'longitude', + 'finished_well_depth', + 'diameter', + 'surface_seal_depth', + 'surface_seal_thickness', + 'aquifer_lithology', + 'well_status', + 'work_start_date', + 'work_end_date', + 'person_responsible', + 'company_of_person_responsible', + 'create_date', + 'create_user', + 'natural_resource_region', + 'internal_comments' + ] diff --git a/app/backend/wells/urls.py b/app/backend/wells/urls.py index 06774b3f2e..a175038761 100644 --- a/app/backend/wells/urls.py +++ b/app/backend/wells/urls.py @@ -114,5 +114,16 @@ # Well Licensing status endpoint from e-Licensing. url(api_path_prefix() + r'/wells/licensing$', - views.well_licensing, name='well-licensing') + views.well_licensing, name='well-licensing'), + + # QA/QC Endpoints + url(api_path_prefix() + r'/qaqc/crossreferencing$', + never_cache(views_v2.CrossReferencingListView.as_view()), name='qaqc-cross-referencing'), + + url(api_path_prefix() + r'/qaqc/mislocatedwells$', + never_cache(views_v2.MislocatedWellsListView.as_view()), name='qaqc-mislocated-wells'), + + url(api_path_prefix() + r'/qaqc/recordcompliance$', + never_cache(views_v2.RecordComplianceListView.as_view()), name='qaqc-record-compliance'), + ] diff --git a/app/backend/wells/utils.py b/app/backend/wells/utils.py new file mode 100644 index 0000000000..cb013bb395 --- /dev/null +++ b/app/backend/wells/utils.py @@ -0,0 +1,151 @@ +import requests +import geopandas as gpd +from time import sleep +from shapely.geometry import Point +from wells.constants import ADDRESS_COLUMNS, GEOCODER_ENDPOINT +from thefuzz import fuzz + +def calculate_pid_distance_for_well(well): + """ + Calculate the distance from a single well to the nearest parcel using a WFS query. + :param well: A well instance with latitude, longitude attributes + :return: Distance to the nearest parcel + """ + # Base URL for the WFS request + base_url = "https://openmaps.gov.bc.ca/geo/pub/wfs" + params = { + "service": "WFS", + "version": "2.0.0", + "request": "GetFeature", + "typeName": "WHSE_CADASTRE.PMBC_PARCEL_FABRIC_POLY_SVW", + "outputFormat": "json", + "srsName": "EPSG:4326", + "CQL_FILTER": f"DWITHIN(geometry, POINT({well.longitude} {well.latitude}), 0.1, meters)" + } + + # Construct the request URL + request_url = f"{base_url}?{'&'.join([f'{k}={v}' for k, v in params.items()])}" + + # Make the request + response = requests.get(request_url) + if response.status_code != 200: + print("Error making request to WFS service.") + return None + + # Load response into GeoDataFrame + data = response.json() + parcels_gdf = gpd.GeoDataFrame.from_features(data["features"]) + + if parcels_gdf.empty: + print("No parcels found near well location.") + return None + + # Calculate the distance from the well to the nearest parcel + well_point = Point(well.longitude, well.latitude) + parcels_gdf["distance"] = parcels_gdf.distance(well_point) + + return parcels_gdf["distance"].min() + + +def calculate_natural_resource_region_for_well(well): + """ + Retrieve the natural resource region name that a well is within using a WFS query. + :param well: A well instance with latitude, longitude attributes + :return: Natural Resource Region name + """ + # Base URL for the WFS request + base_url = "https://openmaps.gov.bc.ca/geo/pub/wfs" + params = { + "service": "WFS", + "version": "2.0.0", + "request": "GetFeature", + "typeName": "WHSE_ADMIN_BOUNDARIES.ADM_NR_DISTRICTS_SPG", + "outputFormat": "json", + "srsName": "EPSG:4326", + "CQL_FILTER": f"CONTAINS(geometry, POINT({well.longitude} {well.latitude}))" + } + + # Construct the request URL + request_url = f"{base_url}?{'&'.join([f'{k}={v}' for k, v in params.items()])}" + + # Make the request + response = requests.get(request_url) + if response.status_code != 200: + print("Error making request to WFS service.") + return None + + # Load response into GeoDataFrame + data = response.json() + regions_gdf = gpd.GeoDataFrame.from_features(data["features"]) + + if regions_gdf.empty: + print("No natural resource regions found near well location.") + return None + + # Assuming the well can only be in one region, return the name of the first region found + return regions_gdf.iloc[0]['properties']['REGION_ORG_UNIT_NAME'] + + +def reverse_geocode( + x, + y, + distance_start=200, + distance_increment=200, + distance_max=2000, +): + """ + Provided a location as x/y coordinates (EPSG:4326), request an address + from BC Geocoder within given distance_start (metres) + + If no result is found, request using an expanding search radius in + distance_increment steps, until distance_max is reached. + + A dict with 'distance' = 99999 is returned if no result is found. + + """ + result = False + distance = distance_start + # expand the search distance until we get a result or hit the max distance + while result is False and distance <= distance_max: + params = { + "point": str(x) + "," + str(y), + "apikey": 'fake_api_key', # api key not required to be valid + "outputFormat": "json", + "maxDistance": distance, + } + r = requests.get(GEOCODER_ENDPOINT, params=params) + + # pause for 2s per request if near limit of 1000 requests/min + if int(r.headers["RateLimit-Remaining"]) < 30: + sleep(2) + if r.status_code == 200: + result = True + else: + distance = distance + distance_increment + if r.status_code == 200: + address = r.json()["properties"] + address["distance"] = distance + return address + else: + empty_result = dict([(k, "") for k in ADDRESS_COLUMNS]) + empty_result["distance"] = 99999 + return empty_result + + +def calculate_geocode_distance(well): + response = reverse_geocode(well.longitude, well.latitude) + return response.get('distance', None) if response else None + + +def calculate_score_address(well): + geocoded_address = reverse_geocode(well.longitude, well.latitude) + if not geocoded_address: + return None + return fuzz.token_set_ratio(well.street_address.lower(), geocoded_address.get('fullAddress', '').lower()) + + +def calculate_score_city(well): + geocoded_address = reverse_geocode(well.longitude, well.latitude) + if not geocoded_address: + return None + return fuzz.token_set_ratio(well.city.lower(), geocoded_address.get('localityName', '').lower()) diff --git a/app/backend/wells/views_v2.py b/app/backend/wells/views_v2.py index b5a32c66c2..ed4f8234d3 100644 --- a/app/backend/wells/views_v2.py +++ b/app/backend/wells/views_v2.py @@ -20,8 +20,8 @@ from django.contrib.gis.db.models.functions import Distance from django.contrib.gis.geos import GEOSException, GEOSGeometry from django.contrib.gis.gdal import GDALException -from django.db.models import FloatField -from django.db.models.functions import Cast +from django.db.models.functions import Cast, Lower +from django.db.models import FloatField, Q, Case, When, F, Value, DateField from rest_framework import status, filters from rest_framework.exceptions import PermissionDenied, NotFound, ValidationError @@ -38,7 +38,8 @@ GeometryFilterBackend, RadiusFilterBackend ) -from wells.models import Well, WellAttachment +from wells.models import Well, WellAttachment, \ + WELL_STATUS_CODE_ALTERATION, WELL_STATUS_CODE_CONSTRUCTION, WELL_STATUS_CODE_DECOMMISSION from wells.serializers_v2 import ( WellLocationSerializerV2, WellVerticalAquiferExtentSerializerV2, @@ -47,7 +48,10 @@ WellExportSerializerV2, WellExportAdminSerializerV2, WellSubsurfaceSerializer, - WellDetailSerializer + WellDetailSerializer, + MislocatedWellsSerializer, + CrossReferencingSerializer, + RecordComplianceSerializer ) from wells.permissions import WellsEditOrReadOnly from wells.renderers import WellListCSVRenderer, WellListExcelRenderer @@ -573,3 +577,170 @@ class WellDetail(WellDetailV1): This view is open to all, and has no permissions. """ serializer_class = WellDetailSerializer + + +class MislocatedWellsListView(ListAPIView): + """ + API view to retrieve mislocated wells. + """ + serializer_class = MislocatedWellsSerializer + + swagger_schema = None + permission_classes = (WellsEditOrReadOnly,) + model = Well + pagination_class = APILimitOffsetPagination + + # Allow searching on name fields, names of related companies, etc. + filter_backends = (WellListFilterBackend, BoundingBoxFilterBackend, + filters.SearchFilter, WellListOrderingFilter, GeometryFilterBackend) + ordering = ('well_tag_number',) + + def get_queryset(self): + """ + This view should return a list of all mislocated wells + for the currently authenticated user. + """ + queryset = Well.objects.all() + + queryset = Well.objects.select_related('well_status').annotate( + work_start_date=Case( + When(well_status__well_status_code=WELL_STATUS_CODE_CONSTRUCTION, then=F('construction_start_date')), + When(well_status__well_status_code=WELL_STATUS_CODE_ALTERATION, then=F('alteration_start_date')), + When(well_status__well_status_code=WELL_STATUS_CODE_DECOMMISSION, then=F('decommission_start_date')), + default=Value(None), + output_field=DateField() + ), + work_end_date=Case( + When(well_status__well_status_code=WELL_STATUS_CODE_CONSTRUCTION, then=F('construction_end_date')), + When(well_status__well_status_code=WELL_STATUS_CODE_ALTERATION, then=F('alteration_end_date')), + When(well_status__well_status_code=WELL_STATUS_CODE_DECOMMISSION, then=F('decommission_end_date')), + default=Value(None), + output_field=DateField() + ) + ) + + return queryset + + +class RecordComplianceListView(ListAPIView): + serializer_class = RecordComplianceSerializer + + swagger_schema = None + permission_classes = (WellsEditOrReadOnly,) + model = Well + pagination_class = APILimitOffsetPagination + + # Allow searching on name fields, names of related companies, etc. + filter_backends = (WellListFilterBackend, BoundingBoxFilterBackend, + filters.SearchFilter, WellListOrderingFilter, GeometryFilterBackend) + ordering = ('well_tag_number',) + + def get_queryset(self): + """ + Retrieves wells that are missing information in any of the specified fields. + """ + queryset = Well.objects.all() + + queryset = Well.objects.select_related('well_status').annotate( + work_start_date=Case( + When(well_status__well_status_code=WELL_STATUS_CODE_CONSTRUCTION, then=F('construction_start_date')), + When(well_status__well_status_code=WELL_STATUS_CODE_ALTERATION, then=F('alteration_start_date')), + When(well_status__well_status_code=WELL_STATUS_CODE_DECOMMISSION, then=F('decommission_start_date')), + default=Value(None), + output_field=DateField() + ), + work_end_date=Case( + When(well_status__well_status_code=WELL_STATUS_CODE_CONSTRUCTION, then=F('construction_end_date')), + When(well_status__well_status_code=WELL_STATUS_CODE_ALTERATION, then=F('alteration_end_date')), + When(well_status__well_status_code=WELL_STATUS_CODE_DECOMMISSION, then=F('decommission_end_date')), + default=Value(None), + output_field=DateField() + ) + ) + + # Filtering for records missing any of the specified fields + missing_info_filter = ( + Q(well_tag_number__isnull=True) | + Q(identification_plate_number__isnull=True) | + Q(well_class__isnull=True) | + Q(geom__isnull=True) | # for latitude and longitude + Q(finished_well_depth__isnull=True) | + Q(surface_seal_depth__isnull=True) | + Q(surface_seal_thickness__isnull=True) | + Q(aquifer_lithology__isnull=True) | + Q(well_status__isnull=True) | + Q(work_start_date__isnull=True) | + Q(work_end_date__isnull=True) | + Q(person_responsible__isnull=True) | + Q(company_of_person_responsible__isnull=True) | + Q(create_date__isnull=True) | + Q(create_user__isnull=True) + # Q(natural_resource_region__isnull=True) | + # Q(internal_comments__isnull=True) + ) + + queryset = queryset.filter(missing_info_filter) + + # Additional filtering based on query parameters + work_start_date = self.request.query_params.get('work_start_date') + work_end_date = self.request.query_params.get('work_end_date') + + if work_start_date: + queryset = queryset.filter(work_start_date__gte=work_start_date) + if work_end_date: + queryset = queryset.filter(work_end_date__lte=work_end_date) + + return queryset + + +class CrossReferencingListView(ListAPIView): + serializer_class = CrossReferencingSerializer + + swagger_schema = None + permission_classes = (WellsEditOrReadOnly,) + model = Well + pagination_class = APILimitOffsetPagination + + # Allow searching on name fields, names of related companies, etc. + filter_backends = (WellListFilterBackend, BoundingBoxFilterBackend, + filters.SearchFilter, WellListOrderingFilter, GeometryFilterBackend) + ordering = ('well_tag_number',) + + def get_queryset(self): + """ + Optionally restricts the returned wells to those that have certain keywords like 'x-ref'd' or 'cross-ref' + in their internal_comments. + """ + queryset = Well.objects.all() + + queryset = Well.objects.select_related('well_status').annotate( + work_start_date=Case( + When(well_status__well_status_code=WELL_STATUS_CODE_CONSTRUCTION, then=F('construction_start_date')), + When(well_status__well_status_code=WELL_STATUS_CODE_ALTERATION, then=F('alteration_start_date')), + When(well_status__well_status_code=WELL_STATUS_CODE_DECOMMISSION, then=F('decommission_start_date')), + default=Value(None), + output_field=DateField() + ), + work_end_date=Case( + When(well_status__well_status_code=WELL_STATUS_CODE_CONSTRUCTION, then=F('construction_end_date')), + When(well_status__well_status_code=WELL_STATUS_CODE_ALTERATION, then=F('alteration_end_date')), + When(well_status__well_status_code=WELL_STATUS_CODE_DECOMMISSION, then=F('decommission_end_date')), + default=Value(None), + output_field=DateField() + ) + ) + + search_terms = ["x-ref'd", "x-ref", "cross-ref", "cross r", "cross-r", "ref'd", "referenced", "refd", "xref", "x-r", "x r"] + + # Annotate the queryset to add a lowercase version of internal_comments + queryset = Well.objects.annotate(lower_internal_comments=Lower('internal_comments')) + + # Build a Q object for the search terms, against the lowercase internal_comments + comments_query = Q() + for term in search_terms: + comments_query |= Q(lower_internal_comments__icontains=term) + + # Filter the queryset based on the search terms + queryset = queryset.filter(comments_query) + + return queryset diff --git a/app/frontend/package-lock.json b/app/frontend/package-lock.json index c4cc294dc8..b4662a3eeb 100644 --- a/app/frontend/package-lock.json +++ b/app/frontend/package-lock.json @@ -2681,12 +2681,6 @@ "integrity": "sha1-l6ERlkmyEa0zaR2fn0hqjsn74KM=", "dev": true }, - "amdefine": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz", - "integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=", - "dev": true - }, "ansi-colors": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-3.2.4.tgz", @@ -2767,16 +2761,6 @@ "integrity": "sha512-BLM56aPo9vLLFVa8+/+pJLnrZ7QGGTVHWsCwieAWT9o9K8UeGaQbzZbGoabWLOo2ksBCztoXdqBZBplqLDDCSg==", "dev": true }, - "are-we-there-yet": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", - "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", - "dev": true, - "requires": { - "delegates": "^1.0.0", - "readable-stream": "^2.0.6" - } - }, "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -2810,12 +2794,6 @@ "integrity": "sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM=", "dev": true }, - "array-find-index": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/array-find-index/-/array-find-index-1.0.2.tgz", - "integrity": "sha1-3wEKoSh+Fku9pvlyOwqWoexBh6E=", - "dev": true - }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -2941,12 +2919,6 @@ "integrity": "sha512-z/WhQ5FPySLdvREByI2vZiTWwCnF0moMJ1hK9YQwDTHKh6I7/uSckMetoRGb5UBZPC1z0jlw+n/XCgjeH7y1AQ==", "dev": true }, - "async-foreach": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/async-foreach/-/async-foreach-0.1.3.tgz", - "integrity": "sha1-NhIfhFwFeBct5Bmpfb6x0W7DRUI=", - "dev": true - }, "async-limiter": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz", @@ -3585,15 +3557,6 @@ "file-uri-to-path": "1.0.0" } }, - "block-stream": { - "version": "0.0.9", - "resolved": "https://registry.npmjs.org/block-stream/-/block-stream-0.0.9.tgz", - "integrity": "sha1-E+v+d4oDIFz+A3UUgeu0szAMEmo=", - "dev": true, - "requires": { - "inherits": "~2.0.0" - } - }, "bluebird": { "version": "3.5.5", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.5.tgz", @@ -4098,24 +4061,6 @@ "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==" }, - "camelcase-keys": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", - "integrity": "sha1-MIvur/3ygRkFHvodkyITyRuPkuc=", - "dev": true, - "requires": { - "camelcase": "^2.0.0", - "map-obj": "^1.0.0" - }, - "dependencies": { - "camelcase": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-2.1.1.tgz", - "integrity": "sha1-fB0W1nmhu+WcoCys7PsBHiAfWh8=", - "dev": true - } - } - }, "caniuse-api": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/caniuse-api/-/caniuse-api-3.0.0.tgz", @@ -4222,6 +4167,19 @@ "path-is-absolute": "^1.0.0", "readdirp": "^2.2.1", "upath": "^1.1.1" + }, + "dependencies": { + "fsevents": { + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz", + "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==", + "dev": true, + "optional": true, + "requires": { + "bindings": "^1.5.0", + "nan": "^2.12.1" + } + } } }, "chownr": { @@ -4791,12 +4749,6 @@ "date-now": "^0.1.4" } }, - "console-control-strings": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", - "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=", - "dev": true - }, "consolidate": { "version": "0.15.1", "resolved": "https://registry.npmjs.org/consolidate/-/consolidate-0.15.1.tgz", @@ -5385,15 +5337,6 @@ "integrity": "sha1-8xz35PPiGLBybnOMqSoC00iO9hU=", "dev": true }, - "currently-unhandled": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/currently-unhandled/-/currently-unhandled-0.4.1.tgz", - "integrity": "sha1-mI3zP+qxke95mmE2nddsF635V+o=", - "dev": true, - "requires": { - "array-find-index": "^1.0.1" - } - }, "cyclist": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cyclist/-/cyclist-1.0.1.tgz", @@ -5649,12 +5592,6 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" }, - "delegates": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", - "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=", - "dev": true - }, "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -7506,806 +7443,185 @@ "dev": true }, "fsevents": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.9.tgz", - "integrity": "sha512-oeyj2H3EjjonWcFjD5NvZNE9Rqe4UW+nQBU2HNeKw0koVLEFIhtyETyAakeAM3de7Z/SW5kcA+fZUait9EApnw==", + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "optional": true + }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, + "function.prototype.name": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/function.prototype.name/-/function.prototype.name-1.1.5.tgz", + "integrity": "sha512-uN7m/BzVKQnCUF/iW8jYea67v++2u7m5UgENbHRtdDVclOUP+FMPlCNdmk0h/ysGyo2tavMJEDqJAkJdRa1vMA==", + "dev": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "es-abstract": "^1.19.0", + "functions-have-names": "^1.2.2" + } + }, + "functional-red-black-tree": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", + "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", + "dev": true + }, + "functions-have-names": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz", + "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", + "dev": true + }, + "fuzzy": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/fuzzy/-/fuzzy-0.1.3.tgz", + "integrity": "sha1-THbsL/CsGjap3M+aAN+GIweNTtg=" + }, + "geojson-vt": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/geojson-vt/-/geojson-vt-3.2.1.tgz", + "integrity": "sha512-EvGQQi/zPrDA6zr6BnJD/YhwAkBP8nnJ9emh3EnHQKVMfg/MRVtPbMYdgVy/IaEmn4UfagD2a6fafPDL5hbtwg==" + }, + "get-caller-file": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz", + "integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==", + "dev": true + }, + "get-intrinsic": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz", + "integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==", + "dev": true, + "requires": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + } + }, + "get-stream": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", + "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "dev": true, + "requires": { + "pump": "^3.0.0" + } + }, + "get-symbol-description": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz", + "integrity": "sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==", + "dev": true, + "requires": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.1.1" + } + }, + "get-value": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz", + "integrity": "sha1-3BXKHGcjh8p2vTesCjlbogQqLCg=", + "dev": true + }, + "getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "dev": true, + "requires": { + "assert-plus": "^1.0.0" + } + }, + "gl-matrix": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/gl-matrix/-/gl-matrix-3.3.0.tgz", + "integrity": "sha512-COb7LDz+SXaHtl/h4LeaFcNdJdAQSDeVqjiIihSXNrkWObZLhDI4hIkZC11Aeqp7bcE72clzB0BnDXr2SmslRA==" + }, + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "dev": true, - "optional": true, "requires": { - "nan": "^2.12.1", - "node-pre-gyp": "^0.12.0" + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" }, "dependencies": { - "abbrev": { - "version": "1.1.1", - "bundled": true, - "dev": true, - "optional": true - }, - "ansi-regex": { - "version": "2.1.1", - "bundled": true, - "dev": true, - "optional": true - }, - "aproba": { - "version": "1.2.0", - "bundled": true, + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "dev": true, - "optional": true - }, - "are-we-there-yet": { - "version": "1.1.5", - "bundled": true, + "requires": { + "brace-expansion": "^1.1.7" + } + } + } + }, + "glob-base": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/glob-base/-/glob-base-0.3.0.tgz", + "integrity": "sha1-27Fk9iIbHAscz4Kuoyi0l98Oo8Q=", + "dev": true, + "requires": { + "glob-parent": "^2.0.0", + "is-glob": "^2.0.0" + }, + "dependencies": { + "glob-parent": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-2.0.0.tgz", + "integrity": "sha1-gTg9ctsFT8zPUzbaqQLxgvbtuyg=", "dev": true, - "optional": true, "requires": { - "delegates": "^1.0.0", - "readable-stream": "^2.0.6" + "is-glob": "^2.0.0" } }, - "balanced-match": { + "is-extglob": { "version": "1.0.0", - "bundled": true, - "dev": true, - "optional": true + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-1.0.0.tgz", + "integrity": "sha1-rEaBd8SUNAWgkvyPKXYMb/xiBsA=", + "dev": true }, - "brace-expansion": { - "version": "1.1.11", - "bundled": true, + "is-glob": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-2.0.1.tgz", + "integrity": "sha1-0Jb5JqPe1WAPP9/ZEZjLCIjC2GM=", "dev": true, - "optional": true, "requires": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" + "is-extglob": "^1.0.0" } - }, - "chownr": { - "version": "1.1.1", - "bundled": true, - "dev": true, - "optional": true - }, - "code-point-at": { - "version": "1.1.0", - "bundled": true, - "dev": true, - "optional": true - }, - "concat-map": { - "version": "0.0.1", - "bundled": true, - "dev": true, - "optional": true - }, - "console-control-strings": { - "version": "1.1.0", - "bundled": true, - "dev": true, - "optional": true - }, - "core-util-is": { - "version": "1.0.2", - "bundled": true, - "dev": true, - "optional": true - }, - "debug": { - "version": "4.1.1", - "bundled": true, + } + } + }, + "glob-parent": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-3.1.0.tgz", + "integrity": "sha1-nmr2KZ2NO9K9QEMIMr0RPfkGxa4=", + "dev": true, + "requires": { + "is-glob": "^3.1.0", + "path-dirname": "^1.0.0" + }, + "dependencies": { + "is-glob": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-3.1.0.tgz", + "integrity": "sha1-e6WuJCF4BKxwcHuWkiVnSGzD6Eo=", "dev": true, - "optional": true, "requires": { - "ms": "^2.1.1" - } - }, - "deep-extend": { - "version": "0.6.0", - "bundled": true, - "dev": true, - "optional": true - }, - "delegates": { - "version": "1.0.0", - "bundled": true, - "dev": true, - "optional": true - }, - "detect-libc": { - "version": "1.0.3", - "bundled": true, - "dev": true, - "optional": true - }, - "fs-minipass": { - "version": "1.2.5", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "minipass": "^2.2.1" - } - }, - "fs.realpath": { - "version": "1.0.0", - "bundled": true, - "dev": true, - "optional": true - }, - "gauge": { - "version": "2.7.4", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "aproba": "^1.0.3", - "console-control-strings": "^1.0.0", - "has-unicode": "^2.0.0", - "object-assign": "^4.1.0", - "signal-exit": "^3.0.0", - "string-width": "^1.0.1", - "strip-ansi": "^3.0.1", - "wide-align": "^1.1.0" - } - }, - "glob": { - "version": "7.1.3", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "has-unicode": { - "version": "2.0.1", - "bundled": true, - "dev": true, - "optional": true - }, - "iconv-lite": { - "version": "0.4.24", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "safer-buffer": ">= 2.1.2 < 3" - } - }, - "ignore-walk": { - "version": "3.0.1", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "minimatch": "^3.0.4" - } - }, - "inflight": { - "version": "1.0.6", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "inherits": { - "version": "2.0.3", - "bundled": true, - "dev": true, - "optional": true - }, - "ini": { - "version": "1.3.5", - "bundled": true, - "dev": true, - "optional": true - }, - "is-fullwidth-code-point": { - "version": "1.0.0", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "number-is-nan": "^1.0.0" - } - }, - "isarray": { - "version": "1.0.0", - "bundled": true, - "dev": true, - "optional": true - }, - "minimatch": { - "version": "3.0.4", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "minimist": { - "version": "0.0.8", - "bundled": true, - "dev": true, - "optional": true - }, - "minipass": { - "version": "2.3.5", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "safe-buffer": "^5.1.2", - "yallist": "^3.0.0" - } - }, - "minizlib": { - "version": "1.2.1", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "minipass": "^2.2.1" - } - }, - "mkdirp": { - "version": "0.5.1", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "minimist": "0.0.8" - } - }, - "ms": { - "version": "2.1.1", - "bundled": true, - "dev": true, - "optional": true - }, - "needle": { - "version": "2.3.0", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "debug": "^4.1.0", - "iconv-lite": "^0.4.4", - "sax": "^1.2.4" - } - }, - "node-pre-gyp": { - "version": "0.12.0", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "detect-libc": "^1.0.2", - "mkdirp": "^0.5.1", - "needle": "^2.2.1", - "nopt": "^4.0.1", - "npm-packlist": "^1.1.6", - "npmlog": "^4.0.2", - "rc": "^1.2.7", - "rimraf": "^2.6.1", - "semver": "^5.3.0", - "tar": "^4" - } - }, - "nopt": { - "version": "4.0.1", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "abbrev": "1", - "osenv": "^0.1.4" - } - }, - "npm-bundled": { - "version": "1.0.6", - "bundled": true, - "dev": true, - "optional": true - }, - "npm-packlist": { - "version": "1.4.1", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "ignore-walk": "^3.0.1", - "npm-bundled": "^1.0.1" - } - }, - "npmlog": { - "version": "4.1.2", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "are-we-there-yet": "~1.1.2", - "console-control-strings": "~1.1.0", - "gauge": "~2.7.3", - "set-blocking": "~2.0.0" - } - }, - "number-is-nan": { - "version": "1.0.1", - "bundled": true, - "dev": true, - "optional": true - }, - "object-assign": { - "version": "4.1.1", - "bundled": true, - "dev": true, - "optional": true - }, - "once": { - "version": "1.4.0", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "wrappy": "1" - } - }, - "os-homedir": { - "version": "1.0.2", - "bundled": true, - "dev": true, - "optional": true - }, - "os-tmpdir": { - "version": "1.0.2", - "bundled": true, - "dev": true, - "optional": true - }, - "osenv": { - "version": "0.1.5", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "os-homedir": "^1.0.0", - "os-tmpdir": "^1.0.0" - } - }, - "path-is-absolute": { - "version": "1.0.1", - "bundled": true, - "dev": true, - "optional": true - }, - "process-nextick-args": { - "version": "2.0.0", - "bundled": true, - "dev": true, - "optional": true - }, - "rc": { - "version": "1.2.8", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "deep-extend": "^0.6.0", - "ini": "~1.3.0", - "minimist": "^1.2.0", - "strip-json-comments": "~2.0.1" - }, - "dependencies": { - "minimist": { - "version": "1.2.0", - "bundled": true, - "dev": true, - "optional": true - } - } - }, - "readable-stream": { - "version": "2.3.6", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "rimraf": { - "version": "2.6.3", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "glob": "^7.1.3" - } - }, - "safe-buffer": { - "version": "5.1.2", - "bundled": true, - "dev": true, - "optional": true - }, - "safer-buffer": { - "version": "2.1.2", - "bundled": true, - "dev": true, - "optional": true - }, - "sax": { - "version": "1.2.4", - "bundled": true, - "dev": true, - "optional": true - }, - "semver": { - "version": "5.7.0", - "bundled": true, - "dev": true, - "optional": true - }, - "set-blocking": { - "version": "2.0.0", - "bundled": true, - "dev": true, - "optional": true - }, - "signal-exit": { - "version": "3.0.2", - "bundled": true, - "dev": true, - "optional": true - }, - "string-width": { - "version": "1.0.2", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "code-point-at": "^1.0.0", - "is-fullwidth-code-point": "^1.0.0", - "strip-ansi": "^3.0.0" - } - }, - "string_decoder": { - "version": "1.1.1", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "safe-buffer": "~5.1.0" - } - }, - "strip-ansi": { - "version": "3.0.1", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "ansi-regex": "^2.0.0" - } - }, - "strip-json-comments": { - "version": "2.0.1", - "bundled": true, - "dev": true, - "optional": true - }, - "tar": { - "version": "4.4.8", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "chownr": "^1.1.1", - "fs-minipass": "^1.2.5", - "minipass": "^2.3.4", - "minizlib": "^1.1.1", - "mkdirp": "^0.5.0", - "safe-buffer": "^5.1.2", - "yallist": "^3.0.2" - } - }, - "util-deprecate": { - "version": "1.0.2", - "bundled": true, - "dev": true, - "optional": true - }, - "wide-align": { - "version": "1.1.3", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "string-width": "^1.0.2 || 2" - } - }, - "wrappy": { - "version": "1.0.2", - "bundled": true, - "dev": true, - "optional": true - }, - "yallist": { - "version": "3.0.3", - "bundled": true, - "dev": true, - "optional": true - } - } - }, - "fstream": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz", - "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", - "dev": true, - "requires": { - "graceful-fs": "^4.1.2", - "inherits": "~2.0.0", - "mkdirp": ">=0.5 0", - "rimraf": "2" - } - }, - "function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" - }, - "function.prototype.name": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/function.prototype.name/-/function.prototype.name-1.1.5.tgz", - "integrity": "sha512-uN7m/BzVKQnCUF/iW8jYea67v++2u7m5UgENbHRtdDVclOUP+FMPlCNdmk0h/ysGyo2tavMJEDqJAkJdRa1vMA==", - "dev": true, - "requires": { - "call-bind": "^1.0.2", - "define-properties": "^1.1.3", - "es-abstract": "^1.19.0", - "functions-have-names": "^1.2.2" - } - }, - "functional-red-black-tree": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", - "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", - "dev": true - }, - "functions-have-names": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz", - "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", - "dev": true - }, - "fuzzy": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/fuzzy/-/fuzzy-0.1.3.tgz", - "integrity": "sha1-THbsL/CsGjap3M+aAN+GIweNTtg=" - }, - "gauge": { - "version": "2.7.4", - "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", - "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", - "dev": true, - "requires": { - "aproba": "^1.0.3", - "console-control-strings": "^1.0.0", - "has-unicode": "^2.0.0", - "object-assign": "^4.1.0", - "signal-exit": "^3.0.0", - "string-width": "^1.0.1", - "strip-ansi": "^3.0.1", - "wide-align": "^1.1.0" - }, - "dependencies": { - "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", - "dev": true - }, - "is-fullwidth-code-point": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", - "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", - "dev": true, - "requires": { - "number-is-nan": "^1.0.0" - } - }, - "string-width": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", - "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", - "dev": true, - "requires": { - "code-point-at": "^1.0.0", - "is-fullwidth-code-point": "^1.0.0", - "strip-ansi": "^3.0.0" - } - }, - "strip-ansi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "dev": true, - "requires": { - "ansi-regex": "^2.0.0" - } - } - } - }, - "gaze": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/gaze/-/gaze-1.1.3.tgz", - "integrity": "sha512-BRdNm8hbWzFzWHERTrejLqwHDfS4GibPoq5wjTPIoJHoBtKGPg3xAFfxmM+9ztbXelxcf2hwQcaz1PtmFeue8g==", - "dev": true, - "requires": { - "globule": "^1.0.0" - } - }, - "geojson-vt": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/geojson-vt/-/geojson-vt-3.2.1.tgz", - "integrity": "sha512-EvGQQi/zPrDA6zr6BnJD/YhwAkBP8nnJ9emh3EnHQKVMfg/MRVtPbMYdgVy/IaEmn4UfagD2a6fafPDL5hbtwg==" - }, - "get-caller-file": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz", - "integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==", - "dev": true - }, - "get-intrinsic": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz", - "integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==", - "dev": true, - "requires": { - "function-bind": "^1.1.1", - "has": "^1.0.3", - "has-symbols": "^1.0.3" - } - }, - "get-stdin": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz", - "integrity": "sha1-uWjGsKBDhDJJAui/Gl3zJXmkUP4=", - "dev": true - }, - "get-stream": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", - "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", - "dev": true, - "requires": { - "pump": "^3.0.0" - } - }, - "get-symbol-description": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz", - "integrity": "sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==", - "dev": true, - "requires": { - "call-bind": "^1.0.2", - "get-intrinsic": "^1.1.1" - } - }, - "get-value": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz", - "integrity": "sha1-3BXKHGcjh8p2vTesCjlbogQqLCg=", - "dev": true - }, - "getpass": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", - "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", - "dev": true, - "requires": { - "assert-plus": "^1.0.0" - } - }, - "gl-matrix": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/gl-matrix/-/gl-matrix-3.3.0.tgz", - "integrity": "sha512-COb7LDz+SXaHtl/h4LeaFcNdJdAQSDeVqjiIihSXNrkWObZLhDI4hIkZC11Aeqp7bcE72clzB0BnDXr2SmslRA==" - }, - "glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "dev": true, - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "dependencies": { - "minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "requires": { - "brace-expansion": "^1.1.7" - } - } - } - }, - "glob-base": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/glob-base/-/glob-base-0.3.0.tgz", - "integrity": "sha1-27Fk9iIbHAscz4Kuoyi0l98Oo8Q=", - "dev": true, - "requires": { - "glob-parent": "^2.0.0", - "is-glob": "^2.0.0" - }, - "dependencies": { - "glob-parent": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-2.0.0.tgz", - "integrity": "sha1-gTg9ctsFT8zPUzbaqQLxgvbtuyg=", - "dev": true, - "requires": { - "is-glob": "^2.0.0" - } - }, - "is-extglob": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-1.0.0.tgz", - "integrity": "sha1-rEaBd8SUNAWgkvyPKXYMb/xiBsA=", - "dev": true - }, - "is-glob": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-2.0.1.tgz", - "integrity": "sha1-0Jb5JqPe1WAPP9/ZEZjLCIjC2GM=", - "dev": true, - "requires": { - "is-extglob": "^1.0.0" - } - } - } - }, - "glob-parent": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-3.1.0.tgz", - "integrity": "sha1-nmr2KZ2NO9K9QEMIMr0RPfkGxa4=", - "dev": true, - "requires": { - "is-glob": "^3.1.0", - "path-dirname": "^1.0.0" - }, - "dependencies": { - "is-glob": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-3.1.0.tgz", - "integrity": "sha1-e6WuJCF4BKxwcHuWkiVnSGzD6Eo=", - "dev": true, - "requires": { - "is-extglob": "^2.1.0" + "is-extglob": "^2.1.0" } } } @@ -8355,33 +7671,6 @@ } } }, - "globule": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/globule/-/globule-1.2.1.tgz", - "integrity": "sha512-g7QtgWF4uYSL5/dn71WxubOrS7JVGCnFPEnoeChJmBnyR9Mw8nGoEwOgJL/RC2Te0WhbsEUCejfH8SZNJ+adYQ==", - "dev": true, - "requires": { - "glob": "~7.1.1", - "lodash": "~4.17.10", - "minimatch": "~3.0.2" - }, - "dependencies": { - "glob": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz", - "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==", - "dev": true, - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - } - } - }, "gopd": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", @@ -8550,12 +7839,6 @@ "has-symbols": "^1.0.2" } }, - "has-unicode": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", - "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=", - "dev": true - }, "has-value": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz", @@ -9018,6 +8301,12 @@ "integrity": "sha512-Pgs951kaMm5GXP7MOvxERINe3gsaVjUWFm+UZPSq9xYriQAksyhg0csnS0KXSNRD5NmNdapXEpjxG49+AKh/ug==", "dev": true }, + "immutable": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.4.tgz", + "integrity": "sha512-fsXeu4J4i6WNWSikpI88v/PcVflZz+6kMhUfIwc5SY+poQRPnaf5V7qds6SUyUN3cVxEzuCab7QIoLOQ+DQ1wA==", + "dev": true + }, "import-cwd": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/import-cwd/-/import-cwd-2.1.0.tgz", @@ -9098,21 +8387,6 @@ "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", "dev": true }, - "in-publish": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/in-publish/-/in-publish-2.0.0.tgz", - "integrity": "sha1-4g/146KvwmkDILbcVSaCqcf631E=", - "dev": true - }, - "indent-string": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-2.1.0.tgz", - "integrity": "sha1-ji1INIdCEhtKghi3oTfppSBJ3IA=", - "dev": true, - "requires": { - "repeating": "^2.0.0" - } - }, "indexes-of": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/indexes-of/-/indexes-of-1.0.1.tgz", @@ -10883,12 +10157,6 @@ "integrity": "sha512-v28EW9DWDFpzcD9O5iyJXg3R3+q+mET5JhnjJzQUZMHOv67bpSIHq81GEYpPNZHG+XXHsfSme3nxp/hndKEcsQ==", "dev": true }, - "js-base64": { - "version": "2.6.4", - "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-2.6.4.tgz", - "integrity": "sha512-pZe//GGmwJndub7ZghVHz7vjb2LgC1m8B07Au3eYqeqv9emhESByMXxaEgkUkEqJe87oBbSniGYoQNIBklc7IQ==", - "dev": true - }, "js-beautify": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.10.0.tgz", @@ -11348,16 +10616,6 @@ "js-tokens": "^3.0.0 || ^4.0.0" } }, - "loud-rejection": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/loud-rejection/-/loud-rejection-1.6.0.tgz", - "integrity": "sha1-W0b4AUft7leIcPCG0Eghz5mOVR8=", - "dev": true, - "requires": { - "currently-unhandled": "^0.4.1", - "signal-exit": "^3.0.0" - } - }, "lower-case": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-1.1.4.tgz", @@ -11536,24 +10794,6 @@ "readable-stream": "^2.0.1" } }, - "meow": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/meow/-/meow-3.7.0.tgz", - "integrity": "sha1-cstmi0JSKCkKu/qFaJJYcwioAfs=", - "dev": true, - "requires": { - "camelcase-keys": "^2.0.0", - "decamelize": "^1.1.2", - "loud-rejection": "^1.0.0", - "map-obj": "^1.0.1", - "minimist": "^1.1.3", - "normalize-package-data": "^2.3.4", - "object-assign": "^4.0.1", - "read-pkg-up": "^1.0.1", - "redent": "^1.0.0", - "trim-newlines": "^1.0.0" - } - }, "merge": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/merge/-/merge-1.2.1.tgz", @@ -11899,7 +11139,8 @@ "version": "2.14.0", "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz", "integrity": "sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg==", - "dev": true + "dev": true, + "optional": true }, "nanoid": { "version": "2.1.11", @@ -12017,43 +11258,6 @@ "integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==", "dev": true }, - "node-gyp": { - "version": "3.8.0", - "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-3.8.0.tgz", - "integrity": "sha512-3g8lYefrRRzvGeSowdJKAKyks8oUpLEd/DyPV4eMhVlhJ0aNaZqIrNUIPuEWWTAoPqyFkfGrM67MC69baqn6vA==", - "dev": true, - "requires": { - "fstream": "^1.0.0", - "glob": "^7.0.3", - "graceful-fs": "^4.1.2", - "mkdirp": "^0.5.0", - "nopt": "2 || 3", - "npmlog": "0 || 1 || 2 || 3 || 4", - "osenv": "0", - "request": "^2.87.0", - "rimraf": "2", - "semver": "~5.3.0", - "tar": "^2.0.0", - "which": "1" - }, - "dependencies": { - "nopt": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz", - "integrity": "sha1-xkZdvwirzU2zWTF/eaxopkayj/k=", - "dev": true, - "requires": { - "abbrev": "1" - } - }, - "semver": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz", - "integrity": "sha1-myzl094C0XxgEq0yaqa00M9U+U8=", - "dev": true - } - } - }, "node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -12077,151 +11281,58 @@ "integrity": "sha512-5MQunG/oyOaBdttrL40dA7bUfPORLRWMUJLQtMg7nluxUvk5XwnLdL9twQHFAjRx/y7mIMkLKT9++qPbbk6BZA==", "dev": true, "requires": { - "assert": "^1.1.1", - "browserify-zlib": "^0.2.0", - "buffer": "^4.3.0", - "console-browserify": "^1.1.0", - "constants-browserify": "^1.0.0", - "crypto-browserify": "^3.11.0", - "domain-browser": "^1.1.1", - "events": "^3.0.0", - "https-browserify": "^1.0.0", - "os-browserify": "^0.3.0", - "path-browserify": "0.0.0", - "process": "^0.11.10", - "punycode": "^1.2.4", - "querystring-es3": "^0.2.0", - "readable-stream": "^2.3.3", - "stream-browserify": "^2.0.1", - "stream-http": "^2.7.2", - "string_decoder": "^1.0.0", - "timers-browserify": "^2.0.4", - "tty-browserify": "0.0.0", - "url": "^0.11.0", - "util": "^0.11.0", - "vm-browserify": "0.0.4" - }, - "dependencies": { - "punycode": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=", - "dev": true - } - } - }, - "node-notifier": { - "version": "5.4.5", - "resolved": "https://registry.npmjs.org/node-notifier/-/node-notifier-5.4.5.tgz", - "integrity": "sha512-tVbHs7DyTLtzOiN78izLA85zRqB9NvEXkAf014Vx3jtSvn/xBl6bR8ZYifj+dFcFrKI21huSQgJZ6ZtL3B4HfQ==", - "dev": true, - "requires": { - "growly": "^1.3.0", - "is-wsl": "^1.1.0", - "semver": "^5.5.0", - "shellwords": "^0.1.1", - "which": "^1.3.0" - } - }, - "node-releases": { - "version": "2.0.10", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.10.tgz", - "integrity": "sha512-5GFldHPXVG/YZmFzJvKK2zDSzPKhEp0+ZR5SVaoSag9fsL5YgHbUHDfnG5494ISANDcK4KwPXAx2xqVEydmd7w==", - "dev": true - }, - "node-sass": { - "version": "4.14.1", - "resolved": "https://registry.npmjs.org/node-sass/-/node-sass-4.14.1.tgz", - "integrity": "sha512-sjCuOlvGyCJS40R8BscF5vhVlQjNN069NtQ1gSxyK1u9iqvn6tf7O1R4GNowVZfiZUCRt5MmMs1xd+4V/7Yr0g==", - "dev": true, - "requires": { - "async-foreach": "^0.1.3", - "chalk": "^1.1.1", - "cross-spawn": "^3.0.0", - "gaze": "^1.0.0", - "get-stdin": "^4.0.1", - "glob": "^7.0.3", - "in-publish": "^2.0.0", - "lodash": "^4.17.15", - "meow": "^3.7.0", - "mkdirp": "^0.5.1", - "nan": "^2.13.2", - "node-gyp": "^3.8.0", - "npmlog": "^4.0.0", - "request": "^2.88.0", - "sass-graph": "2.2.5", - "stdout-stream": "^1.4.0", - "true-case-path": "^1.0.2" - }, - "dependencies": { - "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", - "dev": true - }, - "ansi-styles": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", - "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=", - "dev": true - }, - "chalk": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", - "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", - "dev": true, - "requires": { - "ansi-styles": "^2.2.1", - "escape-string-regexp": "^1.0.2", - "has-ansi": "^2.0.0", - "strip-ansi": "^3.0.0", - "supports-color": "^2.0.0" - } - }, - "cross-spawn": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-3.0.1.tgz", - "integrity": "sha1-ElYDfsufDF9549bvE14wdwGEuYI=", - "dev": true, - "requires": { - "lru-cache": "^4.0.1", - "which": "^1.2.9" - } - }, - "lru-cache": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", - "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", - "dev": true, - "requires": { - "pseudomap": "^1.0.2", - "yallist": "^2.1.2" - } - }, - "strip-ansi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "dev": true, - "requires": { - "ansi-regex": "^2.0.0" - } - }, - "supports-color": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", - "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=", - "dev": true - }, - "yallist": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", - "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=", + "assert": "^1.1.1", + "browserify-zlib": "^0.2.0", + "buffer": "^4.3.0", + "console-browserify": "^1.1.0", + "constants-browserify": "^1.0.0", + "crypto-browserify": "^3.11.0", + "domain-browser": "^1.1.1", + "events": "^3.0.0", + "https-browserify": "^1.0.0", + "os-browserify": "^0.3.0", + "path-browserify": "0.0.0", + "process": "^0.11.10", + "punycode": "^1.2.4", + "querystring-es3": "^0.2.0", + "readable-stream": "^2.3.3", + "stream-browserify": "^2.0.1", + "stream-http": "^2.7.2", + "string_decoder": "^1.0.0", + "timers-browserify": "^2.0.4", + "tty-browserify": "0.0.0", + "url": "^0.11.0", + "util": "^0.11.0", + "vm-browserify": "0.0.4" + }, + "dependencies": { + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=", "dev": true } } }, + "node-notifier": { + "version": "5.4.5", + "resolved": "https://registry.npmjs.org/node-notifier/-/node-notifier-5.4.5.tgz", + "integrity": "sha512-tVbHs7DyTLtzOiN78izLA85zRqB9NvEXkAf014Vx3jtSvn/xBl6bR8ZYifj+dFcFrKI21huSQgJZ6ZtL3B4HfQ==", + "dev": true, + "requires": { + "growly": "^1.3.0", + "is-wsl": "^1.1.0", + "semver": "^5.5.0", + "shellwords": "^0.1.1", + "which": "^1.3.0" + } + }, + "node-releases": { + "version": "2.0.10", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.10.tgz", + "integrity": "sha512-5GFldHPXVG/YZmFzJvKK2zDSzPKhEp0+ZR5SVaoSag9fsL5YgHbUHDfnG5494ISANDcK4KwPXAx2xqVEydmd7w==", + "dev": true + }, "nopt": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.1.tgz", @@ -12275,18 +11386,6 @@ "path-key": "^2.0.0" } }, - "npmlog": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", - "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", - "dev": true, - "requires": { - "are-we-there-yet": "~1.1.2", - "console-control-strings": "~1.1.0", - "gauge": "~2.7.3", - "set-blocking": "~2.0.0" - } - }, "nth-check": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", @@ -12898,8 +11997,7 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, - "optional": true + "dev": true }, "pify": { "version": "4.0.1", @@ -14001,27 +13099,6 @@ "util.promisify": "^1.0.0" } }, - "redent": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/redent/-/redent-1.0.0.tgz", - "integrity": "sha1-z5Fqsf1fHxbfsggi3W7H9zDCr94=", - "dev": true, - "requires": { - "indent-string": "^2.1.0", - "strip-indent": "^1.0.1" - }, - "dependencies": { - "strip-indent": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-1.0.1.tgz", - "integrity": "sha1-DHlipq3vp7vUrDZkYKY4VSrhoKI=", - "dev": true, - "requires": { - "get-stdin": "^4.0.1" - } - } - } - }, "regenerate": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.2.tgz", @@ -14549,134 +13626,122 @@ "minimist": "^1.1.1", "walker": "~1.0.5", "watch": "~0.18.0" + }, + "dependencies": { + "fsevents": { + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz", + "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==", + "dev": true, + "optional": true, + "requires": { + "bindings": "^1.5.0", + "nan": "^2.12.1" + } + } } }, - "sass-graph": { - "version": "2.2.5", - "resolved": "https://registry.npmjs.org/sass-graph/-/sass-graph-2.2.5.tgz", - "integrity": "sha512-VFWDAHOe6mRuT4mZRd4eKE+d8Uedrk6Xnh7Sh9b4NGufQLQjOrvf/MQoOdx+0s92L89FeyUUNfU597j/3uNpag==", + "sass": { + "version": "1.69.5", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.69.5.tgz", + "integrity": "sha512-qg2+UCJibLr2LCVOt3OlPhr/dqVHWOa9XtZf2OjbLs/T4VPSJ00udtgJxH3neXZm+QqX8B+3cU7RaLqp1iVfcQ==", "dev": true, "requires": { - "glob": "^7.0.0", - "lodash": "^4.0.0", - "scss-tokenizer": "^0.2.3", - "yargs": "^13.3.2" + "chokidar": ">=3.0.0 <4.0.0", + "immutable": "^4.0.0", + "source-map-js": ">=0.6.2 <2.0.0" }, "dependencies": { - "cliui": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-5.0.0.tgz", - "integrity": "sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA==", - "dev": true, - "requires": { - "string-width": "^3.1.0", - "strip-ansi": "^5.2.0", - "wrap-ansi": "^5.1.0" - } - }, - "find-up": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", - "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", "dev": true, "requires": { - "locate-path": "^3.0.0" + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" } }, - "get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "binary-extensions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", "dev": true }, - "locate-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", - "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", "dev": true, "requires": { - "p-locate": "^3.0.0", - "path-exists": "^3.0.0" + "fill-range": "^7.0.1" } }, - "p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "chokidar": { + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", + "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", "dev": true, "requires": { - "p-try": "^2.0.0" + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "fsevents": "~2.3.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" } }, - "p-locate": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", - "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", "dev": true, "requires": { - "p-limit": "^2.0.0" + "to-regex-range": "^5.0.1" } }, - "p-try": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", - "dev": true - }, - "require-main-filename": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", - "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", - "dev": true - }, - "string-width": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", - "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", "dev": true, "requires": { - "emoji-regex": "^7.0.1", - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^5.1.0" + "is-glob": "^4.0.1" } }, - "wrap-ansi": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz", - "integrity": "sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q==", + "is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", "dev": true, "requires": { - "ansi-styles": "^3.2.0", - "string-width": "^3.0.0", - "strip-ansi": "^5.0.0" + "binary-extensions": "^2.0.0" } }, - "yargs": { - "version": "13.3.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-13.3.2.tgz", - "integrity": "sha512-AX3Zw5iPruN5ie6xGRIDgqkT+ZhnRlZMLMHAs8tg7nRruy2Nb+i5o9bwghAogtM08q1dpr2LVoS8KSTMYpWXUw==", + "is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true + }, + "readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", "dev": true, "requires": { - "cliui": "^5.0.0", - "find-up": "^3.0.0", - "get-caller-file": "^2.0.1", - "require-directory": "^2.1.1", - "require-main-filename": "^2.0.0", - "set-blocking": "^2.0.0", - "string-width": "^3.0.0", - "which-module": "^2.0.0", - "y18n": "^4.0.0", - "yargs-parser": "^13.1.2" + "picomatch": "^2.2.1" } }, - "yargs-parser": { - "version": "13.1.2", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.2.tgz", - "integrity": "sha512-3lbsNRf/j+A4QuSZfDRA7HRSfWrzO0YjqTJd5kjAq37Zep1CEgaYmrH9Q3GwPiB9cHyd1Y1UwggGhJGoxipbzg==", + "to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", "dev": true, "requires": { - "camelcase": "^5.0.0", - "decamelize": "^1.2.0" + "is-number": "^7.0.0" } } } @@ -14719,16 +13784,6 @@ "ajv-keywords": "^3.1.0" } }, - "scss-tokenizer": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/scss-tokenizer/-/scss-tokenizer-0.2.3.tgz", - "integrity": "sha1-jrBtualyMzOCTT9VMGQRSYR85dE=", - "dev": true, - "requires": { - "js-base64": "^2.1.8", - "source-map": "^0.4.2" - } - }, "select-hose": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz", @@ -15237,14 +14292,11 @@ "integrity": "sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw==", "dev": true }, - "source-map": { - "version": "0.4.4", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz", - "integrity": "sha1-66T12pwNyZneaAMti092FzZSA2s=", - "dev": true, - "requires": { - "amdefine": ">=0.0.4" - } + "source-map-js": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", + "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==", + "dev": true }, "source-map-resolve": { "version": "0.5.2", @@ -15474,15 +14526,6 @@ "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=", "dev": true }, - "stdout-stream": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/stdout-stream/-/stdout-stream-1.4.1.tgz", - "integrity": "sha512-j4emi03KXqJWcIeF8eIXkjMFN1Cmb8gUlDYGeBALLPo5qdyTfA9bOtl8m33lRoC+vFMkP3gl0WsDr6+gzxbbTA==", - "dev": true, - "requires": { - "readable-stream": "^2.0.1" - } - }, "stealthy-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", @@ -15831,17 +14874,6 @@ "integrity": "sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==", "dev": true }, - "tar": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/tar/-/tar-2.2.2.tgz", - "integrity": "sha512-FCEhQ/4rE1zYv9rYXJw/msRqsnmlje5jHP6huWeBZ704jUTy02c5AZyWujpMR1ax6mVw9NyJMfuK2CMDWVIfgA==", - "dev": true, - "requires": { - "block-stream": "*", - "fstream": "^1.0.12", - "inherits": "2" - } - }, "terser": { "version": "4.8.1", "resolved": "https://registry.npmjs.org/terser/-/terser-4.8.1.tgz", @@ -16243,27 +15275,12 @@ "punycode": "^2.1.0" } }, - "trim-newlines": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz", - "integrity": "sha1-WIeWa7WCpFA6QetST301ARgVphM=", - "dev": true - }, "trim-right": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/trim-right/-/trim-right-1.0.1.tgz", "integrity": "sha1-yy4SAwZ+DI3h9hQJS5/kVwTqYAM=", "dev": true }, - "true-case-path": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/true-case-path/-/true-case-path-1.0.3.tgz", - "integrity": "sha512-m6s2OdQe5wgpFMC+pAJ+q9djG82O2jcHPOI6RNg1yy9rCYR+WD6Nbpl32fDpfC56nirdRy+opFa/Vk7HYhqaew==", - "dev": true, - "requires": { - "glob": "^7.1.2" - } - }, "tryer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/tryer/-/tryer-1.0.1.tgz", @@ -16967,13 +15984,6 @@ "to-regex-range": "^5.0.1" } }, - "fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, - "optional": true - }, "glob-parent": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", @@ -17563,15 +16573,6 @@ "is-typed-array": "^1.1.10" } }, - "wide-align": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", - "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", - "dev": true, - "requires": { - "string-width": "^1.0.2 || 2" - } - }, "wkt-parser": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/wkt-parser/-/wkt-parser-1.2.3.tgz", diff --git a/app/frontend/package.json b/app/frontend/package.json index 48f16853d7..608efd51d5 100644 --- a/app/frontend/package.json +++ b/app/frontend/package.json @@ -54,7 +54,7 @@ "jest-transform-stub": "^2.0.0", "jquery": "^3.4.1", "moxios": "^0.4.0", - "node-sass": "^4.9.0", + "sass": "^1.32.0", "sass-loader": "^7.1.0", "vue-template-compiler": "^2.5.21" }, diff --git a/app/frontend/src/common/components/Header.vue b/app/frontend/src/common/components/Header.vue index 79a7872135..764174ff98 100644 --- a/app/frontend/src/common/components/Header.vue +++ b/app/frontend/src/common/components/Header.vue @@ -44,6 +44,7 @@ Registry Search Submit Report Bulk Upload + QA/QC Dashboard Admin Groundwater Information @@ -89,6 +90,7 @@ export default { admin: adminMeta ? adminMeta.content === 'true' : false, aquifers: this.hasConfig && this.config.enable_aquifers_search === true, surveys: this.hasConfig && this.userRoles.surveys.edit === true, + qaqc: this.hasConfig && this.userRoles.submissions.edit === true, bulk } } diff --git a/app/frontend/src/qaqc/components/QaQcExports.vue b/app/frontend/src/qaqc/components/QaQcExports.vue new file mode 100644 index 0000000000..c3b5e968bf --- /dev/null +++ b/app/frontend/src/qaqc/components/QaQcExports.vue @@ -0,0 +1,146 @@ +/* + Licensed under the Apache License, Version 2.0 (the "License") + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + + + + \ No newline at end of file diff --git a/app/frontend/src/qaqc/components/QaQcFilters.vue b/app/frontend/src/qaqc/components/QaQcFilters.vue new file mode 100644 index 0000000000..d265325b84 --- /dev/null +++ b/app/frontend/src/qaqc/components/QaQcFilters.vue @@ -0,0 +1,210 @@ +/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + + + + + diff --git a/app/frontend/src/qaqc/components/QaQcTable.vue b/app/frontend/src/qaqc/components/QaQcTable.vue new file mode 100644 index 0000000000..1122fd985e --- /dev/null +++ b/app/frontend/src/qaqc/components/QaQcTable.vue @@ -0,0 +1,433 @@ +/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + + + + + diff --git a/app/frontend/src/qaqc/store/actions.types.js b/app/frontend/src/qaqc/store/actions.types.js new file mode 100644 index 0000000000..a670ce3a5f --- /dev/null +++ b/app/frontend/src/qaqc/store/actions.types.js @@ -0,0 +1,16 @@ +/** + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +export const QAQC_SEARCH = 'QAQC_SEARCH' +export const RESET_QAQC_SEARCH = 'RESET_QAQC_SEARCH' +export const RESET_QAQC_DATA = 'RESET_QAQC_DATA' +export const FETCH_QAQC_WELL_DOWNLOAD_LINKS = 'FETCH_QAQC_WELL_DOWNLOAD_LINKS' +export const SET_QAQC_SELECTED_TAB_ACTION = 'SET_QAQC_SELECTED_TAB_ACTION' diff --git a/app/frontend/src/qaqc/store/index.js b/app/frontend/src/qaqc/store/index.js new file mode 100644 index 0000000000..e219e3fa26 --- /dev/null +++ b/app/frontend/src/qaqc/store/index.js @@ -0,0 +1,323 @@ +/** + Licensed under the Apache License, Version 2.0 (the "License") + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +import Vue from 'vue' +import Vuex from 'vuex' +import axios from 'axios' +import ApiService from '@/common/services/ApiService.js' + +import { + QAQC_SEARCH, + RESET_QAQC_SEARCH, + FETCH_QAQC_WELL_DOWNLOAD_LINKS, + SET_QAQC_SELECTED_TAB_ACTION +} from './actions.types.js' +import { + SET_QAQC_ERROR, + SET_QAQC_LAST_SEARCH_TRIGGER, + SET_QAQC_PENDING_SEARCH, + SET_QAQC_HAS_SEARCHED, + SET_QAQC_SEARCH_BOUNDS, + SET_QAQC_LIMIT, + SET_QAQC_OFFSET, + SET_QAQC_ORDERING, + SET_QAQC_ERRORS, + SET_QAQC_PARAMS, + SET_QAQC_RESULT_COLUMNS, + SET_QAQC_RESULT_COUNT, + SET_QAQC_RESULT_FILTERS, + SET_QAQC_RESULTS, + SET_QAQC_SELECTED_TAB +} from './mutations.types.js' + +Vue.use(Vuex) + +const cleanParams = (payload) => { + // Clear any null or empty string values, to keep URLs clean. + return Object.entries(payload).filter(([key, value]) => { + return !(value === undefined || value === '' || value === null) + }).reduce((cleanedParams, [key, value]) => { + cleanedParams[key] = value + return cleanedParams + }, {}) +} + +function buildSearchParams (state) { + const params = { ...state.qaqcParams } + + if (Object.entries(state.qaqcResultFilters).length > 0) { + params['filter_group'] = JSON.stringify(state.qaqcResultFilters) + } + + return params +} + +export const RECORD_COMPLIANCE_COLUMNS = [ + 'wellTagNumber', + 'identificationPlateNumber', + 'wellClass', + 'latitude', + 'longitude', + 'finishedWellDepth', + 'diameter', + 'surfaceSealDepth', + 'surfaceSealThickness', + 'aquiferLithology', + 'wellStatus', + 'dateOfWork', + 'personResponsible', + 'orgResponsible', + 'createDate', + 'createUser', + 'naturalResourceRegion', + 'internalComments' +] + +export const MISLOCATED_WELLS_COLUMNS = [ + 'wellTagNumber', + 'geocodeDistance', + 'distanceToPid', + 'scoreAddress', + 'scoreCity', + 'wellStatus', + 'dateOfWork', + 'orgResponsible', + 'naturalResourceRegion', + 'createDate', + 'createUser', + 'internalComments' +] + +export const CROSS_REFERENCING_COLUMNS = [ + 'wellTagNumber', + 'wellStatus', + 'dateOfWork', + 'createUser', + 'createDate', + 'updateUser', + 'updateDate', + 'naturalResourceRegion', + 'internalComments' +] + +const DEFAULT_ORDERING = '-well_tag_number' +const DEFAULT_LIMIT = 10 + +const wellsStore = { + state: { + error: null, + lastSearchTrigger: null, + qaqcPendingSearch: null, + qaqcHasSearched: false, + qaqcBounds: {}, + qaqcErrors: {}, + qaqcLimit: DEFAULT_LIMIT, + qaqcOffset: 0, + qaqcOrdering: DEFAULT_ORDERING, + qaqcParams: {}, + qaqcResultColumns: CROSS_REFERENCING_COLUMNS, + qaqcResultFilters: {}, + qaqcResults: null, + qaqcResultCount: 0, + downloads: null, + selectedTab: 0 + }, + mutations: { + [SET_QAQC_SELECTED_TAB] (state, payload) { + state.selectedTab = payload + }, + [SET_QAQC_ERROR] (state, payload) { + state.error = payload + }, + [SET_QAQC_LAST_SEARCH_TRIGGER] (state, payload) { + state.lastSearchTrigger = payload + }, + [SET_QAQC_PENDING_SEARCH] (state, payload) { + state.qaqcPendingSearch = payload + }, + [SET_QAQC_HAS_SEARCHED] (state, payload) { + state.qaqcHasSearched = payload + }, + [SET_QAQC_ERRORS] (state, payload) { + state.qaqcErrors = payload + }, + [SET_QAQC_LIMIT] (state, payload) { + if (!(payload === 10 || payload === 25 || payload === 50)) { + return + } + state.qaqcLimit = payload + }, + [SET_QAQC_OFFSET] (state, payload) { + state.qaqcOffset = payload + }, + [SET_QAQC_ORDERING] (state, payload) { + state.qaqcOrdering = payload + }, + [SET_QAQC_PARAMS] (state, payload) { + state.qaqcParams = cleanParams(payload) + }, + [SET_QAQC_RESULT_COLUMNS] (state, payload) { + state.qaqcResultColumns = payload + }, + [SET_QAQC_RESULT_FILTERS] (state, payload) { + state.qaqcResultFilters = cleanParams(payload) + }, + [SET_QAQC_RESULTS] (state, payload) { + state.qaqcResults = payload + }, + [SET_QAQC_RESULT_COUNT] (state, payload) { + state.qaqcResultCount = payload + } + }, + actions: { + [SET_QAQC_SELECTED_TAB_ACTION] ({ commit, dispatch }, tab) { + commit('SET_QAQC_SELECTED_TAB', tab) + // Determine which columns to use based on the selected tab + let columns = [] + switch (tab) { + case 0: + columns = RECORD_COMPLIANCE_COLUMNS + break + case 1: + columns = MISLOCATED_WELLS_COLUMNS + break + case 2: + columns = CROSS_REFERENCING_COLUMNS + break + default: + columns = [] // Default case or you can set a default column set + } + // Commit a mutation to set the qaqc result columns + commit('SET_QAQC_RESULT_COLUMNS', columns) + commit('SET_QAQC_RESULT_COLUMNS', columns) + dispatch('QAQC_SEARCH', {}) + }, + [FETCH_QAQC_WELL_DOWNLOAD_LINKS] ({ commit, state }) { + if (state.downloads === null) { + ApiService.query('wells/extracts').then((response) => { + state.downloads = response.data + }) + } + }, + [RESET_QAQC_SEARCH] ({ commit, state }) { + if (state.qaqcPendingSearch !== null) { + state.qaqcPendingSearch.cancel() + } + commit(SET_QAQC_HAS_SEARCHED, false) + commit(SET_QAQC_PENDING_SEARCH, null) + commit(SET_QAQC_SEARCH_BOUNDS, {}) + commit(SET_QAQC_ORDERING, DEFAULT_ORDERING) + commit(SET_QAQC_LIMIT, DEFAULT_LIMIT) + commit(SET_QAQC_OFFSET, 0) + commit(SET_QAQC_PARAMS, {}) + commit(SET_QAQC_ERRORS, {}) + commit(SET_QAQC_RESULTS, null) + commit(SET_QAQC_RESULT_COUNT, 0) + commit(SET_QAQC_RESULT_FILTERS, {}) + }, + [QAQC_SEARCH] ({ commit, state }, { constrain = null, trigger = null }) { + commit(SET_QAQC_LAST_SEARCH_TRIGGER, trigger) + commit(SET_QAQC_HAS_SEARCHED, true) + + if (state.qaqcPendingSearch !== null) { + state.qaqcPendingSearch.cancel() + } + + const cancelSource = axios.CancelToken.source() + commit(SET_QAQC_PENDING_SEARCH, cancelSource) + + const params = { + ...buildSearchParams(state), + ordering: state.qaqcOrdering, + limit: state.qaqcLimit, + offset: state.qaqcOffset + } + + // Modify the endpoint or parameters based on the selectedTab + let endpoint = 'qaqc' + if (state.selectedTab === 0) { + endpoint += '/recordcompliance' + } else if (state.selectedTab === 1) { + endpoint += '/mislocatedwells' + } else if (state.selectedTab === 2) { + endpoint += '/crossreferencing' + } + + ApiService.query(endpoint, params, { cancelToken: cancelSource.token }).then((response) => { + commit(SET_QAQC_ERRORS, {}) + commit(SET_QAQC_RESULTS, response.data.results) + commit(SET_QAQC_RESULT_COUNT, response.data.count) + }).catch((err) => { + // If the qaqc was cancelled, a new one is pending, so don't bother resetting. + if (axios.isCancel(err)) { + return + } + + if (err.response && err.response.data) { + commit(SET_QAQC_ERRORS, err.response.data) + } + commit(SET_QAQC_RESULTS, null) + commit(SET_QAQC_RESULT_COUNT, 0) + }).finally(() => { + commit(SET_QAQC_PENDING_SEARCH, null) + }) + } + }, + getters: { + qaqcHasSearched (state) { + return state.qaqcHasSearched + }, + qaqcPendingSearch (state) { + return state.qaqcPendingSearch + }, + qaqcInProgress (state) { + return Boolean(state.qaqcPendingSearch) + }, + qaqcBounds (state) { + return state.qaqcBounds + }, + qaqcErrors (state) { + return state.qaqcErrors + }, + qaqcLimit (state) { + return state.qaqcLimit + }, + qaqcOffset (state) { + return state.qaqcOffset + }, + qaqcOrdering (state) { + return state.qaqcOrdering + }, + qaqcParams (state) { + return state.qaqcParams + }, + qaqcResultColumns (state) { + return state.qaqcResultColumns + }, + qaqcResultFilters (state) { + return state.qaqcResultFilters + }, + qaqcResultCount (state) { + return state.qaqcResultCount + }, + qaqcResults (state) { + return state.qaqcResults + }, + qaqcWellFileDownloads (state) { + return state.downloads + }, + searchQueryParams (state) { + return buildSearchParams(state) + } + } +} + +export default wellsStore diff --git a/app/frontend/src/qaqc/store/mutations.types.js b/app/frontend/src/qaqc/store/mutations.types.js new file mode 100644 index 0000000000..f1ce2255e7 --- /dev/null +++ b/app/frontend/src/qaqc/store/mutations.types.js @@ -0,0 +1,27 @@ +/** + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +export const SET_QAQC_ERROR = 'SET_QAQC_ERROR' +export const SET_QAQC_LAST_SEARCH_TRIGGER = 'SET_QAQC_LAST_SEARCH_TRIGGER' +export const SET_QAQC_PENDING_LOCATION_SEARCH = 'SET_QAQC_PENDING_LOCATION_SEARCH' +export const SET_QAQC_HAS_SEARCHED = 'SET_QAQC_HAS_SEARCHED' +export const SET_QAQC_PENDING_SEARCH = 'SET_QAQC_PENDING_SEARCH' +export const SET_QAQC_BOUNDS = 'SET_QAQC_BOUNDS' +export const SET_QAQC_ERRORS = 'SET_QAQC_ERRORS' +export const SET_QAQC_LIMIT = 'SET_QAQC_LIMIT' +export const SET_QAQC_OFFSET = 'SET_QAQC_OFFSET' +export const SET_QAQC_ORDERING = 'SET_QAQC_ORDERING' +export const SET_QAQC_PARAMS = 'SET_QAQC_PARAMS' +export const SET_QAQC_RESULT_COLUMNS = 'SET_QAQC_RESULT_COLUMNS' +export const SET_QAQC_RESULT_FILTERS = 'SET_QAQC_RESULT_FILTERS' +export const SET_QAQC_RESULTS = 'SET_QAQC_RESULTS' +export const SET_QAQC_RESULT_COUNT = 'SET_QAQC_RESULT_COUNT' +export const SET_QAQC_SELECTED_TAB = 'SET_QAQC_SELECTED_TAB' diff --git a/app/frontend/src/qaqc/store/triggers.types.js b/app/frontend/src/qaqc/store/triggers.types.js new file mode 100644 index 0000000000..1db736471c --- /dev/null +++ b/app/frontend/src/qaqc/store/triggers.types.js @@ -0,0 +1,21 @@ +/** + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +// the QUERY trigger means the search was triggered by a querystring in the URL +// e.g. the user bookmarked a search or shared a link. +export const QUERY_TRIGGER = 'QUERY_TRIGGER' + +// the search trigger means the basic or advanced search form was used to search for wells. +export const SEARCH_TRIGGER = 'SEARCH_TRIGGER' + +// the filter trigger means that the search was triggered via search result filters. +export const FILTER_TRIGGER = 'FILTER_TRIGGER' diff --git a/app/frontend/src/qaqc/views/QaQcDashboard.vue b/app/frontend/src/qaqc/views/QaQcDashboard.vue new file mode 100644 index 0000000000..2ecbe96806 --- /dev/null +++ b/app/frontend/src/qaqc/views/QaQcDashboard.vue @@ -0,0 +1,42 @@ + + + diff --git a/app/frontend/src/router.js b/app/frontend/src/router.js index 21313ee3b6..44aadc723c 100644 --- a/app/frontend/src/router.js +++ b/app/frontend/src/router.js @@ -40,6 +40,9 @@ import PageNotFound from '@/common/components/PageNotFound.vue' // Surveys import Surveys from '@/surveys/views/Surveys.vue' +// QaQc +import QaQcDashboard from '@/qaqc/views/QaQcDashboard.vue' + Vue.use(Router) const router = new Router({ @@ -229,6 +232,11 @@ const router = new Router({ app: 'surveys' } }, + { + path: '/qaqc', + name: 'qaqc', + component: QaQcDashboard + }, { path: '/search', redirect: '/' }, { path: '/', diff --git a/app/frontend/src/store/index.js b/app/frontend/src/store/index.js index a97a052e8e..0362b6c22b 100644 --- a/app/frontend/src/store/index.js +++ b/app/frontend/src/store/index.js @@ -7,6 +7,7 @@ import registriesStore from '@/registry/store/index.js' import submissionStore from '@/submissions/store/index.js' import aquiferStore from '@/aquifers/store/index.js' import wellsStore from '@/wells/store/index.js' +import qaqcStore from '@/qaqc/store/index.js' Vue.use(Vuex) @@ -18,6 +19,7 @@ export const store = new Vuex.Store({ registriesStore, submissionStore, aquiferStore, - wellsStore + wellsStore, + qaqcStore } }) diff --git a/app/frontend/src/wells/components/mixins/filters.js b/app/frontend/src/wells/components/mixins/filters.js index 5067bac6c5..a6c1c83265 100644 --- a/app/frontend/src/wells/components/mixins/filters.js +++ b/app/frontend/src/wells/components/mixins/filters.js @@ -874,7 +874,39 @@ const SEARCH_FIELDS = { textField: 'description', valueField: 'well_disinfected_code', sortable: true, - } + }, + latitude: { param: 'latitude', label: 'Latitude', type: 'number' }, + longitude: { param: 'longitude', label: 'Longitude', type: 'number' }, + geocodeDistance: { + param: 'geocode_distance', + label: 'Geocode Distance', + type: 'range', + sortable: true, + }, + distanceToPid: { + param: 'distance_to_pid', + label: 'Distance to Matching PID', + type: 'range', + sortable: true, + }, + scoreAddress: { + param: 'score_address', + label: 'Score Address', + type: 'range', + sortable: true, + }, + scoreCity: { + param: 'score_city', + label: 'Score City', + type: 'range', + sortable: true, + }, + naturalResourceRegion: { + param: 'natural_resource_region', + label: 'Natural Resource Region', + type: 'text', + sortable: true, + }, } export default { diff --git a/app/scripts/qaqc/README.md b/app/scripts/qaqc/README.md new file mode 100644 index 0000000000..605c025d71 --- /dev/null +++ b/app/scripts/qaqc/README.md @@ -0,0 +1,107 @@ +# GWELLS Location Data QA + +This suite of scripts is designed to generate Quality Assurance and Quality Control (QAQC) data for all wells registered in the GWELLS system. The primary objective is to build a comprehensive dataset that highlights potential data inconsistencies or issues within existing well records, thereby enabling internal staff to more effectively locate and address these issues. + +## Acknowledgements + +These scripts have been repurposed and adapted from an existing repository: [smnorris/gwells_locationqa](https://github.com/smnorris/gwells_locationqa). + +## Installation/Requirements + +1. Create and activate a Python virtual environment: + + ``` + python -m venv venv + source venv/bin/activate # On Windows, use `venv\Scripts\activate` + ``` + +2. Install the required Python packages: + + ``` + pip install -r requirements.txt + ``` + +## Download ESA WorldCover 2020 + +The ESA WorldCover dataset is a crucial component for the script but requires separate downloading. To download and prepare the tiff tiles for British Columbia (BC), ensure you have `awscli` and `gdal` installed and available at the command line. Then, execute the provided bash script: + + ```bash + ./get_esa_worldcover_bc.sh + ``` + +This script is designed for Unix-based systems (`bash`). It can be modified for Windows with minimal changes. + +For more information on the ESA WorldCover dataset, visit: + +- [ESA WorldCover Website](https://esa-worldcover.org/en) +- [ESA WorldCover Dataset Details](https://esa-worldcover.s3.amazonaws.com/readme.html) + +## Usage + +The scripts should be run in the following order: + +1. **Data Download**: + + Download the required data to the `/data` folder (including GWELLS and PMBC parcel fabric datasets): + + ```python + python gwells_locationqa.py download + ``` + +2. **Reverse Geocoding**: + + Perform reverse-geocoding for all wells. This process has an optional API key and takes approximately 6 hours: + + ```python + python gwells_locationqa.py geocode + ``` + +3. **Quality Assurance (QA)**: + + Match well PIDs to PMBC, match addresses, and overlay with agricultural data sources: + + ```python + python gwells_locationqa.py qa + ``` + +4. **Data Extraction**: + + Run the `extract_data.py` script to extract specific columns from the generated `gwells_locationqa.csv` file. This step focuses on key data points for further analysis in GWELLS: + + ```python + python extract_data.py + ``` + + This script isolates and extracts essential columns such as `well_tag_number`, `distance_geocode`, `distance_to_matching_pid`, `score_address`, `score_city`, and `xref_ind`, saving the refined data into a new file named `extracted_wells_data.csv`. This file is used in a migration to populate the database with this information. + +## Output + +The output file generated is `gwells_locationqa.csv`, which includes all wells with latitude and longitude data. + +The additional columns added to the output file for QAQC purposes are as follows: + +| Column | Description | +| ---------------------------- | ------------- | +| nr_district_name | Natural Resource District | +| nr_region_name | Natural Resource Region | +| fullAddress | Geocoder result | +| streetAddress | Geocoder results parsed to match GWELLS address | +| civicNumber | Geocoder result | +| civicNumberSuffix | Geocoder result | +| streetName | Geocoder result | +| streetType | Geocoder result | +| isStreetTypePrefix | Geocoder result | +| streetDirection | Geocoder result | +| isStreetDirectionPrefix | Geocoder result | +| streetQualifier | Geocoder result | +| localityName | Geocoder result | +| distance_geocode | Distance from well to result of geocode (value 99999 indicates no result) | +| distance_to_matching_pid | Distance from well to BC Parcel Fabric polygon with matching PID | +| score_address | Token Set Ratio score for matching well's address to reverse geocoded address | +| score_location_description | Token Set Ratio score for matching well's location description to reverse geocoded full address | +| score_city | Token Set Ratio score for matching well's city to reverse geocoded locality | +| xref_ind | Indicates if specific strings found in comments column | +| alr_ind | Indicates if the well is within the ALR as defined by relevant datasets | +| btm_label | BTM Present Land Use Label at well location | +| esa_landclass | ESA WorldCover land class at well location | + diff --git a/app/scripts/qaqc/extract_data.py b/app/scripts/qaqc/extract_data.py new file mode 100644 index 0000000000..326e747c30 --- /dev/null +++ b/app/scripts/qaqc/extract_data.py @@ -0,0 +1,26 @@ +import pandas as pd + +def extract_columns_from_csv(file_path, output_file_path): + """ + Extracts specific columns from a CSV file and saves them into another CSV file. + + Parameters: + file_path (str): The path to the input CSV file. + output_file_path (str): The path to the output CSV file. + """ + # Read the CSV file + df = pd.read_csv(file_path) + + # Extract the required columns + extracted_df = df[['well_tag_number', 'distance_geocode', 'distance_to_matching_pid', + 'score_address', 'score_city', 'xref_ind', 'nr_region_name']] + + # Save the extracted data to a new CSV file + extracted_df.to_csv(output_file_path, index=False) + + print(f"Extracted data saved to {output_file_path}") + + +file_path = 'gwells_locationqa.csv' +output_file_path = 'qaqc_well_data.csv' # Output file path +extract_columns_from_csv(file_path, output_file_path) diff --git a/app/scripts/qaqc/get_esa_worldcover_bc.sh b/app/scripts/qaqc/get_esa_worldcover_bc.sh new file mode 100755 index 0000000000..b869ae9945 --- /dev/null +++ b/app/scripts/qaqc/get_esa_worldcover_bc.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -euxo pipefail + +# create data folder if not present +mkdir -p data + +# Download BC tiles of ESA world landcover from S3 bucket +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N54W123_Map.tif data/ESA_WorldCover_10m_2020_v100_N54W123_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N57W120_Map.tif data/ESA_WorldCover_10m_2020_v100_N57W120_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N51W126_Map.tif data/ESA_WorldCover_10m_2020_v100_N51W126_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N57W138_Map.tif data/ESA_WorldCover_10m_2020_v100_N57W138_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N54W132_Map.tif data/ESA_WorldCover_10m_2020_v100_N54W132_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N48W117_Map.tif data/ESA_WorldCover_10m_2020_v100_N48W117_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N57W141_Map.tif data/ESA_WorldCover_10m_2020_v100_N57W141_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N48W120_Map.tif data/ESA_WorldCover_10m_2020_v100_N48W120_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N51W135_Map.tif data/ESA_WorldCover_10m_2020_v100_N51W135_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N51W120_Map.tif data/ESA_WorldCover_10m_2020_v100_N51W120_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N48W132_Map.tif data/ESA_WorldCover_10m_2020_v100_N48W132_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N54W129_Map.tif data/ESA_WorldCover_10m_2020_v100_N54W129_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N54W135_Map.tif data/ESA_WorldCover_10m_2020_v100_N54W135_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N57W135_Map.tif data/ESA_WorldCover_10m_2020_v100_N57W135_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N57W129_Map.tif data/ESA_WorldCover_10m_2020_v100_N57W129_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N51W129_Map.tif data/ESA_WorldCover_10m_2020_v100_N51W129_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N48W129_Map.tif data/ESA_WorldCover_10m_2020_v100_N48W129_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N54W120_Map.tif data/ESA_WorldCover_10m_2020_v100_N54W120_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N54W126_Map.tif data/ESA_WorldCover_10m_2020_v100_N54W126_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N51W123_Map.tif data/ESA_WorldCover_10m_2020_v100_N51W123_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N51W132_Map.tif data/ESA_WorldCover_10m_2020_v100_N51W132_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N57W126_Map.tif data/ESA_WorldCover_10m_2020_v100_N57W126_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N57W132_Map.tif data/ESA_WorldCover_10m_2020_v100_N57W132_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N51W117_Map.tif data/ESA_WorldCover_10m_2020_v100_N51W117_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N48W123_Map.tif data/ESA_WorldCover_10m_2020_v100_N48W123_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N57W123_Map.tif data/ESA_WorldCover_10m_2020_v100_N57W123_Map.tif --no-sign-request +aws s3 cp s3://esa-worldcover/v100/2020/map/ESA_WorldCover_10m_2020_v100_N48W126_Map.tif data/ESA_WorldCover_10m_2020_v100_N48W126_Map.tif --no-sign-request + +# merge tiles by building vrt +gdalbuildvrt data/esa_merged.vrt data/ESA_WorldCover_10m_2020_v100_*.tif + +# warp data to BC Albers, writing to compressed geotiff +gdalwarp -co COMPRESS=DEFLATE -co PREDICTOR=2 -co NUM_THREADS=ALL_CPUS -t_srs EPSG:3153 data/esa_merged.vrt esa_bc.tif diff --git a/app/scripts/qaqc/gwells_locationqa.py b/app/scripts/qaqc/gwells_locationqa.py new file mode 100644 index 0000000000..6f089cc7f2 --- /dev/null +++ b/app/scripts/qaqc/gwells_locationqa.py @@ -0,0 +1,642 @@ +import csv +import logging +import sys +from zipfile import ZipFile, BadZipFile +from io import BytesIO +from time import sleep +import os +from pathlib import Path +import tarfile +import tempfile +from urllib.parse import urlparse +import urllib.request + + +import fiona +import rasterio +import geopandas as gpd +import pandas as pd +import requests +import click +from thefuzz import fuzz +import bcdata + + +""" +BC Geocoder ADDRESS API + +Overview +https://www2.gov.bc.ca/gov/content/data/geographic-data-services/location-services/geocoder + +Specs +https://github.com/bcgov/api-specs/tree/master/geocoder + +Developer guide +https://github.com/bcgov/api-specs/blob/master/geocoder/geocoder-developer-guide.md + +More +https://www2.gov.bc.ca/assets/gov/data/geographic/location-services/geocoder/understanding_geocoder_results.pdf + +Status +https://stats.uptimerobot.com/KZ3Nvh29l1/787375578 +""" + +# source gwells data file +WELLS_URL = "https://s3.ca-central-1.amazonaws.com/gwells-export/export/v2/gwells.zip" + +# bc geocoder endpoint of interest +GEOCODER_ENDPOINT = "https://geocoder.api.gov.bc.ca/sites/nearest.json" +ADDRESS_COLUMNS = [ + "fullAddress", + "siteName", + "unitDesignator", + "unitNumber", + "unitNumberSuffix", + "civicNumber", + "civicNumberSuffix", + "streetName", + "streetType", + "isStreetTypePrefix", + "streetDirection", + "isStreetDirectionPrefix", + "streetQualifier", + "localityName", + "localityType", + "electoralArea", + "provinceCode", + "locationPositionalAccuracy", + "locationDescriptor", + "siteID", + "blockID", + "fullSiteDescriptor", + "accessNotes", + "siteStatus", + "siteRetireDate", + "changeDate", + "isOfficial", + "distance", +] + +PARCELFABRIC_URL = "https://pub.data.gov.bc.ca/datasets/4cf233c2-f020-4f7a-9b87-1923252fbc24/pmbc_parcel_fabric_poly_svw.zip" + +WELLS_FILENAME = "wells.csv" +PIDMATCH_FILENAME = "pidmatch.csv" +EPSG_3154 = "EPSG:3153" + + +LOG = logging.getLogger(__name__) + + +class ZipCompatibleTarFile(tarfile.TarFile): + """ + Wrapper around TarFile to make it more compatible with ZipFile + Modified from https://github.com/OpenBounds/Processing/blob/master/utils.py + """ + + def infolist(self): + members = self.getmembers() + for m in members: + m.filename = m.name + return members + + def namelist(self): + return self.getnames() + + +def get_compressed_file_wrapper(path): + """From https://github.com/OpenBounds/Processing/blob/master/utils.py""" + ARCHIVE_FORMAT_ZIP = "zip" + ARCHIVE_FORMAT_TAR = "tar" + ARCHIVE_FORMAT_TAR_GZ = "tar.gz" + ARCHIVE_FORMAT_TAR_BZ2 = "tar.bz2" + archive_format = None + if path.endswith(".zip"): + archive_format = ARCHIVE_FORMAT_ZIP + elif path.endswith(".tar.gz") or path.endswith(".tgz"): + archive_format = ARCHIVE_FORMAT_TAR_GZ + elif path.endswith(".tar.bz2"): + archive_format = ARCHIVE_FORMAT_TAR_BZ2 + else: + try: + with ZipFile(path, "r") as f: + archive_format = ARCHIVE_FORMAT_ZIP + except BadZipFile: + try: + with tarfile.TarFile.open(path, "r") as f: + archive_format = ARCHIVE_FORMAT_TAR + except tarfile.TarError: + pass + if archive_format is None: + raise Exception("Unable to determine archive format") + + if archive_format == ARCHIVE_FORMAT_ZIP: + return ZipFile(path, "r") + + elif archive_format == ARCHIVE_FORMAT_TAR_GZ: + return ZipCompatibleTarFile.open(path, "r:gz") + + elif archive_format == ARCHIVE_FORMAT_TAR_BZ2: + return ZipCompatibleTarFile.open(path, "r:bz2") + + +def download_file(url, out_path, filename): + """Download and extract a zipfile to unique location""" + out_file = os.path.join(out_path, filename) + if not os.path.exists(os.path.join(out_path, filename)): + LOG.info("Downloading " + url) + parsed_url = urlparse(url) + urlfile = parsed_url.path.split("/")[-1] + _, extension = os.path.split(urlfile) + fp = tempfile.NamedTemporaryFile("wb", suffix=extension, delete=False) + if parsed_url.scheme == "http" or parsed_url.scheme == "https": + res = requests.get(url, stream=True) + if not res.ok: + raise IOError + + for chunk in res.iter_content(1024): + fp.write(chunk) + elif parsed_url.scheme == "ftp": + dl = urllib.request.urlopen(url) + file_size_dl = 0 + block_sz = 8192 + while True: + buffer = dl.read(block_sz) + if not buffer: + break + file_size_dl += len(buffer) + fp.write(buffer) + fp.close() + # extract zipfile + Path(out_path).mkdir(parents=True, exist_ok=True) + LOG.info("Extracting %s to %s" % (fp.name, out_path)) + zipped_file = get_compressed_file_wrapper(fp.name) + zipped_file.extractall(out_path) + zipped_file.close() + layer = fiona.listlayers(os.path.join(out_path, filename))[0] + return (out_file, layer) + + +def get_gwells(outfile=os.path.join("data", WELLS_FILENAME)): + """ + - get wells csv if not already present + - retain only records of interest + - write to file + - return wells as dataframe + """ + if os.path.exists(outfile): + LOG.info(f"Loading {outfile}") + return pd.read_csv(outfile) + else: + r = requests.get(WELLS_URL) + zip_file = ZipFile(BytesIO(r.content)) + dfs = { + text_file.filename: pd.read_csv(zip_file.open(text_file.filename)) + for text_file in zip_file.infolist() + if text_file.filename.endswith(".csv") + } + # only retain records with coordinates + df = dfs["well.csv"].dropna(subset=["latitude_Decdeg", "longitude_Decdeg"]) + # only retain records that are unlicensed + # (we presume locations of licensed wells are correct) + # df = df[df["licenced_status_code"] == "UNLICENSED"] + + # save as intermediate file + df.to_csv(outfile, index=False) + return df + + +def reverse_geocode( + x, + y, + geocoder_api_key, + distance_start=200, + distance_increment=200, + distance_max=2000, +): + """ + Provided a location as x/y coordinates (EPSG:4326), request an address + from BC Geocoder within given distance_start (metres) + + If no result is found, request using an expanding search radius in + distance_increment steps, until distance_max is reached. + + A dict with 'distance' = 99999 is returned if no result is found. + + This could sped up by: + - just make the request using the max distance and derive distance between + source point and returned location + - make requests either in parallel or async + + """ + result = False + distance = distance_start + # expand the search distance until we get a result or hit the max distance + while result is False and distance <= distance_max: + params = { + "point": str(x) + "," + str(y), + "apikey": geocoder_api_key, + "outputFormat": "json", + "maxDistance": distance, + } + r = requests.get(GEOCODER_ENDPOINT, params=params) + LOG.debug(r.request.url) + + # pause for 2s per request if near limit of 1000 requests/min + if int(r.headers["RateLimit-Remaining"]) < 30: + LOG.info("Approaching API limit, sleeping for 2 seconds to refresh.") + sleep(2) + if r.status_code == 200: + result = True + else: + distance = distance + distance_increment + if r.status_code == 200: + address = r.json()["properties"] + address["distance"] = distance + return address + else: + empty_result = dict([(k, "") for k in ADDRESS_COLUMNS]) + empty_result["distance"] = 99999 + return empty_result + + +def pidmatch(wells_gdf): + if os.path.exists(os.path.join("data", PIDMATCH_FILENAME)): + LOG.info( + "Loading data/pidmatch.csv, cached result of wells/parcels PID matching" + ) + pid_distances = gpd.read_file(os.path.join("data", PIDMATCH_FILENAME)) + pid_distances["well_tag_number"] = pid_distances["well_tag_number"].astype(int) + + else: + # load parcel data + LOG.info("Loading data/pmbc_parcel_fabric_poly_svw.gdb") + parcels = gpd.read_file( + os.path.join("data", "pmbc_parcel_fabric_poly_svw.gdb"), + layer="pmbc_parcel_fabric_poly_svw", + ) + # ------ + # ** PID matching ** + # Derive 'distance_to_matching_pid', the distance from well to parcel + # polygon with the same PID + # ------ + # get well records with PID values + wells_with_pid = wells_gdf[wells_gdf["legal_pid"].notna()][ + ["well_tag_number", "legal_pid", "geometry"] + ] + # join to parcel fabric on PID + wells_parcels_pid = pd.merge( + wells_with_pid, + parcels, + how="inner", + left_on="legal_pid", + right_on="PID", + validate="many_to_one" + ) + # find distance from well point to parcel polygon with matching PID + distances_series = wells_parcels_pid["geometry_x"].distance( + wells_parcels_pid["geometry_y"], align=True + ) + # put distance values back into merged df but keep just columns of interest + pid_distances = wells_parcels_pid.assign( + distance_to_matching_pid=distances_series + )[["well_tag_number", "distance_to_matching_pid"]] + # dump to file + pid_distances.to_csv(os.path.join("data", PIDMATCH_FILENAME), index=False) + + return pd.merge(wells_gdf, pid_distances, on="well_tag_number", how="left") + + +def agriculture_overlays(in_gdf): + """Overlay wells with several definitions of agricultural lands or a proxy: + - BTM + - ALR + - ESA land cover + """ + if os.path.exists(os.path.join("data", "agriculture_overlays.csv")): + LOG.info( + "Loading data/agriculture_overlays.csv, cached result of ALR/BTM/ESA overlays" + ) + ag_overlays = gpd.read_file(os.path.join("data", "agriculture_overlays.csv")) + ag_overlays["well_tag_number"] = ag_overlays["well_tag_number"].astype(int) + + else: + LOG.info("Overlaying wells with ALR") + alr = ( + bcdata.get_data("WHSE_LEGAL_ADMIN_BOUNDARIES.OATS_ALR_POLYS", as_gdf=True)[ + ["STATUS", "geometry"] + ] + .to_crs(EPSG_3154) + .rename( + columns={ + "STATUS": "alr_ind", + } + ) + ) + alr_overlay = gpd.sjoin(in_gdf, alr, how="left", predicate="within") + alr_overlay = alr_overlay[["well_tag_number", "alr_ind"]] + + LOG.info("Overlaying wells with BTM") + btm = ( + bcdata.get_data( + "WHSE_BASEMAPPING.BTM_PRESENT_LAND_USE_V1_SVW", as_gdf=True + )[["PRESENT_LAND_USE_LABEL", "geometry"]] + .to_crs(EPSG_3154) + .rename( + columns={ + "PRESENT_LAND_USE_LABEL": "btm_label", + } + ) + ) + btm_overlay = gpd.sjoin(in_gdf, btm, how="left", predicate="within") + btm_overlay = btm_overlay[["well_tag_number", "btm_label"]] + + # overlay input with ESA land cover + LOG.info("Overlaying wells with ESA 10m landcover (2020)") + landclass_lookup = { + 0: "NULL", + 10: "Tree cover", + 20: "Shrubland", + 30: "Grassland", + 40: "Cropland", + 50: "Built-up", + 60: "Bare/sparse vegetation", + 70: "Snow and ice", + 80: "Permanent water bodies", + 90: "Herbaceous wetland", + 95: "Mangroves", + 100: "Moss and lichen", + } + esa = in_gdf.copy() + esa.index = range(len(in_gdf)) + coords = [(x, y) for x, y in zip(esa.geometry.x, in_gdf.geometry.y)] + raster = rasterio.open("data/esa_bc.tif") + esa["esa_landclass"] = [landclass_lookup[x[0]] for x in raster.sample(coords)] + esa = esa[["well_tag_number", "esa_landclass"]] + + # join all three dfs together + a = pd.merge(alr_overlay, btm_overlay, "left", on="well_tag_number") + b = pd.merge(a, esa, "left", on="well_tag_number") + + # make sure we only have columns of interest + ag_overlays = b[["well_tag_number", "alr_ind", "btm_label", "esa_landclass"]] + + # write to csv + ag_overlays.to_csv("data/agriculture_overlays.csv") + + return ag_overlays + + +def compare_strings(x): + return fuzz.token_set_ratio(x[0], x[1]) + + +@click.group() +def cli(): + """ + This is the main command group for the CLI. + + The function `cli` is decorated with `@click.group()`, which turns it into a command group. + This group will serve as the base for nesting other sub-commands. The `pass` statement is used + here because the function itself does not need to execute any code. Its primary purpose is to + serve as a foundation for the CLI structure. Sub-commands will be attached to this group, + and they will be the ones performing the actual operations or functionalities. + """ + pass + + +@cli.command() +def download(): + """Download required data to data folder""" + Path("data").mkdir(parents=True, exist_ok=True) + get_gwells(os.path.join("data", WELLS_FILENAME)) + download_file(PARCELFABRIC_URL, "data", "pmbc_parcel_fabric_poly_svw.gdb") + + +@cli.command() +@click.argument("geocoder_api_key", envvar="GEOCODER_API_KEY") +@click.option( + "--out_file", + "-o", + default=os.path.join("data", "wells_geocoded.csv"), + help="Name of output file.", +) +def geocode(geocoder_api_key, out_file): + """Reverse geocode well locations with BC Geocoder API""" + # only process if output file does not already exist + if not os.path.exists(out_file): + # get wells csv as pandas dataframe + df = get_gwells(os.path.join("data", WELLS_FILENAME)) + + # extract just id and coords + well_locations = df[ + ["well_tag_number", "longitude_Decdeg", "latitude_Decdeg"] + ].to_dict("records") + + LOG.info("Reverse geocoding well locations") + with open(out_file, "w", newline="") as csvfile: + writer = csv.DictWriter( + csvfile, fieldnames=ADDRESS_COLUMNS + ["well_tag_number"] + ) + writer.writeheader() + with click.progressbar(well_locations) as bar: + for row in bar: + r = reverse_geocode( + row["longitude_Decdeg"], + row["latitude_Decdeg"], + geocoder_api_key, + ) + r["well_tag_number"] = row["well_tag_number"] + writer.writerow(r) + + +@cli.command() +def qa(): + """Create several columns to use for location QA + - distance from well pt to parcel with matching PID + - wells street address / geocoder street address similarity + - wells location description / geocoder streeet address similarity + - wells city / geocoder locality similarity + + While running the analysis, reports on a few counts. + """ + # load source wells data + gwells_df = get_gwells() + + # create a copy + wells_copy = gwells_df.copy() + wells = gpd.GeoDataFrame( + wells_copy, + geometry=gpd.points_from_xy( + x=wells_copy.longitude_Decdeg, y=wells_copy.latitude_Decdeg, crs="EPSG:4326" + ), + ).to_crs(EPSG_3154) + # convert id to integer + wells["well_tag_number"] = wells["well_tag_number"].astype(int) + # retain just columns of interest + wells = wells[ + [ + "well_tag_number", + "street_address", + "city", + "well_location_description", + "legal_pid", + "geometry", + ] + ] + + # get NR regions/districts + nrd = ( + bcdata.get_data("WHSE_ADMIN_BOUNDARIES.ADM_NR_DISTRICTS_SPG", as_gdf=True)[ + ["DISTRICT_NAME", "REGION_ORG_UNIT_NAME", "geometry"] + ] + .to_crs(EPSG_3154) + .rename( + columns={ + "DISTRICT_NAME": "nr_district_name", + "REGION_ORG_UNIT_NAME": "nr_region_name", + } + ) + ) + # overlay with wells + wells_nrd = gpd.sjoin(wells, nrd, how="left", predicate="within") + + # load geocode results as string, retain only columns of interest + geocode_df = pd.read_csv(os.path.join("data", "wells_geocoded.csv"), dtype=str)[ + [ + "well_tag_number", + "fullAddress", + "civicNumber", + "civicNumberSuffix", + "streetName", + "streetType", + "isStreetTypePrefix", + "streetDirection", + "isStreetDirectionPrefix", + "streetQualifier", + "localityName", + "distance", + ] + ].rename(columns={"distance": "distance_geocode"}) + # convert tag and distance_geocode to integer + geocode_df["well_tag_number"] = geocode_df["well_tag_number"].astype(int) + geocode_df["distance_geocode"] = geocode_df["distance_geocode"].astype(int) + + # combine wells and geocode results + wells_nrd_geocoded = pd.merge(wells_nrd, geocode_df, "inner", on="well_tag_number") + + # match wells pid values to parcels pid values + wells_nrd_geocoded_pidmatched = pidmatch(wells_nrd_geocoded) + + scoring = wells_nrd_geocoded_pidmatched # shorter name + + # ------ + # ** Address matching ** + # ------ + LOG.info("Generating address matching scores") + + # before trying matching, clean the addresses by filling in nulls and + # do some very simple string standardization + scoring["street_address"] = scoring["street_address"].fillna("") + scoring["well_location_description"] = scoring["well_location_description"].fillna( + "" + ) + scoring["city"] = scoring["city"].fillna("") + # lowercasify the scoring address strings + scoring["street_address"] = scoring["street_address"].str.lower() + # abbreviate road types to match geocoder + street_abbreviations = { + "road": "rd", + "drive": "dr", + "avenue": "ave", + "highway": "hwy", + "street": "st", + "boulevard": "blvd", + "crescent": "cres", + "frontage": "frtg", + "place": "pl", + "court": "crt", + "terrace": "terr", + "lookout": "lkout", + "heights": "hts", + } + for k in street_abbreviations: + scoring["street_address"] = scoring["street_address"].str.replace( + k, street_abbreviations[k] + ) + # combine geocoder number/name/type/direction into a single slug + t = scoring[["civicNumber", "streetName", "streetType", "streetDirection"]] + scoring["streetAddress"] = t.apply( + lambda x: " ".join(x.dropna().astype(str).values), axis=1 + ) + + # + # "score_address" + # + # compares gwells address to geocoder address + t = scoring[["street_address", "streetAddress"]] + scoring["score_address"] = t.apply(compare_strings, axis=1) + + # + # "score_location_description" + # + # compares well_location_description with geocoder full street address + t = scoring[["well_location_description", "fullAddress"]] + scoring["score_location_description"] = t.apply(compare_strings, axis=1) + + # + # "city_score" + # + # compares gwells city to geocoder locality + t = scoring[["city", "localityName"]] + scoring["score_city"] = t.apply(compare_strings, axis=1) + + # extract only columns of interest + scoring = scoring[ + [ + "well_tag_number", + "nr_district_name", + "nr_region_name", + "fullAddress", + "streetAddress", + "civicNumber", + "civicNumberSuffix", + "streetName", + "streetType", + "isStreetTypePrefix", + "streetDirection", + "isStreetDirectionPrefix", + "streetQualifier", + "localityName", + "distance_geocode", + "distance_to_matching_pid", + "score_address", + "score_location_description", + "score_city", + ] + ] + + # Join the NR/geocoding/pidmatch scoring df back to source wells + out_df = pd.merge(gwells_df, scoring, "left", on="well_tag_number") + + # addtional QA - + # find records that have been cross referenced + # (fuzzy match scoring doesn't seem appropriate here, we can just + # search for exact matches of several permutations to catch most) + out_df["xref_ind"] = ( + out_df["comments"] + .str.upper() + .fillna("") + .str.contains("CROSS R|CROSS-R|REF'D|REFERENCED|REFD|XREF|X-R|X R") + ) + + # finally, overlay with ALR/BTM/ESA Landcover to find wells in agricultural areas + # ag_overlays = agriculture_overlays(wells) + # out_df = pd.merge(out_df, ag_overlays, "left", on="well_tag_number") + + # and dump results to file + LOG.info("Writing output file gwells_locationqa.csv") + out_df.to_csv("gwells_locationqa.csv", index=False) + + +if __name__ == "__main__": + cli() diff --git a/app/scripts/qaqc/requirements.txt b/app/scripts/qaqc/requirements.txt new file mode 100644 index 0000000000..bd4c919c58 --- /dev/null +++ b/app/scripts/qaqc/requirements.txt @@ -0,0 +1,7 @@ +requests>=2.26 +rasterio +geopandas>=0.10 +jupyterlab>=3.2.1 +python-levenshtein==0.12.2 +thefuzz==0.19.0 +bcdata>=0.4.5