From f5051d33bcf02cd3afbd132e67f3d20ed7024c72 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Fri, 23 Jun 2023 11:33:25 -0700 Subject: [PATCH 1/3] feat(acquisition): remove and deactivate norostat --- deploy.json | 9 - .../norostat/norostat_add_history.py | 45 -- src/acquisition/norostat/norostat_raw.py | 112 ----- src/acquisition/norostat/norostat_sql.py | 434 ------------------ src/acquisition/norostat/norostat_update.py | 28 -- src/acquisition/norostat/norostat_utils.py | 44 -- .../norostat/sample_content.pickle | Bin 37801 -> 0 bytes 7 files changed, 672 deletions(-) delete mode 100644 src/acquisition/norostat/norostat_add_history.py delete mode 100644 src/acquisition/norostat/norostat_raw.py delete mode 100644 src/acquisition/norostat/norostat_sql.py delete mode 100644 src/acquisition/norostat/norostat_update.py delete mode 100644 src/acquisition/norostat/norostat_utils.py delete mode 100644 src/acquisition/norostat/sample_content.pickle diff --git a/deploy.json b/deploy.json index 59d141ba4..3396dbbf6 100644 --- a/deploy.json +++ b/deploy.json @@ -138,15 +138,6 @@ "add-header-comment": true }, - "// acquisition - norostat", - { - "type": "move", - "src": "src/acquisition/norostat/", - "dst": "[[package]]/acquisition/norostat/", - "match": "^.*\\.(py)$", - "add-header-comment": true - }, - "// acquisition - paho", { "type": "move", diff --git a/src/acquisition/norostat/norostat_add_history.py b/src/acquisition/norostat/norostat_add_history.py deleted file mode 100644 index 64fd11ff7..000000000 --- a/src/acquisition/norostat/norostat_add_history.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -Parses historical versions of the NoroSTAT data-table and updates the -appropriate databases. Currently uses snapshots from the WayBack Machine -(archive.org). A more comprehensive archival service may be mementoweb.org, -which appears to pull from many services that implement the Memento protocol, -including archive.org. Manually downloaded snapshots could be recorded via this -script as well. -""" - -# standard library -import re -import os -import time -import collections - -# first party -from . import norostat_sql -from . import norostat_raw - - - -def main(): - norostat_sql.ensure_tables_exist() - snapshot_dir = os.path.expanduser("~/norostat_history/wayback/websites/www.cdc.gov/norovirus/reporting/norostat/data-table.html/") - snapshot_version_counter = collections.Counter() - for subdir in os.listdir(snapshot_dir): - if re.match(r'[0-9]+', subdir) is not None: - # appears to be snapshot dir - snapshot_version_counter[subdir] = 0 # register that loop found this snapshot directory - for norostat_capitalization in ["norostat","noroSTAT"]: - time.sleep(0.002) # ensure parse times are unique, assuming OS can accurately sleep and measure to ms precision - path = os.path.join(snapshot_dir,subdir,"norovirus","reporting",norostat_capitalization,"data-table.html") - if os.path.isfile(path): - print("Processing file ", path) - with open(path, 'r') as datatable_file: - content = datatable_file.read() - wide_raw = norostat_raw.parse_content_to_wide_raw(content) - long_raw = norostat_raw.melt_wide_raw_to_long_raw(wide_raw) - norostat_sql.record_long_raw(long_raw) - snapshot_version_counter[subdir] += 1 - print('Successfully uploaded the following snapshots, with the count indicating the number of data-table versions found inside each snapshot (expected to be 1, or maybe 2 if there was a change in capitalization; 0 indicates the NoroSTAT page was not found within a snapshot directory); just "Counter()" indicates no snapshot directories were found:', snapshot_version_counter) - norostat_sql.update_point() - -if __name__ == '__main__': - main() diff --git a/src/acquisition/norostat/norostat_raw.py b/src/acquisition/norostat/norostat_raw.py deleted file mode 100644 index 582de9684..000000000 --- a/src/acquisition/norostat/norostat_raw.py +++ /dev/null @@ -1,112 +0,0 @@ -""" -Functions to fetch, save, load, and format the NoroSTAT data-table. Formatting -functions include conversion from html content to "wide_raw" --- a wide data -frame in a tuple along with metadata --- and then to "long_raw" --- a long/tall -data frame in a tuple along with metadata. Metadata: release_date, parse_time, -and (constant) location. Here, the location will be (a str representing) a set -of states. -""" - - - -# standard library -import datetime -import re -import pickle - -# third party -import requests -import lxml.html -import pandas as pd - -# first party -from .norostat_utils import * - -def fetch_content(norostat_datatable_url="https://www.cdc.gov/norovirus/reporting/norostat/data-table.html"): - """Download NoroSTAT data-table. Returns the html content.""" - headers = { - 'User-Agent': 'delphibot/1.0 (+https://delphi.cmu.edu/)', - } - resp = requests.get(norostat_datatable_url, headers=headers) - expect_value_eq(resp.status_code, 200, - 'Wanted status code {}. Received: ') - expect_value_eq(resp.headers.get("Content-Type"), "text/html", - 'Expected Content-Type "{}"; Received ') - return resp.content - -def save_sample_content(content, f="sample_content.pickle"): - """Save the content from fetch_content into a pickle file for most testing (don't download unnecessarily).""" - with open(f, "wb") as handle: - pickle.dump(content, handle) - -def load_sample_content(f="sample_content.pickle"): - """Load data from a past call to fetch_content from a pickle file for most testing (don't download unnecessarily).""" - with open(f, "rb") as handle: - content = pickle.load(handle) - return content - -def parse_content_to_wide_raw(content): - """Convert the html content for the data-table into a wide data frame, then stick it in a tuple along with the release_date, parse_time, and (constant) location.""" - parse_time = datetime.datetime.now() - html_root = lxml.html.fromstring(content) - # Extract the release date, a.k.a. dateModified, a.k.a. "Page last updated" date; ~Dec 2018 this is only available in a meta tag; previously, it was available in a visible span - dateModified_meta_elts = html_root.xpath('//meta[@property="cdc:last_updated"]') - dateModified_span_elts = html_root.xpath('//span[@itemprop="dateModified"]') - if len(dateModified_meta_elts) == 1: - [dateModified_meta_elt] = dateModified_meta_elts - dateModified = dateModified_meta_elt.attrib['content'] - elif len(dateModified_span_elts) == 1: - [dateModified_span_elt] = dateModified_span_elts - dateModified = dateModified_span_elt.text - else: - raise Exception("Could not find the expected number of dateModified meta or span tags.") - # FIXME check/enforce locale - release_date = datetime.datetime.strptime(dateModified, "%B %d, %Y").date() - # Check that table description still specifies suspected&confirmed norovirus - # outbreaks (insensitive to case of certain letters and allowing for both old - # "to the" and new "through the" text), then extract list of states from the - # description: - [description_elt] = html_root.xpath('''//p[ - contains(translate(text(), "SCNORHD", "scnorhd"), "suspected and confirmed norovirus outbreaks reported by state health departments in") and - ( - contains(text(), "to the") or - contains(text(), "through the") - ) - ]''') - location = re.match(".*?[Dd]epartments in (.*?) (?:to)|(?:through) the.*$", description_elt.text).group(1) - # Attempt to find exactly 1 table (note: it would be nice to filter on the - # associated caption, but no such caption is present in earlier versions): - [table] = html_root.xpath('//table') - # Convert html table to DataFrame: - # Directly reading in the table with pd.read_html performs unwanted dtype - # inference, but reveals the column names: - [wide_raw_df_with_unwanted_conversions] = pd.read_html(lxml.html.tostring(table)) - # We want all columns to be string columns. However, there does not appear - # to be an option to disable dtype inference in pd.read_html. Hide all - # entries inside 1-tuple wrappers using pre-dtype-inference converters, - # then unpack afterward (the entries fed to the converters should already - # be strings, but "convert" them to strings just in case): - [wide_raw_df_with_wrappers] = pd.read_html( - lxml.html.tostring(table), - converters= {col: lambda entry: (str(entry),) - for col in wide_raw_df_with_unwanted_conversions.columns} - ) - # Unwrap entries: - wide_raw_df = wide_raw_df_with_wrappers.applymap(lambda wrapper: wrapper[0]) - # Check format: - expect_value_eq(wide_raw_df.columns[0], "Week", - 'Expected raw_colnames[0] to be "{}"; encountered ') - for colname in wide_raw_df.columns: - expect_result_eq(dtype_kind, wide_raw_df[colname].head(), "O", - 'Expected (head of) "%s" column to have dtype kind "{}"; instead had dtype kind & head '%(colname)) - # Pack up df with metadata: - wide_raw = (wide_raw_df, release_date, parse_time, location) - return wide_raw - -def melt_wide_raw_to_long_raw(wide_raw): - (wide_raw_df, release_date, parse_time, location) = wide_raw - long_raw_df = wide_raw_df \ - .melt(id_vars=["Week"], var_name="measurement_type", value_name="value") \ - .rename(index=str, columns={"Week": "week"}) - long_raw = (long_raw_df, release_date, parse_time, location) - return long_raw diff --git a/src/acquisition/norostat/norostat_sql.py b/src/acquisition/norostat/norostat_sql.py deleted file mode 100644 index 168e275eb..000000000 --- a/src/acquisition/norostat/norostat_sql.py +++ /dev/null @@ -1,434 +0,0 @@ -# standard library -import re - -# third party -import mysql.connector - -# first party -from .norostat_utils import * -import delphi.operations.secrets as secrets - -# Column names: -# `release_date` :: release date as stated in the web page in the dateModified -# span, displayed on the web page with the label "Page last updated:" -# `parse_time` :: time that we attempted to parse the data out of a downloaded -# version of the web page; when the scraper is running, this may be similar -# to a fetch time, but when loading in past versions that have been saved, -# it probably won't mean the same thing; this is tracked (a) in case the -# provided release date ever is out of date so that the raw data will still -# be recorded and we can recover later on, and (b) to provide a record of -# when parses/fetches happened; if there is a request for the data for a -# particular `release_date` with no restrictions on `parse_time`, the -# version with the latest `parse_time` should be selected -# (`release_date`, `parse_time`) :: uniquely identify a version of the table -# `measurement_type_id` :: "pointer" to an interned measurement_type string -# `measurement_type` :: the name of some column other than "Week" in the -# data-table -# `location_id` :: "pointer" to an interned location string -# `location` :: a string containing the list of reporting states -# `week_id` :: "pointer" to an interned week string -# `week` :: a string entry from the "Week" column -# `value` :: an string entry from some column other than "Week" in the -# data-table -# `new_value` :: an update to a `value` provided by a new version of the data -# table: either a string representing an added or revised entry (or a -# redundant repetition of a value retained from a past issue --- although -# no such entries should be generated by the code in this file), or NULL -# representing a deletion of a cell/entry from the table -# -# Tables: -# `norostat_raw_datatable_version_list` :: list of all versions of the raw -# data-table that have ever been successfully parsed -# `_pool` :: maps each encountered value of string `` to a unique ID -# `_id`, so that the string's character data is not duplicated in the -# tables on disk; serves a purpose similar to Java's interned string pool -# `norostat_raw_datatable_diffs` :: contains diffs between consecutive versions -# of the raw data-table (when arranged according to the tuple -# (`release_date`,`parse_time`) using lexicographical tuple ordering) -# `norostat_raw_datatable_parsed` :: a temporary table to hold the version of -# the raw data-table (in long/melted format) to be recorded; uses string -# values instead of interned string id's, so will need to be joined with -# `*_pool` tables for operations with other tables -# `norostat_raw_datatable_previous` :: a temporary table to hold an -# already-recorded version of the raw data-table with the latest -# `release_date`, `parse_time` before those of the version to be recorded; -# if there is no such version, this table will be empty (as if we recorded -# an empty version of the table before all other versions); uses interned -# string id's -# `norostat_raw_datatable_next` :: a temporary table to hold an -# already-recorded version of the raw data-table with the earliest -# `release_date`, `parse_time` after those of the version to be recorded; -# if there is no such version, this table will not be created or used; uses -# interned string id's - -def ensure_tables_exist(): - (u, p) = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database='epidata') - try: - cursor = cnx.cursor() - cursor.execute(''' - CREATE TABLE IF NOT EXISTS `norostat_raw_datatable_version_list` ( - `release_date` DATE NOT NULL, - `parse_time` DATETIME(6) NOT NULL, - PRIMARY KEY (`release_date`, `parse_time`) - ); - ''') - cursor.execute(''' - CREATE TABLE IF NOT EXISTS `norostat_raw_datatable_measurement_type_pool` ( - `measurement_type_id` INT NOT NULL PRIMARY KEY AUTO_INCREMENT, - `measurement_type` NVARCHAR(255) NOT NULL UNIQUE KEY - ); - ''') - cursor.execute(''' - CREATE TABLE IF NOT EXISTS `norostat_raw_datatable_location_pool` ( - `location_id` INT NOT NULL PRIMARY KEY AUTO_INCREMENT, - `location` NVARCHAR(255) NOT NULL UNIQUE KEY - ); - ''') - cursor.execute(''' - CREATE TABLE IF NOT EXISTS `norostat_raw_datatable_week_pool` ( - `week_id` INT NOT NULL PRIMARY KEY AUTO_INCREMENT, - `week` NVARCHAR(255) NOT NULL UNIQUE KEY - ); - ''') - cursor.execute(''' - CREATE TABLE IF NOT EXISTS `norostat_raw_datatable_diffs` ( - `release_date` DATE NOT NULL, - `parse_time` DATETIME(6) NOT NULL, - `measurement_type_id` INT NOT NULL, - `location_id` INT NOT NULL, - `week_id` INT NOT NULL, - `new_value` NVARCHAR(255), -- allow NULL, with meaning "removed" - FOREIGN KEY (`release_date`,`parse_time`) REFERENCES `norostat_raw_datatable_version_list` (`release_date`,`parse_time`), - FOREIGN KEY (`measurement_type_id`) REFERENCES `norostat_raw_datatable_measurement_type_pool` (`measurement_type_id`), - FOREIGN KEY (`location_id`) REFERENCES `norostat_raw_datatable_location_pool` (`location_id`), - FOREIGN KEY (`week_id`) REFERENCES `norostat_raw_datatable_week_pool` (`week_id`), - UNIQUE KEY (`measurement_type_id`, `location_id`, `week_id`, `release_date`, `parse_time`, `new_value`), - PRIMARY KEY (`release_date`, `parse_time`, `measurement_type_id`, `location_id`, `week_id`) - -- (the indices here are larger than the data, but reducing the key - -- sizes and adding an id somehow seems to result in larger index sizes - -- somehow) - ); - ''') - cnx.commit() - finally: - cnx.close() - -def dangerously_drop_all_norostat_tables(): - (u, p) = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database='epidata') - try: - cursor = cnx.cursor() - # Drop tables in reverse order (to avoid foreign key related errors): - cursor.execute(''' - DROP TABLE IF EXISTS `norostat_point_diffs`, - `norostat_point_version_list`, - `norostat_raw_datatable_diffs`, - `norostat_raw_datatable_week_pool`, - `norostat_raw_datatable_location_pool`, - `norostat_raw_datatable_measurement_type_pool`, - `norostat_raw_datatable_version_list`; - ''') - cnx.commit() # (might do nothing; each DROP commits itself anyway) - finally: - cnx.close() - -def record_long_raw(long_raw): - (long_raw_df, release_date, parse_time, location) = long_raw - (u, p) = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database='epidata') - try: - cursor = cnx.cursor() - cnx.start_transaction(isolation_level='SERIALIZABLE') - # Create, populate `norostat_raw_datatable_parsed`: - cursor.execute(''' - CREATE TEMPORARY TABLE `norostat_raw_datatable_parsed` ( - `measurement_type` NVARCHAR(255) NOT NULL, - `location` NVARCHAR(255) NOT NULL, - `week` NVARCHAR(255) NOT NULL, - `value` NVARCHAR(255) NOT NULL, -- forbid NULL; has special external meaning (see above) - PRIMARY KEY (`measurement_type`, `location`, `week`) - ) ENGINE=MEMORY; - ''') - cursor.executemany(''' - INSERT INTO `norostat_raw_datatable_parsed` (`week`,`measurement_type`,`value`,`location`) - VALUES (%s, %s, %s, %s); - ''', [(week, measurement_type, value, location) for - (week, measurement_type, value) in long_raw_df[["week","measurement_type","value"]].astype(str).itertuples(index=False, name=None) - ]) - # Create, populate `norostat_raw_datatable_previous`: - cursor.execute(''' - CREATE TEMPORARY TABLE `norostat_raw_datatable_previous` ( - `measurement_type_id` INT NOT NULL, - `location_id` INT NOT NULL, - `week_id` INT NOT NULL, - `value` NVARCHAR(255) NOT NULL, -- forbid NULL; has special external meaning (see above) - -- would like but not allowed: FOREIGN KEY (`measurement_type_id`) REFERENCES `norostat_raw_datatable_measurement_type_pool` (`measurement_type_id`), - -- would like but not allowed: FOREIGN KEY (`location_id`) REFERENCES `norostat_raw_datatable_location_pool` (`location_id`), - -- would like but not allowed: FOREIGN KEY (`week_id`) REFERENCES `norostat_raw_datatable_week_pool` (`week_id`), - PRIMARY KEY (`measurement_type_id`, `location_id`, `week_id`) - ) ENGINE=MEMORY; - ''') - cursor.execute(''' - INSERT INTO `norostat_raw_datatable_previous` (`measurement_type_id`, `location_id`, `week_id`, `value`) - SELECT `latest`.`measurement_type_id`, `latest`.`location_id`, `latest`.`week_id`, `latest`.`new_value` - FROM `norostat_raw_datatable_diffs` AS `latest` - -- Get the latest `new_value` by "group" (measurement_type, location, week) - -- using the fact that there are no later measurements belonging to the - -- same group (find NULL entries in `later`.{release_date,parse_time} - -- in the LEFT JOIN below); if the latest `new_value` is NULL, don't - -- include it in the result; it means that the corresponding cell/entry - -- has been removed from the data-table: - LEFT JOIN ( - SELECT * FROM `norostat_raw_datatable_diffs` - WHERE (`release_date`,`parse_time`) <= (%s,%s) - ) `later` - ON `latest`.`measurement_type_id` = `later`.`measurement_type_id` AND - `latest`.`location_id` = `later`.`location_id` AND - `latest`.`week_id` = `later`.`week_id` AND - (`latest`.`release_date`, `latest`.`parse_time`) < - (`later`.`release_date`, `later`.`parse_time`) - WHERE (`latest`.`release_date`, `latest`.`parse_time`) <= (%s, %s) AND - `later`.`parse_time` IS NULL AND - `latest`.`new_value` IS NOT NULL; - ''', (release_date, parse_time, release_date, parse_time)) - # Find next recorded `release_date`, `parse_time` if any; create, populate - # `norostat_raw_datatable_next` if there is such a version: - cursor.execute(''' - SELECT `release_date`, `parse_time` - FROM `norostat_raw_datatable_version_list` - WHERE (`release_date`, `parse_time`) > (%s,%s) - ORDER BY `release_date`, `parse_time` - LIMIT 1 - ''', (release_date, parse_time)) - next_version_if_any = cursor.fetchall() - expect_result_in(len, next_version_if_any, (0,1), - 'Bug: expected next-version query to return a number of results in {}; instead have len & val ') - if len(next_version_if_any) != 0: - cursor.execute(''' - CREATE TEMPORARY TABLE `norostat_raw_datatable_next` ( - `measurement_type_id` INT NOT NULL, - `location_id` INT NOT NULL, - `week_id` INT NOT NULL, - `value` NVARCHAR(255) NOT NULL, -- forbid NULL; has special external meaning (see above) - -- would like but not allowed: FOREIGN KEY (`measurement_type_id`) REFERENCES `norostat_raw_datatable_measurement_type_pool` (`measurement_type_id`), - -- would like but not allowed: FOREIGN KEY (`location_id`) REFERENCES `norostat_raw_datatable_location_pool` (`location_id`), - -- would like but not allowed: FOREIGN KEY (`week_id`) REFERENCES `norostat_raw_datatable_week_pool` (`week_id`), - PRIMARY KEY (`measurement_type_id`, `location_id`, `week_id`) - ) ENGINE=MEMORY; - ''') - cursor.execute(''' - INSERT INTO `norostat_raw_datatable_next` (`measurement_type_id`, `location_id`, `week_id`, `value`) - SELECT `latest`.`measurement_type_id`, `latest`.`location_id`, `latest`.`week_id`, `latest`.`new_value` - FROM `norostat_raw_datatable_diffs` AS `latest` - -- Get the latest `new_value` by "group" (measurement_type, location, week) - -- using the fact that there are no later measurements belonging to the - -- same group (find NULL entries in `later`.{release_date,parse_time} - -- in the LEFT JOIN below); if the latest `new_value` is NULL, don't - -- include it in the result; it means that the corresponding cell/entry - -- has been removed from the data-table: - LEFT JOIN ( - SELECT * FROM `norostat_raw_datatable_diffs` - WHERE (`release_date`,`parse_time`) <= (%s, %s) - ) `later` - ON `latest`.`measurement_type_id` = `later`.`measurement_type_id` AND - `latest`.`location_id` = `later`.`location_id` AND - `latest`.`week_id` = `later`.`week_id` AND - (`latest`.`release_date`, `latest`.`parse_time`) < - (`later`.`release_date`, `later`.`parse_time`) - WHERE (`latest`.`release_date`, `latest`.`parse_time`) <= (%s, %s) AND - `later`.`parse_time` IS NULL AND - `latest`.`new_value` IS NOT NULL -- NULL means value was removed - ''', next_version_if_any[0]+next_version_if_any[0]) - # Register new version in version list: - try: - cursor.execute(''' - INSERT INTO `norostat_raw_datatable_version_list` (`release_date`, `parse_time`) - VALUES (%s, %s) - ''', (release_date, parse_time)) - except mysql.connector.errors.IntegrityError as e: - raise Exception(['Encountered an IntegrityError when updating the norostat_raw_datatable_version_list table; this probably indicates that a version with the same `release_date` and `parse_time` was already added to the database; parse_time has limited resolution, so this can happen from populating the database too quickly when there are duplicate release dates; original error: ', e]) - # Add any new measurement_type, location, or week strings to the associated - # string pools: - cursor.execute(''' - INSERT INTO `norostat_raw_datatable_measurement_type_pool` (`measurement_type`) - SELECT DISTINCT `measurement_type` - FROM `norostat_raw_datatable_parsed` - WHERE `measurement_type` NOT IN ( - SELECT `norostat_raw_datatable_measurement_type_pool`.`measurement_type` - FROM `norostat_raw_datatable_measurement_type_pool` - ); - ''') - cursor.execute(''' - INSERT INTO `norostat_raw_datatable_location_pool` (`location`) - SELECT DISTINCT `location` - FROM `norostat_raw_datatable_parsed` - WHERE `location` NOT IN ( - SELECT `norostat_raw_datatable_location_pool`.`location` - FROM `norostat_raw_datatable_location_pool` - ); - ''') - cursor.execute(''' - INSERT INTO `norostat_raw_datatable_week_pool` (`week`) - SELECT DISTINCT `week` - FROM `norostat_raw_datatable_parsed` - WHERE `week` NOT IN ( - SELECT `norostat_raw_datatable_week_pool`.`week` - FROM `norostat_raw_datatable_week_pool` - ); - ''') - # Record diff: [newly parsed version "minus" previous version] (first, - # record additions/updates, then record deletions): - cursor.execute(''' - INSERT INTO `norostat_raw_datatable_diffs` (`measurement_type_id`, `location_id`, `week_id`, `release_date`, `parse_time`, `new_value`) - SELECT `measurement_type_id`, `location_id`, `week_id`, %s, %s, `value` - FROM `norostat_raw_datatable_parsed` - LEFT JOIN `norostat_raw_datatable_measurement_type_pool` USING (`measurement_type`) - LEFT JOIN `norostat_raw_datatable_location_pool` USING (`location`) - LEFT JOIN `norostat_raw_datatable_week_pool` USING (`week`) - WHERE (`measurement_type_id`, `location_id`, `week_id`, `value`) NOT IN ( - SELECT `norostat_raw_datatable_previous`.`measurement_type_id`, - `norostat_raw_datatable_previous`.`location_id`, - `norostat_raw_datatable_previous`.`week_id`, - `norostat_raw_datatable_previous`.`value` - FROM `norostat_raw_datatable_previous` - ); - ''', (release_date, parse_time)) - cursor.execute(''' - INSERT INTO `norostat_raw_datatable_diffs` (`measurement_type_id`, `location_id`, `week_id`, `release_date`, `parse_time`, `new_value`) - SELECT `measurement_type_id`, `location_id`, `week_id`, %s, %s, NULL - FROM `norostat_raw_datatable_previous` - WHERE (`measurement_type_id`, `location_id`, `week_id`) NOT IN ( - SELECT `norostat_raw_datatable_measurement_type_pool`.`measurement_type_id`, - `norostat_raw_datatable_location_pool`.`location_id`, - `norostat_raw_datatable_week_pool`.`week_id` - FROM `norostat_raw_datatable_parsed` - LEFT JOIN `norostat_raw_datatable_measurement_type_pool` USING (`measurement_type`) - LEFT JOIN `norostat_raw_datatable_location_pool` USING (`location`) - LEFT JOIN `norostat_raw_datatable_week_pool` USING (`week`) - ); - ''', (release_date, parse_time)) - # If there is an already-recorded next version, its diff is invalidated by - # the insertion of the newly parsed version; delete the [next version - # "minus" previous version] diff and record the [next version "minus" newly - # parsed] diff: - if len(next_version_if_any) != 0: - cursor.execute(''' - DELETE FROM `norostat_raw_datatable_diffs` - WHERE `release_date`=%s AND `parse_time`=%s; - ''', next_version_if_any[0]) - cursor.execute(''' - INSERT INTO `norostat_raw_datatable_diffs` (`measurement_type_id`, `location_id`, `week_id`, `release_date`, `parse_time`, `new_value`) - SELECT `measurement_type_id`, `location_id`, `week_id`, %s, %s, `value` - FROM `norostat_raw_datatable_next` - WHERE (`measurement_type_id`, `location_id`, `week_id`, `value`) NOT IN ( - SELECT - `norostat_raw_datatable_measurement_type_pool`.`measurement_type_id`, - `norostat_raw_datatable_location_pool`.`location_id`, - `norostat_raw_datatable_week_pool`.`week_id`, - `norostat_raw_datatable_parsed`.`value` - FROM `norostat_raw_datatable_parsed` - LEFT JOIN `norostat_raw_datatable_measurement_type_pool` USING (`measurement_type`) - LEFT JOIN `norostat_raw_datatable_location_pool` USING (`location`) - LEFT JOIN `norostat_raw_datatable_week_pool` USING (`week`) - ); - ''', next_version_if_any[0]) - cursor.execute(''' - INSERT INTO `norostat_raw_datatable_diffs` (`measurement_type_id`, `location_id`, `week_id`, `release_date`, `parse_time`, `new_value`) - SELECT `measurement_type_id`, `location_id`, `week_id`, %s, %s, NULL - FROM `norostat_raw_datatable_parsed` - LEFT JOIN `norostat_raw_datatable_measurement_type_pool` USING (`measurement_type`) - LEFT JOIN `norostat_raw_datatable_location_pool` USING (`location`) - LEFT JOIN `norostat_raw_datatable_week_pool` USING (`week`) - WHERE (`measurement_type_id`, `location_id`, `week_id`) NOT IN ( - SELECT `norostat_raw_datatable_next`.`measurement_type_id`, - `norostat_raw_datatable_next`.`location_id`, - `norostat_raw_datatable_next`.`week_id` - FROM `norostat_raw_datatable_next` - ); - ''', next_version_if_any[0]) - cursor.execute(''' - CREATE TABLE IF NOT EXISTS `norostat_point_version_list` ( - `release_date` DATE NOT NULL, - `parse_time` DATETIME(6) NOT NULL, - FOREIGN KEY (`release_date`,`parse_time`) REFERENCES `norostat_raw_datatable_version_list` (`release_date`,`parse_time`), - PRIMARY KEY (`release_date`, `parse_time`) - ); - ''') - cursor.execute(''' - CREATE TABLE IF NOT EXISTS `norostat_point_diffs` ( - `release_date` DATE NOT NULL, - `parse_time` datetime(6) NOT NULL, - `location_id` INT NOT NULL, - `epiweek` INT NOT NULL, - `new_value` NVARCHAR(255), -- allow NULL, with meaning "removed" - FOREIGN KEY (`release_date`,`parse_time`) REFERENCES `norostat_point_version_list` (`release_date`,`parse_time`), - FOREIGN KEY (`location_id`) REFERENCES norostat_raw_datatable_location_pool (`location_id`), - UNIQUE KEY (`location_id`, `epiweek`, `release_date`, `parse_time`, `new_value`), - PRIMARY KEY (`release_date`, `parse_time`, `location_id`, `epiweek`) - ); - ''') - cnx.commit() # (might do nothing; each statement above takes effect and/or commits immediately) - finally: - cnx.close() - -def update_point(): - (u, p) = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database='epidata') - try: - cursor = cnx.cursor() - cnx.start_transaction(isolation_level='serializable') - cursor.execute(''' - SELECT `release_date`, `parse_time`, `measurement_type`, `location_id`, `week`, `new_value` - FROM `norostat_raw_datatable_diffs` - LEFT JOIN `norostat_raw_datatable_measurement_type_pool` USING (`measurement_type_id`) - LEFT JOIN `norostat_raw_datatable_week_pool` USING (`week_id`) - WHERE (`release_date`, `parse_time`) NOT IN ( - SELECT `norostat_point_version_list`.`release_date`, - `norostat_point_version_list`.`parse_time` - FROM `norostat_point_version_list` - ); - ''') - raw_datatable_diff_selection = cursor.fetchall() - prog = re.compile(r"[0-9]+-[0-9]+$") - point_diff_insertion = [ - (release_date, parse_time, location_id, - season_db_to_epiweek(measurement_type, week), - int(new_value_str) if new_value_str is not None else None - ) - for (release_date, parse_time, measurement_type, location_id, week, new_value_str) - in raw_datatable_diff_selection - if prog.match(measurement_type) is not None and - new_value_str != "" - ] - cursor.execute(''' - INSERT INTO `norostat_point_version_list` (`release_date`, `parse_time`) - SELECT DISTINCT `release_date`, `parse_time` - FROM `norostat_raw_datatable_version_list` - WHERE (`release_date`, `parse_time`) NOT IN ( - SELECT `norostat_point_version_list`.`release_date`, - `norostat_point_version_list`.`parse_time` - FROM `norostat_point_version_list` - ); - ''') - cursor.executemany(''' - INSERT INTO `norostat_point_diffs` (`release_date`, `parse_time`, `location_id`, `epiweek`, `new_value`) - VALUES (%s, %s, %s, %s, %s) - ''', point_diff_insertion) - cnx.commit() - finally: - cnx.close() - -# note there are more efficient ways to calculate diffs without forming ..._next table -# todo give indices names -# todo trim pool functionality for if data is deleted? -# todo make classes to handle pool, keyval store, and diff table query formation -# todo test mode w/ rollback -# todo record position of rows and columns in raw data-table (using additional diff tables) -# todo consider measurement index mapping to another id -# todo add fetch_time to version list -# xxx replace "import *"'s -# xxx should cursor be closed? -# xxx is cnx auto-closed on errors? -# xxx drop temporary tables? -# fixme time zone issues diff --git a/src/acquisition/norostat/norostat_update.py b/src/acquisition/norostat/norostat_update.py deleted file mode 100644 index 4b0021dd5..000000000 --- a/src/acquisition/norostat/norostat_update.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -=============== -=== Purpose === -=============== - -Fetch NoroSTAT data table from -; -process and record it in the appropriate databases. -""" - -# first party -from . import norostat_sql -from . import norostat_raw - - -def main(): - # Download the data: - # content = norostat_raw.load_sample_content() - content = norostat_raw.fetch_content() - # norostat_raw.save_sample_content(content) - wide_raw = norostat_raw.parse_content_to_wide_raw(content) - long_raw = norostat_raw.melt_wide_raw_to_long_raw(wide_raw) - norostat_sql.ensure_tables_exist() - norostat_sql.record_long_raw(long_raw) - norostat_sql.update_point() - -if __name__ == '__main__': - main() diff --git a/src/acquisition/norostat/norostat_utils.py b/src/acquisition/norostat/norostat_utils.py deleted file mode 100644 index a99a4dc96..000000000 --- a/src/acquisition/norostat/norostat_utils.py +++ /dev/null @@ -1,44 +0,0 @@ -# standard library -import re -import datetime - -# first party -from delphi.utils.epidate import EpiDate - -# helper funs for checking expectations, throwing exceptions on violations: -def expect_value_eq(encountered, expected, mismatch_format): - if encountered != expected: - raise Exception([mismatch_format.format(expected), encountered]) -def expect_result_eq(f, value, expected, mismatch_format): - result = f(value) - if result != expected: - raise Exception([mismatch_format.format(expected), result, value]) -def expect_value_in(encountered, expected_candidates, mismatch_format): - if encountered not in expected_candidates: - raise Exception([mismatch_format.format(expected_candidates), encountered]) -def expect_result_in(f, value, expected_candidates, mismatch_format): - result = f(value) - if result not in expected_candidates: - raise Exception([mismatch_format.format(expected_candidates), result, value]) -def expect_str_contains(encountered, regex, mismatch_format): - if re.search(regex, encountered) is None: - raise Exception([mismatch_format.format(regex), encountered]) - -# helper fun used with expect_* funs to check value of .dtype.kind: -def dtype_kind(numpy_like): - return numpy_like.dtype.kind - -# helper fun used to convert season string ("YYYY-YY" or "YYYY-YYYY") and -# "Week" string (strptime format "%d-%b") to the corresponding epiweek; assumes -# by default that dates >= 1-Aug correspond to weeks of the first year: -def season_db_to_epiweek(season_str, db_date_str, first_db_date_of_season_str="1-Aug"): - year_strs = season_str.split("-") - first_year = int(year_strs[0]) - second_year = first_year + 1 - # FIXME check/enforce locale - first_date_of_season = datetime.datetime.strptime(first_db_date_of_season_str+"-"+str(first_year), "%d-%b-%Y").date() - date_using_first_year = datetime.datetime.strptime(db_date_str+"-"+str(first_year), "%d-%b-%Y").date() - date_using_second_year = datetime.datetime.strptime(db_date_str+"-"+str(second_year), "%d-%b-%Y").date() - date = date_using_first_year if date_using_first_year >= first_date_of_season else date_using_second_year - epiweek = EpiDate(date.year, date.month, date.day).get_ew() - return epiweek diff --git a/src/acquisition/norostat/sample_content.pickle b/src/acquisition/norostat/sample_content.pickle deleted file mode 100644 index 1518dde0deb517bb8641573d685ba808d9e39c8d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 37801 zcmeHwZFAd5lIBF*Cp)75Lp9xJOFaT1l6tc(aYB~lwt8(zE6VnE_jM>hvM9m^0U7`$ z%N|F}f4KW{pYFfyFWD!vUQpnRM9RH8J9F;cu|S|ID=RBAD=RB2>u>+#+5h|BfBMt* z_xs0tXTO{rh<-kZn%~{u{@(L`3wt8U#o>Y2`kh|UC(((5EZeEXiT5rOad0W}z8pyJ zB9$^mX)lzuRrU9=`m-e}A`0S*or;XTQC4Y+?eBk+aX0M!?s-m2cW-8H_>ULt#(aRk zKRo!oN%WeJMrELa$l-ZoBSCQjurNpsm1c_OGp>umRIMeGNF^;faCzqjqH+TY!`ZN$Mq?o_%m z>!jf@50kheI!T<%7|l}qxybs-RTy80cr<9sR3tso2{M_{Lu|1~3=UO|CU-yqe=o1E zlC+ySkPUL83*1(w!4awMi~5@7LB1mLpdCpxpy{K)UxxCEy6u3!3cGoKrzk4G~9@Y!*z?3F0IU!L(+8`guOgp7{RN)m0T}R49Fh?PQrBg>Ao`V9@e37}nRSc&CZy5N-i@kb8phmqxW- zZB(}mEIk!o@KlI7gmwL-)@D-BoQauilV+N)=~fTQ*9)#>mSA!X<)+)W()4LtRWmhD zad&Mvls7a}u%b)Rf$WCCPGy*e@$KWqrKV9>%{n4WJ2P149S?Tq50b7-N^+|+rFyL^J4LMFYGOSk4aBfwDu0WAud<3e?k(lZO|NEYL7%>S(S+#*z~>8r%6H^a zH|dN9unLGs2N5j4IM1G4p9L33Fd&vHtXP(RTm8Mtb|Ss3X2U4Vmnx#Nyi!rdVrBU$ zdTkN3Fgq|`a;d>CRfywIG_LZb>~I}k<<~>lE4lob`y};vqd5`d4@;C@+@d@fmCUdF zIsbEo+5V};V@BVgwJO{<=m{9Gdmv|bDaKGN^T--r!O-R4lMEyO`3t6xE{u?ne#N6G zA`Wo)m3}!U5*Z%Q@x+}TO-jzh0JDu${A{g#<1|##XHJ8I%x&@v{IQorN$RxwV0Cq) z76e8Y&a9h#7zL@^T>ZGY8YLHr17Z4Hpsb{~VY~&oisU)h#HB} zzdN<1>(130%tqO$-MuYccdoWn&!`6ecc-4|x-(jv&qN-xkXN!kl>2)avtBREcjBbN zQ`%wp#TVE|z)H5wg^lN^E z@(n~wFEWnoB4dvi8E0hin`MhK8rbkSI(Vxa-))KG=VHUJ6vuIJ)gD?zYtmza)6oPdb zo&>a3$8r-IaBy*hX{XoAgCN6<9CXjKQJWa4nq6L0L=eIM z@LzzZk&u4a?ZQo!#Jt?2*?BSKYmmC=?}Pefn1xW11@204v~(OEw;l`Tv-aphzz_2t z>)HhT0~I_7-b;~70S;0-?!sdVeuqp~@fhU1vB|>(Zk~^({v20bYBhm2n#K9$u_$oDB)?6gStD;~R z^O#?m`H|Wn2%0FWC>AS5G9OgnJ|(AHY0hZ2c+mg~w|FWs0?7*1;=c`%9T>=}{&T#k z2$*O61R9N}pMaM+P;c5(pbA=Nf%>VnUN4kU7nW-os7LL1uChg^B6$Jpx2f@?b{!=3 zN2amcebtzP*Xhg7`*!lt=)mnLXiMu^hWrB$5Gss^aPBBW-&Qd+Q-$Cf1t^IQ>xYWC z45AT;N~=N!t2K8q@>8(lmd3hS8a$1Cdz9x;&Ia#$+vn z*GUc-_<}<$+=A;Ta68GT5D{pX5DU5MCyJAws+9vA$Vr*pA;&FX;6^D>gKBWpW1(V2x?2ZTm*xXxQ-!Qe+sQtT%%CCr@+4 z5bH-T4KMn+(KZ+w^T>+FRW4-MC`+}N;Aj$++t5QMTKp#K>{|Dp0%y-ZI5U_iuf!Fxy0qL z#-BA=tToxj=7LN;IC1j?N~;f6S{uSBne)3&e1z>Mg&cCx|OxkUOIvz6og z65sFah(~0a_OSZsK3W!^-Oq5;#G~^lfyrBPjfpQHI`2MdN;{J$QyLy9u#U1R_2fz= z4S#?=iqSSxp6eVBKKX(2IyZOA-J7FkTi1NHaLtaciCs5`k>U=Y?y&*ERoT;hK@I`Euc!16}jg!ZqOr zrM;f!(ZV$oUGsS1nnPXlWZ{}0bj^QVxMr$ro-SN7(=}TQ*UWXzvxRGpbj|YQGLKHU z)HUBM+~$?8`F7!&A9c-tTe#-6uKCNtH9zT^zb;&JRo8sr*3>iDA2WerHoAHKDjUV@ zH5xaRYCbkoDPly=WXh??OFmj?dNAk6l`tAmAV%Rhs;}uaD;#n>8(Q;k!kSXB%f_U( zZi{*H+alDgOuj8bX>Dtn=G!9l|LaAlQIw>erY`hK`U@qX$XZy=(ds)+lo@ZMLa?|M z*tidS*W?+dBG@;^4oTd(hQlh$$lELSkHyjP8SR#e5bImG@~hw7XASvvs}0-K%_>_d zV(#h<4t{o=(_V;?6VOKy=xrb2kG$z%_F)C~d`B5}AE^5Q{G`LMBl-yE;KdM;fhtIR zjv^IVhk&M0)D>-V{)_E?t%*Rli>jR_&{Dr%sBS0Nhj;Pyv}d>3X!^U4O{{v8kTm~| z53D7R-ZwBlPy3!W1k^4P^_iG~{z$YH;pS?J z*KsEqVEqTT9&L$dv{`^a4BG2p_YV`qHf@;0a6{q0xuyMVXc2h$ptezax(h5i;ll^@ zwZ_v}p>hyWGzVA_jzJ`q0$mw^$n3>%6W zIp@0qlm|`Y1fiOvp$-6TUyEznu4!XyDwAndz+J0o4;uS#wMvyo;KqRN;VBW{O z3ZsZvaRg^c5Wy9bM zttAkI)Mr5F@RT9WL38si!-k)PVjqa3tA>HM!T;7>@O3J}0J>d6n8q>0N=u9n+W3y4 zk`!`0(HC%;)*LPaJG+LgYa`jqIkH-DkCDjfTe{krR(1pma^bCWQ`@MS8b_*9#dZMI^6;yyB-YT?Xzn* z{nMKMw^T>mV4MxYjKVfG$DKK^<^?k>HcHiMyVyCM7eo$u%t;MdH1ux=0D0k{(LvLh z-GZE!$odkLPJ=v2ui^Z)B1AUb2z3At=2 zJBVw;MhT`prT#U{mi~gxB=coHkO)A$fCz_%37kIfn@!o& zXC0ep_uC0N0kS=Uw>tJ-w{-WP@o(B%wL%~cqbP=b1Z~R`Q6fnXR^RVOST)j zr9Di+Q1vYnrdDdQ)TNtwZK5IUDo0(4m(mRt9eHcM+yQ+R5z&R|sKK6chy=Z$&F7+# zX{gUJ*hQqYtB=$X3ToqFFc>}+txs9ta~!|#vL|spMa>b=d6~lka%gMdEKFJp?2l%&0`N>eGlCAx3 zb!a{x^MgjFY)>eRMD;c(ETv>k{jEj&pMaArmiHOz>Spo`?42cTB-)w@Z%IERfwFW*NQk|Zxy8U zFoyY?k1+SsYD>}iA4KTP%!7q@&c~0;takHcMy5yyZo6vS$wpP#9Zh3ABHA9E-(Y#e z3Lieti|c;?!kFkJp2xNG)wL$r{mOH{eJXyzDXZPHsyhB<-yFH@dT_xYDw_6VjU~gU za&1X^a6NhCcB?%u$~pxnTH(6mO_!k2Vg7C>#Yq@MV%xZM_U+k5+}63g#}%CNtD;f&i$l#KrqXuV_RT(&=~eW3!kiBOnnSj+=FeZ?O9b)?aiL|5HE|<&+qMdj_3A;I{NrvEbz~X z%ZGifc4Dw?5DQb$#C}qzR6qhD?lh#)FfCgkl+>e$8^yTKL^{6;u_c69PkjdE%>2F` z<+CfC2{+A*p=iNs4E1lWGJ)I-f||n$4|ukXrj49HRQf!JSri0cAN}+ib#)+*PqQ#H z&~eIA94?RSGmi5-fVa;%L4kd#0H>W1A&(tt8V`hc)1MM0T>j8da13e&6gU9cAA)U6 z`)HskHooQSBF8y+Ul%BD`5_DRx*fX`_NU!bwhf;S|Y;7TpT(T^^#uT)F)mNf^WkNbjZl*Oy}sNvuOuIf<2tZlrk$CjW${9t?LXcS+PW3&}AA} z>d9Lv-;=Rr9NNb?;f6de_0`qK9v#5^@nJW(xRB{n__v3cu)rQTZlswSCkj16hTrkO zi1DUSZ*nl)-|hr_;?R_KnBK{bV-R(6WdV&FO&X3lm&__8DVfr}5{LuBIQ| zX3g6j!MB!k-|(||Bt-RF=W$)XRbUl0y3MUJl+~hYtx-HXk2O=I45zj}6A*RRAZaS& z#dLh?^?Bf|6(9}tGCK9gWia)kf>YOdGzH3F*1eV-7F8V@n`Y4w=W$DyEkmi*%V11f zsB5W-XH(e$IR@wPOgN7VB%MypAn`^Sj8nluf!SRTn;HHxIt}l*lNaqey>h@f3|KP_ zmBG|jXTou*IhSJ7QL5KwLaIBE>V|z> zl9h3)Zy?xm!=;YwB?ht&<>9u`d1_Y@sL2}1v&Be*! z0h~4seq*2>kAZSAaYk&-&>5g?6J=2KjWJLTCL7L>xuDh!*fCJv&p}*J%CZX@-!L_( zPq+8)aT!wGP+<(xy3mpc)RS4O4~s=TOk{ zS{YPhbqthK#SS@j*1>89(Ko#3vRx3p;gD+6>7T6)sA~rCGE%k17$|3)woDaNx9pAQ zh8fo~sQT&{D2F^7jxu+prf!&R4Af>BlskZrO%-5LH<%DshMZ+k8{VrREl5>&5OK%M z>Vet)1F1@&T;1jhjU^92SyfX;3ZyE7a!6GgG=~-HW@N@dZH|F*#=q1-2UNq5unel< z?G7DMx&2#n$W|h{VUaQ<%*G{9?l75d(D1o9nddP`n`KB29<~<*$&modEM-XbwK61! zzP*vYGm($;K14t5TFGqzHe85D)A=$O zOv5EOPG=nQI&ZD*Fa`&cl#7o+nmXW;+Uc;ttqce=@GL-vq)un>XlPyrgV||>ECcw? z_MC%`1I6i~DNu$2aV>#ydt%x}6Q(B)Vq-JHR4@Dr>NNxV;$`Xt5PXkD7YFJ znVcc?18e+sueL$w#1P<({jO`ADtoHCu<)vLxI@No z5ieE|Lc^L2EdIBMmx37IB3=-*I7hIe+1B|M@!~{r-Yw##*ePYB$Hy5L?AvjmE`p=V zmXF%I)c#FZUjHM6+Mhb z*pu~CK0Ui}t==B@dR$?sscxTn0N=BS#QJRZ@b+DCsu(Wf9B9A_mRf#OM%`(>1Xh?l8wWME7Hbf;~B`^^}s(>J&5%8*Q za#L{T-q4ga^7v3B1t2P0QXN=(Vy#4bsL@Q~w zB<=ccd!v*x=$W85mA9Hbq=$pS$e_SpP&%6&>I3&WrFus8(25&G$ME;x?5hKI%v-8o z+|GVTrY53t5ibX)Djc{jAbliToxkIOw6={JR{UFD9?45knq5~v>TV8V2v&X^%LHUHuWdVaK`gw!4_q+Nk~ghGJHivI3CUw;P=y1GCL z7u9HJ$VS{MBB+^Es)3Ru6(y8G?6_Mku1e525r#JXRLUJ-qh-+8s2~-N$IYR;A>?7C zq+^D$gIx+_EFnWqO9P9lZr_bjJRne}5L?dXgx+jqO#C&*W8mZX8O765WbRJoyp0p; z$P_cp{sKy-;_YbJ(C!6L(?Qn;E4r;_Uc^lN_O0Y91%b~=E@Z&TM{V3~1!^H+HYJkb z$}kxq0nmjy|1&qD8rJOu{3RKkq0t4%N4aW4oJ|TYV4^uhgesbIcx~<5(W669+|X16 ztp>3teDAsP1Igi|7juhGy<@Y^50ftLjl?~Bvv`y?2`GbfiiT z3K$Omxiv7Y>TgiA?x${t4{#esaX*=?q67KD^^an?S> z(Zc7)hvkIiw__n?g5&8pZPGzDbL{go8Mjf+88g6#R z0TI{`$A@F7yvgV!+1FVg3{SuU^rsteld`Jpd&6C>X=>CF-nm+$+o2T#I%&kp{dj{4 zL3fQ3h>*H3Qnz7QFtCzuG(CTcl6Da3rfaVCsdE-GArLBoPnB>PC_woGkfbye&|VGlWCk)8KQ!9m`@^@j)>y@WNY_ z2?5VHnJ_aYnHh^TkeO!jTqq4qM`N5UqAu$H4b{cpl`UpZF_h8H$*)x8V-OF;i_rk5 zkXtH8iZ*V>dBO-Ra`NzhH;dUHP6X5J{1ReF;lqDeWOlc5@Zv?wKgO4m1ejLFG=Y6h zL@}Z@F;D#AeJ1}Bw>XDBsE)EgrA{*V*`7lG(w9<4Nf^xtf3>K)QaCB;kdrGUhQ%HvGK36ZYgnL&zDeyWSiHY&%m$q;z`}6XxT@f8y1_+3De43xUnZ^W7%GA@G?( zx+jVjFUm=nalayAKxTnV@pcjBeU1E4dYF;-WwOhMdKPF3??|@tWO%Qa{hI)kG0P;skdM(vH*~c7BopawHQvZ1RPmOUNhlFq8Q zf^S8@fyC+imvq&8S(Uc2)*fRTsnI?6_r5IipLt={Z-!aF3(UsNFdKJ)i7nNe`o8AC z{K9iR2m@I;qT5~(t25#>-<<#E>3#U{`T-JgAi0YTzOLZ4!>FKp{*u_Owp8K0$;$Fm zUhz;&mP~OC^wHko={^)1>2|T@*VgF@^g7fJ^CMl*&he! z%{tmNLE+Ty^iqwb%CJAg_vxS4mMUHBg=w?}UD~b?S`$cp2p+Wfu*#(5DfoZIPW7ED zWQs|ys^rF@w8d4*6jaTVmlRw@x5Fdr91NCc?DDq8^ z{xPr^+1DH+z-NwF0pe;@Z3vh=Sdh|OhH6b!+Lc~E5zB{vuqK7Q!{*Uf|=0=;{muCm3V)y8m zvloX)KZ$1tFOS~}Jh#Nl!=nT7y>Ki2_@n>beGpZsX<{d<=FHwA1J_=vC|R;L&n(XBAB%E>x>6 z)g~fKGNiweOOO6%WAD-OpCA3N@4ZKVe&juZh>zi6F4vgcRws{otyZtrlg%m)w81X1 zxwT9f=2#IQ_XlJ?USj3w*L<3?J@2UHue#4j<0a{i+?`6T0uk!weSYHHTL{V#)>z^k zTMyr5jMKP8vuhVPcTt))Ek zPan52(-7v8x>s(h9YXjn+`L|(a)%2e*gTajXo*V_=Cq%+E%J3{KiVs=P)kNX&$Pzq z`C-X$>@oy6=w2@sGxleaFEG(d+=xPP?>i!vS1jB54W>Nj7LyfGaWVmy^N3T{Lm$CV zI#>3J$kP$>Y4wmn*!;nqShNLPa4l{Mk-iVx$PrgP Date: Fri, 23 Jun 2023 11:33:56 -0700 Subject: [PATCH 2/3] docs(norostat): note that updates stopped November 2020 --- docs/api/norostat.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/api/norostat.md b/docs/api/norostat.md index 6e801116c..734dd9aa9 100644 --- a/docs/api/norostat.md +++ b/docs/api/norostat.md @@ -13,6 +13,8 @@ General topics not specific to any particular endpoint are discussed in the [contributing](README.md#contributing), [citing](README.md#citing), and [data licensing](README.md#data-licensing). +**NOTE**: This data source stopped acquiring data in November 2020. + ## NoroSTAT Data ... From 32b05e976d66f50c6b0d7c5e04bf4f2923e503a5 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Fri, 23 Jun 2023 11:38:18 -0700 Subject: [PATCH 3/3] docs(norostat): correct wording --- docs/api/norostat.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api/norostat.md b/docs/api/norostat.md index 734dd9aa9..dded4ec13 100644 --- a/docs/api/norostat.md +++ b/docs/api/norostat.md @@ -13,7 +13,7 @@ General topics not specific to any particular endpoint are discussed in the [contributing](README.md#contributing), [citing](README.md#citing), and [data licensing](README.md#data-licensing). -**NOTE**: This data source stopped acquiring data in November 2020. +**NOTE**: Delphi stopped stopped acquiring data from this data source in November 2020. ## NoroSTAT Data