diff --git a/README.md b/README.md index 17ab9aa..027c855 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,7 @@ -# openownership-cove-bods-alpha +# Open Ownership BODS Cove + +Checks data complies with the Beneficial Ownership Data Standard (BODS) versions 0.1-0.4. +Based on: https://github.com/OpenDataServices/cove ## Dev installation diff --git a/cove_bods/process.py b/cove_bods/process.py index 66a8663..e0e4cb7 100644 --- a/cove_bods/process.py +++ b/cove_bods/process.py @@ -5,6 +5,7 @@ from libcovebods.config import LibCoveBODSConfig from libcovebods.jsonschemavalidate import JSONSchemaValidator from libcovebods.additionalfields import AdditionalFields +from libcovebods.schema_dir import schema_registry import libcovebods.run_tasks import libcovebods.data_reader from typing import List @@ -23,6 +24,9 @@ from libcoveweb2.utils import get_file_type_for_flatten_tool from libcoveweb2.utils import group_data_list_by +from logging import getLogger + +logger = getLogger(__name__) def create_error_file(directory: str, name: str, data: dict): """Create temporary error file""" @@ -234,7 +238,11 @@ def process(self, process_data: dict) -> dict: process_data["json_data_filename"], sample_mode=process_data['sample_mode'] ) process_data['config'] = LibCoveBODSConfig() - process_data['schema'] = SchemaBODS(process_data['data_reader'], process_data['config']) + try: + process_data['schema'] = SchemaBODS(process_data['data_reader'], process_data['config']) + except json.decoder.JSONDecodeError: + raise ValueError("JSON: Data parsing error") + logger.info("Schema version:", process_data['schema'].schema_version) # Save some to disk for templates if not os.path.exists(self.data_filename): save_data = { @@ -294,13 +302,18 @@ def process(self, process_data: dict) -> dict: os.makedirs(self.output_dir, exist_ok=True) + if os.path.isdir(process_data['schema'].pkg_schema_url): + schema = schema_registry(process_data['schema'].pkg_schema_url).contents("urn:statement") + else: + schema = process_data['schema'].pkg_schema_url + flatten_kwargs = { "output_name": self.output_dir, "root_list_path": "there-is-no-root-list-path", "root_id": "statementID", "id_name": "statementID", "root_is_list": True, - "schema": process_data['schema'].pkg_schema_url, + "schema": schema, } try: diff --git a/cove_bods/templates/cove_bods/additional_checks_table.html b/cove_bods/templates/cove_bods/additional_checks_table.html index 5931edc..463883a 100644 --- a/cove_bods/templates/cove_bods/additional_checks_table.html +++ b/cove_bods/templates/cove_bods/additional_checks_table.html @@ -13,10 +13,10 @@ {% if additional_check.type == 'entity_identifier_scheme_not_known' %} - {% trans 'The statement has an identifier scheme which is not valid.' %} + {% blocktrans %}scheme is not valid. Check the BODS documentation for guidance on identifiers. {% endblocktrans %} - {% trans 'Invalid Scheme' %}: {{ additional_check.scheme }} + scheme: {{ additional_check.scheme }} {{ additional_check.entity_statement }} @@ -25,10 +25,10 @@ {% elif additional_check.type == 'entity_statement_out_of_order' %} - {% trans 'This statement references an entity but that entity is defined after this statement.' %} + {% blocktrans %}Entity statement not in correct order. Check that the Entity statement is placed in the array before any statement referencing it.{% endblocktrans %} - {% trans 'Entity that is out of order' %}: {{ additional_check.entity_statement_out_of_order }} + {% trans 'Entity statement' %}: {{ additional_check.entity_statement_out_of_order }} {{ additional_check.seen_in_ownership_or_control_statement }} @@ -37,10 +37,10 @@ {% elif additional_check.type == 'person_statement_out_of_order' %} - {% trans 'This statement references a person but that person is defined after this statement.' %} + {% blocktrans %}Person statement not in correct order. Check that the Person statement is placed in the array before any statement referencing it.{% endblocktrans %} - {% trans 'Entity that is out of order' %}: {{ additional_check.person_statement_out_of_order }} + {% trans 'Person statement' %}: {{ additional_check.person_statement_out_of_order }} {{ additional_check.seen_in_ownership_or_control_statement }} @@ -49,7 +49,7 @@ {% elif additional_check.type == 'entity_statement_not_used_in_ownership_or_control_statement' %} - {% trans 'This Entity Statement is not used in any ownership or control statements.' %} + {% blocktrans %}Entity statement is not referenced from any Relationship statements. Check whether it should be the subject or interestedParty of a relationship.{% endblocktrans %} @@ -60,7 +60,7 @@ {% elif additional_check.type == 'person_statement_not_used_in_ownership_or_control_statement' %} - {% trans 'This Person Statement is not used in any ownership or control statements.' %} + {% blocktrans %}Person statement is not referenced from any Relationship statements. Check whether it should be the interestedParty of a relationship.{% endblocktrans %} @@ -71,10 +71,10 @@ {% elif additional_check.type == 'entity_statement_missing' %} - {% trans 'This Entity Statement is referenced from an ownership or control statement, but it is missing.' %} + {% blocktrans %}Entity statement is missing. Check whether an Entity statement is incorrectly referenced from interestedParty or subject, or whether an Entity statement is missing.{% endblocktrans %} - {% trans 'Entity that is missing' %}: {{ additional_check.entity_statement_missing }} + {% trans 'Entity statement' %}: {{ additional_check.entity_statement_missing }} {{ additional_check.seen_in_ownership_or_control_statement }} @@ -83,10 +83,10 @@ {% elif additional_check.type == 'person_statement_missing' %} - {% trans 'This Person Statement is referenced from an ownership or control statement, but it is missing.' %} + {% blocktrans %}Person statement is missing. Check whether a Person statement is incorrectly referenced from interestedParty, or whether a Person statement is missing.{% endblocktrans %} - {% trans 'Person that is missing' %}: {{ additional_check.person_statement_missing }} + {% trans 'Person statement' %}: {{ additional_check.person_statement_missing }} {{ additional_check.seen_in_ownership_or_control_statement }} @@ -95,10 +95,10 @@ {% elif additional_check.type == 'duplicate_statement_id' %} - {% trans 'This statement ID has been used more than once.' %} + {% blocktrans %}statementId value used in multiple statements. Different statements should not have the same statementId value.{% endblocktrans %} - {% trans 'Statement ID' %}: {{ additional_check.id }} + statementId: {{ additional_check.id }}   @@ -107,10 +107,10 @@ {% elif additional_check.type == 'person_birth_year_too_early' %} - {% trans 'This Person Statement has a birthday that is to early.' %} + {% blocktrans %}birthDate value is invalid. The year is too far in the past. Check that the date is correct and well formatted.{% endblocktrans %} - {% trans 'Year' %}: {{ additional_check.year }} + birthDate {% trans 'year' %}: {{ additional_check.year }} {{ additional_check.person_statement }} @@ -119,10 +119,10 @@ {% elif additional_check.type == 'person_birth_year_too_late' %} - {% trans 'This Person Statement has a birthday that is to late.' %} + {% blocktrans %}birthDate value is invalid. The date is in the future. Check that the date is correct and well formatted.{% endblocktrans %} - {% trans 'Year' %}: {{ additional_check.year }} + birthDate {% trans 'year' %}: {{ additional_check.year }} {{ additional_check.person_statement }} @@ -131,10 +131,10 @@ {% elif additional_check.type == 'wrong_address_type_used' and additional_check.statement_type == 'entity' %} - {% trans 'This Entity Statement has an address type that is not allowed in entity statements.' %} + {% blocktrans %}type of address is invalid in an Entity statement. Check that the address type is correct.{% endblocktrans %} - {% trans 'Type' %}: {{ additional_check.address_type }} + type: {{ additional_check.address_type }} {{ additional_check.statement }} @@ -143,10 +143,10 @@ {% elif additional_check.type == 'wrong_address_type_used' and additional_check.statement_type == 'person' %} - {% trans 'This Person Statement has an address type that is not allowed in person statements.' %} + {% blocktrans %}type of address is invalid in a Person statement. Check that the address type is correct.{% endblocktrans %} - {% trans 'Type' %}: {{ additional_check.address_type }} + type: {{ additional_check.address_type }} {{ additional_check.statement }} @@ -155,7 +155,7 @@ {% elif additional_check.type == 'alternative_address_with_no_other_address_types' and additional_check.statement_type == 'entity' %} - {% trans 'This Entity Statement has an alternate address but no other addresses.' %} + {% blocktrans %}type of address is 'alternative' when no other addresses are published. Check that the address type is correct.{% endblocktrans %} @@ -167,7 +167,7 @@ {% elif additional_check.type == 'alternative_address_with_no_other_address_types' and additional_check.statement_type == 'person' %} - {% trans 'This Person Statement has an alternate address but no other addresses.' %} + {% blocktrans %}type of address is 'alternative' when no other addresses are published. Check that the address type is correct.{% endblocktrans %} @@ -179,10 +179,10 @@ {% elif additional_check.type == 'component_statement_id_not_in_package' %} - {% trans 'This Ownership-or-control Statement has a component statement that is not in this package.' %} + {% blocktrans %}componentStatementIDs contains a statementID not included in this dataset. Check that this is expected.{% endblocktrans %} - {% trans 'Component Statement ID' %}: {{ additional_check.component_statement_id }} + statementID: {{ additional_check.component_statement_id }} {{ additional_check.seen_in_ownership_or_control_statement }} @@ -191,7 +191,7 @@ {% elif additional_check.type == 'ownership_or_control_statement_has_is_compontent_and_component_statement_ids' %} - {% trans 'An Ownership-or-control Statement cannot both be a component statement (isComponent) and have component statements (componentStatementIDs).' %} + {% blocktrans %}Ownership-or-control statement has an isComponent value ('true') incompatible with having its own components in componentStatementIDs.{% endblocktrans %} @@ -202,7 +202,7 @@ {% elif additional_check.type == 'statement_is_component_but_not_used_in_component_statement_ids' and additional_check.statement_type == 'person' %} - {% trans 'This Person Statement is a component (isComponent) but no primary Ownership-or-control Statement references it (from componentStatementIDs)' %} + {% blocktrans %}Person statement has an isComponent value of 'true' but does not appear in any componentStatementIDs list. Check that this is expected.{% endblocktrans %} @@ -213,7 +213,7 @@ {% elif additional_check.type == 'statement_is_component_but_not_used_in_component_statement_ids' and additional_check.statement_type == 'entity' %} - {% trans 'This Entity Statement is a component (isComponent) but no primary Ownership-or-control Statement references it (from componentStatementIDs)' %} + {% blocktrans %}Entity statement has an isComponent value of 'true' but does not appear in any componentStatementIDs list. Check that this is expected.{% endblocktrans %} @@ -224,7 +224,7 @@ {% elif additional_check.type == 'statement_is_component_but_not_used_in_component_statement_ids' and additional_check.statement_type == 'ownership_or_control' %} - {% trans 'This Ownership-or-control Statement is a component (isComponent) but no primary Ownership-or-control Statement references it (from componentStatementIDs)' %} + {% blocktrans %}Ownership-or-control statement has an isComponent value of 'true' but does not appear in any componentStatementIDs list. Check that this is expected.{% endblocktrans %} @@ -235,7 +235,7 @@ {% elif additional_check.type == 'statement_is_component_but_is_after_use_in_component_statement_id' and additional_check.statement_type == 'person' %} - {% blocktrans %}This Person Statement is a component (isComponent) and should appear before the primary Ownership-or-control Statement that references it (from componentStatementIDs).{%endblocktrans%} + {% blocktrans %}Person statement not in the correct position. As a component (isComponent 'true'), it must appear before the primary Ownership-or-control statement that references it (from componentStatementIDs @@ -246,7 +246,7 @@ {% elif additional_check.type == 'statement_is_component_but_is_after_use_in_component_statement_id' and additional_check.statement_type == 'entity' %} - {% blocktrans %}This Entity Statement is a component (isComponent) and should appear before the primary Ownership-or-control Statement that references it (from componentStatementIDs).{%endblocktrans%} + {% blocktrans %}Entity statement not in the correct position. As a component (isComponent 'true'), it must appear before the primary Ownership-or-control statement that references it (from componentStatementIDs @@ -257,7 +257,7 @@ {% elif additional_check.type == 'statement_is_component_but_is_after_use_in_component_statement_id' and additional_check.statement_type == 'ownership_or_control' %} - {% blocktrans %}This Ownership-or-control Statement is a component (isComponent) and should appear before the primary Ownership-or-control Statement that references it (from componentStatementIDs).{%endblocktrans%} + {% blocktrans %}Ownership-or-control statement not in correct order. As a component (isComponent 'true'), it must appear before the primary Ownership-or-control statement that references it (from componentStatementIDs @@ -268,10 +268,10 @@ {% elif additional_check.type == 'inconsistent_schema_version_used' and additional_check.statement_type == 'person' %} - {% blocktrans %}This Person Statement and the first statement of the submitted data reference different BODS versions.{%endblocktrans%} + {% blocktrans %}bodsVersion is different than that in the first statement of the dataset. Check that the schema versions are compatible.{% endblocktrans %} - {% trans 'Schema Version Used' %}: {{ additional_check.schema_version }} + bodsVersion: {{ additional_check.schema_version }} {{ additional_check.statement }} @@ -280,7 +280,7 @@ {% elif additional_check.type == 'inconsistent_schema_version_used' and additional_check.statement_type == 'entity' %} - {% blocktrans %}This Entity Statement and the first statement of the submitted data reference different BODS versions.{%endblocktrans%} + {% blocktrans %}bodsVersion is different than that in the first statement of the dataset. Check that the schema versions are compatible.{% endblocktrans %} {% trans 'Schema Version Used' %}: {{ additional_check.schema_version }} @@ -292,7 +292,7 @@ {% elif additional_check.type == 'inconsistent_schema_version_used' and additional_check.statement_type == 'ownership_or_control' %} - {% blocktrans %}This Ownership-or-control Statement and the first statement of the submitted data reference different BODS versions.{%endblocktrans%} + {% blocktrans %}bodsVersion is different than that in the first statement of the dataset. Check that the schema versions are compatible.{% endblocktrans %} {% trans 'Schema Version Used' %}: {{ additional_check.schema_version }} @@ -304,10 +304,10 @@ {% elif additional_check.type == 'unknown_schema_version_used' %} - {% blocktrans %}This data attempted to use a schema version that was not recognised.{%endblocktrans%} + {% blocktrans %}bodsVersion not valid. Check that the value is correctly formatted.{% endblocktrans %} - {{ additional_check.schema_version }} + bodsVersion: {{ additional_check.schema_version }} @@ -315,7 +315,7 @@ {% elif additional_check.type == 'statement_is_beneficialOwnershipOrControl_but_no_person_specified' %} - {% blocktrans %}This Ownership-or-control Statement contains interests where beneficialOwnershipOrControl is true. Therefore interestedParty must reference a Person Statement.{%endblocktrans%} + {% blocktrans %}Ownership-or-control statement asserts beneficialOwnershipOrControl is 'true' but interestedParty does not reference a Person statement. Check that information is correctly represented.{% endblocktrans %} @@ -326,7 +326,7 @@ {% elif additional_check.type == 'statement_entity_type_and_entity_sub_type_do_not_align' %} - {% blocktrans %}The specified entitySubtype is not valid for the specified entityType.{%endblocktrans%} + {% blocktrans %}entitySubtype is not valid for the specified entityType.{% endblocktrans %} @@ -337,7 +337,7 @@ {% elif additional_check.type == 'has_public_listing_information_but_has_public_listing_is_false' %} - {% blocktrans %}This Entity Statement hasPublicListing that does not exist or is false. Information has been provided under companyFilingsURLs or securitiesListings so hasPublicListing must be true.{%endblocktrans%} + {% blocktrans %}hasPublicListing has incorrect value. Value of companyFilingsURLs or securitiesListings suggests hasPublicListing must be 'true'.{% endblocktrans %} @@ -348,7 +348,7 @@ {% elif additional_check.type == 'entity_security_listing_market_identifier_code_set_but_not_operating_market_identifier_code' %} - {% blocktrans %}This Entity Statement has a security listing where marketIdentifierCode is set but operatingMarketIdentifierCode is not set.{%endblocktrans%} + {% blocktrans %}operatingMarketIdentifierCode should be set alongside marketIdentifierCode.{% endblocktrans %} @@ -359,7 +359,7 @@ {% elif additional_check.type == 'entity_security_listing_operating_market_identifier_code_set_but_not_market_identifier_code' %} - {% blocktrans %}This Entity Statement has a security listing where operatingMarketIdentifierCode is set but marketIdentifierCode is not set.{%endblocktrans%} + {% blocktrans %}marketIdentifierCode should be set alongside operatingMarketIdentifierCode.{% endblocktrans %} @@ -371,7 +371,7 @@ {# Currently this applies to 0.2 only #} - {% blocktrans %}This Person Statement has some PEP details without missing info but their PEP status has not been declared as True.{%endblocktrans%} + {% blocktrans %}hasPepStatus has incorrect value. pepStatusDetails are substantive, suggesting that hasPepStatus should be 'true'.{% endblocktrans %} @@ -383,7 +383,7 @@ {# Currently this applies to 0.3+ only #} - {% blocktrans %}This Person Statement has some PEP details but their PEP status is missing or has been declared as 'isNotPep'.{%endblocktrans%} + {% blocktrans %}politicalExposure.status has incorrect value. politicalExposure.details are substantive, suggesting that politicalExposure.status should be 'isPep'.{% endblocktrans %} @@ -396,9 +396,9 @@ {% if schema_version_used == '0.2' %} - {% blocktrans %}This Person Statement has some PEP details with missing info but their status has been declared as True.{%endblocktrans%} + {% blocktrans %}hasPepStatus has incorrect value. pepStatusDetails contains missingInfoReason, suggesting that hasPepStatus should have no value.{% endblocktrans %} {% else %} - {% blocktrans %}This Person Statement has a missingInfoReason for PEP status details, so PEP status should be declared as 'unknown'.{%endblocktrans%} + {% blocktrans %}politicalExposure.status has incorrect value. politicalExposure.details contains missingInfoReason, suggesting that status should be 'unknown'.{% endblocktrans %} {% endif %} @@ -407,78 +407,634 @@ {{ additional_check.statement }} + {% elif additional_check.type == "statement_annotation_creation_date_is_future_date" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}creationDate of an annotation is in the future. Check that dates are correctly generated and well formatted.{% endblocktrans %} + + + creationDate: {{ additional_check.creation_date }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "statement_publication_date_is_future_date" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}publicationDate is in the future. Check that dates are correctly generated and well formatted.{% endblocktrans %} + + + publicationDate: {{ additional_check.publication_date }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "statement_entity_is_component_not_in_component_details" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}Entity statement has an isComponent value of 'true' but does not appear in the componentRecords list of a later Relationship statement in the dataset. Check that component records are correctly listed and that statements are in the correct order.{% endblocktrans %} + + + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "statement_person_is_component_not_in_component_details" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}Person statement has an isComponent value of 'true' but appears in the componentRecords list of no later Relationship statement in the dataset. Check that component records are correctly listed and that statements are in the correct order.{% endblocktrans %} + + + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "statement_relationship_is_component_not_in_component_details" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}Relationship statement has an isComponent value of 'true' but does not appear in the componentRecords list of a later Relationship statement in the dataset. Check that component records are correctly listed and that statements are in the correct order.{% endblocktrans %} + + + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "statement_source_retrieved_at_future_date" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}retrievedAt is in the future. Check that dates are correctly generated and well formatted.{% endblocktrans %} + + + retrievedAt: {{ additional_check.retrieval_date }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "statement_date_is_future_date" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}statementDate is in the future. Check that dates are correctly generated and well formatted.{% endblocktrans %} + + + statementDate: {{ additional_check.statement_date }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "statement_person_birth_date_in_future" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}birthDate is in the future. Check that dates are correctly generated and well formatted.{% endblocktrans %} + + + birthDate: {{ additional_check.birth_date }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "statement_person_birth_date_too_far_in_past" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}birthDate is before 1800. Check that dates are correctly generated and well formatted.{% endblocktrans %} + + + birthDate: {{ additional_check.birth_date }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "statement_entity_dissolution_before_founding_date" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}foundingDate is later than dissolutionDate. Check that dates are correctly generated and well formatted.{% endblocktrans %} + + + foundingDate: {{ additional_check.founding_date }}
+ dissolutionDate: {{ additional_check.dissolution_date }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "statement_person_death_date_not_sensible_value" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}deathDate is incorrect. Check that the date is not before the birthDate or in the future.{% endblocktrans %} + + + deathDate: {{ additional_check.death_date }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "statement_relationship_interests_start_after_end_date" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}startDate is later than endDate in an Interest. Check that dates are correctly generated and well formatted.{% endblocktrans %} + + + startDate: {{ additional_check.start_date }}
+ endDate: {{ additional_check.end_date }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "statement_entity_securities_listings_haspubliclisting_is_false" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}hasPublicListing has incorrect value. Value of securitiesListings suggests hasPublicListing must be 'true'.{% endblocktrans %} + + + securitiesListings: {{ additional_check.securities_listings }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "statement_relationship_interests_exact_has_min_max" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}share.exact is provided, alongside a range (minimum and maximum values). Provide either an exact value, or a range.{% endblocktrans %} + + + share.exact: {{ additional_check.share }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "statement_relationship_interests_not_exact_max_greater_than_min" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}share is an invalid range. The maximum value is less than the minimum value.{% endblocktrans %} + + + share {% trans '(exclusive) minimum' %}: {{ additional_check.minval }}
+ share {% trans '(exclusive) maximum' %}: {{ additional_check.maxval }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "statement_relationship_interests_exact_max_equals_min" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}share is an invalid range. The maximum value is the same as the minimum value.{% endblocktrans %} + + + share {% trans '(exclusive) minimum' %}: {{ additional_check.minval }}
+ share {% trans '(exclusive) maximum' %}: {{ additional_check.maxval }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "statement_relationship_interests_share_min_and_exclusivemin" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}share is an invalid range. Only one of minimum and exclusiveMinimum must be provided.{% endblocktrans %} + + + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "statement_relationship_interests_share_max_and_exclusivemax" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}share is an invalid range. Only one of maximum and exclusiveMaximum must be provided.{% endblocktrans %} + + + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "statement_declaration_subject_not_exist" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}declarationSubject does not appear in the dataset. Check that the dataset is complete, or that this is expected.{% endblocktrans %} + + + declarationSubject: {{ additional_check.declaration_subject }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "statement_declaration_subject_not_entity_person" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}declarationSubject must reference an entity or person. Check that recordId values are being correctly generated and used.{% endblocktrans %} + + + declarationSubject: {{ additional_check.record_id }}
+ recordType of {{ additional_check.record_id }}: {{ additional_check.record_type }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "multiple_statements_in_series_with_record_status_new" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}recordStatus reported as 'new' in multiple Statements for a single Record. Check that recordStatus is ‘new’ only the first time a Statement is published for the record.{% endblocktrans %} + + + recordId: {{ additional_check.record_id }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "statement_with_record_status_new_must_be_first" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}statementDate is too early. The recordStatus is 'updated' or 'closed' but statementDate is earlier than that of the corresponding 'new' Statement.{% endblocktrans %} + + + recordId: {{ additional_check.record_id }}
+ statementId: {{ additional_check.statement_id }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "multiple_statements_in_series_with_record_status_closed" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}recordStatus reported as 'closed' in multiple Statements for a single Record. Check that recordStatus is 'closed' only the final time a Statement is published for a record.{% endblocktrans %} + + + recordId: {{ additional_check.record_id }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "statement_with_record_status_closed_must_be_last" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}statementDate is too late. recordStatus is 'new' or 'updated' but statementDate is later than that of the corresponding 'closed' Statement.{% endblocktrans %} + + + recordId:: {{ additional_check.record_id }}
+ statementId:: {{ additional_check.statement_id }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "statements_in_series_with_different_record_types" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}recordType varies across Statements for the same record. Check that Statements relating to the same record all have the same type.{%endblocktrans%} + + + recordId: {{ additional_check.record_id }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "component_record_is_statement_id" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}componentRecords contains a statement ID value. componentRecords entries must be record IDs.{% endblocktrans %} + + + componentRecords: {{ additional_check.component_id }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "component_record_id_not_in_dataset" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}componentRecords contains a recordId not included in this dataset. Check that this is expected.{% endblocktrans %} + + + componentRecords: {{ additional_check.component_id }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "subject_must_be_record_id" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}subject has unexpected value. subject must be either a record ID for a Statement in the dataset or an Unspecified Record object.{% endblocktrans %} + + + subject: {{ additional_check.subject }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "subject_can_only_refer_to_entity" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}subject must be the recordId of an entity (not a person or relationship). Check that recordId values are being correctly generated and used.{% endblocktrans %} + + + subject: {{ additional_check.subject }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "interested_party_must_be_record_id" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}interestedParty not recognised. Check that the value matches a recordId in the dataset or is an Unspecified Record object.{% endblocktrans %} + + + interestedParty: {{ additional_check.interested_party }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "interested_party_can_only_refer_to_entity_or_person" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}interestedParty is invalid. The value should be a recordId for a person or an entity in the dataset (not a relationship).{% endblocktrans %} + + + interestedParty: {{ additional_check.interested_party }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "interest_beneficial_ownership_interested_party_not_person" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}beneficialOwnershipOrControl is 'true' but interestedParty is not a person. Check that the interested party is correct and that beneficialOwnershipOrControl is used correctly.{% endblocktrans %} + + + interestedParty: {{ additional_check.interested_party }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "annotation_statement_pointer_target_invalid" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}statementPointerTarget is invalid. Check that it is a valid JSON pointer and that it points to an existing field in the Statement.{% endblocktrans %} + + + statementPointerTarget: {{ additional_check.pointer }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "relationship_interests_subject_should_be_entity_nomination_arrangement" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}subject has unexpected type. Interests in this Relationship statement suggest that the subject's entityType.subtype should be 'nomination'.{% endblocktrans %} + + + entityType.subtype: {{ additional_check.subject_record_subtype }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "relationship_interests_subject_should_be_entity_trust" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}subject has unexpected type. Interests in this Relationship statement suggest that the subject's entityType.subtype should be 'trust'.{% endblocktrans %} + + + entityType.subtype: {{ additional_check.subject_record_subtype }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "relationship_subject_not_before_relationship_in_dataset" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}subject not found. The subject must match the recordId of at least one prior Statement in the dataset. Check that Statements are ordered correctly.{% endblocktrans %} + + + subject: {{ additional_check.subject_id }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "relationship_interested_party_not_before_relationship_in_dataset" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}interestedParty not found. The interested party must match the recordId of at least one prior Statement in the dataset. Check that Statements are ordered correctly.{% endblocktrans %} + + + interestedParty: {{ additional_check.interested_party_id }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "person_identifiers_invalid_composition" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}scheme has incorrect formatting. Check the field description for guidance.{% endblocktrans %} + + + scheme: {{ additional_check.scheme }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "person_identifiers_no_valid_iso_3166_1_alpha_3_code" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}scheme contains an unrecognised jurisdiction. An ISO 3166-1 3-digit country code is expected. Check the field description for guidance.{% endblocktrans %} + + + scheme: {{ additional_check.scheme }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "person_identifiers_not_passport_taxid_idcard" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}scheme has unrecognised type. 'PASSPORT', 'TAXID' or 'IDCARD' is expected. Check the field description for guidance.{% endblocktrans %} + + + scheme: {{ additional_check.scheme }} + + + {{ additional_check.statement }} + + + {% elif additional_check.type == "entity_identifiers_not_known_scheme" %} + {# Currently this applies to 0.4+ #} + + + {% blocktrans %}scheme is unrecognised. A code from org-id.guide is expected. Check the field description for guidance.{% endblocktrans %} + + + scheme: {{ additional_check.scheme }} + + + {{ additional_check.statement }} + + + {% endif %} {% endfor %} {% if not statistics.count_ownership_or_control_statement_with_at_least_one_interest_beneficial %} - {% blocktrans %}No individuals are disclosed as beneficial owners. beneficialOwnershipOrControl must be set to true within an Interest object to indicate that the interested party is a beneficial owner.{%endblocktrans%} + {% blocktrans %}beneficialOwnershipOrControl expected to be 'true' in at least one Relationship statement with a person as an interested party. If this dataset contains beneficial owners, check that beneficialOwnershipOrControl is correctly used.{% endblocktrans %} - {% blocktrans %}All Ownership-or-control statements{%endblocktrans%} + {% blocktrans %}All Relationship statements.{% endblocktrans %} {% endif %} {% for check_not_run_in_sample_mode in checks_not_run_in_sample_mode %} {% if check_not_run_in_sample_mode == "entity_statement_missing" %} - {% trans 'This Entity Statement is referenced from an ownership or control statement, but it is missing.' %} + + {% blocktrans %}Entity statement is missing. Check whether an Entity statement is incorrectly referenced from interestedParty or subject, or whether an Entity statement is missing.{% endblocktrans %} + This check is not carried out in Sample mode. {% elif check_not_run_in_sample_mode == "person_statement_out_of_order" %} - {% trans 'This statement references a person but that person is defined after this statement.' %} + + {% blocktrans %}Person statement not in correct order. Check that the Person statement is placed in the array before any statement referencing it.{% endblocktrans %} + This check is not carried out in Sample mode. {% elif check_not_run_in_sample_mode == "statement_is_component_but_is_after_use_in_component_statement_id" %} - {% trans 'This Statement is a component (isComponent) and should appear before the primary Ownership-or-control Statement that references it (from componentStatementIDs).' %} + + {% blocktrans %}Ownership-or-control statement not in correct order. As a component (isComponent 'true'), it must appear before the primary Ownership-or-control statement that references it (from componentStatementIDs This check is not carried out in Sample mode. {% elif check_not_run_in_sample_mode == "statement_is_component_but_not_used_in_component_statement_ids" %} - {% trans 'This Statement is a component (isComponent) but no primary Ownership-or-control Statement references it (from componentStatementIDs)' %} + + {% blocktrans %}Ownership-or-control statement has an isComponent value of 'true' but appears in no componentStatementIDs list. Check that this is expected.{% endblocktrans %} + This check is not carried out in Sample mode. {% elif check_not_run_in_sample_mode == "person_statement_missing" %} - {% trans 'This Person Statement is referenced from an ownership or control statement, but it is missing.' %} + + {% blocktrans %}Person statement is missing. Check whether a Person statement is incorrectly referenced from interestedParty, or whether a Person statement is missing.{% endblocktrans %} + This check is not carried out in Sample mode. {% elif check_not_run_in_sample_mode == "person_statement_not_used_in_ownership_or_control_statement" %} - {% trans 'This Person Statement is not used in any ownership or control statements.' %} + + {% blocktrans %}Person statement is not referenced from any Relationship statements. Check whether it should be the interestedParty of a relationship.{% endblocktrans %} + This check is not carried out in Sample mode. {% elif check_not_run_in_sample_mode == "entity_statement_not_used_in_ownership_or_control_statement" %} - {% trans 'This Entity Statement is not used in any ownership or control statements.' %} + + {% blocktrans %}Entity statement is not referenced from any Relationship statements. Check whether it should be the subject or interestedParty of a relationship.{% endblocktrans %} + This check is not carried out in Sample mode. {% elif check_not_run_in_sample_mode == "duplicate_statement_id" %} - {% trans 'This statement ID has been used more than once.' %} + + {% blocktrans %}statementId value used in multiple statements. Different statements should not have the same statementId value.{% endblocktrans %} + This check is not carried out in Sample mode. {% elif check_not_run_in_sample_mode == "entity_statement_out_of_order" %} - {% trans 'This statement references an entity but that entity is defined after this statement.' %} + + {% blocktrans %}Entity statement not in correct order. Check that the Entity statement is placed in the array before any statement referencing it.{% endblocktrans %} + This check is not carried out in Sample mode. {% elif check_not_run_in_sample_mode == "component_statement_id_not_in_package" %} - {% trans 'This Ownership-or-control Statement has a component statement that is not in this package.' %} + + {% blocktrans %}componentStatementIDs contains a statementID not included in this dataset. Check that this is expected.{% endblocktrans %} + This check is not carried out in Sample mode. diff --git a/cove_bods/templates/cove_bods/additional_fields_table.html b/cove_bods/templates/cove_bods/additional_fields_table.html new file mode 100644 index 0000000..2031604 --- /dev/null +++ b/cove_bods/templates/cove_bods/additional_fields_table.html @@ -0,0 +1,93 @@ +{% load i18n %} + + + + + + + + + + + + + + + {% for full_path, info in additional_fields.items %} + {% if info.root_additional_field %} + + + + + + + + {% endif %} + {% endfor %} + +
{% trans 'Field Name' %}{% trans 'Field Path' %}{% trans 'Usage Count' %}{% trans 'First 3 Values' %}{% trans 'Child Fields' %}
+ {{ info.field_name }} + + {{ full_path }} + + {{ info.count }} + +
    + {% for example in info.examples|slice:":3" %} +
  • + {{ example }} +
    • + {% endfor %} +
+ 		+ {% if info.additional_field_descendance %} + {{info.additional_field_descendance|length}} + {% trans "(See child fields)" %} + {% endif %} +
+ +{% for parent_full_path, parent_info in additional_fields.items %} + {% if parent_info.root_additional_field and parent_info.additional_field_descendance %} + + {% endif %} +{% endfor %} + diff --git a/cove_bods/templates/cove_bods/explore.html b/cove_bods/templates/cove_bods/explore.html index a53e4b2..478217f 100644 --- a/cove_bods/templates/cove_bods/explore.html +++ b/cove_bods/templates/cove_bods/explore.html @@ -142,7 +142,7 @@

- {% if additional_fields_count %} + {% if additional_fields_count or any_additional_fields_exist %}
@@ -152,7 +152,7 @@

- {% include "additional_fields_table.html" %} + {% include "cove_bods/additional_fields_table.html" %}
{% else %} diff --git a/cove_project/urls.py b/cove_project/urls.py index 7d56bb5..2b6fa78 100644 --- a/cove_project/urls.py +++ b/cove_project/urls.py @@ -1,4 +1,3 @@ -from django.conf.urls import url from django.conf.urls.static import static from django.conf import settings from libcoveweb2.urls import urlpatterns @@ -6,7 +5,7 @@ from django.urls import re_path urlpatterns += [re_path(r"^$", cove_bods.views.NewInput.as_view(), name="index")] -urlpatterns += [url(r'^data/(.+)$', cove_bods.views.ExploreBODSView.as_view(), name='explore')] +urlpatterns += [re_path(r'^data/(.+)$', cove_bods.views.ExploreBODSView.as_view(), name='explore')] urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) diff --git a/requirements.in b/requirements.in index 226c357..7cecb73 100644 --- a/requirements.in +++ b/requirements.in @@ -1,6 +1,6 @@ dealer sentry-sdk -Django>3.2,<3.3 +Django>5.0,<5.1 jsonschema libcovebods>=0.15.0 libcoveweb2>=0.1.0 diff --git a/requirements.txt b/requirements.txt index 4d6e1b2..8401062 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,8 +8,6 @@ amqp==5.1.1 # via kombu asgiref==3.7.2 # via django -async-timeout==4.0.2 - # via redis attrs==23.1.0 # via jsonschema backports-datetime-fromisoformat==2.0.0 @@ -46,7 +44,7 @@ dealer==2.1.0 # via -r requirements.in defusedxml==0.7.1 # via odfpy -django==3.2.19 +django==5.0.7 # via # -r requirements.in # django-bootstrap3 @@ -59,7 +57,8 @@ django-environ==0.10.0 # via libcoveweb2 et-xmlfile==1.1.0 # via openpyxl -flattentool==0.20.1 +#flattentool==0.20.1 +flattentool @ git+https://github.com/OpenDataServices/flatten-tool.git@handle_bods_0.4 # via -r requirements.in gunicorn==20.1.0 # via -r requirements.in @@ -71,17 +70,20 @@ ijson==3.2.0.post0 # libcovebods jsonref==1.1.0 # via flattentool -jsonschema==4.9.1 +jsonschema==4.23.0 # via # -r requirements.in # libcovebods kombu==5.2.4 # via celery -libcove2==0.1.0 +#libcove2==0.1.0 +libcove2 @ git+https://github.com/OpenDataServices/lib-cove-2.git@handle_new_jsonschema # via libcovebods -libcovebods==0.15.0 +#libcovebods==0.15.0 +libcovebods @ git+https://github.com/openownership/lib-cove-bods.git@add_bods_0.4 # via -r requirements.in -libcoveweb2==0.1.0 +#libcoveweb2==0.1.0 +libcoveweb2 @ git+https://github.com/OpenDataServices/lib-cove-web-2.git@handle_bods_0.4 # via -r requirements.in lxml==4.9.2 # via flattentool @@ -106,7 +108,6 @@ python-dateutil==2.8.2 pytz==2023.3 # via # celery - # django # flattentool # libcovebods redis==4.5.5 @@ -121,7 +122,7 @@ rfc3987==1.3.8 # via libcovebods schema==0.7.5 # via flattentool -sentry-sdk==1.24.0 +sentry-sdk==2.14.0 # via # -r requirements.in # libcoveweb2 diff --git a/requirements_dev.txt b/requirements_dev.txt index fcb0703..f6b3178 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -16,10 +16,6 @@ async-generator==1.10 # via # trio # trio-websocket -async-timeout==4.0.2 - # via - # -r requirements.txt - # redis attrs==23.1.0 # via # -r requirements.txt @@ -89,7 +85,7 @@ defusedxml==0.7.1 # via # -r requirements.txt # odfpy -django==3.2.19 +django==5.0.7 # via # -r requirements.txt # django-bootstrap3 @@ -130,7 +126,7 @@ jsonref==1.1.0 # via # -r requirements.txt # flattentool -jsonschema==4.9.1 +jsonschema==4.23.0 # via # -r requirements.txt # libcovebods @@ -138,11 +134,13 @@ kombu==5.2.4 # via # -r requirements.txt # celery -libcove2==0.1.0 +#libcove2==0.1.0 +libcove2 @ git+https://github.com/OpenDataServices/lib-cove-2.git@handle_new_jsonschema # via # -r requirements.txt # libcovebods -libcovebods==0.15.0 +#libcovebods==0.15.0 +libcovebods @ git+https://github.com/openownership/lib-cove-bods.git@add_bods_0.4 # via -r requirements.txt libcoveweb2==0.1.0 # via -r requirements.txt @@ -213,7 +211,6 @@ pytz==2023.3 # via # -r requirements.txt # celery - # django # flattentool # libcovebods redis==4.5.5 @@ -239,7 +236,7 @@ schema==0.7.5 # flattentool selenium==4.8.0 # via -r requirements_dev.in -sentry-sdk==1.24.0 +sentry-sdk==2.14.0 # via # -r requirements.txt # libcoveweb2