Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix 1611 incorrect errors when validating data with a missiing required column #1620

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions frictionless/resources/table.py
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,6 @@ def row_stream():
for row_number, cells in enumerated_content_stream:
self.stats.rows += 1

# Create row
row = Row(
cells,
field_info=field_info,
Expand Down Expand Up @@ -387,7 +386,16 @@ def row_stream():
# Yield row
yield row

# Crreate row stream
# NB: missing required labels are not included in the
# field_info parameter used for row creation
if self.detector.schema_sync:
for field in self.schema.fields:
if field.name not in self.labels and field.name in field_info["names"]:
field_index = field_info["names"].index(field.name)
del field_info["names"][field_index]
del field_info["objects"][field_index]
del field_info["mapping"][field.name]
# # Create row stream
self.__row_stream = row_stream()

# Read
Expand Down
69 changes: 69 additions & 0 deletions tests/validator/resource/test_schema.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
from copy import deepcopy

import pytest

import frictionless
from frictionless import Checklist, Detector, FrictionlessException, Schema, fields
from frictionless.resources import TableResource

Expand Down Expand Up @@ -304,3 +307,69 @@ def test_resource_validate_less_actual_fields_with_required_constraint_issue_950
[3, 3, "constraint-error"],
[3, 3, "missing-cell"],
]


def test_resource_with_missing_required_header_with_schema_sync_is_true_issue_1611():
schema_descriptor_1 = {
"$schema": "https://frictionlessdata.io/schemas/table-schema.json",
"fields": [
{
"name": "A",
"title": "Field A",
"type": "string",
"constraints": {"required": True},
},
{"name": "B", "title": "Field B", "type": "string"},
{"name": "C", "title": "Field C", "type": "string"},
],
}

schema_descriptor_2 = deepcopy(schema_descriptor_1)
# Add required constraint on "C" field
schema_descriptor_2["fields"][2]["constraints"] = {"required": True}

test_cases = [
{
"schema": schema_descriptor_1,
"source": [["B", "C"], ["b", "c"]],
"expected_flattened_report": [
[None, 3, "A", "missing-label"],
],
},
{
"schema": schema_descriptor_2,
"source": [["B"], ["b"]],
"expected_flattened_report": [
[None, 2, "A", "missing-label"],
[None, 3, "C", "missing-label"],
],
},
{
"schema": schema_descriptor_2,
"source": [
["A", "B"],
["a", "b"],
["a1"],
["a2", "b2"],
[],
["a3", "b3", "c3"],
],
"expected_flattened_report": [
[None, 3, "C", "missing-label"],
[3, 2, "B", "missing-cell"],
[5, None, None, "blank-row"],
[6, 3, "", "extra-cell"],
],
},
]
for tc in test_cases:
schema = Schema.from_descriptor(tc["schema"])
resource = TableResource(
tc["source"], schema=schema, detector=Detector(schema_sync=True)
)
report = frictionless.validate(resource)
print(report.flatten(["rowNumber", "fieldNumber", "fieldName", "type"]))
assert (
report.flatten(["rowNumber", "fieldNumber", "fieldName", "type"])
== tc["expected_flattened_report"]
)
Loading