Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Ingest intl csvs #83

Open
wants to merge 16 commits into
base: main
Choose a base branch
from
Open

Ingest intl csvs #83

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
acc8611
Add logic and tests to improve separation between API and worldbase data
lgarvey Apr 9, 2020
5957bd0
Add blank=True to company source fields
lgarvey Apr 9, 2020
dd4b490
Modify the WB ingest process so that we can import UK and Intl format…
lgarvey Apr 13, 2020
ddd06f4
Add national id numbers and mappings
lgarvey Apr 13, 2020
dbe0059
handle unmapped reg numbers appropriately
lgarvey Apr 13, 2020
7f8cf6e
continue processing if csv column count is wrong
lgarvey Apr 30, 2020
f3e1237
Add interrim ingest logging output and check ingest timestamp
lgarvey May 5, 2020
a5624d6
Add gunicorn worker timeout
lgarvey Jun 10, 2020
b8ea296
fixup! Add logic and tests to improve separation between API and worl…
lgarvey Jun 10, 2020
a5693c5
Add db indexes for company timestamp fields
lgarvey Jun 24, 2020
72e7ba9
Add missing APM vars to pytest.ini
lgarvey Jun 24, 2020
6a732e4
Bump codecov
lgarvey Jun 24, 2020
98dde0f
Disable APM when running tests
lgarvey Jun 24, 2020
987a9f1
Fix pytest bug on py 3.8
lgarvey Nov 19, 2020
377be24
Add extra tests
lgarvey Dec 8, 2020
91135b5
add blank=True to date fields
lgarvey Dec 8, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Procfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
web: python manage.py migrate && gunicorn -b 0.0.0.0:$PORT config.wsgi:application
web: python manage.py migrate && gunicorn -b 0.0.0.0:$PORT config.wsgi:application --timeout 120
celery_worker: celery -A config worker -l info
celery_beat: celery -A config beat -l info -S django
94 changes: 47 additions & 47 deletions api/tests/test_views.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -396,49 +396,49 @@ def test_no_params_returns_all_results(self, auth_client):
"""
duns_numbers = [
ChangeRequestFactory(
changes={'primary_name': 'bar'},
status='pending',
changes={'primary_name': 'bar'},
status='pending',
id='00000000-0000-0000-0000-000000000001'
).duns_number,
).duns_number,
ChangeRequestFactory(
changes={'primary_name': 'baz'},
status='submitted',
changes={'primary_name': 'baz'},
status='submitted',
id='00000000-0000-0000-0000-000000000002'
).duns_number
]

response = auth_client.get(
reverse('api:change-request'),
{},
)

assert response.status_code == 200

result_data = response.json()
assert result_data['count'] == 2

expected_result_data = {
'count': 2,
'next': None,
'previous': None,
'count': 2,
'next': None,
'previous': None,
'results': [
{
'id': '00000000-0000-0000-0000-000000000001',
'duns_number': '000000000',
'changes': {'primary_name': 'bar'},
'status': 'pending',
'duns_number': '000000000',
'changes': {'primary_name': 'bar'},
'status': 'pending',
'created_on': '2020-05-18T12:00:01Z'
},
},
{
'id': '00000000-0000-0000-0000-000000000002',
'duns_number': '000000001',
'changes': {'primary_name': 'baz'},
'status': 'submitted',
'duns_number': '000000001',
'changes': {'primary_name': 'baz'},
'status': 'submitted',
'created_on': '2020-05-18T12:00:01Z'
}
]
}

assert result_data == expected_result_data

@freeze_time('2020-05-18 12:00:01 UTC')
Expand All @@ -447,14 +447,14 @@ def test_only_returns_pending_requests(self, auth_client):
Test that all pending change requests can be returned, for any company.
"""
pending_duns_numbers = [
ChangeRequestFactory(changes={'primary_name': 'test1'}, status='pending', id='00000000-0000-0000-0000-000000000001'),
ChangeRequestFactory(changes={'primary_name': 'test2'}, status='pending', id='00000000-0000-0000-0000-000000000002'),
ChangeRequestFactory(changes={'primary_name': 'test1'}, status='pending', id='00000000-0000-0000-0000-000000000001'),
ChangeRequestFactory(changes={'primary_name': 'test2'}, status='pending', id='00000000-0000-0000-0000-000000000002'),
]

ChangeRequestFactory(changes={'primary_name': 'test3'}, status='submitted')
ChangeRequestFactory(changes={'primary_name': 'test4'}, status='submitted')


response = auth_client.get(
reverse('api:change-request'),
{'status': 'pending'},
Expand All @@ -468,43 +468,43 @@ def test_only_returns_pending_requests(self, auth_client):
print(result_data)

expected_result_data = {
'count': 2,
'next': None,
'previous': None,
'count': 2,
'next': None,
'previous': None,
'results': [
{
'id': '00000000-0000-0000-0000-000000000002',
'duns_number': '000000003',
'changes': {'primary_name': 'test2'},
'status': 'pending',
'duns_number': '000000003',
'changes': {'primary_name': 'test2'},
'status': 'pending',
'created_on': '2020-05-18T12:00:01Z'
},
{
'id': '00000000-0000-0000-0000-000000000001',
'duns_number': '000000002',
'changes': {'primary_name': 'test1'},
'status': 'pending',
'duns_number': '000000002',
'changes': {'primary_name': 'test1'},
'status': 'pending',
'created_on': '2020-05-18T12:00:01Z'
}

]
}

assert result_data == expected_result_data

def test_only_returns_submitted_requests(self, auth_client):
"""
Test that all submitted change requests can be returned, for any company.
"""
submitted_duns_numbers = [
ChangeRequestFactory(changes={'primary_name': 'test1'}, status='submitted'),
ChangeRequestFactory(changes={'primary_name': 'test2'}, status='submitted'),
ChangeRequestFactory(changes={'primary_name': 'test1'}, status='submitted'),
ChangeRequestFactory(changes={'primary_name': 'test2'}, status='submitted'),
]

ChangeRequestFactory(changes={'primary_name': 'test3'}, status='pending')
ChangeRequestFactory(changes={'primary_name': 'test4'}, status='pending')


response = auth_client.get(
reverse('api:change-request'),
{'status': 'submitted'},
Expand All @@ -515,16 +515,16 @@ def test_only_returns_submitted_requests(self, auth_client):
result_data = response.json()
assert len(result_data['results']) == 2
assert result_data['count'] == 2

def test_only_returns_pending_requests_with_specific_duns_number(self, auth_client):
"""
Test that all pending change requests are returned for a specific company.
"""
change_requests = [
ChangeRequestFactory(changes={'primary_name': 'test1'}, status='pending', duns_number='123456789', id='00000000-0000-0000-0000-000000000001'),
ChangeRequestFactory(changes={'primary_name': 'test2'}, status='pending', duns_number='123456789', id='00000000-0000-0000-0000-000000000002'),
ChangeRequestFactory(changes={'primary_name': 'test3'}, status='pending', duns_number='123056789', id='00000000-0000-0000-0000-000000000003'),
ChangeRequestFactory(changes={'primary_name': 'test4'}, status='pending', duns_number='123406780', id='00000000-0000-0000-0000-000000000004'),
ChangeRequestFactory(changes={'primary_name': 'test1'}, status='pending', duns_number='123456789', id='00000000-0000-0000-0000-000000000001'),
ChangeRequestFactory(changes={'primary_name': 'test2'}, status='pending', duns_number='123456789', id='00000000-0000-0000-0000-000000000002'),
ChangeRequestFactory(changes={'primary_name': 'test3'}, status='pending', duns_number='123056789', id='00000000-0000-0000-0000-000000000003'),
ChangeRequestFactory(changes={'primary_name': 'test4'}, status='pending', duns_number='123406780', id='00000000-0000-0000-0000-000000000004'),
]

test_ids = []
Expand All @@ -547,16 +547,16 @@ def test_only_returns_pending_requests_with_specific_duns_number(self, auth_clie

for result in result_data['results']:
assert result['id'] in test_ids

def test_only_duns_param_returns_all_results(self, auth_client):
"""
Test that all change requests for a company can be returned.
"""
change_requests = [
ChangeRequestFactory(changes={'primary_name': 'test1'}, duns_number='123456789', id='00000000-0000-0000-0000-000000000001'),
ChangeRequestFactory(changes={'primary_name': 'test2'}, duns_number='123456789', id='00000000-0000-0000-0000-000000000002'),
ChangeRequestFactory(changes={'primary_name': 'test3'}, duns_number='123456789', id='00000000-0000-0000-0000-000000000003'),
ChangeRequestFactory(changes={'primary_name': 'test4'}, duns_number='123456700', id='00000000-0000-0000-0000-000000000004'),
ChangeRequestFactory(changes={'primary_name': 'test1'}, duns_number='123456789', id='00000000-0000-0000-0000-000000000001'),
ChangeRequestFactory(changes={'primary_name': 'test2'}, duns_number='123456789', id='00000000-0000-0000-0000-000000000002'),
ChangeRequestFactory(changes={'primary_name': 'test3'}, duns_number='123456789', id='00000000-0000-0000-0000-000000000003'),
ChangeRequestFactory(changes={'primary_name': 'test4'}, duns_number='123456700', id='00000000-0000-0000-0000-000000000004'),
]

test_ids = []
Expand Down
Loading