diff --git a/tap_typeform/schemas/questions.json b/tap_typeform/schemas/questions.json index 93ac608..24820e3 100644 --- a/tap_typeform/schemas/questions.json +++ b/tap_typeform/schemas/questions.json @@ -1,18 +1,36 @@ { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "form_id": { - "type": ["null", "string"] - }, - "question_id": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "ref": { - "type": ["null", "string"] + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "form_id": { + "type": ["null", "string"] + }, + "question_id": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "ref": { + "type": ["null", "string"] + }, + "sub_questions": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "question_id": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "ref": { + "type": ["null", "string"] + } + } + } } - } +}, +"key_properties": ["form_id","question_id"] } diff --git a/tap_typeform/streams.py b/tap_typeform/streams.py index ec47160..7c23c87 100644 --- a/tap_typeform/streams.py +++ b/tap_typeform/streams.py @@ -211,6 +211,7 @@ def sync_obj(self, client, state, catalogs, max_bookmark = bookmark for records in self.get_forms(client): + max_bookmark = self.write_records(records, catalogs, selected_stream_ids, None, max_bookmark, state, start_date) write_bookmarks(self.tap_stream_id, selected_stream_ids, None, max_bookmark, state) @@ -223,15 +224,35 @@ class Questions(FullTableStream): endpoint = 'forms/{}' data_key = 'fields' + def fetch_sub_questions(self, row): + '''This function fetches records for each sub_question in a question group and returns a list of fetched sub_questions''' + sub_questions = [] #Creating blank list to accommodate each sub-question's records + + #Appending each sub-question to the list + for question in row['properties'].get('fields',[]): + sub_questions.append({ + "question_id": question['id'], + "title": question['title'], + "ref": question['ref'] + }) + + return sub_questions + def add_fields_at_1st_level(self, record, additional_data={}): """ Add additional data and nested fields to top level """ + sub_questions ={} #Creating a blank dictionary to store records of sub_questions,if any + + #If type of question is group, i.e. it has sub_questions, then fetch those sub_questions + if record.get('type') == 'group': + sub_questions['sub_questions'] = self.fetch_sub_questions(record) + #If sub_questions are fetched then add those in this field and display the same, else don't display this field record.update({ "form_id": additional_data['form_id'], "question_id": record['id'] - }) + }|sub_questions) class SubmittedLandings(IncrementalStream): tap_stream_id = 'submitted_landings' diff --git a/tests/unittests/test_sub_questions.py b/tests/unittests/test_sub_questions.py new file mode 100644 index 0000000..5af2790 --- /dev/null +++ b/tests/unittests/test_sub_questions.py @@ -0,0 +1,44 @@ +import unittest +from unittest import mock +from tap_typeform.streams import Questions +from parameterized import parameterized + +class TestSubQuestionTest(unittest.TestCase): + @parameterized.expand([ + ["Question group have sub-questions", + { + 'case1a': 'value1a', 'case1b': 'value1b', 'case1c': 'value1c', + 'properties': { + 'description': 'Group question', + 'case1': 'value1','case2':'value2', + 'fields': [ + { + 'id': 'id1', 'title': 'title1', 'ref': 'ref1' + }, + { + 'id': 'id2','title': 'title2', 'ref': 'title2' + } + ] + }, + 'type': 'group' + }, + [{'question_id': 'id1', 'title': 'title1', 'ref': 'ref1'}, {'question_id': 'id2', 'title': 'title2', 'ref': 'title2'}] + ], + ["Question group don't have sub-questions", + { + 'case2a': 'value2a', 'case2b': 'value2b', 'case2c': 'value2c', + 'properties': { + 'description': 'Group question', 'case3':'value3', 'case4':'value4' + }, + 'type': 'group' + }, + [] + ] + ]) + def test_fetched_sub_question(self, test_name, test_value, expected_value): + """ + To verify that we are getting expected response or not for question group + """ + question_stream = Questions() + self.assertEqual(expected_value, question_stream.fetch_sub_questions(test_value)) +