diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl index 27bf2dc60ca84..9989f84792de5 100644 --- a/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl @@ -1,9 +1,9 @@ {"stream": "users", "data": {"object": "user", "id": "5612c094-99ec-4ba3-ac7f-df8d84c8d6be", "name": "Sherif Nada", "avatar_url": "https://s3-us-west-2.amazonaws.com/public.notion-static.com/305f7efc-2862-4342-ba99-5023f3e34717/6246757.png", "type": "person", "person": {"email": "sherif@airbyte.io"}}, "emitted_at": 1697023279924} {"stream": "users", "data": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}, "emitted_at": 1697023279925} {"stream": "users", "data": {"object": "user", "id": "c1ff0160-b2af-497a-aab7-8b61e625e4e3", "name": "Gil Cho", "avatar_url": "https://lh3.googleusercontent.com/a/ALm5wu0ElXfvy3YfVUyRn-aB9EZy5AZ1ougHuNyCGmO2=s100", "type": "person", "person": {"email": "gil@airbyte.io"}}, "emitted_at": 1697023279925} -{"stream": "databases", "data": {"object": "database", "id": "b75d2e55-cc80-4afa-a273-c78178ac6b3f", "cover": null, "icon": {"type": "emoji", "emoji": "\ud83d\ude4b"}, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_time": "2021-10-19T13:33:00.000Z", "title": [{"type": "text", "text": {"content": "Engineering Directory ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Engineering Directory ", "href": null}], "description": [{"type": "text", "text": {"content": "Have a question about part of our codebase?\nFind the most knowledgeable person in this directory.\nLearn more about ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Have a question about part of our codebase?\nFind the most knowledgeable person in this directory.\nLearn more about ", "href": null}, {"type": "text", "text": {"content": "Notion databases", "link": {"url": "https://www.notion.so/notion/Database-101-build-and-view-fd8cd2d212f74c50954c11086d85997e"}}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Notion databases", "href": "https://www.notion.so/notion/Database-101-build-and-view-fd8cd2d212f74c50954c11086d85997e"}, {"type": "text", "text": {"content": ".", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": ".", "href": null}], "is_inline": false, "properties": [{"name": "Date Added", "value": {"id": "%2Fkv%22", "name": "Date Added", "type": "created_time", "created_time": {}}}, {"name": "Notes", "value": {"id": "mq%22D", "name": "Notes", "type": "rich_text", "rich_text": {}}}, {"name": "Person", "value": {"id": "uiZ%26", "name": "Person", "type": "people", "people": {}}}, {"name": "Name", "value": {"id": "title", "name": "Name", "type": "title", "title": {}}}], "parent": {"type": "block_id", "block_id": "b81f8caf-3ec4-4455-9a0b-25c2bd3b60cb"}, "url": "https://www.notion.so/b75d2e55cc804afaa273c78178ac6b3f", "public_url": null, "archived": false}, "emitted_at": 1697023281967} -{"stream": "databases", "data": {"object": "database", "id": "fbff7d4e-eca4-4432-91e6-ec64ba4b5a98", "cover": null, "icon": null, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_time": "2021-10-19T13:33:00.000Z", "title": [{"type": "text", "text": {"content": "Questions", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Questions", "href": null}], "description": [], "is_inline": true, "properties": [{"name": "Difficulty", "value": {"id": "'i6%2F", "name": "Difficulty", "type": "select", "select": {"options": [{"id": "f00068b9-7612-45da-91ad-1a7b1d259375", "name": "Easy", "color": "green"}, {"id": "8e244bfe-d4c7-48c5-9088-ffd6926b4ba0", "name": "Medium", "color": "yellow"}, {"id": "9ab57ef4-eab1-4b20-a502-047610b5c97d", "name": "Hard", "color": "red"}]}}}, {"name": "Skills", "value": {"id": "K%3AtR", "name": "Skills", "type": "multi_select", "multi_select": {"options": [{"id": "72f4d134-a773-48c1-ba3d-b529f55c6818", "name": "Front end", "color": "default"}, {"id": "c20f5d57-3e35-4b39-b556-05071203cc1a", "name": "Backend", "color": "default"}, {"id": "31d5735c-d6ba-4bd7-940f-bdcb36091c02", "name": "Architecture", "color": "default"}, {"id": "0398de54-af68-4c3a-9953-3788e8eaadbf", "name": "Algorithms", "color": "default"}, {"id": "df9dff09-7dea-4409-a10f-b5e2b546ad94", "name": "Data Structures", "color": "default"}]}}}, {"name": "Question Name", "value": {"id": "title", "name": "Question Name", "type": "title", "title": {}}}], "parent": {"type": "page_id", "page_id": "4999109d-1b7b-41a2-abb4-84f6b961ee74"}, "url": "https://www.notion.so/fbff7d4eeca4443291e6ec64ba4b5a98", "public_url": null, "archived": false}, "emitted_at": 1697023281968} -{"stream": "databases", "data": {"object": "database", "id": "9b1ce91e-a93a-437c-8c92-81083cd98540", "cover": null, "icon": {"type": "emoji", "emoji": "\u270f\ufe0f"}, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "ec324c09-af75-40f0-b91a-49ded74fdaf5"}, "last_edited_time": "2023-09-13T00:06:00.000Z", "title": [{"type": "text", "text": {"content": "Meeting Notes", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Meeting Notes", "href": null}], "description": [{"type": "text", "text": {"content": "Use this template to capture notes from all meetings in one accessible spot.\nNotes can be tagged by meeting type to make them easy to find. \nSee when each meeting took place and who was there.\n\n", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Use this template to capture notes from all meetings in one accessible spot.\nNotes can be tagged by meeting type to make them easy to find. \nSee when each meeting took place and who was there.\n\n", "href": null}, {"type": "text", "text": {"content": "\u2193", "link": null}, "annotations": {"bold": true, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "\u2193", "href": null}, {"type": "text", "text": {"content": " Click ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " Click ", "href": null}, {"type": "text", "text": {"content": "List View", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": true, "color": "default"}, "plain_text": "List View", "href": null}, {"type": "text", "text": {"content": " to create and see other views, including a board organized by meeting type.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " to create and see other views, including a board organized by meeting type.", "href": null}], "is_inline": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "name": "Last Edited Time", "type": "last_edited_time", "last_edited_time": {}}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "name": "Created By", "type": "created_by", "created_by": {}}}, {"name": "Created", "value": {"id": "Ird4", "name": "Created", "type": "created_time", "created_time": {}}}, {"name": "Type", "value": {"id": "_%7B%5C7", "name": "Type", "type": "select", "select": {"options": [{"id": "3a8fd64c-899d-4c39-ba97-ac4f565d6e94", "name": "Post-mortem", "color": "red"}, {"id": "28b68013-20d5-4824-b810-45cde8784581", "name": "Standup", "color": "green"}, {"id": "8ee247a9-cb60-430a-9ea6-d5c053253334", "name": "Weekly Sync", "color": "blue"}, {"id": "5fb57c36-999f-49e2-b153-96531d086862", "name": "Sprint Planning", "color": "yellow"}, {"id": "1747fcca-8207-42c8-802f-fd43965c016a", "name": "Ad Hoc", "color": "orange"}]}}}, {"name": "Participants", "value": {"id": "b%3AeA", "name": "Participants", "type": "people", "people": {}}}, {"name": "Name", "value": {"id": "title", "name": "Name", "type": "title", "title": {}}}], "parent": {"type": "workspace", "workspace": true}, "url": "https://www.notion.so/9b1ce91ea93a437c8c9281083cd98540", "public_url": null, "archived": false}, "emitted_at": 1697023281968} +{"stream": "databases", "data": {"object": "database", "id": "b75d2e55-cc80-4afa-a273-c78178ac6b3f", "cover": null, "icon": {"type": "emoji", "emoji": "\ud83d\ude4b"}, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_time": "2021-10-19T13:33:00.000Z", "title": [{"type": "text", "text": {"content": "Engineering Directory ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Engineering Directory ", "href": null}], "description": [{"type": "text", "text": {"content": "Have a question about part of our codebase?\nFind the most knowledgeable person in this directory.\nLearn more about ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Have a question about part of our codebase?\nFind the most knowledgeable person in this directory.\nLearn more about ", "href": null}, {"type": "text", "text": {"content": "Notion databases", "link": {"url": "https://www.notion.so/notion/Database-101-build-and-view-fd8cd2d212f74c50954c11086d85997e"}}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Notion databases", "href": "https://www.notion.so/notion/Database-101-build-and-view-fd8cd2d212f74c50954c11086d85997e"}, {"type": "text", "text": {"content": ".", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": ".", "href": null}], "is_inline": false, "properties": [{"name": "Date Added", "value": {"id": "%2Fkv%22", "name": "Date Added", "type": "created_time", "created_time": {}}}, {"name": "Notes", "value": {"id": "mq%22D", "name": "Notes", "type": "rich_text", "rich_text": {}}}, {"name": "Person", "value": {"id": "uiZ%26", "name": "Person", "type": "people", "people": {}}}, {"name": "Name", "value": {"id": "title", "name": "Name", "type": "title", "title": {}}}], "parent": {"type": "block_id", "block_id": "b81f8caf-3ec4-4455-9a0b-25c2bd3b60cb"}, "url": "https://www.notion.so/b75d2e55cc804afaa273c78178ac6b3f", "public_url": null, "archived": false}, "emitted_at": 1708341487319} +{"stream": "databases", "data": {"object": "database", "id": "fbff7d4e-eca4-4432-91e6-ec64ba4b5a98", "cover": null, "icon": null, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_time": "2021-10-19T13:33:00.000Z", "title": [{"type": "text", "text": {"content": "Questions", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Questions", "href": null}], "description": [], "is_inline": true, "properties": [{"name": "Difficulty", "value": {"id": "'i6%2F", "name": "Difficulty", "type": "select", "select": {"options": [{"id": "f00068b9-7612-45da-91ad-1a7b1d259375", "name": "Easy", "color": "green", "description": null}, {"id": "8e244bfe-d4c7-48c5-9088-ffd6926b4ba0", "name": "Medium", "color": "yellow", "description": null}, {"id": "9ab57ef4-eab1-4b20-a502-047610b5c97d", "name": "Hard", "color": "red", "description": null}]}}}, {"name": "Skills", "value": {"id": "K%3AtR", "name": "Skills", "type": "multi_select", "multi_select": {"options": [{"id": "72f4d134-a773-48c1-ba3d-b529f55c6818", "name": "Front end", "color": "default", "description": null}, {"id": "c20f5d57-3e35-4b39-b556-05071203cc1a", "name": "Backend", "color": "default", "description": null}, {"id": "31d5735c-d6ba-4bd7-940f-bdcb36091c02", "name": "Architecture", "color": "default", "description": null}, {"id": "0398de54-af68-4c3a-9953-3788e8eaadbf", "name": "Algorithms", "color": "default", "description": null}, {"id": "df9dff09-7dea-4409-a10f-b5e2b546ad94", "name": "Data Structures", "color": "default", "description": null}]}}}, {"name": "Question Name", "value": {"id": "title", "name": "Question Name", "type": "title", "title": {}}}], "parent": {"type": "page_id", "page_id": "4999109d-1b7b-41a2-abb4-84f6b961ee74"}, "url": "https://www.notion.so/fbff7d4eeca4443291e6ec64ba4b5a98", "public_url": null, "archived": false}, "emitted_at": 1708341487319} +{"stream": "databases", "data": {"object": "database", "id": "9b1ce91e-a93a-437c-8c92-81083cd98540", "cover": null, "icon": {"type": "emoji", "emoji": "\u270f\ufe0f"}, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "ec324c09-af75-40f0-b91a-49ded74fdaf5"}, "last_edited_time": "2023-09-13T00:06:00.000Z", "title": [{"type": "text", "text": {"content": "Meeting Notes", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Meeting Notes", "href": null}], "description": [{"type": "text", "text": {"content": "Use this template to capture notes from all meetings in one accessible spot.\nNotes can be tagged by meeting type to make them easy to find. \nSee when each meeting took place and who was there.\n\n", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Use this template to capture notes from all meetings in one accessible spot.\nNotes can be tagged by meeting type to make them easy to find. \nSee when each meeting took place and who was there.\n\n", "href": null}, {"type": "text", "text": {"content": "\u2193", "link": null}, "annotations": {"bold": true, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "\u2193", "href": null}, {"type": "text", "text": {"content": " Click ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " Click ", "href": null}, {"type": "text", "text": {"content": "List View", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": true, "color": "default"}, "plain_text": "List View", "href": null}, {"type": "text", "text": {"content": " to create and see other views, including a board organized by meeting type.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " to create and see other views, including a board organized by meeting type.", "href": null}], "is_inline": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "name": "Last Edited Time", "type": "last_edited_time", "last_edited_time": {}}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "name": "Created By", "type": "created_by", "created_by": {}}}, {"name": "Created", "value": {"id": "Ird4", "name": "Created", "type": "created_time", "created_time": {}}}, {"name": "Type", "value": {"id": "_%7B%5C7", "name": "Type", "type": "select", "select": {"options": [{"id": "3a8fd64c-899d-4c39-ba97-ac4f565d6e94", "name": "Post-mortem", "color": "red", "description": null}, {"id": "28b68013-20d5-4824-b810-45cde8784581", "name": "Standup", "color": "green", "description": null}, {"id": "8ee247a9-cb60-430a-9ea6-d5c053253334", "name": "Weekly Sync", "color": "blue", "description": null}, {"id": "5fb57c36-999f-49e2-b153-96531d086862", "name": "Sprint Planning", "color": "yellow", "description": null}, {"id": "1747fcca-8207-42c8-802f-fd43965c016a", "name": "Ad Hoc", "color": "orange", "description": null}]}}}, {"name": "Participants", "value": {"id": "b%3AeA", "name": "Participants", "type": "people", "people": {}}}, {"name": "Name", "value": {"id": "title", "name": "Name", "type": "title", "title": {}}}], "parent": {"type": "workspace", "workspace": true}, "url": "https://www.notion.so/9b1ce91ea93a437c8c9281083cd98540", "public_url": null, "archived": false}, "emitted_at": 1708341487319} {"stream": "pages", "data": {"object": "page", "id": "39a69b4e-7cc2-4f7a-a656-dd128f3ce855", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "9b1ce91e-a93a-437c-8c92-81083cd98540"}, "archived": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "type": "last_edited_time", "last_edited_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "type": "created_by", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}}}, {"name": "Created", "value": {"id": "Ird4", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Type", "value": {"id": "_%7B%5C7", "type": "select", "select": {"id": "28b68013-20d5-4824-b810-45cde8784581", "name": "Standup", "color": "green"}}}, {"name": "Participants", "value": {"id": "b%3AeA", "type": "people", "people": []}}, {"name": "Name", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Daily Standup", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Daily Standup", "href": null}]}}], "url": "https://www.notion.so/Daily-Standup-39a69b4e7cc24f7aa656dd128f3ce855", "public_url": null}, "emitted_at": 1697023284463} {"stream": "pages", "data": {"object": "page", "id": "621d3dc4-55fe-46ce-a3ff-83da06e5f9fb", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "9b1ce91e-a93a-437c-8c92-81083cd98540"}, "archived": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "type": "last_edited_time", "last_edited_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "type": "created_by", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}}}, {"name": "Created", "value": {"id": "Ird4", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Type", "value": {"id": "_%7B%5C7", "type": "select", "select": {"id": "5fb57c36-999f-49e2-b153-96531d086862", "name": "Sprint Planning", "color": "yellow"}}}, {"name": "Participants", "value": {"id": "b%3AeA", "type": "people", "people": []}}, {"name": "Name", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Sprint Planning ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Sprint Planning ", "href": null}]}}], "url": "https://www.notion.so/Sprint-Planning-621d3dc455fe46cea3ff83da06e5f9fb", "public_url": null}, "emitted_at": 1697023284465} {"stream": "pages", "data": {"object": "page", "id": "6eb2dedc-8b88-486c-8648-d1878bafb106", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "9b1ce91e-a93a-437c-8c92-81083cd98540"}, "archived": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "type": "last_edited_time", "last_edited_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "type": "created_by", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}}}, {"name": "Created", "value": {"id": "Ird4", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Type", "value": {"id": "_%7B%5C7", "type": "select", "select": {"id": "1747fcca-8207-42c8-802f-fd43965c016a", "name": "Ad Hoc", "color": "orange"}}}, {"name": "Participants", "value": {"id": "b%3AeA", "type": "people", "people": []}}, {"name": "Name", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Ad Hoc Meeting", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Ad Hoc Meeting", "href": null}]}}], "url": "https://www.notion.so/Ad-Hoc-Meeting-6eb2dedc8b88486c8648d1878bafb106", "public_url": null}, "emitted_at": 1697023284465} diff --git a/airbyte-integrations/connectors/source-notion/metadata.yaml b/airbyte-integrations/connectors/source-notion/metadata.yaml index 8a95d375f3cf2..58b329ffe793b 100644 --- a/airbyte-integrations/connectors/source-notion/metadata.yaml +++ b/airbyte-integrations/connectors/source-notion/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6e00b415-b02e-4160-bf02-58176a0ae687 - dockerImageTag: 2.0.9 + dockerImageTag: 2.1.0 dockerRepository: airbyte/source-notion documentationUrl: https://docs.airbyte.com/integrations/sources/notion githubIssueLabel: source-notion diff --git a/airbyte-integrations/connectors/source-notion/pyproject.toml b/airbyte-integrations/connectors/source-notion/pyproject.toml index 4218b07b35bae..3c4250763099d 100644 --- a/airbyte-integrations/connectors/source-notion/pyproject.toml +++ b/airbyte-integrations/connectors/source-notion/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "2.0.9" +version = "2.1.0" name = "source-notion" description = "Source implementation for Notion." authors = [ "Airbyte ",] diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json index cc003048b8755..e8ecf832da801 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json @@ -32,6 +32,17 @@ "properties": { "type": { "type": ["null", "string"] + }, + "info": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + } + } } } }, diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json index d893b1147b7e1..8c5000dba2058 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json @@ -37,10 +37,38 @@ "type": { "type": "string" }, + "info": { + "avatar_url": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "person": { + "type": ["null", "object"], + "properties": { + "email": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + } + }, "workspace": { "type": ["null", "boolean"] } } + }, + "workspace_name": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-notion/source_notion/streams.py b/airbyte-integrations/connectors/source-notion/source_notion/streams.py index a0546d2116c49..f7cb9e456e250 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/streams.py +++ b/airbyte-integrations/connectors/source-notion/source_notion/streams.py @@ -252,6 +252,22 @@ def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> params["start_cursor"] = next_page_token["next_cursor"] return params + def transform(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + owner = record.get("bot", {}).get("owner") + if owner: + owner_type = owner.get("type") + owner_info = owner.get(owner_type) + if owner_type and owner_info: + record["bot"]["owner"]["info"] = owner_info + del record["bot"]["owner"][owner_type] + return record + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + # sometimes notion api returns response without results object + data = response.json().get("results", []) + for record in data: + yield self.transform(record) + class Databases(IncrementalNotionStream): """ @@ -313,6 +329,20 @@ def stream_slices( yield {"page_id": page_id} + def transform(self, record: Mapping[str, Any]) -> Mapping[str, Any]: + transform_object_field = record.get("type") + + if transform_object_field: + rich_text = record.get(transform_object_field, {}).get("rich_text", []) + for r in rich_text: + mention = r.get("mention") + if mention: + type_info = mention[mention["type"]] + record[transform_object_field]["rich_text"][rich_text.index(r)]["mention"]["info"] = type_info + del record[transform_object_field]["rich_text"][rich_text.index(r)]["mention"][mention["type"]] + + return record + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: # pages and databases blocks are already fetched in their streams, so no # need to do it again @@ -321,7 +351,7 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str, records = super().parse_response(response, stream_state=stream_state, **kwargs) for record in records: if record["type"] not in ("child_page", "child_database", "ai_block"): - yield record + yield self.transform(record) def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: # if reached recursive limit, don't read anymore diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py index ceeb5b1ca5cb5..d369b201beccc 100644 --- a/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py @@ -312,3 +312,41 @@ def test_request_throttle(initial_page_size, expected_page_size, mock_response, stream.should_retry(response=response) assert stream.page_size == expected_page_size + + +def test_users_record_transformer(): + stream = Users(config=MagicMock()) + response_record = { + "object": "user", "id": "id", "name": "Airbyte", "avatar_url": "some url", "type": "bot", + "bot": {"owner": {"type": "user", "user": {"object": "user", "id": "id", "name": "Test User", "avatar_url": None, "type": "person", + "person": {"email": "email"}}}, "workspace_name": "test"} + } + expected_record = { + "object": "user", "id": "id", "name": "Airbyte", "avatar_url": "some url", "type": "bot", + "bot": {"owner": {"type": "user", "info": {"object": "user", "id": "id", "name": "Test User", "avatar_url": None, "type": "person", + "person": {"email": "email"}}}, "workspace_name": "test"} + } + assert stream.transform(response_record) == expected_record + + +def test_block_record_transformer(): + stream = Blocks(parent=None, config=MagicMock()) + response_record = { + "object": "block", "id": "id", "parent": {"type": "page_id", "page_id": "id"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", + "created_by": {"object": "user", "id": "id"}, "last_edited_by": {"object": "user", "id": "id"}, "has_children": False, "archived": False, "type": "paragraph", + "paragraph": {"rich_text": [{"type": "text", "text": {"content": "test", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": None}, + {"type": "text", "text": {"content": "@", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": True, "color": "default"}, "plain_text": "@", "href": None}, + {"type": "text", "text": {"content": "test", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": None}, + {"type": "mention", "mention": {"type": "page", "page": {"id": "id"}}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, + "plain_text": "test", "href": "https://www.notion.so/id"}], "color": "default"} + } + expected_record = { + "object": "block", "id": "id", "parent": {"type": "page_id", "page_id": "id"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", + "created_by": {"object": "user", "id": "id"}, "last_edited_by": {"object": "user", "id": "id"}, "has_children": False, "archived": False, "type": "paragraph", + "paragraph": {"rich_text": [{"type": "text", "text": {"content": "test", "link": None}, "annotations":{"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text":"test", "href": None}, + {"type": "text", "text": {"content": "@", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": True, "color": "default"}, "plain_text": "@", "href": None}, + {"type": "text", "text": {"content": "test", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": None}, + {"type": "mention", "mention": {"type": "page", "info": {"id": "id"}}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": "https://www.notion.so/id"}], + "color": "default"} + } + assert stream.transform(response_record) == expected_record diff --git a/docs/integrations/sources/notion.md b/docs/integrations/sources/notion.md index 84319c018bc90..04d96e2a9be2e 100644 --- a/docs/integrations/sources/notion.md +++ b/docs/integrations/sources/notion.md @@ -110,43 +110,44 @@ The connector is restricted by Notion [request limits](https://developers.notion ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------- | -| 2.0.9 | 2024-02-12 | [35155](https://github.com/airbytehq/airbyte/pull/35155) | Manage dependencies with Poetry. | -| 2.0.8 | 2023-11-01 | [31899](https://github.com/airbytehq/airbyte/pull/31899) | Fix `table_row.cells` property in `Blocks` stream | -| 2.0.7 | 2023-10-31 | [32004](https://github.com/airtybehq/airbyte/pull/32004) | Reduce page_size on 504 errors | -| 2.0.6 | 2023-10-25 | [31825](https://github.com/airbytehq/airbyte/pull/31825) | Increase max_retries on retryable errors | -| 2.0.5 | 2023-10-23 | [31742](https://github.com/airbytehq/airbyte/pull/31742) | Add 'synced_block' property to Blocks schema | -| 2.0.4 | 2023-10-19 | [31625](https://github.com/airbytehq/airbyte/pull/31625) | Fix check_connection method | -| 2.0.3 | 2023-10-19 | [31612](https://github.com/airbytehq/airbyte/pull/31612) | Add exponential backoff for 500 errors | -| 2.0.2 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | -| 2.0.1 | 2023-10-17 | [31507](https://github.com/airbytehq/airbyte/pull/31507) | Add start_date validation checks | -| 2.0.0 | 2023-10-09 | [30587](https://github.com/airbytehq/airbyte/pull/30587) | Source-wide schema update | -| 1.3.0 | 2023-10-09 | [30324](https://github.com/airbytehq/airbyte/pull/30324) | Add `Comments` stream | -| 1.2.2 | 2023-10-09 | [30780](https://github.com/airbytehq/airbyte/pull/30780) | Update Start Date in config to optional field | -| 1.2.1 | 2023-10-08 | [30750](https://github.com/airbytehq/airbyte/pull/30750) | Add availability strategy | -| 1.2.0 | 2023-10-04 | [31053](https://github.com/airbytehq/airbyte/pull/31053) | Add undeclared fields for blocks and pages streams | -| 1.1.2 | 2023-08-30 | [29999](https://github.com/airbytehq/airbyte/pull/29999) | Update error handling during connection check | -| 1.1.1 | 2023-06-14 | [26535](https://github.com/airbytehq/airbyte/pull/26535) | Migrate from deprecated `authSpecification` to `advancedAuth` | -| 1.1.0 | 2023-06-08 | [27170](https://github.com/airbytehq/airbyte/pull/27170) | Fix typo in `blocks` schema | -| 1.0.9 | 2023-06-08 | [27062](https://github.com/airbytehq/airbyte/pull/27062) | Skip streams with `invalid_start_cursor` error | -| 1.0.8 | 2023-06-07 | [27073](https://github.com/airbytehq/airbyte/pull/27073) | Add empty results handling for stream `Blocks` | -| 1.0.7 | 2023-06-06 | [27060](https://github.com/airbytehq/airbyte/pull/27060) | Add skipping 404 error in `Blocks` stream | -| 1.0.6 | 2023-05-18 | [26286](https://github.com/airbytehq/airbyte/pull/26286) | Add `parent` field to `Blocks` stream | -| 1.0.5 | 2023-05-01 | [25709](https://github.com/airbytehq/airbyte/pull/25709) | Fixed `ai_block is unsupported by API` issue, while fetching `Blocks` stream | -| 1.0.4 | 2023-04-11 | [25041](https://github.com/airbytehq/airbyte/pull/25041) | Improve error handling for API /search | -| 1.0.3 | 2023-03-02 | [22931](https://github.com/airbytehq/airbyte/pull/22931) | Specified date formatting in specification | -| 1.0.2 | 2023-02-24 | [23437](https://github.com/airbytehq/airbyte/pull/23437) | Add retry for 400 error (validation_error) | -| 1.0.1 | 2023-01-27 | [22018](https://github.com/airbytehq/airbyte/pull/22018) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 1.0.0 | 2022-12-19 | [20639](https://github.com/airbytehq/airbyte/pull/20639) | Fix `Pages` stream schema | -| 0.1.10 | 2022-09-28 | [17298](https://github.com/airbytehq/airbyte/pull/17298) | Use "Retry-After" header for backoff | -| 0.1.9 | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state | -| 0.1.8 | 2022-09-05 | [16272](https://github.com/airbytehq/airbyte/pull/16272) | Update spec description to include working timestamp example | -| 0.1.7 | 2022-07-26 | [15042](https://github.com/airbytehq/airbyte/pull/15042) | Update `additionalProperties` field to true from shared schemas | -| 0.1.6 | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from schemas and spec | -| 0.1.5 | 2022-07-14 | [14706](https://github.com/airbytehq/airbyte/pull/14706) | Added OAuth2.0 authentication | -| 0.1.4 | 2022-07-07 | [14505](https://github.com/airbytehq/airbyte/pull/14505) | Fixed bug when normalization didn't run through | -| 0.1.3 | 2022-04-22 | [11452](https://github.com/airbytehq/airbyte/pull/11452) | Use pagination for User stream | -| 0.1.2 | 2022-01-11 | [9084](https://github.com/airbytehq/airbyte/pull/9084) | Fix documentation URL | -| 0.1.1 | 2021-12-30 | [9207](https://github.com/airbytehq/airbyte/pull/9207) | Update connector fields title/description | -| 0.1.0 | 2021-10-17 | [7092](https://github.com/airbytehq/airbyte/pull/7092) | Initial Release | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------| +| 2.1.0 | 2024-02-19 | [35409](https://github.com/airbytehq/airbyte/pull/35409) | Update users stream schema with bot type info fields and block schema with mention type info fields. | +| 2.0.9 | 2024-02-12 | [35155](https://github.com/airbytehq/airbyte/pull/35155) | Manage dependencies with Poetry. | +| 2.0.8 | 2023-11-01 | [31899](https://github.com/airbytehq/airbyte/pull/31899) | Fix `table_row.cells` property in `Blocks` stream | +| 2.0.7 | 2023-10-31 | [32004](https://github.com/airtybehq/airbyte/pull/32004) | Reduce page_size on 504 errors | +| 2.0.6 | 2023-10-25 | [31825](https://github.com/airbytehq/airbyte/pull/31825) | Increase max_retries on retryable errors | +| 2.0.5 | 2023-10-23 | [31742](https://github.com/airbytehq/airbyte/pull/31742) | Add 'synced_block' property to Blocks schema | +| 2.0.4 | 2023-10-19 | [31625](https://github.com/airbytehq/airbyte/pull/31625) | Fix check_connection method | +| 2.0.3 | 2023-10-19 | [31612](https://github.com/airbytehq/airbyte/pull/31612) | Add exponential backoff for 500 errors | +| 2.0.2 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 2.0.1 | 2023-10-17 | [31507](https://github.com/airbytehq/airbyte/pull/31507) | Add start_date validation checks | +| 2.0.0 | 2023-10-09 | [30587](https://github.com/airbytehq/airbyte/pull/30587) | Source-wide schema update | +| 1.3.0 | 2023-10-09 | [30324](https://github.com/airbytehq/airbyte/pull/30324) | Add `Comments` stream | +| 1.2.2 | 2023-10-09 | [30780](https://github.com/airbytehq/airbyte/pull/30780) | Update Start Date in config to optional field | +| 1.2.1 | 2023-10-08 | [30750](https://github.com/airbytehq/airbyte/pull/30750) | Add availability strategy | +| 1.2.0 | 2023-10-04 | [31053](https://github.com/airbytehq/airbyte/pull/31053) | Add undeclared fields for blocks and pages streams | +| 1.1.2 | 2023-08-30 | [29999](https://github.com/airbytehq/airbyte/pull/29999) | Update error handling during connection check | +| 1.1.1 | 2023-06-14 | [26535](https://github.com/airbytehq/airbyte/pull/26535) | Migrate from deprecated `authSpecification` to `advancedAuth` | +| 1.1.0 | 2023-06-08 | [27170](https://github.com/airbytehq/airbyte/pull/27170) | Fix typo in `blocks` schema | +| 1.0.9 | 2023-06-08 | [27062](https://github.com/airbytehq/airbyte/pull/27062) | Skip streams with `invalid_start_cursor` error | +| 1.0.8 | 2023-06-07 | [27073](https://github.com/airbytehq/airbyte/pull/27073) | Add empty results handling for stream `Blocks` | +| 1.0.7 | 2023-06-06 | [27060](https://github.com/airbytehq/airbyte/pull/27060) | Add skipping 404 error in `Blocks` stream | +| 1.0.6 | 2023-05-18 | [26286](https://github.com/airbytehq/airbyte/pull/26286) | Add `parent` field to `Blocks` stream | +| 1.0.5 | 2023-05-01 | [25709](https://github.com/airbytehq/airbyte/pull/25709) | Fixed `ai_block is unsupported by API` issue, while fetching `Blocks` stream | +| 1.0.4 | 2023-04-11 | [25041](https://github.com/airbytehq/airbyte/pull/25041) | Improve error handling for API /search | +| 1.0.3 | 2023-03-02 | [22931](https://github.com/airbytehq/airbyte/pull/22931) | Specified date formatting in specification | +| 1.0.2 | 2023-02-24 | [23437](https://github.com/airbytehq/airbyte/pull/23437) | Add retry for 400 error (validation_error) | +| 1.0.1 | 2023-01-27 | [22018](https://github.com/airbytehq/airbyte/pull/22018) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 1.0.0 | 2022-12-19 | [20639](https://github.com/airbytehq/airbyte/pull/20639) | Fix `Pages` stream schema | +| 0.1.10 | 2022-09-28 | [17298](https://github.com/airbytehq/airbyte/pull/17298) | Use "Retry-After" header for backoff | +| 0.1.9 | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state | +| 0.1.8 | 2022-09-05 | [16272](https://github.com/airbytehq/airbyte/pull/16272) | Update spec description to include working timestamp example | +| 0.1.7 | 2022-07-26 | [15042](https://github.com/airbytehq/airbyte/pull/15042) | Update `additionalProperties` field to true from shared schemas | +| 0.1.6 | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from schemas and spec | +| 0.1.5 | 2022-07-14 | [14706](https://github.com/airbytehq/airbyte/pull/14706) | Added OAuth2.0 authentication | +| 0.1.4 | 2022-07-07 | [14505](https://github.com/airbytehq/airbyte/pull/14505) | Fixed bug when normalization didn't run through | +| 0.1.3 | 2022-04-22 | [11452](https://github.com/airbytehq/airbyte/pull/11452) | Use pagination for User stream | +| 0.1.2 | 2022-01-11 | [9084](https://github.com/airbytehq/airbyte/pull/9084) | Fix documentation URL | +| 0.1.1 | 2021-12-30 | [9207](https://github.com/airbytehq/airbyte/pull/9207) | Update connector fields title/description | +| 0.1.0 | 2021-10-17 | [7092](https://github.com/airbytehq/airbyte/pull/7092) | Initial Release |