Fix bug for lineage metadata

Author: shubhamjagtap639

metadata-ingestion/setup.py

@@ -395,7 +395,7 @@
     # databricks is alias for unity-catalog and needs to be kept in sync
     "databricks": databricks | sql_common | sqllineage_lib,
     "fivetran": snowflake_common | bigquery_common,
-    "qlik-sense": {"requests", "websocket-client"},
+    "qlik-sense": {"requests", "websocket-client"} | sqlglot_lib,
 }
 
 # This is mainly used to exclude plugins from the Docker image.

metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py

@@ -191,7 +191,6 @@ def update_values(cls, values: Dict) -> Dict:
 
 class QlikTable(BaseModel):
     tableName: str
-    databaseName: str
     type: BoxType = Field(alias="boxType")
     tableAlias: str
     dataconnectorid: str
@@ -200,22 +199,17 @@ class QlikTable(BaseModel):
     spaceId: str
     datasetSchema: List[SchemaField] = Field(alias="fields")
     tableQri: Optional[str] = None
+    selectStatement: Optional[str] = None
+    databaseName: Optional[str] = None
     schemaName: Optional[str] = None
 
     @root_validator(pre=True)
     def update_values(cls, values: Dict) -> Dict:
-        if values["boxType"] == BoxType.BLACKBOX.value:
-            values[Constant.DATABASENAME] = values[Constant.CONNECTORPROPERTIES][
-                Constant.TABLEQUALIFIERS
-            ][0]
-            values[Constant.SCHEMANAME] = values[Constant.CONNECTORPROPERTIES][
-                Constant.TABLEQUALIFIERS
-            ][1]
         values[Constant.DATACONNECTORID] = values[Constant.CONNECTIONINFO][Constant.ID]
         values[Constant.DATACONNECTORPLATFORM] = values[Constant.CONNECTIONINFO][
             Constant.SOURCECONNECTORID
         ]
-        values[Constant.SPACEID] = values[Constant.CONNECTIONINFO]["space"]
+        values[Constant.SPACEID] = values[Constant.CONNECTIONINFO][Constant.SPACE]
         return values
 
 

metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py

@@ -124,6 +124,9 @@ def _get_sheet(
             )
             response = websocket_connection.websocket_send_request(method="GetLayout")
             sheet_dict = response[Constant.QLAYOUT]
+            if Constant.OWNERID not in sheet_dict[Constant.QMETA]:
+                # That means sheet is private sheet
+                return None
             sheet = Sheet.parse_obj(sheet_dict[Constant.QMETA])
             for i, chart_dict in enumerate(
                 sheet_dict[Constant.QCHILDLIST][Constant.QITEMS]
@@ -159,20 +162,27 @@ def _add_qri_of_tables(self, tables: List[QlikTable], app_id: str) -> None:
                     f"{self.rest_api_url}/lineage-graphs/nodes/{app_qri}/actions/expand?node={table_node_qri}&level=FIELD"
                 )
                 response.raise_for_status()
-                field_node_qri = list(
+                field_nodes_qris = list(
                     response.json()[Constant.GRAPH][Constant.NODES].keys()
-                )[0]
-                response = self.session.post(
-                    f"{self.rest_api_url}/lineage-graphs/nodes/{app_qri}/overview",
-                    json=[field_node_qri],
                 )
-                response.raise_for_status()
-                for each_lineage in response.json()[Constant.RESOURCES][0][
-                    Constant.LINEAGE
-                ]:
-                    table_qri_dict[each_lineage[Constant.TABLELABEL]] = each_lineage[
-                        Constant.TABLEQRI
-                    ]
+                for field_node_qri in field_nodes_qris:
+                    response = self.session.post(
+                        f"{self.rest_api_url}/lineage-graphs/nodes/{app_qri}/overview",
+                        json=[field_node_qri],
+                    )
+                    response.raise_for_status()
+                    # Some fields might not have lineage overview, in that case status code is 207
+                    if response.status_code == 200:
+                        for each_lineage in response.json()[Constant.RESOURCES][0][
+                            Constant.LINEAGE
+                        ]:
+                            table_name = (
+                                each_lineage[Constant.TABLELABEL]
+                                .replace('"', "")
+                                .split(".")[-1]
+                            )
+                            table_qri_dict[table_name] = each_lineage[Constant.TABLEQRI]
+                        break
             for table in tables:
                 if table.tableName in table_qri_dict:
                     table.tableQri = table_qri_dict[table.tableName]

metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py

@@ -84,6 +84,7 @@
     OwnershipTypeClass,
     SchemaFieldDataTypeClass,
 )
+from datahub.sql_parsing.sqlglot_lineage import create_lineage_sql_parsed_result
 
 # Logger instance
 logger = logging.getLogger(__name__)
@@ -331,14 +332,17 @@ def _gen_app_table_upstream_lineage(
                     ),
                 )
             )
-            upstream_dataset_urn = builder.make_dataset_urn_with_platform_instance(
-                name=f"{table.databaseName}.{table.schemaName}.{table.tableName}".lower(),
+            if not table.selectStatement:
+                return None
+            upstream_dataset_urn = create_lineage_sql_parsed_result(
+                query=table.selectStatement.strip(),
+                default_db=None,
                 platform=KNOWN_DATA_PLATFORM_MAPPING.get(
                     table.dataconnectorPlatform, table.dataconnectorPlatform
                 ),
                 env=upstream_dataset_platform_detail.env,
                 platform_instance=upstream_dataset_platform_detail.platform_instance,
-            )
+            ).in_tables[0]
         elif table.type == BoxType.LOADFILE:
             upstream_dataset_urn = self._gen_qlik_dataset_urn(
                 f"{table.spaceId}.{table.databaseName}".lower()