|
| 1 | +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. |
| 2 | + |
| 3 | +from __future__ import annotations |
| 4 | + |
| 5 | +from uuid import UUID |
| 6 | + |
| 7 | +import httpx |
| 8 | +from httpx import HTTPError, HTTPStatusError |
| 9 | +from tenacity import Retrying, retry_if_exception_type, stop_after_attempt, wait_random_exponential |
| 10 | + |
| 11 | +from destination_deepset import util |
| 12 | +from destination_deepset.models import DeepsetCloudConfig, DeepsetCloudFile |
| 13 | + |
| 14 | + |
| 15 | +class APIError(RuntimeError): |
| 16 | + """Raised when any error occurs while using the API.""" |
| 17 | + |
| 18 | + |
| 19 | +class ConfigurationError(ValueError, APIError): |
| 20 | + """Raised when the configuration is missing or incorrect.""" |
| 21 | + |
| 22 | + |
| 23 | +class FileUploadError(APIError): |
| 24 | + """Raised when the server is unable to successfully upload the file.""" |
| 25 | + |
| 26 | + def __str__(self) -> str: |
| 27 | + return "File upload failed." |
| 28 | + |
| 29 | + |
| 30 | +class DeepsetCloudApi: |
| 31 | + def __init__(self, config: DeepsetCloudConfig) -> None: |
| 32 | + self.config = config |
| 33 | + self._client: httpx.Client | None = None |
| 34 | + |
| 35 | + # retry settings in seconds |
| 36 | + self.max = 60 |
| 37 | + self.multiplier = 0.5 |
| 38 | + |
| 39 | + @property |
| 40 | + def client(self) -> httpx.Client: |
| 41 | + if not self.config: |
| 42 | + raise ConfigurationError |
| 43 | + |
| 44 | + if self._client is None: |
| 45 | + self._client = httpx.Client( |
| 46 | + base_url=self.config.base_url.removesuffix("/"), |
| 47 | + headers={ |
| 48 | + "Accept": "application/json", |
| 49 | + "Authorization": f"Bearer {self.config.api_key}", |
| 50 | + "X-Client-Source": "airbyte-destination-deepset", |
| 51 | + }, |
| 52 | + follow_redirects=True, |
| 53 | + ) |
| 54 | + |
| 55 | + return self._client |
| 56 | + |
| 57 | + def retry(self) -> Retrying: |
| 58 | + """Retrial configurations |
| 59 | +
|
| 60 | + Returns: |
| 61 | + Retrying: The instance |
| 62 | + """ |
| 63 | + return Retrying( |
| 64 | + retry=retry_if_exception_type(HTTPError), |
| 65 | + stop=stop_after_attempt(self.config.retries), |
| 66 | + wait=wait_random_exponential(multiplier=self.multiplier, max=self.max), |
| 67 | + reraise=True, |
| 68 | + ) |
| 69 | + |
| 70 | + def health_check(self) -> None: |
| 71 | + """Check the health of deepset cloud API |
| 72 | +
|
| 73 | + Raises: |
| 74 | + APIError: Raised when an error is encountered. |
| 75 | + """ |
| 76 | + try: |
| 77 | + for attempt in self.retry(): |
| 78 | + with attempt: |
| 79 | + response = self.client.get("/api/v1/me") |
| 80 | + response.raise_for_status() |
| 81 | + |
| 82 | + workspaces = util.get(response.json(), "organization.workspaces", []) |
| 83 | + access = next((True for workspace in workspaces if workspace["name"] == self.config.workspace), False) |
| 84 | + except Exception as ex: |
| 85 | + raise APIError from ex |
| 86 | + else: |
| 87 | + if access: |
| 88 | + return |
| 89 | + |
| 90 | + error = "User does not have access to the selected workspace!" |
| 91 | + raise ConfigurationError(error) |
| 92 | + |
| 93 | + def upload(self, file: DeepsetCloudFile, write_mode: str = "KEEP") -> UUID: |
| 94 | + """Upload file to deepset Cloud. |
| 95 | +
|
| 96 | + Args: |
| 97 | + file (DeepsetCloudFile): The file to upload |
| 98 | + write_mode (str, Optional): The write mode. Defaults to `KEEP`. |
| 99 | +
|
| 100 | + Raises: |
| 101 | + APIError: Raised whenever the file upload fails |
| 102 | +
|
| 103 | + Returns: |
| 104 | + UUID: The unique identifier of the uploaded file |
| 105 | + """ |
| 106 | + |
| 107 | + try: |
| 108 | + for attempt in self.retry(): |
| 109 | + with attempt: |
| 110 | + response = self.client.post( |
| 111 | + f"/api/v1/workspaces/{self.config.workspace}/files", |
| 112 | + files={"file": (file.name, file.content)}, |
| 113 | + data={"meta": file.meta_as_string}, |
| 114 | + params={"write_mode": write_mode}, |
| 115 | + ) |
| 116 | + response.raise_for_status() |
| 117 | + |
| 118 | + if file_id := response.json().get("file_id"): |
| 119 | + return UUID(file_id) |
| 120 | + |
| 121 | + except HTTPStatusError as ex: |
| 122 | + status_code, response_text = ex.response.status_code, ex.response.text |
| 123 | + message = f"File upload failed: {status_code = }, {response_text = }." |
| 124 | + raise FileUploadError(message) from ex |
| 125 | + except Exception as ex: |
| 126 | + raise FileUploadError from ex |
| 127 | + |
| 128 | + raise FileUploadError |
0 commit comments