Merge branch 'feature/aquav1.0.1' of https://github.com/oracle/accelerated-data-science into ODSC-55771/fix_telemetry_name

Author: lu-ohai

README.md

@@ -181,4 +181,4 @@ Consult the security guide [SECURITY.md](https://github.com/oracle/accelerated-d
 
 ## License
 
-Copyright (c) 2020, 2022 Oracle and/or its affiliates. Licensed under the [Universal Permissive License v1.0](https://oss.oracle.com/licenses/upl/)
+Copyright (c) 2020, 2024 Oracle and/or its affiliates. Licensed under the [Universal Permissive License v1.0](https://oss.oracle.com/licenses/upl/)

ads/aqua/decorator.py

@@ -17,6 +17,7 @@
     RequestException,
     ServiceError,
 )
+from tornado.web import HTTPError
 
 from ads.aqua.exception import AquaError
 from ads.aqua.extension.base_handler import AquaAPIhandler
@@ -58,6 +59,7 @@ def inner_function(self: AquaAPIhandler, *args, **kwargs):
         except ServiceError as error:
             self.write_error(
                 status_code=error.status or 500,
+                message=error.message,
                 reason=error.message,
                 service_payload=error.args[0] if error.args else None,
                 exc_info=sys.exc_info(),
@@ -91,6 +93,12 @@ def inner_function(self: AquaAPIhandler, *args, **kwargs):
                 service_payload=error.service_payload,
                 exc_info=sys.exc_info(),
             )
+        except HTTPError as e:
+            self.write_error(
+                status_code=e.status_code,
+                reason=e.log_message,
+                exc_info=sys.exc_info(),
+            )
         except Exception as ex:
             self.write_error(
                 status_code=500,

ads/aqua/deployment.py

@@ -391,40 +391,30 @@ def create(
             .with_runtime(container_runtime)
         ).deploy(wait_for_completion=False)
 
-        if is_fine_tuned_model:
-            # tracks unique deployments that were created in the user compartment
-            self.telemetry.record_event_async(
-                category="aqua/custom/deployment", action="create", detail=model_name
-            )
-            # tracks the shape used for deploying the custom models
-            self.telemetry.record_event_async(
-                category="aqua/custom/deployment/create",
-                action="shape",
-                detail=instance_shape,
-            )
-            # tracks the shape used for deploying the custom models by name
-            self.telemetry.record_event_async(
-                category=f"aqua/custom/{model_name}/deployment/create",
-                action="shape",
-                detail=instance_shape,
-            )
-        else:
-            # tracks unique deployments that were created in the user compartment
-            self.telemetry.record_event_async(
-                category="aqua/service/deployment", action="create", detail=model_name
-            )
-            # tracks the shape used for deploying the service models
-            self.telemetry.record_event_async(
-                category="aqua/service/deployment/create",
-                action="shape",
-                detail=instance_shape,
-            )
-            # tracks the shape used for deploying the service models by name
-            self.telemetry.record_event_async(
-                category=f"aqua/service/{model_name}/deployment/create",
-                action="shape",
-                detail=instance_shape,
-            )
+        model_type = "custom" if is_fine_tuned_model else "service"
+        deployment_id = deployment.dsc_model_deployment.id
+        telemetry_kwargs = (
+            {"ocid": deployment_id[-8:]} if len(deployment_id) > 8 else {}
+        )
+        # tracks unique deployments that were created in the user compartment
+        self.telemetry.record_event_async(
+            category=f"aqua/{model_type}/deployment",
+            action="create",
+            detail=model_name,
+            **telemetry_kwargs,
+        )
+        # tracks the shape used for deploying the custom or service models
+        self.telemetry.record_event_async(
+            category=f"aqua/{model_type}/deployment/create",
+            action="shape",
+            detail=instance_shape,
+        )
+        # tracks the shape used for deploying the custom or service models by name
+        self.telemetry.record_event_async(
+            category=f"aqua/{model_type}/{model_name}/deployment/create",
+            action="shape",
+            detail=instance_shape,
+        )
 
         return AquaDeployment.from_oci_model_deployment(
             deployment.dsc_model_deployment, self.region
@@ -471,6 +461,18 @@ def list(self, **kwargs) -> List["AquaDeployment"]:
                     )
                 )
 
+                # log telemetry if MD is in active or failed state
+                deployment_id = model_deployment.id
+                state = model_deployment.lifecycle_state.upper()
+                if state in ["ACTIVE", "FAILED"]:
+                    # tracks unique deployments that were listed in the user compartment
+                    self.telemetry.record_event_async(
+                        category=f"aqua/deployment",
+                        action="list",
+                        detail=deployment_id[-8:] if len(deployment_id) > 8 else "",
+                        value=state,
+                    )
+
         # tracks number of times deployment listing was called
         self.telemetry.record_event_async(category="aqua/deployment", action="list")
 

ads/aqua/evaluation.py

@@ -79,7 +79,7 @@ class EvaluationJobExitCode(Enum):
     SUCCESS = 0
     COMMON_ERROR = 1
 
-    # Configuration-related issues
+    # Configuration-related issues 10-19
     INVALID_EVALUATION_CONFIG = 10
     EVALUATION_CONFIG_NOT_PROVIDED = 11
     INVALID_OUTPUT_DIR = 12
@@ -88,7 +88,7 @@ class EvaluationJobExitCode(Enum):
     INVALID_TARGET_EVALUATION_ID = 15
     INVALID_EVALUATION_CONFIG_VALIDATION = 16
 
-    # Evaluation process issues
+    # Evaluation process issues 20-39
     OUTPUT_DIR_NOT_FOUND = 20
     INVALID_INPUT_DATASET = 21
     INPUT_DATA_NOT_FOUND = 22
@@ -101,6 +101,7 @@ class EvaluationJobExitCode(Enum):
     MODEL_INFERENCE_WRONG_RESPONSE_FORMAT = 29
     UNSUPPORTED_METRICS = 30
     METRIC_CALCULATION_FAILURE = 31
+    EVALUATION_MODEL_CATALOG_RECORD_CREATION_FAILED = 32
 
 
 EVALUATION_JOB_EXIT_CODE_MESSAGE = {
@@ -125,6 +126,11 @@ class EvaluationJobExitCode(Enum):
     EvaluationJobExitCode.MODEL_INFERENCE_WRONG_RESPONSE_FORMAT.value: "Evaluation encountered unsupported, or unexpected model output, verify the target evaluation model is compatible and produces the correct format.",
     EvaluationJobExitCode.UNSUPPORTED_METRICS.value: "None of the provided metrics are supported by the framework.",
     EvaluationJobExitCode.METRIC_CALCULATION_FAILURE.value: "All attempted metric calculations were unsuccessful. Please review the metric configurations and input data.",
+    EvaluationJobExitCode.EVALUATION_MODEL_CATALOG_RECORD_CREATION_FAILED.value: (
+        "Failed to create a Model Catalog record for the evaluation. "
+        "This could be due to missing required permissions. "
+        "Please check the log for more information."
+    ),
 }
 
 
@@ -312,6 +318,8 @@ class CreateAquaEvaluationDetails(DataClassSerializable):
         The log id for the evaluation job infrastructure.
     metrics: (list, optional). Defaults to `None`.
         The metrics for the evaluation.
+    force_overwrite: (bool, optional). Defaults to `False`.
+        Whether to force overwrite the existing file in object storage.
     """
 
     evaluation_source_id: str
@@ -332,6 +340,7 @@ class CreateAquaEvaluationDetails(DataClassSerializable):
     log_group_id: Optional[str] = None
     log_id: Optional[str] = None
     metrics: Optional[List] = None
+    force_overwrite: Optional[bool] = False
 
 
 class AquaEvaluationApp(AquaApp):
@@ -435,12 +444,12 @@ def create(
                     src_uri=evaluation_dataset_path,
                     dst_uri=dst_uri,
                     auth=default_signer(),
-                    force_overwrite=False,
+                    force_overwrite=create_aqua_evaluation_details.force_overwrite,
                 )
             except FileExistsError:
                 raise AquaFileExistsError(
                     f"Dataset {dataset_file} already exists in {create_aqua_evaluation_details.report_path}. "
-                    "Please use a new dataset file name or report path."
+                    "Please use a new dataset file name, report path or set `force_overwrite` as True."
                 )
             logger.debug(
                 f"Uploaded local file {evaluation_dataset_path} to object storage {dst_uri}."
@@ -878,13 +887,17 @@ def get(self, eval_id) -> AquaEvaluationDetail:
             loggroup_id = ""
 
         loggroup_url = get_log_links(region=self.region, log_group_id=loggroup_id)
-        log_url = get_log_links(
-            region=self.region,
-            log_group_id=loggroup_id,
-            log_id=log_id,
-            compartment_id=job_run_details.compartment_id,
-            source_id=jobrun_id
-        ) if job_run_details else ""
+        log_url = (
+            get_log_links(
+                region=self.region,
+                log_group_id=loggroup_id,
+                log_id=log_id,
+                compartment_id=job_run_details.compartment_id,
+                source_id=jobrun_id,
+            )
+            if job_run_details
+            else ""
+        )
 
         log_name = None
         loggroup_name = None
@@ -945,6 +958,7 @@ def list(
         List[AquaEvaluationSummary]:
             The list of the `ads.aqua.evalution.AquaEvaluationSummary`.
         """
+        compartment_id = compartment_id or COMPARTMENT_OCID
         logger.info(f"Fetching evaluations from compartment {compartment_id}.")
         models = utils.query_resources(
             compartment_id=compartment_id,
@@ -960,7 +974,6 @@ def list(
         evaluations = []
         async_tasks = []
         for model in models:
-
             if model.identifier in self._eval_cache.keys():
                 logger.debug(f"Retrieving evaluation {model.identifier} from cache.")
                 evaluations.append(self._eval_cache.get(model.identifier))
@@ -1078,13 +1091,17 @@ def get_status(self, eval_id: str) -> dict:
             loggroup_id = ""
 
         loggroup_url = get_log_links(region=self.region, log_group_id=loggroup_id)
-        log_url = get_log_links(
-            region=self.region,
-            log_group_id=loggroup_id,
-            log_id=log_id,
-            compartment_id=job_run_details.compartment_id,
-            source_id=jobrun_id
-        ) if job_run_details else ""
+        log_url = (
+            get_log_links(
+                region=self.region,
+                log_group_id=loggroup_id,
+                log_id=log_id,
+                compartment_id=job_run_details.compartment_id,
+                source_id=jobrun_id,
+            )
+            if job_run_details
+            else ""
+        )
 
         return dict(
             id=eval_id,
@@ -1129,6 +1146,19 @@ def get_supported_metrics(self) -> dict:
                 ),
                 "args": {},
             },
+            {
+                "use_case": ["text_generation"],
+                "key": "bleu",
+                "name": "bleu",
+                "description": (
+                    "BLEU (Bilingual Evaluation Understudy) is an algorithm for evaluating the "
+                    "quality of text which has been machine-translated from one natural language to another. "
+                    "Quality is considered to be the correspondence between a machine's output and that of a "
+                    "human: 'the closer a machine translation is to a professional human translation, "
+                    "the better it is'."
+                ),
+                "args": {},
+            },
         ]
 
     @telemetry(entry_point="plugin=evaluation&action=load_metrics", name="aqua")

ads/aqua/extension/base_handler.py

@@ -8,14 +8,16 @@
 import traceback
 import uuid
 from dataclasses import asdict, is_dataclass
+from http.client import responses
 from typing import Any
 
 from notebook.base.handlers import APIHandler
-from tornado.web import HTTPError, Application
 from tornado import httputil
-from ads.telemetry.client import TelemetryClient
-from ads.config import AQUA_TELEMETRY_BUCKET, AQUA_TELEMETRY_BUCKET_NS
+from tornado.web import Application, HTTPError
+
 from ads.aqua import logger
+from ads.config import AQUA_TELEMETRY_BUCKET, AQUA_TELEMETRY_BUCKET_NS
+from ads.telemetry.client import TelemetryClient
 
 
 class AquaAPIhandler(APIHandler):
@@ -66,12 +68,15 @@ def finish(self, payload=None):  # pylint: disable=W0221
 
     def write_error(self, status_code, **kwargs):
         """AquaAPIhandler errors are JSON, not human pages."""
-
         self.set_header("Content-Type", "application/json")
         reason = kwargs.get("reason")
         self.set_status(status_code, reason=reason)
         service_payload = kwargs.get("service_payload", {})
-        message = self.get_default_error_messages(service_payload, str(status_code))
+        default_msg = responses.get(status_code, "Unknown HTTP Error")
+        message = self.get_default_error_messages(
+            service_payload, str(status_code), kwargs.get("message", default_msg)
+        )
+
         reply = {
             "status": status_code,
             "message": message,
@@ -84,7 +89,7 @@ def write_error(self, status_code, **kwargs):
             e = exc_info[1]
             if isinstance(e, HTTPError):
                 reply["message"] = e.log_message or message
-                reply["reason"] = e.reason
+                reply["reason"] = e.reason if e.reason else reply["reason"]
                 reply["request_id"] = str(uuid.uuid4())
             else:
                 reply["request_id"] = str(uuid.uuid4())
@@ -102,15 +107,18 @@ def write_error(self, status_code, **kwargs):
         self.finish(json.dumps(reply))
 
     @staticmethod
-    def get_default_error_messages(service_payload: dict, status_code: str):
+    def get_default_error_messages(
+        service_payload: dict,
+        status_code: str,
+        default_msg: str = "Unknown HTTP Error.",
+    ):
         """Method that maps the error messages based on the operation performed or the status codes encountered."""
 
         messages = {
             "400": "Something went wrong with your request.",
             "403": "We're having trouble processing your request with the information provided.",
             "404": "Authorization Failed: The resource you're looking for isn't accessible.",
             "408": "Server is taking too long to response, please try again.",
-            "500": "An error occurred while creating the resource.",
             "create": "Authorization Failed: Could not create resource.",
             "get": "Authorization Failed: The resource you're looking for isn't accessible.",
         }
@@ -119,7 +127,7 @@ def get_default_error_messages(service_payload: dict, status_code: str):
             operation_name = service_payload["operation_name"]
             if operation_name:
                 if operation_name.startswith("create"):
-                    return messages["create"]
+                    return messages["create"] + f" Operation Name: {operation_name}."
                 elif operation_name.startswith("list") or operation_name.startswith(
                     "get"
                 ):
@@ -128,7 +136,7 @@ def get_default_error_messages(service_payload: dict, status_code: str):
         if status_code in messages:
             return messages[status_code]
         else:
-            return "Unknown HTTP Error."
+            return default_msg
 
 
 # todo: remove after error handler is implemented