Merge branch 'feature/aquav1.0.1' into update_telemetry_for_model_shapes

Author: VipulMascarenhas

ads/aqua/__init__.py

@@ -5,30 +5,58 @@
 
 
 import logging
-import sys
 import os
+import sys
+
+from ads import set_auth
 from ads.aqua.utils import fetch_service_compartment
 from ads.config import NB_SESSION_OCID, OCI_RESOURCE_PRINCIPAL_VERSION
-from ads import set_auth
 
-logger = logging.getLogger(__name__)
-handler = logging.StreamHandler(sys.stdout)
-logger.setLevel(logging.INFO)
+ENV_VAR_LOG_LEVEL = "ADS_AQUA_LOG_LEVEL"
+
+
+def get_logger_level():
+    """Retrieves logging level from environment variable `LOG_LEVEL`."""
+    level = os.environ.get(ENV_VAR_LOG_LEVEL, "INFO").upper()
+    return level
+
+
+def configure_aqua_logger():
+    """Configures the AQUA logger."""
+    log_level = get_logger_level()
+    logger = logging.getLogger(__name__)
+    logger.setLevel(log_level)
+
+    handler = logging.StreamHandler(sys.stdout)
+    formatter = logging.Formatter(
+        "%(asctime)s - %(name)s.%(module)s - %(levelname)s - %(message)s"
+    )
+    handler.setFormatter(formatter)
+    handler.setLevel(log_level)
+
+    logger.addHandler(handler)
+    logger.propagate = False
+    return logger
+
+
+logger = configure_aqua_logger()
+
+
+def set_log_level(log_level: str):
+    """Global for setting logging level."""
+
+    log_level = log_level.upper()
+    logger.setLevel(log_level.upper())
+    logger.handlers[0].setLevel(log_level)
+
 
 if OCI_RESOURCE_PRINCIPAL_VERSION:
     set_auth("resource_principal")
 
-ODSC_MODEL_COMPARTMENT_OCID = os.environ.get("ODSC_MODEL_COMPARTMENT_OCID")
+ODSC_MODEL_COMPARTMENT_OCID = (
+    os.environ.get("ODSC_MODEL_COMPARTMENT_OCID") or fetch_service_compartment()
+)
 if not ODSC_MODEL_COMPARTMENT_OCID:
-    try:
-        ODSC_MODEL_COMPARTMENT_OCID = fetch_service_compartment()
-    except:
-        pass
-
-if not ODSC_MODEL_COMPARTMENT_OCID:
-    logger.error(
-        f"ODSC_MODEL_COMPARTMENT_OCID environment variable is not set for Aqua."
-    )
     if NB_SESSION_OCID:
         logger.error(
             f"Aqua is not available for this notebook session {NB_SESSION_OCID}."

ads/aqua/base.py

@@ -19,7 +19,6 @@
     get_artifact_path,
     is_valid_ocid,
     load_config,
-    logger,
 )
 from ads.common import oci_client as oc
 from ads.common.auth import default_signer
@@ -164,7 +163,7 @@ def create_model_version_set(
         tag = Tags.AQUA_FINE_TUNING.value
 
         if not model_version_set_id:
-            tag = Tags.AQUA_FINE_TUNING.value # TODO: Fix this
+            tag = Tags.AQUA_FINE_TUNING.value  # TODO: Fix this
             try:
                 model_version_set = ModelVersionSet.from_name(
                     name=model_version_set_name,

ads/aqua/cli.py

@@ -3,17 +3,41 @@
 
 # Copyright (c) 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+import os
 
+from ads.aqua import ENV_VAR_LOG_LEVEL, set_log_level
 from ads.aqua.deployment import AquaDeploymentApp
+from ads.aqua.evaluation import AquaEvaluationApp
 from ads.aqua.finetune import AquaFineTuningApp
 from ads.aqua.model import AquaModelApp
-from ads.aqua.evaluation import AquaEvaluationApp
 
 
 class AquaCommand:
-    """Contains the command groups for project Aqua."""
+    """Contains the command groups for project Aqua.
+
+    Acts as an entry point for managing different components of the Aqua
+    project including model management, fine-tuning, deployment, and
+    evaluation.
+    """
 
     model = AquaModelApp
     fine_tuning = AquaFineTuningApp
     deployment = AquaDeploymentApp
     evaluation = AquaEvaluationApp
+
+    def __init__(
+        self,
+        log_level: str = os.environ.get(ENV_VAR_LOG_LEVEL, "ERROR").upper(),
+    ):
+        """
+        Initialize the command line interface settings for the Aqua project.
+
+        FLAGS
+        -----
+        log_level (str):
+            Sets the logging level for the application.
+            Default is retrieved from environment variable `LOG_LEVEL`,
+            or 'ERROR' if not set. Example values include 'DEBUG', 'INFO',
+            'WARNING', 'ERROR', and 'CRITICAL'.
+        """
+        set_log_level(log_level)

ads/aqua/evaluation.py

@@ -38,6 +38,7 @@
     JOB_INFRASTRUCTURE_TYPE_DEFAULT_NETWORKING,
     NB_SESSION_IDENTIFIER,
     UNKNOWN,
+    extract_id_and_name_from_tag,
     fire_and_forget,
     get_container_image,
     is_valid_ocid,
@@ -694,7 +695,7 @@ def create(
         self.telemetry.record_event_async(
             category="aqua/evaluation",
             action="create",
-            detail=evaluation_source.display_name,
+            detail=self._get_service_model_name(evaluation_source),
         )
 
         return AquaEvaluationSummary(
@@ -784,6 +785,34 @@ def _build_evaluation_runtime(
         )
 
         return runtime
+    
+    @staticmethod
+    def _get_service_model_name(
+        source: Union[ModelDeployment, DataScienceModel]
+    ) -> str:
+        """Gets the service model name from source. If it's ModelDeployment, needs to check
+        if its model has been fine tuned or not.
+
+        Parameters
+        ----------
+        source: Union[ModelDeployment, DataScienceModel]
+            An instance of either ModelDeployment or DataScienceModel
+
+        Returns
+        -------
+        str:
+            The service model name of source.
+        """
+        if isinstance(source, ModelDeployment):
+            fine_tuned_model_tag = source.freeform_tags.get(
+                Tags.AQUA_FINE_TUNED_MODEL_TAG.value, UNKNOWN
+            )
+            if not fine_tuned_model_tag:
+                return source.freeform_tags.get(Tags.AQUA_MODEL_NAME_TAG.value)
+            else:
+                return extract_id_and_name_from_tag(fine_tuned_model_tag)[1]
+        
+        return source.display_name
 
     @staticmethod
     def _get_evaluation_container(source_id: str) -> str:
@@ -983,7 +1012,7 @@ def list(
                         self._process_evaluation_summary(model=model, jobrun=jobrun)
                     )
                 except Exception as exc:
-                    logger.error(
+                    logger.debug(
                         f"Processing evaluation: {model.identifier} generated an exception: {exc}"
                     )
                     evaluations.append(
@@ -1028,7 +1057,7 @@ def _if_eval_artifact_exist(
             return True if response.status == 200 else False
         except oci.exceptions.ServiceError as ex:
             if ex.status == 404:
-                logger.info("Evaluation artifact not found.")
+                logger.debug(f"Evaluation artifact not found for {model.identifier}.")
                 return False
 
     @telemetry(entry_point="plugin=evaluation&action=get_status", name="aqua")
@@ -1574,8 +1603,9 @@ def _build_resource_identifier(
                 ),
             )
         except Exception as e:
-            logger.error(
-                f"Failed to construct AquaResourceIdentifier from given id=`{id}`, and name=`{name}`, {str(e)}"
+            logger.debug(
+                f"Failed to construct AquaResourceIdentifier from given id=`{id}`, and name=`{name}`. "
+                f"DEBUG INFO: {str(e)}"
             )
             return AquaResourceIdentifier()
 
@@ -1621,7 +1651,7 @@ def _fetch_runtime_params(
             )
             if not params.get(EvaluationConfig.PARAMS):
                 raise AquaMissingKeyError(
-                    "model parameters have not been saved in correct format in model taxonomy.",
+                    "model parameters have not been saved in correct format in model taxonomy. ",
                     service_payload={"params": params},
                 )
             # TODO: validate the format of parameters.
@@ -1653,7 +1683,7 @@ def _build_job_identifier(
 
         except Exception as e:
             logger.debug(
-                f"Failed to get job details from job_run_details: {job_run_details}"
+                f"Failed to get job details from job_run_details: {job_run_details} "
                 f"DEBUG INFO:{str(e)}"
             )
             return AquaResourceIdentifier()

ads/aqua/extension/common_handler.py

@@ -10,7 +10,7 @@
 from ads.aqua.decorator import handle_exceptions
 from ads.aqua.exception import AquaResourceAccessError
 from ads.aqua.extension.base_handler import AquaAPIhandler
-from ads.aqua.utils import known_realm
+from ads.aqua.utils import known_realm, fetch_service_compartment
 
 
 class ADSVersionHandler(AquaAPIhandler):
@@ -39,7 +39,7 @@ def get(self):
             AquaResourceAccessError: raised when aqua is not accessible in the given session/region.
 
         """
-        if ODSC_MODEL_COMPARTMENT_OCID:
+        if ODSC_MODEL_COMPARTMENT_OCID or fetch_service_compartment():
             return self.finish(dict(status="ok"))
         elif known_realm():
             return self.finish(dict(status="compatible"))

ads/aqua/finetune.py

@@ -15,7 +15,7 @@
     UpdateModelProvenanceDetails,
 )
 
-from ads.aqua import ODSC_MODEL_COMPARTMENT_OCID
+from ads.aqua import ODSC_MODEL_COMPARTMENT_OCID, logger
 from ads.aqua.base import AquaApp
 from ads.aqua.data import AquaResourceIdentifier, Resource, Tags
 from ads.aqua.exception import AquaFileExistsError, AquaValueError
@@ -29,7 +29,6 @@
     UNKNOWN,
     UNKNOWN_DICT,
     get_container_image,
-    logger,
     upload_local_to_os,
 )
 from ads.common.auth import default_signer
@@ -470,6 +469,7 @@ def create(
             **telemetry_kwargs,
         )
         # tracks unique fine-tuned models that were created in the user compartment
+        # TODO: retrieve the service model name for FT custom models.
         self.telemetry.record_event_async(
             category="aqua/service/finetune",
             action="create",

ads/aqua/model.py

@@ -14,7 +14,7 @@
 from cachetools import TTLCache
 from oci.data_science.models import JobRun, Model
 
-from ads.aqua import logger, utils
+from ads.aqua import ODSC_MODEL_COMPARTMENT_OCID, logger, utils
 from ads.aqua.base import AquaApp
 from ads.aqua.constants import (
     TRAINING_METRICS_FINAL,
@@ -26,7 +26,6 @@
 )
 from ads.aqua.data import AquaResourceIdentifier, Tags
 from ads.aqua.exception import AquaRuntimeError
-
 from ads.aqua.training.exceptions import exit_code_dict
 from ads.aqua.utils import (
     LICENSE_TXT,
@@ -50,7 +49,6 @@
     PROJECT_OCID,
     TENANCY_OCID,
 )
-from ads.aqua import ODSC_MODEL_COMPARTMENT_OCID
 from ads.model import DataScienceModel
 from ads.model.model_metadata import MetadataTaxonomyKeys, ModelCustomMetadata
 from ads.telemetry import telemetry
@@ -228,7 +226,7 @@ def __post_init__(
             ).value
         except Exception as e:
             logger.debug(
-                f"Failed to extract model hyperparameters from {model.id}:" f"{str(e)}"
+                f"Failed to extract model hyperparameters from {model.id}: " f"{str(e)}"
             )
             model_hyperparameters = {}