Merge branch 'feature/aquav1.0.1' into ODSC-55768/update_error

Author: lu-ohai

.github/workflows/run-unittests-default_setup.yml

@@ -4,6 +4,12 @@ on:
   workflow_dispatch:
   pull_request:
     branches: [ "main" ]
+    paths:
+      - "ads/**"
+      - "!ads/opctl/operator/**"
+      - "!ads/feature_store/**"
+      - "pyproject.toml"
+
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value

.github/workflows/run-unittests-py38-cov-report.yml

@@ -4,6 +4,11 @@ on:
   workflow_dispatch:
   pull_request:
     branches: [ "main" ]
+    paths:
+      - "ads/**"
+      - "!ads/opctl/operator/**"
+      - "!ads/feature_store/**"
+      - "pyproject.toml"
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value

.github/workflows/run-unittests-py39-py310.yml

@@ -3,6 +3,12 @@ name: "[Py3.9][Py3.10] - tests/unitary/**"
 on:
   workflow_dispatch:
   pull_request:
+    branches: [ "main" ]
+    paths:
+      - "ads/**"
+      - "!ads/opctl/operator/**"
+      - "!ads/feature_store/**"
+      - "pyproject.toml"
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value

ads/aqua/evaluation.py

@@ -833,13 +833,13 @@ def get(self, eval_id) -> AquaEvaluationDetail:
         logger.info(f"Fetching evaluation: {eval_id} details ...")
 
         resource = utils.query_resource(eval_id)
-        model_provenance = self.ds_client.get_model_provenance(eval_id).data
-
         if not resource:
             raise AquaRuntimeError(
                 f"Failed to retrieve evalution {eval_id}."
                 "Please check if the OCID is correct."
             )
+        model_provenance = self.ds_client.get_model_provenance(eval_id).data
+
         jobrun_id = model_provenance.training_id
         job_run_details = self._fetch_jobrun(
             resource, use_rqs=False, jobrun_id=jobrun_id
@@ -1038,14 +1038,14 @@ def get_status(self, eval_id: str) -> dict:
         """
         eval = utils.query_resource(eval_id)
 
-        # TODO: add job_run_id as input param to skip the query below
-        model_provenance = self.ds_client.get_model_provenance(eval_id).data
-
         if not eval:
             raise AquaRuntimeError(
                 f"Failed to retrieve evalution {eval_id}."
                 "Please check if the OCID is correct."
             )
+
+        model_provenance = self.ds_client.get_model_provenance(eval_id).data
+
         jobrun_id = model_provenance.training_id
         job_run_details = self._fetch_jobrun(eval, use_rqs=False, jobrun_id=jobrun_id)
 
@@ -1295,7 +1295,10 @@ def cancel(self, eval_id) -> dict:
             raise AquaRuntimeError(
                 f"Failed to get evaluation details for model {eval_id}"
             )
-        job_run_id = model.provenance_metadata.training_id
+
+        job_run_id = (
+            model.provenance_metadata.training_id if model.provenance_metadata else None
+        )
         if not job_run_id:
             raise AquaMissingKeyError(
                 "Model provenance is missing job run training_id key"
@@ -1358,7 +1361,7 @@ def delete(self, eval_id):
             job_id = model.custom_metadata_list.get(
                 EvaluationCustomMetadata.EVALUATION_JOB_ID.value
             ).value
-        except ValueError:
+        except Exception:
             raise AquaMissingKeyError(
                 f"Custom metadata is missing {EvaluationCustomMetadata.EVALUATION_JOB_ID.value} key"
             )
@@ -1390,7 +1393,7 @@ def _delete_job_and_model(job, model):
             )
 
     def load_evaluation_config(self, eval_id):
-        # TODO
+        """Loads evaluation config."""
         return {
             "model_params": {
                 "max_tokens": 500,
@@ -1568,20 +1571,6 @@ def _build_resource_identifier(
             )
             return AquaResourceIdentifier()
 
-    def _get_jobrun(
-        self, model: oci.resource_search.models.ResourceSummary, mapping: dict = {}
-    ) -> Union[
-        oci.resource_search.models.ResourceSummary, oci.data_science.models.JobRun
-    ]:
-        jobrun_id = self._get_attribute_from_model_metadata(
-            model, EvaluationCustomMetadata.EVALUATION_JOB_RUN_ID.value
-        )
-        job_run = mapping.get(jobrun_id)
-
-        if not job_run:
-            job_run = self._fetch_jobrun(model, use_rqs=True, jobrun_id=jobrun_id)
-        return job_run
-
     def _fetch_jobrun(
         self,
         resource: oci.resource_search.models.ResourceSummary,
@@ -1758,7 +1747,7 @@ def _extract_job_lifecycle_details(self, lifecycle_details: str) -> str:
         Examples
         --------
         >>> _extract_job_lifecycle_details("Job run artifact execution failed with exit code 16")
-        'The evaluation configuration is invalid due to content validation errors.'
+        'Validation errors in the evaluation config. Exit code: 16.'
 
         >>> _extract_job_lifecycle_details("Job completed successfully.")
         'Job completed successfully.'

ads/aqua/extension/common_handler.py

@@ -7,10 +7,10 @@
 from importlib import metadata
 
 from ads.aqua import ODSC_MODEL_COMPARTMENT_OCID
-from ads.aqua.exception import AquaResourceAccessError
-from ads.aqua.utils import known_realm
 from ads.aqua.decorator import handle_exceptions
+from ads.aqua.exception import AquaResourceAccessError
 from ads.aqua.extension.base_handler import AquaAPIhandler
+from ads.aqua.utils import known_realm
 
 
 class ADSVersionHandler(AquaAPIhandler):

docs/source/release_notes.rst

@@ -4,9 +4,10 @@ Release Notes
 
 2.11.7
 ------
-Release date: April 8, 2024
+Release date: April 18, 2024
 
-* Fixed bugs and introduced enhancements following our recent release, which included internal adjustments for future features and updates for the Jupyter Lab 3 upgrade.
+* Fixed the bug in ``ADSDataset.show_in_notebook()``.
+* Updated langchain version. 
 
 
 2.11.6

tests/unitary/with_extras/aqua/test_data/valid_eval_artifact/report.html

@@ -1 +1,2 @@
 This is a sample evaluation report.html.
+Standard deviation (σ)

tests/unitary/with_extras/aqua/test_evaluation.py

@@ -569,7 +569,6 @@ def test_get(self, mock_get_model_provenance_response, mock_get_job_run_response
     def test_get_fail(self, mock_query_resource):
         """Tests get evaluation details failed because of invalid eval id."""
         mock_query_resource.return_value = None
-        self.app.ds_client.get_model_provenance = MagicMock()
         with self.assertRaises(AquaRuntimeError) as context:
             self.app.get(TestDataset.INVALID_EVAL_ID)
 
@@ -624,9 +623,10 @@ def test_download_report(
         mock_dsc_model_from_id.assert_called_with(TestDataset.EVAL_ID)
         self.print_expected_response(response, "DOWNLOAD REPORT")
         self.assert_payload(response, AquaEvalReport)
-        read_content = base64.b64decode(response.content)
+        read_content = base64.b64decode(response.content).decode()
         assert (
-            read_content == b"This is a sample evaluation report.html.\n"
+            read_content
+            == "This is a sample evaluation report.html.\nStandard deviation (σ)\n"
         ), read_content
         assert self.app._report_cache.currsize == 1
 
@@ -676,10 +676,10 @@ def test_cancel_evaluation(
     @parameterized.expand(
         [
             (None, AquaRuntimeError),
-            # (
-            #     DataScienceModel(),
-            #     AquaMissingKeyError,
-            # ),
+            (
+                DataScienceModel(),
+                AquaMissingKeyError,
+            ),
         ]
     )
     @patch.object(DataScienceModel, "from_id")
@@ -821,7 +821,6 @@ def test_get_status_when_missing_jobrun(
     def test_get_status_failed(self, mock_query_resource):
         """Tests when no correct evaluation found."""
         mock_query_resource.return_value = None
-        self.app.ds_client.get_model_provenance = MagicMock()
         with self.assertRaises(AquaRuntimeError) as context:
             self.app.get_status(TestDataset.INVALID_EVAL_ID)