Odsc 45131/ads interfaces (#282)

Author: guptadivyank

ads/feature_store/dataset.py

@@ -245,6 +245,16 @@ def id(self) -> str:
         """
         return self.get_spec(self.CONST_ID)
 
+    @property
+    def features(self) -> List[DatasetFeature]:
+        return [
+            DatasetFeature(**feature_dict)
+            for feature_dict in self.get_spec(self.CONST_OUTPUT_FEATURE_DETAILS)[
+                self.CONST_ITEMS
+            ]
+            or []
+        ]
+
     def with_id(self, id: str) -> "Dataset":
         return self.set_spec(self.CONST_ID, id)
 
@@ -709,6 +719,28 @@ def delete(self):
 
         dataset_execution_strategy.delete_dataset(self, dataset_job)
 
+    def get_features(self) -> List[DatasetFeature]:
+        """
+        Returns all the features in the dataset.
+
+        Returns:
+            List[DatasetFeature]
+        """
+
+        return self.features
+
+    def get_features_df(self) -> "pandas.DataFrame":
+        """
+        Returns all the features as pandas dataframe.
+
+        Returns:
+            pandas.DataFrame
+        """
+        records = []
+        for feature in self.features:
+            records.append({"name": feature.feature_name, "type": feature.feature_type})
+        return pandas.DataFrame.from_records(records)
+
     def update(self, **kwargs) -> "Dataset":
         """Updates Dataset in the feature store.
 
@@ -752,7 +784,18 @@ def _update_from_oci_dataset_model(self, oci_dataset: OCIDataset) -> "Dataset":
 
         for infra_attr, dsc_attr in self.attribute_map.items():
             if infra_attr in dataset_details:
-                self.set_spec(infra_attr, dataset_details[infra_attr])
+                if infra_attr == self.CONST_OUTPUT_FEATURE_DETAILS:
+                    # May not need if we fix the backend and add dataset_id to the output_feature
+                    features_list = []
+                    for output_feature in dataset_details[infra_attr]["items"]:
+                        output_feature["datasetId"] = dataset_details[self.CONST_ID]
+                        features_list.append(output_feature)
+
+                    value = {self.CONST_ITEMS: features_list}
+                else:
+                    value = dataset_details[infra_attr]
+
+                self.set_spec(infra_attr, value)
 
         return self
 
@@ -792,6 +835,33 @@ def materialise(
 
         dataset_execution_strategy.ingest_dataset(self, dataset_job)
 
+    def get_last_job(self) -> "DatasetJob":
+        """Gets the Job details for the last running Dataset job.
+
+        Returns:
+            DatasetJob
+        """
+
+        if not self.id:
+            raise ValueError(
+                "Dataset needs to be saved to the feature store before getting associated jobs."
+            )
+
+        if not self.job_id:
+            ds_job = DatasetJob.list(
+                dataset_id=self.id,
+                compartment_id=self.compartment_id,
+                sort_by="timeCreated",
+                limit="1",
+            )
+            if not ds_job:
+                raise ValueError(
+                    "Unable to retrieve the associated last job. Please make sure you materialized the data."
+                )
+            self.with_job_id(ds_job[0].id)
+            return ds_job[0]
+        return DatasetJob.from_id(self.job_id)
+
     @deprecated(details="preview functionality is deprecated. Please use as_of.")
     def preview(
         self,
@@ -947,14 +1017,8 @@ def get_statistics(self, job_id: str = None) -> "Statistics":
             raise ValueError(
                 "Dataset needs to be saved to the feature store before retrieving the statistics"
             )
-        stat_job_id = job_id
-        if job_id is None:
-            if self.job_id is None:
-                raise ValueError(
-                    "Unable to retrieve the last job,please provide the job id,make sure you materialised the data'"
-                )
-            else:
-                stat_job_id = self.job_id
+
+        stat_job_id = job_id if job_id is not None else self.get_last_job().id
 
         # TODO: take the one in memory or will list down job ids and find the latest
         dataset_job = DatasetJob.from_id(stat_job_id)
@@ -980,14 +1044,8 @@ def get_validation_output(self, job_id: str = None) -> "ValidationOutput":
             raise ValueError(
                 "Dataset needs to be saved to the feature store before retrieving the validation report"
             )
-        validation_job_id = job_id
-        if job_id is None:
-            if self.job_id is None:
-                raise ValueError(
-                    "Unable to retrieve the last job,please provide the job id,make sure you materialised the data'"
-                )
-            else:
-                validation_job_id = self.job_id
+
+        validation_job_id = job_id if job_id is not None else self.get_last_job().id
 
         # retrieve the validation output JSON from data_flow_batch_execution_output
         dataset_job = DatasetJob.from_id(validation_job_id)

ads/feature_store/docs/source/dataset.rst

@@ -227,23 +227,14 @@ The ``get_statistics()`` method takes the following optional parameter:
 
 Get features
 ============
-You can call the ``get_features_dataframe()`` method of the Dataset instance to fetch features in a dataset.
+You can call the ``get_features_df()`` method of the Dataset instance to fetch features in a dataset.
 
 .. code-block:: python3
 
   # Fetch features for a dataset
-  df = dataset.get_features_dataframe()
+  df = dataset.get_features_df()
   df.show()
 
-Get input schema details
-========================
-You can call the ``get_input_schema_dataframe()`` method of the Dataset instance to fetch input schema details of a dataset.
-
-.. code-block:: python3
-
-  # Fetch input schema details for a dataset
-  df = dataset.get_input_schema_dataframe()
-  df.show()
 
 Preview
 ========

ads/feature_store/feature_group.py

@@ -750,13 +750,8 @@ def get_features_df(self) -> "pd.DataFrame":
         """
         records = []
         for feature in self.features:
-            records.append(
-                {
-                    "name": feature.feature_name,
-                    "type": feature.feature_type,
-                    "feature_group_id": feature.feature_group_id,
-                }
-            )
+            records.append({"name": feature.feature_name, "type": feature.feature_type})
+
         return pd.DataFrame.from_records(records)
 
     def get_input_features_df(self) -> "pd.DataFrame":
@@ -927,10 +922,18 @@ def get_last_job(self) -> "FeatureGroupJob":
             )
 
         if not self.job_id:
-            raise ValueError(
-                "Associated jobs cannot be retrieved before calling 'materialise' or 'delete'."
+            fg_job = FeatureGroupJob.list(
+                feature_group_id=self.id,
+                compartment_id=self.compartment_id,
+                sort_by="timeCreated",
+                limit="1",
             )
-
+            if not fg_job:
+                raise ValueError(
+                    "Unable to retrieve the associated last job. Please make sure you materialized the data."
+                )
+            self.with_job_id(fg_job[0].id)
+            return fg_job[0]
         return FeatureGroupJob.from_id(self.job_id)
 
     def select(self, features: Optional[List[str]] = []) -> Query:
@@ -1350,7 +1353,7 @@ def get_statistics(self, job_id: str = None) -> "Statistics":
                 "FeatureGroup needs to be saved to the feature store before retrieving the statistics"
             )
 
-        stat_job_id = self._get_job_id(job_id)
+        stat_job_id = job_id if job_id is not None else self.get_last_job().id
 
         # TODO: take the one in memory or will list down job ids and find the latest
         fg_job = FeatureGroupJob.from_id(stat_job_id)
@@ -1379,7 +1382,7 @@ def get_validation_output(self, job_id: str = None) -> "ValidationOutput":
                 "FeatureGroup needs to be saved to the feature store before retrieving the validation report"
             )
 
-        validation_job_id = self._get_job_id(job_id)
+        validation_job_id = job_id if job_id is not None else self.get_last_job().id
 
         # Retrieve the validation output JSON from data_flow_batch_execution_output.
         fg_job = FeatureGroupJob.from_id(validation_job_id)

ads/feature_store/mixin/oci_feature_store.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
+from ads.common.decorator.utils import class_or_instance_method
 
 # Copyright (c) 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
@@ -77,3 +78,44 @@ def init_client(
     @property
     def client(self) -> oci.feature_store.feature_store_client.FeatureStoreClient:
         return super().client
+
+    @class_or_instance_method
+    def list_resource(
+        cls, compartment_id: str = None, limit: int = 0, **kwargs
+    ) -> list:
+        """Generic method to list OCI resources
+
+        Parameters
+        ----------
+        compartment_id : str
+            Compartment ID of the OCI resources. Defaults to None.
+            If compartment_id is not specified,
+            the value of NB_SESSION_COMPARTMENT_OCID in environment variable will be used.
+        limit : int
+            The maximum number of items to return. Defaults to 0, All items will be returned
+        **kwargs :
+            Additional keyword arguments to filter the resource.
+            The kwargs are passed into OCI API.
+
+        Returns
+        -------
+        list
+            A list of OCI resources
+
+        Raises
+        ------
+        NotImplementedError
+            List method is not supported or implemented.
+
+        """
+        if limit:
+            items = cls._find_oci_method("list")(
+                cls.check_compartment_id(compartment_id), limit=limit, **kwargs
+            ).data.items
+        else:
+            items = oci.pagination.list_call_get_all_results(
+                cls._find_oci_method("list"),
+                cls.check_compartment_id(compartment_id),
+                **kwargs,
+            ).data
+        return [cls.from_oci_model(item) for item in items]