Merge branch 'main' of https://github.com/oracle/accelerated-data-science into anomaly_detect_target_columns

Author: govarsha

.github/workflows/codeql.yml

@@ -38,11 +38,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -56,7 +56,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -69,6 +69,6 @@ jobs:
     #   ./location_of_script_within_repo/buildscript.sh
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3
       with:
         category: "/language:${{matrix.language}}"

.github/workflows/publish-to-pypi.yml

@@ -9,9 +9,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.x"
       - name: Build distribution 📦

.github/workflows/run-operators-unit-tests.yml

@@ -31,12 +31,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.10.8"]
+        python-version: ["3.8"]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"

.github/workflows/run-unittests-default_setup.yml

@@ -3,11 +3,7 @@ name: "[Py3.8][Py3.9][Py3.10] tests/unitary/default_setup/**"
 on:
   workflow_dispatch:
   pull_request:
-    paths:
-      - "ads/**"
-      - pyproject.toml
-      - "**requirements.txt"
-      - ".github/workflows/run-unittests*.yml"
+    branches: [ "main" ]
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -34,9 +30,9 @@ jobs:
         python-version: ["3.8", "3.9", "3.10"]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
@@ -48,13 +44,15 @@ jobs:
         name: "Test config setup"
 
       - name: "Run default_setup tests folder ONLY with minimum ADS dependencies"
-        timeout-minutes: 15
+        timeout-minutes: 30
         shell: bash
         env:
           NoDependency: True
         run: |
           set -x # print commands that are executed
           $CONDA/bin/conda init
           source /home/runner/.bashrc
+          conda install python=${{ matrix.python-version }}
           pip install -r test-requirements.txt
+          conda list
           python -m pytest -v -p no:warnings --durations=5 tests/unitary/default_setup

.github/workflows/run-unittests-py38-cov-report.yml

@@ -3,11 +3,7 @@ name: "[Py3.8][COV REPORT] tests/unitary/**"
 on:
   workflow_dispatch:
   pull_request:
-    paths:
-      - "ads/**"
-      - pyproject.toml
-      - "**requirements.txt"
-      - ".github/workflows/run-unittests*.yml"
+    branches: [ "main" ]
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -38,20 +34,23 @@ jobs:
             test-path: "tests/unitary"
             # `model` tests running in "slow_tests",
             # `feature_store` tests has its own test suite
+            # `forecast` tests has its own test suite
+            # 'hpo' tests hangs if run together with all unitary tests. Tests running in separate command before running all unitary
             ignore-path: |
               --ignore tests/unitary/with_extras/model \
               --ignore tests/unitary/with_extras/feature_store \
+              --ignore tests/unitary/with_extras/operator/forecast \
               --ignore tests/unitary/with_extras/hpo
           - name: "slow_tests"
             test-path: "tests/unitary/with_extras/model"
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - uses: ./.github/workflows/create-more-space
         name: "Create more disk space"
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.8"
           cache: "pip"
@@ -64,14 +63,16 @@ jobs:
 
       - uses: ./.github/workflows/test-env-setup
         name: "Test env setup"
-        timeout-minutes: 20
+        timeout-minutes: 30
 
-      # Installing forecast deps for python3.8 test setup only, it will not work with python3.9/3.10, because
-      # automlx do not support py3.9 and some versions of py3.10. This step omitted in -py39-py30.yml workflow
-      - name: "Install Forecasting dependencies"
+      # Installing pii deps for python3.8 test setup only, it will not work with python3.9/3.10, because
+      # 'datapane' library conflicts with pandas>2.2.0, which used in py3.9/3.10 setup
+      - name: "Install PII dependencies"
         run: |
-          pip install -e ".[forecast]"
-
+          pip install -e ".[pii]"
+      - name: "Install featurestore marketplace dependencies"
+        run: |
+          pip install -e ".[feature-store-marketplace]"
       - name: "Run unitary tests folder with maximum ADS dependencies"
         timeout-minutes: 40
         shell: bash
@@ -101,7 +102,7 @@ jobs:
           ${{ matrix.test-path }} ${{ matrix.ignore-path }}
 
       - name: "Save coverage files"
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: cov-reports-${{ matrix.name }}
           path: |
@@ -114,17 +115,17 @@ jobs:
     runs-on: ubuntu-latest
     continue-on-error: true
     needs: test
-    if: ${{ success() }} && && github.event_name == 'pull_request'
+    if: ${{ success() }} && github.event_name == 'pull_request'
     env:
       COMPARE_BRANCH: main
 
     steps:
       - name: "Checkout current branch"
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: "Download coverage files"
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
       - name: "Calculate overall coverage"
         run: |
           set -x # print commands that are executed
@@ -153,7 +154,6 @@ jobs:
           fi
 
       - name: "Calculate coverage diff"
-        if: always()
         run: |
           set -x # print commands that are executed
 
@@ -183,8 +183,8 @@ jobs:
           fi
 
       - name: "Add comment with cov diff to PR"
-        uses: actions/github-script@v6
-        if: always()
+        uses: actions/github-script@v7
+        if: github.event_name == 'pull_request'
         with:
           github-token: ${{ github.token }}
           script: |
@@ -200,7 +200,7 @@ jobs:
           --compare-branch=origin/${{ env.COMPARE_BRANCH }} \
           --html-report=cov-diff.html
       - name: "Save coverage difference report"
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: cov-html-reports
           path: |

.github/workflows/run-unittests-py39-py310.yml

@@ -3,11 +3,6 @@ name: "[Py3.9][Py3.10] - tests/unitary/**"
 on:
   workflow_dispatch:
   pull_request:
-    paths:
-      - "ads/**"
-      - pyproject.toml
-      - "**requirements.txt"
-      - ".github/workflows/run-unittests*.yml"
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -40,22 +35,24 @@ jobs:
             # `model` tests running in "slow_tests",
             # `feature_store` tests has its own test suite
             # `forecast` tests not supported in python 3.9,3.10 (automlx dependency). Tests are running in python3.8 test env, see run-unittests-py38-cov-report.yml
+            # 'pii' tests run only with py3.8, 'datapane' library conflicts with pandas>2.2.0, which used in py3.9/3.10 setup
             # 'hpo' tests hangs if run together with all unitary tests. Tests running in separate command before running all unitary
             ignore-path: |
               --ignore tests/unitary/with_extras/model \
               --ignore tests/unitary/with_extras/feature_store \
               --ignore tests/unitary/with_extras/operator/forecast \
+              --ignore tests/unitary/with_extras/operator/pii \
               --ignore tests/unitary/with_extras/hpo
           - name: "slow_tests"
             test-path: "tests/unitary/with_extras/model"
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - uses: ./.github/workflows/create-more-space
         name: "Create more disk space"
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
@@ -68,7 +65,7 @@ jobs:
 
       - uses: ./.github/workflows/test-env-setup
         name: "Test env setup"
-        timeout-minutes: 20
+        timeout-minutes: 30
 
       - name: "Run hpo tests"
         timeout-minutes: 10

.github/workflows/test-env-setup/action.yml

@@ -14,5 +14,6 @@ runs:
         sudo apt-get install libkrb5-dev graphviz
         $CONDA/bin/conda init
         source /home/runner/.bashrc
-
+        conda install python=${{ matrix.python-version }}
         pip install -r dev-requirements.txt
+        conda list

.pre-commit-config.yaml

@@ -1,52 +1,52 @@
 repos:
-# Standard hooks
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
-    hooks:
-    -   id: check-ast
-        exclude: ^docs/
-    -   id: check-docstring-first
-        exclude: ^(docs/|tests/)
-    -   id: check-json
-    -   id: check-merge-conflict
-    -   id: check-yaml
-        exclude: feature_store_*.yaml
-        args: ['--allow-multiple-documents']
-    -   id: detect-private-key
-    -   id: end-of-file-fixer
-        exclude: '\.ipynb?$'
-    -   id: pretty-format-json
-        args: ['--autofix']
-    -   id: trailing-whitespace
-        args: [--markdown-linebreak-ext=md]
-        exclude: ^docs/
-# Black, the code formatter, natively supports pre-commit
--   repo: https://github.com/psf/black
-    rev: 23.3.0
-    hooks:
-    -   id: black
-        exclude: ^docs/
-# Regex based rst files common mistakes detector
--   repo: https://github.com/pre-commit/pygrep-hooks
-    rev: v1.10.0
-    hooks:
-    -   id: rst-backticks
-        files: ^docs/
-    -   id: rst-inline-touching-normal
-        files: ^docs/
-# Hardcoded secrets and ocids detector
--   repo: https://github.com/gitleaks/gitleaks
-    rev: v8.17.0
-    hooks:
-    -   id: gitleaks
-        exclude: .github/workflows/reusable-actions/set-dummy-conf.yml
-# Oracle copyright checker
--   repo: https://github.com/oracle-samples/oci-data-science-ai-samples/
-    rev: cbe0136f7aaffe463b31ddf3f34b0e16b4b124ff
-    hooks:
-    -   id: check-copyright
-        name: check-copyright
-        entry: .pre-commit-scripts/check-copyright.py
-        language: script
-        types_or: ['python', 'shell', 'bash']
-        exclude: ^docs/
+    # Standard hooks
+    - repo: https://github.com/pre-commit/pre-commit-hooks
+      rev: v4.4.0
+      hooks:
+          - id: check-ast
+            exclude: ^docs/
+          - id: check-docstring-first
+            exclude: ^(docs/|tests/)
+          - id: check-json
+          - id: check-merge-conflict
+          - id: check-yaml
+            exclude: feature_store_*.yaml
+            args: ["--allow-multiple-documents"]
+          - id: detect-private-key
+          - id: end-of-file-fixer
+            exclude: '\.ipynb?$'
+          - id: pretty-format-json
+            args: ["--autofix"]
+          - id: trailing-whitespace
+            args: [--markdown-linebreak-ext=md]
+            exclude: ^docs/
+    # Black, the code formatter, natively supports pre-commit
+    - repo: https://github.com/psf/black
+      rev: 23.3.0
+      hooks:
+          - id: black
+            exclude: ^docs/
+    # Regex based rst files common mistakes detector
+    - repo: https://github.com/pre-commit/pygrep-hooks
+      rev: v1.10.0
+      hooks:
+          - id: rst-backticks
+            files: ^docs/
+          - id: rst-inline-touching-normal
+            files: ^docs/
+    # Hardcoded secrets and ocids detector
+    - repo: https://github.com/gitleaks/gitleaks
+      rev: v8.17.0
+      hooks:
+          - id: gitleaks
+            exclude: .github/workflows/reusable-actions/set-dummy-conf.yml
+    # Oracle copyright checker
+    - repo: https://github.com/oracle-samples/oci-data-science-ai-samples/
+      rev: 1bc5270a443b791c62f634233c0f4966dfcc0dd6
+      hooks:
+          - id: check-copyright
+            name: check-copyright
+            entry: .pre-commit-scripts/check-copyright.py
+            language: script
+            types_or: ["python", "shell", "bash"]
+            exclude: ^docs/

CODEOWNERS

@@ -0,0 +1 @@
+* @darenr @mayoor @mrDzurb @VipulMascarenhas @qiuosier