PHPLIB-607 Support Azure and GCP keystores in FLE (#809)

* Require ext-mongodb 1.10

* Install dev version of ext-mongodb

* Sync spec tests

* Copy spec test changes from libmongoc

* Add support for GCP and Azure KMS providers

* Refactor double encryption test

* Refactor custom endpoint test

* Test spec tests against GCP/Azure

* Expose secrets as env variables

* Expose CSFLE secrets in evergreen

* Slim down build pipeline

* Address code review feedback

* Fix coding standards violation

* Skip Azure FLE test due to missing certificates

* Revise driver-versions axis

* Add more detailed comment about re-enabling driver versions
* Rename latest-stable to latest to account for varying stability
* Prepare correct versions for 1.10-dev and 1.11-dev

Author: alcaeus

.evergreen/config.yml

@@ -207,6 +207,13 @@ functions:
         working_dir: "src"
         script: |
           ${PREPARE_SHELL}
+          export AWS_ACCESS_KEY_ID="${client_side_encryption_aws_access_key_id}"
+          export AWS_SECRET_ACCESS_KEY="${client_side_encryption_aws_secret_access_key}"
+          export AZURE_TENANT_ID="${client_side_encryption_azure_tenant_id}"
+          export AZURE_CLIENT_ID="${client_side_encryption_azure_client_id}"
+          export AZURE_CLIENT_SECRET="${client_side_encryption_azure_client_secret}"
+          export GCP_EMAIL="${client_side_encryption_gcp_email}"
+          export GCP_PRIVATEKEY="${client_side_encryption_gcp_privatekey}"
           PHP_VERSION=${PHP_VERSION} AUTH=${AUTH} SSL=${SSL} MONGODB_URI="${MONGODB_URI}" sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
 
   "run atlas data lake test":
@@ -436,22 +443,27 @@ axes:
   - id: driver-versions
     display_name: Driver Version
     values:
-      - id: "lowest-supported"
-        display_name: "1.9.0"
-        variables:
-          DRIVER_VERSION: "1.9.0"
-      - id: "latest-stable"
-        display_name: "1.9-stable"
-        variables:
-          DRIVER_VERSION: "stable"
-      - id: "1.9-dev"
-        display_name: "1.9-dev"
-        variables:
-          DRIVER_BRANCH: "v1.9"
-      - id: "latest-dev"
+      # TODO: this axis can be cleaned up as we move towards a 1.10 release:
+      # * lowest-supported can be enabled once a 1.10 patch release has been tagged
+      # * latest-stable can be updated when we start tagging 1.10 releases (even beta)
+      # * 1.10-dev can be enabled once 1.10 has been branched
+      # * latest-dev can be enabled once 1.10 has been branched
+#      - id: "lowest-supported"
+#        display_name: "1.10-dev (master)"
+#        variables:
+#          DRIVER_BRANCH: "master"
+      - id: "latest"
         display_name: "1.10-dev (master)"
         variables:
           DRIVER_BRANCH: "master"
+#      - id: "1.10-dev"
+#        display_name: "1.10-dev"
+#        variables:
+#          DRIVER_BRANCH: "v1.10"
+#      - id: "latest-dev"
+#        display_name: "1.11-dev (master)"
+#        variables:
+#          DRIVER_BRANCH: "master"
 
   - id: os-php7
     display_name: OS
@@ -541,7 +553,7 @@ buildvariants:
 # Tests all PHP versions on all operating systems.
 # Only tests against latest MongoDB and ext-mongodb versions
 - matrix_name: "test-php-versions"
-  matrix_spec: {"os-php7": "*", "php-versions": "*", "edge-versions": "latest-stable", "driver-versions": "latest-stable" }
+  matrix_spec: {"os-php7": "*", "php-versions": "*", "edge-versions": "latest-stable", "driver-versions": "latest" }
   exclude_spec:
     # rhel71-power8 fails due to not reaching pecl
     - { "os-php7": "rhel71-power8", "php-versions": "*", edge-versions: "*", "driver-versions": "*" }
@@ -567,7 +579,7 @@ buildvariants:
 # Only tests on Ubuntu 18.04, with latest stable PHP and driver versions
 # Tests against various topologies
 - matrix_name: "test-mongodb-versions"
-  matrix_spec: {"os-php7": "rhel70-test", "php-edge-versions": "latest-stable", "versions": "*", "driver-versions": "latest-stable" }
+  matrix_spec: {"os-php7": "rhel70-test", "php-edge-versions": "latest-stable", "versions": "*", "driver-versions": "latest" }
   display_name: "MongoDB ${versions}, PHP ${php-edge-versions}, ${os-php7}, ext-mongodb ${driver-versions}"
   tasks:
     - name: "test-standalone"
@@ -576,8 +588,9 @@ buildvariants:
 
 # Tests oldest supported version
 # Enables --prefer-lowest for composer to test oldest dependencies against all server versions
+# TODO: driver-versions can be changed back to lowest-supported when that version is enabled in the axis
 - matrix_name: "test-dependencies"
-  matrix_spec: { "dependencies": "lowest", "os-php7": "rhel70-test", "php-edge-versions": "oldest-supported", "versions": "*", "driver-versions": "lowest-supported" }
+  matrix_spec: { "dependencies": "lowest", "os-php7": "rhel70-test", "php-edge-versions": "oldest-supported", "versions": "*", "driver-versions": "latest" }
   display_name: "Dependencies: ${dependencies}, MongoDB ${versions}, PHP ${php-edge-versions}, ${os-php7}, ext-mongodb ${driver-versions}"
   tasks:
     - name: "test-standalone"

.github/workflows/coding-standards.yml

@@ -19,6 +19,8 @@ jobs:
       matrix:
         php-version:
           - "7.4"
+        driver-version:
+          - "mongodb/mongo-php-driver@master"
 
     steps:
       - name: "Checkout"
@@ -29,7 +31,7 @@ jobs:
         uses: shivammathur/cache-extensions@v1
         with:
           php-version: ${{ matrix.php-version }}
-          extensions: "mongodb"
+          extensions: "mongodb-${{ matrix.driver-version }}"
           key: "extcache-v1"
 
       - name: Cache extensions
@@ -40,10 +42,11 @@ jobs:
           restore-keys: ${{ steps.extcache.outputs.key }}
 
       - name: "Install PHP"
-        uses: "shivammathur/setup-php@v2"
+        # Todo: switch to @v2 once a tag has been created
+        uses: "shivammathur/setup-php@develop"
         with:
           coverage: "none"
-          extensions: "mongodb"
+          extensions: "mongodb-${{ matrix.driver-version }}"
           php-version: "${{ matrix.php-version }}"
           tools: "cs2pr"
 

.github/workflows/tests.yml

@@ -26,19 +26,19 @@ jobs:
         mongodb-version:
           - "4.4"
         driver-version:
-          - "stable"
+          - "mongodb/mongo-php-driver@master"
         topology:
           - "server"
         include:
           - os: "ubuntu-20.04"
             php-version: "8.0"
             mongodb-version: "4.4"
-            driver-version: "stable"
+            driver-version: "mongodb/mongo-php-driver@master"
             topology: "replica_set"
           - os: "ubuntu-20.04"
             php-version: "8.0"
             mongodb-version: "4.4"
-            driver-version: "stable"
+            driver-version: "mongodb/mongo-php-driver@master"
             topology: "sharded_cluster"
 
     steps:
@@ -69,7 +69,8 @@ jobs:
           restore-keys: ${{ steps.extcache.outputs.key }}
 
       - name: "Install PHP"
-        uses: "shivammathur/setup-php@v2"
+        # Todo: switch to @v2 once a tag has been created
+        uses: "shivammathur/setup-php@develop"
         with:
           php-version: "${{ matrix.php-version }}"
           tools: "pecl"
@@ -95,3 +96,10 @@ jobs:
         env:
           SYMFONY_DEPRECATIONS_HELPER: 999999
           MONGODB_URI: ${{ steps.setup-mongodb.outputs.cluster-uri }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
+          GCP_EMAIL: ${{ secrets.GCP_EMAIL }}
+          GCP_PRIVATE_KEY: ${{ secrets.GCP_PRIVATE_KEY }}

composer.json

@@ -12,7 +12,7 @@
         "php": "^7.0 || ^8.0",
         "ext-hash": "*",
         "ext-json": "*",
-        "ext-mongodb": "^1.9.0",
+        "ext-mongodb": "^1.10.0",
         "jean85/pretty-package-versions": "^1.2 || ^2.0.1",
         "symfony/polyfill-php80": "^1.19"
     },