Merge branch 'develop' into PSQL-773

Author: hunleyd

.github/actionlint.yaml

@@ -0,0 +1,9 @@
+self-hosted-runner:
+  labels:
+    - aarch64-darwin
+    - aarch64-linux
+    - blacksmith-32vcpu-ubuntu-2404
+    - blacksmith-2vcpu-ubuntu-2404
+    - blacksmith-2vcpu-ubuntu-2404-arm
+    - blacksmith-4vcpu-ubuntu-2404
+    - large-linux-arm

.github/actions/build-ami/action.yml

@@ -0,0 +1,87 @@
+name: Build AMI
+description: Build both stage 1 and stage 2 AMIs
+
+inputs:
+  postgres_version:
+    description: 'PostgreSQL major version (e.g., 15)'
+    required: true
+  region:
+    description: 'AWS region'
+    required: true
+  ami_regions:
+    description: 'AMI regions as JSON array (e.g., ["us-east-1"])'
+    required: true
+  git_sha:
+    description: 'Git SHA for this build'
+    required: true
+  ami_name_prefix:
+    description: 'Prefix for the AMI name'
+    required: false
+    default: 'supabase-postgres'
+
+outputs:
+  stage2_ami_id:
+    description: 'The AMI ID of the stage 2 build'
+    value: ${{ steps.build-stage2.outputs.stage2_ami_id }}
+  postgres_release_version:
+    description: 'The PostgreSQL release version'
+    value: ${{ steps.generate-vars.outputs.version }}
+  execution_id:
+    description: 'The execution ID for this build'
+    value: ${{ steps.set-execution-id.outputs.execution_id }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set execution ID
+      id: set-execution-id
+      shell: bash
+      run: |
+        EXECUTION_ID="${{ github.run_id }}-${{ inputs.postgres_version }}"
+        echo "EXECUTION_ID=$EXECUTION_ID" >> $GITHUB_ENV
+        echo "execution_id=$EXECUTION_ID" >> $GITHUB_OUTPUT
+
+    - name: Generate common-nix.vars.pkr.hcl
+      id: generate-vars
+      shell: bash
+      run: |
+        PG_VERSION=$(nix run nixpkgs#yq -- -r '.postgres_release["postgres${{ inputs.postgres_version }}"]' ansible/vars.yml)
+        echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl
+        echo "" >> common-nix.vars.pkr.hcl
+        git add -f common-nix.vars.pkr.hcl
+        echo "version=$PG_VERSION" >> $GITHUB_OUTPUT
+
+    - name: Build AMI stage 1
+      shell: bash
+      env:
+        POSTGRES_MAJOR_VERSION: ${{ inputs.postgres_version }}
+        POSTGRES_VERSION: ${{ steps.generate-vars.outputs.version }}
+        AWS_MAX_ATTEMPTS: 10
+        AWS_RETRY_MODE: adaptive
+        AWS_REGION: ${{ inputs.region }}
+      run: |
+        nix run .#build-ami -- stage1 \
+          -var "git-head-version=${{ inputs.git_sha }}" \
+          -var "packer-execution-id=${{ env.EXECUTION_ID }}" \
+          -var "ansible_arguments=-e postgresql_major=${{ inputs.postgres_version }}" \
+          -var 'ami_regions=${{ inputs.ami_regions }}' \
+          amazon-arm64-nix.pkr.hcl
+
+    - name: Build AMI stage 2
+      id: build-stage2
+      shell: bash
+      env:
+        POSTGRES_MAJOR_VERSION: ${{ inputs.postgres_version }}
+        POSTGRES_VERSION: ${{ steps.generate-vars.outputs.version }}
+        PACKER_EXECUTION_ID: ${{ env.EXECUTION_ID }}
+        AWS_MAX_ATTEMPTS: 10
+        AWS_RETRY_MODE: adaptive
+        AWS_REGION: ${{ inputs.region }}
+      run: |
+        nix run .#build-ami -- stage2 \
+          -var "git-head-version=${{ inputs.git_sha }}" \
+          -var "packer-execution-id=${{ env.EXECUTION_ID }}" \
+          -var "postgres_major_version=${{ inputs.postgres_version }}" \
+          -var "ami_name=${{ inputs.ami_name_prefix }}" \
+          -var "git_sha=${{ inputs.git_sha }}" \
+          stage2-nix-psql.pkr.hcl

.github/actions/nix-install-ephemeral/action.yml

@@ -5,6 +5,10 @@ inputs:
     description: 'Whether to push build outputs to the Nix binary cache'
     required: false
     default: 'false'
+  aws-region:
+    description: 'AWS region for the Nix binary cache S3 bucket'
+    required: false
+    default: 'us-east-1'
 runs:
   using: 'composite'
   steps:
@@ -13,7 +17,7 @@ runs:
       if: ${{ inputs.push-to-cache == 'true' }}
       with:
         role-to-assume: ${{ env.DEV_AWS_ROLE }}
-        aws-region: "us-east-1"
+        aws-region: ${{ inputs.aws-region }}
         output-credentials: true
         role-duration-seconds: 7200
     - name: Setup AWS credentials for Nix

.github/actions/nix-install-self-hosted/action.yml

@@ -0,0 +1,30 @@
+name: 'Configure Nix on self hosted runners'
+description: 'Sets up AWS credentials to push to the Nix binary cache'
+inputs:
+  aws-role-duration:
+    description: 'AWS role session duration in seconds'
+    required: false
+    default: '18000'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: aws-creds
+      uses: aws-actions/configure-aws-credentials@v4.3.1
+      with:
+        disable-retry: true
+        aws-region: us-east-2
+        role-to-assume: arn:aws:iam::436098097459:role/nix-artifacts-deploy-role # supabase-dev
+        role-session-name: gha-oidc-${{ github.run_id }}
+        role-duration-seconds: ${{ inputs.aws-role-duration }}
+
+    - name: Write creds files
+      shell: bash
+      run: |
+        umask 006
+        cat > /etc/nix/aws/nix-aws-credentials <<EOF
+        [ci-uploader]
+        aws_access_key_id = ${AWS_ACCESS_KEY_ID}
+        aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY}
+        aws_session_token = ${AWS_SESSION_TOKEN}
+        EOF

.github/workflows/ami-release-nix-single.yml

@@ -27,6 +27,7 @@ jobs:
         uses: supabase/postgres/.github/actions/shared-checkout@HEAD
         with:
           ref: ${{ github.event.inputs.branch }}
+
       - name: aws-creds
         uses: aws-actions/configure-aws-credentials@v4
         with:
@@ -38,56 +39,35 @@ jobs:
       - name: Get current branch SHA
         id: get_sha
         run: |
-          echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+          echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
 
       - name: Install nix
-        uses: cachix/install-nix-action@v27
+        uses: ./.github/actions/nix-install-ephemeral
         with:
-          install_url: https://releases.nixos.org/nix/nix-2.29.1/install
-          extra_nix_config: |
-            substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com
-            trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
-
-      - name: Set PostgreSQL version environment variable
-        run: |
-          echo "POSTGRES_MAJOR_VERSION=${{ github.event.inputs.postgres_version }}" >> $GITHUB_ENV
-          echo "EXECUTION_ID=${{ github.run_id }}-${{ matrix.postgres_version }}" >> $GITHUB_ENV
-
-      - name: Generate common-nix.vars.pkr.hcl
-        run: |
-          PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ env.POSTGRES_MAJOR_VERSION }}'"]' ansible/vars.yml)
-          PG_VERSION=$(echo "$PG_VERSION" | tr -d '"')  # Remove any surrounding quotes
-          echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl
-          # Ensure there's a newline at the end of the file
-          echo "" >> common-nix.vars.pkr.hcl
-
-      - name: Build AMI stage 1
+          push-to-cache: 'true'
         env:
-          POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
-        run: |
-          GIT_SHA=${{ steps.get_sha.outputs.sha }}
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}"  -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}"  amazon-arm64-nix.pkr.hcl
+          DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }}
+          NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
 
-      - name: Build AMI stage 2
-        env:
-          POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
-        run: |
-          GIT_SHA=${{ steps.get_sha.outputs.sha }}
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- init stage2-nix-psql.pkr.hcl
-          POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl
+      - name: Build AMI
+        id: build-ami
+        uses: ./.github/actions/build-ami
+        with:
+          postgres_version: ${{ github.event.inputs.postgres_version }}
+          region: us-east-1
+          ami_regions: '["us-east-1"]'
+          git_sha: ${{ steps.get_sha.outputs.sha }}
 
       - name: Grab release version
         id: process_release_version
         run: |
-          VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
-          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          VERSION="${{ steps.build-ami.outputs.postgres_release_version }}"
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
 
       - name: Create nix flake revision tarball
         run: |
           GIT_SHA=${{ steps.get_sha.outputs.sha }}
-          MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
+          MAJOR_VERSION=${{ github.event.inputs.postgres_version }}
 
           mkdir -p "/tmp/pg_upgrade_bin/${MAJOR_VERSION}"
           echo "$GIT_SHA" >> "/tmp/pg_upgrade_bin/${MAJOR_VERSION}/nix_flake_version"
@@ -105,7 +85,7 @@ jobs:
           ansible-playbook -i localhost \
             -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
             -e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \
-            -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \
+            -e "postgres_major_version=${{ github.event.inputs.postgres_version }}" \
             manifest-playbook.yml
 
       - name: Upload nix flake revision to s3 staging
@@ -126,7 +106,7 @@ jobs:
           ansible-playbook -i localhost \
             -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
             -e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \
-            -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \
+            -e "postgres_major_version=${{ github.event.inputs.postgres_version }}" \
             manifest-playbook.yml
     
       - name: Upload nix flake revision to s3 prod
@@ -155,10 +135,12 @@ jobs:
       - name: Cleanup resources after build
         if: ${{ always() }}
         run: |
+          EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}"
           aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids
 
       - name: Cleanup resources on build cancellation
         if: ${{ cancelled() }}
         run: |
+          EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}"
           aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids