From 233e15335c176076f42db78574e5dfc056e1cc6c Mon Sep 17 00:00:00 2001 From: adela Date: Fri, 15 Nov 2024 16:15:58 +0800 Subject: [PATCH 1/6] update --- .github/workflows/hello-world.yml | 182 ++++++++++++++++++ .../hr_test/column-masking.json | 18 ++ 2 files changed, 200 insertions(+) create mode 100644 .github/workflows/hello-world.yml create mode 100644 masking/databases/test-sample-instance/hr_test/column-masking.json diff --git a/.github/workflows/hello-world.yml b/.github/workflows/hello-world.yml new file mode 100644 index 0000000..74fc4a4 --- /dev/null +++ b/.github/workflows/hello-world.yml @@ -0,0 +1,182 @@ +name: Hello World Action +on: + pull_request: + types: [opened, synchronize, reopened] + branches: + - main + - develop + workflow_dispatch: + inputs: + name: + description: "Who to greet" + required: true + default: "World" + +jobs: + hello-world: + runs-on: ubuntu-latest + permissions: + pull-requests: write + issues: write + contents: read + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Login Bytebase + id: bytebase-login + uses: bytebase/login-action@0.0.2 + with: + bytebase-url: ${{ secrets.BYTEBASE_URL }} + service-key: ${{ secrets.BYTEBASE_SERVICE_KEY }} + service-secret: ${{ secrets.BYTEBASE_SERVICE_SECRET }} + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v42 + with: + files: | + masking/databases/**/**/column-masking.json + masking/projects/**/masking-exception.json + + - name: Debug changed files in detail + run: | + echo "All changed files:" + echo "${{ steps.changed-files.outputs.all_changed_files }}" + echo "Contains column-masking.json: ${{ contains(steps.changed-files.outputs.all_changed_files, 'column-masking.json') }}" + echo "Contains masking-exception.json: ${{ contains(steps.changed-files.outputs.all_changed_files, 'masking-exception.json') }}" + echo "Raw output:" + echo "${{ toJSON(steps.changed-files.outputs) }}" + + - name: Apply column masking policy + id: apply-column-masking + if: ${{ steps.changed-files.outputs.any_changed == 'true' && contains(steps.changed-files.outputs.all_changed_files, '/column-masking.json') }} + run: | + # Process all column-masking.json files + echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep "column-masking.json" | while read -r CHANGED_FILE; do + echo "Processing: $CHANGED_FILE" + INSTANCE_NAME=$(echo "$CHANGED_FILE" | sed -n 's/masking\/databases\/\([^/]*\)\/\([^/]*\).*/\1/p') + DATABASE_NAME=$(echo "$CHANGED_FILE" | sed -n 's/masking\/databases\/\([^/]*\)\/\([^/]*\).*/\2/p') + echo "INSTANCE_NAME=$INSTANCE_NAME" + echo "DATABASE_NAME=$DATABASE_NAME" + + response=$(curl -s -w "\n%{http_code}" --request PATCH "${{ steps.bytebase-login.outputs.api_url }}/instances/${INSTANCE_NAME}/databases/${DATABASE_NAME}/policies/masking?allow_missing=true&update_mask=payload" \ + --header "Authorization: Bearer ${{ steps.bytebase-login.outputs.token }}" \ + --header "Content-Type: application/json" \ + --data @"$CHANGED_FILE") + + # Extract status code and response body + status_code=$(echo "$response" | tail -n1) + body=$(echo "$response" | sed '$d') + + echo "Status code: $status_code" + echo "Response body: $body" + + # Append to outputs (with unique identifiers) + echo "status_code_${DATABASE_NAME}=${status_code}" >> $GITHUB_OUTPUT + echo "response_${DATABASE_NAME}<> $GITHUB_OUTPUT + echo "${body}" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + if [[ $status_code -lt 200 || $status_code -ge 300 ]]; then + echo "Failed with status code: $status_code for database: $DATABASE_NAME" + exit 1 + fi + done + + - name: Apply masking exception policy + id: apply-masking-exception + if: ${{ steps.changed-files.outputs.any_changed == 'true' && contains(steps.changed-files.outputs.all_changed_files, '/masking-exception.json') }} + run: | + # Process all masking-exception.json files + echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep "masking-exception.json" | while read -r CHANGED_FILE; do + echo "Processing: $CHANGED_FILE" + PROJECT_NAME=$(echo "$CHANGED_FILE" | sed -n 's/masking\/projects\/\([^/]*\).*/\1/p') + echo "PROJECT_NAME=$PROJECT_NAME" + + response=$(curl -s -w "\n%{http_code}" --request PATCH "${{ steps.bytebase-login.outputs.api_url }}/projects/${PROJECT_NAME}/policies/masking_exception?allow_missing=true&update_mask=payload" \ + --header "Authorization: Bearer ${{ steps.bytebase-login.outputs.token }}" \ + --header "Content-Type: application/json" \ + --data @"$CHANGED_FILE") + + # Extract status code and response body + status_code=$(echo "$response" | tail -n1) + body=$(echo "$response" | sed '$d') + + echo "Status code: $status_code" + echo "Response body: $body" + + # Append to outputs (with unique identifiers) + echo "status_code_${PROJECT_NAME}=${status_code}" >> $GITHUB_OUTPUT + echo "response_${PROJECT_NAME}<> $GITHUB_OUTPUT + echo "${body}" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + if [[ $status_code -lt 200 || $status_code -ge 300 ]]; then + echo "Failed with status code: $status_code for project: $PROJECT_NAME" + exit 1 + fi + done + + - name: Comment on PR + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + env: + CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + with: + script: | + const changedFiles = process.env.CHANGED_FILES || ''; + let commentBody = `### Masking Policy Update Summary\n\n`; + + // Add changed files section + commentBody += `📝 **Changed Files:**\n\n`; + if (changedFiles.trim()) { + commentBody += changedFiles.split(' ').map(f => `- ${f}`).join('\n'); + } else { + commentBody += `None`; + } + commentBody += '\n\n'; + + // Add API calls summary + commentBody += `🔄 **API Calls:**\n\n`; + let apiCallsFound = false; + + if (changedFiles.includes('column-masking.json')) { + const maskingStatuses = Object.keys(${{ toJSON(steps.apply-column-masking.outputs) }} || {}) + .filter(key => key.startsWith('status_code_')) + .map(key => ({ + name: key.replace('status_code_', ''), + status: ${{ toJSON(steps.apply-column-masking.outputs) }}[key] + })); + + maskingStatuses.forEach(({name, status}) => { + apiCallsFound = true; + const success = status >= 200 && status < 300; + commentBody += `- Column Masking (${name}): ${success ? '✅' : '❌'} ${status}\n`; + }); + } + + if (changedFiles.includes('masking-exception.json')) { + const exceptionStatuses = Object.keys(${{ toJSON(steps.apply-masking-exception.outputs) }} || {}) + .filter(key => key.startsWith('status_code_')) + .map(key => ({ + name: key.replace('status_code_', ''), + status: ${{ toJSON(steps.apply-masking-exception.outputs) }}[key] + })); + + exceptionStatuses.forEach(({name, status}) => { + apiCallsFound = true; + const success = status >= 200 && status < 300; + commentBody += `- Masking Exception (${name}): ${success ? '✅' : '❌'} ${status}\n`; + }); + } + + if (!apiCallsFound) { + commentBody += `None`; + } + + await github.rest.issues.createComment({ + ...context.repo, + issue_number: context.issue.number, + body: commentBody + }); diff --git a/masking/databases/test-sample-instance/hr_test/column-masking.json b/masking/databases/test-sample-instance/hr_test/column-masking.json new file mode 100644 index 0000000..3d70b0d --- /dev/null +++ b/masking/databases/test-sample-instance/hr_test/column-masking.json @@ -0,0 +1,18 @@ +{ + "inheritFromParent": false, + "type": "MASKING", + "maskingPolicy": { + "maskData": [ + { + "schema": "public", + "table": "salary", + "column": "amount", + "maskingLevel": "PARTIAL", + "fullMaskingAlgorithmId": "", + "partialMaskingAlgorithmId": "" + } + ] + }, + "enforce": true, + "resourceType": "DATABASE" +} From 748528a8c3b0b4f0d774d82dca67f29bb445ab2a Mon Sep 17 00:00:00 2001 From: adela Date: Fri, 15 Nov 2024 17:09:43 +0800 Subject: [PATCH 2/6] update --- .github/workflows/{hello-world.yml => bb-masking.yml} | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) rename .github/workflows/{hello-world.yml => bb-masking.yml} (98%) diff --git a/.github/workflows/hello-world.yml b/.github/workflows/bb-masking.yml similarity index 98% rename from .github/workflows/hello-world.yml rename to .github/workflows/bb-masking.yml index 74fc4a4..39f97c1 100644 --- a/.github/workflows/hello-world.yml +++ b/.github/workflows/bb-masking.yml @@ -1,4 +1,4 @@ -name: Hello World Action +name: Bytebase Masking Policy Update on: pull_request: types: [opened, synchronize, reopened] @@ -6,11 +6,6 @@ on: - main - develop workflow_dispatch: - inputs: - name: - description: "Who to greet" - required: true - default: "World" jobs: hello-world: From 00105e13cf6098774d7ce87fdb336a7804e2bd9c Mon Sep 17 00:00:00 2001 From: adela Date: Fri, 15 Nov 2024 17:15:05 +0800 Subject: [PATCH 3/6] update --- .../databases/test-sample-instance/hr_test/column-masking.json | 2 +- masking/projects/project-sample/masking-exception.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/masking/databases/test-sample-instance/hr_test/column-masking.json b/masking/databases/test-sample-instance/hr_test/column-masking.json index 3d70b0d..f381fd6 100644 --- a/masking/databases/test-sample-instance/hr_test/column-masking.json +++ b/masking/databases/test-sample-instance/hr_test/column-masking.json @@ -7,7 +7,7 @@ "schema": "public", "table": "salary", "column": "amount", - "maskingLevel": "PARTIAL", + "maskingLevel": "FULL", "fullMaskingAlgorithmId": "", "partialMaskingAlgorithmId": "" } diff --git a/masking/projects/project-sample/masking-exception.json b/masking/projects/project-sample/masking-exception.json index 65d4ea5..6573882 100644 --- a/masking/projects/project-sample/masking-exception.json +++ b/masking/projects/project-sample/masking-exception.json @@ -25,7 +25,7 @@ }, { "action": "QUERY", - "maskingLevel": "NONE", + "maskingLevel": "PARTIAL", "member": "group:contractor@example.com", "condition": { "expression": "resource.database_name == \"hr_prod\" && resource.instance_id == \"prod-sample-instance\" && resource.schema_name == \"public\" && resource.table_name == \"salary\" && resource.column_name == \"amount\"", From 3a6357cdc051c793713506be4fd734d9b4ee0555 Mon Sep 17 00:00:00 2001 From: adela Date: Fri, 15 Nov 2024 17:16:11 +0800 Subject: [PATCH 4/6] rename --- .github/workflows/bb-masking.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bb-masking.yml b/.github/workflows/bb-masking.yml index 39f97c1..471f6cb 100644 --- a/.github/workflows/bb-masking.yml +++ b/.github/workflows/bb-masking.yml @@ -8,7 +8,7 @@ on: workflow_dispatch: jobs: - hello-world: + bytebase-masking: runs-on: ubuntu-latest permissions: pull-requests: write From 731b97330aaf0ef5c82f654010919d7d16c724b0 Mon Sep 17 00:00:00 2001 From: adela Date: Fri, 15 Nov 2024 17:18:04 +0800 Subject: [PATCH 5/6] update --- .../projects/project-sample/masking-exception.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/masking/projects/project-sample/masking-exception.json b/masking/projects/project-sample/masking-exception.json index 6573882..18f16f9 100644 --- a/masking/projects/project-sample/masking-exception.json +++ b/masking/projects/project-sample/masking-exception.json @@ -6,9 +6,9 @@ { "action": "EXPORT", "maskingLevel": "PARTIAL", - "member": "user:dev1@example.com", + "member": "user:dev@x.com", "condition": { - "expression": "resource.database_name == \"hr_prod\" && resource.instance_id == \"prod-sample-instance\" && resource.schema_name == \"public\" && resource.table_name == \"salary\" && resource.column_name == \"amount\"", + "expression": "resource.instance_id == \"prod-sample-instance\" && resource.database_name == \"hr_prod\" && resource.schema_name == \"public\" && resource.table_name == \"salary\" && resource.column_name == \"amount\"", "title": "", "description": "" } @@ -16,19 +16,19 @@ { "action": "QUERY", "maskingLevel": "PARTIAL", - "member": "user:dev2@example.com", + "member": "user:dev2@x.com", "condition": { - "expression": "resource.database_name == \"hr_prod\" && resource.instance_id == \"prod-sample-instance\" && resource.schema_name == \"public\" && resource.table_name == \"salary\" && resource.column_name == \"amount\"", + "expression": "resource.instance_id == \"prod-sample-instance\" && resource.database_name == \"hr_prod\" && resource.schema_name == \"public\" && resource.table_name == \"salary\" && resource.column_name == \"amount\"", "title": "", "description": "" } }, { "action": "QUERY", - "maskingLevel": "PARTIAL", - "member": "group:contractor@example.com", + "maskingLevel": "NONE", + "member": "group:contractor@x.com", "condition": { - "expression": "resource.database_name == \"hr_prod\" && resource.instance_id == \"prod-sample-instance\" && resource.schema_name == \"public\" && resource.table_name == \"salary\" && resource.column_name == \"amount\"", + "expression": "resource.instance_id == \"prod-sample-instance\" && resource.database_name == \"hr_prod\" && resource.schema_name == \"public\" && resource.table_name == \"salary\" && resource.column_name == \"amount\"", "title": "", "description": "" } From 4e51083fc041a5af05ea6cb9b959f1071ff251b6 Mon Sep 17 00:00:00 2001 From: adela Date: Fri, 15 Nov 2024 17:20:19 +0800 Subject: [PATCH 6/6] update --- .github/workflows/bb-masking.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bb-masking.yml b/.github/workflows/bb-masking.yml index 471f6cb..7bdb0b2 100644 --- a/.github/workflows/bb-masking.yml +++ b/.github/workflows/bb-masking.yml @@ -1,14 +1,14 @@ name: Bytebase Masking Policy Update on: pull_request: - types: [opened, synchronize, reopened] + types: [closed] branches: - main - - develop workflow_dispatch: jobs: bytebase-masking: + if: github.event.pull_request.merged == true runs-on: ubuntu-latest permissions: pull-requests: write