Skip to content

acc: Add EnvMatrixExclude config option #4178

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
denik merged 1 commit into from
Dec 29, 2025
+128 −23
Merged

acc: Add EnvMatrixExclude config option #4178

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 1 addition & 21 deletions acceptance/acceptance_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,7 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
t.Parallel()
}

expanded := internal.ExpandEnvMatrix(config.EnvMatrix)
expanded := internal.ExpandEnvMatrix(config.EnvMatrix, config.EnvMatrixExclude)

if testdiff.OverwriteMode && len(expanded) > 1 {
// All variants of the test are producing the same output,
Expand All @@ -351,9 +351,6 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
runTest(t, dir, 0, coverDir, repls.Clone(), config, expanded[0], envFilters)
} else {
for ind, envset := range expanded {
if forbiddenEnvSet(envset) {
continue
}
envname := strings.Join(envset, "/")
t.Run(envname, func(t *testing.T) {
if !inprocessMode {
Expand All @@ -371,23 +368,6 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
return selectedDirs - skippedDirs
}

func forbiddenEnvSet(envset []string) bool {
hasTerraform := false
hasReadplan := false

for _, pair := range envset {
if pair == "DATABRICKS_BUNDLE_ENGINE=terraform" {
hasTerraform = true
}
if pair == "READPLAN=1" {
hasReadplan = true
}
}

// Do not run terraform tests with --plan option:
return hasTerraform && hasReadplan
}

func getEnvFilters(t *testing.T) []string {
envFilterValue := os.Getenv(EnvFilterVar)
if envFilterValue == "" {
Expand Down
56 changes: 55 additions & 1 deletion acceptance/internal/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,13 @@ type TestConfig struct {
// similar to github actions matrix strategy.
EnvMatrix map[string][]string

// Environment variables matrix exclusion list.
// Exclude certain configuration from the grid generated by EnvMatrix.
// They key is arbitrary string and the value is a list of KEY=value pairs to exclude.
// For example:
// EnvMatrixExclude.noplantf = ["READPLAN=1", "DATABRICKS_BUNDLE_ENGINE=terraform"]
EnvMatrixExclude map[string][]string

// List of keys for which to do string replacement value -> [KEY]. If not set, defaults to true.
EnvRepl map[string]bool

Expand Down Expand Up @@ -269,7 +276,8 @@ func (t mapTransformer) Transformer(typ reflect.Type) func(dst, src reflect.Valu
// output: [["KEY=A", "OTHER=VALUE"], ["KEY=B", "OTHER=VALUE"]]
//
// If any entries is an empty list, that variable is dropped from the matrix before processing.
func ExpandEnvMatrix(matrix map[string][]string) [][]string {
// The exclude parameter specifies combinations to exclude from the result.
func ExpandEnvMatrix(matrix, exclude map[string][]string) [][]string {
result := [][]string{{}}

if len(matrix) == 0 {
Expand Down Expand Up @@ -314,5 +322,51 @@ func ExpandEnvMatrix(matrix map[string][]string) [][]string {
result = newResult
}

// Filter out excluded combinations
if len(exclude) > 0 {
result = filterExcludedEnvSets(result, exclude)
}

return result
}

// filterExcludedEnvSets removes environment sets that match exclusion rules.
// An environment set is excluded if it contains all KEY=value pairs from any exclusion rule.
func filterExcludedEnvSets(envSets [][]string, exclude map[string][]string) [][]string {
var filtered [][]string

for _, envSet := range envSets {
excluded := false

// Check each exclusion rule
for _, excludeRule := range exclude {
if matchesExclusionRule(envSet, excludeRule) {
excluded = true
break
}
}

if !excluded {
filtered = append(filtered, envSet)
}
}

return filtered
}

// matchesExclusionRule returns true if envSet contains all KEY=value pairs from excludeRule.
func matchesExclusionRule(envSet, excludeRule []string) bool {
for _, excludePair := range excludeRule {
found := false
for _, envPair := range envSet {
if envPair == excludePair {
found = true
break
}
}
if !found {
return false
}
}
return true
}
72 changes: 71 additions & 1 deletion acceptance/internal/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ func TestExpandEnvMatrix(t *testing.T) {
tests := []struct {
name string
matrix map[string][]string
exclude map[string][]string
expected [][]string
}{
{
Expand Down Expand Up @@ -90,11 +91,80 @@ func TestExpandEnvMatrix(t *testing.T) {
{"KEY=B", "OTHER=VALUE"},
},
},
{
name: "exclude single combination",
matrix: map[string][]string{
"KEY1": {"A", "B"},
"KEY2": {"C", "D"},
},
exclude: map[string][]string{
"rule1": {"KEY1=A", "KEY2=C"},
},
expected: [][]string{
{"KEY1=A", "KEY2=D"},
{"KEY1=B", "KEY2=C"},
{"KEY1=B", "KEY2=D"},
},
},
{
name: "exclude multiple combinations",
matrix: map[string][]string{
"KEY1": {"A", "B"},
"KEY2": {"C", "D"},
},
exclude: map[string][]string{
"rule1": {"KEY1=A", "KEY2=C"},
"rule2": {"KEY1=B", "KEY2=D"},
},
expected: [][]string{
{"KEY1=A", "KEY2=D"},
{"KEY1=B", "KEY2=C"},
},
},
{
name: "exclude with terraform and readplan example",
matrix: map[string][]string{
"DATABRICKS_BUNDLE_ENGINE": {"terraform", "direct"},
"READPLAN": {"0", "1"},
},
exclude: map[string][]string{
"noplantf": {"READPLAN=1", "DATABRICKS_BUNDLE_ENGINE=terraform"},
},
expected: [][]string{
{"DATABRICKS_BUNDLE_ENGINE=terraform", "READPLAN=0"},
{"DATABRICKS_BUNDLE_ENGINE=direct", "READPLAN=0"},
{"DATABRICKS_BUNDLE_ENGINE=direct", "READPLAN=1"},
},
},
{
name: "exclude rule with subset of keys matches",
matrix: map[string][]string{
"KEY1": {"A"},
"KEY2": {"B"},
"KEY3": {"C"},
},
exclude: map[string][]string{
"rule1": {"KEY1=A", "KEY2=B"},
},
expected: nil,
},
{
name: "exclude rule with more keys than envset does not match",
matrix: map[string][]string{
"KEY1": {"A"},
},
exclude: map[string][]string{
"rule1": {"KEY1=A", "KEY2=B"},
},
expected: [][]string{
{"KEY1=A"},
},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := ExpandEnvMatrix(tt.matrix)
result := ExpandEnvMatrix(tt.matrix, tt.exclude)
assert.Equal(t, tt.expected, result)
})
}
Expand Down
1 change: 1 addition & 0 deletions acceptance/test.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ Env.PYTHONUNBUFFERED = "1"
Env.PYTHONUTF8 = "1"

EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"]
Copy link
Contributor

@shreyas-goenka shreyas-goenka Dec 29, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we combine these to completely enumerate all environments the test would run in? Something like:

EnvMatrix = [
  {"DATABRICKS_BUNDLE_ENGINE":"terraform", "READPLAN":0},
 {"DATABRICKS_BUNDLE_ENGINE":"direct", "READPLAN":0},
 {"DATABRICKS_BUNDLE_ENGINE":"direct", "READPLAN":1},
]

Much easier reason about.

Copy link
Contributor Author

@denik denik Dec 29, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, this does not allow composition. For example, we set today DATABRICKS_BUNDLE_ENGINE variants for all tests but READPLAN for some subset, they don't know about each other can be configured separately.

EnvMatrixExclude.noplantf = ["DATABRICKS_BUNDLE_ENGINE=terraform", "READPLAN=1"]
EnvRepl.DATABRICKS_BUNDLE_ENGINE = false

# >>> datetime.datetime.fromtimestamp(18000000000)
Expand Down