From cc72c864849c534c5cd1ba0185cfb439ef39c15a Mon Sep 17 00:00:00 2001 From: Diogo Basto Date: Thu, 30 Oct 2025 17:29:13 +0000 Subject: [PATCH 1/6] TG-939 --- .../commands/queries/execute_tests_query.py | 32 +++++----- testgen/commands/run_launch_db_config.py | 1 + testgen/commands/run_test_execution.py | 16 +++-- testgen/common/models/test_run.py | 3 +- testgen/settings.py | 8 +++ .../050_populate_new_schema_metadata.sql | 3 +- .../test_types_Schema_Drift.yaml | 63 +++++++++++++++++++ .../execution/ex_get_tests_metadata.sql | 51 +++++++++++++++ .../ex_schema_drift_generic.sql | 31 +++++++++ .../gen_schema_drift_tests.sql | 44 +++++++++++++ testgen/ui/assets/dk_icon.svg | 0 testgen/ui/views/test_definitions.py | 48 +++++++++----- testgen/ui/views/test_results.py | 14 +++-- 13 files changed, 265 insertions(+), 49 deletions(-) create mode 100644 testgen/template/dbsetup_test_types/test_types_Schema_Drift.yaml create mode 100644 testgen/template/execution/ex_get_tests_metadata.sql create mode 100644 testgen/template/flavors/generic/exec_query_tests/ex_schema_drift_generic.sql create mode 100644 testgen/template/gen_query_tests/gen_schema_drift_tests.sql mode change 100755 => 100644 testgen/ui/assets/dk_icon.svg diff --git a/testgen/commands/queries/execute_tests_query.py b/testgen/commands/queries/execute_tests_query.py index 6caa1567..d71572b6 100644 --- a/testgen/commands/queries/execute_tests_query.py +++ b/testgen/commands/queries/execute_tests_query.py @@ -6,7 +6,7 @@ from testgen.common import read_template_sql_file from testgen.common.clean_sql import concat_columns -from testgen.common.database.database_service import get_flavor_service, replace_params +from testgen.common.database.database_service import get_flavor_service, get_tg_schema, replace_params from testgen.common.models.connection import Connection from testgen.common.models.table_group import TableGroup from testgen.common.models.test_definition import TestRunType, TestScope @@ -107,7 +107,7 @@ def _get_params(self, test_def: TestExecutionDef | None = None) -> dict: "TEST_SUITE_ID": self.test_run.test_suite_id, "TEST_RUN_ID": self.test_run.id, "RUN_DATE": self.run_date, - "SQL_FLAVOR": self.flavor, + "SQL_FLAVOR": self.flavor, "VARCHAR_TYPE": self.flavor_service.varchar_type, "QUOTE": quote, } @@ -116,7 +116,9 @@ def _get_params(self, test_def: TestExecutionDef | None = None) -> dict: params.update({ "TEST_TYPE": test_def.test_type, "TEST_DEFINITION_ID": test_def.id, + "APP_SCHEMA_NAME": get_tg_schema(), "SCHEMA_NAME": test_def.schema_name, + "TABLE_GROUPS_ID": self.table_group.id, "TABLE_NAME": test_def.table_name, "COLUMN_NAME": f"{quote}{test_def.column_name or ''}{quote}", "COLUMN_NAME_NO_QUOTES": test_def.column_name, @@ -146,7 +148,7 @@ def _get_params(self, test_def: TestExecutionDef | None = None) -> dict: "MATCH_HAVING_CONDITION": f"HAVING {test_def.match_having_condition}" if test_def.match_having_condition else "", "CUSTOM_QUERY": test_def.custom_query, "COLUMN_TYPE": test_def.column_type, - "INPUT_PARAMETERS": self._get_input_parameters(test_def), + "INPUT_PARAMETERS": self._get_input_parameters(test_def), }) return params @@ -169,11 +171,11 @@ def _get_query( query = query.replace(":", "\\:") return query, None if no_bind else params - + def get_active_test_definitions(self) -> tuple[dict]: # Runs on App database return self._get_query("get_active_test_definitions.sql") - + def get_target_identifiers(self, schemas: Iterable[str]) -> tuple[str, dict]: # Runs on Target database filename = "get_target_identifiers.sql" @@ -185,7 +187,7 @@ def get_target_identifiers(self, schemas: Iterable[str]) -> tuple[str, dict]: return self._get_query(filename, f"flavors/{self.connection.sql_flavor}/validate_tests", extra_params=params) except ModuleNotFoundError: return self._get_query(filename, "flavors/generic/validate_tests", extra_params=params) - + def get_test_errors(self, test_defs: list[TestExecutionDef]) -> list[list[UUID | str | datetime]]: return [ [ @@ -205,15 +207,15 @@ def get_test_errors(self, test_defs: list[TestExecutionDef]) -> list[list[UUID | None, # No result_measure on errors ] for td in test_defs if td.errors ] - + def disable_invalid_test_definitions(self) -> tuple[str, dict]: # Runs on App database return self._get_query("disable_invalid_test_definitions.sql") - + def update_historic_thresholds(self) -> tuple[str, dict]: # Runs on App database return self._get_query("update_historic_thresholds.sql") - + def run_query_test(self, test_def: TestExecutionDef) -> tuple[str, dict]: # Runs on Target database folder = "generic" if test_def.template_name.endswith("_generic.sql") else self.flavor @@ -225,7 +227,7 @@ def run_query_test(self, test_def: TestExecutionDef) -> tuple[str, dict]: extra_params={"DATA_SCHEMA": test_def.schema_name}, test_def=test_def, ) - + def aggregate_cat_tests( self, test_defs: list[TestExecutionDef], @@ -265,7 +267,7 @@ def add_query(test_defs: list[TestExecutionDef]) -> str: aggregate_queries.append((query, None)) aggregate_test_defs.append(test_defs) - + if single: for td in test_defs: # Add separate query for each test @@ -296,9 +298,9 @@ def add_query(test_defs: list[TestExecutionDef]) -> str: current_test_defs.append(td) add_query(current_test_defs) - + return aggregate_queries, aggregate_test_defs - + def get_cat_test_results( self, aggregate_results: list[AggregateResult], @@ -309,7 +311,7 @@ def get_cat_test_results( test_defs = aggregate_test_defs[result["query_index"]] result_measures = result["result_measures"].split("|") result_codes = result["result_codes"].split(",") - + for index, td in enumerate(test_defs): test_results.append([ self.test_run.id, @@ -329,7 +331,7 @@ def get_cat_test_results( ]) return test_results - + def update_test_results(self) -> list[tuple[str, dict]]: # Runs on App database return [ diff --git a/testgen/commands/run_launch_db_config.py b/testgen/commands/run_launch_db_config.py index 0d926fbe..83da5d09 100644 --- a/testgen/commands/run_launch_db_config.py +++ b/testgen/commands/run_launch_db_config.py @@ -53,6 +53,7 @@ def _get_params_mapping() -> dict: "TABLE_GROUPS_NAME": settings.DEFAULT_TABLE_GROUPS_NAME, "TEST_SUITE": settings.DEFAULT_TEST_SUITE_KEY, "TEST_SUITE_DESCRIPTION": settings.DEFAULT_TEST_SUITE_DESCRIPTION, + "MONITOR_TEST_SUITE": settings.DEFAULT_MONITOR_TEST_SUITE_KEY, "MAX_THREADS": settings.PROJECT_CONNECTION_MAX_THREADS, "MAX_QUERY_CHARS": settings.PROJECT_CONNECTION_MAX_QUERY_CHAR, "OBSERVABILITY_API_URL": settings.OBSERVABILITY_API_URL, diff --git a/testgen/commands/run_test_execution.py b/testgen/commands/run_test_execution.py index bb91f70c..374b27b5 100644 --- a/testgen/commands/run_test_execution.py +++ b/testgen/commands/run_test_execution.py @@ -54,7 +54,7 @@ def run_test_execution_in_background(test_suite_id: str | UUID): def run_test_execution(test_suite_id: str | UUID, username: str | None = None, run_date: datetime | None = None) -> str: if test_suite_id is None: raise ValueError("Test Suite ID was not specified") - + LOG.info(f"Starting test run for test suite {test_suite_id}") time_delta = (run_date - datetime.now(UTC)) if run_date else timedelta() @@ -112,9 +112,7 @@ def run_test_execution(test_suite_id: str | UUID, username: str | None = None, r "CAT": partial(_run_cat_tests, sql_generator), } # Run metadata tests last so that results for other tests are available to them - # TODO: TURN ON WHEN ADDING METADATA TESTS - # for run_type in ["QUERY", "CAT", "METADATA"]: - for run_type in ["QUERY", "CAT"]: + for run_type in ["QUERY", "CAT", "METADATA"]: if (run_test_defs := [td for td in valid_test_defs if td.run_type == run_type]): run_functions[run_type](run_test_defs) else: @@ -198,7 +196,7 @@ def update_test_progress(progress: ThreadedProgress) -> None: LOG.info(f"Writing {run_type} test errors") for index, error in error_data.items(): test_defs[index].errors.append(error) - + error_results = sql_generator.get_test_errors(test_defs) write_to_app_db(error_results, sql_generator.result_columns, sql_generator.test_results_table) @@ -219,7 +217,7 @@ def _run_cat_tests(sql_generator: TestExecutionSQL, test_defs: list[TestExecutio total_count = len(test_defs) LOG.info(f"Aggregating CAT tests: {total_count}") aggregate_queries, aggregate_test_defs = sql_generator.aggregate_cat_tests(test_defs) - + def update_aggegate_progress(progress: ThreadedProgress) -> None: processed_count = sum(len(aggregate_test_defs[index]) for index in progress["indexes"]) test_run.set_progress( @@ -251,7 +249,7 @@ def update_aggegate_progress(progress: ThreadedProgress) -> None: error_test_defs: list[TestExecutionDef] = [] for index in aggregate_errors: error_test_defs.extend(aggregate_test_defs[index]) - + single_queries, single_test_defs = sql_generator.aggregate_cat_tests(error_test_defs, single=True) test_run.set_progress( @@ -260,7 +258,7 @@ def update_aggegate_progress(progress: ThreadedProgress) -> None: error="Rerunning errored tests singly", ) test_run.save() - + def update_single_progress(progress: ThreadedProgress) -> None: test_run.set_progress( "CAT", @@ -293,7 +291,7 @@ def update_single_progress(progress: ThreadedProgress) -> None: td = single_test_defs[index][0] td.errors.append(error) error_test_defs.append(td) - + error_results = sql_generator.get_test_errors(error_test_defs) write_to_app_db(error_results, sql_generator.result_columns, sql_generator.test_results_table) diff --git a/testgen/common/models/test_run.py b/testgen/common/models/test_run.py index 4451fcfc..7c0701a8 100644 --- a/testgen/common/models/test_run.py +++ b/testgen/common/models/test_run.py @@ -275,8 +275,7 @@ def init_progress(self) -> None: "validation": {"label": "Validating test definitions"}, "QUERY": {"label": "Running query tests"}, "CAT": {"label": "Running aggregated tests"}, - # TODO: TURN ON WHEN ADDING METADATA TESTS - # "METADATA": {"label": "Running metadata tests"}, + "METADATA": {"label": "Running metadata tests"}, } for key in self._progress: self._progress[key].update({"key": key, "status": "Pending"}) diff --git a/testgen/settings.py b/testgen/settings.py index cf71768d..98d1c7f5 100644 --- a/testgen/settings.py +++ b/testgen/settings.py @@ -299,6 +299,14 @@ defaults to: `default_suite_desc` """ +DEFAULT_MONITOR_TEST_SUITE_KEY: str = os.getenv("DEFAULT_MONITOR_TEST_SUITE_NAME", "default-monitor-suite-1") +""" +Key to be assgined to the auto generated monitoring test suite. + +from env variable: `DEFAULT_MONITOR_TEST_SUITE_NAME` +defaults to: `default-monitor-suite-1` +""" + DEFAULT_PROFILING_TABLE_SET = os.getenv("DEFAULT_PROFILING_TABLE_SET", "") """ Comma separated list of specific table names to include when running diff --git a/testgen/template/dbsetup/050_populate_new_schema_metadata.sql b/testgen/template/dbsetup/050_populate_new_schema_metadata.sql index 4b1c20a7..4628a4ae 100644 --- a/testgen/template/dbsetup/050_populate_new_schema_metadata.sql +++ b/testgen/template/dbsetup/050_populate_new_schema_metadata.sql @@ -23,7 +23,8 @@ VALUES ('Monitor', 'Recency'), ('Monitor', 'Daily_Record_Ct'), ('Monitor', 'Monthly_Rec_Ct'), ('Monitor', 'Weekly_Rec_Ct'), - ('Monitor', 'Table_Freshness'); + ('Monitor', 'Table_Freshness'), + ('Monitor', 'Schema_Drift'); TRUNCATE TABLE test_templates; diff --git a/testgen/template/dbsetup_test_types/test_types_Schema_Drift.yaml b/testgen/template/dbsetup_test_types/test_types_Schema_Drift.yaml new file mode 100644 index 00000000..10392d6a --- /dev/null +++ b/testgen/template/dbsetup_test_types/test_types_Schema_Drift.yaml @@ -0,0 +1,63 @@ +test_types: + id: '1512' + test_type: Schema_Drift + test_name_short: Schema Drift + test_name_long: Table Schema Changed + test_description: |- + Checks whether table schema has changed + except_message: |- + Table schema has changed. + measure_uom: Was Schema Change Detected + measure_uom_description: null + selection_criteria: |- + TEMPLATE + dq_score_prevalence_formula: null + dq_score_risk_factor: null + column_name_prompt: null + column_name_help: null + default_parm_columns: null + default_parm_values: null + default_parm_prompts: null + default_parm_help: null + default_severity: Warning + run_type: METADATA + test_scope: tablegroup + dq_dimension: null + health_dimension: null + threshold_description: null + result_visualization: binary_chart + result_visualization_params: '{"legend":{"labels":{"0":"No Changes","1":"Changes"}}}' + usage_notes: |- + This test compares the current table column types with previous data, to check whether the table schema has changed. This test allows you to track any changes to the table structure. + active: Y + cat_test_conditions: [] + target_data_lookups: [] + test_templates: + - id: '2514' + test_type: Schema_Drift + sql_flavor: bigquery + template_name: ex_schema_drift_generic.sql + - id: '2414' + test_type: Schema_Drift + sql_flavor: databricks + template_name: ex_schema_drift_generic.sql + - id: '2214' + test_type: Schema_Drift + sql_flavor: mssql + template_name: ex_schema_drift_generic.sql + - id: '2314' + test_type: Schema_Drift + sql_flavor: postgresql + template_name: ex_schema_drift_generic.sql + - id: '2014' + test_type: Schema_Drift + sql_flavor: redshift + template_name: ex_schema_drift_generic.sql + - id: '2614' + test_type: Schema_Drift + sql_flavor: redshift_spectrum + template_name: ex_schema_drift_generic.sql + - id: '2114' + test_type: Schema_Drift + sql_flavor: snowflake + template_name: ex_schema_drift_generic.sql diff --git a/testgen/template/execution/ex_get_tests_metadata.sql b/testgen/template/execution/ex_get_tests_metadata.sql new file mode 100644 index 00000000..1068b017 --- /dev/null +++ b/testgen/template/execution/ex_get_tests_metadata.sql @@ -0,0 +1,51 @@ +SELECT tt.test_type, + td.id::VARCHAR AS test_definition_id, + COALESCE(td.test_description, tt.test_description) AS test_description, + COALESCE(td.test_action, ts.test_action, '') AS test_action, + schema_name, + table_name, + column_name, + cast(coalesce(skip_errors, 0) as varchar(50)) as skip_errors, + coalesce(baseline_ct, '') as baseline_ct, + coalesce(baseline_unique_ct, '') as baseline_unique_ct, + coalesce(baseline_value, '') as baseline_value, + coalesce(baseline_value_ct, '') as baseline_value_ct, + coalesce(threshold_value, '') as threshold_value, + coalesce(baseline_sum, '') as baseline_sum, + coalesce(baseline_avg, '') as baseline_avg, + coalesce(baseline_sd, '') as baseline_sd, + coalesce(lower_tolerance, '') as lower_tolerance, + coalesce(upper_tolerance, '') as upper_tolerance, + case + when nullif(subset_condition, '') is null then '1=1' + else subset_condition end as subset_condition, + coalesce(groupby_names, '') as groupby_names, + case + when having_condition is null then '' + else concat('HAVING ', having_condition) end as having_condition, + coalesce(window_date_column, '') as window_date_column, + cast(coalesce(window_days, '0') as varchar(50)) as window_days, + coalesce(match_schema_name, '') as match_schema_name, + coalesce(match_table_name, '') as match_table_name, + coalesce(match_column_names, '') as match_column_names, + case + when nullif(match_subset_condition, '') is null then '1=1' + else match_subset_condition end as match_subset_condition, + coalesce(match_groupby_names, '') as match_groupby_names, + case + when match_having_condition is null then '' + else concat('HAVING ', match_having_condition) + END as match_having_condition, + coalesce(custom_query, '') as custom_query, + coalesce(tm.template_name, '') as template_name +FROM test_definitions td + INNER JOIN test_suites ts + ON (td.test_suite_id = ts.id) + INNER JOIN test_types tt + ON (td.test_type = tt.test_type) + LEFT JOIN test_templates tm + ON (td.test_type = tm.test_type + AND :SQL_FLAVOR = tm.sql_flavor) +WHERE td.test_suite_id = :TEST_SUITE_ID + AND tt.run_type = 'METADATA' + AND td.test_active = 'Y'; diff --git a/testgen/template/flavors/generic/exec_query_tests/ex_schema_drift_generic.sql b/testgen/template/flavors/generic/exec_query_tests/ex_schema_drift_generic.sql new file mode 100644 index 00000000..c6b7f9ef --- /dev/null +++ b/testgen/template/flavors/generic/exec_query_tests/ex_schema_drift_generic.sql @@ -0,0 +1,31 @@ +WITH prev_test AS ( + SELECT MAX(test_time) as last_run_time + from {APP_SCHEMA_NAME}.test_results + where test_definition_id = '{TEST_DEFINITION_ID}' +), +change_counts AS ( + SELECT COUNT(*) FILTER (WHERE dsl.change = 'A') AS schema_adds, + COUNT(*) FILTER (WHERE dsl.change = 'D') AS schema_drops, + COUNT(*) FILTER (WHERE dsl.change = 'M') AS schema_mods + FROM prev_test, {APP_SCHEMA_NAME}.data_structure_log dsl + LEFT JOIN {APP_SCHEMA_NAME}.data_column_chars dcc ON dcc.column_id = dsl.element_id + WHERE dcc.table_groups_id = '{TABLE_GROUPS_ID}' + -- if no previous tests, this comparision yelds null and nothing is counted. + AND change_date > prev_test.last_run_time +) +SELECT '{TEST_TYPE}' AS test_type, + '{TEST_DEFINITION_ID}' AS test_definition_id, + '{TEST_SUITE_ID}' AS test_suite_id, + '{TEST_RUN_ID}' AS test_run_id, + '{RUN_DATE}' AS test_time, + '1' AS threshold_value, + 1 AS skip_errors, + '{INPUT_PARAMETERS}' AS input_parameters, + schema_adds::VARCHAR || '|' || schema_mods::VARCHAR || '|' || schema_drops::VARCHAR AS result_signal, + CASE WHEN schema_adds+schema_mods+schema_drops > 0 THEN 0 ELSE 1 END AS result_code, + CASE WHEN schema_adds+schema_mods+schema_drops > 0 THEN + 'Table schema changes detected' + ELSE 'No table schema changes found.' + END AS result_message, + schema_adds+schema_mods+schema_drops AS result_measure +FROM change_counts diff --git a/testgen/template/gen_query_tests/gen_schema_drift_tests.sql b/testgen/template/gen_query_tests/gen_schema_drift_tests.sql new file mode 100644 index 00000000..02817428 --- /dev/null +++ b/testgen/template/gen_query_tests/gen_schema_drift_tests.sql @@ -0,0 +1,44 @@ +INSERT INTO test_definitions (table_groups_id, profile_run_id, test_type, test_suite_id, + schema_name, + skip_errors, test_active, last_auto_gen_date, profiling_as_of_date) +WITH last_run AS (SELECT r.table_groups_id, MAX(run_date) AS last_run_date, p.schema_name, p.profile_run_id + FROM profile_results p + INNER JOIN profiling_runs r + ON (p.profile_run_id = r.id) + INNER JOIN test_suites ts + ON p.project_code = ts.project_code + AND p.connection_id = ts.connection_id + WHERE p.project_code = '{PROJECT_CODE}' + AND r.table_groups_id = '{TABLE_GROUPS_ID}'::UUID + AND ts.id = '{TEST_SUITE_ID}' + AND p.run_date::DATE <= '{AS_OF_DATE}' + GROUP BY r.table_groups_id, p.schema_name, p.profile_run_id), + locked AS (SELECT schema_name + FROM test_definitions + WHERE table_groups_id = '{TABLE_GROUPS_ID}'::UUID + AND test_suite_id = '{TEST_SUITE_ID}' + AND test_type = 'Schema_Drift' + AND lock_refresh = 'Y'), + newtests AS (SELECT * + FROM last_run lr + INNER JOIN test_types t + ON ('Schema_Drift' = t.test_type + AND 'Y' = t.active) + LEFT JOIN generation_sets s + ON (t.test_type = s.test_type + AND '{GENERATION_SET}' = s.generation_set) + WHERE lr.schema_name = '{DATA_SCHEMA}' + AND (s.generation_set IS NOT NULL + OR '{GENERATION_SET}' = '') ) +SELECT '{TABLE_GROUPS_ID}'::UUID as table_groups_id, + n.profile_run_id, + 'Schema_Drift' AS test_type, + '{TEST_SUITE_ID}' AS test_suite_id, + n.schema_name, + 0 as skip_errors, 'Y' as test_active, + '{RUN_DATE}'::TIMESTAMP as last_auto_gen_date, + '{AS_OF_DATE}'::TIMESTAMP as profiling_as_of_date +FROM newtests n +LEFT JOIN locked l + ON (n.schema_name = l.schema_name) +WHERE l.schema_name IS NULL; diff --git a/testgen/ui/assets/dk_icon.svg b/testgen/ui/assets/dk_icon.svg old mode 100755 new mode 100644 diff --git a/testgen/ui/views/test_definitions.py b/testgen/ui/views/test_definitions.py index 1f02a906..63241249 100644 --- a/testgen/ui/views/test_definitions.py +++ b/testgen/ui/views/test_definitions.py @@ -297,7 +297,7 @@ def render_selected_details(selected_test: dict, table_group: TableGroupMinimal) "export_to_observability", ] - additional_columns = [val.strip() for val in selected_test["default_parm_columns"].split(",")] + additional_columns = [val.strip() for val in selected_test["default_parm_columns"].split(",")] if selected_test["default_parm_columns"] else [] columns = columns + additional_columns labels = labels + additional_columns labels = list(map(snake_case_to_title_case, labels)) @@ -391,7 +391,7 @@ def show_test_form( # run type run_type = selected_test_type_row["run_type"] # Can be "QUERY" or "CAT" - test_scope = selected_test_type_row["test_scope"] # Can be "column", "table", "referential", "custom" + test_scope = selected_test_type_row["test_scope"] # Can be "column", "table", "referential", "custom", "tablegroup" # test_description test_description = empty_if_null(selected_test_def["test_description"]) if mode == "edit" else "" @@ -475,7 +475,7 @@ def show_test_form( export_to_observability_index = export_to_observability_options.index(export_to_observability) # dynamic attributes - dynamic_attributes_raw = selected_test_type_row["default_parm_columns"] + dynamic_attributes_raw = selected_test_type_row["default_parm_columns"] or "" dynamic_attributes = dynamic_attributes_raw.split(",") dynamic_attributes_labels_raw = selected_test_type_row["default_parm_prompts"] @@ -617,14 +617,15 @@ def show_test_form( has_match_attributes = any(attribute.startswith("match_") for attribute in dynamic_attributes) left_column, right_column = st.columns([0.5, 0.5]) if has_match_attributes else (st.container(), None) - # schema_name test_definition["schema_name"] = left_column.text_input( - label="Schema", max_chars=100, value=schema_name, disabled=True - ) + label="Schema", max_chars=100, value=schema_name, disabled=True + ) # table_name table_column_list = get_columns(table_groups_id) - if test_scope == "custom": + if test_scope == "tablegroup": + test_definition["table_name"] = None + elif test_scope == "custom": test_definition["table_name"] = left_column.text_input( label="Table", max_chars=100, value=table_name, disabled=False ) @@ -643,7 +644,7 @@ def show_test_form( ) column_name_label = None - if test_scope == "table": + if test_scope in ("table", "tablegroup"): test_definition["column_name"] = None elif test_scope in ("referential", "custom"): column_name_label = selected_test_type_row["column_name_prompt"] if selected_test_type_row["column_name_prompt"] else "Test Focus" @@ -670,7 +671,7 @@ def show_test_form( leftover_attributes = dynamic_attributes.copy() def render_dynamic_attribute(attribute: str, container: DeltaGenerator): - if not attribute in dynamic_attributes: + if not attribute in dynamic_attributes or not attribute: return choice_fields = { @@ -762,7 +763,8 @@ def render_dynamic_attribute(attribute: str, container: DeltaGenerator): for attribute in ["match_schema_name", "match_table_name", "match_column_names"]: render_dynamic_attribute(attribute, right_column) - st.divider() + if test_scope != "tablegroup": + st.divider() mid_left_column, mid_right_column = st.columns([0.5, 0.5]) @@ -954,11 +956,19 @@ def prompt_for_test_type(): col0, col1, col2, col3, col4, col5 = st.columns([0.1, 0.2, 0.2, 0.2, 0.2, 0.1]) col0.write("Show Types") + include_referential=col1.checkbox(":green[⧉] Referential", True), + include_table=col2.checkbox(":green[⊞] Table", True), + include_column=col3.checkbox(":green[≣] Column", True), + include_custom=col4.checkbox(":green[⛭] Custom", True), + # always exclude tablegroup scopes from showing + include_all = not any(include_referential, include_table, include_column, include_custom) + df = run_test_type_lookup_query( - include_referential=col1.checkbox(":green[⧉] Referential", True), - include_table=col2.checkbox(":green[⊞] Table", True), - include_column=col3.checkbox(":green[≣] Column", True), - include_custom=col4.checkbox(":green[⛭] Custom", True), + include_referential=include_referential or include_all, + include_table=include_table or include_all, + include_column=include_column or include_all, + include_custom=include_custom or include_all, + include_tablegroup=False, ) lst_choices = df["select_name"].tolist() @@ -1089,12 +1099,14 @@ def run_test_type_lookup_query( include_table: bool = True, include_column: bool = True, include_custom: bool = True, + include_tablegroup: bool = True, ) -> pd.DataFrame: scope_map = { "referential": include_referential, "table": include_table, "column": include_column, "custom": include_custom, + "tablegroup": include_tablegroup, } scopes = [ key for key, include in scope_map.items() if include ] @@ -1112,6 +1124,7 @@ def run_test_type_lookup_query( WHEN 'custom' THEN '⛭ ' WHEN 'table' THEN '⊞ ' WHEN 'column' THEN '≣ ' + WHEN 'tablegroup' THEN '▦ ' ELSE '? ' END || tt.test_name_short @@ -1131,7 +1144,8 @@ def run_test_type_lookup_query( WHEN 'custom' THEN 2 WHEN 'table' THEN 3 WHEN 'column' THEN 4 - ELSE 5 + WHEN 'tablegroup' THEN 5 + ELSE 6 END, tt.test_name_short; """ @@ -1165,7 +1179,7 @@ def get_test_definitions( clauses.append(TestDefinition.column_name.ilike(column_name)) if test_type: clauses.append(TestDefinition.test_type == test_type) - + sort_funcs = {"ASC": asc, "DESC": desc} test_definitions = TestDefinition.select_where( *clauses, @@ -1203,7 +1217,7 @@ def get_test_definitions_collision( target_table_group_id: str, target_test_suite_id: str, ) -> pd.DataFrame: - table_tests = [(item["table_name"], item["test_type"]) for item in test_definitions if item["column_name"] is None] + table_tests = [(item["table_name"], item["test_type"]) for item in test_definitions if item["column_name"] is None and item["table_name"] is not None] column_tests = [(item["table_name"], item["column_name"], item["test_type"]) for item in test_definitions if item["column_name"] is not None] results = TestDefinition.select_minimal_where( TestDefinition.table_groups_id == target_table_group_id, diff --git a/testgen/ui/views/test_results.py b/testgen/ui/views/test_results.py index 9bfbb653..f91029c4 100644 --- a/testgen/ui/views/test_results.py +++ b/testgen/ui/views/test_results.py @@ -454,11 +454,15 @@ def readable_boolean(v: bool): dynamic_attributes_labels_raw = "" dynamic_attributes_labels = dynamic_attributes_labels_raw.split(",") - dynamic_attributes_raw = test_definition.default_parm_columns - dynamic_attributes_fields = dynamic_attributes_raw.split(",") - dynamic_attributes_values = attrgetter(*dynamic_attributes_fields)(test_definition)\ - if len(dynamic_attributes_fields) > 1\ - else (getattr(test_definition, dynamic_attributes_fields[0]),) + dynamic_attributes_raw = test_definition.default_parm_columns or "" + if not dynamic_attributes_raw: + dynamic_attributes_fields = [] + dynamic_attributes_values = [] + else: + dynamic_attributes_fields = dynamic_attributes_raw.split(",") + dynamic_attributes_values = attrgetter(*dynamic_attributes_fields)(test_definition)\ + if len(dynamic_attributes_fields) > 1\ + else (getattr(test_definition, dynamic_attributes_fields[0]),) for field_name in dynamic_attributes_fields[len(dynamic_attributes_labels):]: dynamic_attributes_labels.append(snake_case_to_title_case(field_name)) From 275e16a60be473f24195215e120997023b8d5d3a Mon Sep 17 00:00:00 2001 From: Diogo Basto Date: Tue, 9 Dec 2025 18:37:01 +0000 Subject: [PATCH 2/6] fix: pass signal --- testgen/__main__.py | 46 +++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/testgen/__main__.py b/testgen/__main__.py index e4871f0c..1f0f4480 100644 --- a/testgen/__main__.py +++ b/testgen/__main__.py @@ -385,7 +385,7 @@ def quick_start( test_suite_id = "9df7489d-92b3-49f9-95ca-512160d7896f" click.echo(f"run-profile with table_group_id: {table_group_id}") - message = run_profiling(table_group_id, run_date=now_date + time_delta) + message = run_profiling(table_group_id, run_date=now_date + time_delta) click.echo("\n" + message) LOG.info(f"run-test-generation with table_group_id: {table_group_id} test_suite: {settings.DEFAULT_TEST_SUITE_KEY}") @@ -640,8 +640,6 @@ def list_ui_plugins(): def run_ui(): from testgen.ui.scripts import patch_streamlit - status_code: int = -1 - use_ssl = os.path.isfile(settings.SSL_CERT_FILE) and os.path.isfile(settings.SSL_KEY_FILE) patch_streamlit.patch(force=True) @@ -656,25 +654,29 @@ def cancel_all_running(): cancel_all_running() - try: - app_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "ui/app.py") - status_code = subprocess.check_call( - [ # noqa: S607 - "streamlit", - "run", - app_file, - "--browser.gatherUsageStats=false", - "--client.showErrorDetails=none", - "--client.toolbarMode=minimal", - f"--server.sslCertFile={settings.SSL_CERT_FILE}" if use_ssl else "", - f"--server.sslKeyFile={settings.SSL_KEY_FILE}" if use_ssl else "", - "--", - f"{'--debug' if settings.IS_DEBUG else ''}", - ], - env={**os.environ, "TG_JOB_SOURCE": "UI"} - ) - except Exception: - LOG.exception(f"Testgen UI exited with status code {status_code}") + app_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "ui/app.py") + process= subprocess.Popen( + [ # noqa: S607 + "streamlit", + "run", + app_file, + "--browser.gatherUsageStats=false", + "--client.showErrorDetails=none", + "--client.toolbarMode=minimal", + f"--server.sslCertFile={settings.SSL_CERT_FILE}" if use_ssl else "", + f"--server.sslKeyFile={settings.SSL_KEY_FILE}" if use_ssl else "", + "--", + f"{'--debug' if settings.IS_DEBUG else ''}", + ], + env={**os.environ, "TG_JOB_SOURCE": "UI"} + ) + def term_ui(signum, _): + LOG.info(f"Sending termination signal {signum} to Testgen UI") + process.send_signal(signum) + signal.signal(signal.SIGINT, term_ui) + signal.signal(signal.SIGTERM, term_ui) + status_code = process.wait() + LOG.log(logging.ERROR if status_code != 0 else logging.INFO, f"Testgen UI exited with status code {status_code}") @cli.command("run-app", help="Runs TestGen's application modules") From 62de7baa95bf0b329703ea0b36bed9207faf88ab Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Tue, 9 Dec 2025 15:26:54 -0500 Subject: [PATCH 3/6] fix: typing for filter options --- .../components/frontend/js/components/score_breakdown.js | 2 +- testgen/ui/components/frontend/js/components/select.js | 1 - testgen/ui/components/frontend/js/pages/data_catalog.js | 3 ++- testgen/ui/components/frontend/js/pages/profiling_runs.js | 6 ++---- .../ui/components/frontend/js/pages/project_dashboard.js | 2 +- testgen/ui/components/frontend/js/pages/test_runs.js | 7 +++---- testgen/ui/components/frontend/js/pages/test_suites.js | 5 ++--- testgen/ui/components/frontend/js/types.js | 6 ++++++ 8 files changed, 17 insertions(+), 15 deletions(-) diff --git a/testgen/ui/components/frontend/js/components/score_breakdown.js b/testgen/ui/components/frontend/js/components/score_breakdown.js index 5fc2fd1d..acd2ffe1 100644 --- a/testgen/ui/components/frontend/js/components/score_breakdown.js +++ b/testgen/ui/components/frontend/js/components/score_breakdown.js @@ -42,7 +42,7 @@ const ScoreBreakdown = (score, breakdown, category, scoreType, onViewDetails) => return Select({ label: '', value: selectedScoreType, - options: scoreTypeOptions.map((s) => ({ label: SCORE_TYPE_LABEL[s], value: s, selected: s === scoreType })), + options: scoreTypeOptions.map((s) => ({ label: SCORE_TYPE_LABEL[s], value: s })), height: 32, onChange: (value) => emitEvent('ScoreTypeChanged', { payload: value }), testId: 'score-type-selector', diff --git a/testgen/ui/components/frontend/js/components/select.js b/testgen/ui/components/frontend/js/components/select.js index e7350424..455b7783 100644 --- a/testgen/ui/components/frontend/js/components/select.js +++ b/testgen/ui/components/frontend/js/components/select.js @@ -3,7 +3,6 @@ * @type {object} * @property {string} label * @property {string} value - * @property {boolean?} selected * @property {string?} icon * * @typedef Properties diff --git a/testgen/ui/components/frontend/js/pages/data_catalog.js b/testgen/ui/components/frontend/js/pages/data_catalog.js index 77f65e4a..2edc1efd 100644 --- a/testgen/ui/components/frontend/js/pages/data_catalog.js +++ b/testgen/ui/components/frontend/js/pages/data_catalog.js @@ -1,7 +1,7 @@ /** * @import { Column, Table } from '../data_profiling/data_profiling_utils.js'; * @import { TreeNode, SelectedNode } from '../components/tree.js'; - * @import { ProjectSummary } from '../types.js'; + * @import { FilterOption, ProjectSummary } from '../types.js'; * * @typedef ColumnPath * @type {object} @@ -42,6 +42,7 @@ * @typedef Properties * @type {object} * @property {ProjectSummary} project_summary + * @property {FilterOption[]} table_group_filter_options * @property {ColumnPath[]} columns * @property {Table | Column} selected_item * @property {Object.} tag_values diff --git a/testgen/ui/components/frontend/js/pages/profiling_runs.js b/testgen/ui/components/frontend/js/pages/profiling_runs.js index 4fc62ba5..0b4e2948 100644 --- a/testgen/ui/components/frontend/js/pages/profiling_runs.js +++ b/testgen/ui/components/frontend/js/pages/profiling_runs.js @@ -1,7 +1,5 @@ /** - * @import { ProjectSummary } from '../types.js'; - * @import { SelectOption } from '../components/select.js'; - * + * @import { FilterOption, ProjectSummary } from '../types.js'; * * * @typedef ProgressStep * @type {object} @@ -40,7 +38,7 @@ * @type {object} * @property {ProjectSummary} project_summary * @property {ProfilingRun[]} profiling_runs - * @property {SelectOption[]} table_group_options + * @property {FilterOption[]} table_group_options * @property {Permissions} permissions */ import van from '../van.min.js'; diff --git a/testgen/ui/components/frontend/js/pages/project_dashboard.js b/testgen/ui/components/frontend/js/pages/project_dashboard.js index 1eeee31a..734298e4 100644 --- a/testgen/ui/components/frontend/js/pages/project_dashboard.js +++ b/testgen/ui/components/frontend/js/pages/project_dashboard.js @@ -1,5 +1,5 @@ /** - * @import { ProjectSummary } from '../types.js'; + * @import { FilterOption, ProjectSummary } from '../types.js'; * @import { TestSuiteSummary } from '../types.js'; * * @typedef TableGroupSummary diff --git a/testgen/ui/components/frontend/js/pages/test_runs.js b/testgen/ui/components/frontend/js/pages/test_runs.js index b9796462..31aa7413 100644 --- a/testgen/ui/components/frontend/js/pages/test_runs.js +++ b/testgen/ui/components/frontend/js/pages/test_runs.js @@ -1,6 +1,5 @@ /** - * @import { ProjectSummary } from '../types.js'; - * @import { SelectOption } from '../components/select.js'; + * @import { FilterOption, ProjectSummary } from '../types.js'; * * @typedef ProgressStep * @type {object} @@ -37,8 +36,8 @@ * @type {object} * @property {ProjectSummary} project_summary * @property {TestRun[]} test_runs - * @property {SelectOption[]} table_group_options - * @property {SelectOption[]} test_suite_options + * @property {FilterOption[]} table_group_options + * @property {FilterOption[]} test_suite_options * @property {Permissions} permissions */ import van from '../van.min.js'; diff --git a/testgen/ui/components/frontend/js/pages/test_suites.js b/testgen/ui/components/frontend/js/pages/test_suites.js index 05e2464a..85e84abf 100644 --- a/testgen/ui/components/frontend/js/pages/test_suites.js +++ b/testgen/ui/components/frontend/js/pages/test_suites.js @@ -1,7 +1,6 @@ /** - * @import { ProjectSummary } from '../types.js'; + * @import { FilterOption, ProjectSummary } from '../types.js'; * @import { TestSuiteSummary } from '../types.js'; - * @import { SelectOption } from '../components/select.js'; * * @typedef Permissions * @type {object} @@ -11,7 +10,7 @@ * @type {object} * @property {ProjectSummary} project_summary * @property {TestSuiteSummary} test_suites - * @property {SelectOption[]} table_group_filter_options + * @property {FilterOption[]} table_group_filter_options * @property {Permissions} permissions */ import van from '../van.min.js'; diff --git a/testgen/ui/components/frontend/js/types.js b/testgen/ui/components/frontend/js/types.js index 0155396e..cbf5812d 100644 --- a/testgen/ui/components/frontend/js/types.js +++ b/testgen/ui/components/frontend/js/types.js @@ -1,4 +1,10 @@ /** + * @typedef FilterOption + * @type {object} + * @property {string} label + * @property {string} value + * @property {boolean} selected + * * @typedef ProjectSummary * @type {object} * @property {string} project_code From 81a63afa1d37598c4816702b0439f6484d8e090b Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Tue, 9 Dec 2025 15:34:38 -0500 Subject: [PATCH 4/6] fix(score-breakdown): prevent infinite looping and make defaulting logic consistent --- testgen/ui/views/score_details.py | 23 +++++++++--------- testgen/ui/views/score_explorer.py | 39 +++++++++++++++++++++--------- 2 files changed, 39 insertions(+), 23 deletions(-) diff --git a/testgen/ui/views/score_details.py b/testgen/ui/views/score_details.py index 9148128c..f457bc91 100644 --- a/testgen/ui/views/score_details.py +++ b/testgen/ui/views/score_details.py @@ -66,14 +66,19 @@ def render( ], ) - if category not in typing.get_args(Categories): - category = None - - if not category and score_definition.category: - category = score_definition.category.value + if not category or category not in typing.get_args(Categories): + category = ( + score_definition.category.value + if score_definition.category + else ScoreCategory.dq_dimension.value + ) - if not category: - category = ScoreCategory.dq_dimension.value + if not score_type or score_type not in typing.get_args(ScoreTypes): + score_type = ( + "cde_score" + if score_definition.cde_score and not score_definition.total_score + else "score" + ) score_card = None score_breakdown = None @@ -81,10 +86,6 @@ def render( with st.spinner(text="Loading data :gray[:small[(This might take a few minutes)]] ..."): user_can_edit = session.auth.user_has_permission("edit") score_card = format_score_card(score_definition.as_cached_score_card(include_definition=True)) - if score_type not in typing.get_args(ScoreTypes): - score_type = None - if not score_type: - score_type = "cde_score" if score_card["cde_score"] and not score_card["score"] else "score" if not drilldown: score_breakdown = ScoreDefinitionBreakdownItem.filter( definition_id=definition_id, diff --git a/testgen/ui/views/score_explorer.py b/testgen/ui/views/score_explorer.py index 1522967b..8846629e 100644 --- a/testgen/ui/views/score_explorer.py +++ b/testgen/ui/views/score_explorer.py @@ -1,4 +1,5 @@ import json +import typing from datetime import datetime from functools import partial from io import BytesIO @@ -13,7 +14,14 @@ ) from testgen.common.mixpanel_service import MixpanelService from testgen.common.models.profiling_run import ProfilingRun -from testgen.common.models.scores import ScoreCategory, ScoreDefinition, ScoreDefinitionCriteria, SelectedIssue +from testgen.common.models.scores import ( + Categories, + ScoreCategory, + ScoreDefinition, + ScoreDefinitionCriteria, + ScoreTypes, + SelectedIssue, +) from testgen.common.models.test_run import TestRun from testgen.ui.components import widgets as testgen from testgen.ui.components.widgets.download_dialog import FILE_DATA_TYPE, download_dialog, zip_multi_file_data @@ -46,7 +54,7 @@ def render( category: str | None = None, filters: str | None = None, breakdown_category: str | None = None, - breakdown_score_type: str | None = "score", + breakdown_score_type: str | None = None, drilldown: str | None = None, definition_id: str | None = None, project_code: str | None = None, @@ -65,20 +73,15 @@ def render( ) return - if not breakdown_category and original_score_definition.category: - breakdown_category = original_score_definition.category.value - project_code = original_score_definition.project_code page_title = "Edit Scorecard" last_breadcrumb = original_score_definition.name + testgen.page_header(page_title, breadcrumbs=[ {"path": "quality-dashboard", "label": "Quality Dashboard", "params": {"project_code": project_code}}, {"label": last_breadcrumb}, ]) - if not breakdown_category: - breakdown_category = ScoreCategory.dq_dimension.value - score_breakdown = None issues = None filter_values = {} @@ -117,9 +120,21 @@ def render( group_by_field=filter_by_columns != "true", ) - score_card = None - if score_definition: - score_card = score_definition.as_score_card() + score_card = score_definition.as_score_card() + + if not breakdown_category or breakdown_category not in typing.get_args(Categories): + breakdown_category = ( + score_definition.category.value + if score_definition.category + else ScoreCategory.dq_dimension.value + ) + + if not breakdown_score_type or breakdown_score_type not in typing.get_args(ScoreTypes): + breakdown_score_type = ( + "cde_score" + if score_definition.cde_score and not score_definition.total_score + else "score" + ) if score_definition.criteria.has_filters() and not drilldown: score_breakdown = format_score_card_breakdown( @@ -129,7 +144,7 @@ def render( ), breakdown_category, ) - if score_card and drilldown: + if drilldown: issues = format_score_card_issues( score_definition.get_score_card_issues(breakdown_score_type, breakdown_category, drilldown), breakdown_category, From a8d6549c9b404e42514041b69967144a051931e8 Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Mon, 15 Dec 2025 10:54:04 -0500 Subject: [PATCH 5/6] fix: temporarily disable Schema Drift test --- .../template/dbsetup_test_types/test_types_Schema_Drift.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgen/template/dbsetup_test_types/test_types_Schema_Drift.yaml b/testgen/template/dbsetup_test_types/test_types_Schema_Drift.yaml index 10392d6a..7dbd646b 100644 --- a/testgen/template/dbsetup_test_types/test_types_Schema_Drift.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Schema_Drift.yaml @@ -29,7 +29,7 @@ test_types: result_visualization_params: '{"legend":{"labels":{"0":"No Changes","1":"Changes"}}}' usage_notes: |- This test compares the current table column types with previous data, to check whether the table schema has changed. This test allows you to track any changes to the table structure. - active: Y + active: N cat_test_conditions: [] target_data_lookups: [] test_templates: From 94693f7616e2284fbced8178886b930e7747cbbc Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Mon, 15 Dec 2025 13:34:48 -0500 Subject: [PATCH 6/6] release: 4.38.3 -> 4.38.6 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b7714a67..db252eb7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta" [project] name = "dataops-testgen" -version = "4.38.3" +version = "4.38.6" description = "DataKitchen's Data Quality DataOps TestGen" authors = [ { "name" = "DataKitchen, Inc.", "email" = "info@datakitchen.io" },