From 18a0d63556bd134d0c1b610d68859af90e554403 Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Wed, 27 Aug 2025 19:21:23 -0400 Subject: [PATCH 01/28] fix(analytics): error in username --- testgen/common/mixpanel_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgen/common/mixpanel_service.py b/testgen/common/mixpanel_service.py index 2863a0b4..dba5c74f 100644 --- a/testgen/common/mixpanel_service.py +++ b/testgen/common/mixpanel_service.py @@ -57,7 +57,7 @@ def send_event(self, event_name, include_usage=False, **properties): properties.setdefault("instance_id", self.instance_id) properties.setdefault("edition", settings.DOCKER_HUB_REPOSITORY) properties.setdefault("version", settings.VERSION) - properties.setdefault("username", session.auth.user_display) + properties.setdefault("username", session.auth.user_display if session.auth else None) properties.setdefault("distinct_id", self.get_distinct_id(properties["username"])) if include_usage: properties.update(self.get_usage()) From f3d40cb6221ab8fad6680a0047dbe34c144110a1 Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Wed, 27 Aug 2025 19:22:07 -0400 Subject: [PATCH 02/28] fix(test suites): allow spaces in name --- testgen/ui/views/dialogs/generate_tests_dialog.py | 2 +- testgen/ui/views/dialogs/run_tests_dialog.py | 2 +- testgen/ui/views/test_suites.py | 7 +++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/testgen/ui/views/dialogs/generate_tests_dialog.py b/testgen/ui/views/dialogs/generate_tests_dialog.py index c0c159b1..a548b476 100644 --- a/testgen/ui/views/dialogs/generate_tests_dialog.py +++ b/testgen/ui/views/dialogs/generate_tests_dialog.py @@ -55,7 +55,7 @@ def generate_tests_dialog(test_suite: TestSuiteMinimal) -> None: if testgen.expander_toggle(expand_label="Show CLI command", key="test_suite:keys:generate-tests-show-cli"): st.code( - f"testgen run-test-generation --table-group-id {table_group_id} --test-suite-key {test_suite_name}", + f"testgen run-test-generation --table-group-id {table_group_id} --test-suite-key '{test_suite_name}'", language="shellSession", ) diff --git a/testgen/ui/views/dialogs/run_tests_dialog.py b/testgen/ui/views/dialogs/run_tests_dialog.py index 40e3a05e..808451db 100644 --- a/testgen/ui/views/dialogs/run_tests_dialog.py +++ b/testgen/ui/views/dialogs/run_tests_dialog.py @@ -42,7 +42,7 @@ def run_tests_dialog(project_code: str, test_suite: TestSuiteMinimal | None = No if testgen.expander_toggle(expand_label="Show CLI command", key="run_tests_dialog:keys:show-cli"): st.code( - f"testgen run-tests --project-key {project_code} --test-suite-key {test_suite_name}", + f"testgen run-tests --project-key {project_code} --test-suite-key '{test_suite_name}'", language="shellSession" ) diff --git a/testgen/ui/views/test_suites.py b/testgen/ui/views/test_suites.py index b95a7e04..31a75f0f 100644 --- a/testgen/ui/views/test_suites.py +++ b/testgen/ui/views/test_suites.py @@ -185,10 +185,9 @@ def show_test_suite(mode, project_code, table_groups: Iterable[TableGroupMinimal ) if submit: - if " " in entity["test_suite"]: - proposed_test_suite = entity["test_suite"].replace(" ", "-") + if not entity["test_suite"]: st.error( - f"Blank spaces not allowed in field 'Test Suite Name'. Use dash or underscore instead. i.e.: {proposed_test_suite}" + "Test Suite Name is required" ) else: test_suite = selected or TestSuite() @@ -264,7 +263,7 @@ def observability_export_dialog(test_suite_id: str) -> None: if testgen.expander_toggle(expand_label="Show CLI command", key="test_suite:keys:export-tests-show-cli"): st.code( - f"testgen export-observability --project-key {project_key} --test-suite-key {test_suite_key}", + f"testgen export-observability --project-key {project_key} --test-suite-key '{test_suite_key}'", language="shellSession" ) From 7bd44effdc255554a0b8ea4cee3dcf40d0c4e40e Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Wed, 27 Aug 2025 19:26:22 -0400 Subject: [PATCH 03/28] fix(exports): add schema header detail and remove column --- testgen/common/models/profiling_run.py | 2 ++ testgen/ui/views/hygiene_issues.py | 6 +++--- testgen/ui/views/profiling_results.py | 9 +++------ testgen/ui/views/test_definitions.py | 8 +++----- testgen/ui/views/test_results.py | 8 +++++--- 5 files changed, 16 insertions(+), 17 deletions(-) diff --git a/testgen/common/models/profiling_run.py b/testgen/common/models/profiling_run.py index ca05f5ac..efc19313 100644 --- a/testgen/common/models/profiling_run.py +++ b/testgen/common/models/profiling_run.py @@ -24,6 +24,7 @@ class ProfilingRunMinimal(EntityMinimal): project_code: str table_groups_id: UUID table_groups_name: str + table_group_schema: str profiling_starttime: datetime dq_score_profiling: float is_latest_run: bool @@ -81,6 +82,7 @@ class ProfilingRun(Entity): project_code, table_groups_id, TableGroup.table_groups_name, + TableGroup.table_group_schema, profiling_starttime, dq_score_profiling, case( diff --git a/testgen/ui/views/hygiene_issues.py b/testgen/ui/views/hygiene_issues.py index 3dd75fb0..164d7c39 100644 --- a/testgen/ui/views/hygiene_issues.py +++ b/testgen/ui/views/hygiene_issues.py @@ -233,7 +233,7 @@ def open_download_dialog(data: pd.DataFrame | None = None) -> None: download_dialog( dialog_title="Download Excel Report", file_content_func=get_excel_report_data, - args=(run.table_groups_name, run_date, run_id, data), + args=(run.table_groups_name, run.table_group_schema, run_date, run_id, data), ) with popover_container.container(key="tg--export-popover"): @@ -550,6 +550,7 @@ def get_profiling_anomaly_summary(profile_run_id: str) -> list[dict]: def get_excel_report_data( update_progress: PROGRESS_UPDATE_TYPE, table_group: str, + schema: str, run_date: str, run_id: str, data: pd.DataFrame | None = None, @@ -558,7 +559,6 @@ def get_excel_report_data( data = get_profiling_anomalies(run_id) columns = { - "schema_name": {"header": "Schema"}, "table_name": {"header": "Table"}, "column_name": {"header": "Column"}, "anomaly_name": {"header": "Issue Type"}, @@ -571,7 +571,7 @@ def get_excel_report_data( return get_excel_file_data( data, "Hygiene Issues", - details={"Table group": table_group, "Profiling run date": run_date}, + details={"Table group": table_group, "Schema": schema, "Profiling run date": run_date}, columns=columns, update_progress=update_progress, ) diff --git a/testgen/ui/views/profiling_results.py b/testgen/ui/views/profiling_results.py index c1139616..ee3eebb4 100644 --- a/testgen/ui/views/profiling_results.py +++ b/testgen/ui/views/profiling_results.py @@ -86,7 +86,6 @@ def render(self, run_id: str, table_name: str | None = None, column_name: str | with sort_column: sortable_columns = ( - ("Schema", "schema_name"), ("Table", "table_name"), ("Column", "column_name"), ("Data Type", "column_type"), @@ -107,7 +106,6 @@ def render(self, run_id: str, table_name: str | None = None, column_name: str | ) show_columns = [ - "schema_name", "table_name", "column_name", "column_type", @@ -115,7 +113,6 @@ def render(self, run_id: str, table_name: str | None = None, column_name: str | "hygiene_issues", ] show_column_headers = [ - "Schema", "Table", "Column", "Data Type", @@ -142,7 +139,7 @@ def open_download_dialog(data: pd.DataFrame | None = None) -> None: download_dialog( dialog_title="Download Excel Report", file_content_func=get_excel_report_data, - args=(run.table_groups_name, run_date, run_id, data), + args=(run.table_groups_name, run.table_group_schema, run_date, run_id, data), ) with popover_container.container(key="tg--export-popover"): @@ -179,6 +176,7 @@ def open_download_dialog(data: pd.DataFrame | None = None) -> None: def get_excel_report_data( update_progress: PROGRESS_UPDATE_TYPE, table_group: str, + schema: str, run_date: str, run_id: str, data: pd.DataFrame | None = None, @@ -217,7 +215,6 @@ def get_excel_report_data( ) columns = { - "schema_name": {"header": "Schema"}, "table_name": {"header": "Table"}, "column_name": {"header": "Column"}, "position": {}, @@ -273,7 +270,7 @@ def get_excel_report_data( return get_excel_file_data( data, "Profiling Results", - details={"Table group": table_group, "Profiling run date": run_date}, + details={"Table group": table_group, "Schema": schema, "Profiling run date": run_date}, columns=columns, update_progress=update_progress, ) diff --git a/testgen/ui/views/test_definitions.py b/testgen/ui/views/test_definitions.py index 9e018d51..5bbf7260 100644 --- a/testgen/ui/views/test_definitions.py +++ b/testgen/ui/views/test_definitions.py @@ -877,7 +877,6 @@ def show_test_defs_grid( df = get_test_definitions(test_suite, table_name, column_name, test_type) lst_show_columns = [ - "schema_name", "table_name", "column_name", "test_name_short", @@ -889,7 +888,6 @@ def show_test_defs_grid( "last_manual_update", ] show_column_headers = [ - "Schema", "Table", "Columns / Focus", "Test Type", @@ -925,7 +923,7 @@ def open_download_dialog(data: pd.DataFrame | None = None) -> None: download_dialog( dialog_title="Download Excel Report", file_content_func=get_excel_report_data, - args=(test_suite, data), + args=(test_suite, table_group.table_group_schema, data), ) with popover_container.container(key="tg--export-popover"): @@ -1006,6 +1004,7 @@ def open_download_dialog(data: pd.DataFrame | None = None) -> None: def get_excel_report_data( update_progress: PROGRESS_UPDATE_TYPE, test_suite: TestSuite, + schema: str, data: pd.DataFrame | None = None, ) -> FILE_DATA_TYPE: if data is not None: @@ -1022,7 +1021,6 @@ def get_excel_report_data( ) columns = { - "schema_name": {"header": "Schema"}, "table_name": {"header": "Table"}, "column_name": {"header": "Column/Focus"}, "test_name_short": {"header": "Test type"}, @@ -1038,7 +1036,7 @@ def get_excel_report_data( return get_excel_file_data( data, "Test Definitions", - details={"Test suite": test_suite.test_suite}, + details={"Test suite": test_suite.test_suite, "Schema": schema}, columns=columns, update_progress=update_progress, ) diff --git a/testgen/ui/views/test_results.py b/testgen/ui/views/test_results.py index ae6080f8..3f4d6586 100644 --- a/testgen/ui/views/test_results.py +++ b/testgen/ui/views/test_results.py @@ -18,6 +18,7 @@ from testgen.common import date_service from testgen.common.mixpanel_service import MixpanelService from testgen.common.models import with_database_session +from testgen.common.models.table_group import TableGroup from testgen.common.models.test_definition import TestDefinition from testgen.common.models.test_run import TestRun from testgen.common.models.test_suite import TestSuite @@ -454,6 +455,7 @@ def show_result_detail( df["action"] = df["test_result_id"].map(action_map).fillna(df["action"]) test_suite = TestSuite.get_minimal(test_suite_id) + table_group = TableGroup.get_minimal(test_suite.table_groups_id) lst_show_columns = [ "table_name", @@ -497,7 +499,7 @@ def open_download_dialog(data: pd.DataFrame | None = None) -> None: download_dialog( dialog_title="Download Excel Report", file_content_func=get_excel_report_data, - args=(test_suite.test_suite, run_date, run_id, data), + args=(test_suite.test_suite, table_group.table_group_schema, run_date, run_id, data), ) with popover_container.container(key="tg--export-popover"): @@ -608,6 +610,7 @@ def open_download_dialog(data: pd.DataFrame | None = None) -> None: def get_excel_report_data( update_progress: PROGRESS_UPDATE_TYPE, test_suite: str, + schema: str, run_date: str, run_id: str, data: pd.DataFrame | None = None, @@ -616,7 +619,6 @@ def get_excel_report_data( data = test_result_queries.get_test_results(run_id) columns = { - "schema_name": {"header": "Schema"}, "table_name": {"header": "Table"}, "column_names": {"header": "Columns/Focus"}, "test_name_short": {"header": "Test type"}, @@ -634,7 +636,7 @@ def get_excel_report_data( return get_excel_file_data( data, "Test Results", - details={"Test suite": test_suite, "Test run date": run_date}, + details={"Test suite": test_suite, "Schema": schema, "Test run date": run_date}, columns=columns, update_progress=update_progress, ) From c05c344fb9ab9f5e2163e60a9dba0dddeb53e57f Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Wed, 27 Aug 2025 19:27:33 -0400 Subject: [PATCH 04/28] fix: apply case insensitive sorting --- testgen/common/models/connection.py | 3 +- testgen/common/models/project.py | 4 +-- testgen/common/models/table_group.py | 4 +-- testgen/common/models/test_definition.py | 3 +- testgen/common/models/test_suite.py | 6 ++-- .../get_score_card_issues_by_column.sql | 6 ++-- .../get_score_card_issues_by_dimension.sql | 6 ++-- .../frontend/js/components/input.js | 3 +- .../frontend/js/components/score_breakdown.js | 3 +- .../frontend/js/components/score_issues.js | 8 ++--- .../components/frontend/js/components/tree.js | 3 +- .../components/frontend/js/display_utils.js | 20 ++++++++++++- .../frontend/js/pages/data_catalog.js | 4 +-- .../frontend/js/pages/project_dashboard.js | 6 ++-- .../frontend/js/pages/quality_dashboard.js | 5 ++-- .../frontend/js/pages/table_group_list.js | 3 +- testgen/ui/queries/profiling_queries.py | 8 ++--- testgen/ui/queries/scoring_queries.py | 29 ++++++++++--------- testgen/ui/views/data_catalog.py | 6 ++-- testgen/ui/views/hygiene_issues.py | 8 ++--- testgen/ui/views/profiling_results.py | 8 +++-- testgen/ui/views/test_definitions.py | 4 +-- testgen/ui/views/test_results.py | 8 ++--- 23 files changed, 96 insertions(+), 62 deletions(-) diff --git a/testgen/common/models/connection.py b/testgen/common/models/connection.py index 5f8cd65e..90de7daa 100644 --- a/testgen/common/models/connection.py +++ b/testgen/common/models/connection.py @@ -13,6 +13,7 @@ Integer, String, asc, + func, select, ) from sqlalchemy.dialects import postgresql @@ -60,7 +61,7 @@ class Connection(Entity): http_path: str = Column(String) _get_by = "connection_id" - _default_order_by = (asc(connection_name),) + _default_order_by = (asc(func.lower(connection_name)),) _minimal_columns = ConnectionMinimal.__annotations__.keys() @classmethod diff --git a/testgen/common/models/project.py b/testgen/common/models/project.py index e1e32a7b..e39daecd 100644 --- a/testgen/common/models/project.py +++ b/testgen/common/models/project.py @@ -2,7 +2,7 @@ from uuid import UUID, uuid4 import streamlit as st -from sqlalchemy import Column, String, asc, text +from sqlalchemy import Column, String, asc, func, text from sqlalchemy.dialects import postgresql from testgen.common.models import get_current_session @@ -34,7 +34,7 @@ class Project(Entity): observability_api_key: str = Column(NullIfEmptyString) _get_by = "project_code" - _default_order_by = (asc(project_name),) + _default_order_by = (asc(func.lower(project_name)),) @classmethod @st.cache_data(show_spinner=False) diff --git a/testgen/common/models/table_group.py b/testgen/common/models/table_group.py index 1ed09223..8f823305 100644 --- a/testgen/common/models/table_group.py +++ b/testgen/common/models/table_group.py @@ -4,7 +4,7 @@ from uuid import UUID, uuid4 import streamlit as st -from sqlalchemy import BigInteger, Boolean, Column, Float, ForeignKey, Integer, String, asc, text, update +from sqlalchemy import BigInteger, Boolean, Column, Float, ForeignKey, Integer, String, asc, func, text, update from sqlalchemy.dialects import postgresql from sqlalchemy.orm import InstrumentedAttribute @@ -80,7 +80,7 @@ class TableGroup(Entity): dq_score_profiling: float = Column(Float) dq_score_testing: float = Column(Float) - _default_order_by = (asc(table_groups_name),) + _default_order_by = (asc(func.lower(table_groups_name)),) _minimal_columns = TableGroupMinimal.__annotations__.keys() _update_exclude_columns = ( id, diff --git a/testgen/common/models/test_definition.py b/testgen/common/models/test_definition.py index a938e12f..f7e30116 100644 --- a/testgen/common/models/test_definition.py +++ b/testgen/common/models/test_definition.py @@ -14,6 +14,7 @@ Text, TypeDecorator, asc, + func, insert, select, text, @@ -194,7 +195,7 @@ class TestDefinition(Entity): last_manual_update: datetime = Column(UpdateTimestamp, nullable=False) export_to_observability: bool = Column(YNString) - _default_order_by = (asc(schema_name), asc(table_name), asc(column_name), asc(test_type)) + _default_order_by = (asc(func.lower(schema_name)), asc(func.lower(table_name)), asc(func.lower(column_name)), asc(test_type)) _summary_columns = ( *[key for key in TestDefinitionSummary.__annotations__.keys() if key != "default_test_description"], TestType.test_description.label("default_test_description"), diff --git a/testgen/common/models/test_suite.py b/testgen/common/models/test_suite.py index 95935b8d..781153f7 100644 --- a/testgen/common/models/test_suite.py +++ b/testgen/common/models/test_suite.py @@ -4,7 +4,7 @@ from uuid import UUID, uuid4 import streamlit as st -from sqlalchemy import BigInteger, Boolean, Column, ForeignKey, String, asc, text +from sqlalchemy import BigInteger, Boolean, Column, ForeignKey, String, asc, func, text from sqlalchemy.dialects import postgresql from sqlalchemy.orm import InstrumentedAttribute @@ -66,7 +66,7 @@ class TestSuite(Entity): last_complete_test_run_id: UUID = Column(postgresql.UUID(as_uuid=True)) dq_score_exclude: bool = Column(Boolean, default=False) - _default_order_by = (asc(test_suite),) + _default_order_by = (asc(func.lower(test_suite)),) _minimal_columns = TestSuiteMinimal.__annotations__.keys() @classmethod @@ -182,7 +182,7 @@ def select_summary(cls, project_code: str, table_group_id: str | UUID | None = N ON (groups.id = suites.table_groups_id) WHERE suites.project_code = :project_code {"AND suites.table_groups_id = :table_group_id" if table_group_id else ""} - ORDER BY suites.test_suite; + ORDER BY LOWER(suites.test_suite); """ params = {"project_code": project_code, "table_group_id": table_group_id} db_session = get_current_session() diff --git a/testgen/template/score_cards/get_score_card_issues_by_column.sql b/testgen/template/score_cards/get_score_card_issues_by_column.sql index 804caac9..c0e9724e 100644 --- a/testgen/template/score_cards/get_score_card_issues_by_column.sql +++ b/testgen/template/score_cards/get_score_card_issues_by_column.sql @@ -73,11 +73,13 @@ FROM ( ) issues ORDER BY CASE - status + issues.status WHEN 'Definite' THEN 1 WHEN 'Failed' THEN 2 WHEN 'Likely' THEN 3 WHEN 'Possible' THEN 4 WHEN 'Warning' THEN 5 ELSE 6 - END + END, + LOWER(issues.table), + LOWER(issues.column) diff --git a/testgen/template/score_cards/get_score_card_issues_by_dimension.sql b/testgen/template/score_cards/get_score_card_issues_by_dimension.sql index 43ea8d24..8afb5d85 100644 --- a/testgen/template/score_cards/get_score_card_issues_by_dimension.sql +++ b/testgen/template/score_cards/get_score_card_issues_by_dimension.sql @@ -75,11 +75,13 @@ FROM ( ) issues ORDER BY CASE - status + issues.status WHEN 'Definite' THEN 1 WHEN 'Failed' THEN 2 WHEN 'Likely' THEN 3 WHEN 'Possible' THEN 4 WHEN 'Warning' THEN 5 ELSE 6 - END + END, + LOWER(issues.table), + LOWER(issues.column) diff --git a/testgen/ui/components/frontend/js/components/input.js b/testgen/ui/components/frontend/js/components/input.js index d0a48413..1cc4e449 100644 --- a/testgen/ui/components/frontend/js/components/input.js +++ b/testgen/ui/components/frontend/js/components/input.js @@ -38,6 +38,7 @@ import { debounce, getValue, loadStylesheet, getRandomId } from '../utils.js'; import { Icon } from './icon.js'; import { withTooltip } from './tooltip.js'; import { Portal } from './portal.js'; +import { caseInsensitiveIncludes } from '../display_utils.js'; const { div, input, label, i, small, span } = van.tags; const defaultHeight = 38; @@ -84,7 +85,7 @@ const Input = (/** @type Properties */ props) => { const autocompleteOpened = van.state(false); const autocompleteOptions = van.derive(() => { - const filtered = getValue(props.autocompleteOptions)?.filter(option => option.toLowerCase().includes(value.val.toLowerCase())); + const filtered = getValue(props.autocompleteOptions)?.filter(option => caseInsensitiveIncludes(option, value.val)); if (!filtered?.length) { autocompleteOpened.val = false; } diff --git a/testgen/ui/components/frontend/js/components/score_breakdown.js b/testgen/ui/components/frontend/js/components/score_breakdown.js index 3c8e5e9b..5fc2fd1d 100644 --- a/testgen/ui/components/frontend/js/components/score_breakdown.js +++ b/testgen/ui/components/frontend/js/components/score_breakdown.js @@ -3,6 +3,7 @@ import { dot } from '../components/dot.js'; import { Caption } from '../components/caption.js'; import { Select } from '../components/select.js'; import { emitEvent, getValue, loadStylesheet } from '../utils.js'; +import { caseInsensitiveSort } from '../display_utils.js'; import { getScoreColor } from '../score_utils.js'; const { div, i, span } = van.tags; @@ -23,7 +24,7 @@ const ScoreBreakdown = (score, breakdown, category, scoreType, onViewDetails) => label: '', value: selectedCategory, options: Object.entries(CATEGORIES) - .sort((A, B) => A[1].localeCompare(B[1])) + .sort((A, B) => caseInsensitiveSort(A[1], B[1])) .map(([value, label]) => ({ value, label })), height: 32, onChange: (value) => emitEvent('CategoryChanged', { payload: value }), diff --git a/testgen/ui/components/frontend/js/components/score_issues.js b/testgen/ui/components/frontend/js/components/score_issues.js index 06cd5170..46127703 100644 --- a/testgen/ui/components/frontend/js/components/score_issues.js +++ b/testgen/ui/components/frontend/js/components/score_issues.js @@ -27,7 +27,7 @@ import { Checkbox } from '../components/checkbox.js'; import { Select } from './select.js'; import { Paginator } from '../components/paginator.js'; import { emitEvent, loadStylesheet } from '../utils.js'; -import { colorMap, formatTimestamp } from '../display_utils.js'; +import { colorMap, formatTimestamp, caseInsensitiveSort } from '../display_utils.js'; const { div, i, span } = van.tags; const PAGE_SIZE = 100; @@ -203,18 +203,18 @@ const Toolbar = ( ) => { const filterOptions = { table: [ ...new Set(issues.map(({ table }) => table)) ] - .sort() + .sort(caseInsensitiveSort) .map(value => ({ label: value, value })), column: van.derive(() => ( [ ...new Set(issues .filter(({ table }) => table === filters.table.val) .map(({ column }) => column) )] - .sort() + .sort(caseInsensitiveSort) .map(value => ({ label: value, value })) )), type: [ ...new Set(issues.map(({ type }) => type)) ] - .sort() + .sort(caseInsensitiveSort) .map(value => ({ label: value, value })), status: [ 'Definite', 'Failed', 'Likely', 'Possible', 'Warning', 'Potential PII' ] .map(value => ({ diff --git a/testgen/ui/components/frontend/js/components/tree.js b/testgen/ui/components/frontend/js/components/tree.js index e32357b0..1b737b94 100644 --- a/testgen/ui/components/frontend/js/components/tree.js +++ b/testgen/ui/components/frontend/js/components/tree.js @@ -46,6 +46,7 @@ import { Icon } from './icon.js'; import { Checkbox } from './checkbox.js'; import { Toggle } from './toggle.js'; import { withTooltip } from './tooltip.js'; +import { caseInsensitiveIncludes } from '../display_utils.js'; const { div, h3, span } = van.tags; const levelOffset = 14; @@ -121,7 +122,7 @@ const Toolbar = ( const filtersActive = van.state(false); const isNodeHidden = (/** @type TreeNode */ node) => props.isNodeHidden ? props.isNodeHidden?.(node, search.val) - : !node.label.toLowerCase().includes(search.val.toLowerCase()); + : !caseInsensitiveIncludes(node.label, search.val); return div( { class: 'tg-tree--actions' }, diff --git a/testgen/ui/components/frontend/js/display_utils.js b/testgen/ui/components/frontend/js/display_utils.js index 9eb5e8c1..070e0d8a 100644 --- a/testgen/ui/components/frontend/js/display_utils.js +++ b/testgen/ui/components/frontend/js/display_utils.js @@ -74,6 +74,14 @@ function humanReadableSize(bytes) { return `${bytes}B`; } +const caseInsensitiveSort = new Intl.Collator('en').compare; +const caseInsensitiveIncludes = (/** @type string */ value, /** @type string */ search) => { + if (value && search) { + return value.toLowerCase().includes(search.toLowerCase()); + } + return !search; +} + // https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors const colorMap = { red: '#EF5350', // Red 400 @@ -98,4 +106,14 @@ const colorMap = { const DISABLED_ACTION_TEXT = 'You do not have permissions to perform this action. Contact your administrator.'; -export { formatTimestamp, formatDuration, formatNumber, capitalize, humanReadableSize, colorMap, DISABLED_ACTION_TEXT }; +export { + formatTimestamp, + formatDuration, + formatNumber, + capitalize, + humanReadableSize, + caseInsensitiveSort, + caseInsensitiveIncludes, + colorMap, + DISABLED_ACTION_TEXT, +}; diff --git a/testgen/ui/components/frontend/js/pages/data_catalog.js b/testgen/ui/components/frontend/js/pages/data_catalog.js index 42fed8cc..8140983b 100644 --- a/testgen/ui/components/frontend/js/pages/data_catalog.js +++ b/testgen/ui/components/frontend/js/pages/data_catalog.js @@ -48,7 +48,7 @@ import { getColumnIcon, TABLE_ICON, LatestProfilingTime } from '../data_profilin import { RadioGroup } from '../components/radio_group.js'; import { Checkbox } from '../components/checkbox.js'; import { Select } from '../components/select.js'; -import { capitalize } from '../display_utils.js'; +import { capitalize, caseInsensitiveIncludes } from '../display_utils.js'; import { TableSizeCard } from '../data_profiling/table_size.js'; import { Card } from '../components/card.js'; import { Button } from '../components/button.js'; @@ -210,7 +210,7 @@ const DataCatalog = (/** @type Properties */ props) => { multiSelectToggleLabel: 'Edit multiple', onMultiSelect: (/** @type string[] | null */ selected) => multiSelectedItems.val = selected, isNodeHidden: (/** @type TreeNode */ node, /** string */ search) => search - && (!node.label.toLowerCase().includes(search.toLowerCase()) + && (!caseInsensitiveIncludes(node.label, search) || (!!node.children && !searchOptions.tableName.val) || (!node.children && !searchOptions.columnName.val)) || ![ node.criticalDataElement, false ].includes(filters.criticalDataElement.val) diff --git a/testgen/ui/components/frontend/js/pages/project_dashboard.js b/testgen/ui/components/frontend/js/pages/project_dashboard.js index 5ebdb89e..8c6b7175 100644 --- a/testgen/ui/components/frontend/js/pages/project_dashboard.js +++ b/testgen/ui/components/frontend/js/pages/project_dashboard.js @@ -36,7 +36,7 @@ import van from '../van.min.js'; import { Streamlit } from '../streamlit.js'; import { getValue, loadStylesheet, resizeFrameHeightOnDOMChange, resizeFrameHeightToElement } from '../utils.js'; -import { formatTimestamp } from '../display_utils.js'; +import { formatTimestamp, caseInsensitiveSort, caseInsensitiveIncludes } from '../display_utils.js'; import { Card } from '../components/card.js'; import { Select } from '../components/select.js'; import { Input } from '../components/input.js'; @@ -60,7 +60,7 @@ const ProjectDashboard = (/** @type Properties */ props) => { const filteredTableGroups = van.state(getValue(tableGroups)); const sortFunctions = { - table_groups_name: (a, b) => a.table_groups_name.toLowerCase().localeCompare(b.table_groups_name.toLowerCase()), + table_groups_name: (a, b) => caseInsensitiveSort(a.table_groups_name, b.table_groups_name), latest_activity_date: (a, b) => Math.max(b.latest_profile_start, b.latest_tests_start) - Math.max(a.latest_profile_start, a.latest_tests_start), lowest_score: (a, b) => { const scoreA = a.dq_score ? (a.dq_score.startsWith('>') ? 99.99 : Number(a.dq_score)) : 101; @@ -73,7 +73,7 @@ const ProjectDashboard = (/** @type Properties */ props) => { const sortByField = getValue(tableGroupsSortOption); const sortFn = sortFunctions[sortByField] ?? sortFunctions.latest_activity_date; - filteredTableGroups.val = getValue(tableGroups).filter(group => group.table_groups_name.toLowerCase().includes(searchTerm.toLowerCase() ?? '')).sort(sortFn); + filteredTableGroups.val = getValue(tableGroups).filter(group => caseInsensitiveIncludes(group.table_groups_name, searchTerm ?? '')).sort(sortFn); } onFiltersChange(); diff --git a/testgen/ui/components/frontend/js/pages/quality_dashboard.js b/testgen/ui/components/frontend/js/pages/quality_dashboard.js index f10116f2..06070158 100644 --- a/testgen/ui/components/frontend/js/pages/quality_dashboard.js +++ b/testgen/ui/components/frontend/js/pages/quality_dashboard.js @@ -22,6 +22,7 @@ import { Button } from '../components/button.js'; import { ScoreCard } from '../components/score_card.js'; import { ScoreLegend } from '../components/score_legend.js'; import { EmptyState, EMPTY_STATE_MESSAGE } from '../components/empty_state.js'; +import { caseInsensitiveSort, caseInsensitiveIncludes } from '../display_utils.js'; const { div, span } = van.tags; @@ -41,8 +42,8 @@ const QualityDashboard = (/** @type {Properties} */ props) => { const sort = getValue(sortedBy) ?? 'name'; const filter = getValue(filterTerm) ?? ''; return getValue(props.scores) - .filter(score => score.name.toLowerCase().includes(filter.toLowerCase())) - .sort((a, b) => a[sort] > b[sort] ? 1 : (b[sort] > a[sort] ? -1 : 0)); + .filter(score => caseInsensitiveIncludes(score.name, filter)) + .sort((a, b) => caseInsensitiveSort(a[sort], b[sort])); }); return div( diff --git a/testgen/ui/components/frontend/js/pages/table_group_list.js b/testgen/ui/components/frontend/js/pages/table_group_list.js index c4bde7e2..7ee53555 100644 --- a/testgen/ui/components/frontend/js/pages/table_group_list.js +++ b/testgen/ui/components/frontend/js/pages/table_group_list.js @@ -28,6 +28,7 @@ import { Select } from '../components/select.js'; import { Icon } from '../components/icon.js'; import { withTooltip } from '../components/tooltip.js'; import { Input } from '../components/input.js'; +import { caseInsensitiveSort } from '../display_utils.js'; const { div, h4, i, span } = van.tags; @@ -292,7 +293,7 @@ const Toolbar = (permissions, connections, selectedConnection, tableGroupNameFil */ const TruncatedText = ({ max, ...options }, ...children) => { const sortedChildren = [...children.sort((a, b) => a.length - b.length)]; - const tooltipText = children.sort((a, b) => a.localeCompare(b)).join(', '); + const tooltipText = children.sort(caseInsensitiveSort).join(', '); return div( { class: () => `${options.class ?? ''}`, style: 'position: relative;' }, diff --git a/testgen/ui/queries/profiling_queries.py b/testgen/ui/queries/profiling_queries.py index 01d6373a..94e18287 100644 --- a/testgen/ui/queries/profiling_queries.py +++ b/testgen/ui/queries/profiling_queries.py @@ -72,7 +72,7 @@ def get_profiling_results(profiling_run_id: str, table_name: str | None = None, column_name: str | None = None, sorting_columns = None) -> pd.DataFrame: order_by = "" if sorting_columns is None: - order_by = "ORDER BY schema_name, table_name, position" + order_by = "ORDER BY LOWER(schema_name), LOWER(table_name), position" elif len(sorting_columns): order_by = "ORDER BY " + ", ".join(" ".join(col) for col in sorting_columns) @@ -252,7 +252,7 @@ def get_tables_by_condition( ) """ if include_active_tests else ""} {filter_condition} - ORDER BY table_name; + ORDER BY LOWER(table_chars.table_name); """ results = fetch_all_from_db(query, filter_params) @@ -410,7 +410,7 @@ def get_columns_by_condition( AND column_chars.column_name = profile_results.column_name ) {filter_condition} - ORDER BY table_name, ordinal_position; + ORDER BY LOWER(column_chars.table_name), ordinal_position; """ results = fetch_all_from_db(query, filter_params) return [ dict(row) for row in results ] @@ -459,7 +459,7 @@ def get_hygiene_issues(profile_run_id: str, table_name: str, column_name: str | WHEN 'Moderate' THEN 2 ELSE 3 END, - column_name; + LOWER(column_name); """ params = { "profile_run_id": profile_run_id, diff --git a/testgen/ui/queries/scoring_queries.py b/testgen/ui/queries/scoring_queries.py index f6199b71..ea2b3cd8 100644 --- a/testgen/ui/queries/scoring_queries.py +++ b/testgen/ui/queries/scoring_queries.py @@ -173,18 +173,21 @@ def get_score_category_values(project_code: str) -> dict[ScoreCategory, list[str quote = lambda v: f"'{v}'" query = f""" - SELECT DISTINCT - UNNEST(array[{', '.join([quote(c) for c in categories])}]) as category, - UNNEST(array[{', '.join(categories)}]) AS value - FROM v_dq_test_scoring_latest_by_column - WHERE project_code = :project_code - UNION - SELECT DISTINCT - UNNEST(array[{', '.join([quote(c) for c in categories])}]) as category, - UNNEST(array[{', '.join(categories)}]) AS value - FROM v_dq_profile_scoring_latest_by_column - WHERE project_code = :project_code - ORDER BY value + SELECT * + FROM ( + SELECT DISTINCT + UNNEST(array[{', '.join([quote(c) for c in categories])}]) as category, + UNNEST(array[{', '.join(categories)}]) AS value + FROM v_dq_test_scoring_latest_by_column + WHERE project_code = :project_code + UNION + SELECT DISTINCT + UNNEST(array[{', '.join([quote(c) for c in categories])}]) as category, + UNNEST(array[{', '.join(categories)}]) AS value + FROM v_dq_profile_scoring_latest_by_column + WHERE project_code = :project_code + ) category_values + ORDER BY LOWER(value) """ results = fetch_all_from_db(query, {"project_code": project_code}) for row in results: @@ -206,7 +209,7 @@ def get_column_filters(project_code: str) -> list[dict]: FROM data_column_chars INNER JOIN table_groups ON (table_groups.id = data_column_chars.table_groups_id) WHERE table_groups.project_code = :project_code - ORDER BY table_name, ordinal_position; + ORDER BY LOWER(table_groups_name), LOWER(table_name), ordinal_position; """ results = fetch_all_from_db(query, {"project_code": project_code}) return [dict(row) for row in results] diff --git a/testgen/ui/views/data_catalog.py b/testgen/ui/views/data_catalog.py index 1df6c2d9..58ef1bee 100644 --- a/testgen/ui/views/data_catalog.py +++ b/testgen/ui/views/data_catalog.py @@ -183,7 +183,7 @@ def get_excel_report_data(update_progress: PROGRESS_UPDATE_TYPE, table_group: Ta data = pd.DataFrame(table_data + column_data) - data = data.sort_values(by=["table_name", "ordinal_position"], na_position="first") + data = data.sort_values(by=["table_name", "ordinal_position"], na_position="first", key=lambda x: x.str.lower() if x.dtype == "object" else x) for key in ["column_type", "datatype_suggestion"]: data[key] = data[key].apply(lambda val: val.lower() if not pd.isna(val) else None) @@ -480,7 +480,7 @@ def get_latest_test_issues(table_group_id: str, table_name: str, column_name: st WHEN 'Warning' THEN 2 ELSE 3 END, - column_name; + LOWER(column_names); """ params = { "table_group_id": table_group_id, @@ -507,7 +507,7 @@ def get_related_test_suites(table_group_id: str, table_name: str, column_name: s AND table_name = :table_name {"AND column_name = :column_name" if column_name else ""} GROUP BY test_suites.id - ORDER BY test_suite; + ORDER BY LOWER(test_suite); """ params = { "table_group_id": table_group_id, diff --git a/testgen/ui/views/hygiene_issues.py b/testgen/ui/views/hygiene_issues.py index 164d7c39..e7d12593 100644 --- a/testgen/ui/views/hygiene_issues.py +++ b/testgen/ui/views/hygiene_issues.py @@ -110,7 +110,7 @@ def render( .groupby("column_name") .first() .reset_index() - .sort_values("column_name") + .sort_values("column_name", key=lambda x: x.str.lower()) ) column_name = testgen.select( options=column_options, @@ -150,8 +150,8 @@ def render( with sort_column: sortable_columns = ( - ("Table", "r.table_name"), - ("Column", "r.column_name"), + ("Table", "LOWER(r.table_name)"), + ("Column", "LOWER(r.column_name)"), ("Issue Type", "t.anomaly_name"), ("Likelihood", "likelihood_order"), ("Action", "r.disposition"), @@ -401,7 +401,7 @@ def get_profiling_run_columns(profiling_run_id: str) -> pd.DataFrame: FROM profile_anomaly_results r LEFT JOIN profile_anomaly_types t on t.id = r.anomaly_id WHERE r.profile_run_id = :profiling_run_id - ORDER BY r.table_name, r.column_name; + ORDER BY LOWER(r.table_name), LOWER(r.column_name); """ return fetch_df_from_db(query, {"profiling_run_id": profiling_run_id}) diff --git a/testgen/ui/views/profiling_results.py b/testgen/ui/views/profiling_results.py index ee3eebb4..9519c314 100644 --- a/testgen/ui/views/profiling_results.py +++ b/testgen/ui/views/profiling_results.py @@ -63,6 +63,7 @@ def render(self, run_id: str, table_name: str | None = None, column_name: str | with table_filter_column: # Table Name filter df = get_profiling_run_tables(run_id) + df = df.sort_values("table_name", key=lambda x: x.str.lower()) table_name = testgen.select( options=df, value_column="table_name", @@ -74,6 +75,7 @@ def render(self, run_id: str, table_name: str | None = None, column_name: str | with column_filter_column: # Column Name filter df = get_profiling_run_columns(run_id, table_name) + df = df.sort_values("column_name", key=lambda x: x.str.lower()) column_name = testgen.select( options=df, value_column="column_name", @@ -86,9 +88,9 @@ def render(self, run_id: str, table_name: str | None = None, column_name: str | with sort_column: sortable_columns = ( - ("Table", "table_name"), - ("Column", "column_name"), - ("Data Type", "column_type"), + ("Table", "LOWER(table_name)"), + ("Column", "LOWER(column_name)"), + ("Data Type", "LOWER(column_type)"), ("Semantic Data Type", "semantic_data_type"), ("Hygiene Issues", "hygiene_issues"), ) diff --git a/testgen/ui/views/test_definitions.py b/testgen/ui/views/test_definitions.py index 5bbf7260..c8973f56 100644 --- a/testgen/ui/views/test_definitions.py +++ b/testgen/ui/views/test_definitions.py @@ -6,7 +6,7 @@ import pandas as pd import streamlit as st -from sqlalchemy import asc, tuple_ +from sqlalchemy import asc, func, tuple_ from streamlit.delta_generator import DeltaGenerator from streamlit_extras.no_default_selectbox import selectbox @@ -1134,7 +1134,7 @@ def run_test_type_lookup_query( def get_test_suite_columns(test_suite_id: str) -> pd.DataFrame: results = TestDefinition.select_minimal_where( TestDefinition.test_suite_id == test_suite_id, - order_by = (asc(TestDefinition.table_name), asc(TestDefinition.column_name)), + order_by = (asc(func.lower(TestDefinition.table_name)), asc(func.lower(TestDefinition.column_name))), ) return to_dataframe(results, TestDefinitionMinimal.columns()) diff --git a/testgen/ui/views/test_results.py b/testgen/ui/views/test_results.py index 3f4d6586..d254f4ec 100644 --- a/testgen/ui/views/test_results.py +++ b/testgen/ui/views/test_results.py @@ -126,7 +126,7 @@ def render( run_columns_df["table_name"] == table_name ]["column_name"].dropna().unique().tolist() else: - column_options = run_columns_df.groupby("column_name").first().reset_index().sort_values("column_name") + column_options = run_columns_df.groupby("column_name").first().reset_index().sort_values("column_name", key=lambda x: x.str.lower()) column_name = testgen.select( options=column_options, value_column="column_name", @@ -158,8 +158,8 @@ def render( with sort_column: sortable_columns = ( - ("Table", "r.table_name"), - ("Columns/Focus", "r.column_names"), + ("Table", "LOWER(r.table_name)"), + ("Columns/Focus", "LOWER(r.column_names)"), ("Test Type", "r.test_type"), ("Unit of Measure", "tt.measure_uom"), ("Result Measure", "result_measure"), @@ -280,7 +280,7 @@ def get_test_run_columns(test_run_id: str) -> pd.DataFrame: FROM test_results r LEFT JOIN test_types t ON t.test_type = r.test_type WHERE test_run_id = :test_run_id - ORDER BY table_name, column_names; + ORDER BY LOWER(r.table_name), LOWER(r.column_names); """ return fetch_df_from_db(query, {"test_run_id": test_run_id}) From f2fbce6b02b3cef94d1f4a9889e4b7c637851162 Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Wed, 27 Aug 2025 19:46:15 -0400 Subject: [PATCH 05/28] fix(application logs): make search case insensitive --- testgen/ui/views/dialogs/application_logs_dialog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgen/ui/views/dialogs/application_logs_dialog.py b/testgen/ui/views/dialogs/application_logs_dialog.py index 72de5d44..629dda6c 100644 --- a/testgen/ui/views/dialogs/application_logs_dialog.py +++ b/testgen/ui/views/dialogs/application_logs_dialog.py @@ -39,7 +39,7 @@ def _filter_by_date(log_data, start_date, end_date): # Function to search text in log data def _search_text(log_data, search_query): - return [line for line in log_data if search_query in line] + return [line for line in log_data if search_query.lower() in line.lower()] @st.dialog(title="Application Logs") From 7859cb51f7667227691cda816fab1e1ff076d015 Mon Sep 17 00:00:00 2001 From: Luis Date: Wed, 27 Aug 2025 18:56:01 -0400 Subject: [PATCH 06/28] feat(schedules): add cron expression editor component --- pyproject.toml | 1 + testgen/ui/components/frontend/css/shared.css | 14 +- .../frontend/js/components/crontab_input.js | 525 ++++++++++++++++++ .../frontend/js/components/input.js | 2 + .../components/frontend/js/components/link.js | 3 +- .../frontend/js/components/portal.js | 2 +- .../frontend/js/components/select.js | 67 ++- .../frontend/js/pages/schedule_list.js | 165 +++++- testgen/ui/components/frontend/js/values.js | 3 + testgen/ui/components/widgets/__init__.py | 1 - testgen/ui/components/widgets/tz_select.py | 70 --- testgen/ui/views/dialogs/manage_schedules.py | 206 +++---- testgen/ui/views/profiling_runs.py | 19 +- testgen/ui/views/test_runs.py | 19 +- 14 files changed, 855 insertions(+), 242 deletions(-) create mode 100644 testgen/ui/components/frontend/js/components/crontab_input.js create mode 100644 testgen/ui/components/frontend/js/values.js delete mode 100644 testgen/ui/components/widgets/tz_select.py diff --git a/pyproject.toml b/pyproject.toml index 26c4cf08..b463aaa4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,7 @@ dependencies = [ "pydantic==1.10.13", "streamlit-pydantic==0.6.0", "cron-converter==1.2.1", + "cron-descriptor==2.0.5", # Pinned to match the manually compiled libs or for security "pyarrow==18.1.0", diff --git a/testgen/ui/components/frontend/css/shared.css b/testgen/ui/components/frontend/css/shared.css index a3d18383..335858ad 100644 --- a/testgen/ui/components/frontend/css/shared.css +++ b/testgen/ui/components/frontend/css/shared.css @@ -641,4 +641,16 @@ code > .tg-icon:hover { input::-ms-reveal, input::-ms-clear { display: none; -} \ No newline at end of file +} + +.text-left { + text-align: left; +} + +.text-right { + text-align: right; +} + +.text-center { + text-align: center; +} diff --git a/testgen/ui/components/frontend/js/components/crontab_input.js b/testgen/ui/components/frontend/js/components/crontab_input.js new file mode 100644 index 00000000..a1f2c638 --- /dev/null +++ b/testgen/ui/components/frontend/js/components/crontab_input.js @@ -0,0 +1,525 @@ +/** + * @typedef EditOptions + * @type {object} + * @property {CronSample?} sample + * @property {(expr: string) => void} onChange + * @property {(() => void)?} onClose + * + * @typedef CronSample + * @type {object} + * @property {string?} error + * @property {string?} sample + * @property {string?} readable_expr + * + * @typedef InitialValue + * @type {object} + * @property {string} timezone + * @property {string} expression + * + * @typedef Options + * @type {object} + * @property {(string|null)} id + * @property {string?} testId + * @property {string?} class + * @property {CronSample?} sample + * @property {InitialValue?} value + * @property {((expr: string) => void)?} onChange + */ +import { getRandomId, getValue, loadStylesheet } from '../utils.js'; +import van from '../van.min.js'; +import { Portal } from './portal.js'; +import { Button } from './button.js'; +import { Input } from './input.js'; +import { required } from '../form_validators.js'; +import { Select } from './select.js'; +import { Checkbox } from './checkbox.js'; +import { Link } from './link.js'; + +const { div, span } = van.tags; + +const CrontabInput = (/** @type Options */ props) => { + loadStylesheet('crontab-input', stylesheet); + + const domId = van.derive(() => props.id?.val ?? `tg-crontab-wrapper-${getRandomId()}`); + const opened = van.state(false); + const expression = van.state(props.value?.rawVal?.expression ?? props.value?.expression ?? ''); + const readableSchedule = van.state(null); + const timezone = van.derive(() => getValue(props.value)?.timezone); + const disabled = van.derive(() => !timezone.val); + const placeholder = van.derive(() => !timezone.val ? 'Select a timezone first' : 'Click to select schedule'); + + const onEditorChange = (cronExpr) => { + expression.val = cronExpr; + const onChange = props.onChange?.val ?? props.onChange; + if (onChange && cronExpr) { + onChange(cronExpr); + } + }; + + van.derive(() => { + const sample = getValue(props.sample) ?? {}; + if (!sample.error && sample.readable_expr) { + readableSchedule.val = `${sample.readable_expr} (${timezone.val})`; + } + }); + + return div( + { + id: domId, + class: () => `tg-crontab-input ${getValue(props.class) ?? ''}`, + 'data-testid': getValue(props.testId) ?? null, + }, + div( + {onclick: () => { + if (!disabled.val) { + opened.val = true; + } + }}, + Input({ + label: 'Schedule', + icon: 'calendar_clock', + readonly: true, + disabled: disabled, + placeholder: placeholder, + value: readableSchedule, + }), + ), + Portal( + {target: domId.val, align: 'right', style: 'width: 450px;', opened}, + () => ContabEditorPortal( + { + onChange: onEditorChange, + onClose: () => opened.val = false, + sample: props.sample, + }, + expression, + ), + ), + ); +}; + +/** + * @param {EditOptions} options + * @param {import('../van.min.js').VanState} expr + * @returns {HTMLElement} + */ +const ContabEditorPortal = ({sample, ...options}, expr) => { + const mode = van.state(expr.rawVal ? determineMode(expr.rawVal) : 'x_hours'); + + const xHoursState = { + hours: van.state(1), + minute: van.state(0), + }; + const xDaysState = { + days: van.state(1), + hour: van.state(1), + minute: van.state(0), + }; + const certainDaysState = { + sunday: van.state(false), + monday: van.state(false), + tuesday: van.state(false), + wednesday: van.state(false), + thursday: van.state(false), + friday: van.state(false), + saturday: van.state(false), + hour: van.state(1), + minute: van.state(0), + }; + + // Populate initial state based on the initial mode and expression + populateInitialModeState(expr.rawVal, mode.rawVal, xHoursState, xDaysState, certainDaysState); + + van.derive(() => { + if (mode.val === 'x_hours') { + const hours = xHoursState.hours.val; + const minute = xHoursState.minute.val; + options.onChange(`${minute ?? 0} ${(hours && hours !== 1) ? '*/' + hours : '*'} * * *`); + } else if (mode.val === 'x_days') { + const days = xDaysState.days.val; + const hour = xDaysState.hour.val; + const minute = xDaysState.minute.val; + options.onChange(`${minute ?? 0} ${hour ?? 0} ${(days && days !== 1) ? '*/' + days : '*'} * *`); + } else if (mode.val === 'certain_days') { + const days = []; + const dayMap = [ + { key: 'sunday', val: certainDaysState.sunday.val, label: 'SUN' }, + { key: 'monday', val: certainDaysState.monday.val, label: 'MON' }, + { key: 'tuesday', val: certainDaysState.tuesday.val, label: 'TUE' }, + { key: 'wednesday', val: certainDaysState.wednesday.val, label: 'WED' }, + { key: 'thursday', val: certainDaysState.thursday.val, label: 'THU' }, + { key: 'friday', val: certainDaysState.friday.val, label: 'FRI' }, + { key: 'saturday', val: certainDaysState.saturday.val, label: 'SAT' }, + ]; + // Collect selected days + dayMap.forEach(d => { if (d.val) days.push(d.label); }); + // If days are consecutive, use range notation + let dayField = '*'; + if (days.length > 0) { + // Find ranges + const indices = days.map(d => dayMap.findIndex(dm => dm.label === d)).sort((a,b) => a-b); + let ranges = [], rangeStart = null, prev = null; + indices.forEach((idx, i) => { + if (rangeStart === null) rangeStart = idx; + if (prev !== null && idx !== prev + 1) { + ranges.push([rangeStart, prev]); + rangeStart = idx; + } + prev = idx; + if (i === indices.length - 1) ranges.push([rangeStart, idx]); + }); + // Convert ranges to crontab format + dayField = ranges.map(([start, end]) => { + if (start === end) return dayMap[start].label; + return `${dayMap[start].label}-${dayMap[end].label}`; + }).join(','); + } + const hour = certainDaysState.hour.val; + const minute = certainDaysState.minute.val; + options.onChange(`${minute ?? 0} ${hour ?? 0} * * ${dayField}`); + } + }); + + return div( + { class: 'tg-crontab-editor flex-column border-radius-1 mt-1' }, + div( + { class: 'tg-crontab-editor-content flex-row' }, + div( + { class: 'tg-crontab-editor-left flex-column' }, + span( + { + class: () => `tg-crontab-editor-mode p-4 ${mode.val === 'x_hours' ? 'selected' : ''}`, + onclick: () => mode.val = 'x_hours', + }, + 'Every x hours', + ), + span( + { + class: () => `tg-crontab-editor-mode p-4 ${mode.val === 'x_days' ? 'selected' : ''}`, + onclick: () => mode.val = 'x_days', + }, + 'Every x days', + ), + span( + { + class: () => `tg-crontab-editor-mode p-4 ${mode.val === 'certain_days' ? 'selected' : ''}`, + onclick: () => mode.val = 'certain_days', + }, + 'On certain days', + ), + span( + { + class: () => `tg-crontab-editor-mode p-4 ${mode.val === 'custom' ? 'selected' : ''}`, + onclick: () => mode.val = 'custom', + }, + 'Custom', + ), + ), + div( + { class: 'tg-crontab-editor-right flex-column p-4 fx-flex' }, + div( + { class: () => `${mode.val === 'x_hours' ? '' : 'hidden'}`}, + div( + {class: 'flex-row fx-gap-2 mb-2'}, + span({}, 'Every'), + () => Select({ + label: "", + options: Array.from({length: 24}, (_, i) => i).map(i => ({label: i.toString(), value: i})), + triggerStyle: 'inline', + portalClass: 'tg-crontab--select-portal', + value: xHoursState.hours, + onChange: (value) => xHoursState.hours.val = value, + }), + span({}, 'hours'), + ), + div( + {class: 'flex-row fx-gap-2'}, + span({}, 'on'), + span({}, 'minute'), + () => Select({ + label: "", + options: Array.from({length: 60}, (_, i) => i).map(i => ({label: i.toString().padStart(2, '0'), value: i})), + triggerStyle: 'inline', + portalClass: 'tg-crontab--select-portal', + value: xHoursState.minute, + onChange: (value) => xHoursState.minute.val = value, + }), + ), + ), + div( + { class: () => `${mode.val === 'x_days' ? '' : 'hidden'}`}, + div( + {class: 'flex-row fx-gap-2 mb-2'}, + span({}, 'Every'), + () => Select({ + label: "", + options: Array.from({length: 31}, (_, i) => i + 1).map(i => ({label: i.toString(), value: i})), + triggerStyle: 'inline', + portalClass: 'tg-crontab--select-portal', + value: xDaysState.days, + onChange: (value) => xDaysState.days.val = value, + }), + span({}, 'days'), + ), + div( + {class: 'flex-row fx-gap-2'}, + span({}, 'at'), + () => Select({ + label: "", + options: Array.from({length: 24}, (_, i) => i).map(i => ({label: i.toString(), value: i})), + triggerStyle: 'inline', + portalClass: 'tg-crontab--select-portal', + value: xDaysState.hour, + onChange: (value) => xDaysState.hour.val = value, + }), + () => Select({ + label: "", + options: Array.from({length: 60}, (_, i) => i).map(i => ({label: i.toString().padStart(2, '0'), value: i})), + triggerStyle: 'inline', + portalClass: 'tg-crontab--select-portal', + value: xDaysState.minute, + onChange: (value) => xDaysState.minute.val = value, + }), + ), + ), + div( + { class: () => `${mode.val === 'certain_days' ? '' : 'hidden'}`}, + div( + {class: 'flex-row fx-gap-2 mb-2'}, + Checkbox({ + label: 'Sunday', + checked: certainDaysState.sunday, + onChange: (v) => certainDaysState.sunday.val = v, + }), + Checkbox({ + label: 'Monday', + checked: certainDaysState.monday, + onChange: (v) => certainDaysState.monday.val = v, + }), + Checkbox({ + label: 'Tuesday', + checked: certainDaysState.tuesday, + onChange: (v) => certainDaysState.tuesday.val = v, + }), + ), + div( + {class: 'flex-row fx-gap-2 mb-2'}, + Checkbox({ + label: 'Wednesday', + checked: certainDaysState.wednesday, + onChange: (v) => certainDaysState.wednesday.val = v, + }), + Checkbox({ + label: 'Thursday', + checked: certainDaysState.thursday, + onChange: (v) => certainDaysState.thursday.val = v, + }), + ), + div( + {class: 'flex-row fx-gap-2 mb-2'}, + Checkbox({ + label: 'Friday', + checked: certainDaysState.friday, + onChange: (v) => certainDaysState.friday.val = v, + }), + Checkbox({ + label: 'Saturday', + checked: certainDaysState.saturday, + onChange: (v) => certainDaysState.saturday.val = v, + }), + ), + div( + {class: 'flex-row fx-gap-2'}, + span({}, 'at'), + () => Select({ + label: "", + options: Array.from({length: 24}, (_, i) => i).map(i => ({label: i.toString(), value: i})), + triggerStyle: 'inline', + portalClass: 'tg-crontab--select-portal shorter', + value: certainDaysState.hour, + onChange: (value) => certainDaysState.hour.val = value, + }), + () => Select({ + label: "", + options: Array.from({length: 60}, (_, i) => i).map(i => ({label: i.toString().padStart(2, '0'), value: i})), + triggerStyle: 'inline', + portalClass: 'tg-crontab--select-portal shorter', + value: certainDaysState.minute, + onChange: (value) => certainDaysState.minute.val = value, + }), + ), + ), + div( + { class: () => `${mode.val === 'custom' ? '' : 'hidden'}`}, + () => Input({ + name: 'cron_expr', + label: 'Cron Expression', + value: expr, + validators: [ + required, + ((sampleState) => { + return () => { + const sample = getValue(sampleState) ?? {}; + return sample.error || null; + }; + })(sample), + ], + onChange: (value, state) => mode.val === 'custom' && options.onChange(value), + }), + ), + span({class: 'fx-flex'}, ''), + div( + {class: 'flex-column fx-gap-1 mt-3 text-secondary'}, + () => span({}, `Cron Expression: ${expr.val ?? ''}`), + () => span({}, `Next Run: ${(getValue(sample) ?? {})?.sample ?? ''}`), + () => div( + {class: 'flex-row fx-gap-1 text-caption'}, + span({}, 'Learn more about'), + Link({ + open_new: true, + label: 'cron expressions', + href: 'https://crontab.guru/', + right_icon: 'open_in_new', + right_icon_size: 13, + }), + ), + ), + ), + ), + div( + { class: 'flex-row fx-justify-space-between p-3' }, + span({class: 'fx-flex'}, ''), + div( + { class: 'flex-row fx-gap-2' }, + Button({ + type: 'stroked', + color: 'primary', + label: 'Close', + style: 'width: auto;', + onclick: options?.onClose, + }), + ), + ), + ); +}; + +/** + * Populates the state variables for the initial mode based on the cron expression + * @param {string} expr + * @param {string} mode + * @param {object} xHoursState + * @param {object} xDaysState + * @param {object} certainDaysState + */ +function populateInitialModeState(expr, mode, xHoursState, xDaysState, certainDaysState) { + const parts = (expr || '').trim().split(/\s+/); + if (mode === 'x_hours' && parts.length === 5) { + // e.g. "M */H * * *" or "M * * * *" + xHoursState.minute.val = Number(parts[0]) || 0; + if (parts[1].startsWith('*/')) { + xHoursState.hours.val = Number(parts[1].slice(2)) || 1; + } else { + xHoursState.hours.val = 1; + } + } else if (mode === 'x_days' && parts.length === 5) { + // e.g. "M H */D * *" or "M H * * *" + xDaysState.minute.val = Number(parts[0]) || 0; + xDaysState.hour.val = Number(parts[1]) || 0; + if (parts[2].startsWith('*/')) { + xDaysState.days.val = Number(parts[2].slice(2)) || 1; + } else { + xDaysState.days.val = 1; + } + } else if (mode === 'certain_days' && parts.length === 5) { + // e.g. "M H * * DAY[,DAY...]" + certainDaysState.minute.val = Number(parts[0]) || 0; + certainDaysState.hour.val = Number(parts[1]) || 0; + const days = parts[4].split(','); + const dayKeys = ['sunday','monday','tuesday','wednesday','thursday','friday','saturday']; + const dayLabels = ['SUN','MON','TUE','WED','THU','FRI','SAT']; + dayKeys.forEach((key, idx) => { + certainDaysState[key].val = days.some(d => { + if (d.includes('-')) { + // Range, e.g. MON-WED + const [start, end] = d.split('-'); + const startIdx = dayLabels.indexOf(start); + const endIdx = dayLabels.indexOf(end); + return idx >= startIdx && idx <= endIdx; + } + return d === dayLabels[idx]; + }); + }); + } +} + +/** + * @param {string} expression + * @returns {'x_hours'|'x_days'|'certain_days'|'custom'} + */ +function determineMode(expression) { + // Normalize whitespace + const expr = (expression || '').trim().replace(/\s+/g, ' '); + // x_hours: "M */H * * *" or "M * * * *" + if (/^\d{1,2} \*\/\d+ \* \* \*$/.test(expr) || /^\d{1,2} \* \* \* \*$/.test(expr)) { + return 'x_hours'; + } + // x_days: "M H */D * *" or "M H * * *" + if (/^\d{1,2} \d{1,2} \*\/\d+ \* \*$/.test(expr) || /^\d{1,2} \d{1,2} \* \* \*$/.test(expr)) { + return 'x_days'; + } + // certain_days: "M H * * DAY[,DAY...]" (DAY = SUN,MON,...) + if (/^\d{1,2} \d{1,2} \* \* ((SUN|MON|TUE|WED|THU|FRI|SAT)(-(SUN|MON|TUE|WED|THU|FRI|SAT))?(,)?)+$/.test(expr)) { + return 'certain_days'; + } + return 'custom'; +} + +const stylesheet = new CSSStyleSheet(); +stylesheet.replace(` +.tg-crontab-display { + border-bottom: 1px dashed var(--border-color); +} + +.tg-crontab-editor { + border-radius: 8px; + background: var(--portal-background); + box-shadow: var(--portal-box-shadow); + overflow: auto; +} + +.tg-crontab-editor-content { + border-bottom: 1px solid var(--border-color); +} + +.tg-crontab-editor-left { + border-right: 1px solid var(--border-color); +} + +.tg-crontab-editor-right { + place-self: stretch; +} + +.tg-crontab-editor-mode { + cursor: pointer; +} + +.tg-crontab-editor-mode.selected, +.tg-crontab-editor-mode:hover { + background: var(--select-hover-background); +} + +.tg-crontab--select-portal { + max-height: 150px; + -ms-overflow-style: none; /* Internet Explorer 10+ */ + scrollbar-width: none; /* Firefox, Safari 18.2+, Chromium 121+ */ +} +.tg-crontab--select-portal::-webkit-scrollbar { + display: none; /* Older Safari and Chromium */ +} + +.tg-crontab--select-portal.shorter { + max-height: 120px; +} +`); + +export { CrontabInput }; diff --git a/testgen/ui/components/frontend/js/components/input.js b/testgen/ui/components/frontend/js/components/input.js index 1cc4e449..c0324cad 100644 --- a/testgen/ui/components/frontend/js/components/input.js +++ b/testgen/ui/components/frontend/js/components/input.js @@ -23,6 +23,7 @@ * @property {boolean?} passwordSuggestions * @property {function(string, InputState)?} onChange * @property {boolean?} disabled + * @property {boolean?} readonly * @property {function(string, InputState)?} onClear * @property {number?} width * @property {number?} height @@ -135,6 +136,7 @@ const Input = (/** @type Properties */ props) => { name: props.name ?? '', type: inputType, disabled: props.disabled, + ...(props.readonly ? {readonly: true} : {}), ...(props.passwordSuggestions ?? true ? {} : {autocomplete: 'off', 'data-op-ignore': true}), placeholder: () => getValue(props.placeholder) ?? '', oninput: debounce((/** @type Event */ event) => { diff --git a/testgen/ui/components/frontend/js/components/link.js b/testgen/ui/components/frontend/js/components/link.js index ae460d5d..96468963 100644 --- a/testgen/ui/components/frontend/js/components/link.js +++ b/testgen/ui/components/frontend/js/components/link.js @@ -43,6 +43,7 @@ const Link = (/** @type Properties */ props) => { const params = getValue(props.params) ?? {}; const open_new = !!getValue(props.open_new); const showTooltip = van.state(false); + const isExternal = /http(s)?:\/\//.test(href); return a( { @@ -51,7 +52,7 @@ const Link = (/** @type Properties */ props) => { ${getValue(props.disabled) ? 'disabled' : ''} ${getValue(props.class) ?? ''}`, style: props.style, - href: `/${href}${getQueryFromParams(params)}`, + href: isExternal ? href : `/${href}${getQueryFromParams(params)}`, target: open_new ? '_blank' : '', onclick: open_new ? null : (event) => { event.preventDefault(); diff --git a/testgen/ui/components/frontend/js/components/portal.js b/testgen/ui/components/frontend/js/components/portal.js index ad2287e5..51c82b63 100644 --- a/testgen/ui/components/frontend/js/components/portal.js +++ b/testgen/ui/components/frontend/js/components/portal.js @@ -3,7 +3,7 @@ * * NOTE: Ensure options is an object and turn individual properties into van.state * if dynamic updates are needed. - * + * * @typedef Options * @type {object} * @property {string} target diff --git a/testgen/ui/components/frontend/js/components/select.js b/testgen/ui/components/frontend/js/components/select.js index 28879e04..95e1e741 100644 --- a/testgen/ui/components/frontend/js/components/select.js +++ b/testgen/ui/components/frontend/js/components/select.js @@ -20,6 +20,8 @@ * @property {number?} height * @property {string?} style * @property {string?} testId + * @property {number?} portalClass + * @property {('normal' | 'inline')?} triggerStyle */ import van from '../van.min.js'; import { getRandomId, getValue, loadStylesheet, isState, isEqual } from '../utils.js'; @@ -90,31 +92,48 @@ const Select = (/** @type {Properties} */ props) => { ? span({ class: 'text-error' }, '*') : '', ), - div( - { - class: () => `flex-row tg-select--field ${opened.val ? 'opened' : ''}`, - style: () => getValue(props.height) ? `height: ${getValue(props.height)}px;` : '', - 'data-testid': 'select-input', - }, - () => div( - { class: 'tg-select--field--content', 'data-testid': 'select-input-display' }, - valueIcon.val - ? Icon({ classes: 'mr-2' }, valueIcon.val) - : undefined, - valueLabel.val, - ), - div( - { class: 'tg-select--field--icon', 'data-testid': 'select-input-trigger' }, - i( - { class: 'material-symbols-rounded' }, - 'expand_more', + + () => getValue(props.triggerStyle) === 'inline' + ? div( + {class: 'tg-select--inline-trigger flex-row'}, + span({}, valueLabel.val ?? '---'), + div( + { class: 'tg-select--field--icon ', 'data-testid': 'select-input-trigger' }, + i( + { class: 'material-symbols-rounded' }, + 'expand_more', + ), + ), + ) + : div( + { + class: () => `flex-row tg-select--field ${opened.val ? 'opened' : ''}`, + style: () => getValue(props.height) ? `height: ${getValue(props.height)}px;` : '', + 'data-testid': 'select-input', + }, + () => div( + { class: 'tg-select--field--content', 'data-testid': 'select-input-display' }, + valueIcon.val + ? Icon({ classes: 'mr-2' }, valueIcon.val) + : undefined, + valueLabel.val, + ), + div( + { class: 'tg-select--field--icon', 'data-testid': 'select-input-trigger' }, + i( + { class: 'material-symbols-rounded' }, + 'expand_more', + ), ), ), - ), + Portal( {target: domId.val, targetRelative: true, opened}, () => div( - { class: 'tg-select--options-wrapper mt-1', 'data-testid': 'select-options' }, + { + class: () => `tg-select--options-wrapper mt-1 ${getValue(props.portalClass) ?? ''}`, + 'data-testid': 'select-options', + }, getValue(options).map(option => div( { @@ -226,6 +245,14 @@ stylesheet.replace(` background: var(--select-hover-background); color: var(--primary-color); } + +.tg-select--inline-trigger { + border-bottom: 1px solid var(--border-color); +} + +.tg-select--inline-trigger > span { + min-width: 24px; +} `); export { Select }; diff --git a/testgen/ui/components/frontend/js/pages/schedule_list.js b/testgen/ui/components/frontend/js/pages/schedule_list.js index f8c54f96..bc403a3e 100644 --- a/testgen/ui/components/frontend/js/pages/schedule_list.js +++ b/testgen/ui/components/frontend/js/pages/schedule_list.js @@ -1,4 +1,6 @@ /** + * @import {CronSample} from '../components/crontab_input.js' + * * @typedef Schedule * @type {object} * @property {string} argValue @@ -10,23 +12,38 @@ * @type {object} * @property {boolean} can_edit * + * @typedef Results + * @type {object} + * @property {boolean} success + * @property {string} message + * * @typedef Properties * @type {object} * @property {Schedule[]} items * @property {Permissions} permissions * @property {string} arg_label + * @property {import('../components/select.js').Option[]} arg_values + * @property {CronSample?} sample + * @property {Results?} results */ import van from '../van.min.js'; import { Button } from '../components/button.js'; import { Streamlit } from '../streamlit.js'; -import { emitEvent, getValue, resizeFrameHeightToElement, resizeFrameHeightOnDOMChange } from '../utils.js'; +import { emitEvent, getValue, loadStylesheet } from '../utils.js'; import { withTooltip } from '../components/tooltip.js'; +import { ExpansionPanel } from '../components/expansion_panel.js'; +import { Select } from '../components/select.js'; +import { CrontabInput } from '../components/crontab_input.js'; +import { timezones } from '../values.js'; +import { Alert } from '../components/alert.js'; - -const { div, span, i, rawHTML } = van.tags; +const minHeight = 380; +const { div, span, i } = van.tags; const ScheduleList = (/** @type Properties */ props) => { - window.testgen.isPage = false; + loadStylesheet('schedule-list', stylesheet); + + window.testgen.isPage = true; const scheduleItems = van.derive(() => { let items = []; @@ -35,37 +52,117 @@ const ScheduleList = (/** @type Properties */ props) => { } catch (e) { console.log(e) } - Streamlit.setFrameHeight(100 * items.length || 150); + Streamlit.setFrameHeight(Math.max(minHeight, 100 * items.length || 150)); return items; }); - const columns = ['40%', '50%', '10%']; - const tableId = 'profiling-schedules-table'; - resizeFrameHeightToElement(tableId); - resizeFrameHeightOnDOMChange(tableId); + const newScheduleForm = { + argValue: van.state(''), + timezone: van.state(Intl.DateTimeFormat().resolvedOptions().timeZone), + expression: van.state(null), + }; + const cronEditorValue = van.derive(() => ({ + timezone: newScheduleForm.timezone.val, + expression: newScheduleForm.expression.val, + })); + + const columns = ['40%', '50%', '10%']; + const domId = 'schedules-table'; return div( - { class: 'table', id: tableId }, - div( - { class: 'table-header flex-row' }, - span( - { style: `flex: ${columns[0]}` }, - getValue(props.arg_label), + { id: domId, class: 'flex-column fx-gap-2', style: 'height: 100%; overflow-y: auto;' }, + ExpansionPanel( + {title: 'Add Schedule', testId: 'scheduler-cron-editor'}, + div( + { class: 'flex-row fx-gap-2' }, + () => Select({ + label: getValue(props.arg_label), + options: props.arg_values, + value: newScheduleForm.argValue, + style: 'flex: 1;', + onChange: (value) => newScheduleForm.argValue.val = value, + portalClass: 'short-select-portal', + }), + () => Select({ + label: 'Timezone', + options: timezones.map(tz_ => ({label: tz_, value: tz_})), + value: newScheduleForm.timezone, + allowNull: false, + onChange: (value) => { + newScheduleForm.timezone.val = value; + if (newScheduleForm.expression.val && newScheduleForm.timezone.val) { + emitEvent('GetCronSample', {payload: {cron_expr: newScheduleForm.expression.val, tz: newScheduleForm.timezone.val}}); + } + }, + portalClass: 'short-select-portal', + }), + CrontabInput({ + class: 'fx-flex', + sample: props.sample, + value: cronEditorValue, + onChange: (value) => { + newScheduleForm.expression.val = value; + if (newScheduleForm.expression.val && newScheduleForm.timezone.val) { + emitEvent('GetCronSample', {payload: {cron_expr: newScheduleForm.expression.val, tz: newScheduleForm.timezone.val}}); + } + }, + }), ), - span( - { style: `flex: ${columns[1]}` }, - 'Cron Expression | Timezone', + div( + { class: 'flex-row fx-justify-content-flex-end mt-3' }, + Button({ + type: 'stroked', + label: 'Add Schedule', + width: '150px', + onclick: () => emitEvent('AddSchedule', {payload: { + arg_value: newScheduleForm.argValue.val, + cron_expr: newScheduleForm.expression.val, + cron_tz: newScheduleForm.timezone.val, + }}), + }), ), - span( - { style: `flex: ${columns[2]}` }, - 'Actions', + () => { + const results = getValue(props.results); + if (!results) { + return ''; + } + + if (results.success) { + newScheduleForm.argValue.val = ''; + newScheduleForm.expression.val = null; + newScheduleForm.timezone.val = Intl.DateTimeFormat().resolvedOptions().timeZone; + } + + return Alert({ + type: results.success ? 'success' : 'error', + class: 'mt-3', + closeable: true, + }, results.message); + }, + ), + div( + { class: 'table fx-flex' }, + div( + { class: 'table-header flex-row' }, + span( + { style: `flex: ${columns[0]}` }, + getValue(props.arg_label), + ), + span( + { style: `flex: ${columns[1]}` }, + 'Cron Expression | Timezone', + ), + span( + { style: `flex: ${columns[2]}` }, + 'Actions', + ), ), + () => scheduleItems.val?.length + ? div( + scheduleItems.val.map(item => ScheduleListItem(item, columns, getValue(props.permissions))), + ) + : div({ class: 'mt-5 mb-3 ml-3 text-secondary', style: 'text-align: center;' }, 'No schedules defined yet.'), ), - () => scheduleItems.val?.length - ? div( - scheduleItems.val.map(item => ScheduleListItem(item, columns, getValue(props.permissions))), - ) - : div({ class: 'mt-5 mb-3 ml-3 text-secondary' }, 'No schedules defined yet.'), ); } @@ -94,7 +191,13 @@ const ScheduleListItem = ( }, 'info', ), - { text: [div("Next runs:"), ...item.sample?.map(v => div(v))] }, + { + text: [ + div({class: 'text-center mb-1'}, item.readableExpr), + div({class: 'text-left'}, "Next runs:"), + ...item.sample?.map(v => div({class: 'text-left'}, v)) + ], + }, ), ), div( @@ -115,4 +218,12 @@ const ScheduleListItem = ( ); } + +const stylesheet = new CSSStyleSheet(); +stylesheet.replace(` +.short-select-portal { + max-height: 250px !important; +} +`); + export { ScheduleList }; diff --git a/testgen/ui/components/frontend/js/values.js b/testgen/ui/components/frontend/js/values.js new file mode 100644 index 00000000..99a23a36 --- /dev/null +++ b/testgen/ui/components/frontend/js/values.js @@ -0,0 +1,3 @@ +const timezones = Intl.supportedValuesOf('timeZone'); + +export { timezones }; diff --git a/testgen/ui/components/widgets/__init__.py b/testgen/ui/components/widgets/__init__.py index 5716f6c9..f3812ef7 100644 --- a/testgen/ui/components/widgets/__init__.py +++ b/testgen/ui/components/widgets/__init__.py @@ -26,5 +26,4 @@ from testgen.ui.components.widgets.sorting_selector import sorting_selector from testgen.ui.components.widgets.summary_bar import summary_bar from testgen.ui.components.widgets.testgen_component import testgen_component -from testgen.ui.components.widgets.tz_select import tz_select from testgen.ui.components.widgets.wizard import WizardStep, wizard diff --git a/testgen/ui/components/widgets/tz_select.py b/testgen/ui/components/widgets/tz_select.py deleted file mode 100644 index bbd09a0b..00000000 --- a/testgen/ui/components/widgets/tz_select.py +++ /dev/null @@ -1,70 +0,0 @@ -import zoneinfo - -import streamlit as st - -HANDY_TIMEZONES = [ - "Africa/Abidjan", # +00:00 - "Africa/Johannesburg", # +02:00 - "Africa/Lagos", # +01:00 - "America/Anchorage", # -09:00 - "America/Argentina/Buenos_Aires", # -03:00 - "America/Bogota", # -05:00 - "America/Chicago", # -06:00 - "America/Denver", # -07:00 - "America/Halifax", # -03:00 - "America/Los_Angeles", # -08:00 - "America/Mexico_City", # -06:00 - "America/New_York", # -04:00 (during DST) - "America/Phoenix", # -07:00 - "America/Sao_Paulo", # -03:00 - "America/Toronto", # -04:00 (during DST) - "America/Vancouver", # -08:00 - "Asia/Almaty", # +06:00 - "Asia/Baku", # +04:00 - "Asia/Bangkok", # +07:00 - "Asia/Colombo", # +05:30 - "Asia/Dhaka", # +06:00 - "Asia/Dubai", # +04:00 - "Asia/Jakarta", # +07:00 - "Asia/Kabul", # +04:30 - "Asia/Kolkata", # +05:30 - "Asia/Manila", # +08:00 - "Asia/Riyadh", # +03:00 - "Asia/Seoul", # +09:00 - "Asia/Shanghai", # +08:00 - "Asia/Singapore", # +08:00 - "Asia/Tokyo", # +09:00 - "Atlantic/Azores", # -01:00 - "Atlantic/South_Georgia", # -02:00 - "Australia/Sydney", # +10:00 - "Europe/Amsterdam", # +01:00 - "Europe/Athens", # +02:00 - "Europe/Berlin", # +01:00 - "Europe/Bucharest", # +02:00 - "Europe/Helsinki", # +02:00 - "Europe/Istanbul", # +03:00 - "Europe/London", # +00:00 - "Europe/Moscow", # +03:00 - "Europe/Paris", # +01:00 - "Pacific/Auckland", # +12:00 - "Pacific/Honolulu", # -10:00 - "Pacific/Noumea", # +11:00 - "Pacific/Port_Moresby", # +10:00 -] - - -def tz_select(*, default="America/New_York", **kwargs): - tz_options = HANDY_TIMEZONES[:] - tz_options.extend(sorted(tz for tz in zoneinfo.available_timezones() if tz not in HANDY_TIMEZONES)) - - if "index" in kwargs: - raise ValueError("Use the Session State API instead.") - - # This is wierd, but apparently Streamlit likes it this way - if "key" in kwargs and st.session_state.get(kwargs["key"], None) in tz_options: - kwargs["index"] = tz_options.index(st.session_state[kwargs["key"]]) - del st.session_state[kwargs["key"]] - else: - kwargs["index"] = tz_options.index(st.session_state.get("browser_timezone", default)) - - return st.selectbox(options=tz_options, format_func=lambda v: v.replace("_", " "), **kwargs) diff --git a/testgen/ui/views/dialogs/manage_schedules.py b/testgen/ui/views/dialogs/manage_schedules.py index 702e41b4..820cd782 100644 --- a/testgen/ui/views/dialogs/manage_schedules.py +++ b/testgen/ui/views/dialogs/manage_schedules.py @@ -5,14 +5,14 @@ from uuid import UUID import cron_converter +import cron_descriptor import streamlit as st from sqlalchemy.exc import IntegrityError from testgen.common.models import Session, with_database_session from testgen.common.models.scheduler import JobSchedule from testgen.ui.components import widgets as testgen -from testgen.ui.components.widgets import tz_select -from testgen.ui.session import session +from testgen.ui.session import session, temp_value class ScheduleDialog: @@ -30,18 +30,108 @@ def init(self) -> None: def get_arg_value(self, job): raise NotImplementedError - def arg_value_input(self) -> tuple[bool, list[Any], dict[str, Any]]: + def get_arg_value_options(self) -> list[dict[str, str]]: + raise NotImplementedError + + def get_job_arguments(self, arg_value: str) -> tuple[list[Any], dict[str, Any]]: raise NotImplementedError @with_database_session def open(self, project_code: str) -> None: - st.session_state["schedule_form_success"] = None - st.session_state["schedule_cron_expr"] = "" self.project_code = project_code self.init() return st.dialog(title=self.title)(self.render)() def render(self) -> None: + def on_delete_sched(item): + with Session() as db_session: + try: + sched, = db_session.query(JobSchedule).where(JobSchedule.id == UUID(item["id"])) + db_session.delete(sched) + except ValueError: + db_session.rollback() + else: + db_session.commit() + st.rerun(scope="fragment") + + def on_cron_sample(payload: dict[str, str]): + try: + cron_expr = payload["cron_expr"] + cron_tz = payload.get("tz", "America/New_York") + + cron_obj = cron_converter.Cron(cron_expr) + cron_schedule = cron_obj.schedule(datetime.now(zoneinfo.ZoneInfo(cron_tz))) + readble_cron_schedule = cron_descriptor.get_description( + cron_expr, + ) + + set_cron_sample({ + "sample": cron_schedule.next().strftime("%a %b %-d, %-I:%M %p"), + "readable_expr": readble_cron_schedule, + }) + except ValueError as e: + set_cron_sample({"error": str(e)}) + except Exception as e: + set_cron_sample({"error": "Error validating the Cron expression"}) + + def on_add_schedule(payload: dict[str, str]): + set_arg_value(payload["arg_value"]) + set_timezone(payload["cron_tz"]) + set_cron_expr(payload["cron_expr"]) + + set_should_save(True) + + user_can_edit = session.auth.user_has_permission("edit") + cron_sample_result, set_cron_sample = temp_value("schedule_dialog:cron_expr_validation", default={}) + get_arg_value, set_arg_value = temp_value("schedule_dialog:new:arg_value", default=None) + get_timezone, set_timezone = temp_value("schedule_dialog:new:timezone", default=None) + get_cron_expr, set_cron_expr = temp_value("schedule_dialog:new:cron_expr", default=None) + should_save, set_should_save = temp_value("schedule_dialog:new:should_save", default=False) + + results = None + if should_save(): + success = True + message = "Schedule added" + + try: + arg_value = get_arg_value() + cron_expr = get_cron_expr() + cron_tz = get_timezone() + + is_form_valid = ( + bool(arg_value) + and bool(cron_tz) + and bool(cron_expr) + ) + + if is_form_valid: + cron_obj = cron_converter.Cron(cron_expr) + args, kwargs = self.get_job_arguments(arg_value) + with Session() as db_session: + sched_model = JobSchedule( + project_code=self.project_code, + key=self.job_key, + cron_expr=cron_obj.to_string(), + cron_tz=cron_tz, + args=args, + kwargs=kwargs, + ) + db_session.add(sched_model) + db_session.commit() + else: + success = False + message = "Complete all the fields before adding the schedule" + except IntegrityError: + success = False + message = "This schedule already exists." + except ValueError as e: + success = False + message = str(e) + except Exception as e: + success = False + message = "Error validating the Cron expression" + results = {"success": success, "message": message} + with Session() as db_session: scheduled_jobs = ( db_session.query(JobSchedule) @@ -53,6 +143,7 @@ def render(self) -> None: "id": str(job.id), "argValue": self.get_arg_value(job), "cronExpr": job.cron_expr, + "readableExpr": cron_descriptor.get_description(job.cron_expr), "cronTz": job.cron_tz_str, "sample": [ sample.strftime("%a %b %-d, %-I:%M %p") @@ -61,104 +152,21 @@ def render(self) -> None: } scheduled_jobs_json.append(job_json) - def on_delete_sched(item): - with Session() as db_session: - try: - sched, = db_session.query(JobSchedule).where(JobSchedule.id == UUID(item["id"])) - db_session.delete(sched) - except ValueError: - db_session.rollback() - else: - db_session.commit() - st.rerun(scope="fragment") - - user_can_edit = session.auth.user_has_permission("edit") testgen.testgen_component( "schedule_list", props={ "items": json.dumps(scheduled_jobs_json), - "arg_abel": self.arg_label, + "arg_label": self.arg_label, + "arg_values": self.get_arg_value_options(), "permissions": {"can_edit": user_can_edit}, + "sample": cron_sample_result(), + "results": results, + }, + event_handlers={ + "DeleteSchedule": on_delete_sched, + }, + on_change_handlers={ + "GetCronSample": on_cron_sample, + "AddSchedule": on_add_schedule, }, - event_handlers={"DeleteSchedule": on_delete_sched} ) - - if user_can_edit: - with st.container(border=True): - self.add_schedule_form() - - def add_schedule_form(self): - st.html("Add schedule") - arg_column, expr_column, tz_column, button_column = st.columns([.3, .4, .3, .1], vertical_alignment="bottom") - status_container = st.empty() - - with status_container: - match st.session_state.get("schedule_form_success", None): - case True: - st.success("Schedule added.", icon=":material/check:") - st.session_state["schedule_cron_expr"] = "" - del st.session_state["schedule_cron_tz"] - del st.session_state["schedule_form_success"] - case False: - st.error("This schedule already exists.", icon=":material/block:") - case None: - testgen.whitespace(56, "px") - - with arg_column: - args_valid, args, kwargs = self.arg_value_input() - - with expr_column: - cron_expr = st.text_input( - label="Cron Expression", - help="Examples: Every day at 6:00 AM: 0 6 * * * — Every Monday at 5:30 PM: 30 17 * * 1", - key="schedule_cron_expr", - ) - - with tz_column: - cron_tz = tz_select(label="Timezone", key="schedule_cron_tz") - - cron_obj = None - if cron_expr: - with status_container: - try: - cron_obj = cron_converter.Cron(cron_expr) - cron_schedule = cron_obj.schedule(datetime.now(zoneinfo.ZoneInfo(cron_tz))) - sample = [cron_schedule.next().strftime("%a %b %-d, %-I:%M %p") for _ in range(3)] - except ValueError as e: - st.warning(str(e), icon=":material/warning:") - except Exception as e: - st.error("Error validating the Cron expression") - else: - # We postpone the validation status update when the previous rerun had a failed - # attempt to insert a schedule. This prevents the error message of being overridden - if st.session_state.get("schedule_form_success", None) is None: - st.info( - f"**Next runs:** {' | '.join(sample)} ({cron_tz.replace('_', ' ')})", - icon=":material/check:", - ) - else: - st.session_state["schedule_form_success"] = None - - is_form_valid = bool(args_valid and cron_obj) - with button_column: - add_button = st.button("Add", use_container_width=True, disabled=not is_form_valid) - - # We also check for `is_form_valid` here because apparently it's possible to click a disabled button =) - if add_button and is_form_valid: - with Session() as db_session: - try: - sched_model = JobSchedule( - project_code=self.project_code, - key=self.job_key, - cron_expr=cron_obj.to_string(), - cron_tz=cron_tz, - args=args, - kwargs=kwargs, - ) - db_session.add(sched_model) - db_session.commit() - except IntegrityError: - st.session_state["schedule_form_success"] = False - else: - st.session_state["schedule_form_success"] = True - st.rerun(scope="fragment") diff --git a/testgen/ui/views/profiling_runs.py b/testgen/ui/views/profiling_runs.py index 01760874..9fd38890 100644 --- a/testgen/ui/views/profiling_runs.py +++ b/testgen/ui/views/profiling_runs.py @@ -131,17 +131,14 @@ def init(self) -> None: def get_arg_value(self, job): return next(item.table_groups_name for item in self.table_groups if str(item.id) == job.kwargs["table_group_id"]) - def arg_value_input(self) -> tuple[bool, list[typing.Any], dict[str, typing.Any]]: - table_groups_df = to_dataframe(self.table_groups, TableGroupMinimal.columns()) - tg_id = testgen.select( - label="Table Group", - options=table_groups_df, - value_column="id", - display_column="table_groups_name", - required=True, - placeholder="Select table group", - ) - return bool(tg_id), [], {"table_group_id": str(tg_id)} + def get_arg_value_options(self) -> list[dict[str, str]]: + return [ + {"value": str(table_group.id), "label": table_group.table_groups_name} + for table_group in self.table_groups + ] + + def get_job_arguments(self, arg_value: str) -> tuple[list[typing.Any], dict[str, typing.Any]]: + return [], {"table_group_id": str(arg_value)} def render_empty_state(project_code: str, user_can_run: bool) -> bool: diff --git a/testgen/ui/views/test_runs.py b/testgen/ui/views/test_runs.py index 72312c8c..1d1f43b0 100644 --- a/testgen/ui/views/test_runs.py +++ b/testgen/ui/views/test_runs.py @@ -149,17 +149,14 @@ def init(self) -> None: def get_arg_value(self, job): return job.kwargs["test_suite_key"] - def arg_value_input(self) -> tuple[bool, list[typing.Any], dict[str, typing.Any]]: - test_suites_df = to_dataframe(self.test_suites, TestSuiteMinimal.columns()) - ts_name = testgen.select( - label="Test Suite", - options=test_suites_df, - value_column="test_suite", - display_column="test_suite", - required=True, - placeholder="Select test suite", - ) - return bool(ts_name), [], {"project_key": self.project_code, "test_suite_key": ts_name} + def get_arg_value_options(self) -> list[dict[str, str]]: + return [ + {"value": test_suite.test_suite, "label": test_suite.test_suite} + for test_suite in self.test_suites + ] + + def get_job_arguments(self, arg_value: str) -> tuple[list[typing.Any], dict[str, typing.Any]]: + return [], {"project_key": self.project_code, "test_suite_key": arg_value} def render_empty_state(project_code: str, user_can_run: bool) -> bool: From 3712b8b485464d04dca544b51ffdbc1abdadb5da Mon Sep 17 00:00:00 2001 From: Luis Date: Fri, 29 Aug 2025 16:19:42 -0400 Subject: [PATCH 07/28] fix(connections): set snowflake account in connection url also dynamically refreshes the url in the connection form while the user edits the relevant fields --- .../common/database/flavor/flavor_service.py | 45 ++++- .../flavor/snowflake_flavor_service.py | 32 ++-- testgen/common/models/connection.py | 18 +- .../frontend/js/components/alert.js | 1 + .../frontend/js/components/connection_form.js | 181 ++++++++---------- .../frontend/js/pages/connections.js | 2 + testgen/ui/views/connections.py | 9 + 7 files changed, 162 insertions(+), 126 deletions(-) diff --git a/testgen/common/database/flavor/flavor_service.py b/testgen/common/database/flavor/flavor_service.py index 986cb64f..23f2a013 100644 --- a/testgen/common/database/flavor/flavor_service.py +++ b/testgen/common/database/flavor/flavor_service.py @@ -1,5 +1,6 @@ from abc import abstractmethod -from typing import Literal, TypedDict +from typing import Any, Literal, TypedDict +from urllib.parse import parse_qs, urlparse from testgen.common.encrypt import DecryptText @@ -90,3 +91,45 @@ def get_connection_string_from_fields(self) -> str: @abstractmethod def get_connection_string_head(self) -> str: raise NotImplementedError("Subclasses must implement this method") + + def get_parts_from_connection_string(self) -> dict[str, Any]: + if self.connect_by_url: + if not self.url: + return {} + + parsed_url = urlparse(self.get_connection_string()) + credentials, location = ( + parsed_url.netloc if "@" in parsed_url.netloc else f"@{parsed_url.netloc}" + ).split("@") + username, password = ( + credentials if ":" in credentials else f"{credentials}:" + ).split(":") + host, port = ( + location if ":" in location else f"{location}:" + ).split(":") + + database = (path_patrs[0] if (path_patrs := parsed_url.path.strip("/").split("/")) else "") + + extras = { + param_name: param_values[0] + for param_name, param_values in parse_qs(parsed_url.query or "").items() + } + + return { + "username": username, + "password": password, + "host": host, + "port": port, + "dbname": database, + **extras, + } + + return { + "username": self.username, + "password": self.password, + "host": self.host, + "port": self.port, + "dbname": self.dbname, + "http_path": self.http_path, + "catalog": self.catalog, + } diff --git a/testgen/common/database/flavor/snowflake_flavor_service.py b/testgen/common/database/flavor/snowflake_flavor_service.py index c1636d7f..7662f832 100644 --- a/testgen/common/database/flavor/snowflake_flavor_service.py +++ b/testgen/common/database/flavor/snowflake_flavor_service.py @@ -2,6 +2,7 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization +from snowflake.sqlalchemy import URL from testgen.common.database.flavor.flavor_service import FlavorService @@ -38,25 +39,18 @@ def get_connection_string_from_fields(self): # optionally + '/[schema]' + '?warehouse=xxx' # NOTE: Snowflake host should NOT include ".snowflakecomputing.com" - def get_raw_host_name(host): - endings = [ - ".snowflakecomputing.com", - ] - for ending in endings: - if host.endswith(ending): - i = host.index(ending) - return host[0:i] - return host - - raw_host = get_raw_host_name(self.host) - host = raw_host - if self.port != "443": - host += ":" + self.port - - if self.connect_by_key: - return f"snowflake://{self.username}@{host}/{self.dbname}/{self.dbschema}" - else: - return f"snowflake://{self.username}:{quote_plus(self.password)}@{host}/{self.dbname}/{self.dbschema}" + account, _ = self.host.split(".", maxsplit=1) if "." in self.host else ("", "") + connection_url = URL( + host=self.host, + port=int(self.port if str(self.port).isdigit() else 443), + account=account, + user=self.username, + password="" if self.connect_by_key else self.password, + database=self.dbname, + schema=self.dbschema or "", + ) + + return connection_url def get_pre_connection_queries(self): return [ diff --git a/testgen/common/models/connection.py b/testgen/common/models/connection.py index 90de7daa..6a7142dc 100644 --- a/testgen/common/models/connection.py +++ b/testgen/common/models/connection.py @@ -19,6 +19,7 @@ from sqlalchemy.dialects import postgresql from sqlalchemy.orm import InstrumentedAttribute +from testgen.common.database.database_service import get_flavor_service from testgen.common.database.flavor.flavor_service import SQLFlavor from testgen.common.models import get_current_session from testgen.common.models.custom_types import EncryptedBytea @@ -115,13 +116,14 @@ def clear_cache(cls) -> bool: def save(self) -> None: if self.connect_by_url and self.url: - url_sections = self.url.split("/") - if url_sections: - host_port = url_sections[0] - host_port_sections = host_port.split(":") - self.project_host = host_port_sections[0] if host_port_sections else host_port - self.project_port = "".join(host_port_sections[1:]) if host_port_sections else "" - if len(url_sections) > 1: - self.project_db = url_sections[1] + flavor_service = get_flavor_service(self.sql_flavor) + flavor_service.init(self.to_dict()) + + connection_parts = flavor_service.get_parts_from_connection_string() + if connection_parts: + self.project_host = connection_parts["host"] + self.project_port = connection_parts["port"] + self.project_db = connection_parts["dbname"] + self.http_path = connection_parts.get("http_path") or None super().save() diff --git a/testgen/ui/components/frontend/js/components/alert.js b/testgen/ui/components/frontend/js/components/alert.js index cda6afda..d6d9c716 100644 --- a/testgen/ui/components/frontend/js/components/alert.js +++ b/testgen/ui/components/frontend/js/components/alert.js @@ -55,6 +55,7 @@ const Alert = (/** @type Properties */ props, /** @type Array */ .. type: 'icon', icon: 'close', style: `margin-left: auto;`, + onclick: close, }); }, ); diff --git a/testgen/ui/components/frontend/js/components/connection_form.js b/testgen/ui/components/frontend/js/components/connection_form.js index ed0f0848..23d0ad75 100644 --- a/testgen/ui/components/frontend/js/components/connection_form.js +++ b/testgen/ui/components/frontend/js/components/connection_form.js @@ -50,6 +50,7 @@ * @property {Array.} flavors * @property {boolean} disableFlavor * @property {FileValue?} cachedPrivateKeyFile + * @property {string?} dynamicConnectionUrl * @property {(c: Connection, state: FormState, cache?: FieldsCache) => void} onChange */ import van from '../van.min.js'; @@ -97,10 +98,6 @@ const ConnectionForm = (props, saveButton) => { const connectionQueryChars = van.state(connection?.max_query_chars ?? 9000); const privateKeyFile = van.state(getValue(props.cachedPrivateKeyFile) ?? null); - const flavor = getValue(props.flavors).find(f => f.value === connectionFlavor.rawVal); - const originalURLTemplate = flavor.connection_string; - const [_, urlSuffix] = originalURLTemplate.split('@'); - const updatedConnection = van.state({ project_code: connection.project_code, connection_id: connection.connection_id, @@ -115,21 +112,27 @@ const ConnectionForm = (props, saveButton) => { private_key: isEditMode ? '' : (connection?.private_key ?? ''), private_key_passphrase: isEditMode ? '' : (connection?.private_key_passphrase ?? ''), http_path: connection?.http_path ?? '', - url: connection?.connect_by_url - ? (connection?.url ?? '') - : formatURL( - urlSuffix ?? '', - connection?.project_host ?? '', - connection?.project_port ?? defaultPort ?? '', - connection?.project_db ?? '', - connection?.http_path ?? '', - ), - + url: connection?.url ?? '', sql_flavor_code: connectionFlavor.rawVal ?? '', connection_name: connectionName.rawVal ?? '', max_threads: connectionMaxThreads.rawVal ?? 4, max_query_chars: connectionQueryChars.rawVal ?? 9000, }); + const dynamicConnectionUrl = van.state(props.dynamicConnectionUrl?.rawVal ?? ''); + + van.derive(() => { + const previousValue = updatedConnection.oldVal; + const currentValue = updatedConnection.rawVal; + + if (shouldRefreshUrl(previousValue, currentValue)) { + emitEvent('ConnectionUpdated', {payload: updatedConnection.rawVal}); + } + }); + + van.derive(() => { + const updatedUrl = getValue(props.dynamicConnectionUrl); + dynamicConnectionUrl.val = updatedUrl; + }); const dirty = van.derive(() => !isEqual(updatedConnection.val, connection)); const validityPerField = van.state({}); @@ -143,6 +146,7 @@ const ConnectionForm = (props, saveButton) => { setFieldValidity('redshift_form', isValid); }, connection, + dynamicConnectionUrl, ), azure_mssql: () => AzureMSSQLForm( updatedConnection, @@ -152,6 +156,7 @@ const ConnectionForm = (props, saveButton) => { setFieldValidity('mssql_form', isValid); }, connection, + dynamicConnectionUrl, ), synapse_mssql: () => SynapseMSSQLForm( updatedConnection, @@ -161,6 +166,7 @@ const ConnectionForm = (props, saveButton) => { setFieldValidity('mssql_form', isValid); }, connection, + dynamicConnectionUrl, ), mssql: () => MSSQLForm( updatedConnection, @@ -170,6 +176,7 @@ const ConnectionForm = (props, saveButton) => { setFieldValidity('mssql_form', isValid); }, connection, + dynamicConnectionUrl, ), postgresql: () => PostgresqlForm( updatedConnection, @@ -179,6 +186,7 @@ const ConnectionForm = (props, saveButton) => { setFieldValidity('mssql_form', isValid); }, connection, + dynamicConnectionUrl, ), snowflake: () => SnowflakeForm( @@ -191,6 +199,7 @@ const ConnectionForm = (props, saveButton) => { }, connection, getValue(props.cachedPrivateKeyFile) ?? null, + dynamicConnectionUrl, ), databricks: () => DatabricksForm( updatedConnection, @@ -200,6 +209,7 @@ const ConnectionForm = (props, saveButton) => { setFieldValidity('databricks_form', isValid); }, connection, + dynamicConnectionUrl, ), }; @@ -328,6 +338,7 @@ const ConnectionForm = (props, saveButton) => { * @param {boolean} maskPassword * @param {(params: Partial, isValid: boolean) => void} onChange * @param {Connection?} originalConnection + * @param {VanState} dynamicConnectionUrl * @returns {HTMLElement} */ const RedshiftForm = ( @@ -335,9 +346,8 @@ const RedshiftForm = ( flavor, onChange, originalConnection, + dynamicConnectionUrl, ) => { - const originalURLTemplate = flavor.connection_string; - const isValid = van.state(true); const connectByUrl = van.state(connection.rawVal.connect_by_url ?? false); const connectionHost = van.state(connection.rawVal.project_host ?? ''); @@ -345,29 +355,10 @@ const RedshiftForm = ( const connectionDatabase = van.state(connection.rawVal.project_db ?? ''); const connectionUsername = van.state(connection.rawVal.project_user ?? ''); const connectionPassword = van.state(connection.rawVal?.project_pw_encrypted ?? ''); - - const [prefixPart, sufixPart] = originalURLTemplate.split('@'); - const connectionStringPrefix = van.state(`${prefixPart}@`); - const connectionStringSuffix = van.state(connection.rawVal?.url ?? ''); + const connectionUrl = van.state(connection.rawVal?.url ?? ''); const validityPerField = {}; - if (!connectionStringSuffix.rawVal) { - connectionStringSuffix.val = formatURL(sufixPart ?? '', connectionHost.rawVal, connectionPort.rawVal, connectionDatabase.rawVal); - } - - van.derive(() => { - const connectionHost_ = connectionHost.val; - const connectionPort_ = connectionPort.val; - const connectionDatabase_ = connectionDatabase.val; - - if (!connectByUrl.rawVal && originalURLTemplate.includes('@')) { - const [originalURLPrefix, originalURLSuffix] = originalURLTemplate.split('@'); - connectionStringPrefix.val = `${originalURLPrefix}@`; - connectionStringSuffix.val = formatURL(originalURLSuffix, connectionHost_, connectionPort_, connectionDatabase_); - } - }); - van.derive(() => { onChange({ project_host: connectionHost.val, @@ -376,17 +367,23 @@ const RedshiftForm = ( project_user: connectionUsername.val, project_pw_encrypted: connectionPassword.val, connect_by_url: connectByUrl.val, - url: connectByUrl.val ? connectionStringSuffix.val : connectionStringSuffix.rawVal, + url: connectByUrl.val ? connectionUrl.val : connectionUrl.rawVal, connect_by_key: false, }, isValid.val); }); + van.derive(() => { + const newUrlValue = (dynamicConnectionUrl.val ?? '').replace(extractPrefix(dynamicConnectionUrl.rawVal), ''); + if (!connectByUrl.rawVal) { + connectionUrl.val = newUrlValue; + } + }); + return div( {class: 'flex-column fx-gap-3 fx-flex'}, div( { class: 'flex-column border border-radius-1 p-3 mt-1 fx-gap-1', style: 'position: relative;' }, Caption({content: 'Server', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), - RadioGroup({ label: 'Connect by', options: [ @@ -448,12 +445,12 @@ const RedshiftForm = ( { class: 'flex-row fx-gap-3 fx-align-stretch', style: 'position: relative;' }, Input({ label: 'URL', - value: connectionStringSuffix, + value: connectionUrl, class: 'fx-flex', name: 'url_suffix', - prefix: span({ style: 'white-space: nowrap; color: var(--disabled-text-color)' }, connectionStringPrefix), + prefix: span({ style: 'white-space: nowrap; color: var(--disabled-text-color)' }, extractPrefix(dynamicConnectionUrl.val)), disabled: !connectByUrl.val, - onChange: (value, state) => connectionStringSuffix.val = value, + onChange: (value, state) => connectionUrl.val = value, }), ), ), @@ -504,6 +501,7 @@ const MSSQLForm = RedshiftForm; * @param {boolean} maskPassword * @param {(params: Partial, isValid: boolean) => void} onChange * @param {Connection?} originalConnection + * @param {VanState} dynamicConnectionUrl * @returns {HTMLElement} */ const DatabricksForm = ( @@ -511,9 +509,8 @@ const DatabricksForm = ( flavor, onChange, originalConnection, + dynamicConnectionUrl, ) => { - const originalURLTemplate = flavor.connection_string; - const isValid = van.state(true); const connectByUrl = van.state(connection.rawVal?.connect_by_url ?? false); const connectionHost = van.state(connection.rawVal?.project_host ?? ''); @@ -522,29 +519,10 @@ const DatabricksForm = ( const connectionDatabase = van.state(connection.rawVal?.project_db ?? ''); const connectionUsername = van.state(connection.rawVal?.project_user ?? ''); const connectionPassword = van.state(connection.rawVal?.project_pw_encrypted ?? ''); - - const [prefixPart, sufixPart] = originalURLTemplate.split('@'); - const connectionStringPrefix = van.state(`${prefixPart}@`); - const connectionStringSuffix = van.state(connection.rawVal?.url ?? ''); + const connectionUrl = van.state(connection.rawVal?.url ?? ''); const validityPerField = {}; - if (!connectionStringSuffix.rawVal) { - connectionStringSuffix.val = formatURL(sufixPart ?? '', connectionHost.rawVal, connectionPort.rawVal, connectionDatabase.rawVal, connectionHttpPath.rawVal); - } - - van.derive(() => { - const connectionHost_ = connectionHost.val; - const connectionPort_ = connectionPort.val; - const connectionDatabase_ = connectionDatabase.val; - const connectionHttpPath_ = connectionHttpPath.val; - - if (!connectByUrl.rawVal && originalURLTemplate.includes('@')) { - const [, originalURLSuffix] = originalURLTemplate.split('@'); - connectionStringSuffix.val = formatURL(originalURLSuffix, connectionHost_, connectionPort_, connectionDatabase_, connectionHttpPath_); - } - }); - van.derive(() => { onChange({ project_host: connectionHost.val, @@ -554,11 +532,18 @@ const DatabricksForm = ( project_pw_encrypted: connectionPassword.val, http_path: connectionHttpPath.val, connect_by_url: connectByUrl.val, - url: connectByUrl.val ? connectionStringSuffix.val : connectionStringSuffix.rawVal, + url: connectByUrl.val ? connectionUrl.val : connectionUrl.rawVal, connect_by_key: false, }, isValid.val); }); + van.derive(() => { + const newUrlValue = (dynamicConnectionUrl.val ?? '').replace(extractPrefix(dynamicConnectionUrl.rawVal), ''); + if (!connectByUrl.rawVal) { + connectionUrl.val = newUrlValue; + } + }); + return div( {class: 'flex-column fx-gap-3 fx-flex'}, div( @@ -639,12 +624,12 @@ const DatabricksForm = ( { class: 'flex-row fx-gap-3 fx-align-stretch', style: 'position: relative;' }, Input({ label: 'URL', - value: connectionStringSuffix, + value: connectionUrl, class: 'fx-flex', name: 'url_suffix', - prefix: span({ style: 'white-space: nowrap; color: var(--disabled-text-color)' }, connectionStringPrefix), + prefix: span({ style: 'white-space: nowrap; color: var(--disabled-text-color)' }, extractPrefix(dynamicConnectionUrl.val)), disabled: !connectByUrl.val, - onChange: (value, state) => connectionStringSuffix.val = value, + onChange: (value, state) => connectionUrl.val = value, }), ), ), @@ -687,7 +672,8 @@ const DatabricksForm = ( * @param {boolean} maskPassword * @param {(params: Partial, isValid: boolean) => void} onChange * @param {Connection?} originalConnection - * @param {string?} originalConnection + * @param {string?} cachedFile + * @param {VanState} dynamicConnectionUrl * @returns {HTMLElement} */ const SnowflakeForm = ( @@ -696,9 +682,8 @@ const SnowflakeForm = ( onChange, originalConnection, cachedFile, + dynamicConnectionUrl, ) => { - const originalURLTemplate = flavor.connection_string; - const isValid = van.state(false); const clearPrivateKeyPhrase = van.state(connection.rawVal?.private_key_passphrase === clearSentinel); const connectByUrl = van.state(connection.rawVal.connect_by_url ?? false); @@ -714,29 +699,12 @@ const SnowflakeForm = ( ? '' : (connection.rawVal?.private_key_passphrase ?? '') ); + const connectionUrl = van.state(connection.rawVal?.url ?? ''); + const validityPerField = {}; const privateKeyFileRaw = van.state(cachedFile); - const [prefixPart, sufixPart] = originalURLTemplate.split('@'); - const connectionStringPrefix = van.state(`${prefixPart}@`); - const connectionStringSuffix = van.state(connection.rawVal?.url ?? ''); - - if (!connectionStringSuffix.rawVal) { - connectionStringSuffix.val = formatURL(sufixPart ?? '', connectionHost.rawVal, connectionPort.rawVal, connectionDatabase.rawVal); - } - - van.derive(() => { - const connectionHost_ = connectionHost.val; - const connectionPort_ = connectionPort.val; - const connectionDatabase_ = connectionDatabase.val; - - if (!connectByUrl.rawVal && originalURLTemplate.includes('@')) { - const [, originalURLSuffix] = originalURLTemplate.split('@'); - connectionStringSuffix.val = formatURL(originalURLSuffix, connectionHost_, connectionPort_, connectionDatabase_); - } - }); - van.derive(() => { onChange({ project_host: connectionHost.val, @@ -745,13 +713,20 @@ const SnowflakeForm = ( project_user: connectionUsername.val, project_pw_encrypted: connectionPassword.val, connect_by_url: connectByUrl.val, - url: connectByUrl.val ? connectionStringSuffix.val : connectionStringSuffix.rawVal, + url: connectByUrl.val ? connectionUrl.val : connectionUrl.rawVal, connect_by_key: connectByKey.val, private_key: connectionPrivateKey.val, private_key_passphrase: clearPrivateKeyPhrase.val ? clearSentinel : connectionPrivateKeyPassphrase.val, }, privateKeyFileRaw.val, isValid.val); }); + van.derive(() => { + const newUrlValue = (dynamicConnectionUrl.val ?? '').replace(extractPrefix(dynamicConnectionUrl.rawVal), ''); + if (!connectByUrl.rawVal) { + connectionUrl.val = newUrlValue; + } + }); + return div( {class: 'flex-column fx-gap-3 fx-flex'}, div( @@ -819,13 +794,13 @@ const SnowflakeForm = ( { class: 'flex-row fx-gap-3 fx-align-stretch', style: 'position: relative;' }, Input({ label: 'URL', - value: connectionStringSuffix, + value: connectionUrl, class: 'fx-flex', name: 'url_suffix', - prefix: span({ style: 'white-space: nowrap; color: var(--disabled-text-color)' }, connectionStringPrefix), + prefix: span({ style: 'white-space: nowrap; color: var(--disabled-text-color)' }, extractPrefix(dynamicConnectionUrl.val)), disabled: !connectByUrl.val, onChange: (value, state) => { - connectionStringSuffix.val = value; + connectionUrl.val = value; validityPerField['url_suffix'] = state.valid; isValid.val = Object.values(validityPerField).every(v => v); }, @@ -938,11 +913,21 @@ const SnowflakeForm = ( ); }; -function formatURL(url, host, port, database, httpPath) { - return url.replace('', host) - .replace('', port) - .replace('', database) - .replace('', httpPath); +function extractPrefix(url) { + const parts = (url ?? '').split('@'); + if (!parts[0]) { + return ''; + } + return `${parts[0]}@`; +} + +function shouldRefreshUrl(previous, current) { + if (current.connect_by_url) { + return false; + } + + const fields = ['sql_flavor', 'project_host', 'project_port', 'project_db', 'project_user', 'connect_by_key', 'http_path']; + return fields.some((fieldName) => previous[fieldName] !== current[fieldName]); } const stylesheet = new CSSStyleSheet(); diff --git a/testgen/ui/components/frontend/js/pages/connections.js b/testgen/ui/components/frontend/js/pages/connections.js index 8da64475..959510dc 100644 --- a/testgen/ui/components/frontend/js/pages/connections.js +++ b/testgen/ui/components/frontend/js/pages/connections.js @@ -17,6 +17,7 @@ * @property {boolean} has_table_groups * @property {Array} flavors * @property {Permissions} permissions + * @property {string?} generated_connection_url * @property {Results?} results */ import van from '../van.min.js'; @@ -79,6 +80,7 @@ const Connections = (props) => { connection: props.connection, flavors: props.flavors, disableFlavor: false, + dynamicConnectionUrl: props.generated_connection_url, onChange: (connection, state) => { formState.val = state; updatedConnection.val = connection; diff --git a/testgen/ui/views/connections.py b/testgen/ui/views/connections.py index 6ece4ce7..bb821d04 100644 --- a/testgen/ui/views/connections.py +++ b/testgen/ui/views/connections.py @@ -77,6 +77,9 @@ def render(self, project_code: str, **_kwargs) -> None: default=False, ) + def on_connection_updated(connection: dict) -> None: + set_updated_connection(self._sanitize_connection_input(connection)) + def on_save_connection_clicked(updated_connection): is_pristine = lambda value: value in ["", "***"] @@ -140,6 +143,10 @@ def on_test_connection_clicked(updated_connection: dict) -> None: for key, value in get_updated_connection().items(): setattr(connection, key, value) + flavor_service = get_flavor_service(connection.sql_flavor) + flavor_service.init({**connection.to_dict(), "project_pw_encrypted": ""}) + connection_string = flavor_service.get_connection_string().replace("%3E", ">").replace("%3C", "<") + if should_save(): success = True try: @@ -165,12 +172,14 @@ def on_test_connection_clicked(updated_connection: dict) -> None: "permissions": { "is_admin": user_is_admin, }, + "generated_connection_url": connection_string, "results": results, }, on_change_handlers={ "TestConnectionClicked": on_test_connection_clicked, "SaveConnectionClicked": on_save_connection_clicked, "SetupTableGroupClicked": lambda _: self.setup_data_configuration(project_code, connection.connection_id), + "ConnectionUpdated": on_connection_updated, }, ) From 1702008062dc053a9700219e44e68a83147f8aa2 Mon Sep 17 00:00:00 2001 From: Luis Date: Tue, 2 Sep 2025 16:40:57 -0400 Subject: [PATCH 08/28] refactor: validate required fields for connections --- .../frontend/js/components/connection_form.js | 80 +++++++++++++++---- .../frontend/js/components/file_input.js | 14 +++- .../frontend/js/components/input.js | 12 +-- .../components/frontend/js/form_validators.js | 17 ++++ testgen/ui/components/frontend/js/utils.js | 12 ++- 5 files changed, 111 insertions(+), 24 deletions(-) diff --git a/testgen/ui/components/frontend/js/components/connection_form.js b/testgen/ui/components/frontend/js/components/connection_form.js index 23d0ad75..c4966efb 100644 --- a/testgen/ui/components/frontend/js/components/connection_form.js +++ b/testgen/ui/components/frontend/js/components/connection_form.js @@ -60,7 +60,7 @@ import { getValue, emitEvent, loadStylesheet, isEqual } from '../utils.js'; import { Input } from './input.js'; import { Slider } from './slider.js'; import { Select } from './select.js'; -import { maxLength, minLength, required, sizeLimit } from '../form_validators.js'; +import { maxLength, minLength, required, requiredIf, sizeLimit } from '../form_validators.js'; import { RadioGroup } from './radio_group.js'; import { FileInput } from './file_input.js'; import { ExpansionPanel } from './expansion_panel.js'; @@ -413,7 +413,10 @@ const RedshiftForm = ( validityPerField['db_host'] = state.valid; isValid.val = Object.values(validityPerField).every(v => v); }, - validators: [ maxLength(250) ], + validators: [ + maxLength(250), + requiredIf(() => !connectByUrl.val), + ], }), Input({ name: 'db_port', @@ -426,7 +429,11 @@ const RedshiftForm = ( validityPerField['db_port'] = state.valid; isValid.val = Object.values(validityPerField).every(v => v); }, - validators: [ minLength(3), maxLength(5) ], + validators: [ + minLength(3), + maxLength(5), + requiredIf(() => !connectByUrl.val), + ], }) ), Input({ @@ -439,7 +446,10 @@ const RedshiftForm = ( validityPerField['db_name'] = state.valid; isValid.val = Object.values(validityPerField).every(v => v); }, - validators: [ maxLength(100) ], + validators: [ + maxLength(100), + requiredIf(() => !connectByUrl.val), + ], }), () => div( { class: 'flex-row fx-gap-3 fx-align-stretch', style: 'position: relative;' }, @@ -451,6 +461,9 @@ const RedshiftForm = ( prefix: span({ style: 'white-space: nowrap; color: var(--disabled-text-color)' }, extractPrefix(dynamicConnectionUrl.val)), disabled: !connectByUrl.val, onChange: (value, state) => connectionUrl.val = value, + validators: [ + requiredIf(() => connectByUrl.val), + ], }), ), ), @@ -468,7 +481,10 @@ const RedshiftForm = ( validityPerField['db_user'] = state.valid; isValid.val = Object.values(validityPerField).every(v => v); }, - validators: [ maxLength(50) ], + validators: [ + required, + maxLength(50), + ], }), Input({ name: 'password', @@ -579,7 +595,10 @@ const DatabricksForm = ( validityPerField['db_host'] = state.valid; isValid.val = Object.values(validityPerField).every(v => v); }, - validators: [ maxLength(250) ], + validators: [ + requiredIf(() => !connectByUrl.val), + maxLength(250), + ], }), Input({ name: 'db_port', @@ -592,7 +611,11 @@ const DatabricksForm = ( validityPerField['db_port'] = state.valid; isValid.val = Object.values(validityPerField).every(v => v); }, - validators: [ minLength(3), maxLength(5) ], + validators: [ + requiredIf(() => !connectByUrl.val), + minLength(3), + maxLength(5), + ], }) ), Input({ @@ -606,7 +629,10 @@ const DatabricksForm = ( validityPerField['http_path'] = state.valid; isValid.val = Object.values(validityPerField).every(v => v); }, - validators: [ maxLength(50) ], + validators: [ + requiredIf(() => !connectByUrl.val), + maxLength(50), + ], }), Input({ name: 'db_name', @@ -618,7 +644,10 @@ const DatabricksForm = ( validityPerField['db_name'] = state.valid; isValid.val = Object.values(validityPerField).every(v => v); }, - validators: [ maxLength(100) ], + validators: [ + requiredIf(() => !connectByUrl.val), + maxLength(100), + ], }), () => div( { class: 'flex-row fx-gap-3 fx-align-stretch', style: 'position: relative;' }, @@ -630,6 +659,9 @@ const DatabricksForm = ( prefix: span({ style: 'white-space: nowrap; color: var(--disabled-text-color)' }, extractPrefix(dynamicConnectionUrl.val)), disabled: !connectByUrl.val, onChange: (value, state) => connectionUrl.val = value, + validators: [ + requiredIf(() => connectByUrl.val), + ], }), ), ), @@ -647,7 +679,10 @@ const DatabricksForm = ( validityPerField['db_user'] = state.valid; isValid.val = Object.values(validityPerField).every(v => v); }, - validators: [ maxLength(50) ], + validators: [ + required, + maxLength(50), + ], }), Input({ name: 'password', @@ -762,7 +797,10 @@ const SnowflakeForm = ( validityPerField['db_host'] = state.valid; isValid.val = Object.values(validityPerField).every(v => v); }, - validators: [ maxLength(250) ], + validators: [ + requiredIf(() => !connectByUrl.val), + maxLength(250), + ], }), Input({ name: 'db_port', @@ -775,7 +813,11 @@ const SnowflakeForm = ( validityPerField['db_port'] = state.valid; isValid.val = Object.values(validityPerField).every(v => v); }, - validators: [ minLength(3), maxLength(5) ], + validators: [ + requiredIf(() => !connectByUrl.val), + minLength(3), + maxLength(5), + ], }) ), Input({ @@ -788,7 +830,10 @@ const SnowflakeForm = ( validityPerField['db_name'] = state.valid; isValid.val = Object.values(validityPerField).every(v => v); }, - validators: [ maxLength(100) ], + validators: [ + requiredIf(() => !connectByUrl.val), + maxLength(100), + ], }), () => div( { class: 'flex-row fx-gap-3 fx-align-stretch', style: 'position: relative;' }, @@ -804,6 +849,9 @@ const SnowflakeForm = ( validityPerField['url_suffix'] = state.valid; isValid.val = Object.values(validityPerField).every(v => v); }, + validators: [ + requiredIf(() => connectByUrl.val), + ], }), ), ), @@ -832,7 +880,10 @@ const SnowflakeForm = ( validityPerField['db_user'] = state.valid; isValid.val = Object.values(validityPerField).every(v => v); }, - validators: [ maxLength(50) ], + validators: [ + required, + maxLength(50), + ], }), () => { if (connectByKey.val) { @@ -889,6 +940,7 @@ const SnowflakeForm = ( isValid.val = Object.values(validityPerField).every(v => v); }, validators: [ + required, sizeLimit(200 * 1024 * 1024), ], }), diff --git a/testgen/ui/components/frontend/js/components/file_input.js b/testgen/ui/components/frontend/js/components/file_input.js index fb869f64..8bf247e3 100644 --- a/testgen/ui/components/frontend/js/components/file_input.js +++ b/testgen/ui/components/frontend/js/components/file_input.js @@ -20,7 +20,7 @@ * */ import van from '../van.min.js'; -import { getRandomId, getValue, loadStylesheet } from "../utils.js"; +import { checkIsRequired, getRandomId, getValue, loadStylesheet } from "../utils.js"; import { Icon } from './icon.js'; import { Button } from './button.js'; import { humanReadableSize } from '../display_utils.js'; @@ -48,9 +48,14 @@ const FileInput = (options) => { const validators = getValue(options.validators) ?? []; return validators.map(v => v(value.val)).filter(error => error); }); + const isRequired = van.state(false); + + van.derive(() => { + isRequired.val = checkIsRequired(getValue(options.validators) ?? []); + }); let sizeLimit = undefined; - let sizeLimitValidator = (getValue(options.validators) ?? []).filter(v => v.args.name === 'sizeLimit')[0]; + let sizeLimitValidator = (getValue(options.validators) ?? []).filter(v => v.args?.name === 'sizeLimit')[0]; if (sizeLimitValidator) { sizeLimit = sizeLimitValidator.args.limit; } @@ -106,8 +111,11 @@ const FileInput = (options) => { return div( { class: cssClass }, label( - { class: 'tg-file-uploader--label' }, + { class: 'tg-file-uploader--label text-caption flex-row fx-gap-1' }, options.label, + () => isRequired.val + ? span({ class: 'text-error' }, '*') + : '', ), div( { class: () => `tg-file-uploader--dropzone flex-column clickable ${fileOver.val ? 'on-dragover' : ''}` }, diff --git a/testgen/ui/components/frontend/js/components/input.js b/testgen/ui/components/frontend/js/components/input.js index c0324cad..5b77975f 100644 --- a/testgen/ui/components/frontend/js/components/input.js +++ b/testgen/ui/components/frontend/js/components/input.js @@ -35,7 +35,7 @@ * @property {Array?} validators */ import van from '../van.min.js'; -import { debounce, getValue, loadStylesheet, getRandomId } from '../utils.js'; +import { debounce, getValue, loadStylesheet, getRandomId, checkIsRequired } from '../utils.js'; import { Icon } from './icon.js'; import { withTooltip } from './tooltip.js'; import { Portal } from './portal.js'; @@ -55,10 +55,6 @@ const Input = (/** @type Properties */ props) => { const domId = van.derive(() => getValue(props.id) ?? getRandomId()); const value = van.derive(() => getValue(props.value) ?? ''); - const isRequired = van.derive(() => { - const validators = getValue(props.validators) ?? []; - return validators.some(v => v.name === 'required'); - }); const errors = van.derive(() => { const validators = getValue(props.validators) ?? []; return validators.map(v => v(value.val)).filter(error => error); @@ -66,10 +62,10 @@ const Input = (/** @type Properties */ props) => { const firstError = van.derive(() => { return errors.val[0] ?? ''; }); - const originalInputType = van.derive(() => getValue(props.type) ?? 'text'); const inputType = van.state(originalInputType.rawVal); + const isRequired = van.state(false); const isDirty = van.state(false); const onChange = props.onChange?.val ?? props.onChange; if (onChange) { @@ -82,6 +78,10 @@ const Input = (/** @type Properties */ props) => { } }); + van.derive(() => { + isRequired.val = checkIsRequired(getValue(props.validators) ?? []); + }); + const onClear = props.onClear?.val ?? props.onClear ?? (() => value.val = ''); const autocompleteOpened = van.state(false); diff --git a/testgen/ui/components/frontend/js/form_validators.js b/testgen/ui/components/frontend/js/form_validators.js index d57a2cc4..0d9d78ad 100644 --- a/testgen/ui/components/frontend/js/form_validators.js +++ b/testgen/ui/components/frontend/js/form_validators.js @@ -13,6 +13,22 @@ function required(value) { return null; } +/** + * @param {(v: any) => bool} condition + * @returns {Validator} + */ +function requiredIf(condition) { + const validator = (value) => { + if (condition(value)) { + return required(value); + } + return null; + } + validator['args'] = { name: 'requiredIf', condition }; + + return validator; +} + function noSpaces(value) { if (value?.includes(' ')) { return `Value cannot contain spaces.`; @@ -76,5 +92,6 @@ export { minLength, noSpaces, required, + requiredIf, sizeLimit, }; diff --git a/testgen/ui/components/frontend/js/utils.js b/testgen/ui/components/frontend/js/utils.js index 1e8b4f7f..3a81eb17 100644 --- a/testgen/ui/components/frontend/js/utils.js +++ b/testgen/ui/components/frontend/js/utils.js @@ -197,4 +197,14 @@ function isDataURL(/** @type string */ url) { return url.startsWith('data:'); } -export { afterMount, debounce, emitEvent, enforceElementWidth, getRandomId, getValue, getParents, isEqual, isState, loadStylesheet, resizeFrameHeightToElement, resizeFrameHeightOnDOMChange, friendlyPercent, slugify, isDataURL }; +function checkIsRequired(validators) { + let isRequired = validators.some(v => v.name === 'required'); + if (!isRequired) { + isRequired = validators + .filter((v) => v.args?.name === 'requiredIf') + .some((v) => v.args?.condition?.()) + } + return isRequired; +} + +export { afterMount, debounce, emitEvent, enforceElementWidth, getRandomId, getValue, getParents, isEqual, isState, loadStylesheet, resizeFrameHeightToElement, resizeFrameHeightOnDOMChange, friendlyPercent, slugify, isDataURL, checkIsRequired }; From 68b0dc299f64389dc28eb41edf936e90bd865a59 Mon Sep 17 00:00:00 2001 From: Luis Date: Tue, 2 Sep 2025 17:55:13 -0400 Subject: [PATCH 09/28] feat(connections): add warehouse field to snowflake form --- .../common/database/flavor/flavor_service.py | 2 ++ .../flavor/snowflake_flavor_service.py | 6 +++++ testgen/common/models/connection.py | 2 ++ .../030_initialize_new_schema_structure.sql | 3 ++- .../dbupgrade/0149_incremental_upgrade.sql | 4 ++++ .../frontend/js/components/connection_form.js | 22 +++++++++++++++-- testgen/ui/views/connections.py | 24 ------------------- 7 files changed, 36 insertions(+), 27 deletions(-) create mode 100644 testgen/template/dbupgrade/0149_incremental_upgrade.sql diff --git a/testgen/common/database/flavor/flavor_service.py b/testgen/common/database/flavor/flavor_service.py index 23f2a013..ece9efc0 100644 --- a/testgen/common/database/flavor/flavor_service.py +++ b/testgen/common/database/flavor/flavor_service.py @@ -38,6 +38,7 @@ class FlavorService: private_key_passphrase = None http_path = None catalog = None + warehouse = None def init(self, connection_params: ConnectionParams): self.url = connection_params.get("url", None) @@ -51,6 +52,7 @@ def init(self, connection_params: ConnectionParams): self.connect_by_key = connection_params.get("connect_by_key", False) self.http_path = connection_params.get("http_path", None) self.catalog = connection_params.get("catalog", None) + self.warehouse = connection_params.get("warehouse", None) password = connection_params.get("project_pw_encrypted", None) if isinstance(password, memoryview) or isinstance(password, bytes): diff --git a/testgen/common/database/flavor/snowflake_flavor_service.py b/testgen/common/database/flavor/snowflake_flavor_service.py index 7662f832..07f620f4 100644 --- a/testgen/common/database/flavor/snowflake_flavor_service.py +++ b/testgen/common/database/flavor/snowflake_flavor_service.py @@ -40,6 +40,11 @@ def get_connection_string_from_fields(self): # NOTE: Snowflake host should NOT include ".snowflakecomputing.com" account, _ = self.host.split(".", maxsplit=1) if "." in self.host else ("", "") + + extra_params = {} + if self.warehouse: + extra_params["warehouse"] = self.warehouse + connection_url = URL( host=self.host, port=int(self.port if str(self.port).isdigit() else 443), @@ -48,6 +53,7 @@ def get_connection_string_from_fields(self): password="" if self.connect_by_key else self.password, database=self.dbname, schema=self.dbschema or "", + **extra_params, ) return connection_url diff --git a/testgen/common/models/connection.py b/testgen/common/models/connection.py index 6a7142dc..660f51fd 100644 --- a/testgen/common/models/connection.py +++ b/testgen/common/models/connection.py @@ -60,6 +60,7 @@ class Connection(Entity): private_key: str = Column(EncryptedBytea) private_key_passphrase: str = Column(EncryptedBytea) http_path: str = Column(String) + warehouse: str = Column(String) _get_by = "connection_id" _default_order_by = (asc(func.lower(connection_name)),) @@ -125,5 +126,6 @@ def save(self) -> None: self.project_port = connection_parts["port"] self.project_db = connection_parts["dbname"] self.http_path = connection_parts.get("http_path") or None + self.warehouse = connection_parts.get("warehouse") or None super().save() diff --git a/testgen/template/dbsetup/030_initialize_new_schema_structure.sql b/testgen/template/dbsetup/030_initialize_new_schema_structure.sql index 469e2776..d2b4b022 100644 --- a/testgen/template/dbsetup/030_initialize_new_schema_structure.sql +++ b/testgen/template/dbsetup/030_initialize_new_schema_structure.sql @@ -77,7 +77,8 @@ CREATE TABLE connections ( connect_by_key BOOLEAN DEFAULT FALSE, private_key BYTEA, private_key_passphrase BYTEA, - http_path VARCHAR(200) + http_path VARCHAR(200), + warehouse VARCHAR(200) ); CREATE TABLE table_groups diff --git a/testgen/template/dbupgrade/0149_incremental_upgrade.sql b/testgen/template/dbupgrade/0149_incremental_upgrade.sql new file mode 100644 index 00000000..8b30ebbc --- /dev/null +++ b/testgen/template/dbupgrade/0149_incremental_upgrade.sql @@ -0,0 +1,4 @@ +SET SEARCH_PATH TO {SCHEMA_NAME}; + +ALTER TABLE connections + ADD COLUMN warehouse VARCHAR(200); diff --git a/testgen/ui/components/frontend/js/components/connection_form.js b/testgen/ui/components/frontend/js/components/connection_form.js index c4966efb..711322e8 100644 --- a/testgen/ui/components/frontend/js/components/connection_form.js +++ b/testgen/ui/components/frontend/js/components/connection_form.js @@ -33,6 +33,7 @@ * @property {string?} private_key * @property {string?} private_key_passphrase * @property {string?} http_path + * @property {string?} warehouse * @property {ConnectionStatus?} status * * @typedef FormState @@ -66,7 +67,7 @@ import { FileInput } from './file_input.js'; import { ExpansionPanel } from './expansion_panel.js'; import { Caption } from './caption.js'; -const { div, i, span } = van.tags; +const { div, span } = van.tags; const clearSentinel = ''; const secretsPlaceholder = ''; const defaultPorts = { @@ -112,6 +113,7 @@ const ConnectionForm = (props, saveButton) => { private_key: isEditMode ? '' : (connection?.private_key ?? ''), private_key_passphrase: isEditMode ? '' : (connection?.private_key_passphrase ?? ''), http_path: connection?.http_path ?? '', + warehouse: connection?.warehouse ?? '', url: connection?.url ?? '', sql_flavor_code: connectionFlavor.rawVal ?? '', connection_name: connectionName.rawVal ?? '', @@ -726,6 +728,7 @@ const SnowflakeForm = ( const connectionHost = van.state(connection.rawVal.project_host ?? ''); const connectionPort = van.state(connection.rawVal.project_port || defaultPorts[flavor.flavor]); const connectionDatabase = van.state(connection.rawVal.project_db ?? ''); + const connectionWarehouse = van.state(connection.rawVal.warehouse ?? ''); const connectionUsername = van.state(connection.rawVal.project_user ?? ''); const connectionPassword = van.state(connection.rawVal?.project_pw_encrypted ?? ''); const connectionPrivateKey = van.state(connection.rawVal?.private_key ?? ''); @@ -752,6 +755,7 @@ const SnowflakeForm = ( connect_by_key: connectByKey.val, private_key: connectionPrivateKey.val, private_key_passphrase: clearPrivateKeyPhrase.val ? clearSentinel : connectionPrivateKeyPassphrase.val, + warehouse: connectionWarehouse.val, }, privateKeyFileRaw.val, isValid.val); }); @@ -835,6 +839,20 @@ const SnowflakeForm = ( maxLength(100), ], }), + Input({ + name: 'warehouse', + label: 'Warehouse', + value: connectionWarehouse, + disabled: connectByUrl, + onChange: (value, state) => { + connectionWarehouse.val = value; + validityPerField['warehouse'] = state.valid; + isValid.val = Object.values(validityPerField).every(v => v); + }, + validators: [ + maxLength(100), + ], + }), () => div( { class: 'flex-row fx-gap-3 fx-align-stretch', style: 'position: relative;' }, Input({ @@ -978,7 +996,7 @@ function shouldRefreshUrl(previous, current) { return false; } - const fields = ['sql_flavor', 'project_host', 'project_port', 'project_db', 'project_user', 'connect_by_key', 'http_path']; + const fields = ['sql_flavor', 'project_host', 'project_port', 'project_db', 'project_user', 'connect_by_key', 'http_path', 'warehouse']; return fields.some((fieldName) => previous[fieldName] !== current[fieldName]); } diff --git a/testgen/ui/views/connections.py b/testgen/ui/views/connections.py index bb821d04..f67e8b3d 100644 --- a/testgen/ui/views/connections.py +++ b/testgen/ui/views/connections.py @@ -405,29 +405,12 @@ def format_connection(connection: Connection | ConnectionMinimal) -> dict: return formatted_connection -def get_connection_string(flavor: str) -> str: - connection_params: ConnectionParams = { - "sql_flavor": flavor, - "project_host": "", - "project_port": "", - "project_user": "", - "project_db": "", - "project_pw_encrypted": "", - "http_path": "", - "table_group_schema": "", - } - flavor_service = get_flavor_service(flavor) - flavor_service.init(connection_params) - return flavor_service.get_connection_string().replace("%3E", ">").replace("%3C", "<") - - @dataclass(frozen=True, slots=True, kw_only=True) class ConnectionFlavor: value: str label: str icon: str flavor: str - connection_string: str FLAVOR_OPTIONS = [ @@ -436,48 +419,41 @@ class ConnectionFlavor: value="redshift", flavor="redshift", icon=get_asset_data_url("flavors/redshift.svg"), - connection_string=get_connection_string("redshift"), ), ConnectionFlavor( label="Azure SQL Database", value="azure_mssql", flavor="mssql", icon=get_asset_data_url("flavors/azure_sql.svg"), - connection_string=get_connection_string("mssql"), ), ConnectionFlavor( label="Azure Synapse Analytics", value="synapse_mssql", flavor="mssql", icon=get_asset_data_url("flavors/azure_synapse_table.svg"), - connection_string=get_connection_string("mssql"), ), ConnectionFlavor( label="Microsoft SQL Server", value="mssql", flavor="mssql", icon=get_asset_data_url("flavors/mssql.svg"), - connection_string=get_connection_string("mssql"), ), ConnectionFlavor( label="PostgreSQL", value="postgresql", flavor="postgresql", icon=get_asset_data_url("flavors/postgresql.svg"), - connection_string=get_connection_string("postgresql"), ), ConnectionFlavor( label="Snowflake", value="snowflake", flavor="snowflake", icon=get_asset_data_url("flavors/snowflake.svg"), - connection_string=get_connection_string("snowflake"), ), ConnectionFlavor( label="Databricks", value="databricks", flavor="databricks", icon=get_asset_data_url("flavors/databricks.svg"), - connection_string=get_connection_string("databricks"), ), ] From 86610838564af56833274236c81f94fa1670e3d8 Mon Sep 17 00:00:00 2001 From: Luis Date: Tue, 2 Sep 2025 18:22:29 -0400 Subject: [PATCH 10/28] refactor(connections): stop creating an initial connection --- testgen/commands/run_launch_db_config.py | 6 ++- testgen/commands/run_quick_start.py | 10 ++++- .../040_populate_new_schema_project.sql | 42 ------------------ .../quick_start/initial_data_seeding.sql | 43 +++++++++++++++++++ testgen/ui/views/connections.py | 6 ++- 5 files changed, 61 insertions(+), 46 deletions(-) create mode 100644 testgen/template/quick_start/initial_data_seeding.sql diff --git a/testgen/commands/run_launch_db_config.py b/testgen/commands/run_launch_db_config.py index 01daa060..71e3c650 100644 --- a/testgen/commands/run_launch_db_config.py +++ b/testgen/commands/run_launch_db_config.py @@ -91,4 +91,8 @@ def run_launch_db_config(delete_db: bool) -> None: project_code=settings.PROJECT_KEY, table_groups_name=settings.DEFAULT_TABLE_GROUPS_NAME, ) - ).save() \ No newline at end of file + ).save() + + +def get_app_db_params_mapping() -> dict: + return _get_params_mapping() diff --git a/testgen/commands/run_quick_start.py b/testgen/commands/run_quick_start.py index 659109a0..7ae59036 100644 --- a/testgen/commands/run_quick_start.py +++ b/testgen/commands/run_quick_start.py @@ -3,7 +3,7 @@ import click from testgen import settings -from testgen.commands.run_launch_db_config import run_launch_db_config +from testgen.commands.run_launch_db_config import get_app_db_params_mapping, run_launch_db_config from testgen.common.credentials import get_tg_schema from testgen.common.database.database_service import ( create_database, @@ -117,6 +117,14 @@ def run_quick_start(delete_target_db: bool) -> None: delete_db = True run_launch_db_config(delete_db) + click.echo(f"Seeding the application db") + app_db_params = get_app_db_params_mapping() + execute_db_queries( + [ + (replace_params(read_template_sql_file("initial_data_seeding.sql", "quick_start"), app_db_params), app_db_params), + ], + ) + # Schema and Populate target db click.echo(f"Populating target db : {target_db_name}") execute_db_queries( diff --git a/testgen/template/dbsetup/040_populate_new_schema_project.sql b/testgen/template/dbsetup/040_populate_new_schema_project.sql index ae4eddf8..c6c959f5 100644 --- a/testgen/template/dbsetup/040_populate_new_schema_project.sql +++ b/testgen/template/dbsetup/040_populate_new_schema_project.sql @@ -7,48 +7,6 @@ SELECT '{PROJECT_CODE}' as project_code, '{OBSERVABILITY_API_KEY}' as observability_api_key, '{OBSERVABILITY_API_URL}' as observability_api_url; -INSERT INTO connections -(project_code, sql_flavor, sql_flavor_code, - project_host, project_port, project_user, project_db, - connection_name, project_pw_encrypted, http_path, max_threads, max_query_chars) -SELECT '{PROJECT_CODE}' as project_code, - '{SQL_FLAVOR}' as sql_flavor, - '{SQL_FLAVOR}' as sql_flavor_code, - NULLIF('{PROJECT_HOST}', '') as project_host, - NULLIF('{PROJECT_PORT}', '') as project_port, - NULLIF('{PROJECT_USER}', '') as project_user, - NULLIF('{PROJECT_DB}', '') as project_db, - '{CONNECTION_NAME}' as connection_name, - NULLIF('{PROJECT_PW_ENCRYPTED}', ''::BYTEA) as project_pw_encrypted, - NULLIF('{PROJECT_HTTP_PATH}', '') as http_path, - '{MAX_THREADS}'::INTEGER as max_threads, - '{MAX_QUERY_CHARS}'::INTEGER as max_query_chars; - -INSERT INTO table_groups -(id, project_code, connection_id, table_groups_name, table_group_schema, profiling_table_set, profiling_include_mask, profiling_exclude_mask, - profile_sample_min_count) -SELECT '0ea85e17-acbe-47fe-8394-9970725ad37d'::UUID as id, - '{PROJECT_CODE}' as project_code, - 1 as connection_id, - '{TABLE_GROUPS_NAME}' as table_groups_name, - '{PROJECT_SCHEMA}' as table_group_schema, - NULLIF('{PROFILING_TABLE_SET}', '') as profiling_table_set, - NULLIF('{PROFILING_INCLUDE_MASK}', '') as profiling_include_mask, - NULLIF('{PROFILING_EXCLUDE_MASK}', '') as profiling_exclude_mask, - 15000 as profile_sample_min_count; - -INSERT INTO test_suites - (project_code, test_suite, connection_id, table_groups_id, test_suite_description, - export_to_observability, component_key, component_type) -SELECT '{PROJECT_CODE}' as project_code, - '{TEST_SUITE}' as test_suite, - 1 as connection_id, - '0ea85e17-acbe-47fe-8394-9970725ad37d'::UUID as table_groups_id, - '{TEST_SUITE} Test Suite' as test_suite_description, - 'Y' as export_to_observability, - NULL as component_key, - '{OBSERVABILITY_COMPONENT_TYPE}' as component_type; - INSERT INTO auth_users (username, email, name, password, role) SELECT diff --git a/testgen/template/quick_start/initial_data_seeding.sql b/testgen/template/quick_start/initial_data_seeding.sql new file mode 100644 index 00000000..6d9e76ed --- /dev/null +++ b/testgen/template/quick_start/initial_data_seeding.sql @@ -0,0 +1,43 @@ +SET SEARCH_PATH TO {SCHEMA_NAME}; + +INSERT INTO connections +(project_code, sql_flavor, sql_flavor_code, + project_host, project_port, project_user, project_db, + connection_name, project_pw_encrypted, http_path, max_threads, max_query_chars) +SELECT '{PROJECT_CODE}' as project_code, + '{SQL_FLAVOR}' as sql_flavor, + '{SQL_FLAVOR}' as sql_flavor_code, + NULLIF('{PROJECT_HOST}', '') as project_host, + NULLIF('{PROJECT_PORT}', '') as project_port, + NULLIF('{PROJECT_USER}', '') as project_user, + NULLIF('{PROJECT_DB}', '') as project_db, + '{CONNECTION_NAME}' as connection_name, + NULLIF('{PROJECT_PW_ENCRYPTED}', ''::BYTEA) as project_pw_encrypted, + NULLIF('{PROJECT_HTTP_PATH}', '') as http_path, + '{MAX_THREADS}'::INTEGER as max_threads, + '{MAX_QUERY_CHARS}'::INTEGER as max_query_chars; + +INSERT INTO table_groups +(id, project_code, connection_id, table_groups_name, table_group_schema, profiling_table_set, profiling_include_mask, profiling_exclude_mask, + profile_sample_min_count) +SELECT '0ea85e17-acbe-47fe-8394-9970725ad37d'::UUID as id, + '{PROJECT_CODE}' as project_code, + 1 as connection_id, + '{TABLE_GROUPS_NAME}' as table_groups_name, + '{PROJECT_SCHEMA}' as table_group_schema, + NULLIF('{PROFILING_TABLE_SET}', '') as profiling_table_set, + NULLIF('{PROFILING_INCLUDE_MASK}', '') as profiling_include_mask, + NULLIF('{PROFILING_EXCLUDE_MASK}', '') as profiling_exclude_mask, + 15000 as profile_sample_min_count; + +INSERT INTO test_suites + (project_code, test_suite, connection_id, table_groups_id, test_suite_description, + export_to_observability, component_key, component_type) +SELECT '{PROJECT_CODE}' as project_code, + '{TEST_SUITE}' as test_suite, + 1 as connection_id, + '0ea85e17-acbe-47fe-8394-9970725ad37d'::UUID as table_groups_id, + '{TEST_SUITE} Test Suite' as test_suite_description, + 'Y' as export_to_observability, + NULL as component_key, + '{OBSERVABILITY_COMPONENT_TYPE}' as component_type; diff --git a/testgen/ui/views/connections.py b/testgen/ui/views/connections.py index f67e8b3d..704c2783 100644 --- a/testgen/ui/views/connections.py +++ b/testgen/ui/views/connections.py @@ -59,10 +59,11 @@ def render(self, project_code: str, **_kwargs) -> None: ) connections = Connection.select_where(Connection.project_code == project_code) - connection: Connection = connections[0] + connection: Connection = connections[0] if len(connections) > 0 else Connection(sql_flavor="postgresql", sql_flavor_code="postgresql") has_table_groups = ( - len(TableGroup.select_minimal_where(TableGroup.connection_id == connection.connection_id) or []) > 0 + connection.id and len(TableGroup.select_minimal_where(TableGroup.connection_id == connection.connection_id) or []) > 0 ) + user_is_admin = session.auth.user_has_permission("administer") should_check_status, set_check_status = temp_value( "connections:status_check", @@ -143,6 +144,7 @@ def on_test_connection_clicked(updated_connection: dict) -> None: for key, value in get_updated_connection().items(): setattr(connection, key, value) + connection_string: str | None = None flavor_service = get_flavor_service(connection.sql_flavor) flavor_service.init({**connection.to_dict(), "project_pw_encrypted": ""}) connection_string = flavor_service.get_connection_string().replace("%3E", ">").replace("%3C", "<") From ce5122c1b1027596c2549e730c0277e394393a8c Mon Sep 17 00:00:00 2001 From: Luis Date: Wed, 3 Sep 2025 12:47:33 -0400 Subject: [PATCH 11/28] fix(connections): enforce required url and private key --- .../common/database/flavor/flavor_service.py | 16 +++++----- .../frontend/js/components/alert.js | 3 +- .../frontend/js/components/connection_form.js | 32 ++++++++++++++----- .../frontend/js/components/file_input.js | 4 ++- testgen/ui/views/connections.py | 31 +++++++++++++----- 5 files changed, 60 insertions(+), 26 deletions(-) diff --git a/testgen/common/database/flavor/flavor_service.py b/testgen/common/database/flavor/flavor_service.py index ece9efc0..a257b2da 100644 --- a/testgen/common/database/flavor/flavor_service.py +++ b/testgen/common/database/flavor/flavor_service.py @@ -41,18 +41,18 @@ class FlavorService: warehouse = None def init(self, connection_params: ConnectionParams): - self.url = connection_params.get("url", None) + self.url = connection_params.get("url") or "" self.connect_by_url = connection_params.get("connect_by_url", False) - self.username = connection_params.get("project_user") - self.host = connection_params.get("project_host") - self.port = connection_params.get("project_port") - self.dbname = connection_params.get("project_db") + self.username = connection_params.get("project_user") or "" + self.host = connection_params.get("project_host") or "" + self.port = connection_params.get("project_port") or "" + self.dbname = connection_params.get("project_db") or "" self.flavor = connection_params.get("sql_flavor") self.dbschema = connection_params.get("table_group_schema", None) self.connect_by_key = connection_params.get("connect_by_key", False) - self.http_path = connection_params.get("http_path", None) - self.catalog = connection_params.get("catalog", None) - self.warehouse = connection_params.get("warehouse", None) + self.http_path = connection_params.get("http_path") or "" + self.catalog = connection_params.get("catalog") or "" + self.warehouse = connection_params.get("warehouse") or "" password = connection_params.get("project_pw_encrypted", None) if isinstance(password, memoryview) or isinstance(password, bytes): diff --git a/testgen/ui/components/frontend/js/components/alert.js b/testgen/ui/components/frontend/js/components/alert.js index d6d9c716..dfb28edd 100644 --- a/testgen/ui/components/frontend/js/components/alert.js +++ b/testgen/ui/components/frontend/js/components/alert.js @@ -6,6 +6,7 @@ * @property {boolean?} closeable * @property {string?} class * @property {'info'|'success'|'warn'|'error'} type + * @property {Function?} onClose */ import van from '../van.min.js'; import { getValue, loadStylesheet, getRandomId } from '../utils.js'; @@ -19,7 +20,7 @@ const Alert = (/** @type Properties */ props, /** @type Array */ .. const elementId = getValue(props.id) ?? 'tg-alert-' + getRandomId(); const close = () => { - document.getElementById(elementId)?.remove(); + props.onClose ? props.onClose() : document.getElementById(elementId)?.remove(); }; const timeout = getValue(props.timeout); if (timeout && timeout > 0) { diff --git a/testgen/ui/components/frontend/js/components/connection_form.js b/testgen/ui/components/frontend/js/components/connection_form.js index 711322e8..e16b29a1 100644 --- a/testgen/ui/components/frontend/js/components/connection_form.js +++ b/testgen/ui/components/frontend/js/components/connection_form.js @@ -93,6 +93,11 @@ const ConnectionForm = (props, saveButton) => { const isEditMode = !!connection?.connection_id; const defaultPort = defaultPorts[connection?.sql_flavor]; + const connectionStatus = van.state(undefined); + van.derive(() => { + connectionStatus.val = getValue(props.connection)?.status; + }); + const connectionFlavor = van.state(connection?.sql_flavor_code); const connectionName = van.state(connection?.connection_name ?? ''); const connectionMaxThreads = van.state(connection?.max_threads ?? 4); @@ -318,15 +323,17 @@ const ConnectionForm = (props, saveButton) => { saveButton, ), () => { - const conn = getValue(props.connection); - const connectionStatus = conn.status; - return connectionStatus + return connectionStatus.val ? Alert( - {type: connectionStatus.successful ? 'success' : 'error', closeable: true}, + { + type: connectionStatus.val.successful ? 'success' : 'error', + closeable: true, + onClose: () => connectionStatus.val = undefined, + }, div( { class: 'flex-column' }, - span(connectionStatus.message), - connectionStatus.details ? span(connectionStatus.details) : '', + span(connectionStatus.val.message), + connectionStatus.val.details ? span(connectionStatus.val.details) : '', ) ) : ''; @@ -462,7 +469,11 @@ const RedshiftForm = ( name: 'url_suffix', prefix: span({ style: 'white-space: nowrap; color: var(--disabled-text-color)' }, extractPrefix(dynamicConnectionUrl.val)), disabled: !connectByUrl.val, - onChange: (value, state) => connectionUrl.val = value, + onChange: (value, state) => { + connectionUrl.val = value; + validityPerField['url_suffix'] = state.valid; + isValid.val = Object.values(validityPerField).every(v => v); + }, validators: [ requiredIf(() => connectByUrl.val), ], @@ -660,7 +671,11 @@ const DatabricksForm = ( name: 'url_suffix', prefix: span({ style: 'white-space: nowrap; color: var(--disabled-text-color)' }, extractPrefix(dynamicConnectionUrl.val)), disabled: !connectByUrl.val, - onChange: (value, state) => connectionUrl.val = value, + onChange: (value, state) => { + connectionUrl.val = value; + validityPerField['url_suffix'] = state.valid; + isValid.val = Object.values(validityPerField).every(v => v); + }, validators: [ requiredIf(() => connectByUrl.val), ], @@ -954,6 +969,7 @@ const SnowflakeForm = ( console.error(err); isFieldValid = false; } + validityPerField['private_key'] = isFieldValid; isValid.val = Object.values(validityPerField).every(v => v); }, diff --git a/testgen/ui/components/frontend/js/components/file_input.js b/testgen/ui/components/frontend/js/components/file_input.js index 8bf247e3..5b49f503 100644 --- a/testgen/ui/components/frontend/js/components/file_input.js +++ b/testgen/ui/components/frontend/js/components/file_input.js @@ -60,10 +60,12 @@ const FileInput = (options) => { sizeLimit = sizeLimitValidator.args.limit; } + let hasBeenChecked = false; van.derive(() => { - if (options.onChange && (value.val !== value.oldVal || errors.val.length !== errors.oldVal.length)) { + if (options.onChange && (!hasBeenChecked || value.val !== value.oldVal || errors.val.length !== errors.oldVal.length)) { options.onChange(value.val, { errors: errors.val, valid: errors.val.length <= 0 }); } + hasBeenChecked = true; }); const browseFile = () => { diff --git a/testgen/ui/views/connections.py b/testgen/ui/views/connections.py index 704c2783..1a3d0622 100644 --- a/testgen/ui/views/connections.py +++ b/testgen/ui/views/connections.py @@ -1,5 +1,6 @@ import base64 import logging +import random import typing from dataclasses import asdict, dataclass, field @@ -59,7 +60,11 @@ def render(self, project_code: str, **_kwargs) -> None: ) connections = Connection.select_where(Connection.project_code == project_code) - connection: Connection = connections[0] if len(connections) > 0 else Connection(sql_flavor="postgresql", sql_flavor_code="postgresql") + connection: Connection = connections[0] if len(connections) > 0 else Connection( + sql_flavor="postgresql", + sql_flavor_code="postgresql", + project_code=project_code, + ) has_table_groups = ( connection.id and len(TableGroup.select_minimal_where(TableGroup.connection_id == connection.connection_id) or []) > 0 ) @@ -242,7 +247,7 @@ def test_connection(self, connection: Connection) -> "ConnectionStatus": return ConnectionStatus(message="Error attempting the connection.", details=details, successful=False) except Exception as error: details = "Try again" - if connection["connect_by_key"] and not connection.get("private_key", ""): + if connection.connect_by_key and not connection.private_key: details = "The private key is missing." LOG.exception("Error testing database connection") return ConnectionStatus(message="Error attempting the connection.", details=details, successful=False) @@ -258,6 +263,7 @@ def on_save_table_group_clicked(payload: dict) -> None: set_new_table_group(table_group) set_table_group_verified(table_group_verified) set_run_profiling(run_profiling) + mark_for_save(True) def on_go_to_profiling_runs(params: dict) -> None: set_navigation_params({ **params, "project_code": project_code }) @@ -301,11 +307,18 @@ def on_preview_table_group(payload: dict) -> None: f"connections:{connection_id}:tg_verified", default=False, ) + should_save, mark_for_save = temp_value( + f"connections:{connection_id}:tg_save", + default=False, + ) + add_scorecard_definition = table_group_data.pop("add_scorecard_definition", False) table_group = TableGroup( - **table_group_data or {}, - project_code = project_code, - connection_id = connection_id, + project_code=project_code, + **{ + **(table_group_data or {}), + "connection_id": connection_id, + }, ) table_group_preview = None @@ -315,13 +328,13 @@ def on_preview_table_group(payload: dict) -> None: verify_table_access=should_verify_access(), ) - if table_group_data: + if should_save(): success = True message = None if is_table_group_verified(): try: - table_group.save() + table_group.save(add_scorecard_definition=add_scorecard_definition) if should_run_profiling: try: @@ -342,7 +355,7 @@ def on_preview_table_group(payload: dict) -> None: results = { "success": success, "message": message, - "table_group_id": table_group.id, + "table_group_id": str(table_group.id), } else: results = { @@ -357,6 +370,7 @@ def on_preview_table_group(payload: dict) -> None: props={ "project_code": project_code, "connection_id": connection_id, + "table_group": table_group.to_dict(json_safe=True), "table_group_preview": table_group_preview, "steps": [ "tableGroup", @@ -378,6 +392,7 @@ class ConnectionStatus: message: str successful: bool details: str | None = field(default=None) + _: float = field(default_factory=random.random) def is_open_ssl_error(error: Exception): From 0ff041a8fb20099df6a6d8de32f019661fe0cf3f Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Thu, 4 Sep 2025 11:58:10 -0400 Subject: [PATCH 12/28] style: fix linting errors --- testgen/commands/run_quick_start.py | 2 +- testgen/ui/views/connections.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/testgen/commands/run_quick_start.py b/testgen/commands/run_quick_start.py index 7ae59036..fd973d95 100644 --- a/testgen/commands/run_quick_start.py +++ b/testgen/commands/run_quick_start.py @@ -117,7 +117,7 @@ def run_quick_start(delete_target_db: bool) -> None: delete_db = True run_launch_db_config(delete_db) - click.echo(f"Seeding the application db") + click.echo("Seeding the application db") app_db_params = get_app_db_params_mapping() execute_db_queries( [ diff --git a/testgen/ui/views/connections.py b/testgen/ui/views/connections.py index 1a3d0622..f6bdb269 100644 --- a/testgen/ui/views/connections.py +++ b/testgen/ui/views/connections.py @@ -6,7 +6,6 @@ import streamlit as st -from testgen.common.database.flavor.flavor_service import ConnectionParams from testgen.ui.queries import table_group_queries try: From 94b67abf2d0c03fbc071ca9e74b919e28ae86455 Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Wed, 3 Sep 2025 23:45:15 -0400 Subject: [PATCH 13/28] fix: incorrect test counts in generate tests warning --- testgen/commands/run_refresh_score_cards_results.py | 2 +- testgen/ui/views/dialogs/generate_tests_dialog.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/testgen/commands/run_refresh_score_cards_results.py b/testgen/commands/run_refresh_score_cards_results.py index abd63f2c..7d56c6b5 100644 --- a/testgen/commands/run_refresh_score_cards_results.py +++ b/testgen/commands/run_refresh_score_cards_results.py @@ -72,7 +72,7 @@ def run_refresh_score_cards_results( history_entry.add_as_cutoff() definition.save() LOG.info( - "CurrentStep: Done rereshing scorecard %s in project %s", + "CurrentStep: Done refreshing scorecard %s in project %s", definition.name, definition.project_code, ) diff --git a/testgen/ui/views/dialogs/generate_tests_dialog.py b/testgen/ui/views/dialogs/generate_tests_dialog.py index a548b476..7dc25500 100644 --- a/testgen/ui/views/dialogs/generate_tests_dialog.py +++ b/testgen/ui/views/dialogs/generate_tests_dialog.py @@ -91,11 +91,8 @@ def get_test_suite_refresh_warning(test_suite_id: str) -> tuple[int, int, int]: SUM(CASE WHEN COALESCE(td.lock_refresh, 'N') = 'N' THEN 1 ELSE 0 END) AS unlocked_test_ct, SUM(CASE WHEN COALESCE(td.lock_refresh, 'N') = 'N' AND td.last_manual_update IS NOT NULL THEN 1 ELSE 0 END) AS unlocked_edits_ct FROM test_definitions td - INNER JOIN test_types tt - ON (td.test_type = tt.test_type) WHERE td.test_suite_id = :test_suite_id - AND tt.run_type = 'CAT' - AND tt.selection_criteria IS NOT NULL; + AND td.last_auto_gen_date IS NOT NULL; """, {"test_suite_id": test_suite_id}, ) From 6b004cbe42654e18f8874ac64c8ce64c39771156 Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Thu, 4 Sep 2025 00:34:16 -0400 Subject: [PATCH 14/28] feat: display sql query in source data dialogs --- testgen/ui/queries/source_data_queries.py | 173 +++++++++++++--------- testgen/ui/views/hygiene_issues.py | 16 +- testgen/ui/views/test_results.py | 33 +++-- 3 files changed, 133 insertions(+), 89 deletions(-) diff --git a/testgen/ui/queries/source_data_queries.py b/testgen/ui/queries/source_data_queries.py index 9bc5965b..d632457b 100644 --- a/testgen/ui/queries/source_data_queries.py +++ b/testgen/ui/queries/source_data_queries.py @@ -16,11 +16,7 @@ LOG = logging.getLogger("testgen") -@st.cache_data(show_spinner=False) -def get_hygiene_issue_source_data( - issue_data: dict, - limit: int | None = None, -) -> tuple[Literal["OK"], None, str, pd.DataFrame] | tuple[Literal["NA", "ND", "ERR"], str, str | None, None]: +def get_hygiene_issue_source_query(issue_data: dict) -> str: def generate_lookup_query(test_id: str, detail_exp: str, column_names: list[str], sql_flavor: SQLFlavor) -> str: if test_id in {"1019", "1020"}: start_index = detail_exp.find("Columns: ") @@ -40,33 +36,44 @@ def generate_lookup_query(test_id: str, detail_exp: str, column_names: list[str] sql_query = "" return sql_query - lookup_query = None - try: - lookup_data = _get_lookup_data(issue_data["table_groups_id"], issue_data["anomaly_id"], "Profile Anomaly") - if not lookup_data: - return "NA", "Source data lookup is not available for this hygiene issue.", None, None + lookup_data = _get_lookup_data(issue_data["table_groups_id"], issue_data["anomaly_id"], "Profile Anomaly") + if not lookup_data: + return None - lookup_query = ( - generate_lookup_query( - issue_data["anomaly_id"], issue_data["detail"], issue_data["column_name"], lookup_data.sql_flavor - ) - if lookup_data.lookup_query == "created_in_ui" - else lookup_data.lookup_query + lookup_query = ( + generate_lookup_query( + issue_data["anomaly_id"], issue_data["detail"], issue_data["column_name"], lookup_data.sql_flavor ) + if lookup_data.lookup_query == "created_in_ui" + else lookup_data.lookup_query + ) + + if not lookup_query: + return None + params = { + "TARGET_SCHEMA": issue_data["schema_name"], + "TABLE_NAME": issue_data["table_name"], + "COLUMN_NAME": issue_data["column_name"], + "DETAIL_EXPRESSION": issue_data["detail"], + "PROFILE_RUN_DATE": issue_data["profiling_starttime"], + } + + lookup_query = replace_params(lookup_query, params) + lookup_query = replace_templated_functions(lookup_query, lookup_data.sql_flavor) + return lookup_query + + +@st.cache_data(show_spinner=False) +def get_hygiene_issue_source_data( + issue_data: dict, + limit: int | None = None, +) -> tuple[Literal["OK"], None, str, pd.DataFrame] | tuple[Literal["NA", "ND", "ERR"], str, str | None, None]: + try: + lookup_query = get_hygiene_issue_source_query(issue_data) if not lookup_query: return "NA", "Source data lookup is not available for this hygiene issue.", None, None - params = { - "TARGET_SCHEMA": issue_data["schema_name"], - "TABLE_NAME": issue_data["table_name"], - "COLUMN_NAME": issue_data["column_name"], - "DETAIL_EXPRESSION": issue_data["detail"], - "PROFILE_RUN_DATE": issue_data["profiling_starttime"], - } - lookup_query = replace_params(lookup_query, params) - lookup_query = replace_templated_functions(lookup_query, lookup_data.sql_flavor) - connection = Connection.get_by_table_group(issue_data["table_groups_id"]) results = fetch_from_target_db(connection, lookup_query) @@ -87,53 +94,62 @@ def generate_lookup_query(test_id: str, detail_exp: str, column_names: list[str] return "ERR", f"Source data lookup encountered an error:\n\n{e.args[0]}", lookup_query, None +def get_test_issue_source_query(issue_data: dict) -> str: + lookup_data = _get_lookup_data(issue_data["table_groups_id"], issue_data["test_type_id"], "Test Results") + if not lookup_data or not lookup_data.lookup_query: + return None + + test_definition = TestDefinition.get(issue_data["test_definition_id_current"]) + if not test_definition: + return None + + params = { + "TARGET_SCHEMA": issue_data["schema_name"], + "TABLE_NAME": issue_data["table_name"], + "COLUMN_NAME": issue_data["column_names"], + "TEST_DATE": str(issue_data["test_date"]), + "CUSTOM_QUERY": test_definition.custom_query, + "BASELINE_VALUE": test_definition.baseline_value, + "BASELINE_CT": test_definition.baseline_ct, + "BASELINE_AVG": test_definition.baseline_avg, + "BASELINE_SD": test_definition.baseline_sd, + "LOWER_TOLERANCE": test_definition.lower_tolerance, + "UPPER_TOLERANCE": test_definition.upper_tolerance, + "THRESHOLD_VALUE": test_definition.threshold_value, + "SUBSET_CONDITION": test_definition.subset_condition or "1=1", + "GROUPBY_NAMES": test_definition.groupby_names, + "HAVING_CONDITION": test_definition.having_condition, + "MATCH_SCHEMA_NAME": test_definition.match_schema_name, + "MATCH_TABLE_NAME": test_definition.match_table_name, + "MATCH_COLUMN_NAMES": test_definition.match_column_names, + "MATCH_SUBSET_CONDITION": test_definition.match_subset_condition or "1=1", + "MATCH_GROUPBY_NAMES": test_definition.match_groupby_names, + "MATCH_HAVING_CONDITION": test_definition.match_having_condition, + "COLUMN_NAME_NO_QUOTES": issue_data["column_names"], + "WINDOW_DATE_COLUMN": test_definition.window_date_column, + "WINDOW_DAYS": test_definition.window_days, + "CONCAT_COLUMNS": ConcatColumnList(issue_data["column_names"], ""), + "CONCAT_MATCH_GROUPBY": ConcatColumnList(test_definition.match_groupby_names, ""), + } + + lookup_query = replace_params(lookup_data.lookup_query, params) + lookup_query = replace_templated_functions(lookup_query, lookup_data.sql_flavor) + return lookup_query + + @st.cache_data(show_spinner=False) def get_test_issue_source_data( issue_data: dict, limit: int | None = None, ) -> tuple[Literal["OK"], None, str, pd.DataFrame] | tuple[Literal["NA", "ND", "ERR"], str, str | None, None]: - lookup_query = None try: - lookup_data = _get_lookup_data(issue_data["table_groups_id"], issue_data["test_type_id"], "Test Results") - - if not lookup_data or not lookup_data.lookup_query: - return "NA", "Source data lookup is not available for this test.", None, None - test_definition = TestDefinition.get(issue_data["test_definition_id_current"]) if not test_definition: return "NA", "Test definition no longer exists.", None, None - - params = { - "TARGET_SCHEMA": issue_data["schema_name"], - "TABLE_NAME": issue_data["table_name"], - "COLUMN_NAME": issue_data["column_names"], - "TEST_DATE": str(issue_data["test_date"]), - "CUSTOM_QUERY": test_definition.custom_query, - "BASELINE_VALUE": test_definition.baseline_value, - "BASELINE_CT": test_definition.baseline_ct, - "BASELINE_AVG": test_definition.baseline_avg, - "BASELINE_SD": test_definition.baseline_sd, - "LOWER_TOLERANCE": test_definition.lower_tolerance, - "UPPER_TOLERANCE": test_definition.upper_tolerance, - "THRESHOLD_VALUE": test_definition.threshold_value, - "SUBSET_CONDITION": test_definition.subset_condition or "1=1", - "GROUPBY_NAMES": test_definition.groupby_names, - "HAVING_CONDITION": test_definition.having_condition, - "MATCH_SCHEMA_NAME": test_definition.match_schema_name, - "MATCH_TABLE_NAME": test_definition.match_table_name, - "MATCH_COLUMN_NAMES": test_definition.match_column_names, - "MATCH_SUBSET_CONDITION": test_definition.match_subset_condition or "1=1", - "MATCH_GROUPBY_NAMES": test_definition.match_groupby_names, - "MATCH_HAVING_CONDITION": test_definition.match_having_condition, - "COLUMN_NAME_NO_QUOTES": issue_data["column_names"], - "WINDOW_DATE_COLUMN": test_definition.window_date_column, - "WINDOW_DAYS": test_definition.window_days, - "CONCAT_COLUMNS": ConcatColumnList(issue_data["column_names"], ""), - "CONCAT_MATCH_GROUPBY": ConcatColumnList(test_definition.match_groupby_names, ""), - } - - lookup_query = replace_params(lookup_data.lookup_query, params) - lookup_query = replace_templated_functions(lookup_query, lookup_data.sql_flavor) + + lookup_query = get_test_issue_source_query(issue_data) + if not lookup_query: + return "NA", "Source data lookup is not available for this test.", None, None connection = Connection.get_by_table_group(issue_data["table_groups_id"]) results = fetch_from_target_db(connection, lookup_query) @@ -150,23 +166,34 @@ def get_test_issue_source_data( return "ERR", f"Source data lookup encountered an error:\n\n{e.args[0]}", lookup_query, None +def get_test_issue_source_query_custom( + issue_data: dict, +) -> str: + lookup_data = _get_lookup_data_custom(issue_data["test_definition_id_current"]) + if not lookup_data or not lookup_data.lookup_query: + return None + + params = { + "DATA_SCHEMA": issue_data["schema_name"], + } + lookup_query = replace_params(lookup_data.lookup_query, params) + return lookup_query + + @st.cache_data(show_spinner=False) def get_test_issue_source_data_custom( issue_data: dict, limit: int | None = None, ) -> tuple[Literal["OK"], None, str, pd.DataFrame] | tuple[Literal["NA", "ND", "ERR"], str, str | None, None]: - lookup_query = None try: - lookup_data = _get_lookup_data_custom(issue_data["test_definition_id_current"]) - - if not lookup_data or not lookup_data.lookup_query: + test_definition = TestDefinition.get(issue_data["test_definition_id_current"]) + if not test_definition: + return "NA", "Test definition no longer exists.", None, None + + lookup_query = get_test_issue_source_query_custom(issue_data) + if not lookup_query: return "NA", "Source data lookup is not available for this test.", None, None - params = { - "DATA_SCHEMA": issue_data["schema_name"], - } - lookup_query = replace_params(lookup_data.lookup_query, params) - connection = Connection.get_by_table_group(issue_data["table_groups_id"]) results = fetch_from_target_db(connection, lookup_query) diff --git a/testgen/ui/views/hygiene_issues.py b/testgen/ui/views/hygiene_issues.py index e7d12593..c65a9579 100644 --- a/testgen/ui/views/hygiene_issues.py +++ b/testgen/ui/views/hygiene_issues.py @@ -22,7 +22,7 @@ from testgen.ui.components.widgets.page import css_class, flex_row_end from testgen.ui.navigation.page import Page from testgen.ui.pdf.hygiene_issue_report import create_report -from testgen.ui.queries.source_data_queries import get_hygiene_issue_source_data +from testgen.ui.queries.source_data_queries import get_hygiene_issue_source_data, get_hygiene_issue_source_query from testgen.ui.services.database_service import ( execute_db_query, fetch_df_from_db, @@ -580,12 +580,18 @@ def get_excel_report_data( @st.dialog(title="Source Data") @with_database_session def source_data_dialog(selected_row): + testgen.caption(f"Table > Column: {selected_row['table_name']} > {selected_row['column_name']}") + st.markdown(f"#### {selected_row['anomaly_name']}") st.caption(selected_row["anomaly_description"]) - fm.show_prompt(f"Column: {selected_row['column_name']}, Table: {selected_row['table_name']}") - - # Show the detail line - fm.render_html_list(selected_row, ["detail"], None, 700, ["Hygiene Issue Detail"]) + + st.markdown("#### Hygiene Issue Detail") + st.caption(selected_row["detail"]) + + st.markdown("#### SQL Query") + query = get_hygiene_issue_source_query(selected_row) + if query: + st.code(query, language="sql") with st.spinner("Retrieving source data..."): bad_data_status, bad_data_msg, _, df_bad = get_hygiene_issue_source_data(selected_row, limit=500) diff --git a/testgen/ui/views/test_results.py b/testgen/ui/views/test_results.py index d254f4ec..575616a2 100644 --- a/testgen/ui/views/test_results.py +++ b/testgen/ui/views/test_results.py @@ -34,7 +34,12 @@ from testgen.ui.navigation.page import Page from testgen.ui.pdf.test_result_report import create_report from testgen.ui.queries import test_result_queries -from testgen.ui.queries.source_data_queries import get_test_issue_source_data, get_test_issue_source_data_custom +from testgen.ui.queries.source_data_queries import ( + get_test_issue_source_data, + get_test_issue_source_data_custom, + get_test_issue_source_query, + get_test_issue_source_query_custom, +) from testgen.ui.services.database_service import execute_db_query, fetch_df_from_db, fetch_one_from_db from testgen.ui.services.string_service import empty_if_null, snake_case_to_title_case from testgen.ui.session import session @@ -794,18 +799,24 @@ def do_disposition_update(selected, str_new_status): @st.dialog(title="Source Data") @with_database_session def source_data_dialog(selected_row): + testgen.caption(f"Table > Column: {selected_row['table_name']} > {selected_row['column_names']}") + st.markdown(f"#### {selected_row['test_name_short']}") st.caption(selected_row["test_description"]) - fm.show_prompt(f"Column: {selected_row['column_names']}, Table: {selected_row['table_name']}") - - # Show detail - fm.render_html_list( - selected_row, - lst_columns=["input_parameters", "result_message"], - str_section_header=None, - int_data_width=0, - lst_labels=["Test Parameters", "Result Detail"], - ) + + st.markdown("#### Test Parameters") + st.caption(selected_row["input_parameters"]) + + st.markdown("#### Result Detail") + st.caption(selected_row["result_message"]) + + st.markdown("#### SQL Query") + if selected_row["test_type"] == "CUSTOM": + query = get_test_issue_source_query_custom(selected_row) + else: + query = get_test_issue_source_query(selected_row) + if query: + st.code(query, language="sql") with st.spinner("Retrieving source data..."): if selected_row["test_type"] == "CUSTOM": From af7dd5190a9071d476decdc1d3634e2cc7d65747 Mon Sep 17 00:00:00 2001 From: Ricardo Boni Date: Wed, 10 Sep 2025 13:12:25 -0400 Subject: [PATCH 15/28] fix: Fixing LOV_All input --- testgen/ui/views/test_definitions.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/testgen/ui/views/test_definitions.py b/testgen/ui/views/test_definitions.py index c8973f56..0c0f5300 100644 --- a/testgen/ui/views/test_definitions.py +++ b/testgen/ui/views/test_definitions.py @@ -283,7 +283,7 @@ def show_test_form( last_auto_gen_date = empty_if_null(selected_test_def["last_auto_gen_date"]) if mode == "edit" else "" profiling_as_of_date = empty_if_null(selected_test_def["profiling_as_of_date"]) if mode == "edit" else "" profile_run_id = empty_if_null(selected_test_def["profile_run_id"]) if mode == "edit" else "" - + # dynamic attributes custom_query = empty_if_null(selected_test_def["custom_query"]) if mode == "edit" else "" @@ -529,7 +529,9 @@ def render_dynamic_attribute(attribute: str, container: DeltaGenerator): choice_fields = { "history_calculation": ["Value", "Minimum", "Maximum", "Sum", "Average"], } - float_numeric_attributes = ["threshold_value", "lower_tolerance", "upper_tolerance"] + float_numeric_attributes = ["lower_tolerance", "upper_tolerance"] + if test_type != "LOV_All": + float_numeric_attributes.append("threshold_value") int_numeric_attributes = ["history_lookback"] default_value = 0 if attribute in [*float_numeric_attributes, *int_numeric_attributes] else "" @@ -1072,7 +1074,7 @@ def generate_test_defs_help(str_test_type): @st.cache_data(show_spinner=False) def run_test_type_lookup_query( - test_type: str | None = None, + test_type: str | None = None, include_referential: bool = True, include_table: bool = True, include_column: bool = True, @@ -1085,7 +1087,7 @@ def run_test_type_lookup_query( "custom": include_custom, } scopes = [ key for key, include in scope_map.items() if include ] - + query = f""" SELECT tt.id, tt.test_type, tt.id as cat_test_id, @@ -1105,7 +1107,7 @@ def run_test_type_lookup_query( || tt.test_name_short || ': ' || lower(tt.test_name_long) - || CASE + || CASE WHEN tt.selection_criteria > '' THEN ' [auto-generated]' ELSE '' END as select_name @@ -1220,7 +1222,7 @@ def validate_test(test_definition, table_group: TableGroupMinimal): condition = test_definition["custom_query"] concat_operator = get_flavor_service(connection.sql_flavor).get_concat_operator() query = f""" - SELECT + SELECT COALESCE( CAST( SUM( From 9cb87e863b72da57b7ef139230badc1a707e6a6d Mon Sep 17 00:00:00 2001 From: Luis Date: Thu, 11 Sep 2025 10:22:04 -0400 Subject: [PATCH 16/28] fix: append snowflake computing domain when missing --- testgen/common/database/flavor/snowflake_flavor_service.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/testgen/common/database/flavor/snowflake_flavor_service.py b/testgen/common/database/flavor/snowflake_flavor_service.py index 07f620f4..49c479bb 100644 --- a/testgen/common/database/flavor/snowflake_flavor_service.py +++ b/testgen/common/database/flavor/snowflake_flavor_service.py @@ -40,13 +40,16 @@ def get_connection_string_from_fields(self): # NOTE: Snowflake host should NOT include ".snowflakecomputing.com" account, _ = self.host.split(".", maxsplit=1) if "." in self.host else ("", "") + host = self.host + if ".snowflakecomputing.com" not in host: + host = f"{host}.snowflakecomputing.com" extra_params = {} if self.warehouse: extra_params["warehouse"] = self.warehouse connection_url = URL( - host=self.host, + host=host, port=int(self.port if str(self.port).isdigit() else 443), account=account, user=self.username, From 646f38f5e547057da5fb9e0a480e4f9d73c3fd1e Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Tue, 26 Aug 2025 17:57:27 -0400 Subject: [PATCH 17/28] fix(table freshness): filter general type in fingerprint --- .../gen_query_tests/gen_table_changed_test.sql | 10 +++++++--- .../mssql/gen_query_tests/gen_table_changed_test.sql | 10 +++++++--- .../gen_query_tests/gen_table_changed_test.sql | 10 +++++++--- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/testgen/template/flavors/databricks/gen_query_tests/gen_table_changed_test.sql b/testgen/template/flavors/databricks/gen_query_tests/gen_table_changed_test.sql index 6f62ac92..b8c4d2bc 100644 --- a/testgen/template/flavors/databricks/gen_query_tests/gen_table_changed_test.sql +++ b/testgen/template/flavors/databricks/gen_query_tests/gen_table_changed_test.sql @@ -38,7 +38,8 @@ id_cols ELSE 3 END, distinct_value_ct, column_name DESC) AS rank FROM curprof - WHERE functional_data_type ILIKE 'ID%'), + WHERE general_type IN ('A', 'D', 'N') + AND functional_data_type ILIKE 'ID%'), -- Process Date - TOP 1 process_date_cols AS (SELECT profile_run_id, schema_name, table_name, column_name, functional_data_type, general_type, @@ -52,7 +53,8 @@ process_date_cols WHEN column_name ILIKE '%in%' THEN 2 END , distinct_value_ct DESC, column_name) AS rank FROM curprof - WHERE functional_data_type ILIKE 'process%'), + WHERE general_type IN ('A', 'D', 'N') + AND functional_data_type ILIKE 'process%'), -- Transaction Date - TOP 1 tran_date_cols AS ( SELECT profile_run_id, schema_name, table_name, column_name, functional_data_type, general_type, @@ -61,7 +63,9 @@ tran_date_cols ORDER BY distinct_value_ct DESC, column_name) AS rank FROM curprof - WHERE functional_data_type ILIKE 'transactional date%' OR functional_data_type ILIKE 'period%' + WHERE general_type IN ('A', 'D', 'N') + AND functional_data_type ILIKE 'transactional date%' + OR functional_data_type ILIKE 'period%' OR functional_data_type = 'timestamp' ), -- Numeric Measures diff --git a/testgen/template/flavors/mssql/gen_query_tests/gen_table_changed_test.sql b/testgen/template/flavors/mssql/gen_query_tests/gen_table_changed_test.sql index 4bd85532..e9517725 100644 --- a/testgen/template/flavors/mssql/gen_query_tests/gen_table_changed_test.sql +++ b/testgen/template/flavors/mssql/gen_query_tests/gen_table_changed_test.sql @@ -38,7 +38,8 @@ id_cols ELSE 3 END, distinct_value_ct, column_name DESC) AS rank FROM curprof - WHERE functional_data_type ILIKE 'ID%'), + WHERE general_type IN ('A', 'D', 'N') + AND functional_data_type ILIKE 'ID%'), -- Process Date - TOP 1 process_date_cols AS (SELECT profile_run_id, schema_name, table_name, column_name, functional_data_type, general_type, @@ -52,7 +53,8 @@ process_date_cols WHEN column_name ILIKE '%in%' THEN 2 END , distinct_value_ct DESC, column_name) AS rank FROM curprof - WHERE functional_data_type ILIKE 'process%'), + WHERE general_type IN ('A', 'D', 'N') + AND functional_data_type ILIKE 'process%'), -- Transaction Date - TOP 1 tran_date_cols AS ( SELECT profile_run_id, schema_name, table_name, column_name, functional_data_type, general_type, @@ -61,7 +63,9 @@ tran_date_cols ORDER BY distinct_value_ct DESC, column_name) AS rank FROM curprof - WHERE functional_data_type ILIKE 'transactional date%' OR functional_data_type ILIKE 'period%' + WHERE general_type IN ('A', 'D', 'N') + AND functional_data_type ILIKE 'transactional date%' + OR functional_data_type ILIKE 'period%' OR functional_data_type = 'timestamp' ), -- Numeric Measures diff --git a/testgen/template/gen_query_tests/gen_table_changed_test.sql b/testgen/template/gen_query_tests/gen_table_changed_test.sql index 8683520c..7779652e 100644 --- a/testgen/template/gen_query_tests/gen_table_changed_test.sql +++ b/testgen/template/gen_query_tests/gen_table_changed_test.sql @@ -38,7 +38,8 @@ id_cols ELSE 3 END, distinct_value_ct, column_name DESC) AS rank FROM curprof - WHERE functional_data_type ILIKE 'ID%'), + WHERE general_type IN ('A', 'D', 'N') + AND functional_data_type ILIKE 'ID%'), -- Process Date - TOP 1 process_date_cols AS (SELECT profile_run_id, schema_name, table_name, column_name, functional_data_type, general_type, @@ -52,7 +53,8 @@ process_date_cols WHEN column_name ILIKE '%in%' THEN 2 END , distinct_value_ct DESC, column_name) AS rank FROM curprof - WHERE functional_data_type ILIKE 'process%'), + WHERE general_type IN ('A', 'D', 'N') + AND functional_data_type ILIKE 'process%'), -- Transaction Date - TOP 1 tran_date_cols AS ( SELECT profile_run_id, schema_name, table_name, column_name, functional_data_type, general_type, @@ -61,7 +63,9 @@ tran_date_cols ORDER BY distinct_value_ct DESC, column_name) AS rank FROM curprof - WHERE functional_data_type ILIKE 'transactional date%' OR functional_data_type ILIKE 'period%' + WHERE general_type IN ('A', 'D', 'N') + AND functional_data_type ILIKE 'transactional date%' + OR functional_data_type ILIKE 'period%' OR functional_data_type = 'timestamp' ), -- Numeric Measures From a893d2c7df911717e0edd9e82d3f719f3f53d833 Mon Sep 17 00:00:00 2001 From: "Chip.Bloche" Date: Wed, 10 Sep 2025 11:22:18 -0400 Subject: [PATCH 18/28] fix: prevent STDEV overflow error --- .../050_populate_new_schema_metadata.sql | 36 ++-- .../gen_table_changed_test.sql | 2 +- .../ex_table_changed_mssql.sql | 33 ++++ .../gen_table_changed_test.sql | 6 +- .../gen_table_changed_test.sql | 157 ++++++++++++++++++ .../gen_table_changed_test.sql | 2 +- 6 files changed, 213 insertions(+), 23 deletions(-) create mode 100644 testgen/template/flavors/mssql/exec_query_tests/ex_table_changed_mssql.sql create mode 100644 testgen/template/flavors/postgresql/gen_query_tests/gen_table_changed_test.sql diff --git a/testgen/template/dbsetup/050_populate_new_schema_metadata.sql b/testgen/template/dbsetup/050_populate_new_schema_metadata.sql index 5c7e3873..47d0e9a9 100644 --- a/testgen/template/dbsetup/050_populate_new_schema_metadata.sql +++ b/testgen/template/dbsetup/050_populate_new_schema_metadata.sql @@ -234,7 +234,7 @@ VALUES ('2001', 'Combo_Match', 'redshift', 'ex_data_match_generic.sql'), ('2012', 'Table_Freshness', 'redshift', 'ex_table_changed_generic.sql'), ('2112', 'Table_Freshness', 'snowflake', 'ex_table_changed_generic.sql'), - ('2212', 'Table_Freshness', 'mssql', 'ex_table_changed_generic.sql'), + ('2212', 'Table_Freshness', 'mssql', 'ex_table_changed_mssql.sql'), ('2312', 'Table_Freshness', 'postgresql', 'ex_table_changed_generic.sql'), ('2412', 'Table_Freshness', 'databricks', 'ex_table_changed_generic.sql') ; @@ -243,7 +243,7 @@ TRUNCATE TABLE cat_test_conditions; INSERT INTO cat_test_conditions (id, test_type, sql_flavor, measure, test_operator, test_condition) VALUES ('1001', 'Alpha_Trunc', 'redshift', 'MAX(LENGTH({COLUMN_NAME}))', '<', '{THRESHOLD_VALUE}'), - ('1002', 'Avg_Shift', 'redshift', 'ABS( (AVG({COLUMN_NAME}::FLOAT) - {BASELINE_AVG}) / SQRT(((COUNT({COLUMN_NAME})::FLOAT-1)*STDDEV({COLUMN_NAME})^2 + ({BASELINE_VALUE_CT}::FLOAT-1) * {BASELINE_SD}::FLOAT^2) /NULLIF(COUNT({COLUMN_NAME})::FLOAT + {BASELINE_VALUE_CT}::FLOAT, 0) ))', '>=', '{THRESHOLD_VALUE}'), + ('1002', 'Avg_Shift', 'redshift', 'ABS( (AVG({COLUMN_NAME}::FLOAT) - {BASELINE_AVG}) / SQRT(((COUNT({COLUMN_NAME})::FLOAT-1)*STDDEV({COLUMN_NAME}::FLOAT)^2 + ({BASELINE_VALUE_CT}::FLOAT-1) * {BASELINE_SD}::FLOAT^2) /NULLIF(COUNT({COLUMN_NAME})::FLOAT + {BASELINE_VALUE_CT}::FLOAT, 0) ))', '>=', '{THRESHOLD_VALUE}'), ('1003', 'Condition_Flag', 'redshift', 'SUM(CASE WHEN {CUSTOM_QUERY} THEN 1 ELSE 0 END)', '>', '{THRESHOLD_VALUE}'), ('1004', 'Constant', 'redshift', 'SUM(CASE WHEN {COLUMN_NAME} <> {BASELINE_VALUE} THEN 1 ELSE 0 END)', '>', '{THRESHOLD_VALUE}'), ('1005', 'Daily_Record_Ct', 'redshift', 'DATEDIFF(''DAY'', MIN({COLUMN_NAME}), MAX({COLUMN_NAME}))+1-COUNT(DISTINCT {COLUMN_NAME})', '>', '{THRESHOLD_VALUE}'), @@ -273,7 +273,7 @@ VALUES ('1001', 'Alpha_Trunc', 'redshift', 'MAX(LENGTH({COLUMN_NAME}))', '<', ' ('1029', 'Unique_Pct', 'redshift', 'ABS( 2.0 * ASIN( SQRT({BASELINE_UNIQUE_CT}::FLOAT / {BASELINE_VALUE_CT}::FLOAT ) ) - 2 * ASIN( SQRT( COUNT( DISTINCT {COLUMN_NAME} )::FLOAT / NULLIF(COUNT( {COLUMN_NAME} ), 0)::FLOAT )) )', '>=', '{THRESHOLD_VALUE}'), ('1030', 'Weekly_Rec_Ct', 'redshift', 'MAX(DATEDIFF(week, ''1800-01-01''::DATE, {COLUMN_NAME})) - MIN(DATEDIFF(week, ''1800-01-01''::DATE, {COLUMN_NAME}))+1 - COUNT(DISTINCT DATEDIFF(week, ''1800-01-01''::DATE, {COLUMN_NAME}))', '>', '{THRESHOLD_VALUE}'), ('2001', 'Alpha_Trunc', 'snowflake', 'MAX(LENGTH({COLUMN_NAME}))', '<', '{THRESHOLD_VALUE}'), - ('2002', 'Avg_Shift', 'snowflake', 'ABS( (AVG({COLUMN_NAME}::FLOAT) - {BASELINE_AVG}) / SQRT(((COUNT({COLUMN_NAME})::FLOAT-1)*POWER(STDDEV({COLUMN_NAME}),2) + ({BASELINE_VALUE_CT}::FLOAT-1) * POWER({BASELINE_SD}::FLOAT,2)) /NULLIF(COUNT({COLUMN_NAME})::FLOAT + {BASELINE_VALUE_CT}::FLOAT, 0) ))', '>=', '{THRESHOLD_VALUE}'), + ('2002', 'Avg_Shift', 'snowflake', 'ABS( (AVG({COLUMN_NAME}::FLOAT) - {BASELINE_AVG}) / SQRT(((COUNT({COLUMN_NAME})::FLOAT-1)*POWER(STDDEV({COLUMN_NAME}::FLOAT),2) + ({BASELINE_VALUE_CT}::FLOAT-1) * POWER({BASELINE_SD}::FLOAT,2)) /NULLIF(COUNT({COLUMN_NAME})::FLOAT + {BASELINE_VALUE_CT}::FLOAT, 0) ))', '>=', '{THRESHOLD_VALUE}'), ('2003', 'Condition_Flag', 'snowflake', 'SUM(CASE WHEN {CUSTOM_QUERY} THEN 1 ELSE 0 END)', '>', '{THRESHOLD_VALUE}'), ('2004', 'Constant', 'snowflake', 'SUM(CASE WHEN {COLUMN_NAME} <> {BASELINE_VALUE} THEN 1 ELSE 0 END)', '>', '{THRESHOLD_VALUE}'), ('2005', 'Daily_Record_Ct', 'snowflake', 'DATEDIFF(day, MIN({COLUMN_NAME}), MAX({COLUMN_NAME}))+1-COUNT(DISTINCT {COLUMN_NAME})', '<', '{THRESHOLD_VALUE}'), @@ -303,7 +303,7 @@ VALUES ('1001', 'Alpha_Trunc', 'redshift', 'MAX(LENGTH({COLUMN_NAME}))', '<', ' ('2029', 'Unique_Pct', 'snowflake', 'ABS( 2.0 * ASIN( SQRT({BASELINE_UNIQUE_CT}::FLOAT / {BASELINE_VALUE_CT}::FLOAT ) ) - 2 * ASIN( SQRT( COUNT( DISTINCT {COLUMN_NAME} )::FLOAT / NULLIF(COUNT( {COLUMN_NAME} ), 0)::FLOAT )) )', '>=', '{THRESHOLD_VALUE}'), ('2030', 'Weekly_Rec_Ct', 'snowflake', 'MAX(DATEDIFF(week, ''1800-01-01''::DATE, {COLUMN_NAME})) - MIN(DATEDIFF(week, ''1800-01-01''::DATE, {COLUMN_NAME}))+1 - COUNT(DISTINCT DATEDIFF(week, ''1800-01-01''::DATE, {COLUMN_NAME}))', '>', '{THRESHOLD_VALUE}'), ('3001', 'Alpha_Trunc', 'mssql', 'MAX(LEN({COLUMN_NAME}))', '<', '{THRESHOLD_VALUE}'), - ('3002', 'Avg_Shift', 'mssql', 'ABS( (AVG(CAST({COLUMN_NAME} AS FLOAT)) - {BASELINE_AVG}) / SQRT(((COUNT({COLUMN_NAME})-1)*POWER(STDEV({COLUMN_NAME}), 2) + ({BASELINE_VALUE_CT}-1) * POWER({BASELINE_SD}, 2)) /NULLIF(COUNT({COLUMN_NAME}) + {BASELINE_VALUE_CT}, 0) ))', '>=', '{THRESHOLD_VALUE}'), + ('3002', 'Avg_Shift', 'mssql', 'ABS( (AVG(CAST({COLUMN_NAME} AS FLOAT)) - CAST({BASELINE_AVG} as FLOAT)) / SQRT(((COUNT({COLUMN_NAME})-1)*POWER(STDEV(CAST({COLUMN_NAME} AS FLOAT)), 2) + ({BASELINE_VALUE_CT}-1) * POWER(CAST({BASELINE_SD} as FLOAT), 2)) /NULLIF(COUNT({COLUMN_NAME}) + {BASELINE_VALUE_CT}, 0) ))', '>=', '{THRESHOLD_VALUE}'), ('3003', 'Condition_Flag', 'mssql', 'SUM(CASE WHEN {CUSTOM_QUERY} THEN 1 ELSE 0 END)', '>', '{THRESHOLD_VALUE}'), ('3004', 'Constant', 'mssql', 'SUM(CASE WHEN {COLUMN_NAME} <> {BASELINE_VALUE} THEN 1 ELSE 0 END)', '>', '{THRESHOLD_VALUE}'), ('3005', 'Daily_Record_Ct', 'mssql', 'DATEDIFF(day, MIN({COLUMN_NAME}), MAX({COLUMN_NAME}))+1-COUNT(DISTINCT {COLUMN_NAME})', '<', '{THRESHOLD_VALUE}'), @@ -333,7 +333,7 @@ VALUES ('1001', 'Alpha_Trunc', 'redshift', 'MAX(LENGTH({COLUMN_NAME}))', '<', ' ('3029', 'Unique_Pct', 'mssql', 'ABS( 2.0 * ASIN( SQRT(CAST({BASELINE_UNIQUE_CT} AS FLOAT) / CAST({BASELINE_VALUE_CT} AS FLOAT) ) ) - 2 * ASIN( SQRT( CAST(COUNT( DISTINCT {COLUMN_NAME} ) AS FLOAT) / CAST(NULLIF(COUNT( {COLUMN_NAME} ), 0) AS FLOAT) )) )', '>=', '{THRESHOLD_VALUE}'), ('3030', 'Weekly_Rec_Ct', 'mssql', 'MAX(DATEDIFF(week, CAST(''1800-01-01'' AS DATE), {COLUMN_NAME})) - MIN(DATEDIFF(week, CAST(''1800-01-01'' AS DATE), {COLUMN_NAME}))+1 - COUNT(DISTINCT DATEDIFF(week, CAST(''1800-01-01'' AS DATE), {COLUMN_NAME}))', '>', '{THRESHOLD_VALUE}'), ('4001', 'Alpha_Trunc', 'postgresql', 'MAX(LENGTH({COLUMN_NAME}))', '<', '{THRESHOLD_VALUE}'), - ('4002', 'Avg_Shift', 'postgresql', 'ABS( (AVG({COLUMN_NAME}::FLOAT) - {BASELINE_AVG}) / SQRT(((COUNT({COLUMN_NAME})::FLOAT-1)*STDDEV({COLUMN_NAME})^2 + ({BASELINE_VALUE_CT}::FLOAT-1) * {BASELINE_SD}::FLOAT^2) /NULLIF(COUNT({COLUMN_NAME})::FLOAT + {BASELINE_VALUE_CT}::FLOAT, 0) ))', '>=', '{THRESHOLD_VALUE}'), + ('4002', 'Avg_Shift', 'postgresql', 'ABS( (AVG({COLUMN_NAME}::FLOAT) - {BASELINE_AVG}) / SQRT(((COUNT({COLUMN_NAME})::FLOAT-1)*STDDEV({COLUMN_NAME}::FLOAT)^2 + ({BASELINE_VALUE_CT}::FLOAT-1) * {BASELINE_SD}::FLOAT^2) /NULLIF(COUNT({COLUMN_NAME})::FLOAT + {BASELINE_VALUE_CT}::FLOAT, 0) ))', '>=', '{THRESHOLD_VALUE}'), ('4003', 'Condition_Flag', 'postgresql', 'SUM(CASE WHEN {CUSTOM_QUERY} THEN 1 ELSE 0 END)', '>', '{THRESHOLD_VALUE}'), ('4004', 'Constant', 'postgresql', 'SUM(CASE WHEN {COLUMN_NAME} <> {BASELINE_VALUE} THEN 1 ELSE 0 END)', '>', '{THRESHOLD_VALUE}'), ('4005', 'Daily_Record_Ct', 'postgresql', '<%DATEDIFF_DAY;MIN({COLUMN_NAME});MAX({COLUMN_NAME})%>+1-COUNT(DISTINCT {COLUMN_NAME})', '>', '{THRESHOLD_VALUE}'), @@ -363,16 +363,16 @@ VALUES ('1001', 'Alpha_Trunc', 'redshift', 'MAX(LENGTH({COLUMN_NAME}))', '<', ' ('4029', 'Unique_Pct', 'postgresql', 'ABS( 2.0 * ASIN( SQRT({BASELINE_UNIQUE_CT}::FLOAT / {BASELINE_VALUE_CT}::FLOAT ) ) - 2 * ASIN( SQRT( COUNT( DISTINCT {COLUMN_NAME} )::FLOAT / NULLIF(COUNT( {COLUMN_NAME} ), 0)::FLOAT )) )', '>=', '{THRESHOLD_VALUE}'), ('4030', 'Weekly_Rec_Ct', 'postgresql', 'MAX(<%DATEDIFF_WEEK;''1800-01-01''::DATE;{COLUMN_NAME}%>) - MIN(<%DATEDIFF_WEEK;''1800-01-01''::DATE;{COLUMN_NAME}%>)+1 - COUNT(DISTINCT <%DATEDIFF_WEEK;''1800-01-01''::DATE;{COLUMN_NAME}%>)', '>', '{THRESHOLD_VALUE}'), - ('1031', 'Variability_Increase', 'redshift', '100.0*STDDEV(CAST({COLUMN_NAME} AS FLOAT))/{BASELINE_SD}', '>', '{THRESHOLD_VALUE}'), - ('1032', 'Variability_Decrease', 'redshift', '100.0*STDDEV(CAST({COLUMN_NAME} AS FLOAT))/{BASELINE_SD}', '<', '{THRESHOLD_VALUE}'), - ('2031', 'Variability_Increase', 'snowflake', '100.0*STDDEV(CAST({COLUMN_NAME} AS FLOAT))/{BASELINE_SD}', '>', '{THRESHOLD_VALUE}'), - ('2032', 'Variability_Decrease', 'snowflake', '100.0*STDDEV(CAST({COLUMN_NAME} AS FLOAT))/{BASELINE_SD}', '<', '{THRESHOLD_VALUE}'), - ('3031', 'Variability_Increase', 'mssql', '100.0*STDEV(CAST({COLUMN_NAME} AS FLOAT))/{BASELINE_SD}', '>', '{THRESHOLD_VALUE}'), - ('3032', 'Variability_Decrease', 'mssql', '100.0*STDEV(CAST({COLUMN_NAME} AS FLOAT))/{BASELINE_SD}', '<', '{THRESHOLD_VALUE}'), - ('4031', 'Variability_Increase', 'postgresql', '100.0*STDDEV(CAST({COLUMN_NAME} AS FLOAT))/{BASELINE_SD}', '>', '{THRESHOLD_VALUE}'), - ('4032', 'Variability_Decrease', 'postgresql', '100.0*STDDEV(CAST({COLUMN_NAME} AS FLOAT))/{BASELINE_SD}', '<', '{THRESHOLD_VALUE}'), - ('6031', 'Variability_Increase', 'databricks', '100.0*STDDEV_SAMP(CAST({COLUMN_NAME} AS FLOAT))/{BASELINE_SD}', '>', '{THRESHOLD_VALUE}'), - ('6032', 'Variability_Decrease', 'databricks', '100.0*STDDEV_SAMP(CAST({COLUMN_NAME} AS FLOAT))/{BASELINE_SD}', '<', '{THRESHOLD_VALUE}'), + ('1031', 'Variability_Increase', 'redshift', '100.0*STDDEV({COLUMN_NAME}::FLOAT)/{BASELINE_SD}::FLOAT', '>', '{THRESHOLD_VALUE}'), + ('1032', 'Variability_Decrease', 'redshift', '100.0*STDDEV({COLUMN_NAME}::FLOAT)/{BASELINE_SD}::FLOAT', '<', '{THRESHOLD_VALUE}'), + ('2031', 'Variability_Increase', 'snowflake', '100.0*STDDEV({COLUMN_NAME}::FLOAT)/{BASELINE_SD}::FLOAT', '>', '{THRESHOLD_VALUE}'), + ('2032', 'Variability_Decrease', 'snowflake', '100.0*STDDEV({COLUMN_NAME}::FLOAT)/{BASELINE_SD}::FLOAT', '<', '{THRESHOLD_VALUE}'), + ('3031', 'Variability_Increase', 'mssql', '100.0*STDEV(CAST({COLUMN_NAME} AS FLOAT))/CAST({BASELINE_SD} AS FLOAT)', '>', '{THRESHOLD_VALUE}'), + ('3032', 'Variability_Decrease', 'mssql', '100.0*STDEV(CAST({COLUMN_NAME} AS FLOAT))/CAST({BASELINE_SD} AS FLOAT)', '<', '{THRESHOLD_VALUE}'), + ('4031', 'Variability_Increase', 'postgresql', '100.0*STDDEV({COLUMN_NAME}::FLOAT)/{BASELINE_SD}::FLOAT', '>', '{THRESHOLD_VALUE}'), + ('4032', 'Variability_Decrease', 'postgresql', '100.0*STDDEV({COLUMN_NAME}::FLOAT)/{BASELINE_SD}::FLOAT', '<', '{THRESHOLD_VALUE}'), + ('6031', 'Variability_Increase', 'databricks', '100.0*STDDEV_SAMP({COLUMN_NAME}::FLOAT)/{BASELINE_SD}::FLOAT', '>', '{THRESHOLD_VALUE}'), + ('6032', 'Variability_Decrease', 'databricks', '100.0*STDDEV_SAMP({COLUMN_NAME}::FLOAT)/{BASELINE_SD}::FLOAT', '<', '{THRESHOLD_VALUE}'), ('5001', 'Alpha_Trunc', 'trino', 'MAX(LENGTH({COLUMN_NAME}))', '<', '{THRESHOLD_VALUE}'), ('5002', 'Avg_Shift', 'trino', 'ABS( (CAST(AVG({COLUMN_NAME} AS REAL)) - {BASELINE_AVG}) / SQRT(((CAST(COUNT({COLUMN_NAME}) AS REAL)-1)*STDDEV({COLUMN_NAME})^2 + (CAST({BASELINE_VALUE_CT} AS REAL)-1) * CAST({BASELINE_SD} AS REAL)^2) /NULLIF(CAST(COUNT({COLUMN_NAME}) AS REAL) + CAST({BASELINE_VALUE_CT} AS REAL), 0) ))', '>=', '{THRESHOLD_VALUE}'), @@ -472,7 +472,7 @@ INSERT INTO target_data_lookups (id, test_id, error_type, test_type, sql_flavor, lookup_type, lookup_query) VALUES ('1001', '1004', 'Test Results', 'Alpha_Trunc', 'redshift', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", LEN("{COLUMN_NAME}") as current_max_length, {THRESHOLD_VALUE} as previous_max_length FROM {TARGET_SCHEMA}.{TABLE_NAME}, (SELECT MAX(LEN("{COLUMN_NAME}")) as max_length FROM {TARGET_SCHEMA}.{TABLE_NAME}) a WHERE LEN("{COLUMN_NAME}") = a.max_length AND a.max_length < {THRESHOLD_VALUE} LIMIT 500;'), - ('1002', '1005', 'Test Results', 'Avg_Shift', 'redshift', NULL, 'SELECT AVG("{COLUMN_NAME}" :: FLOAT) AS current_average FROM {TARGET_SCHEMA}.{TABLE_NAME};'), + ('1002', '1005', 'Test Results', 'Avg_Shift', 'redshift', NULL, 'SELECT AVG("{COLUMN_NAME}"::FLOAT) AS current_average FROM {TARGET_SCHEMA}.{TABLE_NAME};'), ('1003', '1006', 'Test Results', 'Condition_Flag', 'redshift', NULL, 'SELECT * FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE {CUSTOM_QUERY} LIMIT 500;'), ('1004', '1007', 'Test Results', 'Constant', 'redshift', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE "{COLUMN_NAME}" <> {BASELINE_VALUE} GROUP BY "{COLUMN_NAME}" LIMIT 500;'), ('1005', '1009', 'Test Results', 'Daily_Record_Ct', 'redshift', NULL, 'WITH RECURSIVE daterange(all_dates) AS (SELECT MIN("{COLUMN_NAME}") :: DATE AS all_dates FROM {TARGET_SCHEMA}.{TABLE_NAME} UNION ALL SELECT DATEADD(DAY, 1, d.all_dates) :: DATE AS all_dates FROM daterange d WHERE d.all_dates < (SELECT MAX("{COLUMN_NAME}") :: DATE FROM {TARGET_SCHEMA}.{TABLE_NAME}) ), existing_periods AS ( SELECT DISTINCT "{COLUMN_NAME}" :: DATE AS period, COUNT(1) AS period_count FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY "{COLUMN_NAME}" :: DATE ) SELECT d.all_dates AS missing_period, MAX(b.period) AS prior_available_date, (SELECT period_count FROM existing_periods WHERE period = MAX(b.period) ) AS prior_available_date_count, MIN(c.period) AS next_available_date, (SELECT period_count FROM existing_periods WHERE period = MIN(c.period) ) AS next_available_date_count FROM daterange d LEFT JOIN existing_periods a ON d.all_dates = a.period LEFT JOIN existing_periods b ON b.period < d.all_dates LEFT JOIN existing_periods c ON c.period > d.all_dates WHERE a.period IS NULL AND d.all_dates BETWEEN b.period AND c.period GROUP BY d.all_dates ORDER BY d.all_dates LIMIT 500;'), @@ -501,8 +501,8 @@ VALUES ('1028', '1034', 'Test Results', 'Unique', 'redshift', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY "{COLUMN_NAME}" HAVING COUNT(*) > 1 ORDER BY COUNT(*) DESC LIMIT 500;'), ('1029', '1035', 'Test Results', 'Unique_Pct', 'redshift', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY "{COLUMN_NAME}" ORDER BY COUNT(*) DESC LIMIT 500;'), ('1030', '1037', 'Test Results', 'Weekly_Rec_Ct', 'redshift', NULL, 'WITH RECURSIVE daterange(all_dates) AS (SELECT DATE_TRUNC(''week'',MIN("{COLUMN_NAME}")) :: DATE AS all_dates FROM {TARGET_SCHEMA}.{TABLE_NAME} UNION ALL SELECT (d.all_dates + INTERVAL ''1 week'' ) :: DATE AS all_dates FROM daterange d WHERE d.all_dates < (SELECT DATE_TRUNC(''week'', MAX("{COLUMN_NAME}")) :: DATE FROM {TARGET_SCHEMA}.{TABLE_NAME}) ), existing_periods AS ( SELECT DISTINCT DATE_TRUNC(''week'',"{COLUMN_NAME}") :: DATE AS period, COUNT(1) as period_count FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY DATE_TRUNC(''week'',"{COLUMN_NAME}") :: DATE ) SELECT d.all_dates as missing_period, MAX(b.period) AS prior_available_week, (SELECT period_count FROM existing_periods WHERE period = MAX(b.period) ) AS prior_available_week_count, MIN(c.period) AS next_available_week, (SELECT period_count FROM existing_periods WHERE period = MIN(c.period) ) AS next_available_week_count FROM daterange d LEFT JOIN existing_periods a ON d.all_dates = a.period LEFT JOIN existing_periods b ON b.period < d.all_dates LEFT JOIN existing_periods c ON c.period > d.all_dates WHERE a.period IS NULL AND d.all_dates BETWEEN b.period AND c.period GROUP BY d.all_dates ORDER BY d.all_dates;'), - ('1031', '1040', 'Test Results', 'Variability_Increase', 'redshift', NULL, 'SELECT STDDEV(CAST("{COLUMN_NAME}" AS FLOAT)) as current_standard_deviation FROM {TARGET_SCHEMA}.{TABLE_NAME};'), - ('1032', '1041', 'Test Results', 'Variability_Decrease', 'redshift', NULL, 'SELECT STDDEV(CAST("{COLUMN_NAME}" AS FLOAT)) as current_standard_deviation FROM {TARGET_SCHEMA}.{TABLE_NAME};'), + ('1031', '1040', 'Test Results', 'Variability_Increase', 'redshift', NULL, 'SELECT STDDEV("{COLUMN_NAME}"::FLOAT) as current_standard_deviation FROM {TARGET_SCHEMA}.{TABLE_NAME};'), + ('1032', '1041', 'Test Results', 'Variability_Decrease', 'redshift', NULL, 'SELECT STDDEV("{COLUMN_NAME}"::FLOAT) as current_standard_deviation FROM {TARGET_SCHEMA}.{TABLE_NAME};'), ('1033', '1001', 'Profile Anomaly' , 'Suggested_Type', 'redshift', NULL, 'SELECT TOP 20 "{COLUMN_NAME}", COUNT(*) AS record_ct FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY "{COLUMN_NAME}" ORDER BY record_ct DESC;'), ('1034', '1002', 'Profile Anomaly', 'Non_Standard_Blanks', 'redshift', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE CASE WHEN "{COLUMN_NAME}" IN (''.'', ''?'', '' '') THEN 1 WHEN LOWER("{COLUMN_NAME}") SIMILAR TO ''(^.{2,}|-{2,}|0{2,}|9{2,}|x{2,}|z{2,}$)'' THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''blank'',''error'',''missing'',''tbd'', ''n/a'',''#na'',''none'',''null'',''unknown'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''(blank)'',''(error)'',''(missing)'',''(tbd)'', ''(n/a)'',''(#na)'',''(none)'',''(null)'',''(unknown)'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''[blank]'',''[error]'',''[missing]'',''[tbd]'', ''[n/a]'',''[#na]'',''[none]'',''[null]'',''[unknown]'') THEN 1 WHEN "{COLUMN_NAME}" = '''' THEN 1 WHEN "{COLUMN_NAME}" IS NULL THEN 1 ELSE 0 END = 1 GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}";'), diff --git a/testgen/template/flavors/databricks/gen_query_tests/gen_table_changed_test.sql b/testgen/template/flavors/databricks/gen_query_tests/gen_table_changed_test.sql index b8c4d2bc..1c6521bc 100644 --- a/testgen/template/flavors/databricks/gen_query_tests/gen_table_changed_test.sql +++ b/testgen/template/flavors/databricks/gen_query_tests/gen_table_changed_test.sql @@ -121,7 +121,7 @@ newtests CASE WHEN general_type = 'D' THEN 'MIN(@@@)::STRING || ''|'' || MAX(@@@::STRING) || ''|'' || COUNT(DISTINCT @@@)::STRING' WHEN general_type = 'A' THEN 'MIN(@@@)::STRING || ''|'' || MAX(@@@::STRING) || ''|'' || COUNT(DISTINCT @@@)::STRING || ''|'' || SUM(LENGTH(@@@))::STRING' - WHEN general_type = 'N' THEN 'MIN(@@@)::STRING || ''|'' || MAX(@@@::STRING) || ''|'' || SUM(@@@)::STRING || ''|'' || ROUND(AVG(@@@), 5)::STRING || ''|'' || ROUND(STDDEV(@@@), 5)::STRING' + WHEN general_type = 'N' THEN 'MIN(@@@)::STRING || ''|'' || MAX(@@@::STRING) || ''|'' || SUM(@@@)::STRING || ''|'' || ROUND(AVG(@@@), 5)::STRING || ''|'' || ROUND(STDDEV(@@@::FLOAT), 5)::STRING' END, '@@@', '"' || column_name || '"'), ' || ''|'' || ' diff --git a/testgen/template/flavors/mssql/exec_query_tests/ex_table_changed_mssql.sql b/testgen/template/flavors/mssql/exec_query_tests/ex_table_changed_mssql.sql new file mode 100644 index 00000000..5189c442 --- /dev/null +++ b/testgen/template/flavors/mssql/exec_query_tests/ex_table_changed_mssql.sql @@ -0,0 +1,33 @@ +SELECT '{TEST_TYPE}' as test_type, + '{TEST_DEFINITION_ID}' as test_definition_id, + '{TEST_SUITE_ID}' as test_suite_id, + '{TEST_RUN_ID}' as test_run_id, + '{RUN_DATE}' as test_time, + '{START_TIME}' as starttime, + CURRENT_TIMESTAMP as endtime, + '{SCHEMA_NAME}' as schema_name, + '{TABLE_NAME}' as table_name, + '{COLUMN_NAME_NO_QUOTES}' as column_names, + '{SKIP_ERRORS}' as threshold_value, + {SKIP_ERRORS} as skip_errors, + '{INPUT_PARAMETERS}' as input_parameters, + fingerprint as result_signal, + /* Fails if table is the same */ + CASE WHEN fingerprint = '{BASELINE_VALUE}' THEN 0 ELSE 1 END as result_code, + + CASE + WHEN fingerprint = '{BASELINE_VALUE}' + THEN 'No table change detected.' + ELSE 'Table change detected.' + END AS result_message, + CASE + WHEN fingerprint = '{BASELINE_VALUE}' + THEN 0 + ELSE 1 + END as result_measure, + '{SUBSET_DISPLAY}' as subset_condition, + NULL as result_query + FROM ( SELECT {CUSTOM_QUERY} as fingerprint + FROM {SCHEMA_NAME}.{TABLE_NAME} WITH (NOLOCK) + WHERE {SUBSET_CONDITION} + ) test; diff --git a/testgen/template/flavors/mssql/gen_query_tests/gen_table_changed_test.sql b/testgen/template/flavors/mssql/gen_query_tests/gen_table_changed_test.sql index e9517725..4ed918a2 100644 --- a/testgen/template/flavors/mssql/gen_query_tests/gen_table_changed_test.sql +++ b/testgen/template/flavors/mssql/gen_query_tests/gen_table_changed_test.sql @@ -121,9 +121,9 @@ newtests AS ( 'CAST(COUNT(*) AS varchar) + ''|'' + ' || STRING_AGG( REPLACE( CASE - WHEN general_type = 'D' THEN 'CAST(MIN(@@@) AS varchar) + ''|'' + MAX(CAST(@@@ AS varchar)) + ''|'' + CAST(COUNT(DISTINCT @@@) AS varchar)' - WHEN general_type = 'A' THEN 'CAST(MIN(@@@) AS varchar) + ''|'' + MAX(CAST(@@@ AS varchar)) + ''|'' + CAST(COUNT(DISTINCT @@@) AS varchar) + ''|'' + CAST(SUM(LEN(@@@)) AS varchar)' - WHEN general_type = 'N' THEN 'CAST(MIN(@@@) AS varchar) + ''|'' + MAX(CAST(@@@ AS varchar)) + ''|'' + CAST(SUM(@@@) AS varchar) + ''|'' + CAST(ROUND(AVG(@@@), 5) AS varchar) + ''|'' + CAST(ROUND(STDEV(@@@), 5) AS varchar)' + WHEN general_type = 'D' THEN 'CAST(MIN(@@@) AS NVARCHAR) + ''|'' + MAX(CAST(@@@ AS NVARCHAR)) + ''|'' + CAST(COUNT(DISTINCT @@@) AS NVARCHAR)' + WHEN general_type = 'A' THEN 'CAST(MIN(@@@) AS NVARCHAR) + ''|'' + MAX(CAST(@@@ AS NVARCHAR)) + ''|'' + CAST(COUNT(DISTINCT @@@) AS NVARCHAR) + ''|'' + CAST(SUM(LEN(@@@)) AS NVARCHAR)' + WHEN general_type = 'N' THEN 'CAST(MIN(@@@) AS NVARCHAR) + ''|'' + MAX(CAST(@@@ AS NVARCHAR)) + ''|'' + CAST(SUM(@@@) AS NVARCHAR) + ''|'' + CAST(ROUND(AVG(@@@), 5) AS NVARCHAR) + ''|'' + CAST(ROUND(STDEV(CAST(@@@ AS FLOAT)), 5) AS NVARCHAR)' END, '@@@', '"' || column_name || '"' ), diff --git a/testgen/template/flavors/postgresql/gen_query_tests/gen_table_changed_test.sql b/testgen/template/flavors/postgresql/gen_query_tests/gen_table_changed_test.sql new file mode 100644 index 00000000..fd3fe0a1 --- /dev/null +++ b/testgen/template/flavors/postgresql/gen_query_tests/gen_table_changed_test.sql @@ -0,0 +1,157 @@ +INSERT INTO test_definitions (table_groups_id, profile_run_id, test_type, test_suite_id, + schema_name, table_name, + skip_errors, test_active, last_auto_gen_date, profiling_as_of_date, + lock_refresh, history_calculation, history_lookback, custom_query ) +WITH last_run AS (SELECT r.table_groups_id, MAX(run_date) AS last_run_date + FROM profile_results p + INNER JOIN profiling_runs r + ON (p.profile_run_id = r.id) + INNER JOIN test_suites ts + ON p.project_code = ts.project_code + AND p.connection_id = ts.connection_id + WHERE p.project_code = '{PROJECT_CODE}' + AND r.table_groups_id = '{TABLE_GROUPS_ID}'::UUID + AND ts.id = '{TEST_SUITE_ID}' + AND p.run_date::DATE <= '{AS_OF_DATE}' + GROUP BY r.table_groups_id), +curprof AS (SELECT p.profile_run_id, schema_name, table_name, column_name, functional_data_type, general_type, + distinct_value_ct, record_ct, max_value, min_value, avg_value, stdev_value, null_value_ct + FROM last_run lr + INNER JOIN profile_results p + ON (lr.table_groups_id = p.table_groups_id + AND lr.last_run_date = p.run_date) ), +locked AS (SELECT schema_name, table_name + FROM test_definitions + WHERE table_groups_id = '{TABLE_GROUPS_ID}'::UUID + AND test_suite_id = '{TEST_SUITE_ID}' + AND test_type = 'Table_Freshness' + AND lock_refresh = 'Y'), +-- IDs - TOP 2 +id_cols + AS ( SELECT profile_run_id, schema_name, table_name, column_name, functional_data_type, general_type, + distinct_value_ct, + ROW_NUMBER() OVER (PARTITION BY schema_name, table_name + ORDER BY + CASE + WHEN functional_data_type ILIKE 'ID-Unique%' THEN 1 + WHEN functional_data_type = 'ID-Secondary' THEN 2 + ELSE 3 + END, distinct_value_ct, column_name DESC) AS rank + FROM curprof + WHERE general_type IN ('A', 'D', 'N') + AND functional_data_type ILIKE 'ID%'), +-- Process Date - TOP 1 +process_date_cols + AS (SELECT profile_run_id, schema_name, table_name, column_name, functional_data_type, general_type, + distinct_value_ct, + ROW_NUMBER() OVER (PARTITION BY schema_name, table_name + ORDER BY + CASE + WHEN column_name ILIKE '%mod%' THEN 1 + WHEN column_name ILIKE '%up%' THEN 1 + WHEN column_name ILIKE '%cr%' THEN 2 + WHEN column_name ILIKE '%in%' THEN 2 + END , distinct_value_ct DESC, column_name) AS rank + FROM curprof + WHERE general_type IN ('A', 'D', 'N') + AND functional_data_type ILIKE 'process%'), +-- Transaction Date - TOP 1 +tran_date_cols + AS ( SELECT profile_run_id, schema_name, table_name, column_name, functional_data_type, general_type, + distinct_value_ct, + ROW_NUMBER() OVER (PARTITION BY schema_name, table_name + ORDER BY + distinct_value_ct DESC, column_name) AS rank + FROM curprof + WHERE general_type IN ('A', 'D', 'N') + AND functional_data_type ILIKE 'transactional date%' + OR functional_data_type ILIKE 'period%' + OR functional_data_type = 'timestamp' ), + +-- Numeric Measures +numeric_cols + AS ( SELECT profile_run_id, schema_name, table_name, column_name, functional_data_type, general_type, +/* + -- Subscores + distinct_value_ct * 1.0 / NULLIF(record_ct, 0) AS cardinality_score, + (max_value - min_value) / NULLIF(ABS(NULLIF(avg_value, 0)), 1) AS range_score, + LEAST(1, LOG(GREATEST(distinct_value_ct, 2))) / LOG(GREATEST(record_ct, 2)) AS nontriviality_score, + stdev_value / NULLIF(ABS(NULLIF(avg_value, 0)), 1) AS variability_score, + 1.0 - (null_value_ct * 1.0 / NULLIF(NULLIF(record_ct, 0), 1)) AS null_penalty, +*/ + -- Weighted score + ( + 0.25 * (distinct_value_ct * 1.0 / NULLIF(record_ct, 0)) + + 0.15 * ((max_value - min_value) / NULLIF(ABS(NULLIF(avg_value, 0)), 1)) + + 0.10 * (LEAST(1, LOG(GREATEST(distinct_value_ct, 2))) / LOG(GREATEST(record_ct, 2))) + + 0.40 * (stdev_value / NULLIF(ABS(NULLIF(avg_value, 0)), 1)) + + 0.10 * (1.0 - (null_value_ct * 1.0 / NULLIF(NULLIF(record_ct, 0), 1))) + ) AS change_detection_score + FROM curprof + WHERE general_type = 'N' + AND (functional_data_type ILIKE 'Measure%' OR functional_data_type IN ('Sequence', 'Constant')) + ), +numeric_cols_ranked + AS ( SELECT *, + ROW_NUMBER() OVER (PARTITION BY schema_name, table_name + ORDER BY change_detection_score DESC, column_name) as rank + FROM numeric_cols + WHERE change_detection_score IS NOT NULL), +combined + AS ( SELECT profile_run_id, schema_name, table_name, column_name, 'ID' AS element_type, general_type, 10 + rank AS fingerprint_order + FROM id_cols + WHERE rank <= 2 + UNION ALL + SELECT profile_run_id, schema_name, table_name, column_name, 'DATE_P' AS element_type, general_type, 20 + rank AS fingerprint_order + FROM process_date_cols + WHERE rank = 1 + UNION ALL + SELECT profile_run_id, schema_name, table_name, column_name, 'DATE_T' AS element_type, general_type, 30 + rank AS fingerprint_order + FROM tran_date_cols + WHERE rank = 1 + UNION ALL + SELECT profile_run_id, schema_name, table_name, column_name, 'MEAS' AS element_type, general_type, 40 + rank AS fingerprint_order + FROM numeric_cols_ranked + WHERE rank = 1 ), +newtests + AS (SELECT profile_run_id, schema_name, table_name, + 'COUNT(*)::VARCHAR || ''|'' || ' || + STRING_AGG( + REPLACE( + CASE + WHEN general_type = 'D' THEN 'MIN(@@@)::VARCHAR || ''|'' || MAX(@@@::VARCHAR) || ''|'' || COUNT(DISTINCT @@@)::VARCHAR' + WHEN general_type = 'A' THEN 'MIN(@@@)::VARCHAR || ''|'' || MAX(@@@::VARCHAR) || ''|'' || COUNT(DISTINCT @@@)::VARCHAR || ''|'' || SUM(LENGTH(@@@))::VARCHAR' + WHEN general_type = 'N' THEN 'MIN(@@@)::VARCHAR || ''|'' || MAX(@@@::VARCHAR) || ''|'' || SUM(@@@)::VARCHAR || ''|'' || ROUND(AVG(@@@), 5)::VARCHAR || ''|'' || ROUND(STDDEV(@@@::FLOAT)::NUMERIC, 5)::VARCHAR' + END, + '@@@', '"' || column_name || '"'), + ' || ''|'' || ' + ORDER BY element_type, fingerprint_order, column_name) as fingerprint + FROM combined + GROUP BY profile_run_id, schema_name, table_name) +SELECT '{TABLE_GROUPS_ID}'::UUID as table_groups_id, + n.profile_run_id, + 'Table_Freshness' AS test_type, + '{TEST_SUITE_ID}' AS test_suite_id, + n.schema_name, n.table_name, + 0 as skip_errors, 'Y' as test_active, + + '{RUN_DATE}'::TIMESTAMP as last_auto_gen_date, + '{AS_OF_DATE}'::TIMESTAMP as profiling_as_of_date, + 'N' as lock_refresh, + 'Value' as history_calculation, + 1 as history_lookback, + fingerprint as custom_query +FROM newtests n +INNER JOIN test_types t + ON ('Table_Freshness' = t.test_type + AND 'Y' = t.active) +LEFT JOIN generation_sets s + ON (t.test_type = s.test_type + AND '{GENERATION_SET}' = s.generation_set) +LEFT JOIN locked l + ON (n.schema_name = l.schema_name + AND n.table_name = l.table_name) +WHERE (s.generation_set IS NOT NULL + OR '{GENERATION_SET}' = '') + AND l.schema_name IS NULL; + diff --git a/testgen/template/gen_query_tests/gen_table_changed_test.sql b/testgen/template/gen_query_tests/gen_table_changed_test.sql index 7779652e..918af282 100644 --- a/testgen/template/gen_query_tests/gen_table_changed_test.sql +++ b/testgen/template/gen_query_tests/gen_table_changed_test.sql @@ -121,7 +121,7 @@ newtests CASE WHEN general_type = 'D' THEN 'MIN(@@@)::VARCHAR || ''|'' || MAX(@@@::VARCHAR) || ''|'' || COUNT(DISTINCT @@@)::VARCHAR' WHEN general_type = 'A' THEN 'MIN(@@@)::VARCHAR || ''|'' || MAX(@@@::VARCHAR) || ''|'' || COUNT(DISTINCT @@@)::VARCHAR || ''|'' || SUM(LENGTH(@@@))::VARCHAR' - WHEN general_type = 'N' THEN 'MIN(@@@)::VARCHAR || ''|'' || MAX(@@@::VARCHAR) || ''|'' || SUM(@@@)::VARCHAR || ''|'' || ROUND(AVG(@@@), 5)::VARCHAR || ''|'' || ROUND(STDDEV(@@@), 5)::VARCHAR' + WHEN general_type = 'N' THEN 'MIN(@@@)::VARCHAR || ''|'' || MAX(@@@::VARCHAR) || ''|'' || SUM(@@@)::VARCHAR || ''|'' || ROUND(AVG(@@@), 5)::VARCHAR || ''|'' || ROUND(STDDEV(@@@::FLOAT), 5)::VARCHAR' END, '@@@', '"' || column_name || '"'), ' || ''|'' || ' From 0ffba6dc0bdd11fcb565bc9927dc7128439f46b6 Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Wed, 10 Sep 2025 13:16:01 -0400 Subject: [PATCH 19/28] fix(general types): update schema ddf queries --- .../schema_ddf_query_databricks.sql | 9 ++++----- .../data_chars/schema_ddf_query_mssql.sql | 15 +++++++++------ .../schema_ddf_query_postgresql.sql | 6 +++--- .../data_chars/schema_ddf_query_redshift.sql | 14 +++++++------- .../data_chars/schema_ddf_query_snowflake.sql | 19 ++++++++++--------- 5 files changed, 33 insertions(+), 30 deletions(-) diff --git a/testgen/template/flavors/databricks/data_chars/schema_ddf_query_databricks.sql b/testgen/template/flavors/databricks/data_chars/schema_ddf_query_databricks.sql index c486d94a..0f3e4dd4 100644 --- a/testgen/template/flavors/databricks/data_chars/schema_ddf_query_databricks.sql +++ b/testgen/template/flavors/databricks/data_chars/schema_ddf_query_databricks.sql @@ -13,11 +13,10 @@ SELECT '{PROJECT_CODE}' AS project_code, c.character_maximum_length, c.ordinal_position, CASE - WHEN lower(c.data_type) RLIKE '(string|char|varchar|text)' THEN 'A' - WHEN lower(c.data_type) = 'boolean' THEN 'B' - WHEN lower(c.data_type) IN ('date', 'timestamp') THEN 'D' - WHEN lower(c.data_type) IN ('byte', 'short', 'int', 'integer', 'long', 'bigint', 'float', 'double') THEN 'N' - WHEN lower(c.data_type) LIKE 'decimal%' THEN 'N' + WHEN c.data_type IN ('STRING', 'CHAR') THEN 'A' + WHEN c.data_type = 'BOOLEAN' THEN 'B' + WHEN c.data_type IN ('DATE', 'TIMESTAMP', 'TIMESTAMP_NTZ') THEN 'D' + WHEN c.data_type IN ('BYTE', 'SHORT', 'INT', 'LONG', 'DECIMAL', 'FLOAT', 'DOUBLE') THEN 'N' ELSE 'X' END AS general_type, CASE diff --git a/testgen/template/flavors/mssql/data_chars/schema_ddf_query_mssql.sql b/testgen/template/flavors/mssql/data_chars/schema_ddf_query_mssql.sql index 9fc43c5a..44d659ca 100644 --- a/testgen/template/flavors/mssql/data_chars/schema_ddf_query_mssql.sql +++ b/testgen/template/flavors/mssql/data_chars/schema_ddf_query_mssql.sql @@ -16,20 +16,23 @@ SELECT '{PROJECT_CODE}' as project_code, c.ordinal_position, CASE WHEN LOWER(c.data_type) LIKE '%char%' + OR c.data_type LIKE '%text%' THEN 'A' WHEN c.data_type = 'bit' THEN 'B' WHEN c.data_type = 'date' - OR c.data_type LIKE 'datetime%' + OR c.data_type LIKE '%datetime%' THEN 'D' - WHEN c.data_type like 'time%' + WHEN c.data_type = 'time' THEN 'T' - WHEN c.data_type IN ('bigint', 'double precision', 'integer', 'smallint', 'real') - OR c.data_type LIKE 'numeric%' + WHEN c.data_type IN ('real', 'float', 'decimal', 'numeric') + OR c.data_type LIKE '%int' + OR c.data_type LIKE '%money' THEN 'N' ELSE - 'X' END AS general_type, - case when c.numeric_scale > 0 then 1 else 0 END as is_decimal + 'X' + END AS general_type, + CASE WHEN c.numeric_scale > 0 THEN 1 ELSE 0 END AS is_decimal FROM information_schema.columns c WHERE c.table_schema = '{DATA_SCHEMA}' {TABLE_CRITERIA} ORDER BY c.table_schema, c.table_name, c.ordinal_position; diff --git a/testgen/template/flavors/postgresql/data_chars/schema_ddf_query_postgresql.sql b/testgen/template/flavors/postgresql/data_chars/schema_ddf_query_postgresql.sql index 5f2d7805..5e3136ca 100644 --- a/testgen/template/flavors/postgresql/data_chars/schema_ddf_query_postgresql.sql +++ b/testgen/template/flavors/postgresql/data_chars/schema_ddf_query_postgresql.sql @@ -28,11 +28,11 @@ SELECT '{PROJECT_CODE}' as project_code, THEN 'D' WHEN c.data_type ILIKE 'time without time zone' THEN 'T' - WHEN LOWER(c.data_type) IN ('bigint', 'double precision', 'integer', 'smallint', 'real') - OR c.data_type ILIKE 'numeric%' + WHEN LOWER(c.data_type) IN ('bigint', 'integer', 'smallint', 'double precision', 'real', 'numeric', 'money') THEN 'N' ELSE - 'X' END AS general_type, + 'X' + END AS general_type, CASE WHEN c.data_type = 'numeric' THEN COALESCE(numeric_scale, 1) > 0 ELSE numeric_scale > 0 diff --git a/testgen/template/flavors/redshift/data_chars/schema_ddf_query_redshift.sql b/testgen/template/flavors/redshift/data_chars/schema_ddf_query_redshift.sql index 0ba198b5..aa081ff5 100644 --- a/testgen/template/flavors/redshift/data_chars/schema_ddf_query_redshift.sql +++ b/testgen/template/flavors/redshift/data_chars/schema_ddf_query_redshift.sql @@ -15,24 +15,24 @@ SELECT '{PROJECT_CODE}' as project_code, c.character_maximum_length, c.ordinal_position, CASE - WHEN c.data_type ILIKE '%char%' + WHEN c.data_type ILIKE 'char%' THEN 'A' - WHEN c.data_type ILIKE 'boolean' + WHEN c.data_type = 'boolean' THEN 'B' WHEN c.data_type ILIKE 'date' OR c.data_type ILIKE 'timestamp%' THEN 'D' - WHEN c.data_type ILIKE 'time without time zone' + WHEN c.data_type ILIKE 'time with%' THEN 'T' - WHEN LOWER(c.data_type) IN ('bigint', 'double precision', 'integer', 'smallint', 'real') - OR c.data_type ILIKE 'numeric%' + WHEN LOWER(c.data_type) IN ('bigint', 'integer', 'smallint', 'double precision', 'real', 'numeric') THEN 'N' ELSE - 'X' END AS general_type, + 'X' + END AS general_type, CASE WHEN c.data_type = 'numeric' THEN COALESCE(numeric_scale, 1) > 0 ELSE numeric_scale > 0 - END as is_decimal + END AS is_decimal FROM information_schema.columns c WHERE c.table_schema = '{DATA_SCHEMA}' {TABLE_CRITERIA} ORDER BY c.table_schema, c.table_name, c.ordinal_position diff --git a/testgen/template/flavors/snowflake/data_chars/schema_ddf_query_snowflake.sql b/testgen/template/flavors/snowflake/data_chars/schema_ddf_query_snowflake.sql index 4ebd68f1..b2fa5a4f 100644 --- a/testgen/template/flavors/snowflake/data_chars/schema_ddf_query_snowflake.sql +++ b/testgen/template/flavors/snowflake/data_chars/schema_ddf_query_snowflake.sql @@ -18,21 +18,22 @@ SELECT '{PROJECT_CODE}' as project_code, c.character_maximum_length, c.ordinal_position, CASE - WHEN c.data_type ILIKE '%char%' OR c.data_type = 'TEXT' + WHEN c.data_type = 'TEXT' THEN 'A' - WHEN c.data_type ILIKE 'boolean' + WHEN c.data_type = 'BOOLEAN' THEN 'B' - WHEN c.data_type ILIKE 'date' - OR c.data_type ILIKE 'timestamp%' + WHEN c.data_type = 'DATE' + OR c.data_type ILIKE 'TIMESTAMP%' THEN 'D' - WHEN c.data_type = 'time without time zone' + WHEN c.data_type = 'TIME' THEN 'T' - WHEN lower(c.data_type) IN ('bigint', 'double precision', 'integer', 'smallint', 'real', 'float') - OR c.data_type ILIKE 'num%' + WHEN c.data_type = 'NUMBER' + OR c.data_type = 'FLOAT' THEN 'N' ELSE - 'X' END AS general_type, - numeric_scale > 0 as is_decimal + 'X' + END AS general_type, + numeric_scale > 0 AS is_decimal FROM information_schema.columns c WHERE c.table_schema = '{DATA_SCHEMA}' {TABLE_CRITERIA} ORDER BY c.table_schema, c.table_name, c.ordinal_position; From b30bc62dcb2d385d8cb9986644ad01f27cae6289 Mon Sep 17 00:00:00 2001 From: Ricardo Boni Date: Wed, 10 Sep 2025 16:46:01 -0400 Subject: [PATCH 20/28] misc: Allow re-using the existing users and roles --- testgen/commands/run_launch_db_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgen/commands/run_launch_db_config.py b/testgen/commands/run_launch_db_config.py index 71e3c650..68f99336 100644 --- a/testgen/commands/run_launch_db_config.py +++ b/testgen/commands/run_launch_db_config.py @@ -72,10 +72,10 @@ def _get_params_mapping() -> dict: @with_database_session -def run_launch_db_config(delete_db: bool) -> None: +def run_launch_db_config(delete_db: bool, drop_users_and_roles: bool = True) -> None: params_mapping = _get_params_mapping() - create_database(get_tg_db(), params_mapping, drop_existing=delete_db, drop_users_and_roles=True) + create_database(get_tg_db(), params_mapping, drop_existing=delete_db, drop_users_and_roles=drop_users_and_roles) queries = get_queries_for_command("dbsetup", params_mapping) From 97fb07a8c18e79b3bf85259ae01ecac7f5310558 Mon Sep 17 00:00:00 2001 From: Luis Date: Fri, 5 Sep 2025 16:58:10 -0400 Subject: [PATCH 21/28] feat(tests): generate a monitor suite for new table groups --- testgen/commands/run_execute_tests.py | 5 ++- testgen/commands/run_profiling_bridge.py | 23 ++++++++++ testgen/common/get_pipeline_parms.py | 2 + testgen/common/models/scheduler.py | 25 ++++++++++- testgen/common/models/table_group.py | 44 ++++++++++++++++++- testgen/common/models/test_suite.py | 1 + testgen/scheduler/cli_scheduler.py | 2 +- .../030_initialize_new_schema_structure.sql | 15 ++++--- .../dbupgrade/0150_incremental_upgrade.sql | 10 +++++ testgen/template/parms/parms_profiling.sql | 11 ++++- .../js/components/table_group_form.js | 4 ++ testgen/ui/views/connections.py | 7 ++- testgen/ui/views/profiling_runs.py | 3 +- testgen/ui/views/table_groups.py | 9 +++- testgen/ui/views/test_runs.py | 3 +- tests/unit/test_scheduler_cli.py | 2 +- 16 files changed, 150 insertions(+), 16 deletions(-) create mode 100644 testgen/template/dbupgrade/0150_incremental_upgrade.sql diff --git a/testgen/commands/run_execute_tests.py b/testgen/commands/run_execute_tests.py index 4e2996fb..af9dd9fd 100644 --- a/testgen/commands/run_execute_tests.py +++ b/testgen/commands/run_execute_tests.py @@ -125,9 +125,12 @@ def run_execution_steps_in_background(project_code, test_suite): if settings.IS_DEBUG: LOG.info(msg + ". Running in debug mode (new thread instead of new process).") empty_cache() + username = None + if session.auth: + username = session.auth.user_display background_thread = threading.Thread( target=run_execution_steps, - args=(project_code, test_suite, session.auth.user_display), + args=(project_code, test_suite, username), ) background_thread.start() else: diff --git a/testgen/commands/run_profiling_bridge.py b/testgen/commands/run_profiling_bridge.py index 78aa8a56..723e5b5f 100644 --- a/testgen/commands/run_profiling_bridge.py +++ b/testgen/commands/run_profiling_bridge.py @@ -10,6 +10,8 @@ import testgen.common.process_service as process_service from testgen import settings from testgen.commands.queries.profiling_query import CProfilingSQL +from testgen.commands.run_execute_tests import run_execution_steps_in_background +from testgen.commands.run_generate_tests import run_test_gen_queries from testgen.commands.run_refresh_score_cards_results import run_refresh_score_cards_results from testgen.common import ( date_service, @@ -25,6 +27,7 @@ from testgen.common.mixpanel_service import MixpanelService from testgen.common.models import with_database_session from testgen.common.models.connection import Connection +from testgen.common.models.test_suite import TestSuite from testgen.ui.session import session LOG = logging.getLogger("testgen") @@ -238,6 +241,9 @@ def run_profiling_queries(table_group_id: str, username: str | None = None, spin profiling_run_id = str(uuid.uuid4()) params = get_profiling_params(table_group_id) + needs_monitor_tests_generated = ( + bool(params["monitor_test_suite_id"]) and not params["last_complete_profile_run_id"] + ) LOG.info("CurrentStep: Initializing Query Generator") clsProfiling = CProfilingSQL(params["project_code"], connection.sql_flavor, minutes_offset=minutes_offset) @@ -471,7 +477,24 @@ def run_profiling_queries(table_group_id: str, username: str | None = None, spin scoring_duration=(datetime.now(UTC) - end_time).total_seconds(), ) + if needs_monitor_tests_generated: + _generate_monitor_tests(params["project_code"], table_group_id, params["monitor_test_suite_id"]) + return f""" Profiling completed {"with errors. Check log for details." if has_errors else "successfully."} Run ID: {profiling_run_id} """ + + +@with_database_session +def _generate_monitor_tests(project_code: str, table_group_id: str, test_suite_id: str) -> None: + try: + monitor_test_suite = TestSuite.get(test_suite_id) + if not monitor_test_suite: + LOG.info("Skipping test generation on missing monitor test suite") + else: + LOG.info("Generating monitor tests") + run_test_gen_queries(table_group_id, monitor_test_suite.test_suite, "Monitor") + run_execution_steps_in_background(project_code, monitor_test_suite.test_suite) + except Exception: + LOG.exception("Error generating monitor tests") diff --git a/testgen/common/get_pipeline_parms.py b/testgen/common/get_pipeline_parms.py index b651ee1d..79f5c5ed 100644 --- a/testgen/common/get_pipeline_parms.py +++ b/testgen/common/get_pipeline_parms.py @@ -21,6 +21,8 @@ class ProfilingParams(BaseParams): profile_sample_min_count: int profile_do_pair_rules: str profile_pair_rule_pct: int + monitor_test_suite_id: str | None + last_complete_profile_run_id: str | None class TestGenerationParams(BaseParams): diff --git a/testgen/common/models/scheduler.py b/testgen/common/models/scheduler.py index 12a55ffc..9f665e52 100644 --- a/testgen/common/models/scheduler.py +++ b/testgen/common/models/scheduler.py @@ -4,11 +4,16 @@ from uuid import UUID, uuid4 from cron_converter import Cron -from sqlalchemy import Column, String, select +from sqlalchemy import Column, String, func, select from sqlalchemy.dialects import postgresql from sqlalchemy.orm import InstrumentedAttribute from testgen.common.models import Base, get_current_session +from testgen.common.models.test_definition import TestDefinition +from testgen.common.models.test_suite import TestSuite + +RUN_TESTS_JOB_KEY = "run-tests" +RUN_PROFILE_JOB_KEY = "run-profile" class JobSchedule(Base): @@ -25,7 +30,23 @@ class JobSchedule(Base): @classmethod def select_where(cls, *clauses, order_by: str | InstrumentedAttribute | None = None) -> Iterable[Self]: - query = select(cls).where(*clauses).order_by(order_by) + test_definitions_count = ( + select(cls.id) + .join(TestSuite, TestSuite.test_suite == cls.kwargs["test_suite_key"].astext) + .join(TestDefinition, TestDefinition.test_suite_id == TestSuite.id) + .where(cls.key == RUN_TESTS_JOB_KEY) + .group_by(cls.id, TestSuite.test_suite) + .having(func.count(TestDefinition.id) > 0) + .subquery() + ) + test_runs_query = ( + select(cls) + .join(test_definitions_count, test_definitions_count.c.id == cls.id) + .where(*clauses) + ) + non_test_runs_query = select(cls).where(cls.key != RUN_TESTS_JOB_KEY, *clauses) + query = test_runs_query.union_all(non_test_runs_query).order_by(order_by) + return get_current_session().execute(query) @classmethod diff --git a/testgen/common/models/table_group.py b/testgen/common/models/table_group.py index 8f823305..de0282cc 100644 --- a/testgen/common/models/table_group.py +++ b/testgen/common/models/table_group.py @@ -11,6 +11,7 @@ from testgen.common.models import get_current_session from testgen.common.models.custom_types import NullIfEmptyString, YNString from testgen.common.models.entity import ENTITY_HASH_FUNCS, Entity, EntityMinimal +from testgen.common.models.scheduler import RUN_TESTS_JOB_KEY, JobSchedule from testgen.common.models.scores import ScoreDefinition from testgen.common.models.test_suite import TestSuite @@ -52,6 +53,11 @@ class TableGroup(Entity): id: UUID = Column(postgresql.UUID(as_uuid=True), primary_key=True, default=uuid4) project_code: str = Column(String, ForeignKey("projects.project_code")) connection_id: int = Column(BigInteger, ForeignKey("connections.connection_id")) + monitor_test_suite_id: UUID | None = Column( + postgresql.UUID(as_uuid=True), + ForeignKey("test_suites.id"), + default=None, + ) table_groups_name: str = Column(String) table_group_schema: str = Column(String) profiling_table_set: str = Column(NullIfEmptyString) @@ -260,7 +266,12 @@ def clear_cache(cls) -> bool: cls.select_minimal_where.clear() cls.select_summary.clear() - def save(self, add_scorecard_definition: bool = False) -> None: + def save( + self, + add_scorecard_definition: bool = False, + add_monitor_test_suite: bool = False, + monitor_schedule_timezone: str = "UTC", + ) -> None: if self.id: values = { column.key: getattr(self, column.key, None) @@ -273,7 +284,38 @@ def save(self, add_scorecard_definition: bool = False) -> None: db_session.commit() else: super().save() + db_session = get_current_session() + if add_scorecard_definition: ScoreDefinition.from_table_group(self).save() + if add_monitor_test_suite: + test_suite = TestSuite( + project_code=self.project_code, + test_suite=f"{self.table_groups_name} Monitor", + connection_id=self.connection_id, + table_groups_id=self.id, + export_to_observability=False, + dq_score_exclude=True, + view_mode="Monitor", + ) + test_suite.save() + + schedule_job = JobSchedule( + project_code=self.project_code, + key=RUN_TESTS_JOB_KEY, + cron_expr="0 * * * *", + cron_tz=monitor_schedule_timezone, + args=[], + kwargs={"project_key": self.project_code, "test_suite_key": test_suite.test_suite}, + ) + db_session.add(schedule_job) + + self.monitor_test_suite_id = test_suite.id + db_session.execute( + update(TableGroup) + .where(TableGroup.id == self.id).values(monitor_test_suite_id=test_suite.id) + ) + db_session.commit() + TableGroup.clear_cache() diff --git a/testgen/common/models/test_suite.py b/testgen/common/models/test_suite.py index 781153f7..368147a2 100644 --- a/testgen/common/models/test_suite.py +++ b/testgen/common/models/test_suite.py @@ -65,6 +65,7 @@ class TestSuite(Entity): component_name: str = Column(NullIfEmptyString) last_complete_test_run_id: UUID = Column(postgresql.UUID(as_uuid=True)) dq_score_exclude: bool = Column(Boolean, default=False) + view_mode: str | None = Column(NullIfEmptyString, default=None) _default_order_by = (asc(func.lower(test_suite)),) _minimal_columns = TestSuiteMinimal.__annotations__.keys() diff --git a/testgen/scheduler/cli_scheduler.py b/testgen/scheduler/cli_scheduler.py index 7c82809e..907a7e05 100644 --- a/testgen/scheduler/cli_scheduler.py +++ b/testgen/scheduler/cli_scheduler.py @@ -46,7 +46,7 @@ def get_jobs(self) -> Iterable[CliJob]: self.reload_timer.start() jobs = {} - for (job_model,) in JobSchedule.select_where(): + for job_model in JobSchedule.select_where(): if job_model.key not in JOB_REGISTRY: LOG.error("Job '%s' scheduled but not registered", job_model.key) continue diff --git a/testgen/template/dbsetup/030_initialize_new_schema_structure.sql b/testgen/template/dbsetup/030_initialize_new_schema_structure.sql index d2b4b022..09593c39 100644 --- a/testgen/template/dbsetup/030_initialize_new_schema_structure.sql +++ b/testgen/template/dbsetup/030_initialize_new_schema_structure.sql @@ -84,14 +84,15 @@ CREATE TABLE connections ( CREATE TABLE table_groups ( id UUID DEFAULT gen_random_uuid() - CONSTRAINT pk_tg_id + CONSTRAINT pk_tg_id PRIMARY KEY, project_code VARCHAR(30) - CONSTRAINT table_groups_projects_project_code_fk - REFERENCES projects, + CONSTRAINT table_groups_projects_project_code_fk + REFERENCES projects, connection_id BIGINT - CONSTRAINT table_groups_connections_connection_id_fk - REFERENCES connections, + CONSTRAINT table_groups_connections_connection_id_fk + REFERENCES connections, + monitor_test_suite_id UUID DEFAULT NULL, table_groups_name VARCHAR(100), table_group_schema VARCHAR(100), profiling_table_set VARCHAR(2000), @@ -161,10 +162,14 @@ CREATE TABLE test_suites ( component_name VARCHAR(100), last_complete_test_run_id UUID, dq_score_exclude BOOLEAN default FALSE, + view_mode VARCHAR(20) DEFAULT NULL, CONSTRAINT test_suites_id_pk PRIMARY KEY (id) ); +ALTER TABLE table_groups ADD CONSTRAINT table_groups_test_suites_monitor_test_suite_id_fk + FOREIGN KEY (monitor_test_suite_id) REFERENCES test_suites ON DELETE SET NULL; + CREATE TABLE test_definitions ( id UUID DEFAULT gen_random_uuid(), cat_test_id BIGINT GENERATED BY DEFAULT AS IDENTITY diff --git a/testgen/template/dbupgrade/0150_incremental_upgrade.sql b/testgen/template/dbupgrade/0150_incremental_upgrade.sql new file mode 100644 index 00000000..b7e4a129 --- /dev/null +++ b/testgen/template/dbupgrade/0150_incremental_upgrade.sql @@ -0,0 +1,10 @@ +SET SEARCH_PATH TO {SCHEMA_NAME}; + +ALTER TABLE test_suites + ADD COLUMN view_mode VARCHAR(20) DEFAULT NULL; + +ALTER TABLE table_groups + ADD COLUMN monitor_test_suite_id UUID DEFAULT NULL; + +ALTER TABLE table_groups ADD CONSTRAINT table_groups_test_suites_monitor_test_suite_id_fk + FOREIGN KEY (monitor_test_suite_id) REFERENCES test_suites ON DELETE SET NULL; diff --git a/testgen/template/parms/parms_profiling.sql b/testgen/template/parms/parms_profiling.sql index 5eec7a4c..7b98d41f 100644 --- a/testgen/template/parms/parms_profiling.sql +++ b/testgen/template/parms/parms_profiling.sql @@ -1,6 +1,7 @@ SELECT tg.project_code, tg.id::VARCHAR(50) as table_groups_id, tg.table_group_schema, + tg.table_group_schema, CASE WHEN tg.profiling_table_set ILIKE '''%''' THEN tg.profiling_table_set ELSE fn_format_csv_quotes(tg.profiling_table_set) @@ -14,6 +15,14 @@ SELECT tg.project_code, tg.profile_sample_percent, tg.profile_sample_min_count, tg.profile_do_pair_rules, - tg.profile_pair_rule_pct + tg.profile_pair_rule_pct, + CASE + WHEN tg.monitor_test_suite_id IS NULL THEN NULL + ELSE tg.monitor_test_suite_id::VARCHAR(50) + END as monitor_test_suite_id, + CASE + WHEN tg.last_complete_profile_run_id is NULL THEN NULL + ELSE tg.last_complete_profile_run_id::VARCHAR(50) + END as last_complete_profile_run_id FROM table_groups tg WHERE tg.id = :TABLE_GROUP_ID; diff --git a/testgen/ui/components/frontend/js/components/table_group_form.js b/testgen/ui/components/frontend/js/components/table_group_form.js index 6b072255..c2329cdf 100644 --- a/testgen/ui/components/frontend/js/components/table_group_form.js +++ b/testgen/ui/components/frontend/js/components/table_group_form.js @@ -83,6 +83,7 @@ const TableGroupForm = (props) => { const profileFlagCdes = van.state(tableGroup.profile_flag_cdes ?? true); const includeInDashboard = van.state(tableGroup.include_in_dashboard ?? true); const addScorecardDefinition = van.state(tableGroup.add_scorecard_definition ?? true); + const addMonitorTestSuite = van.state(tableGroup.add_monitor_test_suite ?? false); const profileUseSampling = van.state(tableGroup.profile_use_sampling ?? false); const profileSamplePercent = van.state(tableGroup.profile_sample_percent ?? 30); const profileSampleMinCount = van.state(tableGroup.profile_sample_min_count ?? 15000); @@ -122,6 +123,7 @@ const TableGroupForm = (props) => { profile_flag_cdes: profileFlagCdes.val, include_in_dashboard: includeInDashboard.val, add_scorecard_definition: addScorecardDefinition.val, + add_monitor_test_suite: addMonitorTestSuite.val, profile_use_sampling: profileUseSampling.val, profile_sample_percent: profileSamplePercent.val, profile_sample_min_count: profileSampleMinCount.val, @@ -188,6 +190,7 @@ const TableGroupForm = (props) => { profileFlagCdes, includeInDashboard, addScorecardDefinition, + addMonitorTestSuite, ), SamplingForm( { setValidity: setFieldValidity }, @@ -327,6 +330,7 @@ const SettingsForm = ( profileFlagCdes, includeInDashboard, addScorecardDefinition, + addMonitorTestSuite, ) => { return div( { class: 'flex-row fx-gap-3 fx-flex-wrap fx-align-flex-start border border-radius-1 p-3 mt-1', style: 'position: relative;' }, diff --git a/testgen/ui/views/connections.py b/testgen/ui/views/connections.py index f6bdb269..ab52de96 100644 --- a/testgen/ui/views/connections.py +++ b/testgen/ui/views/connections.py @@ -311,6 +311,7 @@ def on_preview_table_group(payload: dict) -> None: default=False, ) + add_monitor_test_suite = table_group_data.pop("add_monitor_test_suite", False) add_scorecard_definition = table_group_data.pop("add_scorecard_definition", False) table_group = TableGroup( project_code=project_code, @@ -333,7 +334,11 @@ def on_preview_table_group(payload: dict) -> None: if is_table_group_verified(): try: - table_group.save(add_scorecard_definition=add_scorecard_definition) + table_group.save( + add_scorecard_definition=add_scorecard_definition, + add_monitor_test_suite=add_monitor_test_suite, + monitor_schedule_timezone=st.session_state["browser_timezone"] or "UTC", + ) if should_run_profiling: try: diff --git a/testgen/ui/views/profiling_runs.py b/testgen/ui/views/profiling_runs.py index 9fd38890..34f2154e 100644 --- a/testgen/ui/views/profiling_runs.py +++ b/testgen/ui/views/profiling_runs.py @@ -11,6 +11,7 @@ from testgen.common.models import with_database_session from testgen.common.models.profiling_run import ProfilingRun from testgen.common.models.project import Project +from testgen.common.models.scheduler import RUN_PROFILE_JOB_KEY from testgen.common.models.table_group import TableGroup, TableGroupMinimal from testgen.ui.components import widgets as testgen from testgen.ui.components.widgets import testgen_component @@ -122,7 +123,7 @@ class ProfilingScheduleDialog(ScheduleDialog): title = "Profiling Schedules" arg_label = "Table Group" - job_key = "run-profile" + job_key = RUN_PROFILE_JOB_KEY table_groups: Iterable[TableGroupMinimal] | None = None def init(self) -> None: diff --git a/testgen/ui/views/table_groups.py b/testgen/ui/views/table_groups.py index 3ee2523b..78da867d 100644 --- a/testgen/ui/views/table_groups.py +++ b/testgen/ui/views/table_groups.py @@ -171,10 +171,13 @@ def on_go_to_profiling_runs(params: dict) -> None: original_table_group_schema = table_group.table_group_schema is_table_group_used = TableGroup.is_in_use([table_group_id]) + add_monitor_test_suite = False add_scorecard_definition = False for key, value in get_table_group().items(): if key == "add_scorecard_definition": add_scorecard_definition = value + elif key == "add_monitor_test_suite": + add_monitor_test_suite = value else: setattr(table_group, key, value) @@ -209,7 +212,11 @@ def on_go_to_profiling_runs(params: dict) -> None: success = True if is_table_group_verified(): try: - table_group.save(add_scorecard_definition) + table_group.save( + add_scorecard_definition, + add_monitor_test_suite=add_monitor_test_suite, + monitor_schedule_timezone=st.session_state["browser_timezone"] or "UTC", + ) if should_run_profiling(): try: run_profiling_in_background(table_group.id) diff --git a/testgen/ui/views/test_runs.py b/testgen/ui/views/test_runs.py index 1d1f43b0..5e4f594b 100644 --- a/testgen/ui/views/test_runs.py +++ b/testgen/ui/views/test_runs.py @@ -10,6 +10,7 @@ import testgen.ui.services.form_service as fm from testgen.common.models import with_database_session from testgen.common.models.project import Project +from testgen.common.models.scheduler import RUN_TESTS_JOB_KEY from testgen.common.models.table_group import TableGroup, TableGroupMinimal from testgen.common.models.test_run import TestRun from testgen.common.models.test_suite import TestSuite, TestSuiteMinimal @@ -140,7 +141,7 @@ class TestRunScheduleDialog(ScheduleDialog): title = "Test Run Schedules" arg_label = "Test Suite" - job_key = "run-tests" + job_key = RUN_TESTS_JOB_KEY test_suites: Iterable[TestSuiteMinimal] | None = None def init(self) -> None: diff --git a/tests/unit/test_scheduler_cli.py b/tests/unit/test_scheduler_cli.py index 4c4c493e..7a7e0854 100644 --- a/tests/unit/test_scheduler_cli.py +++ b/tests/unit/test_scheduler_cli.py @@ -79,7 +79,7 @@ def cli_job(job_data): @pytest.mark.unit def test_get_jobs(scheduler_instance, db_jobs, job_sched): - db_jobs.return_value = iter([[job_sched]]) + db_jobs.return_value = iter([job_sched]) jobs = list(scheduler_instance.get_jobs()) From e41338a4af8a36d20cb442b4cd028eb91423157e Mon Sep 17 00:00:00 2001 From: Luis Date: Tue, 9 Sep 2025 10:34:45 -0400 Subject: [PATCH 22/28] feat(ui): allow filtering timezone dropdown --- .../frontend/js/components/select.js | 75 ++++++++++++++++--- .../frontend/js/pages/schedule_list.js | 1 + 2 files changed, 65 insertions(+), 11 deletions(-) diff --git a/testgen/ui/components/frontend/js/components/select.js b/testgen/ui/components/frontend/js/components/select.js index 95e1e741..967aee4e 100644 --- a/testgen/ui/components/frontend/js/components/select.js +++ b/testgen/ui/components/frontend/js/components/select.js @@ -21,6 +21,7 @@ * @property {string?} style * @property {string?} testId * @property {number?} portalClass + * @property {boolean?} filterable * @property {('normal' | 'inline')?} triggerStyle */ import van from '../van.min.js'; @@ -28,13 +29,14 @@ import { getRandomId, getValue, loadStylesheet, isState, isEqual } from '../util import { Portal } from './portal.js'; import { Icon } from './icon.js'; -const { div, i, label, span } = van.tags; +const { div, i, input, label, span } = van.tags; const Select = (/** @type {Properties} */ props) => { loadStylesheet('select', stylesheet); const domId = van.derive(() => props.id?.val ?? getRandomId()); const opened = van.state(false); + const optionsFilter = van.state(''); const options = van.derive(() => { const options = getValue(props.options) ?? []; const allowNull = getValue(props.allowNull); @@ -48,6 +50,27 @@ const Select = (/** @type {Properties} */ props) => { return options; }); + const filteredOptions = van.derive(() => { + const allOptions = getValue(options); + const isFilterable = getValue(props.filterable); + const filterTerm = getValue(optionsFilter); + if (isFilterable && filterTerm.length) { + const filteredOptions_ = []; + for (let i = 0; i < allOptions.length; i++) { + const option = allOptions[i]; + if (option.label === filterTerm) { + return allOptions; + } + + if (option.label.toLowerCase().includes(filterTerm.toLowerCase())) { + filteredOptions_.push(option); + } + } + return filteredOptions_; + } + return allOptions; + }); + const value = isState(props.value) ? props.value : van.state(props.value ?? null); const initialSelection = options.val?.find((op) => op.value === value.val); const valueLabel = van.state(initialSelection?.label ?? ''); @@ -58,6 +81,16 @@ const Select = (/** @type {Properties} */ props) => { value.val = option.value; }; + const filterOptions = (/** @type InputEvent */ event) => { + optionsFilter.val = event.target.value; + }; + + const showPortal = (/** @type Event */ event) => { + event.stopPropagation(); + event.stopImmediatePropagation(); + opened.val = getValue(props.disabled) ? false : true; + }; + van.derive(() => { const currentOptions = getValue(options); const previousValue = value.oldVal; @@ -82,8 +115,8 @@ const Select = (/** @type {Properties} */ props) => { id: domId, class: () => `flex-column fx-gap-1 text-caption tg-select--label ${getValue(props.disabled) ? 'disabled' : ''}`, style: () => `width: ${props.width ? getValue(props.width) + 'px' : 'auto'}; ${getValue(props.style)}`, - onclick: van.derive(() => !getValue(props.disabled) ? () => opened.val = !opened.val : null), 'data-testid': getValue(props.testId) ?? '', + onclick: showPortal, }, span( { class: 'flex-row fx-gap-1', 'data-testid': 'select-label' }, @@ -111,17 +144,27 @@ const Select = (/** @type {Properties} */ props) => { style: () => getValue(props.height) ? `height: ${getValue(props.height)}px;` : '', 'data-testid': 'select-input', }, - () => div( - { class: 'tg-select--field--content', 'data-testid': 'select-input-display' }, - valueIcon.val - ? Icon({ classes: 'mr-2' }, valueIcon.val) - : undefined, - valueLabel.val, - ), + () => { + return div( + { class: 'tg-select--field--content', 'data-testid': 'select-input-display' }, + valueIcon.val + ? Icon({ classes: 'mr-2' }, valueIcon.val) + : undefined, + getValue(props.filterable) + ? input({ + id: `tg-select--field--${getRandomId()}`, + value: valueLabel.val, + onkeyup: filterOptions, + }) + : valueLabel.val, + ); + }, div( { class: 'tg-select--field--icon', 'data-testid': 'select-input-trigger' }, i( - { class: 'material-symbols-rounded' }, + { + class: 'material-symbols-rounded', + }, 'expand_more', ), ), @@ -134,7 +177,7 @@ const Select = (/** @type {Properties} */ props) => { class: () => `tg-select--options-wrapper mt-1 ${getValue(props.portalClass) ?? ''}`, 'data-testid': 'select-options', }, - getValue(options).map(option => + getValue(filteredOptions).map(option => div( { class: () => `tg-select--option ${getValue(value) === option.value ? 'selected' : ''}`, @@ -196,6 +239,16 @@ stylesheet.replace(` font-weight: 500; } +.tg-select--field--content > input { + border: unset !important; + background: transparent !important; + outline: none !important; + width: 100%; + font-weight: 500; + font-family: 'Roboto', 'Helvetica Neue', sans-serif; + color: var(--primary-text-color); +} + .tg-select--field--icon { display: flex; align-items: center; diff --git a/testgen/ui/components/frontend/js/pages/schedule_list.js b/testgen/ui/components/frontend/js/pages/schedule_list.js index bc403a3e..658421b5 100644 --- a/testgen/ui/components/frontend/js/pages/schedule_list.js +++ b/testgen/ui/components/frontend/js/pages/schedule_list.js @@ -88,6 +88,7 @@ const ScheduleList = (/** @type Properties */ props) => { options: timezones.map(tz_ => ({label: tz_, value: tz_})), value: newScheduleForm.timezone, allowNull: false, + filterable: true, onChange: (value) => { newScheduleForm.timezone.val = value; if (newScheduleForm.expression.val && newScheduleForm.timezone.val) { From 83ece03ec1af85ee2d80389dab8d2cc3387c40a9 Mon Sep 17 00:00:00 2001 From: Diogo Basto Date: Mon, 15 Sep 2025 15:33:25 +0100 Subject: [PATCH 23/28] Fix: Make authentication case insensitive --- testgen/common/models/user.py | 10 ++++++++-- testgen/ui/auth.py | 6 +++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/testgen/common/models/user.py b/testgen/common/models/user.py index 7824f57b..cc6f57c6 100644 --- a/testgen/common/models/user.py +++ b/testgen/common/models/user.py @@ -1,8 +1,9 @@ from datetime import UTC, datetime -from typing import Literal +from typing import Literal, Self from uuid import UUID, uuid4 -from sqlalchemy import Column, String, asc, func, update +import streamlit as st +from sqlalchemy import Column, String, asc, func, select, update from sqlalchemy.dialects import postgresql from testgen.common.models import get_current_session @@ -43,3 +44,8 @@ def save(self, update_latest_login: bool = False) -> None: self.latest_login = datetime.now(UTC) super().save() + @classmethod + @st.cache_data(show_spinner=False) + def get(cls, identifier: str) -> Self | None: + query = select(cls).where(func.lower(User.username) == func.lower(identifier)) + return get_current_session().scalars(query).first() diff --git a/testgen/ui/auth.py b/testgen/ui/auth.py index 05abb2ed..14706465 100644 --- a/testgen/ui/auth.py +++ b/testgen/ui/auth.py @@ -36,7 +36,7 @@ def is_logged_in(self) -> bool: @property def user_display(self) -> str | None: return (self.user.name or self.user.username) if self.user else None - + @property def default_page(self) -> str | None: return "project-dashboard" if self.user else "" @@ -63,7 +63,7 @@ def get_credentials(self): "password": item.password, } return {"usernames": usernames} - + def login_user(self, username: str) -> None: self.user = User.get(username) self.user.save(update_latest_login=True) @@ -83,7 +83,7 @@ def load_user_session(self) -> None: def end_user_session(self) -> None: self._clear_jwt_cookie() self.user = None - + def _clear_jwt_cookie(self) -> None: execute_javascript( f"""await (async function () {{ From 9c7827745f2a55e2b5f0680a1ad74e4a5906c920 Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Fri, 12 Sep 2025 00:06:23 -0400 Subject: [PATCH 24/28] fix(profiling): increase char limit when calculating pattern --- .../flavors/mssql/profiling/project_profiling_query_mssql.yaml | 2 +- .../profiling/project_profiling_query_postgresql.yaml | 2 +- .../redshift/profiling/project_profiling_query_redshift.yaml | 2 +- .../snowflake/profiling/project_profiling_query_snowflake.yaml | 2 +- .../flavors/trino/profiling/project_profiling_query_trino.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/testgen/template/flavors/mssql/profiling/project_profiling_query_mssql.yaml b/testgen/template/flavors/mssql/profiling/project_profiling_query_mssql.yaml index adf38026..2b8aae99 100644 --- a/testgen/template/flavors/mssql/profiling/project_profiling_query_mssql.yaml +++ b/testgen/template/flavors/mssql/profiling/project_profiling_query_mssql.yaml @@ -131,7 +131,7 @@ strTemplate05_else: NULL as distinct_std_value_ct, NULL as std_pattern_match, strTemplate06_A_patterns: ( SELECT LEFT(STRING_AGG(pattern, ' | ') WITHIN GROUP (ORDER BY ct DESC), 1000) AS concat_pats FROM ( - SELECT TOP 5 CAST(COUNT(*) AS VARCHAR(10)) + ' | ' + pattern AS pattern, + SELECT TOP 5 CAST(COUNT(*) AS VARCHAR(40)) + ' | ' + pattern AS pattern, COUNT(*) AS ct FROM ( SELECT TRANSLATE("{COL_NAME}" COLLATE Latin1_General_BIN, 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789', diff --git a/testgen/template/flavors/postgresql/profiling/project_profiling_query_postgresql.yaml b/testgen/template/flavors/postgresql/profiling/project_profiling_query_postgresql.yaml index 7fe97764..763dd4b7 100644 --- a/testgen/template/flavors/postgresql/profiling/project_profiling_query_postgresql.yaml +++ b/testgen/template/flavors/postgresql/profiling/project_profiling_query_postgresql.yaml @@ -107,7 +107,7 @@ strTemplate05_else: NULL as distinct_std_value_ct, NULL as std_pattern_match, strTemplate06_A_patterns: ( SELECT LEFT(STRING_AGG(pattern, ' | ' ORDER BY ct DESC) , 1000) AS concat_pats FROM ( - SELECT CAST(COUNT(*) AS VARCHAR(10)) || ' | ' || pattern AS pattern, + SELECT CAST(COUNT(*) AS VARCHAR(40)) || ' | ' || pattern AS pattern, COUNT(*) AS ct FROM ( SELECT REGEXP_REPLACE(REGEXP_REPLACE( REGEXP_REPLACE( "{COL_NAME}", '[a-z]', 'a', 'g'), diff --git a/testgen/template/flavors/redshift/profiling/project_profiling_query_redshift.yaml b/testgen/template/flavors/redshift/profiling/project_profiling_query_redshift.yaml index 0edbe8a7..1596dd1d 100644 --- a/testgen/template/flavors/redshift/profiling/project_profiling_query_redshift.yaml +++ b/testgen/template/flavors/redshift/profiling/project_profiling_query_redshift.yaml @@ -86,7 +86,7 @@ strTemplate05_else: NULL as distinct_std_value_ct, NULL as date_ct, NULL as std_pattern_match, strTemplate06_A_patterns: (SELECT LEFT(LISTAGG(pattern, ' | ') WITHIN GROUP (ORDER BY ct DESC), 1000) AS concat_pats - FROM ( SELECT TOP 5 CAST(COUNT(*) AS VARCHAR(10)) || ' | ' || pattern AS pattern, + FROM ( SELECT TOP 5 CAST(COUNT(*) AS VARCHAR(40)) || ' | ' || pattern AS pattern, COUNT(*) AS ct FROM ( SELECT REGEXP_REPLACE(REGEXP_REPLACE( REGEXP_REPLACE( "{COL_NAME}", '[a-z]', 'a'), diff --git a/testgen/template/flavors/snowflake/profiling/project_profiling_query_snowflake.yaml b/testgen/template/flavors/snowflake/profiling/project_profiling_query_snowflake.yaml index ce9f3066..bc0f1e7d 100644 --- a/testgen/template/flavors/snowflake/profiling/project_profiling_query_snowflake.yaml +++ b/testgen/template/flavors/snowflake/profiling/project_profiling_query_snowflake.yaml @@ -94,7 +94,7 @@ strTemplate05_else: NULL as distinct_std_value_ct, NULL as std_pattern_match, strTemplate06_A_patterns: ( SELECT LEFT(LISTAGG(pattern, ' | ') WITHIN GROUP (ORDER BY ct DESC), 1000) AS concat_pats FROM ( - SELECT TOP 5 CAST(COUNT(*) AS VARCHAR(10)) || ' | ' || pattern AS pattern, + SELECT TOP 5 CAST(COUNT(*) AS VARCHAR(40)) || ' | ' || pattern AS pattern, COUNT(*) AS ct FROM ( SELECT REGEXP_REPLACE(REGEXP_REPLACE( REGEXP_REPLACE( "{COL_NAME}"::VARCHAR, '[a-z]', 'a'), diff --git a/testgen/template/flavors/trino/profiling/project_profiling_query_trino.yaml b/testgen/template/flavors/trino/profiling/project_profiling_query_trino.yaml index 284605b4..e3ee9f83 100644 --- a/testgen/template/flavors/trino/profiling/project_profiling_query_trino.yaml +++ b/testgen/template/flavors/trino/profiling/project_profiling_query_trino.yaml @@ -107,7 +107,7 @@ strTemplate05_else: NULL as distinct_std_value_ct, NULL as date_ct, NULL as std_pattern_match, strTemplate06_A_patterns: (SELECT SUBSTRING(LISTAGG(pattern, ' | ') WITHIN GROUP (ORDER BY ct DESC), 1, 1000) AS concat_pats - FROM ( SELECT CAST(COUNT(*) AS VARCHAR(10)) || ' | ' || pattern AS pattern, + FROM ( SELECT CAST(COUNT(*) AS VARCHAR(40)) || ' | ' || pattern AS pattern, COUNT(*) AS ct FROM ( SELECT REGEXP_REPLACE(REGEXP_REPLACE( REGEXP_REPLACE( "{COL_NAME}", '[a-z]', 'a'), From f44230c1371466e8e0c1a9d2c5ca36cb37ba700b Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Fri, 12 Sep 2025 16:02:25 -0400 Subject: [PATCH 25/28] fix(mssql): update is_date function --- .../mssql/profiling/templated_functions.yaml | 83 ++++++++++--------- 1 file changed, 42 insertions(+), 41 deletions(-) diff --git a/testgen/template/flavors/mssql/profiling/templated_functions.yaml b/testgen/template/flavors/mssql/profiling/templated_functions.yaml index 86d064b4..c426b4be 100644 --- a/testgen/template/flavors/mssql/profiling/templated_functions.yaml +++ b/testgen/template/flavors/mssql/profiling/templated_functions.yaml @@ -3,44 +3,45 @@ IS_NUM: CASE ELSE 0 END -IS_DATE: CASE WHEN TRY_CAST(NULLIF({$1}, '') AS float) IS NOT NULL - AND LEFT(NULLIF({$1}, ''),4) BETWEEN 1800 AND 2200 THEN - CASE - WHEN LEN((NULLIF({$1}, ''))) > 11 THEN 0 - /* YYYYMMDD */ - WHEN TRY_CONVERT(DATE, NULLIF({$1}, ''), 112) IS NOT NULL THEN 1 - - /* YYYY-MM-DD */ - WHEN TRY_CONVERT(DATE, NULLIF({$1}, ''), 23) IS NOT NULL THEN 1 - - /* MM/DD/YYYY */ - WHEN TRY_CONVERT(DATE, NULLIF({$1}, ''), 101) IS NOT NULL THEN 1 - - /* MM/DD/YY */ - WHEN TRY_CONVERT(DATE, NULLIF({$1}, ''), 1) IS NOT NULL THEN 1 - - /*MM-DD-YYYY */ - WHEN TRY_CONVERT(DATE, NULLIF({$1}, ''), 110) IS NOT NULL THEN 1 - - /*MM-DD-YY */ - WHEN TRY_CONVERT(DATE, NULLIF({$1}, ''), 10) IS NOT NULL THEN 1 - - - ELSE 0 END - /*DD MMM YYYY */ - WHEN (TRY_CONVERT(DATE, NULLIF({$1}, ''), 106) IS NOT NULL - AND LEFT(NULLIF({$1}, ''), 4) BETWEEN 1800 AND 2200) - THEN 1 - - /* YYYY-MM-DD HH:MM:SS SSSSSS */ - WHEN (TRY_CONVERT(DATETIME2, NULLIF({$1}, ''), 121) IS NOT NULL - AND LEFT(NULLIF({$1}, ''), 4) BETWEEN 1800 AND 2200) - THEN 1 - - /* YYYY-MM-DD HH:MM:SS */ - WHEN (TRY_CONVERT(DATETIME2, NULLIF({$1}, ''), 120) IS NOT NULL - AND LEFT(NULLIF({$1}, ''), 4) BETWEEN 1800 AND 2200) - THEN 1 - ELSE 0 - END - +IS_DATE: |- + CASE + WHEN NULLIF(LTRIM(RTRIM({$1})), '') IS NULL THEN 0 + WHEN ( + COALESCE( + TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 112), /* YYYYMMDD */ + TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 23), /* YYYY-MM-DD */ + TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 101), /* MM/DD/YYYY */ + TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 1), /* MM/DD/YY (2-digit year windowing) */ + TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 110), /* MM-DD-YYYY */ + TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 10), /* MM-DD-YY (2-digit year windowing) */ + TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 106), /* DD MMM YYYY */ + TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 107), /* MMM dd, YYYY */ + TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 126), /* ISO 8601 no TZ: YYYY-MM-DDTHH:MM:SS[.fff] */ + TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 120), /* YYYY-MM-DD HH:MM:SS */ + TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 121), /* YYYY-MM-DD HH:MM:SS.mmm */ + TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 111), /* YYYY/MM/DD */ + TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 102), /* YYYY.MM.DD */ + /* Normalize "YYYY-MM-DD HH:MM:SS SSSSSS" -> ISO 8601 with 'T' and '.' */ + TRY_CONVERT( + datetime2, + CASE + WHEN LTRIM(RTRIM({$1})) LIKE + '[0-9][0-9][0-9][0-9]-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9] [0-9]%' + THEN + /* Replace DT space (pos 11) with 'T', and the last space with '.' */ + STUFF( + STUFF(LTRIM(RTRIM({$1})), 11, 1, 'T'), + LEN(LTRIM(RTRIM({$1}))) - CHARINDEX(' ', REVERSE(LTRIM(RTRIM({$1})))) + 1, + 1, + '.' + ) + ELSE LTRIM(RTRIM({$1})) + END, + 126 + )) BETWEEN '1800-01-01T00:00:00' AND '2200-12-31T23:59:59.9999999' ) + OR ( /* ISO 8601 with timezone: style 127 parsed as datetimeoffset, then cast */ + CAST(TRY_CONVERT(datetimeoffset, LTRIM(RTRIM({$1})), 127) AS datetime2) + BETWEEN '1800-01-01T00:00:00' AND '2200-12-31T23:59:59.9999999' ) + THEN 1 + ELSE 0 + END From b3f1c902fe3d822783c2c8b0a05616cd0be9c50d Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Mon, 15 Sep 2025 20:02:44 -0400 Subject: [PATCH 26/28] fix(copy tests): bugs when copying multiple and table tests --- testgen/commands/run_profiling_bridge.py | 2 +- testgen/common/models/test_definition.py | 2 +- testgen/ui/views/test_definitions.py | 11 +++++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/testgen/commands/run_profiling_bridge.py b/testgen/commands/run_profiling_bridge.py index 723e5b5f..691f4c1c 100644 --- a/testgen/commands/run_profiling_bridge.py +++ b/testgen/commands/run_profiling_bridge.py @@ -214,7 +214,7 @@ def run_profiling_in_background(table_group_id): empty_cache() background_thread = threading.Thread( target=run_profiling_queries, - args=(table_group_id, session.auth.user_display), + args=(table_group_id, session.auth.user_display if session.auth else None), ) background_thread.start() else: diff --git a/testgen/common/models/test_definition.py b/testgen/common/models/test_definition.py index f7e30116..b97b9bc2 100644 --- a/testgen/common/models/test_definition.py +++ b/testgen/common/models/test_definition.py @@ -147,7 +147,7 @@ class TestType(Entity): class TestDefinition(Entity): __tablename__ = "test_definitions" - id: UUID = Column(postgresql.UUID(as_uuid=True), default=uuid4) + id: UUID = Column(postgresql.UUID(as_uuid=True), server_default=text("gen_random_uuid()")) cat_test_id: int = Column(BigInteger, Identity(), primary_key=True) table_groups_id: UUID = Column(postgresql.UUID(as_uuid=True)) profile_run_id: UUID = Column(postgresql.UUID(as_uuid=True)) diff --git a/testgen/ui/views/test_definitions.py b/testgen/ui/views/test_definitions.py index 0c0f5300..4dbe4c76 100644 --- a/testgen/ui/views/test_definitions.py +++ b/testgen/ui/views/test_definitions.py @@ -6,7 +6,7 @@ import pandas as pd import streamlit as st -from sqlalchemy import asc, func, tuple_ +from sqlalchemy import and_, asc, func, or_, tuple_ from streamlit.delta_generator import DeltaGenerator from streamlit_extras.no_default_selectbox import selectbox @@ -1184,13 +1184,16 @@ def get_test_definitions_collision( target_table_group_id: str, target_test_suite_id: str, ) -> pd.DataFrame: + table_tests = [(item["table_name"], item["test_type"]) for item in test_definitions if item["column_name"] is None] + column_tests = [(item["table_name"], item["column_name"], item["test_type"]) for item in test_definitions if item["column_name"] is not None] results = TestDefinition.select_minimal_where( TestDefinition.table_groups_id == target_table_group_id, TestDefinition.test_suite_id == target_test_suite_id, TestDefinition.last_auto_gen_date.isnot(None), - tuple_(TestDefinition.table_name, TestDefinition.column_name, TestDefinition.test_type).in_( - [(item["table_name"], item["column_name"], item["test_type"]) for item in test_definitions] - ), + or_( + tuple_(TestDefinition.table_name, TestDefinition.column_name, TestDefinition.test_type).in_(column_tests), + and_(tuple_(TestDefinition.table_name, TestDefinition.test_type).in_(table_tests), TestDefinition.column_name.is_(None)), + ) ) return to_dataframe(results, TestDefinitionMinimal.columns()) From 4dff06860bfd3896c92891836dea828423e17bf9 Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Mon, 15 Sep 2025 23:35:50 -0400 Subject: [PATCH 27/28] fix(hygiene issues): sort by likelihood first --- testgen/common/models/test_definition.py | 2 +- testgen/ui/views/hygiene_issues.py | 10 +++++----- testgen/ui/views/test_results.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/testgen/common/models/test_definition.py b/testgen/common/models/test_definition.py index b97b9bc2..b193dff6 100644 --- a/testgen/common/models/test_definition.py +++ b/testgen/common/models/test_definition.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from datetime import datetime from typing import Literal -from uuid import UUID, uuid4 +from uuid import UUID import streamlit as st from sqlalchemy import ( diff --git a/testgen/ui/views/hygiene_issues.py b/testgen/ui/views/hygiene_issues.py index c65a9579..7cc788d0 100644 --- a/testgen/ui/views/hygiene_issues.py +++ b/testgen/ui/views/hygiene_issues.py @@ -156,7 +156,7 @@ def render( ("Likelihood", "likelihood_order"), ("Action", "r.disposition"), ) - default = [(sortable_columns[i][1], "ASC") for i in (0, 1)] + default = [(sortable_columns[i][1], "ASC") for i in (3, 0, 1)] sorting_columns = testgen.sorting_selector(sortable_columns, default) with actions_column: @@ -434,10 +434,10 @@ def get_profiling_anomalies( THEN 'Potential PII: may require privacy policies, standards and procedures for access, storage and transmission.' END AS likelihood_explanation, CASE - WHEN t.issue_likelihood = 'Potential PII' THEN 1 - WHEN t.issue_likelihood = 'Possible' THEN 2 - WHEN t.issue_likelihood = 'Likely' THEN 3 - WHEN t.issue_likelihood = 'Definite' THEN 4 + WHEN t.issue_likelihood = 'Potential PII' THEN 4 + WHEN t.issue_likelihood = 'Possible' THEN 3 + WHEN t.issue_likelihood = 'Likely' THEN 2 + WHEN t.issue_likelihood = 'Definite' THEN 1 END AS likelihood_order, t.anomaly_description, r.detail, t.suggested_action, r.anomaly_id, r.table_groups_id::VARCHAR, r.id::VARCHAR, p.profiling_starttime, r.profile_run_id::VARCHAR, diff --git a/testgen/ui/views/test_results.py b/testgen/ui/views/test_results.py index 575616a2..80d24bf1 100644 --- a/testgen/ui/views/test_results.py +++ b/testgen/ui/views/test_results.py @@ -805,7 +805,7 @@ def source_data_dialog(selected_row): st.caption(selected_row["test_description"]) st.markdown("#### Test Parameters") - st.caption(selected_row["input_parameters"]) + testgen.caption(selected_row["input_parameters"], styles="max-height: 100px; overflow: auto;") st.markdown("#### Result Detail") st.caption(selected_row["result_message"]) From 9de63e6ae9390fae48c045622c6388fd49d5f9d3 Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Tue, 16 Sep 2025 23:31:23 -0400 Subject: [PATCH 28/28] release: 4.22.2 -> 4.26.1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b463aaa4..2049bac5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta" [project] name = "dataops-testgen" -version = "4.22.2" +version = "4.26.1" description = "DataKitchen's Data Quality DataOps TestGen" authors = [ { "name" = "DataKitchen, Inc.", "email" = "info@datakitchen.io" },