diff --git a/pyproject.toml b/pyproject.toml index 0320b0c9..d1f590c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta" [project] name = "dataops-testgen" -version = "4.12.6" +version = "4.16.3" description = "DataKitchen's Data Quality DataOps TestGen" authors = [ { "name" = "DataKitchen, Inc.", "email" = "info@datakitchen.io" }, diff --git a/testgen/commands/queries/generate_tests_query.py b/testgen/commands/queries/generate_tests_query.py index 374e50f5..460ff73c 100644 --- a/testgen/commands/queries/generate_tests_query.py +++ b/testgen/commands/queries/generate_tests_query.py @@ -43,6 +43,7 @@ def ReplaceParms(self, strInputString): strInputString = strInputString.replace("{GENERATION_SET}", self.generation_set) strInputString = strInputString.replace("{AS_OF_DATE}", self.as_of_date) strInputString = strInputString.replace("{DATA_SCHEMA}", self.data_schema) + strInputString = strInputString.replace("{ID_SEPARATOR}", "`" if self.sql_flavor == "databricks" else '"') return strInputString diff --git a/testgen/commands/run_execute_cat_tests.py b/testgen/commands/run_execute_cat_tests.py index b76e7246..090adb60 100644 --- a/testgen/commands/run_execute_cat_tests.py +++ b/testgen/commands/run_execute_cat_tests.py @@ -63,7 +63,7 @@ def ParseCATResults(clsCATExecute): RunActionQueryList("DKTG", [strQuery]) -def FinalizeTestRun(clsCATExecute: CCATExecutionSQL): +def FinalizeTestRun(clsCATExecute: CCATExecutionSQL, username: str | None = None): _, row_counts = RunActionQueryList(("DKTG"), [ clsCATExecute.FinalizeTestResultsSQL(), clsCATExecute.PushTestRunStatusUpdateSQL(), @@ -89,6 +89,7 @@ def FinalizeTestRun(clsCATExecute: CCATExecutionSQL): MixpanelService().send_event( "run-tests", source=settings.ANALYTICS_JOB_SOURCE, + username=username, sql_flavor=clsCATExecute.flavor, test_count=row_counts[0], run_duration=(end_time - date_service.parse_now(clsCATExecute.run_date)).total_seconds(), @@ -97,7 +98,7 @@ def FinalizeTestRun(clsCATExecute: CCATExecutionSQL): def run_cat_test_queries( - dctParms, strTestRunID, strTestTime, strProjectCode, strTestSuite, error_msg, minutes_offset=0, spinner=None + dctParms, strTestRunID, strTestTime, strProjectCode, strTestSuite, error_msg, username=None, minutes_offset=0, spinner=None ): booErrors = False @@ -167,4 +168,4 @@ def run_cat_test_queries( finally: LOG.info("Finalizing test run") - FinalizeTestRun(clsCATExecute) + FinalizeTestRun(clsCATExecute, username) diff --git a/testgen/commands/run_execute_tests.py b/testgen/commands/run_execute_tests.py index a5799006..511ec1fd 100644 --- a/testgen/commands/run_execute_tests.py +++ b/testgen/commands/run_execute_tests.py @@ -18,6 +18,7 @@ date_service, ) from testgen.common.database.database_service import ExecuteDBQuery, empty_cache +from testgen.ui.session import session from .run_execute_cat_tests import run_cat_test_queries from .run_refresh_data_chars import run_refresh_data_chars_queries @@ -109,7 +110,7 @@ def run_execution_steps_in_background(project_code, test_suite): empty_cache() background_thread = threading.Thread( target=run_execution_steps, - args=(project_code, test_suite), + args=(project_code, test_suite, session.username), ) background_thread.start() else: @@ -121,6 +122,7 @@ def run_execution_steps_in_background(project_code, test_suite): def run_execution_steps( project_code: str, test_suite: str, + username: str | None = None, minutes_offset: int=0, spinner: Spinner=None, ) -> str: @@ -178,7 +180,7 @@ def run_execution_steps( LOG.info("CurrentStep: Execute Step - CAT Test Execution") if run_cat_test_queries( - test_exec_params, test_run_id, test_time, project_code, test_suite, error_msg, minutes_offset, spinner + test_exec_params, test_run_id, test_time, project_code, test_suite, error_msg, username, minutes_offset, spinner ): has_errors = True diff --git a/testgen/commands/run_get_entities.py b/testgen/commands/run_get_entities.py index 89f0d512..efda24e4 100644 --- a/testgen/commands/run_get_entities.py +++ b/testgen/commands/run_get_entities.py @@ -1,7 +1,6 @@ import logging from testgen.common import RetrieveDBResultsToList, read_template_sql_file -from testgen.common.encrypt import DecryptText LOG = logging.getLogger("testgen") @@ -29,17 +28,6 @@ def run_list_connections(): return RetrieveDBResultsToList("DKTG", sql_template) -def run_get_connection(connection_id): - sql_template = read_template_sql_file("get_connection.sql", "get_entities") - sql_template = sql_template.replace("{CONNECTION_ID}", str(connection_id)) - rows, _ = RetrieveDBResultsToList("DKTG", sql_template) - connection = rows.pop()._asdict() - connection["password"] = DecryptText(connection["project_pw_encrypted"]) if connection["project_pw_encrypted"] else None - connection["private_key"] = DecryptText(connection["private_key"]) if connection["private_key"] else None - connection["private_key_passphrase"] = DecryptText(connection["private_key_passphrase"]) if connection["private_key_passphrase"] else "" - return connection - - def run_table_group_list(project_code): sql_template = read_template_sql_file("get_table_group_list.sql", "get_entities") sql_template = sql_template.replace("{PROJECT_CODE}", project_code) diff --git a/testgen/commands/run_profiling_bridge.py b/testgen/commands/run_profiling_bridge.py index a766c6bd..3782dbff 100644 --- a/testgen/commands/run_profiling_bridge.py +++ b/testgen/commands/run_profiling_bridge.py @@ -22,6 +22,7 @@ ) from testgen.common.database.database_service import empty_cache from testgen.common.mixpanel_service import MixpanelService +from testgen.ui.session import session booClean = True LOG = logging.getLogger("testgen") @@ -238,7 +239,7 @@ def run_profiling_in_background(table_group_id): empty_cache() background_thread = threading.Thread( target=run_profiling_queries, - args=(table_group_id,), + args=(table_group_id, session.username), ) background_thread.start() else: @@ -247,7 +248,7 @@ def run_profiling_in_background(table_group_id): subprocess.Popen(script) # NOQA S603 -def run_profiling_queries(strTableGroupsID, spinner=None): +def run_profiling_queries(strTableGroupsID, username=None, spinner=None): if strTableGroupsID is None: raise ValueError("Table Group ID was not specified") @@ -518,6 +519,7 @@ def run_profiling_queries(strTableGroupsID, spinner=None): MixpanelService().send_event( "run-profiling", source=settings.ANALYTICS_JOB_SOURCE, + username=username, sql_flavor=clsProfiling.flavor, sampling=clsProfiling.profile_use_sampling == "Y", table_count=table_count, diff --git a/testgen/template/dbsetup/030_initialize_new_schema_structure.sql b/testgen/template/dbsetup/030_initialize_new_schema_structure.sql index 2caa893a..96549aaa 100644 --- a/testgen/template/dbsetup/030_initialize_new_schema_structure.sql +++ b/testgen/template/dbsetup/030_initialize_new_schema_structure.sql @@ -101,10 +101,11 @@ CREATE TABLE table_groups profile_use_sampling VARCHAR(3) DEFAULT 'N', profile_sample_percent VARCHAR(3) DEFAULT '30', profile_sample_min_count BIGINT DEFAULT 100000, - profiling_delay_days VARCHAR(3) DEFAULT '0' , + profiling_delay_days VARCHAR(3) DEFAULT '0', profile_flag_cdes BOOLEAN DEFAULT TRUE, profile_do_pair_rules VARCHAR(3) DEFAULT 'N', profile_pair_rule_pct INTEGER DEFAULT 95, + include_in_dashboard BOOLEAN DEFAULT TRUE, description VARCHAR(1000), data_source VARCHAR(40), source_system VARCHAR(40), diff --git a/testgen/template/dbsetup/050_populate_new_schema_metadata.sql b/testgen/template/dbsetup/050_populate_new_schema_metadata.sql index f0a8b8ab..dc94bf04 100644 --- a/testgen/template/dbsetup/050_populate_new_schema_metadata.sql +++ b/testgen/template/dbsetup/050_populate_new_schema_metadata.sql @@ -17,12 +17,13 @@ INSERT INTO profile_anomaly_types VALUES ('1001', 'Suggested_Type', 'Column', 'Suggested Data Type', 'Data stored as text all meets criteria for a more suitable type. ', '(functional_data_type NOT IN (''Boolean'', ''Flag'') ) AND (column_type ILIKE ''%ch ar%'' OR column_type ILIKE ''text'') AND NOT (datatype_suggestion ILIKE ''%char%'' OR datatype_suggestion ILIKE ''text'')', 'p.datatype_suggestion::VARCHAR(200)', 'Likely', 'Consider changing the column data type to tighte n controls over data ingested and to make values more efficient, consistent and suitable for downstream analysis.', NULL, NULL, NULL), - ('1002', 'Non_Standard_Blanks', 'Column', 'Non-Standard Blank Values', 'Values representing missing data may be unexpected or inconsistent. Non-standard values may include empty strings as opposed to nulls, dummy entries such as "MISSING" or repeated characters that may have been used to bypass entry requirements, processing artifacts such as "NULL", or spreadsheet artifacts such as "NA", "ERROR".', '(p.filled_value_ct > 0 OR p.zero_length_ct > 0)', '''Dummy Values: '' || p.filled_value_ct::VARCHAR || '', Empty String: '' || p.zero_length_ct::VARCHAR || '', Null: '' || p.null_value_ct::VARCHAR || '', Records: '' || p.record_ct::VARCHAR', 'Definite', 'Consider cleansing the column upon ingestion to replace all variants of missing data with a standard designation, like Null.', 'p.filled_value_ct::FLOAT/NULLIF(p.record_ct, 0)::FLOAT', '1.0', 'Completeness'), + ('1002', 'Non_Standard_Blanks', 'Column', 'Non-Standard Blank Values', 'Values representing missing data may be unexpected or inconsistent. Non-standard values may include empty strings as opposed to nulls, dummy entries such as "MISSING" or repeated characters that may have been used to bypass entry requirements, processing artifacts such as "NULL", or spreadsheet artifacts such as "NA", "ERROR".', '(p.zero_length_ct > 0 OR (p.filled_value_ct > 0 AND (p.numeric_ct <> p.value_ct OR functional_data_type IN (''Phone'', ''Zip''))))', '''Dummy Values: '' || p.filled_value_ct::VARCHAR || '', Empty String: '' || p.zero_length_ct::VARCHAR || '', Null: '' || p.null_value_ct::VARCHAR || '', Records: '' || p.record_ct::VARCHAR', 'Definite', 'Consider cleansing the column upon ingestion to replace all variants of missing data with a standard designation, like Null.', 'p.filled_value_ct::FLOAT/NULLIF(p.record_ct, 0)::FLOAT', '1.0', 'Completeness'), ('1003', 'Invalid_Zip_USA', 'Column', 'Invalid USA Zip Code Format', 'Some values present do not conform with the expected format of USA Zip Codes.', 'p.functional_data_type = ''Zip'' AND (p.general_type <> ''A'' OR p.filled_value_ct > 0 OR EXISTS (SELECT 1 FROM UNNEST(STRING_TO_ARRAY(p.top_patterns, '' | '')) WITH ORDINALITY AS u(val, idx) WHERE idx % 2 = 0 AND val NOT IN (''NNNNN'',''NNNNN-NNNN'',''NNNNNNNNN'')))', 'CASE WHEN p.general_type = ''N'' THEN ''Type: '' || p.column_type ELSE '''' END || CASE WHEN p.general_type = ''A'' THEN ''Patterns: '' || (SELECT string_agg(val, '','') FROM UNNEST(STRING_TO_ARRAY(top_patterns, '' | '')) WITH ORDINALITY AS u(val, idx) WHERE idx % 2 = 0) || '', Dummy Values: '' || p.filled_value_ct::VARCHAR ELSE '''' END', 'Definite', 'Consider correcting invalid column values or changing them to indicate a missing value if corrections cannot be made.', NULL, '1.0', 'Validity'), ('1004', 'Multiple_Types_Minor', 'Multi-Col', 'Multiple Data Types per Column Name - Minor', 'Columns with the same name have the same general type across tables, but the types do not exactly match. Truncation issues may result if columns are commingled and assumed to be the same format.', 'm.general_type_ct = 1 AND m.type_ct > 1', '''Found '' || m.column_ct::VARCHAR || '' columns, '' || m.type_ct::VARCHAR(10) || '' types, '' || m.min_type || '' to '' || m.max_type', 'Possible', 'Consider changing the column data types to be fully consistent. This will tighten your standards at ingestion and assure that data is consistent between tables.', NULL, NULL, 'Consistency'), ('1005', 'Multiple_Types_Major', 'Multi-Col', 'Multiple Data Types per Column Name - Major', 'Columns with the same name have broadly different types across tables. Differences could be significant enough to cause errors in downstream analysis, extra steps resulting in divergent business logic and inconsistencies in results.', 'm.general_type_ct > 1', '''Found '' || m.column_ct::VARCHAR || '' columns, '' || m.type_ct::VARCHAR(10) || '' types, '' || m.min_type || '' to '' || m.max_type', 'Likely', 'Ideally, you should change the column data types to be fully consistent. If the data is meant to be different, you should change column names so downstream users aren''t led astray.', NULL, NULL, 'Consistency'), ('1006', 'No_Values', 'Column', 'No Column Values Present', 'This column is present in the table, but no values have been ingested or assigned in any records. This could indicate missing data or a processing error. Note that this considers dummy values and zero-length values as missing data. ', '(p.null_value_ct + p.filled_value_ct + p.zero_length_ct) = p.record_ct', '''Null: '' || p.null_value_ct::VARCHAR(10) || '', Dummy: '' || p.filled_value_ct::VARCHAR(10) || '', Zero Len: '' || p.zero_length_ct::VARCHAR(10)', 'Possible', 'Review your source data, ingestion process, and any processing steps that update this column.', '1.0', '0.33', 'Completeness'), - ('1007', 'Column_Pattern_Mismatch', 'Column', 'Pattern Inconsistency Within Column', 'Alpha-numeric string data within this column conforms to 2-4 different patterns, with 95% matching the first pattern. This could indicate data errors in the remaining values. ', 'p.general_type = ''A'' AND functional_data_type NOT ILIKE ''Measurement%'' + ('1007', 'Column_Pattern_Mismatch', 'Column', 'Pattern Inconsistency Within Column', 'Alpha-numeric string data within this column conforms to 2-4 different patterns, with 95% matching the first pattern. This could indicate data errors in the remaining values. ', 'p.general_type = ''A'' + AND functional_data_type NOT ILIKE ''Measurement%'' AND functional_data_type NOT IN (''Category'', ''Code'') AND p.max_length > 3 AND p.value_ct > (p.numeric_ct + p.filled_value_ct + p.zero_length_ct) AND p.distinct_pattern_ct BETWEEN 2 AND 4 @@ -34,6 +35,7 @@ n controls over data ingested and to make values more efficient, consistent and SPLIT_PART(p.top_patterns, ''|'', 3)::NUMERIC / SPLIT_PART(p.top_patterns, ''|'', 1)::NUMERIC < 0.1 )', '''Patterns: '' || p.top_patterns', 'Likely', 'Review the values for any data that doesn''t conform to the most common pattern and correct any data errors.', '(p.record_ct - SPLIT_PART(p.top_patterns, ''|'', 1)::INT)::FLOAT/NULLIF(p.record_ct, 0)::FLOAT', '0.66', 'Validity'), ('1008', 'Table_Pattern_Mismatch', 'Multi-Col', 'Pattern Inconsistency Across Tables', 'Alpha-numeric string data within this column matches a single pattern, but other columns with the same name have data that matches a different single pattern. Inconsistent formatting may contradict user assumptions and cause downstream errors, extra steps and inconsistent business logic.', 'p.general_type = ''A'' + AND functional_data_type NOT ILIKE ''Measurement%'' AND functional_data_type NOT IN (''Category'', ''Code'') AND p.max_length > 3 AND p.value_ct > (p.numeric_ct + p.filled_value_ct + p.zero_length_ct) AND m.max_pattern_ct = 1 @@ -51,10 +53,14 @@ n controls over data ingested and to make values more efficient, consistent and ('1012', 'Char_Column_Date_Values', 'Column', 'Character Column with Mostly Date Values', 'This column is defined as alpha, but more than 95% of its values are dates. Dates in alpha columns might not sort correctly, and might contradict user expectations downstream. It''s also possible that more than one type of information is stored in the column, making it harder to retrieve. ', 'p.general_type = ''A'' AND p.value_ct > p.date_ct AND p.date_ct::NUMERIC > (0.95 * p.value_ct::NUMERIC)', ''' Date Ct: '' || p.date_ct || '' of '' || p.value_ct || '' (Date Percent: '' || ROUND(100.0 * p.date_ct::NUMERIC(18, 5) / p.value_ct::NUMERIC(18, 5), 2) || '' )''::VARCHAR(200)', 'Likely', 'Review your source data and ingestion process. Consider whether it might be better to store the date values as a date or datetime column. If the alpha data is also significant, you could store it in a different column.', 'p.date_ct::FLOAT/NULLIF(p.record_ct, 0)::FLOAT', '0.66', 'Validity'), - ('1013', 'Small Missing Value Ct', 'Column', 'Small Percentage of Missing Values Found', 'Under 3% of values in this column were found to be null, zero-length or dummy values, but values are not universally present. This could indicate unexpected missing values in a required column.', '(p.value_ct - p.zero_length_ct - p.filled_value_ct)::FLOAT / p.record_ct::FLOAT > 0.97 - AND (p.value_ct - p.zero_length_ct - p.filled_value_ct) < p.record_ct', '(p.record_ct - (p.value_ct - p.zero_length_ct - p.filled_value_ct))::VARCHAR(20) || + ('1013', 'Small Missing Value Ct', 'Column', 'Small Percentage of Missing Values Found', 'Under 3% of values in this column were found to be null, zero-length or dummy values, but values are not universally present. This could indicate unexpected missing values in a required column.', '(p.value_ct - p.zero_length_ct - CASE WHEN (p.filled_value_ct > 0 AND (p.numeric_ct <> p.value_ct OR functional_data_type IN (''Phone'', ''Zip''))) THEN p.filled_value_ct ELSE 0 END + )::FLOAT / p.record_ct::FLOAT > 0.97 + AND (p.value_ct - p.zero_length_ct - CASE WHEN (p.filled_value_ct > 0 AND (p.numeric_ct <> p.value_ct OR functional_data_type IN (''Phone'', ''Zip''))) THEN p.filled_value_ct ELSE 0 END + ) < p.record_ct', '(p.record_ct - (p.value_ct - p.zero_length_ct - CASE WHEN (p.filled_value_ct > 0 AND (p.numeric_ct <> p.value_ct OR functional_data_type IN (''Phone'', ''Zip''))) THEN p.filled_value_ct ELSE 0 END + ))::VARCHAR(20) || '' of '' || p.record_ct::VARCHAR(20) || '' blank values: '' || - ROUND(100.0 * (p.record_ct - (p.value_ct - p.zero_length_ct - p.filled_value_ct))::NUMERIC(18, 5) + ROUND(100.0 * (p.record_ct - (p.value_ct - p.zero_length_ct - CASE WHEN (p.filled_value_ct > 0 AND (p.numeric_ct <> p.value_ct OR functional_data_type IN (''Phone'', ''Zip''))) THEN p.filled_value_ct ELSE 0 END + ))::NUMERIC(18, 5) / NULLIF(p.value_ct, 0)::NUMERIC(18, 5), 2)::VARCHAR(40) || ''%''', 'Possible', 'Review your source data and follow-up with data owners to determine whether this data needs to be corrected, supplemented or excluded.', '(p.null_value_ct + filled_value_ct + zero_length_ct)::FLOAT/NULLIF(p.record_ct, 0)::FLOAT', '0.33', 'Completeness'), ('1014', 'Small Divergent Value Ct', 'Column', 'Small Percentage of Divergent Values Found', 'Under 3% of values in this column were found to be different from the most common value. This could indicate a data error.', 'functional_data_type <> ''Boolean'' AND (100.0 * fn_parsefreq(p.top_freq_values, 1, 2)::FLOAT / p.value_ct::FLOAT) > 97::FLOAT @@ -68,7 +74,7 @@ n controls over data ingested and to make values more efficient, consistent and ELSE ''Top Freq: '' || p.top_freq_values END', 'Likely', 'Review your source data and follow-up with data owners to determine whether this data needs to be corrected. ', NULL, '0.66', 'Validity'), ('1016', 'Potential_Duplicates', 'Column', 'Potential Duplicate Values Found', 'This column is largely unique, but some duplicate values are present. This pattern is uncommon and could indicate inadvertant duplication. ', 'p.distinct_value_ct > 1000 AND fn_parsefreq(p.top_freq_values, 1, 2)::BIGINT BETWEEN 2 AND 4', '''Top Freq: '' || p.top_freq_values', 'Possible', 'Review your source data and follow-up with data owners to determine whether this data needs to be corrected. ', '(p.value_ct - p.distinct_value_ct)::FLOAT/NULLIF(p.record_ct, 0)::FLOAT', '0.33', 'Uniqueness'), - ('1017', 'Standardized_Value_Matches', 'Column', 'Similar Values Match When Standardized', 'When column values are standardized (removing spaces, single-quotes, periods and dashes), matching values are found in other records. This may indicate that formats should be further standardized to allow consistent comparisons for merges, joins and roll-ups. It could also indicate the presence of unintended duplicates.', 'p.general_type = ''A'' AND p.distinct_std_value_ct <> p.distinct_value_ct', '''Distinct Values: '' || p.distinct_value_ct::VARCHAR + ('1017', 'Standardized_Value_Matches', 'Column', 'Similar Values Match When Standardized', 'When column values are standardized (removing spaces, single-quotes, periods and dashes), matching values are found in other records. This may indicate that formats should be further standardized to allow consistent comparisons for merges, joins and roll-ups. It could also indicate the presence of unintended duplicates.', 'p.general_type = ''A'' AND p.distinct_std_value_ct <> p.distinct_value_ct AND p.functional_data_type NOT LIKE ''Person%Name'' ', '''Distinct Values: '' || p.distinct_value_ct::VARCHAR || '', Standardized: '' || p.distinct_std_value_ct::VARCHAR', 'Likely', 'Review standardized vs. raw data values for all matches. Correct data if values should be consistent.', '(p.distinct_value_ct - p.distinct_std_value_ct)::FLOAT/NULLIF(p.value_ct, 0)', '0.66', 'Uniqueness'), ('1018', 'Unlikely_Date_Values', 'Column', 'Unlikely Dates out of Typical Range', 'Some date values in this column are earlier than 1900-01-01 or later than 30 years after Profiling date.', 'p.general_type = ''D'' AND (p.min_date BETWEEN ''0001-01-02''::DATE AND ''1900-01-01''::DATE @@ -77,7 +83,7 @@ n controls over data ingested and to make values more efficient, consistent and ('1020', 'Recency_Six_Months', 'Dates', 'Recency - No Table Dates within 6 Months', 'Among all date columns present in the table, the most recent date falls 6 months to 1 year back from Profile date. ', 'MAX(p.max_date) >= CURRENT_DATE - INTERVAL ''1 year'' AND MAX(p.max_date) < CURRENT_DATE - INTERVAL ''6 months''', '''Most Recent Date: '' || MAX(p.max_date)::VARCHAR', 'Possible', 'Review your source data and follow-up with data owners to determine whether dates in table should be more recent.', NULL, NULL, 'Timeliness'), ('1021', 'Unexpected US States', 'Column', 'Unexpected Column Contains US States', 'This column is not labeled as a state, but contains mostly US State abbreviations. This could indicate shifted or switched source data columns.', 'p.std_pattern_match = ''STATE_USA'' AND p.distinct_value_ct > 5 - AND NOT (p.column_name = ''st'' OR p.column_name ILIKE ''%state%'' OR p.column_name ILIKE ''%_st'')', '''Value Range: '' || p.min_text || '' thru '' || max_text || CASE WHEN p.top_freq_values > '''' THEN ''Top Freq Values: '' || REPLACE(p.top_freq_values, CHR(10), '' ; '') ELSE '''' END ', 'Possible', 'Review your source data and follow-up with data owners to determine whether column should be populated with US states.', NULL, '0.33', 'Consistency'), + AND NOT (p.column_name = ''st'' OR p.column_name ILIKE ''%state%'' OR p.column_name ILIKE ''%_st'' OR p.column_name ILIKE ''st_%'')', '''Value Range: '' || p.min_text || '' thru '' || max_text || CASE WHEN p.top_freq_values > '''' THEN '', Top Freq Values: '' || REPLACE(p.top_freq_values, CHR(10), '' ; '') ELSE '''' END ', 'Possible', 'Review your source data and follow-up with data owners to determine whether column should be populated with US states.', NULL, '0.33', 'Consistency'), ('1022', 'Unexpected Emails', 'Column', 'Unexpected Column Contains Emails', 'This column is not labeled as email, but contains mostly email addresses. This could indicate shifted or switched source data columns.', 'p.std_pattern_match = ''EMAIL'' AND NOT (p.column_name ILIKE ''%email%'' OR p.column_name ILIKE ''%addr%'')', '''Value Range: '' || p.min_text || '' thru '' || max_text', 'Possible', 'Review your source data and follow-up with data owners to determine whether column should be populated with email addresses.', NULL, '0.33', 'Consistency'), ('1023', 'Small_Numeric_Value_Ct', 'Column', 'Unexpected Numeric Values Found', 'A small fraction (under 3%) of values in this column were found to be numeric. They could be erroneous.', 'p.general_type = ''A'' @@ -490,7 +496,7 @@ VALUES ('1032', '1041', 'Test Results', 'Variability_Decrease', 'redshift', NULL, 'SELECT STDDEV(CAST("{COLUMN_NAME}" AS FLOAT)) as current_standard_deviation FROM {TARGET_SCHEMA}.{TABLE_NAME};'), ('1033', '1001', 'Profile Anomaly' , 'Suggested_Type', 'redshift', NULL, 'SELECT TOP 20 "{COLUMN_NAME}", COUNT(*) AS record_ct FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY "{COLUMN_NAME}" ORDER BY record_ct DESC;'), - ('1034', '1002', 'Profile Anomaly' , 'Non_Standard_Blanks', 'redshift', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE CASE WHEN "{COLUMN_NAME}" IN (''.'', ''?'', '' '') THEN 1 WHEN LOWER("{COLUMN_NAME}") SIMILAR TO ''(^.{2,}|-{2,}|0{2,}|9{2,}|x{2,}|z{2,}$)'' THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''blank'',''error'',''missing'',''tbd'', ''n/a'',''#na'',''none'',''null'',''unknown'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''(blank)'',''(error)'',''(missing)'',''(tbd)'', ''(n/a)'',''(#na)'',''(none)'',''(null)'',''(unknown)'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''[blank]'',''[error]'',''[missing]'',''[tbd]'', ''[n/a]'',''[#na]'',''[none]'',''[null]'',''[unknown]'') THEN 1 WHEN "{COLUMN_NAME}" = '''' THEN 1 WHEN "{COLUMN_NAME}" IS NULL THEN 1 ELSE 0 END = 1 GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}";'), + ('1034', '1002', 'Profile Anomaly', 'Non_Standard_Blanks', 'redshift', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE CASE WHEN "{COLUMN_NAME}" IN (''.'', ''?'', '' '') THEN 1 WHEN LOWER("{COLUMN_NAME}") SIMILAR TO ''(^.{2,}|-{2,}|0{2,}|9{2,}|x{2,}|z{2,}$)'' THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''blank'',''error'',''missing'',''tbd'', ''n/a'',''#na'',''none'',''null'',''unknown'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''(blank)'',''(error)'',''(missing)'',''(tbd)'', ''(n/a)'',''(#na)'',''(none)'',''(null)'',''(unknown)'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''[blank]'',''[error]'',''[missing]'',''[tbd]'', ''[n/a]'',''[#na]'',''[none]'',''[null]'',''[unknown]'') THEN 1 WHEN "{COLUMN_NAME}" = '''' THEN 1 WHEN "{COLUMN_NAME}" IS NULL THEN 1 ELSE 0 END = 1 GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}";'), ('1035', '1003', 'Profile Anomaly' , 'Invalid_Zip_USA', 'redshift', NULL, 'SELECT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE TRANSLATE("{COLUMN_NAME}",''012345678'',''999999999'') NOT IN (''99999'', ''999999999'', ''99999-9999'') GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}" LIMIT 500;'), ('1036', '1004', 'Profile Anomaly' , 'Multiple_Types_Minor', 'redshift', NULL, 'SELECT DISTINCT column_name, table_name, CASE WHEN data_type = ''timestamp without time zone'' THEN ''timestamp'' WHEN data_type = ''character varying'' THEN ''varchar('' || CAST(character_maximum_length AS VARCHAR) || '')'' WHEN data_type = ''character'' THEN ''char('' || CAST(character_maximum_length AS VARCHAR) || '')'' WHEN data_type = ''numeric'' THEN ''numeric('' || CAST(numeric_precision AS VARCHAR) || '','' || CAST(numeric_scale AS VARCHAR) || '')'' ELSE data_type END AS data_type FROM information_schema.columns WHERE table_schema = ''{TARGET_SCHEMA}'' AND column_name = ''{COLUMN_NAME}'' ORDER BY data_type, table_name;'), ('1037', '1005', 'Profile Anomaly' , 'Multiple_Types_Major', 'redshift', NULL, 'SELECT DISTINCT column_name, table_name, CASE WHEN data_type = ''timestamp without time zone'' THEN ''timestamp'' WHEN data_type = ''character varying'' THEN ''varchar('' || CAST(character_maximum_length AS VARCHAR) || '')'' WHEN data_type = ''character'' THEN ''char('' || CAST(character_maximum_length AS VARCHAR) || '')'' WHEN data_type = ''numeric'' THEN ''numeric('' || CAST(numeric_precision AS VARCHAR) || '','' || CAST(numeric_scale AS VARCHAR) || '')'' ELSE data_type END AS data_type FROM information_schema.columns WHERE table_schema = ''{TARGET_SCHEMA}'' AND column_name = ''{COLUMN_NAME}'' ORDER BY data_type, table_name;'), @@ -501,7 +507,7 @@ VALUES ('1042', '1010', 'Profile Anomaly' , 'Quoted_Values', 'redshift', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE (CASE WHEN "{COLUMN_NAME}" ILIKE ''"%"'' OR "{COLUMN_NAME}" ILIKE ''''''%'''''' THEN 1 ELSE 0 END) = 1 GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}" LIMIT 500;' ), ('1043', '1011', 'Profile Anomaly' , 'Char_Column_Number_Values', 'redshift', NULL, 'SELECT A.* FROM ( SELECT TOP 10 DISTINCT ''Numeric'' as data_type, "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE <%IS_NUM;"{COLUMN_NAME}"%> = 1 GROUP BY "{COLUMN_NAME}" ORDER BY count DESC) AS A UNION ALL SELECT B.* FROM ( SELECT TOP 10 DISTINCT ''Non-Numeric'' as data_type, "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE <%IS_NUM;"{COLUMN_NAME}"%> != 1 GROUP BY "{COLUMN_NAME}" ORDER BY count DESC ) AS B ORDER BY data_type, count DESC;' ), ('1044', '1012', 'Profile Anomaly' , 'Char_Column_Date_Values', 'redshift', NULL, 'SELECT A.* FROM ( SELECT TOP 10 DISTINCT ''Date'' as data_type, "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE <%IS_DATE;"{COLUMN_NAME}"%> = 1 GROUP BY "{COLUMN_NAME}" ORDER BY count DESC ) AS A UNION ALL SELECT B.* FROM ( SELECT TOP 10 DISTINCT ''Non-Date'' as data_type, "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE <%IS_DATE;"{COLUMN_NAME}"%> != 1 GROUP BY "{COLUMN_NAME}" ORDER BY count DESC ) AS B ORDER BY data_type, count DESC;' ), - ('1045', '1013', 'Profile Anomaly' , 'Small Missing Value Ct', 'redshift', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE (CASE WHEN "{COLUMN_NAME}" IN (''.'', ''?'', '' '') THEN 1 WHEN LOWER("{COLUMN_NAME}") SIMILAR TO ''(^.{2,}|-{2,}|0{2,}|9{2,}|x{2,}|z{2,}$)'' THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''blank'',''error'',''missing'',''tbd'', ''n/a'',''#na'',''none'',''null'',''unknown'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''(blank)'',''(error)'',''(missing)'',''(tbd)'', ''(n/a)'',''(#na)'',''(none)'',''(null)'',''(unknown)'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''[blank]'',''[error]'',''[missing]'',''[tbd]'', ''[n/a]'',''[#na]'',''[none]'',''[null]'',''[unknown]'') THEN 1 WHEN "{COLUMN_NAME}" = '''' THEN 1 WHEN "{COLUMN_NAME}" IS NULL THEN 1 ELSE 0 END) = 1 GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}";' ), + ('1045', '1013', 'Profile Anomaly', 'Small Missing Value Ct', 'redshift', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE (CASE WHEN "{COLUMN_NAME}" IN (''.'', ''?'', '' '') THEN 1 WHEN LOWER("{COLUMN_NAME}") SIMILAR TO ''(^.{2,}|-{2,}|0{2,}|9{2,}|x{2,}|z{2,}$)'' THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''blank'',''error'',''missing'',''tbd'', ''n/a'',''#na'',''none'',''null'',''unknown'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''(blank)'',''(error)'',''(missing)'',''(tbd)'', ''(n/a)'',''(#na)'',''(none)'',''(null)'',''(unknown)'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''[blank]'',''[error]'',''[missing]'',''[tbd]'', ''[n/a]'',''[#na]'',''[none]'',''[null]'',''[unknown]'') THEN 1 WHEN "{COLUMN_NAME}" = '''' THEN 1 WHEN "{COLUMN_NAME}" IS NULL THEN 1 ELSE 0 END) = 1 GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}";'), ('1046', '1014', 'Profile Anomaly' , 'Small Divergent Value Ct', 'redshift', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY "{COLUMN_NAME}" ORDER BY COUNT(*) DESC;' ), ('1047', '1015', 'Profile Anomaly' , 'Boolean_Value_Mismatch', 'redshift', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY "{COLUMN_NAME}" ORDER BY COUNT(*) DESC;' ), ('1048', '1016', 'Profile Anomaly' , 'Potential_Duplicates', 'redshift', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY "{COLUMN_NAME}" HAVING COUNT(*)> 1 ORDER BY COUNT(*) DESC LIMIT 500;' ), @@ -516,7 +522,7 @@ VALUES ('1057', '1025', 'Profile Anomaly' , 'Delimited_Data_Embedded', 'redshift', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE "{COLUMN_NAME}" ~ ''^([^,|\t]{1,20}[,|\t]){2,}[^,|\t]{0,20}([,|\t]{0,1}[^,|\t]{0,20})*$'' AND "{COLUMN_NAME}" !~ ''\\s(and|but|or|yet)\\s'' GROUP BY "{COLUMN_NAME}" ORDER BY COUNT(*) DESC LIMIT 500;' ), ('1058', '1001', 'Profile Anomaly' , 'Suggested_Type', 'postgresql', NULL, 'SELECT "{COLUMN_NAME}", COUNT(*) AS record_ct FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY "{COLUMN_NAME}" ORDER BY record_ct DESC LIMIT 20;'), - ('1059', '1002', 'Profile Anomaly' , 'Non_Standard_Blanks', 'postgresql', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE CASE WHEN "{COLUMN_NAME}" IN (''.'', ''?'', '' '') THEN 1 WHEN LOWER("{COLUMN_NAME}") SIMILAR TO ''(^.{2,}|-{2,}|0{2,}|9{2,}|x{2,}|z{2,}$)'' THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''blank'',''error'',''missing'',''tbd'', ''n/a'',''#na'',''none'',''null'',''unknown'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''(blank)'',''(error)'',''(missing)'',''(tbd)'', ''(n/a)'',''(#na)'',''(none)'',''(null)'',''(unknown)'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''[blank]'',''[error]'',''[missing]'',''[tbd]'', ''[n/a]'',''[#na]'',''[none]'',''[null]'',''[unknown]'') THEN 1 WHEN "{COLUMN_NAME}" = '''' THEN 1 WHEN "{COLUMN_NAME}" IS NULL THEN 1 ELSE 0 END = 1 GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}";'), + ('1059', '1002', 'Profile Anomaly', 'Non_Standard_Blanks', 'postgresql', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE CASE WHEN "{COLUMN_NAME}" IN (''.'', ''?'', '' '') THEN 1 WHEN LOWER("{COLUMN_NAME}") SIMILAR TO ''(^.{2,}|-{2,}|0{2,}|9{2,}|x{2,}|z{2,}$)'' THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''blank'',''error'',''missing'',''tbd'', ''n/a'',''#na'',''none'',''null'',''unknown'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''(blank)'',''(error)'',''(missing)'',''(tbd)'', ''(n/a)'',''(#na)'',''(none)'',''(null)'',''(unknown)'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''[blank]'',''[error]'',''[missing]'',''[tbd]'', ''[n/a]'',''[#na]'',''[none]'',''[null]'',''[unknown]'') THEN 1 WHEN "{COLUMN_NAME}" = '''' THEN 1 WHEN "{COLUMN_NAME}" IS NULL THEN 1 ELSE 0 END = 1 GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}";'), ('1060', '1003', 'Profile Anomaly' , 'Invalid_Zip_USA', 'postgresql', NULL, 'SELECT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE TRANSLATE("{COLUMN_NAME}",''012345678'',''999999999'') NOT IN (''99999'', ''999999999'', ''99999-9999'') GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}" LIMIT 500;'), ('1061', '1004', 'Profile Anomaly' , 'Multiple_Types_Minor', 'postgresql', NULL, 'SELECT DISTINCT column_name, columns.table_name, CASE WHEN data_type = ''timestamp without time zone'' THEN ''timestamp'' WHEN data_type = ''character varying'' THEN ''varchar('' || CAST(character_maximum_length AS VARCHAR) || '')'' WHEN data_type = ''character'' THEN ''char('' || CAST(character_maximum_length AS VARCHAR) || '')'' WHEN data_type = ''numeric'' THEN ''numeric('' || CAST(numeric_precision AS VARCHAR) || '','' || CAST(numeric_scale AS VARCHAR) || '')'' ELSE data_type END AS data_type FROM information_schema.columns JOIN information_schema.tables ON columns.table_name = tables.table_name AND columns.table_schema = tables.table_schema WHERE columns.table_schema = ''{TARGET_SCHEMA}'' AND columns.column_name = ''{COLUMN_NAME}'' AND UPPER(tables.table_type) = ''BASE TABLE'' ORDER BY data_type, table_name;'), ('1062', '1005', 'Profile Anomaly' , 'Multiple_Types_Major', 'postgresql', NULL, 'SELECT DISTINCT column_name, columns.table_name, CASE WHEN data_type = ''timestamp without time zone'' THEN ''timestamp'' WHEN data_type = ''character varying'' THEN ''varchar('' || CAST(character_maximum_length AS VARCHAR) || '')'' WHEN data_type = ''character'' THEN ''char('' || CAST(character_maximum_length AS VARCHAR) || '')'' WHEN data_type = ''numeric'' THEN ''numeric('' || CAST(numeric_precision AS VARCHAR) || '','' || CAST(numeric_scale AS VARCHAR) || '')'' ELSE data_type END AS data_type FROM information_schema.columns JOIN information_schema.tables ON columns.table_name = tables.table_name AND columns.table_schema = tables.table_schema WHERE columns.table_schema = ''{TARGET_SCHEMA}'' AND columns.column_name = ''{COLUMN_NAME}'' AND UPPER(tables.table_type) = ''BASE TABLE'' ORDER BY data_type, table_name;'), @@ -527,7 +533,7 @@ VALUES ('1067', '1010', 'Profile Anomaly' , 'Quoted_Values', 'postgresql', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE (CASE WHEN "{COLUMN_NAME}" ILIKE ''"%"'' OR "{COLUMN_NAME}" ILIKE ''''''%'''''' THEN 1 ELSE 0 END) = 1 GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}" LIMIT 500;' ), ('1068', '1011', 'Profile Anomaly' , 'Char_Column_Number_Values', 'postgresql', NULL, 'SELECT A.* FROM ( SELECT DISTINCT ''Numeric'' as data_type, "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE <%IS_NUM;"{COLUMN_NAME}"%> = 1 GROUP BY "{COLUMN_NAME}" ORDER BY count DESC LIMIT 10 ) AS A UNION ALL SELECT B.* FROM ( SELECT DISTINCT ''Non-Numeric'' as data_type, "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE <%IS_NUM;"{COLUMN_NAME}"%> != 1 GROUP BY "{COLUMN_NAME}" ORDER BY count DESC LIMIT 10 ) AS B ORDER BY data_type, count DESC;' ), ('1069', '1012', 'Profile Anomaly' , 'Char_Column_Date_Values', 'postgresql', NULL, 'SELECT A.* FROM ( SELECT DISTINCT ''Date'' as data_type, "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE <%IS_DATE;"{COLUMN_NAME}"%> = 1 GROUP BY "{COLUMN_NAME}" ORDER BY count DESC LIMIT 10) AS A UNION ALL SELECT B.* FROM ( SELECT DISTINCT ''Non-Date'' as data_type, "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE <%IS_DATE;"{COLUMN_NAME}"%> != 1 GROUP BY "{COLUMN_NAME}" ORDER BY count DESC LIMIT 10) AS B ORDER BY data_type, count DESC;' ), - ('1070', '1013', 'Profile Anomaly' , 'Small Missing Value Ct', 'postgresql', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE (CASE WHEN "{COLUMN_NAME}" IN (''.'', ''?'', '' '') THEN 1 WHEN LOWER("{COLUMN_NAME}") SIMILAR TO ''(^.{2,}|-{2,}|0{2,}|9{2,}|x{2,}|z{2,}$)'' THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''blank'',''error'',''missing'',''tbd'', ''n/a'',''#na'',''none'',''null'',''unknown'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''(blank)'',''(error)'',''(missing)'',''(tbd)'', ''(n/a)'',''(#na)'',''(none)'',''(null)'',''(unknown)'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''[blank]'',''[error]'',''[missing]'',''[tbd]'', ''[n/a]'',''[#na]'',''[none]'',''[null]'',''[unknown]'') THEN 1 WHEN "{COLUMN_NAME}" = '''' THEN 1 WHEN "{COLUMN_NAME}" IS NULL THEN 1 ELSE 0 END) = 1 GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}";' ), + ('1070', '1013', 'Profile Anomaly', 'Small Missing Value Ct', 'postgresql', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE (CASE WHEN "{COLUMN_NAME}" IN (''.'', ''?'', '' '') THEN 1 WHEN LOWER("{COLUMN_NAME}") SIMILAR TO ''(^.{2,}|-{2,}|0{2,}|9{2,}|x{2,}|z{2,}$)'' THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''blank'',''error'',''missing'',''tbd'', ''n/a'',''#na'',''none'',''null'',''unknown'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''(blank)'',''(error)'',''(missing)'',''(tbd)'', ''(n/a)'',''(#na)'',''(none)'',''(null)'',''(unknown)'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''[blank]'',''[error]'',''[missing]'',''[tbd]'', ''[n/a]'',''[#na]'',''[none]'',''[null]'',''[unknown]'') THEN 1 WHEN "{COLUMN_NAME}" = '''' THEN 1 WHEN "{COLUMN_NAME}" IS NULL THEN 1 ELSE 0 END) = 1 GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}";'), ('1071', '1014', 'Profile Anomaly' , 'Small Divergent Value Ct', 'postgresql', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY "{COLUMN_NAME}" ORDER BY COUNT(*) DESC;' ), ('1072', '1015', 'Profile Anomaly' , 'Boolean_Value_Mismatch', 'postgresql', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY "{COLUMN_NAME}" ORDER BY COUNT(*) DESC;' ), ('1073', '1016', 'Profile Anomaly' , 'Potential_Duplicates', 'postgresql', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY "{COLUMN_NAME}" HAVING COUNT(*)> 1 ORDER BY COUNT(*) DESC LIMIT 500;' ), @@ -576,7 +582,7 @@ VALUES ('1114', '1041', 'Test Results', 'Variability_Decrease', 'postgresql', NULL, 'SELECT STDDEV(CAST("{COLUMN_NAME}" AS FLOAT)) as current_standard_deviation FROM {TARGET_SCHEMA}.{TABLE_NAME};'), ('1115', '1001', 'Profile Anomaly' , 'Suggested_Type', 'mssql', NULL, 'SELECT TOP 20 "{COLUMN_NAME}", COUNT(*) AS record_ct FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY "{COLUMN_NAME}" ORDER BY record_ct DESC;'), - ('1116', '1002', 'Profile Anomaly' , 'Non_Standard_Blanks', 'mssql', NULL, 'SELECT TOP 500 "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE CASE WHEN "{COLUMN_NAME}" IN (''.'', ''?'') OR "{COLUMN_NAME}" LIKE '' '' THEN 1 WHEN LEN("{COLUMN_NAME}") > 1 AND ( LOWER("{COLUMN_NAME}") LIKE ''%..%'' OR LOWER("{COLUMN_NAME}") LIKE ''%--%'' OR (LEN(REPLACE("{COLUMN_NAME}", ''0'', ''''))= 0 ) OR (LEN(REPLACE("{COLUMN_NAME}", ''9'', ''''))= 0 ) OR (LEN(REPLACE(LOWER("{COLUMN_NAME}"), ''x'', ''''))= 0 ) OR (LEN(REPLACE(LOWER("{COLUMN_NAME}"), ''z'', ''''))= 0 ) ) THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''blank'',''error'',''missing'',''tbd'', ''n/a'',''#na'',''none'',''null'',''unknown'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''(blank)'',''(error)'',''(missing)'',''(tbd)'', ''(n/a)'',''(#na)'',''(none)'',''(null)'',''(unknown)'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''[blank]'',''[error]'',''[missing]'',''[tbd]'', ''[n/a]'',''[#na]'',''[none]'',''[null]'',''[unknown]'') THEN 1 WHEN "{COLUMN_NAME}" = '''' THEN 1 WHEN "{COLUMN_NAME}" IS NULL THEN 1 ELSE 0 END = 1 GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}";'), + ('1116', '1002', 'Profile Anomaly', 'Non_Standard_Blanks', 'mssql', NULL, 'SELECT TOP 500 "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE CASE WHEN "{COLUMN_NAME}" IN (''.'', ''?'') OR "{COLUMN_NAME}" LIKE '' '' THEN 1 WHEN LEN("{COLUMN_NAME}") > 1 AND ( LOWER("{COLUMN_NAME}") LIKE ''%..%'' OR LOWER("{COLUMN_NAME}") LIKE ''%--%'' OR (LEN(REPLACE("{COLUMN_NAME}", ''0'', ''''))= 0 ) OR (LEN(REPLACE("{COLUMN_NAME}", ''9'', ''''))= 0 ) OR (LEN(REPLACE(LOWER("{COLUMN_NAME}"), ''x'', ''''))= 0 ) OR (LEN(REPLACE(LOWER("{COLUMN_NAME}"), ''z'', ''''))= 0 ) ) THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''blank'',''error'',''missing'',''tbd'', ''n/a'',''#na'',''none'',''null'',''unknown'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''(blank)'',''(error)'',''(missing)'',''(tbd)'', ''(n/a)'',''(#na)'',''(none)'',''(null)'',''(unknown)'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''[blank]'',''[error]'',''[missing]'',''[tbd]'', ''[n/a]'',''[#na]'',''[none]'',''[null]'',''[unknown]'') THEN 1 WHEN "{COLUMN_NAME}" = '''' THEN 1 WHEN "{COLUMN_NAME}" IS NULL THEN 1 ELSE 0 END = 1 GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}";'), ('1117', '1003', 'Profile Anomaly' , 'Invalid_Zip_USA', 'mssql', NULL, 'SELECT TOP 500 "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE TRANSLATE("{COLUMN_NAME}",''012345678'',''999999999'') NOT IN (''99999'', ''999999999'', ''99999-9999'') GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}";'), ('1118', '1004', 'Profile Anomaly' , 'Multiple_Types_Minor', 'mssql', NULL, 'SELECT TOP 500 column_name, columns.table_name, CASE WHEN data_type = ''datetime'' THEN ''datetime'' WHEN data_type = ''datetime2'' THEN ''datetime'' WHEN data_type = ''varchar'' THEN ''varchar('' + CAST(character_maximum_length AS VARCHAR) + '')'' WHEN data_type = ''char'' THEN ''char('' + CAST(character_maximum_length AS VARCHAR) + '')'' WHEN data_type = ''numeric'' THEN ''numeric('' + CAST(numeric_precision AS VARCHAR) + '','' + CAST(numeric_scale AS VARCHAR) + '')'' ELSE data_type END AS data_type FROM information_schema.columns JOIN information_schema.tables ON columns.table_name = tables.table_name AND columns.table_schema = tables.table_schema WHERE columns.table_schema = ''{TARGET_SCHEMA}'' AND columns.column_name = ''{COLUMN_NAME}'' AND tables.table_type = ''BASE TABLE'' ORDER BY data_type, table_name;'), ('1119', '1005', 'Profile Anomaly' , 'Multiple_Types_Major', 'mssql', NULL, 'SELECT TOP 500 column_name, columns.table_name, CASE WHEN data_type = ''datetime'' THEN ''datetime'' WHEN data_type = ''datetime2'' THEN ''datetime'' WHEN data_type = ''varchar'' THEN ''varchar('' + CAST(character_maximum_length AS VARCHAR) + '')'' WHEN data_type = ''char'' THEN ''char('' + CAST(character_maximum_length AS VARCHAR) + '')'' WHEN data_type = ''numeric'' THEN ''numeric('' + CAST(numeric_precision AS VARCHAR) + '','' + CAST(numeric_scale AS VARCHAR) + '')'' ELSE data_type END AS data_type FROM information_schema.columns JOIN information_schema.tables ON columns.table_name = tables.table_name AND columns.table_schema = tables.table_schema WHERE columns.table_schema = ''{TARGET_SCHEMA}'' AND columns.column_name = ''{COLUMN_NAME}'' AND tables.table_type = ''BASE TABLE'' ORDER BY data_type, table_name;'), @@ -587,7 +593,7 @@ VALUES ('1124', '1010', 'Profile Anomaly' , 'Quoted_Values', 'mssql', NULL, 'SELECT TOP 500 "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE (CASE WHEN "{COLUMN_NAME}" LIKE ''"%"'' OR "{COLUMN_NAME}" LIKE ''''''%'''''' THEN 1 ELSE 0 END) = 1 GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}";' ), ('1125', '1011', 'Profile Anomaly' , 'Char_Column_Number_Values', 'mssql', NULL, 'SELECT A.* FROM ( SELECT DISTINCT TOP 10 ''Numeric'' as data_type, "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE <%IS_NUM;"{COLUMN_NAME}"%> = 1 GROUP BY "{COLUMN_NAME}" ORDER BY count DESC ) AS A UNION ALL SELECT B.* FROM ( SELECT DISTINCT TOP 10 ''Non-Numeric'' as data_type, "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE <%IS_NUM;"{COLUMN_NAME}"%> != 1 GROUP BY "{COLUMN_NAME}" ORDER BY count DESC ) AS B ORDER BY data_type, count DESC;' ), ('1126', '1012', 'Profile Anomaly' , 'Char_Column_Date_Values', 'mssql', NULL, 'SELECT A.* FROM ( SELECT DISTINCT TOP 10 ''Date'' as data_type, "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE <%IS_DATE;"{COLUMN_NAME}"%> = 1 GROUP BY "{COLUMN_NAME}" ORDER BY count DESC) AS A UNION ALL SELECT B.* FROM ( SELECT DISTINCT TOP 10 ''Non-Date'' as data_type, "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE <%IS_DATE;"{COLUMN_NAME}"%> != 1 GROUP BY "{COLUMN_NAME}" ORDER BY count DESC ) AS B ORDER BY data_type, count DESC;' ), - ('1127', '1013', 'Profile Anomaly' , 'Small Missing Value Ct', 'mssql', NULL, 'SELECT TOP 500 "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE (CASE WHEN "{COLUMN_NAME}" IN (''.'', ''?'', '' '') THEN 1 WHEN LEN("{COLUMN_NAME}") > 1 AND ( LOWER("{COLUMN_NAME}") LIKE ''%..%'' OR LOWER("{COLUMN_NAME}") LIKE ''%--%'' OR (LEN(REPLACE("{COLUMN_NAME}", ''0'', ''''))= 0 ) OR (LEN(REPLACE("{COLUMN_NAME}", ''9'', ''''))= 0 ) OR (LEN(REPLACE(LOWER("{COLUMN_NAME}"), ''x'', ''''))= 0 ) OR (LEN(REPLACE(LOWER("{COLUMN_NAME}"), ''z'', ''''))= 0 ) ) THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''blank'',''error'',''missing'',''tbd'', ''n/a'',''#na'',''none'',''null'',''unknown'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''(blank)'',''(error)'',''(missing)'',''(tbd)'', ''(n/a)'',''(#na)'',''(none)'',''(null)'',''(unknown)'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''[blank]'',''[error]'',''[missing]'',''[tbd]'', ''[n/a]'',''[#na]'',''[none]'',''[null]'',''[unknown]'') THEN 1 WHEN "{COLUMN_NAME}" = '''' THEN 1 WHEN "{COLUMN_NAME}" IS NULL THEN 1 ELSE 0 END) = 1 GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}";' ), + ('1127', '1013', 'Profile Anomaly', 'Small Missing Value Ct', 'mssql', NULL, 'SELECT TOP 500 "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE (CASE WHEN "{COLUMN_NAME}" IN (''.'', ''?'', '' '') THEN 1 WHEN LEN("{COLUMN_NAME}") > 1 AND ( LOWER("{COLUMN_NAME}") LIKE ''%..%'' OR LOWER("{COLUMN_NAME}") LIKE ''%--%'' OR (LEN(REPLACE("{COLUMN_NAME}", ''0'', ''''))= 0 ) OR (LEN(REPLACE("{COLUMN_NAME}", ''9'', ''''))= 0 ) OR (LEN(REPLACE(LOWER("{COLUMN_NAME}"), ''x'', ''''))= 0 ) OR (LEN(REPLACE(LOWER("{COLUMN_NAME}"), ''z'', ''''))= 0 ) ) THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''blank'',''error'',''missing'',''tbd'', ''n/a'',''#na'',''none'',''null'',''unknown'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''(blank)'',''(error)'',''(missing)'',''(tbd)'', ''(n/a)'',''(#na)'',''(none)'',''(null)'',''(unknown)'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''[blank]'',''[error]'',''[missing]'',''[tbd]'', ''[n/a]'',''[#na]'',''[none]'',''[null]'',''[unknown]'') THEN 1 WHEN "{COLUMN_NAME}" = '''' THEN 1 WHEN "{COLUMN_NAME}" IS NULL THEN 1 ELSE 0 END) = 1 GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}";'), ('1128', '1014', 'Profile Anomaly' , 'Small Divergent Value Ct', 'mssql', NULL, 'SELECT TOP 500 "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY "{COLUMN_NAME}" ORDER BY COUNT(*) DESC;' ), ('1129', '1015', 'Profile Anomaly' , 'Boolean_Value_Mismatch', 'mssql', NULL, 'SELECT TOP 500 "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY "{COLUMN_NAME}" ORDER BY COUNT(*) DESC;' ), ('1130', '1016', 'Profile Anomaly' , 'Potential_Duplicates', 'mssql', NULL, 'SELECT TOP 500 "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY "{COLUMN_NAME}" HAVING COUNT(*)> 1 ORDER BY COUNT(*) DESC;' ), @@ -755,7 +761,7 @@ ORDER BY check_period DESC;'), ('1171', '1041', 'Test Results', 'Variability_Decrease', 'mssql', NULL, 'SELECT STDEV(CAST("{COLUMN_NAME}" AS FLOAT)) as current_standard_deviation FROM {TARGET_SCHEMA}.{TABLE_NAME};'), ('1172', '1001', 'Profile Anomaly' , 'Suggested_Type', 'snowflake', NULL, 'SELECT TOP 20 "{COLUMN_NAME}", COUNT(*) AS record_ct FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY "{COLUMN_NAME}" ORDER BY record_ct DESC;'), - ('1173', '1002', 'Profile Anomaly' , 'Non_Standard_Blanks', 'snowflake', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE CASE WHEN "{COLUMN_NAME}" IN (''.'', ''?'', '' '') THEN 1 WHEN LOWER("{COLUMN_NAME}"::VARCHAR) REGEXP ''-{2,}'' OR LOWER("{COLUMN_NAME}"::VARCHAR) REGEXP ''0{2,}'' OR LOWER("{COLUMN_NAME}"::VARCHAR) REGEXP ''9{2,}'' OR LOWER("{COLUMN_NAME}"::VARCHAR) REGEXP ''x{2,}'' OR LOWER("{COLUMN_NAME}"::VARCHAR) REGEXP ''z{2,}'' THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''blank'',''error'',''missing'',''tbd'', ''n/a'',''#na'',''none'',''null'',''unknown'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''(blank)'',''(error)'',''(missing)'',''(tbd)'', ''(n/a)'',''(#na)'',''(none)'',''(null)'',''(unknown)'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''[blank]'',''[error]'',''[missing]'',''[tbd]'', ''[n/a]'',''[#na]'',''[none]'',''[null]'',''[unknown]'') THEN 1 WHEN "{COLUMN_NAME}" = '''' THEN 1 WHEN "{COLUMN_NAME}" IS NULL THEN 1 ELSE 0 END = 1 GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}";'), + ('1173', '1002', 'Profile Anomaly', 'Non_Standard_Blanks', 'snowflake', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE CASE WHEN "{COLUMN_NAME}" IN (''.'', ''?'', '' '') THEN 1 WHEN LOWER("{COLUMN_NAME}"::VARCHAR) REGEXP ''-{2,}'' OR LOWER("{COLUMN_NAME}"::VARCHAR) REGEXP ''0{2,}'' OR LOWER("{COLUMN_NAME}"::VARCHAR) REGEXP ''9{2,}'' OR LOWER("{COLUMN_NAME}"::VARCHAR) REGEXP ''x{2,}'' OR LOWER("{COLUMN_NAME}"::VARCHAR) REGEXP ''z{2,}'' THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''blank'',''error'',''missing'',''tbd'', ''n/a'',''#na'',''none'',''null'',''unknown'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''(blank)'',''(error)'',''(missing)'',''(tbd)'', ''(n/a)'',''(#na)'',''(none)'',''(null)'',''(unknown)'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''[blank]'',''[error]'',''[missing]'',''[tbd]'', ''[n/a]'',''[#na]'',''[none]'',''[null]'',''[unknown]'') THEN 1 WHEN "{COLUMN_NAME}" = '''' THEN 1 WHEN "{COLUMN_NAME}" IS NULL THEN 1 ELSE 0 END = 1 GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}";'), ('1174', '1003', 'Profile Anomaly' , 'Invalid_Zip_USA', 'snowflake', NULL, 'SELECT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE TRANSLATE("{COLUMN_NAME}",''012345678'',''999999999'') NOT IN (''99999'', ''999999999'', ''99999-9999'') GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}" LIMIT 500;'), ('1175', '1004', 'Profile Anomaly' , 'Multiple_Types_Minor', 'snowflake', NULL, 'SELECT DISTINCT column_name, columns.table_name, CASE WHEN data_type ILIKE ''timestamp%'' THEN lower(data_type) WHEN data_type ILIKE ''date'' THEN lower(data_type) WHEN data_type ILIKE ''boolean'' THEN ''boolean'' WHEN data_type = ''TEXT'' THEN ''varchar('' || CAST(character_maximum_length AS VARCHAR) || '')'' WHEN data_type ILIKE ''char%'' THEN ''char('' || CAST(character_maximum_length AS VARCHAR) || '')'' WHEN data_type = ''NUMBER'' AND numeric_precision = 38 AND numeric_scale = 0 THEN ''bigint'' WHEN data_type ILIKE ''num%'' THEN ''numeric('' || CAST(numeric_precision AS VARCHAR) || '','' || CAST(numeric_scale AS VARCHAR) || '')'' ELSE data_type END AS data_type FROM information_schema.columns JOIN information_schema.tables ON columns.table_name = tables.table_name AND columns.table_schema = tables.table_schema WHERE columns.table_schema = ''{TARGET_SCHEMA}'' AND columns.column_name = ''{COLUMN_NAME}'' AND tables.table_type = ''BASE TABLE'' ORDER BY data_type, table_name;'), ('1176', '1005', 'Profile Anomaly' , 'Multiple_Types_Major', 'snowflake', NULL, 'SELECT DISTINCT column_name, columns.table_name, CASE WHEN data_type ILIKE ''timestamp%'' THEN lower(data_type) WHEN data_type ILIKE ''date'' THEN lower(data_type) WHEN data_type ILIKE ''boolean'' THEN ''boolean'' WHEN data_type = ''TEXT'' THEN ''varchar('' || CAST(character_maximum_length AS VARCHAR) || '')'' WHEN data_type ILIKE ''char%'' THEN ''char('' || CAST(character_maximum_length AS VARCHAR) || '')'' WHEN data_type = ''NUMBER'' AND numeric_precision = 38 AND numeric_scale = 0 THEN ''bigint'' WHEN data_type ILIKE ''num%'' THEN ''numeric('' || CAST(numeric_precision AS VARCHAR) || '','' || CAST(numeric_scale AS VARCHAR) || '')'' ELSE data_type END AS data_type FROM information_schema.columns JOIN information_schema.tables ON columns.table_name = tables.table_name AND columns.table_schema = tables.table_schema WHERE columns.table_schema = ''{TARGET_SCHEMA}'' AND columns.column_name = ''{COLUMN_NAME}'' AND tables.table_type = ''BASE TABLE'' ORDER BY data_type, table_name;'), @@ -766,7 +772,7 @@ ORDER BY check_period DESC;'), ('1181', '1010', 'Profile Anomaly' , 'Quoted_Values', 'snowflake', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE (CASE WHEN "{COLUMN_NAME}" ILIKE ''"%"'' OR "{COLUMN_NAME}" ILIKE ''''''%'''''' THEN 1 ELSE 0 END) = 1 GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}" LIMIT 500;' ), ('1182', '1011', 'Profile Anomaly' , 'Char_Column_Number_Values', 'snowflake', NULL, 'SELECT A.* FROM (SELECT DISTINCT TOP 10 ''Numeric'' as data_type, "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE <%IS_NUM;"{COLUMN_NAME}"%> = 1 GROUP BY "{COLUMN_NAME}" ORDER BY count DESC) AS A UNION ALL SELECT B.* FROM (SELECT DISTINCT TOP 10 ''Non-Numeric'' as data_type, "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE <%IS_NUM;"{COLUMN_NAME}"%> != 1 GROUP BY "{COLUMN_NAME}" ORDER BY count DESC) AS B ORDER BY data_type, count DESC;' ), ('1183', '1012', 'Profile Anomaly' , 'Char_Column_Date_Values', 'snowflake', NULL, 'SELECT A.* FROM (SELECT DISTINCT TOP 10 ''Date'' as data_type, "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE <%IS_DATE;"{COLUMN_NAME}"%> = 1 GROUP BY "{COLUMN_NAME}" ORDER BY count DESC) AS A UNION ALL SELECT B.* FROM (SELECT DISTINCT TOP 10 ''Non-Date'' as data_type, "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE <%IS_DATE;"{COLUMN_NAME}"%> != 1 GROUP BY "{COLUMN_NAME}" ORDER BY count DESC) AS B ORDER BY data_type, count DESC;' ), - ('1184', '1013', 'Profile Anomaly' , 'Small Missing Value Ct', 'snowflake', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE (CASE WHEN "{COLUMN_NAME}" IN (''.'', ''?'', '' '') THEN 1 WHEN LOWER("{COLUMN_NAME}"::VARCHAR) REGEXP ''-{2,}'' OR LOWER("{COLUMN_NAME}"::VARCHAR) REGEXP ''0{2,}'' OR LOWER("{COLUMN_NAME}"::VARCHAR) REGEXP ''9{2,}'' OR LOWER("{COLUMN_NAME}"::VARCHAR) REGEXP ''x{2,}'' OR LOWER("{COLUMN_NAME}"::VARCHAR) REGEXP ''z{2,}'' THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''blank'',''error'',''missing'',''tbd'', ''n/a'',''#na'',''none'',''null'',''unknown'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''(blank)'',''(error)'',''(missing)'',''(tbd)'', ''(n/a)'',''(#na)'',''(none)'',''(null)'',''(unknown)'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''[blank]'',''[error]'',''[missing]'',''[tbd]'', ''[n/a]'',''[#na]'',''[none]'',''[null]'',''[unknown]'') THEN 1 WHEN "{COLUMN_NAME}" = '''' THEN 1 WHEN "{COLUMN_NAME}" IS NULL THEN 1 ELSE 0 END) = 1 GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}";' ), + ('1184', '1013', 'Profile Anomaly', 'Small Missing Value Ct', 'snowflake', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE (CASE WHEN "{COLUMN_NAME}" IN (''.'', ''?'', '' '') THEN 1 WHEN LOWER("{COLUMN_NAME}"::VARCHAR) REGEXP ''-{2,}'' OR LOWER("{COLUMN_NAME}"::VARCHAR) REGEXP ''0{2,}'' OR LOWER("{COLUMN_NAME}"::VARCHAR) REGEXP ''9{2,}'' OR LOWER("{COLUMN_NAME}"::VARCHAR) REGEXP ''x{2,}'' OR LOWER("{COLUMN_NAME}"::VARCHAR) REGEXP ''z{2,}'' THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''blank'',''error'',''missing'',''tbd'', ''n/a'',''#na'',''none'',''null'',''unknown'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''(blank)'',''(error)'',''(missing)'',''(tbd)'', ''(n/a)'',''(#na)'',''(none)'',''(null)'',''(unknown)'') THEN 1 WHEN LOWER("{COLUMN_NAME}") IN (''[blank]'',''[error]'',''[missing]'',''[tbd]'', ''[n/a]'',''[#na]'',''[none]'',''[null]'',''[unknown]'') THEN 1 WHEN "{COLUMN_NAME}" = '''' THEN 1 WHEN "{COLUMN_NAME}" IS NULL THEN 1 ELSE 0 END) = 1 GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}";'), ('1185', '1014', 'Profile Anomaly' , 'Small Divergent Value Ct', 'snowflake', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY "{COLUMN_NAME}" ORDER BY COUNT(*) DESC;' ), ('1186', '1015', 'Profile Anomaly' , 'Boolean_Value_Mismatch', 'snowflake', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY "{COLUMN_NAME}" ORDER BY COUNT(*) DESC;' ), ('1187', '1016', 'Profile Anomaly' , 'Potential_Duplicates', 'snowflake', NULL, 'SELECT DISTINCT "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY "{COLUMN_NAME}" HAVING COUNT(*)> 1 ORDER BY COUNT(*) DESC LIMIT 500;' ), @@ -1421,7 +1427,7 @@ WHERE {SUBSET_CONDITION} ('1282', '1010', 'Profile Anomaly' , 'Quoted_Values', 'databricks', NULL, 'SELECT DISTINCT `{COLUMN_NAME}`, COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE (CASE WHEN `{COLUMN_NAME}` ILIKE ''"%"'' OR `{COLUMN_NAME}` ILIKE ''''''%'''''' THEN 1 ELSE 0 END) = 1 GROUP BY `{COLUMN_NAME}` ORDER BY `{COLUMN_NAME}` LIMIT 500;' ), ('1283', '1011', 'Profile Anomaly' , 'Char_Column_Number_Values', 'databricks', NULL, 'SELECT A.* FROM (SELECT DISTINCT ''Numeric'' as data_type, `{COLUMN_NAME}`, COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE <%IS_NUM;`{COLUMN_NAME}`%> = 1 GROUP BY `{COLUMN_NAME}` ORDER BY count DESC LIMIT 10) AS A UNION ALL SELECT B.* FROM (SELECT DISTINCT ''Non-Numeric'' as data_type, `{COLUMN_NAME}`, COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE <%IS_NUM;`{COLUMN_NAME}`%> != 1 GROUP BY `{COLUMN_NAME}` ORDER BY count DESC) AS B ORDER BY data_type, count DESC LIMIT 10;' ), ('1284', '1012', 'Profile Anomaly' , 'Char_Column_Date_Values', 'databricks', NULL, 'SELECT A.* FROM (SELECT DISTINCT ''Date'' as data_type, `{COLUMN_NAME}`, COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE <%IS_DATE;`{COLUMN_NAME}`%> = 1 GROUP BY `{COLUMN_NAME}` ORDER BY count DESC LIMIT 10) AS A UNION ALL SELECT B.* FROM (SELECT DISTINCT ''Non-Date'' as data_type, `{COLUMN_NAME}`, COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE <%IS_DATE;`{COLUMN_NAME}`%> != 1 GROUP BY `{COLUMN_NAME}` ORDER BY count DESC) AS B ORDER BY data_type, count DESC LIMIT 10;' ), - ('1285', '1013', 'Profile Anomaly' , 'Small Missing Value Ct', 'databricks', NULL, 'SELECT DISTINCT `{COLUMN_NAME}`, COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE (CASE WHEN `{COLUMN_NAME}` IN (''.'', ''?'', '' '') THEN 1 WHEN LOWER(`{COLUMN_NAME}`::STRING) REGEXP ''-{2,}'' OR LOWER(`{COLUMN_NAME}`::STRING) REGEXP ''0{2,}'' OR LOWER(`{COLUMN_NAME}`::STRING) REGEXP ''9{2,}'' OR LOWER(`{COLUMN_NAME}`::STRING) REGEXP ''x{2,}'' OR LOWER(`{COLUMN_NAME}`::STRING) REGEXP ''z{2,}'' THEN 1 WHEN LOWER(`{COLUMN_NAME}`) IN (''blank'',''error'',''missing'',''tbd'', ''n/a'',''#na'',''none'',''null'',''unknown'') THEN 1 WHEN LOWER(`{COLUMN_NAME}`) IN (''(blank)'',''(error)'',''(missing)'',''(tbd)'', ''(n/a)'',''(#na)'',''(none)'',''(null)'',''(unknown)'') THEN 1 WHEN LOWER(`{COLUMN_NAME}`) IN (''[blank]'',''[error]'',''[missing]'',''[tbd]'', ''[n/a]'',''[#na]'',''[none]'',''[null]'',''[unknown]'') THEN 1 WHEN `{COLUMN_NAME}` = '''' THEN 1 WHEN `{COLUMN_NAME}` IS NULL THEN 1 ELSE 0 END) = 1 GROUP BY `{COLUMN_NAME}` ORDER BY `{COLUMN_NAME}`;' ), + ('1285', '1013', 'Profile Anomaly', 'Small Missing Value Ct', 'databricks', NULL, 'SELECT DISTINCT `{COLUMN_NAME}`, COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE (CASE WHEN `{COLUMN_NAME}` IN (''.'', ''?'', '' '') THEN 1 WHEN LOWER(`{COLUMN_NAME}`::STRING) REGEXP ''-{2,}'' OR LOWER(`{COLUMN_NAME}`::STRING) REGEXP ''0{2,}'' OR LOWER(`{COLUMN_NAME}`::STRING) REGEXP ''9{2,}'' OR LOWER(`{COLUMN_NAME}`::STRING) REGEXP ''x{2,}'' OR LOWER(`{COLUMN_NAME}`::STRING) REGEXP ''z{2,}'' THEN 1 WHEN LOWER(`{COLUMN_NAME}`) IN (''blank'',''error'',''missing'',''tbd'', ''n/a'',''#na'',''none'',''null'',''unknown'') THEN 1 WHEN LOWER(`{COLUMN_NAME}`) IN (''(blank)'',''(error)'',''(missing)'',''(tbd)'', ''(n/a)'',''(#na)'',''(none)'',''(null)'',''(unknown)'') THEN 1 WHEN LOWER(`{COLUMN_NAME}`) IN (''[blank]'',''[error]'',''[missing]'',''[tbd]'', ''[n/a]'',''[#na]'',''[none]'',''[null]'',''[unknown]'') THEN 1 WHEN `{COLUMN_NAME}` = '''' THEN 1 WHEN `{COLUMN_NAME}` IS NULL THEN 1 ELSE 0 END) = 1 GROUP BY `{COLUMN_NAME}` ORDER BY `{COLUMN_NAME}`;'), ('1286', '1014', 'Profile Anomaly' , 'Small Divergent Value Ct', 'databricks', NULL, 'SELECT DISTINCT `{COLUMN_NAME}`, COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY `{COLUMN_NAME}` ORDER BY count DESC;' ), ('1287', '1015', 'Profile Anomaly' , 'Boolean_Value_Mismatch', 'databricks', NULL, 'SELECT DISTINCT `{COLUMN_NAME}`, COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY `{COLUMN_NAME}` ORDER BY count DESC;' ), ('1288', '1016', 'Profile Anomaly' , 'Potential_Duplicates', 'databricks', NULL, 'SELECT DISTINCT `{COLUMN_NAME}`, COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} GROUP BY `{COLUMN_NAME}` HAVING count > 1 ORDER BY count DESC LIMIT 500;' ), @@ -1652,7 +1658,7 @@ GROUP BY "{COLUMN_NAME}" LIMIT 20)'), ('1260', '1028', 'Profile Anomaly', 'Inconsistent_Casing', 'mssql', NULL, 'SELECT TOP 20 ''Upper Case'' as casing, "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE UPPER("{COLUMN_NAME}") = "{COLUMN_NAME}" GROUP BY "{COLUMN_NAME}" -UNION +UNION ALL SELECT TOP 20 ''Mixed Case'' as casing, "{COLUMN_NAME}", COUNT(*) AS count FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE "{COLUMN_NAME}" <> UPPER("{COLUMN_NAME}") AND "{COLUMN_NAME}" <> LOWER("{COLUMN_NAME}") GROUP BY "{COLUMN_NAME}"'), @@ -1682,9 +1688,9 @@ GROUP BY "{COLUMN_NAME}"'), ('1266', '1029', 'Profile Anomaly', 'Non_Alpha_Name_Address', 'snowflake', NULL, 'SELECT "{COLUMN_NAME}", COUNT(*) as record_ct FROM "{TARGET_SCHEMA}"."{TABLE_NAME}" WHERE "{COLUMN_NAME}" = UPPER("{COLUMN_NAME}") AND "{COLUMN_NAME}" = LOWER("{COLUMN_NAME}") AND "{COLUMN_NAME}" > '''' GROUP BY "{COLUMN_NAME}" LIMIT 500'), - ('1267', '1029', 'Profile Anomaly', 'Non_Alpha_Name_Address', 'databricks', NULL, 'SELECT any_value(`{COLUMN_NAME}`), COUNT(*) as record_ct FROM `{TARGET_SCHEMA}`.`{TABLE_NAME}` + ('1267', '1029', 'Profile Anomaly', 'Non_Alpha_Name_Address', 'databricks', NULL, 'SELECT `{COLUMN_NAME}`, COUNT(*) as record_ct FROM `{TARGET_SCHEMA}`.`{TABLE_NAME}` WHERE `{COLUMN_NAME}` = UPPER(`{COLUMN_NAME}`) AND `{COLUMN_NAME}` = LOWER(`{COLUMN_NAME}`) AND `{COLUMN_NAME}` > '''' -GROUP BY "{COLUMN_NAME}" LIMIT 500'), +GROUP BY `{COLUMN_NAME}` LIMIT 500'), ('1268', '1030', 'Profile Anomaly', 'Non_Alpha_Prefixed_Name', 'redshift', NULL, 'SELECT "{COLUMN_NAME}", COUNT(*) as record_ct FROM "{TARGET_SCHEMA}"."{TABLE_NAME}" WHERE "{COLUMN_NAME}" < ''A'' AND LEFT("{COLUMN_NAME}", 1) NOT IN (''"'', '' '') AND RIGHT("{COLUMN_NAME}", 1) <> '''''''' GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}" LIMIT 500'), @@ -1697,22 +1703,78 @@ GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}"'), ('1271', '1030', 'Profile Anomaly', 'Non_Alpha_Prefixed_Name', 'snowflake', NULL, 'SELECT "{COLUMN_NAME}", COUNT(*) as record_ct FROM "{TARGET_SCHEMA}"."{TABLE_NAME}" WHERE "{COLUMN_NAME}" < ''A'' AND LEFT("{COLUMN_NAME}", 1) NOT IN (''"'', '' '') AND RIGHT("{COLUMN_NAME}", 1) <> '''''''' GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}" LIMIT 500'), - ('1272', '1030', 'Profile Anomaly', 'Non_Alpha_Prefixed_Name', 'databricks', NULL, 'SELECT any_value(`{COLUMN_NAME}`), COUNT(*) as record_ct FROM `{TARGET_SCHEMA}`.`{TABLE_NAME}` + ('1272', '1030', 'Profile Anomaly', 'Non_Alpha_Prefixed_Name', 'databricks', NULL, 'SELECT `{COLUMN_NAME}`, COUNT(*) as record_ct FROM `{TARGET_SCHEMA}`.`{TABLE_NAME}` WHERE `{COLUMN_NAME}` < ''A'' AND LEFT(`{COLUMN_NAME}`, 1) NOT IN (''"'', '' '') AND RIGHT(`{COLUMN_NAME}`, 1) <> '''''''' GROUP BY `{COLUMN_NAME}` ORDER BY `{COLUMN_NAME}` LIMIT 500'), - ('1273', '1031', 'Profile Anomaly', 'Non_Printing_Chars', 'redshift', NULL, 'SELECT "{COLUMN_NAME}", COUNT(*) as record_ct FROM "{TARGET_SCHEMA}"."{TABLE_NAME}" + ('1273', '1031', 'Profile Anomaly', 'Non_Printing_Chars', 'redshift', NULL, 'SELECT REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE("{COLUMN_NAME}", + CHR(160), ''\x160''), + CHR(8201), ''\x8201''), + CHR(8203), ''\x8203''), + CHR(8204), ''\x8204''), + CHR(8205), ''\x8205''), + CHR(8206), ''\x8206''), + CHR(8207), ''\x8207''), + CHR(8239), ''\x8239''), + CHR(12288), ''\x12288''), + CHR(65279), ''\x65279'') as "{COLUMN_NAME}_content", + COUNT(*) as record_ct FROM "{TARGET_SCHEMA}"."{TABLE_NAME}" WHERE TRANSLATE("{COLUMN_NAME}", CHR(160) || CHR(8201) || CHR(8203) || CHR(8204) || CHR(8205) || CHR(8206) || CHR(8207) || CHR(8239) || CHR(12288) || CHR(65279), ''XXXXXXXXXX'') <> "{COLUMN_NAME}" GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}" LIMIT 500'), - ('1274', '1031', 'Profile Anomaly', 'Non_Printing_Chars', 'postgresql', NULL, 'SELECT "{COLUMN_NAME}", COUNT(*) as record_ct FROM "{TARGET_SCHEMA}"."{TABLE_NAME}" + ('1274', '1031', 'Profile Anomaly', 'Non_Printing_Chars', 'postgresql', NULL, 'SELECT REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE("{COLUMN_NAME}", + CHR(160), ''\x160''), + CHR(8201), ''\x8201''), + CHR(8203), ''\x8203''), + CHR(8204), ''\x8204''), + CHR(8205), ''\x8205''), + CHR(8206), ''\x8206''), + CHR(8207), ''\x8207''), + CHR(8239), ''\x8239''), + CHR(12288), ''\x12288''), + CHR(65279), ''\x65279'') as "{COLUMN_NAME}_content", + COUNT(*) as record_ct FROM "{TARGET_SCHEMA}"."{TABLE_NAME}" WHERE TRANSLATE("{COLUMN_NAME}", CHR(160) || CHR(8201) || CHR(8203) || CHR(8204) || CHR(8205) || CHR(8206) || CHR(8207) || CHR(8239) || CHR(12288) || CHR(65279), ''XXXXXXXXXX'') <> "{COLUMN_NAME}" GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}" LIMIT 500'), - ('1275', '1031', 'Profile Anomaly', 'Non_Printing_Chars', 'mssql', NULL, 'SELECT TOP 500 "{COLUMN_NAME}", COUNT(*) as record_ct FROM "{TARGET_SCHEMA}"."{TABLE_NAME}" - WHERE TRANSLATE("{COLUMN_NAME}", NCHAR(160), ''X'') <> "{COLUMN_NAME}" + ('1275', '1031', 'Profile Anomaly', 'Non_Printing_Chars', 'mssql', NULL, 'SELECT TOP 500 REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE("{COLUMN_NAME}", + NCHAR(160), ''\x160''), + NCHAR(8201), ''\x8201''), + NCHAR(8203), ''\x8203''), + NCHAR(8204), ''\x8204''), + NCHAR(8205), ''\x8205''), + NCHAR(8206), ''\x8206''), + NCHAR(8207), ''\x8207''), + NCHAR(8239), ''\x8239''), + NCHAR(12288), ''\x12288''), + NCHAR(65279), ''\x65279'') AS "{COLUMN_NAME}_content", + COUNT(*) AS record_ct +FROM "{TARGET_SCHEMA}"."{TABLE_NAME}" + WHERE TRANSLATE("{COLUMN_NAME}", NCHAR(160) + NCHAR(8201) + NCHAR(8203) + NCHAR(8204) + NCHAR(8205) + NCHAR(8206) + NCHAR(8207) + NCHAR(8239) + NCHAR(12288) + NCHAR(65279), ''XXXXXXXXXX'') <> "{COLUMN_NAME}" GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}"'), - ('1276', '1031', 'Profile Anomaly', 'Non_Printing_Chars', 'snowflake', NULL, 'SELECT "{COLUMN_NAME}", COUNT(*) as record_ct FROM "{TARGET_SCHEMA}"."{TABLE_NAME}" + ('1276', '1031', 'Profile Anomaly', 'Non_Printing_Chars', 'snowflake', NULL, 'SELECT REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE("{COLUMN_NAME}", + CHR(160), ''\x160''), + CHR(8201), ''\x8201''), + CHR(8203), ''\x8203''), + CHR(8204), ''\x8204''), + CHR(8205), ''\x8205''), + CHR(8206), ''\x8206''), + CHR(8207), ''\x8207''), + CHR(8239), ''\x8239''), + CHR(12288), ''\x12288''), + CHR(65279), ''\x65279'') as "{COLUMN_NAME}_content", + COUNT(*) as record_ct FROM "{TARGET_SCHEMA}"."{TABLE_NAME}" WHERE TRANSLATE("{COLUMN_NAME}", CHR(160) || CHR(8201) || CHR(8203) || CHR(8204) || CHR(8205) || CHR(8206) || CHR(8207) || CHR(8239) || CHR(12288) || CHR(65279), ''XXXXXXXXXX'') <> "{COLUMN_NAME}" GROUP BY "{COLUMN_NAME}" ORDER BY "{COLUMN_NAME}" LIMIT 500'), - ('1277', '1031', 'Profile Anomaly', 'Non_Printing_Chars', 'databricks', NULL, 'SELECT any_value(`{COLUMN_NAME}`), COUNT(*) as record_ct FROM `{TARGET_SCHEMA}`.`{TABLE_NAME}` + ('1277', '1031', 'Profile Anomaly', 'Non_Printing_Chars', 'databricks', NULL, 'SELECT REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(`{COLUMN_NAME}`, + ''\u00a0'', ''\x160''), + ''\u2009'', ''\x8201''), + ''\u200b'', ''\x8203''), + ''\u200c'', ''\x8204''), + ''\u200d'', ''\x8205''), + ''\u200e'', ''\x8206''), + ''\u200f'', ''\x8207''), + ''\u202f'', ''\x8239''), + ''\u3000'', ''\x12288''), + ''\ufeff'', ''\x65279'') as `{COLUMN_NAME}_content`, + COUNT(*) as record_ct FROM `{TARGET_SCHEMA}`.`{TABLE_NAME}` WHERE TRANSLATE(`{COLUMN_NAME}`, ''\u00a0\u2009\u200b\u200c\u200d\u200e\u200f\u202f\u3000\ufeff'', ''XXXXXXXXXX'') <> `{COLUMN_NAME}` GROUP BY `{COLUMN_NAME}` ORDER BY `{COLUMN_NAME}` LIMIT 500') ; diff --git a/testgen/template/dbupgrade/0144_incremental_upgrade.sql b/testgen/template/dbupgrade/0144_incremental_upgrade.sql new file mode 100644 index 00000000..abd43d44 --- /dev/null +++ b/testgen/template/dbupgrade/0144_incremental_upgrade.sql @@ -0,0 +1,4 @@ +SET SEARCH_PATH TO {SCHEMA_NAME}; + +ALTER TABLE table_groups +ADD COLUMN include_in_dashboard BOOLEAN DEFAULT TRUE; diff --git a/testgen/template/flavors/postgresql/profiling/project_profiling_query_postgresql.yaml b/testgen/template/flavors/postgresql/profiling/project_profiling_query_postgresql.yaml index 8f035cff..a12c2d50 100644 --- a/testgen/template/flavors/postgresql/profiling/project_profiling_query_postgresql.yaml +++ b/testgen/template/flavors/postgresql/profiling/project_profiling_query_postgresql.yaml @@ -73,8 +73,7 @@ strTemplate05_A: COUNT(DISTINCT UPPER(TRANSLATE("{COL_NAME}", ' '',.-', ''))) a THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.8 THEN 'STREET_ADDR' WHEN SUM(CASE WHEN "{COL_NAME}" IN ('AL','AK','AS','AZ','AR','CA','CO','CT','DE','DC','FM','FL','GA','GU','HI','ID','IL','IN','IA','KS','KY','LA','ME','MH','MD','MA','MI','MN','MS','MO','MT','NE','NV','NH','NJ','NM','NY','NC','ND','MP','OH','OK','OR','PW','PA','PR','RI','SC','SD','TN','TX','UT','VT','VI','VA','WA','WV','WI','WY','AE','AP','AA') THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.9 THEN 'STATE_USA' - WHEN SUM( CASE WHEN "{COL_NAME}" SIMILAR TO '^([\+]1 |1-|)[\+]?[(]?[0-9]{3}[)][ ]?[-\s\.]?[0-9]{3}[-\s\.]?[0-9]{4,6}$' - OR "{COL_NAME}" SIMILAR TO '^([\+]1 |1-|)[2-9][01][0-9][-\s\.]?[0-9]{3}[-\s\.]?[0-9]{4}$' + WHEN SUM( CASE WHEN "{COL_NAME}" ~ E'^(\\+1|1)?[ .-]?(\\([2-9][0-9]{2}\\)|[2-9][0-9]{2})[ .-]?[2-9][0-9]{2}[ .-]?[0-9]{4}$' THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.8 THEN 'PHONE_USA' WHEN SUM( CASE WHEN "{COL_NAME}" ~ '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$' THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.9 THEN 'EMAIL' diff --git a/testgen/template/flavors/redshift/profiling/project_profiling_query_redshift.yaml b/testgen/template/flavors/redshift/profiling/project_profiling_query_redshift.yaml index 0de85a1b..b64d4d54 100644 --- a/testgen/template/flavors/redshift/profiling/project_profiling_query_redshift.yaml +++ b/testgen/template/flavors/redshift/profiling/project_profiling_query_redshift.yaml @@ -53,9 +53,8 @@ strTemplate05_A: COUNT(DISTINCT UPPER(TRANSLATE("{COL_NAME}", ' '',.-', ''))) a THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.8 THEN 'STREET_ADDR' WHEN SUM( CASE WHEN "{COL_NAME}" IN ('AL','AK','AS','AZ','AR','CA','CO','CT','DE','DC','FM','FL','GA','GU','HI','ID','IL','IN','IA','KS','KY','LA','ME','MH','MD','MA','MI','MN','MS','MO','MT','NE','NV','NH','NJ','NM','NY','NC','ND','MP','OH','OK','OR','PW','PA','PR','RI','SC','SD','TN','TX','UT','VT','VI','VA','WA','WV','WI','WY','AE','AP','AA') THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.9 THEN 'STATE_USA' - WHEN SUM( CASE WHEN "{COL_NAME}" ~ '^([\\+]1 |1-|)[\\+]?[(]?[0-9]{3}[)][ ]?[-\\s\\.]?[0-9]{3}[-\\s\.]?[0-9]{4,6}$' - OR "{COL_NAME}" ~ '^([\\+]1 |1-|)[2-9][01][0-9][-\\s\\.]?[0-9]{3}[-\\s\\.]?[0-9]{4}$' - THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.9 THEN 'PHONE_USA' + WHEN SUM( CASE WHEN "{COL_NAME}" ~ '^(\\+1|1)?[ .-]?(\\([2-9][0-9]{2}\\)|[2-9][0-9]{2})[ .-]?[2-9][0-9]{2}[ .-]?[0-9]{4}$' + THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.8 THEN 'PHONE_USA' WHEN SUM( CASE WHEN "{COL_NAME}" ~ '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}$' THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.9 THEN 'EMAIL' WHEN SUM( CASE WHEN TRANSLATE("{COL_NAME}",'012345678','999999999') IN ('99999', '999999999', '99999-9999') diff --git a/testgen/template/flavors/snowflake/profiling/project_profiling_query_snowflake.yaml b/testgen/template/flavors/snowflake/profiling/project_profiling_query_snowflake.yaml index 292dcb38..1caffc0a 100644 --- a/testgen/template/flavors/snowflake/profiling/project_profiling_query_snowflake.yaml +++ b/testgen/template/flavors/snowflake/profiling/project_profiling_query_snowflake.yaml @@ -60,8 +60,7 @@ strTemplate05_A: COUNT(DISTINCT UPPER(TRANSLATE("{COL_NAME}", ' '',.-', ''))) a THEN 1 END) AS FLOAT) / CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.8 THEN 'STREET_ADDR' WHEN CAST(SUM(CASE WHEN "{COL_NAME}" IN ('AL','AK','AS','AZ','AR','CA','CO','CT','DE','DC','FM','FL','GA','GU','HI','ID','IL','IN','IA','KS','KY','LA','ME','MH','MD','MA','MI','MN','MS','MO','MT','NE','NV','NH','NJ','NM','NY','NC','ND','MP','OH','OK','OR','PW','PA','PR','RI','SC','SD','TN','TX','UT','VT','VI','VA','WA','WV','WI','WY','AE','AP','AA') THEN 1 END) AS FLOAT) / CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.9 THEN 'STATE_USA' - WHEN CAST(SUM( CASE WHEN REGEXP_LIKE("{COL_NAME}"::VARCHAR, '^([\\+]1 |1-|)[\\+]?[(]?[0-9]{3}[)][ ]?[-\\s\\.]?[0-9]{3}[-\\s\\.]?[0-9]{4,6}$') - OR REGEXP_LIKE("{COL_NAME}"::VARCHAR, '^([\+]1 |1-|)[2-9][01][0-9][-\\s\\.]?[0-9]{3}[-\\s\\.]?[0-9]{4}$') + WHEN CAST(SUM( CASE WHEN REGEXP_LIKE("{COL_NAME}"::VARCHAR, '^(\\+1|1)?[ .-]?(\\([2-9][0-9]{2}\\)|[2-9][0-9]{2})[ .-]?[2-9][0-9]{2}[ .-]?[0-9]{4}$') THEN 1 END) AS FLOAT) / CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.8 THEN 'PHONE_USA' WHEN CAST(SUM( CASE WHEN REGEXP_LIKE("{COL_NAME}"::VARCHAR, '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}$') THEN 1 END) AS FLOAT) / CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.9 THEN 'EMAIL' diff --git a/testgen/template/gen_query_tests/gen_dupe_rows_test.sql b/testgen/template/gen_query_tests/gen_dupe_rows_test.sql index 5027c111..e28164a8 100644 --- a/testgen/template/gen_query_tests/gen_dupe_rows_test.sql +++ b/testgen/template/gen_query_tests/gen_dupe_rows_test.sql @@ -15,7 +15,7 @@ WITH last_run AS (SELECT r.table_groups_id, MAX(run_date) AS last_run_date AND p.run_date::DATE <= '{AS_OF_DATE}' GROUP BY r.table_groups_id), curprof AS (SELECT p.schema_name, p.table_name, p.profile_run_id, - STRING_AGG(QUOTE_IDENT(p.column_name), ', ' ORDER BY p.position) as unique_by_columns + STRING_AGG('{ID_SEPARATOR}' || p.column_name || '{ID_SEPARATOR}', ', ' ORDER BY p.position) as unique_by_columns FROM last_run lr INNER JOIN profile_results p ON (lr.table_groups_id = p.table_groups_id diff --git a/testgen/template/generation/gen_retrieve_or_insert_test_suite.sql b/testgen/template/generation/gen_retrieve_or_insert_test_suite.sql deleted file mode 100644 index da89bc0b..00000000 --- a/testgen/template/generation/gen_retrieve_or_insert_test_suite.sql +++ /dev/null @@ -1,58 +0,0 @@ -WITH existing_rec - AS ( SELECT tg.project_code, tg.connection_id, - cc.sql_flavor, - cc.project_host, - cc.project_port, - cc.project_user, - cc.project_db, - tg.table_group_schema, - s.export_to_observability, - s.test_suite, - s.id as test_suite_id, - cc.url, - cc.connect_by_url, - CURRENT_TIMESTAMP AT TIME ZONE - 'UTC' - CAST(tg.profiling_delay_days AS INTEGER) * INTERVAL '1 day' AS profiling_as_of_date - FROM table_groups tg - INNER JOIN connections cc - ON (tg.connection_id = cc.connection_id) - INNER JOIN test_suites s - ON (tg.id = s.table_groups_id - AND '{TEST_SUITE}' = s.test_suite) - WHERE tg.id = '{TABLE_GROUPS_ID}' ), -new_rec - AS ( INSERT INTO test_suites - (project_code, test_suite, connection_id, table_groups_id, test_suite_description, - component_type, component_key) - SELECT '{PROJECT_CODE}', '{TEST_SUITE}', {CONNECTION_ID}, '{TABLE_GROUPS_ID}', '{TEST_SUITE} Test Suite', - 'dataset', '{TEST_SUITE}' - WHERE NOT EXISTS - (SELECT 1 - FROM test_suites - WHERE table_groups_id = '{TABLE_GROUPS_ID}' - AND test_suite = '{TEST_SUITE}') - RETURNING id as test_suite_id, test_suite, table_groups_id, export_to_observability ) -SELECT project_code, connection_id, sql_flavor, - project_host, project_port, project_user, project_db, table_group_schema, - export_to_observability, test_suite, test_suite_id, url, connect_by_url, profiling_as_of_date - FROM existing_rec - UNION ALL -SELECT tg.project_code, tg.connection_id, - cc.sql_flavor, - cc.project_host, - cc.project_port, - cc.project_user, - cc.project_db, - tg.table_group_schema, - s.export_to_observability, - s.test_suite, - s.test_suite_id, - cc.url, - cc.connect_by_url, - CURRENT_TIMESTAMP AT TIME ZONE - 'UTC' - CAST(tg.profiling_delay_days AS INTEGER) * INTERVAL '1 day' AS profiling_as_of_date - FROM new_rec s -INNER JOIN table_groups tg - ON (s.table_groups_id = tg.id) -INNER JOIN connections cc - ON (tg.connection_id = cc.connection_id); \ No newline at end of file diff --git a/testgen/template/get_entities/get_connection.sql b/testgen/template/get_entities/get_connection.sql deleted file mode 100644 index 035f9304..00000000 --- a/testgen/template/get_entities/get_connection.sql +++ /dev/null @@ -1,21 +0,0 @@ -SELECT - id, - project_code as project_key, - connection_id, - connection_name, - sql_flavor, - project_host, - project_port, - project_user, - project_db, - project_pw_encrypted, - max_threads, - max_query_chars, - url, - connect_by_url, - connect_by_key, - private_key, - private_key_passphrase, - http_path -FROM connections -WHERE connection_id = {CONNECTION_ID}; diff --git a/testgen/template/get_entities/get_latest.sql b/testgen/template/get_entities/get_latest.sql deleted file mode 100644 index 27a06318..00000000 --- a/testgen/template/get_entities/get_latest.sql +++ /dev/null @@ -1,4 +0,0 @@ -SELECT id -FROM {ENTITY} -ORDER BY {TIME_COLUMN} DESC -LIMIT 1; diff --git a/testgen/template/get_entities/get_profile.sql b/testgen/template/get_entities/get_profile.sql deleted file mode 100644 index 113b422b..00000000 --- a/testgen/template/get_entities/get_profile.sql +++ /dev/null @@ -1,12 +0,0 @@ -SELECT p.profile_run_id, - p.run_date, - p.schema_name, - p.table_name, --- p."position", - p.column_name, - p.general_type, - p.column_type, - p.datatype_suggestion -FROM profile_results p -WHERE profile_run_id = '{PROFILE_RUN_ID}'::UUID -ORDER BY p.schema_name, p.table_name, p.position; diff --git a/testgen/template/validate_tests/ex_get_project_column_list_generic.sql b/testgen/template/validate_tests/ex_get_project_column_list_generic.sql deleted file mode 100644 index eacffa61..00000000 --- a/testgen/template/validate_tests/ex_get_project_column_list_generic.sql +++ /dev/null @@ -1,3 +0,0 @@ -select concat(concat(concat(table_schema, '.'), concat(table_name, '.')), column_name) as columns -from information_schema.columns -where table_schema in ({TEST_SCHEMAS}); diff --git a/testgen/ui/components/frontend/css/shared.css b/testgen/ui/components/frontend/css/shared.css index d9ff025d..bcc96b4b 100644 --- a/testgen/ui/components/frontend/css/shared.css +++ b/testgen/ui/components/frontend/css/shared.css @@ -78,6 +78,8 @@ body { --portal-background: white; --portal-box-shadow: rgba(0, 0, 0, 0.16) 0px 4px 16px; --select-hover-background: rgb(240, 242, 246); + + --app-background-color: #f8f9fa; } @media (prefers-color-scheme: dark) { @@ -119,6 +121,8 @@ body { --portal-background: #14181f; --portal-box-shadow: rgba(0, 0, 0, 0.95) 0px 4px 16px; --select-hover-background: rgb(38, 39, 48); + + --app-background-color: rgb(14, 17, 23); } } @@ -619,3 +623,11 @@ code > .tg-icon:hover { .accent-primary { accent-color: var(--primary-color); } + +.border { + border: var(--button-stroked-border); +} + +.border-radius-1 { + border-radius: 4px; +} diff --git a/testgen/ui/components/frontend/js/components/button.js b/testgen/ui/components/frontend/js/components/button.js index d90b0034..ec543bc5 100644 --- a/testgen/ui/components/frontend/js/components/button.js +++ b/testgen/ui/components/frontend/js/components/button.js @@ -1,8 +1,8 @@ /** * @typedef Properties * @type {object} - * @property {(string)} type - * @property {(string|null)} color + * @property {'basic' | 'flat' | 'icon' | 'stroked'} type + * @property {'basic' | 'primary' | 'warn'} color * @property {(string|null)} width * @property {(string|null)} label * @property {(string|null)} icon @@ -27,11 +27,6 @@ const BUTTON_TYPE = { ICON: 'icon', STROKED: 'stroked', }; -const BUTTON_COLOR = { - BASIC: 'basic', - PRIMARY: 'primary', - WARN: 'warn', -}; const DEFAULT_ICON_SIZE = 18; diff --git a/testgen/ui/components/frontend/js/components/checkbox.js b/testgen/ui/components/frontend/js/components/checkbox.js index 6e5968d8..45591ecc 100644 --- a/testgen/ui/components/frontend/js/components/checkbox.js +++ b/testgen/ui/components/frontend/js/components/checkbox.js @@ -9,6 +9,7 @@ * @property {function(boolean, Event)?} onChange * @property {number?} width * @property {string?} testId + * @property {boolean?} disabled */ import van from '../van.min.js'; import { getValue, loadStylesheet } from '../utils.js'; @@ -36,6 +37,7 @@ const Checkbox = (/** @type Properties */ props) => { const onChange = props.onChange?.val ?? props.onChange; return onChange ? (/** @type Event */ event) => onChange(event.target.checked, event) : null; }), + disabled: props.disabled ?? false, }), span({'data-testid': 'checkbox-label'}, props.label), () => getValue(props.help) @@ -86,6 +88,12 @@ stylesheet.replace(` background-color: var(--primary-color); } +.tg-checkbox--input:checked:disabled, +.tg-checkbox--input:indeterminate:disabled { + cursor: not-allowed; + background-color: var(--disabled-text-color); +} + .tg-checkbox--input:checked::after, .tg-checkbox--input:indeterminate::after { position: absolute; diff --git a/testgen/ui/components/frontend/js/components/connection_form.js b/testgen/ui/components/frontend/js/components/connection_form.js index 7ed5918f..33a02e70 100644 --- a/testgen/ui/components/frontend/js/components/connection_form.js +++ b/testgen/ui/components/frontend/js/components/connection_form.js @@ -1,5 +1,6 @@ /** * @import { FileValue } from './file_input.js'; + * @import { VanState } from '../van.min.js'; * * @typedef Flavor * @type {object} @@ -57,13 +58,14 @@ import { Alert } from './alert.js'; import { getValue, emitEvent, loadStylesheet, isEqual } from '../utils.js'; import { Input } from './input.js'; import { Slider } from './slider.js'; -import { Checkbox } from './checkbox.js'; import { Select } from './select.js'; import { maxLength, minLength, sizeLimit } from '../form_validators.js'; import { RadioGroup } from './radio_group.js'; import { FileInput } from './file_input.js'; +import { ExpansionPanel } from './expansion_panel.js'; +import { Caption } from './caption.js'; -const { div, hr, i, span } = van.tags; +const { div, i, span } = van.tags; const clearSentinel = ''; const secretsPlaceholder = ''; const defaultPorts = { @@ -90,306 +92,210 @@ const ConnectionForm = (props, saveButton) => { const defaultPort = defaultPorts[connection?.sql_flavor]; const connectionFlavor = van.state(connection?.sql_flavor_code); - const connectionName = van.state(connection?.connection_name); - const connectionHost = van.state(connection?.project_host); - const connectionPort = van.state(connection?.project_port ?? defaultPort); - const connectionDatabase = van.state(connection?.project_db); - const connectionUsername = van.state(connection?.project_user); - const connectionPassword = van.state(connection?.password); + const connectionName = van.state(connection?.connection_name ?? ''); const connectionMaxThreads = van.state(connection?.max_threads ?? 4); const connectionQueryChars = van.state(connection?.max_query_chars ?? 9000); - const connectByUrl = van.state(connection?.connect_by_url ?? false); - const connectByKey = van.state(connection?.connect_by_key ?? false); - const privateKey = van.state(connection?.private_key); - const privateKeyPhrase = van.state(connection?.private_key_passphrase); - const httpPath = van.state(connection?.http_path); - const privateKeyFile = van.state(getValue(props.cachedPrivateKeyFile) ?? null); - van.derive(() => { - const fileInputValue = privateKeyFile.val; - if (fileInputValue?.content) { - privateKey.val = fileInputValue.content.split(',')?.[1] ?? ''; - } - }); - const clearPrivateKeyPhrase = van.state(false); - - if (isEditMode) { - connectionPassword.val = ''; - privateKey.val = ''; - privateKeyPhrase.val = ''; - } const flavor = getValue(props.flavors).find(f => f.value === connectionFlavor.rawVal); - const originalURLTemplate = van.state(flavor.connection_string); - const [prefixPart, sufixPart] = originalURLTemplate.rawVal.split('@'); - - const connectionStringPrefix = van.state(prefixPart); - const connectionStringSuffix = van.state(connection?.url ?? ''); - if (!connectionStringSuffix.rawVal) { - connectionStringSuffix.val = formatURL(sufixPart ?? '', connectionHost.rawVal, connectionPort.rawVal, connectionDatabase.rawVal); - } + const originalURLTemplate = flavor.connection_string; + const [_, urlSuffix] = originalURLTemplate.split('@'); + + const updatedConnection = van.state({ + project_code: connection.project_code, + connection_id: connection.connection_id, + sql_flavor: connection?.sql_flavor ?? undefined, + project_host: connection?.project_host ?? '', + project_port: connection?.project_port ?? defaultPort ?? '', + project_db: connection?.project_db ?? '', + project_user: connection?.project_user ?? '', + password: isEditMode ? '' : (connection?.password ?? ''), + connect_by_url: connection?.connect_by_url ?? false, + connect_by_key: connection?.connect_by_key ?? false, + private_key: isEditMode ? '' : (connection?.private_key ?? ''), + private_key_passphrase: isEditMode ? '' : (connection?.private_key_passphrase ?? ''), + http_path: connection?.http_path ?? '', + url: connection?.connect_by_url + ? (connection?.url ?? '') + : formatURL( + urlSuffix ?? '', + connection?.project_host ?? '', + connection?.project_port ?? defaultPort ?? '', + connection?.project_db ?? '', + connection?.http_path ?? '', + ), - const updatedConnection = van.derive(() => { - return { - project_code: connection.project_code, - connection_id: connection.connection_id, - sql_flavor: connection?.sql_flavor ?? undefined, - sql_flavor_code: connectionFlavor.val ?? '', - connection_name: connectionName.val ?? '', - project_host: connectionHost.val ?? '', - project_port: connectionPort.val ?? '', - project_db: connectionDatabase.val ?? '', - project_user: connectionUsername.val ?? '', - password: connectionPassword.val ?? '', - max_threads: connectionMaxThreads.val ?? 4, - max_query_chars: connectionQueryChars.val ?? 9000, - connect_by_url: connectByUrl.val ?? false, - url: connectionStringSuffix.val, - connect_by_key: connectByKey.val ?? false, - private_key: privateKey.val ?? '', - private_key_passphrase: clearPrivateKeyPhrase.val ? clearSentinel : (privateKeyPhrase.val ?? ''), - http_path: httpPath.val ?? '', - }; + sql_flavor_code: connectionFlavor.rawVal ?? '', + connection_name: connectionName.rawVal ?? '', + max_threads: connectionMaxThreads.rawVal ?? 4, + max_query_chars: connectionQueryChars.rawVal ?? 9000, }); + const dirty = van.derive(() => !isEqual(updatedConnection.val, connection)); const validityPerField = van.state({}); - van.derive(() => { - const fieldsValidity = validityPerField.val; - const isValid = Object.keys(fieldsValidity).length > 0 && - Object.values(fieldsValidity).every(v => v); - props.onChange?.(updatedConnection.val, { dirty: dirty.val, valid: isValid }, { privateKey: privateKeyFile.rawVal }); - }); - - const setFieldValidity = (field, validity) => { - validityPerField.val = {...validityPerField.val, [field]: validity}; - } - const authenticationForms = { - redshift: () => PasswordConnectionForm( - connection, - connectionPassword, - (value, state) => { - connectionPassword.val = value; - setFieldValidity('password', state.valid); + redshift: () => RedshiftForm( + updatedConnection, + getValue(props.flavors).find(f => f.value === connectionFlavor.rawVal), + (formValue, isValid) => { + updatedConnection.val = {...updatedConnection.val, ...formValue}; + setFieldValidity('redshift_form', isValid); }, - isEditMode, - ), - mssql: () => PasswordConnectionForm( connection, - connectionPassword, - (value, state) => { - connectionPassword.val = value; - setFieldValidity('password', state.valid); - }, - isEditMode, ), - postgresql: () => PasswordConnectionForm( + azure_mssql: () => AzureMSSQLForm( + updatedConnection, + getValue(props.flavors).find(f => f.value === connectionFlavor.rawVal), + (formValue, isValid) => { + updatedConnection.val = {...updatedConnection.val, ...formValue}; + setFieldValidity('mssql_form', isValid); + }, connection, - connectionPassword, - (value, state) => { - connectionPassword.val = value; - setFieldValidity('password', state.valid); + ), + synapse_mssql: () => SynapseMSSQLForm( + updatedConnection, + getValue(props.flavors).find(f => f.value === connectionFlavor.rawVal), + (formValue, isValid) => { + updatedConnection.val = {...updatedConnection.val, ...formValue}; + setFieldValidity('mssql_form', isValid); }, - isEditMode, + connection, ), - snowflake: () => KeyPairConnectionForm( + mssql: () => MSSQLForm( + updatedConnection, + getValue(props.flavors).find(f => f.value === connectionFlavor.rawVal), + (formValue, isValid) => { + updatedConnection.val = {...updatedConnection.val, ...formValue}; + setFieldValidity('mssql_form', isValid); + }, connection, - connectByKey, - connectionPassword, - privateKeyFile, - privateKeyPhrase, - clearPrivateKeyPhrase, - (value, state) => { - connectByKey.val = value.connect_by_key; - connectionPassword.val = value.password; - privateKeyFile.val = value.private_key; - privateKeyPhrase.val = value.private_key_passphrase; - setFieldValidity('key_pair_form', state.valid); + ), + postgresql: () => PostgresqlForm( + updatedConnection, + getValue(props.flavors).find(f => f.value === connectionFlavor.rawVal), + (formValue, isValid) => { + updatedConnection.val = {...updatedConnection.val, ...formValue}; + setFieldValidity('mssql_form', isValid); }, - isEditMode, + connection, ), - databricks: () => HttpPathConnectionForm( + + snowflake: () => SnowflakeForm( + updatedConnection, + getValue(props.flavors).find(f => f.value === connectionFlavor.rawVal), + (formValue, fileValue, isValid) => { + updatedConnection.val = {...updatedConnection.val, ...formValue}; + privateKeyFile.val = fileValue; + setFieldValidity('snowflake_form', isValid); + }, connection, - connectionPassword, - httpPath, - (value, state) => { - connectionPassword.val = value.password; - httpPath.val = value.http_path; - setFieldValidity('http_path_form', state.valid); + getValue(props.cachedPrivateKeyFile) ?? null, + ), + databricks: () => DatabricksForm( + updatedConnection, + getValue(props.flavors).find(f => f.value === connectionFlavor.rawVal), + (formValue, isValid) => { + updatedConnection.val = {...updatedConnection.val, ...formValue}; + setFieldValidity('databricks_form', isValid); }, - isEditMode, + connection, ), }; + + const setFieldValidity = (field, validity) => { + validityPerField.val = {...validityPerField.val, [field]: validity}; + } + const authenticationForm = van.derive(() => { const selectedFlavorCode = connectionFlavor.val; const flavor = getValue(props.flavors).find(f => f.value === selectedFlavorCode); - return authenticationForms[flavor.flavor](); + return authenticationForms[flavor.value](); }); van.derive(() => { const selectedFlavorCode = connectionFlavor.val; const previousFlavorCode = connectionFlavor.oldVal; - const isCustomPort = connectionPort.rawVal !== defaultPorts[previousFlavorCode]; - if (selectedFlavorCode !== previousFlavorCode && (!isCustomPort || !connectionPort.rawVal)) { - connectionPort.val = defaultPorts[selectedFlavorCode]; + const updatedConnection_ = updatedConnection.rawVal; + + const isCustomPort = updatedConnection_?.project_port !== defaultPorts[previousFlavorCode]; + if (selectedFlavorCode !== previousFlavorCode && (!isCustomPort || !updatedConnection_?.project_port)) { + updatedConnection.val = {...updatedConnection_, project_port: defaultPorts[selectedFlavorCode]}; } }); van.derive(() => { - const connectionHost_ = connectionHost.val; - const connectionPort_ = connectionPort.val; - const connectionDatabase_ = connectionDatabase.val; - const connectionHttpPath_ = httpPath.val; - const urlTemplate = originalURLTemplate.val; + const selectedFlavor = connectionFlavor.val; + const flavorObject = getValue(props.flavors).find(f => f.value === selectedFlavor); + + updatedConnection.val = { + ...updatedConnection.val, + sql_flavor: flavorObject.flavor, + sql_flavor_code: flavorObject.value, + connection_name: connectionName.val, + max_threads: connectionMaxThreads.val, + max_query_chars: connectionQueryChars.val, + }; + }); - if (!connectByUrl.rawVal && urlTemplate.includes('@')) { - const [originalURLPrefix, originalURLSuffix] = urlTemplate.split('@'); - connectionStringPrefix.val = originalURLPrefix; - connectionStringSuffix.val = formatURL(originalURLSuffix, connectionHost_, connectionPort_, connectionDatabase_, connectionHttpPath_); - } + van.derive(() => { + const fieldsValidity = validityPerField.val; + const isValid = Object.keys(fieldsValidity).length > 0 && + Object.values(fieldsValidity).every(v => v); + props.onChange?.(updatedConnection.val, { dirty: dirty.val, valid: isValid }, { privateKey: privateKeyFile.rawVal }); }); return div( { class: 'flex-column fx-gap-3 fx-align-stretch', style: 'overflow-y: auto;' }, - div( - { class: 'flex-row fx-gap-3 fx-align-stretch' }, - div( - { class: 'flex-column fx-gap-3', style: 'flex: 2' }, - Select({ - label: 'Database Type', - value: connectionFlavor, - options: props.flavors, - disabled: props.disableFlavor, - height: 38, - help: 'Type of database server to connect to. This determines the database driver and SQL dialect that will be used by TestGen.', - testId: 'sql_flavor', - onChange: (value) => { - const flavor = getValue(props.flavors).find(f => f.value === value); - originalURLTemplate.val = flavor.connection_string; - }, - }), - Input({ - name: 'connection_name', - label: 'Connection Name', - value: connectionName, - height: 38, - help: 'Unique name to describe the connection', - onChange: (value, state) => { - connectionName.val = value; - setFieldValidity('connection_name', state.valid); - }, - validators: [ minLength(3), maxLength(40) ], - }), - div( - { class: 'flex-row fx-gap-3 fx-flex' }, - Input({ - name: 'db_host', - label: 'Host', - value: connectionHost, - height: 38, - class: 'fx-flex', - disabled: connectByUrl, - onChange: (value, state) => { - connectionHost.val = value; - setFieldValidity('db_host', state.valid); - }, - validators: [ maxLength(250) ], - }), - Input({ - name: 'db_port', - label: 'Port', - value: connectionPort, - height: 38, - type: 'number', - disabled: connectByUrl, - onChange: (value, state) => { - connectionPort.val = value; - setFieldValidity('db_port', state.valid); - }, - validators: [ minLength(3), maxLength(5) ], - }), - ), - Input({ - name: 'db_name', - label: 'Database', - value: connectionDatabase, - height: 38, - disabled: connectByUrl, - onChange: (value, state) => { - connectionDatabase.val = value; - setFieldValidity('db_name', state.valid); - }, - validators: [ maxLength(100) ], - }), - Input({ - name: 'db_user', - label: 'Username', - value: connectionUsername, - height: 38, - onChange: (value, state) => { - connectionUsername.val = value; - setFieldValidity('db_user', state.valid); - }, - validators: [ maxLength(50) ], - }), - ), + Select({ + label: 'Database Type', + value: connectionFlavor, + options: props.flavors, + disabled: props.disableFlavor, + height: 38, + help: 'Type of database server to connect to. This determines the database driver and SQL dialect that will be used by TestGen.', + testId: 'sql_flavor', + }), + Input({ + name: 'connection_name', + label: 'Connection Name', + value: connectionName, + height: 38, + help: 'Unique name to describe the connection', + onChange: (value, state) => { + connectionName.val = value; + setFieldValidity('connection_name', state.valid); + }, + validators: [ minLength(3), maxLength(40) ], + }), + + authenticationForm, + + ExpansionPanel( + { + title: 'Advanced Tuning', + }, div( - { class: 'flex-column fx-gap-3', style: 'padding: 2px; flex: 1;' }, + { class: 'flex-row fx-gap-3' }, Slider({ - label: 'Max Threads (Advanced Tuning)', + label: 'Max Threads', hint: 'Maximum number of concurrent threads that run tests. Default values should be retained unless test queries are failing.', - value: connectionMaxThreads, + value: connectionMaxThreads.rawVal, min: 1, max: 8, onChange: (value) => connectionMaxThreads.val = value, }), Slider({ - label: 'Max Expression Length (Advanced Tuning)', + label: 'Max Expression Length', hint: 'Some tests are consolidated into queries for maximum performance. Default values should be retained unless test queries are failing.', - value: connectionQueryChars, + value: connectionQueryChars.rawVal, min: 500, max: 14000, onChange: (value) => connectionQueryChars.val = value, }), ), ), - authenticationForm, - hr({ style: 'width: 100%;', class: 'mt-2 mb-2' }), - Checkbox({ - name: 'connect_by_url', - label: 'URL Override', - help: 'When checked, the connection string will be driven by the field below, along with the username and password from the fields above', - checked: connectByUrl.val, - onChange: (checked) => connectByUrl.val = checked, - }), - () => { - const connectByUrl_ = getValue(connectByUrl); - - if (!connectByUrl_) { - return ''; - } - return div( - { class: 'flex-row fx-gap-3 fx-align-stretch' }, - Input({ - label: 'URL Prefix', - disabled: true, - value: connectionStringPrefix, - height: 38, - width: 255, - name: 'url_prefix', - }), - Input({ - label: 'URL Suffix', - value: connectionStringSuffix, - class: 'fx-flex', - height: 38, - name: 'url_suffix', - onChange: (value, state) => connectionStringSuffix.val = value, - }), - ); - }, div( { class: 'flex-row fx-gap-3 fx-justify-space-between' }, Button({ @@ -418,185 +324,642 @@ const ConnectionForm = (props, saveButton) => { ); }; -const PasswordConnectionForm = (connection, password, onValueChange, useSecretsPlaceholder) => { +/** + * @param {VanState} connection + * @param {Flavor} flavor + * @param {boolean} maskPassword + * @param {(params: Partial, isValid: boolean) => void} onChange + * @param {Connection?} originalConnection + * @returns {HTMLElement} + */ +const RedshiftForm = ( + connection, + flavor, + onChange, + originalConnection, +) => { + const originalURLTemplate = flavor.connection_string; + + const isValid = van.state(true); + const connectByUrl = van.state(connection.rawVal.connect_by_url ?? false); + const connectionHost = van.state(connection.rawVal.project_host ?? ''); + const connectionPort = van.state(connection.rawVal.project_port || defaultPorts[flavor.flavor]); + const connectionDatabase = van.state(connection.rawVal.project_db ?? ''); + const connectionUsername = van.state(connection.rawVal.project_user ?? ''); + const connectionPassword = van.state(connection.rawVal?.password ?? ''); + + const [prefixPart, sufixPart] = originalURLTemplate.split('@'); + const connectionStringPrefix = van.state(`${prefixPart}@`); + const connectionStringSuffix = van.state(connection.rawVal?.url ?? ''); + + const validityPerField = {}; + + if (!connectionStringSuffix.rawVal) { + connectionStringSuffix.val = formatURL(sufixPart ?? '', connectionHost.rawVal, connectionPort.rawVal, connectionDatabase.rawVal); + } + + van.derive(() => { + const connectionHost_ = connectionHost.val; + const connectionPort_ = connectionPort.val; + const connectionDatabase_ = connectionDatabase.val; + + if (!connectByUrl.rawVal && originalURLTemplate.includes('@')) { + const [originalURLPrefix, originalURLSuffix] = originalURLTemplate.split('@'); + connectionStringPrefix.val = `${originalURLPrefix}@`; + connectionStringSuffix.val = formatURL(originalURLSuffix, connectionHost_, connectionPort_, connectionDatabase_); + } + }); + + van.derive(() => { + onChange({ + project_host: connectionHost.val, + project_port: connectionPort.val, + project_db: connectionDatabase.val, + project_user: connectionUsername.val, + password: connectionPassword.val, + connect_by_url: connectByUrl.val, + url: connectByUrl.val ? connectionStringSuffix.val : connectionStringSuffix.rawVal, + }, isValid.val); + }); + return div( - { class: 'flex-row fx-gap-3 fx-align-stretch' }, + {class: 'flex-column fx-gap-3 fx-flex'}, div( - { class: 'flex-column fx-gap-3', style: 'flex: 2' }, + { class: 'flex-column border border-radius-1 p-3 mt-1 fx-gap-1', style: 'position: relative;' }, + Caption({content: 'Server', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), + + RadioGroup({ + label: 'Connect by', + options: [ + { + label: 'Host', + value: false, + }, + { + label: 'URL', + value: true, + }, + ], + value: connectByUrl, + onChange: (value) => connectByUrl.val = value, + inline: true, + }), + div( + { class: 'flex-row fx-gap-3 fx-flex' }, + Input({ + name: 'db_host', + label: 'Host', + value: connectionHost, + height: 38, + class: 'fx-flex', + disabled: connectByUrl, + onChange: (value, state) => { + connectionHost.val = value; + validityPerField['db_host'] = state.valid; + isValid.val = Object.values(validityPerField).every(v => v); + }, + validators: [ maxLength(250) ], + }), + Input({ + name: 'db_port', + label: 'Port', + value: connectionPort, + height: 38, + type: 'number', + disabled: connectByUrl, + onChange: (value, state) => { + connectionPort.val = value; + validityPerField['db_port'] = state.valid; + isValid.val = Object.values(validityPerField).every(v => v); + }, + validators: [ minLength(3), maxLength(5) ], + }) + ), + Input({ + name: 'db_name', + label: 'Database', + value: connectionDatabase, + height: 38, + disabled: connectByUrl, + onChange: (value, state) => { + connectionDatabase.val = value; + validityPerField['db_name'] = state.valid; + isValid.val = Object.values(validityPerField).every(v => v); + }, + validators: [ maxLength(100) ], + }), + () => div( + { class: 'flex-row fx-gap-3 fx-align-stretch', style: 'position: relative;' }, + Input({ + label: 'URL', + value: connectionStringSuffix, + class: 'fx-flex', + height: 38, + name: 'url_suffix', + prefix: span({ style: 'height: 38px; white-space: nowrap; color: var(--disabled-text-color)' }, connectionStringPrefix), + disabled: !connectByUrl.val, + onChange: (value, state) => connectionStringSuffix.val = value, + }), + ), + ), + + div( + { class: 'flex-column border border-radius-1 p-3 mt-1 fx-gap-1', style: 'position: relative;' }, + Caption({content: 'Authentication', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), + + Input({ + name: 'db_user', + label: 'Username', + value: connectionUsername, + height: 38, + onChange: (value, state) => { + connectionUsername.val = value; + validityPerField['db_user'] = state.valid; + isValid.val = Object.values(validityPerField).every(v => v); + }, + validators: [ maxLength(50) ], + }), Input({ name: 'password', label: 'Password', - value: password, + value: connectionPassword, height: 38, type: 'password', - placeholder: (useSecretsPlaceholder && connection.password) ? secretsPlaceholder : '', - onChange: onValueChange, + placeholder: (originalConnection?.connection_id && originalConnection?.password) ? secretsPlaceholder : '', + onChange: (value, state) => { + connectionPassword.val = value; + validityPerField['password'] = state.valid; + isValid.val = Object.values(validityPerField).every(v => v); + }, }), ), - div( - { class: 'flex-column fx-gap-3', style: 'padding: 2px; flex: 1;' }, - '', - ), ); }; -const HttpPathConnectionForm = ( +const PostgresqlForm = RedshiftForm; + +const AzureMSSQLForm = RedshiftForm; + +const SynapseMSSQLForm = RedshiftForm; + +const MSSQLForm = RedshiftForm; + +/** + * @param {VanState} connection + * @param {Flavor} flavor + * @param {boolean} maskPassword + * @param {(params: Partial, isValid: boolean) => void} onChange + * @param {Connection?} originalConnection + * @returns {HTMLElement} + */ +const DatabricksForm = ( connection, - password, - httpPath, - onValueChange, - useSecretsPlaceholder, + flavor, + onChange, + originalConnection, ) => { - const passwordFieldState = van.state({value: password.val, valid: false}); - const httpPathFieldState = van.state({value: httpPath.val, valid: false}); + const originalURLTemplate = flavor.connection_string; + + const isValid = van.state(true); + const connectByUrl = van.state(connection.rawVal?.connect_by_url ?? false); + const connectionHost = van.state(connection.rawVal?.project_host ?? ''); + const connectionPort = van.state(connection.rawVal?.project_port || defaultPorts[flavor.flavor]); + const connectionHttpPath = van.state(connection.rawVal?.http_path ?? ''); + const connectionDatabase = van.state(connection.rawVal?.project_db ?? ''); + const connectionUsername = van.state(connection.rawVal?.project_user ?? ''); + const connectionPassword = van.state(connection.rawVal?.password ?? ''); + + const [prefixPart, sufixPart] = originalURLTemplate.split('@'); + const connectionStringPrefix = van.state(`${prefixPart}@`); + const connectionStringSuffix = van.state(connection.rawVal?.url ?? ''); + + const validityPerField = {}; + + if (!connectionStringSuffix.rawVal) { + connectionStringSuffix.val = formatURL(sufixPart ?? '', connectionHost.rawVal, connectionPort.rawVal, connectionDatabase.rawVal, connectionHttpPath.rawVal); + } van.derive(() => { - const passwordField = passwordFieldState.val; - const httpPathField = httpPathFieldState.val; - onValueChange({password: passwordField.value, http_path: httpPathField.value}, { valid: passwordField.valid && httpPathField.valid }); + const connectionHost_ = connectionHost.val; + const connectionPort_ = connectionPort.val; + const connectionDatabase_ = connectionDatabase.val; + const connectionHttpPath_ = connectionHttpPath.val; + + if (!connectByUrl.rawVal && originalURLTemplate.includes('@')) { + const [, originalURLSuffix] = originalURLTemplate.split('@'); + connectionStringSuffix.val = formatURL(originalURLSuffix, connectionHost_, connectionPort_, connectionDatabase_, connectionHttpPath_); + } + }); + + van.derive(() => { + onChange({ + project_host: connectionHost.val, + project_port: connectionPort.val, + project_db: connectionDatabase.val, + project_user: connectionUsername.val, + password: connectionPassword.val, + http_path: connectionHttpPath.val, + connect_by_url: connectByUrl.val, + url: connectByUrl.val ? connectionStringSuffix.val : connectionStringSuffix.rawVal, + }, isValid.val); }); return div( - { class: 'flex-row fx-gap-3 fx-align-stretch' }, + {class: 'flex-column fx-gap-3 fx-flex'}, div( - { class: 'flex-column fx-gap-3', style: 'flex: 2' }, - Input({ - name: 'password', - label: 'Password', - value: password, - height: 38, - type: 'password', - placeholder: (useSecretsPlaceholder && connection.password) ? secretsPlaceholder : '', - onChange: (value, state) => passwordFieldState.val = {value, valid: state.valid}, + { class: 'flex-column border border-radius-1 p-3 mt-1 fx-gap-1', style: 'position: relative;' }, + Caption({content: 'Server', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), + + RadioGroup({ + label: 'Connect by', + options: [ + { + label: 'Host', + value: false, + }, + { + label: 'URL', + value: true, + }, + ], + value: connectByUrl, + onChange: (value) => connectByUrl.val = value, + inline: true, }), + div( + { class: 'flex-row fx-gap-3 fx-flex' }, + Input({ + name: 'db_host', + label: 'Host', + value: connectionHost, + height: 38, + class: 'fx-flex', + disabled: connectByUrl, + onChange: (value, state) => { + connectionHost.val = value; + validityPerField['db_host'] = state.valid; + isValid.val = Object.values(validityPerField).every(v => v); + }, + validators: [ maxLength(250) ], + }), + Input({ + name: 'db_port', + label: 'Port', + value: connectionPort, + height: 38, + type: 'number', + disabled: connectByUrl, + onChange: (value, state) => { + connectionPort.val = value; + validityPerField['db_port'] = state.valid; + isValid.val = Object.values(validityPerField).every(v => v); + }, + validators: [ minLength(3), maxLength(5) ], + }) + ), Input({ label: 'HTTP Path', - value: httpPath, + value: connectionHttpPath, class: 'fx-flex', height: 38, name: 'http_path', - onChange: (value, state) => httpPathFieldState.val = {value, valid: state.valid}, + disabled: connectByUrl, + onChange: (value, state) => { + connectionHttpPath.val = value; + validityPerField['http_path'] = state.valid; + isValid.val = Object.values(validityPerField).every(v => v); + }, validators: [ maxLength(50) ], - }) + }), + Input({ + name: 'db_name', + label: 'Database', + value: connectionDatabase, + height: 38, + disabled: connectByUrl, + onChange: (value, state) => { + connectionDatabase.val = value; + validityPerField['db_name'] = state.valid; + isValid.val = Object.values(validityPerField).every(v => v); + }, + validators: [ maxLength(100) ], + }), + () => div( + { class: 'flex-row fx-gap-3 fx-align-stretch', style: 'position: relative;' }, + Input({ + label: 'URL', + value: connectionStringSuffix, + class: 'fx-flex', + height: 38, + name: 'url_suffix', + prefix: span({ style: 'height: 38px; white-space: nowrap; color: var(--disabled-text-color)' }, connectionStringPrefix), + disabled: !connectByUrl.val, + onChange: (value, state) => connectionStringSuffix.val = value, + }), + ), ), + div( - { class: 'flex-column fx-gap-3', style: 'padding: 2px; flex: 1;' }, - '', + { class: 'flex-column border border-radius-1 p-3 mt-1 fx-gap-1', style: 'position: relative;' }, + Caption({content: 'Authentication', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), + + Input({ + name: 'db_user', + label: 'Username', + value: connectionUsername, + height: 38, + onChange: (value, state) => { + connectionUsername.val = value; + validityPerField['db_user'] = state.valid; + isValid.val = Object.values(validityPerField).every(v => v); + }, + validators: [ maxLength(50) ], + }), + Input({ + name: 'password', + label: 'Password', + value: connectionPassword, + height: 38, + type: 'password', + placeholder: (originalConnection?.connection_id && originalConnection?.password) ? secretsPlaceholder : '', + onChange: (value, state) => { + connectionPassword.val = value; + validityPerField['password'] = state.valid; + isValid.val = Object.values(validityPerField).every(v => v); + }, + }), ), ); }; -const KeyPairConnectionForm = ( +/** + * @param {VanState} connection + * @param {Flavor} flavor + * @param {boolean} maskPassword + * @param {(params: Partial, isValid: boolean) => void} onChange + * @param {Connection?} originalConnection + * @param {string?} originalConnection + * @returns {HTMLElement} + */ +const SnowflakeForm = ( connection, - connectByKey, - password, - privateKey, - privateKeyPhrase, - clearPrivateKeyPhrase, - onValueChange, - useSecretsPlaceholder, + flavor, + onChange, + originalConnection, + cachedFile, ) => { - const connectByKeyFieldState = van.state({value: connectByKey.val, valid: true}); - const passwordFieldState = van.state({value: password.val, valid: true}); - const privateKeyFieldState = van.state({value: privateKey.val, valid: true}); - const privateKeyPhraseFieldState = van.state({value: privateKeyPhrase.val, valid: true}); + const originalURLTemplate = flavor.connection_string; + + const isValid = van.state(false); + const clearPrivateKeyPhrase = van.state(connection.rawVal?.private_key_passphrase === clearSentinel); + const connectByUrl = van.state(connection.rawVal.connect_by_url ?? false); + const connectByKey = van.state(connection.rawVal?.connect_by_key ?? false); + const connectionHost = van.state(connection.rawVal.project_host ?? ''); + const connectionPort = van.state(connection.rawVal.project_port || defaultPorts[flavor.flavor]); + const connectionDatabase = van.state(connection.rawVal.project_db ?? ''); + const connectionUsername = van.state(connection.rawVal.project_user ?? ''); + const connectionPassword = van.state(connection.rawVal?.password ?? ''); + const connectionPrivateKey = van.state(connection.rawVal?.private_key ?? ''); + const connectionPrivateKeyPassphrase = van.state( + clearPrivateKeyPhrase.rawVal + ? '' + : (connection.rawVal?.private_key_passphrase ?? '') + ); + const validityPerField = {}; + + const privateKeyFileRaw = van.state(cachedFile); + + const [prefixPart, sufixPart] = originalURLTemplate.split('@'); + const connectionStringPrefix = van.state(`${prefixPart}@`); + const connectionStringSuffix = van.state(connection.rawVal?.url ?? ''); + + if (!connectionStringSuffix.rawVal) { + connectionStringSuffix.val = formatURL(sufixPart ?? '', connectionHost.rawVal, connectionPort.rawVal, connectionDatabase.rawVal); + } van.derive(() => { - const connectByKeyField = connectByKeyFieldState.val; - const passwordField = passwordFieldState.val; - const privateKeyField = privateKeyFieldState.val; - const privateKeyPhraseField = privateKeyPhraseFieldState.val; - - let isValid = passwordField.valid; - if (connectByKeyField.value) { - isValid = privateKeyField.valid && privateKeyPhraseField.valid; + const connectionHost_ = connectionHost.val; + const connectionPort_ = connectionPort.val; + const connectionDatabase_ = connectionDatabase.val; + + if (!connectByUrl.rawVal && originalURLTemplate.includes('@')) { + const [, originalURLSuffix] = originalURLTemplate.split('@'); + connectionStringSuffix.val = formatURL(originalURLSuffix, connectionHost_, connectionPort_, connectionDatabase_); } + }); - onValueChange( - { - connect_by_key: connectByKeyField.value, - password: passwordField.value, - private_key: privateKeyField.value, - private_key_passphrase: privateKeyPhraseField.value, - }, - { valid: isValid }, - ); + van.derive(() => { + onChange({ + project_host: connectionHost.val, + project_port: connectionPort.val, + project_db: connectionDatabase.val, + project_user: connectionUsername.val, + password: connectionPassword.val, + connect_by_url: connectByUrl.val, + url: connectByUrl.val ? connectionStringSuffix.val : connectionStringSuffix.rawVal, + connect_by_key: connectByKey.val, + private_key: connectionPrivateKey.val, + private_key_passphrase: clearPrivateKeyPhrase.val ? clearSentinel : connectionPrivateKeyPassphrase.val, + }, privateKeyFileRaw.val, isValid.val); }); return div( - { class: 'flex-column' }, - hr({ style: 'width: 100%;', class: 'mt-2 mb-2' }), - RadioGroup({ - label: 'Connection Strategy', - options: [ - {label: 'Connect By Password', value: false}, - {label: 'Connect By Key-Pair', value: true}, - ], - value: connectByKey, - onChange: (value) => connectByKeyFieldState.val = {value, valid: true}, - }), - () => { - if (connectByKey.val) { - return div( - { class: 'flex-column fx-gap-3' }, - div( - { class: 'key-pair-passphrase-field'}, - Input({ - name: 'private_key_passphrase', - label: 'Private Key Passphrase', - value: privateKeyPhrase, - height: 38, - type: 'password', - help: 'Passphrase used when creating the private key. Leave empty if the private key is not encrypted.', - placeholder: () => (useSecretsPlaceholder && connection.private_key_passphrase && !clearPrivateKeyPhrase.val) ? secretsPlaceholder : '', + {class: 'flex-column fx-gap-3 fx-flex'}, + div( + { class: 'flex-column border border-radius-1 p-3 mt-1 fx-gap-1', style: 'position: relative;' }, + Caption({content: 'Server', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), + + RadioGroup({ + label: 'Connect by', + options: [ + { + label: 'Host', + value: false, + }, + { + label: 'URL', + value: true, + }, + ], + value: connectByUrl, + onChange: (value) => connectByUrl.val = value, + inline: true, + }), + div( + { class: 'flex-row fx-gap-3 fx-flex' }, + Input({ + name: 'db_host', + label: 'Host', + value: connectionHost, + height: 38, + class: 'fx-flex', + disabled: connectByUrl, + onChange: (value, state) => { + connectionHost.val = value; + validityPerField['db_host'] = state.valid; + isValid.val = Object.values(validityPerField).every(v => v); + }, + validators: [ maxLength(250) ], + }), + Input({ + name: 'db_port', + label: 'Port', + value: connectionPort, + height: 38, + type: 'number', + disabled: connectByUrl, + onChange: (value, state) => { + connectionPort.val = value; + validityPerField['db_port'] = state.valid; + isValid.val = Object.values(validityPerField).every(v => v); + }, + validators: [ minLength(3), maxLength(5) ], + }) + ), + Input({ + name: 'db_name', + label: 'Database', + value: connectionDatabase, + height: 38, + disabled: connectByUrl, + onChange: (value, state) => { + connectionDatabase.val = value; + validityPerField['db_name'] = state.valid; + isValid.val = Object.values(validityPerField).every(v => v); + }, + validators: [ maxLength(100) ], + }), + () => div( + { class: 'flex-row fx-gap-3 fx-align-stretch', style: 'position: relative;' }, + Input({ + label: 'URL', + value: connectionStringSuffix, + class: 'fx-flex', + height: 38, + name: 'url_suffix', + prefix: span({ style: 'height: 38px; white-space: nowrap; color: var(--disabled-text-color)' }, connectionStringPrefix), + disabled: !connectByUrl.val, + onChange: (value, state) => { + connectionStringSuffix.val = value; + validityPerField['url_suffix'] = state.valid; + isValid.val = Object.values(validityPerField).every(v => v); + }, + }), + ), + ), + + div( + { class: 'flex-column border border-radius-1 p-3 mt-1 fx-gap-1', style: 'position: relative;' }, + Caption({content: 'Authentication', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), + + RadioGroup({ + label: 'Connection Strategy', + options: [ + {label: 'Connect By Password', value: false}, + {label: 'Connect By Key-Pair', value: true}, + ], + value: connectByKey, + onChange: (value) => connectByKey.val = value, + inline: true, + }), + + Input({ + name: 'db_user', + label: 'Username', + value: connectionUsername, + height: 38, + onChange: (value, state) => { + connectionUsername.val = value; + validityPerField['db_user'] = state.valid; + isValid.val = Object.values(validityPerField).every(v => v); + }, + validators: [ maxLength(50) ], + }), + () => { + if (connectByKey.val) { + return div( + { class: 'flex-column fx-gap-3' }, + div( + { class: 'key-pair-passphrase-field'}, + Input({ + name: 'private_key_passphrase', + label: 'Private Key Passphrase', + value: connectionPrivateKeyPassphrase, + height: 38, + type: 'password', + help: 'Passphrase used when creating the private key. Leave empty if the private key is not encrypted.', + placeholder: () => (originalConnection?.connection_id && originalConnection?.private_key_passphrase && !clearPrivateKeyPhrase.val) ? secretsPlaceholder : '', + onChange: (value, state) => { + if (value) { + clearPrivateKeyPhrase.val = false; + } + connectionPrivateKeyPassphrase.val = value; + validityPerField['private_key_passphrase'] = state.valid; + isValid.val = Object.values(validityPerField).every(v => v); + }, + }), + () => { + const hasPrivateKeyPhrase = originalConnection?.private_key_passphrase || connectionPrivateKeyPassphrase.val; + if (!hasPrivateKeyPhrase) { + return ''; + } + + return i( + { + class: 'material-symbols-rounded clickable text-secondary', + onclick: () => { + clearPrivateKeyPhrase.val = true; + connectionPrivateKeyPassphrase.val = ''; + }, + }, + 'clear', + ); + }, + ), + FileInput({ + name: 'private_key', + label: 'Upload private key (rsa_key.p8)', + placeholder: (originalConnection?.connection_id && originalConnection?.private_key) + ? 'Drop file here or browse files to replace existing key' + : undefined, + value: privateKeyFileRaw, onChange: (value, state) => { - if (value) { - clearPrivateKeyPhrase.val = false; + let isFieldValid = state.valid; + + privateKeyFileRaw.val = value; + try { + if (value?.content) { + connectionPrivateKey.val = value.content.split(',')?.[1] ?? ''; + } + } catch (err) { + console.error(err); + isFieldValid = false; } - privateKeyPhraseFieldState.val = {value, valid: state.valid}; + validityPerField['private_key'] = state.valid; + isValid.val = Object.values(validityPerField).every(v => v); }, + validators: [ + sizeLimit(200 * 1024 * 1024), + ], }), - () => { - const hasPrivateKeyPhrase = connection.private_key_passphrase || privateKeyPhraseFieldState.val?.value; - if (!hasPrivateKeyPhrase) { - return ''; - } - - return i( - { - class: 'material-symbols-rounded clickable text-secondary', - onclick: () => { - clearPrivateKeyPhrase.val = true; - privateKeyPhraseFieldState.val = {value: '', valid: true}; - }, - }, - 'clear', - ); - }, - ), - FileInput({ - name: 'private_key', - label: 'Upload private key (rsa_key.p8)', - placeholder: connection.private_key ? 'Drop file here or browse files to replace existing key' : undefined, - value: privateKey, - onChange: (value, state) => privateKeyFieldState.val = {value, valid: state.valid}, - validators: [ - sizeLimit(200 * 1024 * 1024), - ], - }), - ); - } - - return Input({ - name: 'password', - label: 'Password', - value: password, - height: 38, - type: 'password', - placeholder: (useSecretsPlaceholder && connection.password) ? secretsPlaceholder : '', - onChange: (value, state) => passwordFieldState.val = {value, valid: state.valid}, - }); - }, + ); + } + + return Input({ + name: 'password', + label: 'Password', + value: connectionPassword, + height: 38, + type: 'password', + placeholder: (originalConnection?.connection_id && originalConnection?.password) ? secretsPlaceholder : '', + onChange: (value, state) => { + connectionPassword.val = value; + validityPerField['password'] = state.valid; + isValid.val = Object.values(validityPerField).every(v => v); + }, + }); + }, + ), ); }; diff --git a/testgen/ui/components/frontend/js/components/input.js b/testgen/ui/components/frontend/js/components/input.js index ad3cdee8..b5f19b5d 100644 --- a/testgen/ui/components/frontend/js/components/input.js +++ b/testgen/ui/components/frontend/js/components/input.js @@ -27,6 +27,7 @@ * @property {string?} type * @property {string?} class * @property {string?} testId + * @property {any?} prefix * @property {Array?} validators */ import van from '../van.min.js'; @@ -109,20 +110,31 @@ const Input = (/** @type Properties */ props) => { }, 'clear', ) : '', - input({ - class: () => `tg-input--field ${getValue(props.disabled) ? 'tg-input--disabled' : ''}`, - style: () => `height: ${getValue(props.height) || defaultHeight}px;`, - value, - name: props.name ?? '', - type: props.type ?? 'text', - disabled: props.disabled, - placeholder: () => getValue(props.placeholder) ?? '', - oninput: debounce((/** @type Event */ event) => value.val = event.target.value, 300), - onclick: van.derive(() => autocompleteOptions.val?.length - ? () => autocompleteOpened.val = true - : null - ), - }), + + div( + { + class: () => `flex-row tg-input--field ${getValue(props.disabled) ? 'tg-input--disabled' : ''}`, + style: () => `height: ${getValue(props.height) || defaultHeight}px;`, + }, + props.prefix + ? div( + { class: 'tg-input--field-prefix' }, + props.prefix, + ) + : undefined, + input({ + value, + name: props.name ?? '', + type: props.type ?? 'text', + disabled: props.disabled, + placeholder: () => getValue(props.placeholder) ?? '', + oninput: debounce((/** @type Event */ event) => value.val = event.target.value, 300), + onclick: van.derive(() => autocompleteOptions.val?.length + ? () => autocompleteOpened.val = true + : null + ), + }), + ), () => getValue(props.validators)?.length > 0 ? small({ class: 'tg-input--error' }, firstError) @@ -183,12 +195,25 @@ stylesheet.replace(` border: 1px solid transparent; transition: border-color 0.3s; background-color: var(--form-field-color); - padding: 4px 8px; color: var(--primary-text-color); font-size: 14px; } +.tg-input--field > .tg-input--field-prefix { + padding-left: 8px; +} +.tg-input--field > input { + width: 100%; + height: 100%; + box-sizing: border-box; + font-size: 14px; + background-color: var(--form-field-color); + color: var(--primary-text-color); + border: unset; + padding: 4px 8px; + border-radius: 8px; +} -.tg-input--field::placeholder { +.tg-input--field > input::placeholder { font-style: italic; color: var(--disabled-text-color); } @@ -232,7 +257,7 @@ stylesheet.replace(` background: var(--select-hover-background); } -.tg-input--disabled { +.tg-input--disabled > input { cursor: not-allowed; color: var(--disabled-text-color); } diff --git a/testgen/ui/components/frontend/js/components/radio_group.js b/testgen/ui/components/frontend/js/components/radio_group.js index aad826e3..26807332 100644 --- a/testgen/ui/components/frontend/js/components/radio_group.js +++ b/testgen/ui/components/frontend/js/components/radio_group.js @@ -11,6 +11,7 @@ * @property {string | number | boolean | null} selected * @property {function(string | number | boolean | null)?} onChange * @property {number?} width + * @property {boolean?} inline */ import van from '../van.min.js'; import { getRandomId, getValue, loadStylesheet } from '../utils.js'; @@ -19,12 +20,13 @@ const { div, input, label } = van.tags; const RadioGroup = (/** @type Properties */ props) => { loadStylesheet('radioGroup', stylesheet); + const groupName = getRandomId(); return div( - { style: () => `width: ${props.width ? getValue(props.width) + 'px' : 'auto'}` }, + { class: () => `${getValue(props.inline) ? 'flex-row fx-gap-2' : ''}`, style: () => `width: ${props.width ? getValue(props.width) + 'px' : 'auto'}` }, div( - { class: 'text-caption mb-1' }, + { class: () => `text-caption ${getValue(props.inline) ? '' : 'mb-1'}` }, props.label, ), () => div( diff --git a/testgen/ui/components/frontend/js/components/score_issues.js b/testgen/ui/components/frontend/js/components/score_issues.js index 2c319249..db62cbea 100644 --- a/testgen/ui/components/frontend/js/components/score_issues.js +++ b/testgen/ui/components/frontend/js/components/score_issues.js @@ -31,7 +31,7 @@ import { colorMap, formatTimestamp } from '../display_utils.js'; const { div, i, span } = van.tags; const PAGE_SIZE = 100; -const SCROLL_CONTAINER = window.top.document.querySelector('.stAppViewMain'); +const SCROLL_CONTAINER = window.top.document.querySelector('.stMain'); const IssuesTable = ( /** @type Issue[] */ issues, @@ -116,7 +116,7 @@ const IssuesTable = ( }), ), ), - Toolbar(filters, issues, category), + () => Toolbar(filters, issues, category), div( { class: 'table-header issues-columns flex-row' }, Checkbox({ @@ -157,8 +157,10 @@ const IssuesTable = ( count: filteredIssues.val.length, pageSize: PAGE_SIZE, onChange: (newIndex) => { - pageIndex.val = newIndex; - SCROLL_CONTAINER.scrollTop = 0; + if (newIndex !== pageIndex.val) { + pageIndex.val = newIndex; + SCROLL_CONTAINER.scrollTop = 0; + } }, }), ); @@ -320,7 +322,7 @@ const TimeCell = (value, row) => { const SCORE_LABEL = { table: 'Table', column: 'Column', - type: 'Issue Name', + type: 'Issue Type', status: 'Likelihood / Status', }; @@ -333,7 +335,7 @@ const COLUMN_LABEL = { const ISSUES_COLUMN_LABEL = { column: 'Table | Column', - type: 'Issue Type | Name', + type: 'Issue Type', status: 'Likelihood / Status', detail: 'Detail', time: 'Test Suite | Start Time', diff --git a/testgen/ui/components/frontend/js/components/table_group_form.js b/testgen/ui/components/frontend/js/components/table_group_form.js index 6d043319..909160cf 100644 --- a/testgen/ui/components/frontend/js/components/table_group_form.js +++ b/testgen/ui/components/frontend/js/components/table_group_form.js @@ -14,6 +14,7 @@ * @property {string?} profile_sk_column_mask * @property {number?} profiling_delay_days * @property {boolean?} profile_flag_cdes + * @property {boolean?} include_in_dashboard * @property {boolean?} add_scorecard_definition * @property {boolean?} profile_use_sampling * @property {number?} profile_sample_percent @@ -38,7 +39,7 @@ * @property {TableGroup} tableGroup * @property {Connection[]} connections * @property {boolean?} showConnectionSelector - * @property {boolean?} enableConnectionSelector + * @property {boolean?} disableConnectionSelector * @property {boolean?} disableSchemaField * @property {(tg: TableGroup, state: FormState) => void} onChange */ @@ -49,8 +50,17 @@ import { Checkbox } from './checkbox.js'; import { ExpansionPanel } from './expansion_panel.js'; import { required } from '../form_validators.js'; import { Select } from './select.js'; +import { Caption } from './caption.js'; +import { Textarea } from './textarea.js'; -const { div, span } = van.tags; +const { div } = van.tags; + +const normalizeTableSet = (value) => { + return value?.split(/[,\n]/) + .map(part => part.trim()) + .filter(part => part) + .join(', '); +} /** * @@ -65,12 +75,13 @@ const TableGroupForm = (props) => { const tableGroupsName = van.state(tableGroup.table_groups_name); const profilingIncludeMask = van.state(tableGroup.profiling_include_mask ?? '%'); const profilingExcludeMask = van.state(tableGroup.profiling_exclude_mask ?? 'tmp%'); - const profilingTableSet = van.state(tableGroup.profiling_table_set); + const profilingTableSet = van.state(normalizeTableSet(tableGroup.profiling_table_set)); const tableGroupSchema = van.state(tableGroup.table_group_schema); const profileIdColumnMask = van.state(tableGroup.profile_id_column_mask ?? '%_id'); const profileSkColumnMask = van.state(tableGroup.profile_sk_column_mask ?? '%_sk'); const profilingDelayDays = van.state(tableGroup.profiling_delay_days ?? 0); const profileFlagCdes = van.state(tableGroup.profile_flag_cdes ?? true); + const includeInDashboard = van.state(tableGroup.include_in_dashboard ?? true); const addScorecardDefinition = van.state(tableGroup.add_scorecard_definition ?? true); const profileUseSampling = van.state(tableGroup.profile_use_sampling ?? false); const profileSamplePercent = van.state(tableGroup.profile_sample_percent ?? 30); @@ -94,7 +105,6 @@ const TableGroupForm = (props) => { })); }); const showConnectionSelector = getValue(props.showConnectionSelector) ?? false; - const disableConnectionSelector = van.derive(() => !getValue(props.enableConnectionSelector) || (getValue(props.connections) ?? []).length <= 0); const disableSchemaField = van.derive(() => getValue(props.disableSchemaField) ?? false) const updatedTableGroup = van.derive(() => { @@ -104,12 +114,13 @@ const TableGroupForm = (props) => { table_groups_name: tableGroupsName.val, profiling_include_mask: profilingIncludeMask.val, profiling_exclude_mask: profilingExcludeMask.val, - profiling_table_set: profilingTableSet.val, + profiling_table_set: normalizeTableSet(profilingTableSet.val), table_group_schema: tableGroupSchema.val, profile_id_column_mask: profileIdColumnMask.val, profile_sk_column_mask: profileSkColumnMask.val, profiling_delay_days: profilingDelayDays.val, profile_flag_cdes: profileFlagCdes.val, + include_in_dashboard: includeInDashboard.val, add_scorecard_definition: addScorecardDefinition.val, profile_use_sampling: profileUseSampling.val, profile_sample_percent: profileSamplePercent.val, @@ -151,7 +162,7 @@ const TableGroupForm = (props) => { value: tableGroupConnectionId.rawVal, options: connectionOptions, height: 38, - disabled: disableConnectionSelector, + disabled: props.disableConnectionSelector, onChange: (value) => { tableGroupConnectionId.val = value; setFieldValidity('connection_id', !!value); @@ -161,14 +172,21 @@ const TableGroupForm = (props) => { MainForm( { disableSchemaField, setValidity: setFieldValidity }, tableGroupsName, + tableGroupSchema, + ), + CriteriaForm( + { setValidity: setFieldValidity }, profilingIncludeMask, profilingExcludeMask, profilingTableSet, - tableGroupSchema, profileIdColumnMask, profileSkColumnMask, + ), + SettingsForm( + { editMode: !!tableGroup.id, setValidity: setFieldValidity }, profilingDelayDays, profileFlagCdes, + includeInDashboard, addScorecardDefinition, ), SamplingForm( @@ -195,23 +213,16 @@ const TableGroupForm = (props) => { const MainForm = ( options, tableGroupsName, - profilingIncludeMask, - profilingExcludeMask, - profilingTableSet, tableGroupSchema, - profileIdColumnMask, - profileSkColumnMask, - profilingDelayDays, - profileFlagCdes, - addScorecardDefinition, ) => { return div( - { class: 'tg-main-form flex-column fx-gap-3 fx-flex-wrap' }, + { class: 'flex-row fx-gap-3 fx-flex-wrap' }, Input({ name: 'table_groups_name', label: 'Name', value: tableGroupsName, height: 38, + class: 'tg-column-flex', help: 'Unique name to describe the table group', helpPlacement: 'bottom-right', onChange: (value, state) => { @@ -220,50 +231,12 @@ const MainForm = ( }, validators: [ required ], }), - Input({ - name: 'profiling_include_mask', - label: 'Tables to Include Mask', - value: profilingIncludeMask, - height: 38, - help: 'SQL filter supported by your database\'s LIKE operator for table names to include', - onChange: (value, state) => { - profilingIncludeMask.val = value; - options.setValidity?.('profiling_include_mask', state.valid); - }, - }), - Input({ - name: 'profiling_exclude_mask', - label: 'Tables to Exclude Mask', - value: profilingExcludeMask, - height: 38, - help: 'SQL filter supported by your database\'s LIKE operator for table names to exclude', - onChange: (value, state) => { - profilingExcludeMask.val = value; - options.setValidity?.('profiling_exclude_mask', state.valid); - }, - }), - Input({ - name: 'profiling_table_set', - label: 'Explicit Table List', - value: profilingTableSet, - height: 38, - help: 'List of specific table names to include, separated by commas', - onChange: (value, state) => { - profilingTableSet.val = value; - options.setValidity?.('profiling_table_set', state.valid); - }, - }), - Checkbox({ - name: 'profile_flag_cdes', - label: 'Detect critical data elements (CDE) during profiling', - checked: profileFlagCdes, - onChange: (value) => profileFlagCdes.val = value, - }), Input({ name: 'table_group_schema', label: 'Schema', value: tableGroupSchema, height: 38, + class: 'tg-column-flex', help: 'Database schema containing the tables for the Table Group', helpPlacement: 'bottom-left', disabled: options.disableSchemaField, @@ -273,47 +246,134 @@ const MainForm = ( }, validators: [ required ], }), - Input({ - name: 'profile_id_column_mask', - label: 'Profiling ID Column Mask', - value: profileIdColumnMask, - height: 38, - help: 'SQL filter supported by your database\'s LIKE operator representing ID columns (optional)', - onChange: (value, state) => { - profileIdColumnMask.val = value; - options.setValidity?.('profile_id_column_mask', state.valid); - }, - }), - Input({ - name: 'profile_sk_column_mask', - label: 'Profiling Surrogate Key Column Mask', - value: profileSkColumnMask, - height: 38, - help: 'SQL filter supported by your database\'s LIKE operator representing surrogate key columns (optional)', - onChange: (value, state) => { - profileSkColumnMask.val = value - options.setValidity?.('profile_sk_column_mask', state.valid); - }, - }), + ); +}; + +const CriteriaForm = ( + options, + profilingIncludeMask, + profilingExcludeMask, + profilingTableSet, + profileIdColumnMask, + profileSkColumnMask, +) => { + return div( + { class: 'flex-column fx-gap-3 border border-radius-1 p-3 mt-1', style: 'position: relative;' }, + Caption({content: 'Criteria', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), + div( + { class: 'flex-row fx-gap-3 fx-flex-wrap fx-align-flex-start' }, + div( + { class: 'tg-column-flex flex-column fx-gap-3', }, + Input({ + name: 'profiling_include_mask', + label: 'Tables to Include Mask', + value: profilingIncludeMask, + height: 38, + help: 'SQL filter supported by your database\'s LIKE operator for table names to include', + onChange: (value, state) => { + profilingIncludeMask.val = value; + options.setValidity?.('profiling_include_mask', state.valid); + }, + }), + Input({ + name: 'profiling_exclude_mask', + label: 'Tables to Exclude Mask', + value: profilingExcludeMask, + height: 38, + help: 'SQL filter supported by your database\'s LIKE operator for table names to exclude', + onChange: (value, state) => { + profilingExcludeMask.val = value; + options.setValidity?.('profiling_exclude_mask', state.valid); + }, + }), + ), + Textarea({ + name: 'profiling_table_set', + label: 'Explicit Table List', + value: profilingTableSet, + height: 108, + class: 'tg-column-flex', + help: 'List of specific table names to include, separated by commas or newlines', + onChange: (value) => profilingTableSet.val = value, + }), + ), + div( + { class: 'flex-row fx-gap-3 fx-flex-wrap' }, + Input({ + name: 'profile_id_column_mask', + label: 'Profiling ID Column Mask', + value: profileIdColumnMask, + height: 38, + class: 'tg-column-flex', + help: 'SQL filter supported by your database\'s LIKE operator representing ID columns', + onChange: (value, state) => { + profileIdColumnMask.val = value; + options.setValidity?.('profile_id_column_mask', state.valid); + }, + }), + Input({ + name: 'profile_sk_column_mask', + label: 'Profiling Surrogate Key Column Mask', + value: profileSkColumnMask, + height: 38, + class: 'tg-column-flex', + help: 'SQL filter supported by your database\'s LIKE operator representing surrogate key columns', + onChange: (value, state) => { + profileSkColumnMask.val = value + options.setValidity?.('profile_sk_column_mask', state.valid); + }, + }), + ), + ); +}; + +const SettingsForm = ( + options, + profilingDelayDays, + profileFlagCdes, + includeInDashboard, + addScorecardDefinition, +) => { + return div( + { class: 'flex-row fx-gap-3 fx-flex-wrap fx-align-flex-start border border-radius-1 p-3 mt-1', style: 'position: relative;' }, + Caption({content: 'Settings', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), + div( + { class: 'tg-column-flex flex-column fx-gap-3' }, + Checkbox({ + name: 'profile_flag_cdes', + label: 'Detect critical data elements (CDE) during profiling', + checked: profileFlagCdes, + onChange: (value) => profileFlagCdes.val = value, + }), + Checkbox({ + name: 'include_in_dashboard', + label: 'Include table group in Project Dashboard', + checked: includeInDashboard, + onChange: (value) => includeInDashboard.val = value, + }), + () => !options.editMode + ? Checkbox({ + name: 'add_scorecard_definition', + label: 'Add scorecard for table group', + help: 'Add a new scorecard to the Quality Dashboard upon creation of this table group', + checked: addScorecardDefinition, + onChange: (value) => addScorecardDefinition.val = value, + }) + : null, + ), Input({ name: 'profiling_delay_days', type: 'number', label: 'Min Profiling Age (in days)', value: profilingDelayDays, height: 38, + class: 'tg-column-flex', help: 'Number of days to wait before new profiling will be available to generate tests', onChange: (value, state) => { profilingDelayDays.val = value; options.setValidity?.('profiling_delay_days', state.valid); }, }), - Checkbox({ - name: 'add_scorecard_definition', - label: 'Add scorecard for table group', - help: 'Add a new scorecard to the Quality Dashboard upon creation of this table group', - checked: addScorecardDefinition, - onChange: (value) => addScorecardDefinition.val = value, - }), ); }; @@ -323,17 +383,17 @@ const SamplingForm = ( profileSamplePercent, profileSampleMinCount, ) => { - return div( - { class: 'tg-sampling-form flex-column fx-gap-3' }, - Checkbox({ - name: 'profile_use_sampling', - label: 'Use profile sampling', - help: 'When checked, profiling will be based on a sample of records instead of the full table', - checked: profileUseSampling, - onChange: (value) => profileUseSampling.val = value, - }), - ExpansionPanel( - { title: 'Sampling Parameters', testId: 'sampling-panel' }, + return ExpansionPanel( + { title: 'Sampling Parameters', testId: 'sampling-panel' }, + div( + { class: 'flex-column fx-gap-3' }, + Checkbox({ + name: 'profile_use_sampling', + label: 'Use profile sampling', + help: 'When checked, profiling will be based on a sample of records instead of the full table', + checked: profileUseSampling, + onChange: (value) => profileUseSampling.val = value, + }), div( { class: 'flex-row fx-gap-3' }, Input({ @@ -488,10 +548,9 @@ const TaggingForm = ( const stylesheet = new CSSStyleSheet(); stylesheet.replace(` -.tg-main-form { - height: 316px; +.tg-column-flex { + flex: 250px; } - .tg-tagging-form-fields { height: 332px; } diff --git a/testgen/ui/components/frontend/js/components/table_group_test.js b/testgen/ui/components/frontend/js/components/table_group_test.js new file mode 100644 index 00000000..c5dfcaf6 --- /dev/null +++ b/testgen/ui/components/frontend/js/components/table_group_test.js @@ -0,0 +1,76 @@ +/** + * @typedef TableGroupPreview + * @type {object} + * @property {string} schema + * @property {string[]?} tables + * @property {number?} column_count + * @property {boolean?} success + * @property {string?} message + */ +import van from '../van.min.js'; +import { getValue } from '../utils.js'; +import { Alert } from '../components/alert.js'; + +const { div, span, strong } = van.tags; + +/** + * + * @param {string} schema + * @param {TableGroupPreview?} preview + * @returns {HTMLElement} + */ +const TableGroupTest = (schema, preview) => { + return div( + { class: 'flex-column fx-gap-2' }, + div( + { class: 'flex-row fx-justify-space-between' }, + div( + { class: 'flex-column fx-gap-2' }, + div( + { class: 'flex-row fx-gap-1' }, + strong({}, 'Schema:'), + span({}, schema), + ), + div( + { class: 'flex-row fx-gap-1' }, + strong({}, 'Table Count:'), + () => span({}, getValue(preview)?.tables?.length ?? '--'), + ), + div( + { class: 'flex-row fx-gap-1' }, + strong({}, 'Column Count:'), + () => span({}, getValue(preview)?.column_count ?? '--'), + ), + ), + ), + () => { + const tableGroupPreview = getValue(preview); + const wasPreviewExecuted = tableGroupPreview && typeof tableGroupPreview.success === 'boolean'; + + if (!wasPreviewExecuted) { + return ''; + } + + return div( + { class: 'table hoverable p-3' }, + div( + { class: 'table-header' }, + span('Tables'), + ), + div( + { class: 'flex-column', style: 'max-height: 200px; overflow-y: auto;' }, + tableGroupPreview?.tables?.length + ? tableGroupPreview.tables.map((table) => + div({ class: 'table-row' }, table), + ) + : div( + { class: 'flex-row fx-justify-center', style: 'height: 50px; font-size: 16px;'}, + tableGroupPreview.message ?? 'No tables found.' + ), + ), + ); + }, + ); +}; + +export { TableGroupTest }; diff --git a/testgen/ui/components/frontend/js/components/textarea.js b/testgen/ui/components/frontend/js/components/textarea.js new file mode 100644 index 00000000..18165312 --- /dev/null +++ b/testgen/ui/components/frontend/js/components/textarea.js @@ -0,0 +1,99 @@ +/** + * @typedef Properties + * @type {object} + * @property {string?} id + * @property {string?} name + * @property {string?} label + * @property {string?} help + * @property {TooltipProperties['position']} helpPlacement + * @property {(string | number)?} value + * @property {string?} placeholder + * @property {string?} icon + * @property {boolean?} disabled + * @property {function(string, InputState)?} onChange + * @property {string?} style + * @property {string?} class + * @property {string?} testId + */ +import van from '../van.min.js'; +import { debounce, getValue, loadStylesheet, getRandomId } from '../utils.js'; +import { Icon } from './icon.js'; +import { withTooltip } from './tooltip.js'; + +const { div, label, textarea } = van.tags; +const defaultHeight = 64; + +const Textarea = (/** @type Properties */ props) => { + loadStylesheet('textarea', stylesheet); + + const domId = van.derive(() => getValue(props.id) ?? getRandomId()); + const value = van.derive(() => getValue(props.value) ?? ''); + + const onChange = props.onChange?.val ?? props.onChange; + if (onChange) { + onChange(value.val); + } + van.derive(() => { + const onChange = props.onChange?.val ?? props.onChange; + if (onChange && value.val !== value.oldVal) { + onChange(value.val); + } + }); + + return label( + { + id: domId, + class: () => `flex-column fx-gap-1 ${getValue(props.class) ?? ''}`, + style: () => `width: ${props.width ? getValue(props.width) + 'px' : 'auto'}; ${getValue(props.style)}`, + 'data-testid': props.testId ?? props.name ?? '', + }, + div( + { class: 'flex-row fx-gap-1 text-caption' }, + props.label, + () => getValue(props.help) + ? withTooltip( + Icon({ size: 16, classes: 'text-disabled' }, 'help'), + { text: props.help, position: getValue(props.helpPlacement) ?? 'top', width: 200 } + ) + : null, + ), + textarea({ + class: () => `tg-textarea--field ${getValue(props.disabled) ? 'tg-textarea--disabled' : ''}`, + style: () => `min-height: ${getValue(props.height) || defaultHeight}px;`, + value, + name: props.name ?? '', + disabled: props.disabled, + placeholder: () => getValue(props.placeholder) ?? '', + oninput: debounce((/** @type Event */ event) => value.val = event.target.value, 300), + }), + ); +}; + +const stylesheet = new CSSStyleSheet(); +stylesheet.replace(` +.tg-textarea--field { + box-sizing: border-box; + width: 100%; + border-radius: 8px; + border: 1px solid transparent; + transition: border-color 0.3s; + background-color: var(--form-field-color); + padding: 4px 8px; + color: var(--primary-text-color); + font-size: 14px; + resize: vertical; +} + +.tg-textarea--field::placeholder { + font-style: italic; + color: var(--disabled-text-color); +} + +.tg-textarea--field:focus, +.tg-textarea--field:focus-visible { + outline: none; + border-color: var(--primary-color); +} +`); + +export { Textarea }; diff --git a/testgen/ui/components/frontend/js/data_profiling/table_create_script.js b/testgen/ui/components/frontend/js/data_profiling/table_create_script.js new file mode 100644 index 00000000..910d38f1 --- /dev/null +++ b/testgen/ui/components/frontend/js/data_profiling/table_create_script.js @@ -0,0 +1,29 @@ +/** + * @import { Table } from './data_profiling_utils.js'; + * + * @typedef Properties + * @type {object} + */ +import van from '../van.min.js'; +import { Card } from '../components/card.js'; +import { Button } from '../components/button.js'; +import { emitEvent } from '../utils.js'; + +const { div } = van.tags; + +const TableCreateScriptCard = (/** @type Properties */ _props, /** @type Table */ item) => { + return Card({ + title: 'Table CREATE Script with Suggested Data Types', + content: div( + Button({ + type: 'stroked', + label: 'View Script', + icon: 'sdk', + width: 'auto', + onclick: () => emitEvent('CreateScriptClicked', { payload: item }), + }), + ), + }); +}; + +export { TableCreateScriptCard }; diff --git a/testgen/ui/components/frontend/js/main.js b/testgen/ui/components/frontend/js/main.js index 2265c595..3fb3bca0 100644 --- a/testgen/ui/components/frontend/js/main.js +++ b/testgen/ui/components/frontend/js/main.js @@ -30,7 +30,6 @@ import { ScheduleList } from './pages/schedule_list.js'; import { Connections } from './pages/connections.js'; import { TableGroupWizard } from './pages/table_group_wizard.js'; import { HelpMenu } from './components/help_menu.js' -import { TableGroup } from './pages/table_group.js'; import { TableGroupList } from './pages/table_group_list.js'; import { TableGroupDeleteConfirmation } from './pages/table_group_delete_confirmation.js'; import { RunProfilingDialog } from './pages/run_profiling_dialog.js'; @@ -64,7 +63,6 @@ const TestGenComponent = (/** @type {string} */ id, /** @type {object} */ props) connections: Connections, table_group_wizard: TableGroupWizard, help_menu: HelpMenu, - table_group: TableGroup, table_group_list: TableGroupList, table_group_delete: TableGroupDeleteConfirmation, run_profiling_dialog: RunProfilingDialog, diff --git a/testgen/ui/components/frontend/js/pages/connections.js b/testgen/ui/components/frontend/js/pages/connections.js index 1bfec4b8..8da64475 100644 --- a/testgen/ui/components/frontend/js/pages/connections.js +++ b/testgen/ui/components/frontend/js/pages/connections.js @@ -73,7 +73,7 @@ const Connections = (props) => { }), ), div( - { class: 'flex-column fx-gap-4 tg-connections--border p-4' }, + { class: 'flex-column fx-gap-4 p-4' }, ConnectionForm( { connection: props.connection, @@ -114,11 +114,6 @@ const Connections = (props) => { const stylesheet = new CSSStyleSheet(); stylesheet.replace(` -.tg-connections--border { - border: var(--button-stroked-border); - border-radius: 8px; -} - .tg-connections--link { margin-left: auto; border-radius: 4px; diff --git a/testgen/ui/components/frontend/js/pages/data_catalog.js b/testgen/ui/components/frontend/js/pages/data_catalog.js index 0a9bedca..246d7a53 100644 --- a/testgen/ui/components/frontend/js/pages/data_catalog.js +++ b/testgen/ui/components/frontend/js/pages/data_catalog.js @@ -61,8 +61,9 @@ import { Button } from '../components/button.js'; import { Link } from '../components/link.js'; import { EMPTY_STATE_MESSAGE, EmptyState } from '../components/empty_state.js'; import { Portal } from '../components/portal.js'; +import { TableCreateScriptCard } from '../data_profiling/table_create_script.js'; -const { div, h2, span, i } = van.tags; +const { div, h2, span } = van.tags; // https://www.sam.today/blog/html5-dnd-globe-icon const EMPTY_IMAGE = new Image(1, 1); @@ -421,6 +422,9 @@ const SelectedDetails = (/** @type Properties */ props, /** @type Table | Column HygieneIssuesCard({ noLinks: !userCanNavigate }, item), TestIssuesCard({ noLinks: !userCanNavigate }, item), TestSuitesCard(item), + item.type === 'table' + ? TableCreateScriptCard({}, item) + : null, ) : ItemEmptyState( 'Select a table or column on the left to view its details.', diff --git a/testgen/ui/components/frontend/js/pages/profiling_runs.js b/testgen/ui/components/frontend/js/pages/profiling_runs.js index 1290e270..2b2c3392 100644 --- a/testgen/ui/components/frontend/js/pages/profiling_runs.js +++ b/testgen/ui/components/frontend/js/pages/profiling_runs.js @@ -238,7 +238,9 @@ const ProfilingRunItem = ( ), div( { style: `flex: ${columns[5]}; font-size: 16px;` }, - item.dq_score_profiling ?? '--', + item.column_ct && item.dq_score_profiling + ? item.dq_score_profiling + : '--', ), ); } diff --git a/testgen/ui/components/frontend/js/pages/project_dashboard.js b/testgen/ui/components/frontend/js/pages/project_dashboard.js index 92c22c0f..1ae77b42 100644 --- a/testgen/ui/components/frontend/js/pages/project_dashboard.js +++ b/testgen/ui/components/frontend/js/pages/project_dashboard.js @@ -73,10 +73,6 @@ const ProjectDashboard = (/** @type Properties */ props) => { Streamlit.setFrameHeight(1); window.testgen.isPage = true; - const isEmpty = van.derive(() => { - const projectSummary = getValue(props.project) - return projectSummary.test_runs_count <= 0 && projectSummary.profiling_runs_count <= 0; - }); const tableGroups = van.derive(() => getValue(props.table_groups)); const tableGroupsSearchTerm = van.state(''); const tableGroupsSortOption = van.state(getValue(props.table_groups_sort_options).find(o => o.selected)?.value); @@ -109,7 +105,7 @@ const ProjectDashboard = (/** @type Properties */ props) => { return div( { id: wrapperId, class: 'flex-column tg-overview' }, - () => !getValue(isEmpty) + () => getValue(tableGroups).length ? div( { class: 'flex-row fx-align-flex-end fx-gap-4' }, Input({ @@ -132,7 +128,7 @@ const ProjectDashboard = (/** @type Properties */ props) => { }), ) : '', - () => !getValue(isEmpty) + () => getValue(tableGroups).length ? div( { class: 'flex-column mt-4' }, getValue(filteredTableGroups).map(tableGroup => TableGroupCard(tableGroup)), diff --git a/testgen/ui/components/frontend/js/pages/table_group.js b/testgen/ui/components/frontend/js/pages/table_group.js deleted file mode 100644 index ce2cba33..00000000 --- a/testgen/ui/components/frontend/js/pages/table_group.js +++ /dev/null @@ -1,174 +0,0 @@ -/** - * @import { TableGroup } from '../components/table_group_form.js'; - * @import { Connection } from '../components/connection_form.js'; - * - * @typedef TableGroupPreview - * @type {object} - * @property {string} schema - * @property {string[]?} tables - * @property {number?} column_count - * @property {boolean?} success - * @property {string?} message - * - * @typedef Result - * @type {object} - * @property {boolean} success - * @property {string} message - * - * @typedef Properties - * @type {object} - * @property {string} project_code - * @property {TableGroup} table_group - * @property {Connection[]} connections - * @property {boolean?} in_used - * @property {TableGroupPreview?} table_group_preview - * @property {Result?} result - */ -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { Button } from '../components/button.js'; -import { getValue, emitEvent, loadStylesheet, resizeFrameHeightToElement, resizeFrameHeightOnDOMChange } from '../utils.js'; -import { TableGroupForm } from '../components/table_group_form.js'; -import { Tab, Tabs } from '../components/tabs.js'; -import { Alert } from '../components/alert.js'; - -const { div, span, strong } = van.tags; - -/** - * @param {Properties} props - * @returns {HTMLElement} - */ -const TableGroup = (props) => { - loadStylesheet('tablegroupchange', stylesheet); - Streamlit.setFrameHeight(1); - window.testgen.isPage = true; - - const connections = getValue(props.connections) ?? []; - const enableConnectionSelector = getValue(props.table_group)?.connection_id === undefined; - const updatedTableGroup = van.state(getValue(props.table_group) ?? {}); - const disableSchemaField = getValue(props.in_used ?? false); - const disableSave = van.state(true); - const wrapperId = 'tablegroup-change-wrapper'; - - resizeFrameHeightToElement(wrapperId); - resizeFrameHeightOnDOMChange(wrapperId); - - return Tabs( - { id: wrapperId }, - Tab( - { label: 'Table Group Settings'}, - () => { - const tableGroup = updatedTableGroup.rawVal; - const result = getValue(props.result); - - return div( - { class: 'flex-column fx-gap-3' }, - TableGroupForm({ - tableGroup, - connections, - enableConnectionSelector, - disableSchemaField, - showConnectionSelector: connections.length > 1, - onChange: (newTableGroup, state) => { - updatedTableGroup.val = newTableGroup; - disableSave.val = !state.valid; - }, - }), - result - ? Alert( - { type: result.success ? 'success' : 'error', closeable: true }, - span({}, result.message), - ) - : undefined, - ); - }, - div( - { class: 'flex-row fx-gap-2 fx-justify-content-flex-end mt-3' }, - Button({ - label: 'Save', - type: 'stroked', - color: 'primary', - style: 'width: auto;', - disabled: disableSave, - onclick: () => emitEvent('TableGroupSaveClicked', { payload: updatedTableGroup.val }), - }), - ), - ), - Tab( - { label: 'Test' }, - () => { - const currentSchema = updatedTableGroup.val.table_group_schema ?? tableGroupPreview?.schema ?? '--'; - const tableGroupPreview = getValue(props.table_group_preview); - const wasPreviewExecuted = tableGroupPreview && typeof tableGroupPreview.success === 'boolean'; - const alertMessage = tableGroupPreview.success ? 'Operation has finished successfully.' : 'Operation was unsuccessful.'; - - return div( - { class: 'flex-column fx-gap-2' }, - div( - { class: 'flex-row fx-justify-space-between' }, - div( - { class: 'flex-column fx-gap-2' }, - div( - { class: 'flex-row fx-gap-1' }, - strong({}, 'Schema:'), - span({}, currentSchema), - ), - div( - { class: 'flex-row fx-gap-1' }, - strong({}, 'Table Count:'), - span({}, tableGroupPreview?.tables?.length ?? '--'), - ), - div( - { class: 'flex-row fx-gap-1' }, - strong({}, 'Column Count:'), - span({}, tableGroupPreview?.column_count ?? '--'), - ), - ), - wasPreviewExecuted - ? Alert( - { type: tableGroupPreview.success ? 'success' : 'error' }, - span({}, alertMessage), - ) - : undefined, - ), - wasPreviewExecuted ? - div( - { class: 'table hoverable p-3' }, - div( - { class: 'table-header' }, - span('Tables'), - ), - div( - { class: 'flex-column', style: 'max-height: 200px; overflow-y: auto;' }, - tableGroupPreview?.tables?.length - ? tableGroupPreview.tables.map((table) => - div({ class: 'table-row' }, table), - ) - : div( - { class: 'flex-row fx-justify-center', style: 'height: 50px; font-size: 16px;'}, - tableGroupPreview.message ?? 'No tables found.' - ), - ), - ) - : undefined, - ); - }, - div( - {class: 'flex-row fx-gap-2 fx-justify-content-flex-end mt-3'}, - Button({ - label: 'Test Table Group', - type: 'stroked', - color: 'primary', - style: 'width: auto;', - onclick: () => emitEvent('PreviewTableGroupClicked', { payload: updatedTableGroup.val }), - }), - ), - ), - ); -} - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -`); - -export { TableGroup }; diff --git a/testgen/ui/components/frontend/js/pages/table_group_delete_confirmation.js b/testgen/ui/components/frontend/js/pages/table_group_delete_confirmation.js index 2037abd0..96b74346 100644 --- a/testgen/ui/components/frontend/js/pages/table_group_delete_confirmation.js +++ b/testgen/ui/components/frontend/js/pages/table_group_delete_confirmation.js @@ -95,6 +95,15 @@ const TableGroupDeleteConfirmation = (props) => { onclick: () => emitEvent('DeleteTableGroupConfirmed'), }), ), + () => { + const result = getValue(props.result); + return result + ? Alert( + { type: result.success ? 'success' : 'error', class: 'mt-3' }, + div(result.message), + ) + : ''; + }, ); }; diff --git a/testgen/ui/components/frontend/js/pages/table_group_list.js b/testgen/ui/components/frontend/js/pages/table_group_list.js index 333b133a..1037c39c 100644 --- a/testgen/ui/components/frontend/js/pages/table_group_list.js +++ b/testgen/ui/components/frontend/js/pages/table_group_list.js @@ -64,10 +64,10 @@ const TableGroupList = (props) => { }); } - return div( + return tableGroups.length > 0 + ? div( Toolbar(permissions, connections, connectionId), - tableGroups.length > 0 - ? tableGroups.map((tableGroup) => Card({ + tableGroups.map((tableGroup) => Card({ testId: 'table-group-card', class: '', title: div( @@ -179,22 +179,22 @@ const TableGroupList = (props) => { ) : undefined, })) - : EmptyState({ - icon: 'table_view', - label: 'No table groups yet', - class: 'mt-4', - message: EMPTY_STATE_MESSAGE.tableGroup, - button: Button({ - type: 'stroked', - icon: 'add', - label: 'Add Table Group', - color: 'primary', - style: 'width: unset;', - disabled: !permissions.can_edit, - onclick: () => emitEvent('AddTableGroupClicked', {}), - }), - }), - ); + ) + : EmptyState({ + icon: 'table_view', + label: 'No table groups yet', + class: 'mt-4', + message: EMPTY_STATE_MESSAGE.tableGroup, + button: Button({ + type: 'stroked', + icon: 'add', + label: 'Add Table Group', + color: 'primary', + style: 'width: unset;', + disabled: !permissions.can_edit, + onclick: () => emitEvent('AddTableGroupClicked', {}), + }), + }); }, ); } diff --git a/testgen/ui/components/frontend/js/pages/table_group_wizard.js b/testgen/ui/components/frontend/js/pages/table_group_wizard.js index c2fcd7ea..a88829d3 100644 --- a/testgen/ui/components/frontend/js/pages/table_group_wizard.js +++ b/testgen/ui/components/frontend/js/pages/table_group_wizard.js @@ -1,26 +1,48 @@ /** + * @import { TableGroupPreview } from '../components/table_group_test.js' + * @import { Connection } from ''../components/connection_form.js' + * @import { TableGroup } from ''../components/table_group_form.js' + * * @typedef WizardResult * @type {object} * @property {boolean} success * @property {string} message - * @property {string} table_group_id + * @property {string?} table_group_id * * @typedef Properties * @type {object} * @property {string} project_code - * @property {string} connection_id + * @property {TableGroup} table_group + * @property {Connection[]} connections + * @property {string[]?} steps + * @property {boolean?} is_in_use + * @property {TableGroupPreview?} table_group_preview * @property {WizardResult?} results */ import van from '../van.min.js'; import { Streamlit } from '../streamlit.js'; import { TableGroupForm } from '../components/table_group_form.js'; +import { TableGroupTest } from '../components/table_group_test.js'; import { emitEvent, getValue, resizeFrameHeightOnDOMChange, resizeFrameHeightToElement } from '../utils.js'; import { Button } from '../components/button.js'; import { Alert } from '../components/alert.js'; import { Checkbox } from '../components/checkbox.js'; import { Icon } from '../components/icon.js'; +import { Caption } from '../components/caption.js'; const { div, i, span, strong } = van.tags; +const stepsTitle = { + tableGroup: 'Configure Table Group', + testTableGroup: 'Preview Table Group', + runProfiling: 'Run Profiling', +}; +const lastStepCustonButtonText = { + runProfiling: (state) => state ? 'Save & Run Profiling' : 'Save', +}; +const defaultSteps = [ + 'tableGroup', + 'testTableGroup', +]; /** * @param {Properties} props @@ -29,16 +51,15 @@ const TableGroupWizard = (props) => { Streamlit.setFrameHeight(1); window.testgen.isPage = true; - const steps = [ - 'tableGroup', - 'runProfiling', - ]; + const steps = props.steps?.val ?? defaultSteps; const stepsState = { - tableGroup: van.state({}), + tableGroup: van.state(props.table_group.val), + testTableGroup: van.state(false), runProfiling: van.state(true), }; const stepsValidity = { tableGroup: van.state(false), + testTableGroup: van.state(false), runProfiling: van.state(true), }; const currentStepIndex = van.state(0); @@ -53,18 +74,30 @@ const TableGroupWizard = (props) => { const nextButtonLabel = van.derive(() => { const isLastStep = currentStepIndex.val === steps.length - 1; if (isLastStep) { - return stepsState.runProfiling.val ? 'Save & Run Profiling' : 'Finish Setup'; + const stepKey = steps[currentStepIndex.val]; + const stepState = stepsState[stepKey]; + return lastStepCustonButtonText[stepKey]?.(stepState.val) ?? 'Save'; } return 'Next'; }); + + van.derive(() => { + const tableGroupPreview = getValue(props.table_group_preview); + stepsValidity.testTableGroup.val = tableGroupPreview?.success ?? false; + stepsState.testTableGroup.val = tableGroupPreview?.success ?? false; + }); + const setStep = (stepIdx) => { currentStepIndex.val = stepIdx; }; const saveTableGroup = () => { - const payload = { - table_group: stepsState.tableGroup.val, - run_profiling: stepsState.runProfiling.val, - }; + const payloadEntries = [ + ['tableGroup', 'table_group', stepsState.tableGroup.val], + ['testTableGroup', 'table_group_verified', stepsState.testTableGroup.val], + ['runProfiling', 'run_profiling', stepsState.runProfiling.val], + ].filter(([stepKey,]) => steps.includes(stepKey)).map(([, eventKey, stepState]) => [eventKey, stepState]); + + const payload = Object.fromEntries(payloadEntries); emitEvent('SaveTableGroupClicked', { payload }); }; @@ -74,78 +107,128 @@ const TableGroupWizard = (props) => { return div( { id: domId, class: 'tg-table-group-wizard flex-column fx-gap-3' }, + div( + {}, + () => { + const stepName = steps[currentStepIndex.val]; + const stepNumber = currentStepIndex.val + 1; + return Caption({ + content: `Step ${stepNumber} of ${steps.length}: ${stepsTitle[stepName]}`, + }); + }, + ), WizardStep(0, currentStepIndex, () => { currentStepIndex.val; + const connections = getValue(props.connections) ?? []; + const tableGroup = stepsState.tableGroup.rawVal; + return TableGroupForm({ - tableGroup: stepsState.tableGroup.rawVal, + connections, + tableGroup: tableGroup, + showConnectionSelector: connections.length > 1, + disableConnectionSelector: false, + disableSchemaField: props.is_in_use ?? false, onChange: (updatedTableGroup, state) => { stepsState.tableGroup.val = updatedTableGroup; stepsValidity.tableGroup.val = state.valid; }, }); }), + WizardStep(1, currentStepIndex, () => { + const tableGroup = stepsState.tableGroup.rawVal; + + if (currentStepIndex.val === 1) { + props.table_group_preview.val = undefined; + stepsValidity.testTableGroup.val = false; + stepsState.testTableGroup.val = false; + + emitEvent('PreviewTableGroupClicked', { payload: tableGroup }); + } + + return TableGroupTest( + tableGroup.table_group_schema ?? '--', + props.table_group_preview, + ); + }), () => { - const results = getValue(props.results); const runProfiling = van.state(stepsState.runProfiling.rawVal); + const results = getValue(props.results) ?? {}; van.derive(() => { stepsState.runProfiling.val = runProfiling.val; }); - return WizardStep(1, currentStepIndex, () => { + return WizardStep(2, currentStepIndex, () => { currentStepIndex.val; return RunProfilingStep( stepsState.tableGroup.rawVal, runProfiling, - results, + results?.success ?? false, ); }); }, div( - { class: 'tg-table-group-wizard--footer flex-row' }, - () => currentStepIndex.val > 0 - ? Button({ - label: 'Previous', - type: 'stroked', - color: 'basic', - width: 'auto', - style: 'margin-right: auto; min-width: 200px;', - onclick: () => setStep(currentStepIndex.val - 1), - }) - : '', + { class: 'flex-column fx-gap-3' }, () => { - const results = getValue(props.results); - const runProfiling = stepsState.runProfiling.val; - - if (results && results.success && runProfiling) { + const results = getValue(props.results) ?? {}; + return Object.keys(results).length > 0 + ? Alert({ type: results.success ? 'success' : 'error' }, span(results.message)) + : ''; + }, + div( + { class: 'flex-row' }, + () => { + const results = getValue(props.results); + + if (currentStepIndex.val <= 0 || results?.success === true) { + return ''; + } + return Button({ + label: 'Previous', type: 'stroked', + color: 'basic', + width: 'auto', + style: 'margin-right: auto; min-width: 200px;', + onclick: () => setStep(currentStepIndex.val - 1), + }); + }, + () => { + const results = getValue(props.results); + const runProfiling = stepsState.runProfiling.val; + const stepKey = steps[currentStepIndex.val]; + + if (results && results.success && stepKey === 'runProfiling' && runProfiling) { + return Button({ + type: 'stroked', + color: 'primary', + label: 'Go to Profiling Runs', + width: 'auto', + icon: 'chevron_right', + style: 'margin-left: auto;', + onclick: () => emitEvent('GoToProfilingRunsClicked', { payload: { table_group_id: results.table_group_id } }), + }); + } + + return Button({ + label: nextButtonLabel, + type: nextButtonType, color: 'primary', - label: 'Go to Profiling Runs', width: 'auto', - icon: 'chevron_right', - onclick: () => emitEvent('GoToProfilingRunsClicked', { payload: { table_group_id: results.table_group_id } }), + style: 'margin-left: auto; min-width: 200px;', + disabled: currentStepIsInvalid, + onclick: () => { + if (currentStepIndex.val < steps.length - 1) { + return setStep(currentStepIndex.val + 1); + } + + saveTableGroup(); + }, }); - } - - return Button({ - label: nextButtonLabel, - type: nextButtonType, - color: 'primary', - width: 'auto', - style: 'margin-left: auto; min-width: 200px;', - disabled: currentStepIsInvalid, - onclick: () => { - if (currentStepIndex.val < steps.length - 1) { - return setStep(currentStepIndex.val + 1); - } - - saveTableGroup(); - }, - }); - }, + }, + ), ), ); }; @@ -153,10 +236,10 @@ const TableGroupWizard = (props) => { /** * @param {object} tableGroup * @param {boolean} runProfiling - * @param {WizardResult} result + * @param {boolean?} disabled * @returns */ -const RunProfilingStep = (tableGroup, runProfiling, results) => { +const RunProfilingStep = (tableGroup, runProfiling, disabled) => { return div( { class: 'flex-column fx-gap-3' }, Checkbox({ @@ -167,6 +250,7 @@ const RunProfilingStep = (tableGroup, runProfiling, results) => { span('?'), ), checked: runProfiling, + disabled: disabled ?? false, onChange: (value) => runProfiling.val = value, }), div( @@ -176,12 +260,6 @@ const RunProfilingStep = (tableGroup, runProfiling, results) => { ? i('Profiling will be performed in a background process.') : i('Profiling will be skipped. You can run this step later from the Profiling Runs page.'), ), - () => { - const results_ = getValue(results) ?? {}; - return Object.keys(results_).length > 0 - ? Alert({ type: results_.success ? 'success' : 'error' }, span(results_.message)) - : ''; - }, ); }; @@ -194,7 +272,7 @@ const WizardStep = (index, currentIndex, content) => { const hidden = van.derive(() => getValue(currentIndex) !== getValue(index)); return div( - { class: () => hidden.val ? 'hidden' : ''}, + { class: () => `flex-column fx-gap-3 ${hidden.val ? 'hidden' : ''}`}, content, ); }; diff --git a/testgen/ui/components/frontend/js/pages/test_definition_summary.js b/testgen/ui/components/frontend/js/pages/test_definition_summary.js index 42984d50..ee315409 100644 --- a/testgen/ui/components/frontend/js/pages/test_definition_summary.js +++ b/testgen/ui/components/frontend/js/pages/test_definition_summary.js @@ -11,7 +11,6 @@ * @property {string} test_suite_name * @property {string} table_name * @property {string} test_focus - * @property {string?} status * @property {string} severity * @property {string} active * @property {string} locked @@ -50,7 +49,6 @@ const TestDefinitionSummary = (props) => { {id: wrapperId}, () => { const testDefinition = getValue(props.test_definition); - console.log(testDefinition); return div( { class: 'flex-column' }, diff --git a/testgen/ui/components/frontend/js/utils.js b/testgen/ui/components/frontend/js/utils.js index f912916d..1e8b4f7f 100644 --- a/testgen/ui/components/frontend/js/utils.js +++ b/testgen/ui/components/frontend/js/utils.js @@ -116,7 +116,7 @@ function friendlyPercent(/** @type number */ value) { } const rounded = Math.round(value); if (rounded === 0 && value > 0) { - return '< 0'; + return '< 1'; } if (rounded === 100 && value < 100) { return '> 99'; diff --git a/testgen/ui/components/frontend/js/van.min.js b/testgen/ui/components/frontend/js/van.min.js index 7e23e03e..57c6b792 100644 --- a/testgen/ui/components/frontend/js/van.min.js +++ b/testgen/ui/components/frontend/js/van.min.js @@ -1,2 +1,10 @@ +/** + * @template T + * @typedef VanState + * @type {object} + * @property {T?} rawVal + * @property {T?} oldVal + * @property {T?} val + */ // https://vanjs.org/code/van-1.5.2.min.js let e,t,r,o,l,n,s=Object.getPrototypeOf,f={isConnected:1},i={},h=s(f),a=s(s),d=(e,t,r,o)=>(e??(setTimeout(r,o),new Set)).add(t),u=(e,t,o)=>{let l=r;r=t;try{return e(o)}catch(e){return console.error(e),o}finally{r=l}},w=e=>e.filter(e=>e.t?.isConnected),_=e=>l=d(l,e,()=>{for(let e of l)e.o=w(e.o),e.l=w(e.l);l=n},1e3),c={get val(){return r?.i?.add(this),this.rawVal},get oldVal(){return r?.i?.add(this),this.h},set val(o){r?.u?.add(this),o!==this.rawVal&&(this.rawVal=o,this.o.length+this.l.length?(t?.add(this),e=d(e,this,v)):this.h=o)}},S=e=>({__proto__:c,rawVal:e,h:e,o:[],l:[]}),g=(e,t)=>{let r={i:new Set,u:new Set},l={f:e},n=o;o=[];let s=u(e,r,t);s=(s??document).nodeType?s:new Text(s);for(let e of r.i)r.u.has(e)||(_(e),e.o.push(l));for(let e of o)e.t=s;return o=n,l.t=s},y=(e,t=S(),r)=>{let l={i:new Set,u:new Set},n={f:e,s:t};n.t=r??o?.push(n)??f,t.val=u(e,l,t.rawVal);for(let e of l.i)l.u.has(e)||(_(e),e.l.push(n));return t},b=(e,...t)=>{for(let r of t.flat(1/0)){let t=s(r??0),o=t===c?g(()=>r.val):t===a?g(r):r;o!=n&&e.append(o)}return e},m=(e,t,...r)=>{let[o,...l]=s(r[0]??0)===h?r:[{},...r],f=e?document.createElementNS(e,t):document.createElement(t);for(let[e,r]of Object.entries(o)){let o=t=>t?Object.getOwnPropertyDescriptor(t,e)??o(s(t)):n,l=t+","+e,h=i[l]??=o(s(f))?.set??0,d=e.startsWith("on")?(t,r)=>{let o=e.slice(2);f.removeEventListener(o,r),f.addEventListener(o,t)}:h?h.bind(f):f.setAttribute.bind(f,e),u=s(r??0);e.startsWith("on")||u===a&&(r=y(r),u=c),u===c?g(()=>(d(r.val,r.h),f)):d(r)}return b(f,l)},x=e=>({get:(t,r)=>m.bind(n,e,r)}),j=(e,t)=>t?t!==e&&e.replaceWith(t):e.remove(),v=()=>{let r=0,o=[...e].filter(e=>e.rawVal!==e.h);do{t=new Set;for(let e of new Set(o.flatMap(e=>e.l=w(e.l))))y(e.f,e.s,e.t),e.t=n}while(++r<100&&(o=[...t]).length);let l=[...e].filter(e=>e.rawVal!==e.h);e=n;for(let e of new Set(l.flatMap(e=>e.o=w(e.o))))j(e.t,g(e.f,e.t)),e.t=n;for(let e of l)e.h=e.rawVal};export default{tags:new Proxy(e=>new Proxy(m,x(e)),x()),hydrate:(e,t)=>j(e,g(t,e)),add:b,state:S,derive:y}; \ No newline at end of file diff --git a/testgen/ui/components/widgets/select.py b/testgen/ui/components/widgets/select.py index 23d65d96..56ce355e 100644 --- a/testgen/ui/components/widgets/select.py +++ b/testgen/ui/components/widgets/select.py @@ -32,11 +32,15 @@ def select( option_display_labels = options if isinstance(options, pd.DataFrame): - value_column = value_column or options.columns[0] - display_column = display_column or value_column - - option_values = options[value_column].values.tolist() - option_display_labels = options[display_column].values.tolist() + if options.empty: + option_values = [] + option_display_labels = [] + else: + value_column = value_column or options.columns[0] + display_column = display_column or value_column + + option_values = options[value_column].values.tolist() + option_display_labels = options[display_column].values.tolist() kwargs["options"] = [*option_display_labels] if default_value in option_values: diff --git a/testgen/ui/components/widgets/testgen_component.py b/testgen/ui/components/widgets/testgen_component.py index d52b2bdc..7ab385ec 100644 --- a/testgen/ui/components/widgets/testgen_component.py +++ b/testgen/ui/components/widgets/testgen_component.py @@ -52,9 +52,9 @@ def on_change(): Router().queue_navigation(to=event_data["href"], with_args=event_data.get("params")) elif on_change_handlers and (handler := on_change_handlers.get(event)): # Prevent handling the same event multiple times - event_id = f"{component_id}:{event_data.get('_id', '')}" - if event_id != session.testgen_event_id: - session.testgen_event_id = event_id + event_id = event_data.get("_id", "") + if event_id != session.testgen_event_id.get(component_id): + session.testgen_event_id[component_id] = event_id handler(event_data.get("payload")) event_data = component( @@ -65,9 +65,9 @@ def on_change(): ) if event_handlers and event_data and (event := event_data.get("event")) and (handler := event_handlers.get(event)): # Prevent handling the same event multiple times - event_id = f"{component_id}:{event_data.get('_id', '')}" - if event_id != session.testgen_event_id: - session.testgen_event_id = event_id + event_id = event_data.get("_id", "") + if event_id != session.testgen_event_id.get(component_id): + session.testgen_event_id[component_id] = event_id # These events are not handled through the component's on_change callback # because they may call st.rerun(), causing the "Calling st.rerun() within a callback is a noop" error handler(event_data.get("payload")) diff --git a/testgen/ui/pdf/dataframe_table.py b/testgen/ui/pdf/dataframe_table.py index ff2f8c26..2516444e 100644 --- a/testgen/ui/pdf/dataframe_table.py +++ b/testgen/ui/pdf/dataframe_table.py @@ -79,6 +79,7 @@ class VerticalHeaderCell(Flowable): """ INITIAL_HEIGHT = 40 + MAX_HEIGHT = 100 HEIGHT_INCR_STEP = 5 def __init__(self, flowable): @@ -94,7 +95,7 @@ def wrap(self, availWidth, _): while True: flowable_height, self.flowable_width = self.flowable.wrap(available_height, self.available_width) - if self.flowable_width > self.available_width: + if self.flowable_width > self.available_width and available_height <= self.MAX_HEIGHT: available_height += self.HEIGHT_INCR_STEP else: break diff --git a/testgen/ui/queries/table_group_queries.py b/testgen/ui/queries/table_group_queries.py index d69e54cb..040a1c6d 100644 --- a/testgen/ui/queries/table_group_queries.py +++ b/testgen/ui/queries/table_group_queries.py @@ -22,7 +22,7 @@ def _get_select_statement(schema): business_domain, stakeholder_group, transform_level, data_product, CASE WHEN profile_use_sampling = 'Y' THEN true ELSE false END AS profile_use_sampling, profile_sample_percent, profile_sample_min_count, - profiling_delay_days, profile_flag_cdes + profiling_delay_days, profile_flag_cdes, include_in_dashboard FROM table_groups """ @@ -122,6 +122,7 @@ def edit(schema, table_group): profile_sample_min_count={int(table_group["profile_sample_min_count"])}, profiling_delay_days='{table_group["profiling_delay_days"]}', profile_flag_cdes={table_group["profile_flag_cdes"]}, + include_in_dashboard={table_group["include_in_dashboard"]}, description='{table_group["description"]}', data_source=NULLIF('{table_group["data_source"]}', ''), source_system=NULLIF('{table_group["source_system"]}', ''), @@ -157,6 +158,7 @@ def add(schema, table_group) -> str: profile_sample_min_count, profiling_delay_days, profile_flag_cdes, + include_in_dashboard, description, data_source, source_system, @@ -182,6 +184,7 @@ def add(schema, table_group) -> str: {table_group["profile_sample_min_count"]}, '{table_group["profiling_delay_days"]}'::character varying, {table_group["profile_flag_cdes"]}, + {table_group["include_in_dashboard"]}, '{table_group["description"]}', NULLIF('{table_group["data_source"]}', ''), NULLIF('{table_group["source_system"]}', ''), diff --git a/testgen/ui/queries/test_definition_queries.py b/testgen/ui/queries/test_definition_queries.py index 161da4de..2294a688 100644 --- a/testgen/ui/queries/test_definition_queries.py +++ b/testgen/ui/queries/test_definition_queries.py @@ -22,7 +22,7 @@ def update_attribute(schema, test_definition_ids, attribute, value): @st.cache_data(show_spinner=False) @with_database_session -def get_test_definitions(_, project_code, test_suite, table_name, column_name, test_definition_ids: list[str] | None): +def get_test_definitions(_, project_code, test_suite, table_name, column_name, test_type, test_definition_ids: list[str] | None): db_session = get_current_session() params = {} order_by = "ORDER BY d.schema_name, d.table_name, d.column_name, d.test_type" @@ -48,6 +48,10 @@ def get_test_definitions(_, project_code, test_suite, table_name, column_name, t if column_name: filters += " AND d.column_name ILIKE :column_name" params["column_name"] = column_name + + if test_type: + filters += " AND d.test_type = :test_type" + params["test_type"] = test_type sql = f""" SELECT @@ -311,7 +315,7 @@ def copy(schema, test_definitions, target_table_group, target_test_suite, target update_target_column = f"'{target_table_column['column_name']}' as column_name" update_target_table = f"'{target_table_column['table_name']}' as table_name" else: - update_target_column = "td.colum_name" + update_target_column = "td.column_name" update_target_table = "td.table_name" test_definition_ids = [f"'{td['id']}'" for td in test_definitions] sql = f""" diff --git a/testgen/ui/services/javascript_service.py b/testgen/ui/services/javascript_service.py index 93eae90c..73c87bfc 100644 --- a/testgen/ui/services/javascript_service.py +++ b/testgen/ui/services/javascript_service.py @@ -7,15 +7,6 @@ LOG = logging.getLogger("testgen") -def copy_to_clipboard(text): - script = f"""await (async function () {{ - window.parent.postMessage({{ type: 'TestgenCopyToClipboard', text: '{text}' }}, '*'); - return 0; - }})() - """ - execute_javascript(script) - - def clear_component_states(): execute_javascript( f"""await (async function () {{ diff --git a/testgen/ui/services/query_service.py b/testgen/ui/services/query_service.py index 1395001a..12c3d7dc 100644 --- a/testgen/ui/services/query_service.py +++ b/testgen/ui/services/query_service.py @@ -210,7 +210,7 @@ def update_result_disposition(selected, str_schema, str_new_status): def finalize_small_update(status, ids): return f"""UPDATE {str_schema}.test_results SET disposition = NULLIF('{status}', 'No Decision') - WHERE id IN ({ids});""" + WHERE id IN ({ids}) and result_status != 'Passed';""" def finalize_big_update(status, ids): return f"""WITH selects @@ -220,7 +220,7 @@ def finalize_big_update(status, ids): FROM {str_schema}.test_results r INNER JOIN selects s ON (r.id = s.selected_id) - WHERE r.id = test_results.id;""" + WHERE r.id = test_results.id and result_status != 'Passed';""" def finalize_test_update(ids): str_lock_test = ", lock_refresh = 'N'" if active_yn == "Y" else ", lock_refresh = 'Y'" diff --git a/testgen/ui/services/table_group_service.py b/testgen/ui/services/table_group_service.py index ca6df20a..66097f51 100644 --- a/testgen/ui/services/table_group_service.py +++ b/testgen/ui/services/table_group_service.py @@ -90,9 +90,41 @@ def get_test_suite_ids_by_table_group_names(table_group_names): return result.to_dict()["id"].values() -def test_table_group(table_group, connection_id, project_code): - # get connection data - connection = connection_service.get_by_id(connection_id, hide_passwords=False) +def get_table_group_preview(project_code: str, connection: dict | None, table_group: dict) -> dict: + table_group_preview = { + "schema": table_group["table_group_schema"], + "tables": set(), + "column_count": 0, + "success": True, + "message": None, + } + if connection: + try: + table_group_results = test_table_group(table_group, connection, project_code) + + for column in table_group_results: + table_group_preview["schema"] = column["table_schema"] + table_group_preview["tables"].add(column["table_name"]) + table_group_preview["column_count"] += 1 + + if len(table_group_results) <= 0: + table_group_preview["success"] = False + table_group_preview["message"] = ( + "No tables found matching the criteria. Please check the Table Group configuration" + " or the database permissions." + ) + except Exception as error: + table_group_preview["success"] = False + table_group_preview["message"] = error.args[0] + else: + table_group_preview["success"] = False + table_group_preview["message"] = "No connection selected. Please select a connection to preview the Table Group." + + table_group_preview["tables"] = list(table_group_preview["tables"]) + return table_group_preview + + +def test_table_group(table_group, connection, project_code): connection_id = str(connection["connection_id"]) # get table group data @@ -143,6 +175,6 @@ def get_profiling_table_set_with_quotes(profiling_table_set): aux_list = [] split = profiling_table_set.split(",") for item in split: - aux_list.append(f"'{item}'") + aux_list.append(f"'{item.strip()}'") profiling_table_set = ",".join(aux_list) return profiling_table_set diff --git a/testgen/ui/services/test_definition_service.py b/testgen/ui/services/test_definition_service.py index 7c4eca29..2d18af89 100644 --- a/testgen/ui/services/test_definition_service.py +++ b/testgen/ui/services/test_definition_service.py @@ -14,11 +14,11 @@ def update_attribute(test_definition_ids, attribute, value): def get_test_definitions( - project_code=None, test_suite=None, table_name=None, column_name=None, test_definition_ids=None + project_code=None, test_suite=None, table_name=None, column_name=None, test_type=None, test_definition_ids=None, ): schema = st.session_state["dbschema"] return test_definition_queries.get_test_definitions( - schema, project_code, test_suite, table_name, column_name, test_definition_ids + schema, project_code, test_suite, table_name, column_name, test_type, test_definition_ids, ) diff --git a/testgen/ui/session.py b/testgen/ui/session.py index 4ac634ed..f64cff4e 100644 --- a/testgen/ui/session.py +++ b/testgen/ui/session.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Literal, TypeVar +from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypeVar if TYPE_CHECKING: from testgen.common.version_service import Version @@ -39,7 +39,7 @@ class TestgenSession(Singleton): add_project: bool version: Version | None - testgen_event_id: str | None + testgen_event_id: ClassVar[dict[str, str]] = {} sidebar_event_id: str | None link_event_id: str | None breadcrumb_event_id: str | None diff --git a/testgen/ui/views/connections.py b/testgen/ui/views/connections.py index 21c76ea4..77b059e2 100644 --- a/testgen/ui/views/connections.py +++ b/testgen/ui/views/connections.py @@ -92,6 +92,12 @@ def on_save_connection_clicked(updated_connection): else: updated_connection["private_key"] = base64.b64decode(updated_connection["private_key"]).decode() + if is_pristine(updated_connection.get("password")): + del updated_connection["password"] + + if updated_connection.get("password") == CLEAR_SENTINEL: + updated_connection["password"] = "" + updated_connection["sql_flavor"] = self._get_sql_flavor_from_value(updated_connection["sql_flavor_code"]).flavor set_save(True) @@ -226,14 +232,20 @@ def test_connection(self, connection: dict) -> "ConnectionStatus": def setup_data_configuration(self, project_code: str, connection_id: str) -> None: def on_save_table_group_clicked(payload: dict) -> None: table_group: dict = payload["table_group"] + table_group_verified: bool = payload.get("table_group_verified", False) run_profiling: bool = payload.get("run_profiling", False) set_new_table_group(table_group) + set_table_group_verified(table_group_verified) set_run_profiling(run_profiling) def on_go_to_profiling_runs(params: dict) -> None: set_navigation_params({ **params, "project_code": project_code }) + def on_preview_table_group(table_group: dict) -> None: + set_new_table_group(table_group) + mark_for_preview(True) + get_navigation_params, set_navigation_params = temp_value( "connections:new_table_group:go_to_profiling_run", default=None, @@ -242,61 +254,94 @@ def on_go_to_profiling_runs(params: dict) -> None: self.router.navigate(to="profiling-runs", with_args=params) get_new_table_group, set_new_table_group = temp_value( - "connections:new_connection:table_group", + f"connections:{connection_id}:table_group", default={}, ) get_run_profiling, set_run_profiling = temp_value( - "connections:new_connection:run_profiling", + f"connections:{connection_id}:run_profiling", default=False, ) results = None table_group = get_new_table_group() should_run_profiling = get_run_profiling() + should_preview, mark_for_preview = temp_value( + f"connections:{connection_id}:tg_preview", + default=False, + ) + is_table_group_verified, set_table_group_verified = temp_value( + f"connections:{connection_id}:tg_verified", + default=False, + ) + + table_group_preview = None + if should_preview(): + connection = connection_service.get_by_id(connection_id, hide_passwords=False) + table_group_preview = table_group_service.get_table_group_preview( + project_code, + connection, + {"id": "temp", **table_group}, + ) + if table_group: success = True message = None table_group_id = None - try: - table_group_id = table_group_service.add({ - **table_group, - "project_code": project_code, - "connection_id": connection_id, - }) - - if should_run_profiling: - try: - run_profiling_in_background(table_group_id) - message = f"Profiling run started for table group {table_group['table_groups_name']}." - except Exception as error: - message = "Profiling run encountered errors" - success = False - LOG.exception(message) - else: - LOG.info("Table group %s created", table_group_id) - st.rerun() - except Exception as error: - message = "Error creating table group" - success = False - LOG.exception(message) - - results = { - "success": success, - "message": message, - "table_group_id": table_group_id, - } + if is_table_group_verified(): + try: + table_group_id = table_group_service.add({ + **table_group, + "project_code": project_code, + "connection_id": connection_id, + }) + + if should_run_profiling: + try: + run_profiling_in_background(table_group_id) + message = f"Profiling run started for table group {table_group['table_groups_name']}." + except Exception as error: + message = "Profiling run encountered errors" + success = False + LOG.exception(message) + else: + LOG.info("Table group %s created", table_group_id) + st.rerun() + except Exception as error: + message = "Error creating table group" + success = False + LOG.exception(message) + + results = { + "success": success, + "message": message, + "table_group_id": table_group_id, + } + else: + results = { + "success": False, + "message": "Verify the table group before saving", + "connection_id": None, + "table_group_id": None, + } testgen.testgen_component( "table_group_wizard", props={ "project_code": project_code, "connection_id": connection_id, + "table_group_preview": table_group_preview, + "steps": [ + "tableGroup", + "testTableGroup", + "runProfiling", + ], "results": results, }, on_change_handlers={ "SaveTableGroupClicked": on_save_table_group_clicked, "GoToProfilingRunsClicked": on_go_to_profiling_runs, + "PreviewTableGroupClicked": on_preview_table_group, }, ) diff --git a/testgen/ui/views/data_catalog.py b/testgen/ui/views/data_catalog.py index 52bd1b68..4abb021c 100644 --- a/testgen/ui/views/data_catalog.py +++ b/testgen/ui/views/data_catalog.py @@ -38,6 +38,7 @@ from testgen.ui.views.dialogs.column_history_dialog import column_history_dialog from testgen.ui.views.dialogs.data_preview_dialog import data_preview_dialog from testgen.ui.views.dialogs.run_profiling_dialog import run_profiling_dialog +from testgen.ui.views.dialogs.table_create_script_dialog import table_create_script_dialog from testgen.utils import format_field, friendly_score, is_uuid4, score PAGE_ICON = "dataset" @@ -126,6 +127,10 @@ def render(self, project_code: str, table_group_id: str | None = None, selected: args=(selected_table_group, items), ), "RemoveTableClicked": remove_table_dialog, + "CreateScriptClicked": lambda item: table_create_script_dialog( + item["table_name"], + columns, + ), "DataPreviewClicked": lambda item: data_preview_dialog( item["table_group_id"], item["schema_name"], @@ -403,8 +408,11 @@ def get_table_group_columns(table_group_id: str) -> pd.DataFrame: CONCAT('table_', table_chars.table_id) AS table_id, column_chars.column_name, table_chars.table_name, + column_chars.schema_name, column_chars.general_type, + column_chars.column_type, column_chars.functional_data_type, + profile_results.datatype_suggestion, table_chars.record_ct, profile_results.value_ct, column_chars.drop_date, diff --git a/testgen/ui/views/dialogs/table_create_script_dialog.py b/testgen/ui/views/dialogs/table_create_script_dialog.py new file mode 100644 index 00000000..59716632 --- /dev/null +++ b/testgen/ui/views/dialogs/table_create_script_dialog.py @@ -0,0 +1,39 @@ +import pandas as pd +import streamlit as st + +from testgen.ui.components import widgets as testgen + + +@st.dialog(title="Table CREATE Script with Suggested Data Types") +def table_create_script_dialog(table_name: str, data: pd.DataFrame) -> None: + testgen.caption( + f"Table: {table_name}" + ) + st.code(generate_create_script(table_name, data), "sql") + + +def generate_create_script(table_name: str, data: pd.DataFrame) -> str: + df = data[data["table_name"] == table_name][["schema_name", "table_name", "column_name", "column_type", "datatype_suggestion"]] + df = df.copy().reset_index(drop=True) + df.fillna("", inplace=True) + + df["comment"] = df.apply( + lambda row: f"-- WAS {row['column_type']}" + if isinstance(row["column_type"], str) + and isinstance(row["datatype_suggestion"], str) + and row["column_type"].lower() != row["datatype_suggestion"].lower() + else "", + axis=1, + ) + max_len_name = df.apply(lambda row: len(row["column_name"]), axis=1).max() + 3 + max_len_type = df.apply(lambda row: len(row["datatype_suggestion"]), axis=1).max() + 3 + + header = f"CREATE TABLE {df.at[0, 'schema_name']}.{df.at[0, 'table_name']} ( " + col_defs = df.apply( + lambda row: f" {row['column_name']:<{max_len_name}} {row['datatype_suggestion']:<{max_len_type}}, {row['comment']}", + axis=1, + ).tolist() + footer = ");" + col_defs[-1] = col_defs[-1].replace(", --", " --") + + return "\n".join([header, *list(col_defs), footer]) diff --git a/testgen/ui/views/hygiene_issues.py b/testgen/ui/views/hygiene_issues.py index ed1f7991..dedea1f4 100644 --- a/testgen/ui/views/hygiene_issues.py +++ b/testgen/ui/views/hygiene_issues.py @@ -46,6 +46,7 @@ def render( issue_type: str | None = None, table_name: str | None = None, column_name: str | None = None, + action: str | None = None, **_kwargs, ) -> None: run_df = profiling_queries.get_run_by_id(run_id) @@ -68,59 +69,88 @@ def render( ], ) - others_summary_column, pii_summary_column, score_column, actions_column = st.columns([.25, .25, .2, .3], vertical_alignment="bottom") - (liklihood_filter_column, issue_type_filter_column, table_filter_column, column_filter_column, sort_column, export_button_column) = ( - st.columns([.15, .2, .2, .2, .1, .15], vertical_alignment="bottom") + others_summary_column, pii_summary_column, score_column, actions_column, export_button_column = st.columns([.2, .2, .15, .3, .15], vertical_alignment="bottom") + (table_filter_column, column_filter_column, issue_type_filter_column, liklihood_filter_column, action_filter_column, sort_column) = ( + st.columns([.15, .2, .2, .2, .15, .1], vertical_alignment="bottom") ) testgen.flex_row_end(actions_column) testgen.flex_row_end(export_button_column) - with liklihood_filter_column: - issue_class = testgen.select( - options=["Definite", "Likely", "Possible", "Potential PII"], - default_value=issue_class, - bind_to_query="issue_class", - label="Issue Class", - ) - - with issue_type_filter_column: - issue_type_options = get_issue_types() - issue_type_id = testgen.select( - options=issue_type_options, - default_value=None if issue_class == "Potential PII" else issue_type, - value_column="id", - display_column="anomaly_name", - bind_to_query="issue_type", - label="Issue Type", - disabled=issue_class == "Potential PII", - ) - run_columns_df = get_profiling_run_columns(run_id) with table_filter_column: table_name = testgen.select( options=list(run_columns_df["table_name"].unique()), default_value=table_name, bind_to_query="table_name", - label="Table Name", + label="Table", ) with column_filter_column: - column_options = list(run_columns_df.loc[run_columns_df["table_name"] == table_name]["column_name"].unique()) + if table_name: + column_options = ( + run_columns_df + .loc[run_columns_df["table_name"] == table_name] + ["column_name"] + .dropna() + .unique() + .tolist() + ) + else: + column_options = ( + run_columns_df + .groupby("column_name") + .first() + .reset_index() + .sort_values("column_name") + ) column_name = testgen.select( options=column_options, value_column="column_name", default_value=column_name, bind_to_query="column_name", - label="Column Name", - disabled=not table_name, + label="Column", accept_new_options=True, ) + with issue_type_filter_column: + issue_type_options = ( + run_columns_df + .groupby("anomaly_name") + .first() + .reset_index() + .sort_values("anomaly_name") + ) + issue_type_id = testgen.select( + options=issue_type_options, + default_value=None if issue_class == "Potential PII" else issue_type, + value_column="anomaly_id", + display_column="anomaly_name", + bind_to_query="issue_type", + label="Issue Type", + disabled=issue_class == "Potential PII", + ) + + with liklihood_filter_column: + issue_class = testgen.select( + options=["Definite", "Likely", "Possible", "Potential PII"], + default_value=issue_class, + bind_to_query="issue_class", + label="Likelihood", + ) + + with action_filter_column: + action = testgen.select( + options=["✓ Confirmed", "✘ Dismissed", "🔇 Muted", "↩ī¸Ž No Action"], + default_value=action, + bind_to_query="action", + label="Action", + ) + with sort_column: sortable_columns = ( ("Table", "r.table_name"), ("Column", "r.column_name"), - ("Anomaly", "t.anomaly_name"), + ("Issue Type", "t.anomaly_name"), ("Likelihood", "likelihood_order"), ("Action", "r.disposition"), ) @@ -134,7 +164,7 @@ def render( with st.container(): with st.spinner("Loading data ..."): # Get hygiene issue list - df_pa = get_profiling_anomalies(run_id, issue_class, issue_type_id, table_name, column_name, sorting_columns) + df_pa = get_profiling_anomalies(run_id, issue_class, issue_type_id, table_name, column_name, action, sorting_columns) # Retrieve disposition action (cache refreshed) df_action = get_anomaly_disposition(run_id) @@ -183,6 +213,14 @@ def render( do_multi_select=do_multi_select, bind_to_query_name="selected", bind_to_query_prop="id", + show_column_headers=[ + "Table", + "Column", + "Likelihood", + "Action", + "Issue Type", + "Detail" + ] ) popover_container = export_button_column.empty() @@ -219,28 +257,17 @@ def open_download_dialog(data: pd.DataFrame | None = None) -> None: if not selected_row: st.markdown(":orange[Select a record to see more information.]") else: - col1, col2 = st.columns([0.8, 0.2]) + _, buttons_column = st.columns([0.5, 0.5]) + + with buttons_column: + col1, col2, col3 = st.columns([.3, .3, .3]) + with col1: - fm.render_html_list( - selected_row, - [ - "anomaly_name", - "table_name", - "column_name", - "column_type", - "anomaly_description", - "detail", - "likelihood_explanation", - "suggested_action", - ], - "Hygiene Issue Detail", - int_data_width=700, - ) - with col2: view_profiling_button( selected_row["column_name"], selected_row["table_name"], selected_row["table_groups_id"] ) + with col2: if st.button( ":material/visibility: Source Data", help="View current source data for highlighted issue", use_container_width=True ): @@ -250,6 +277,8 @@ def open_download_dialog(data: pd.DataFrame | None = None) -> None: issue_type=selected_row["anomaly_name"], ) source_data_dialog(selected_row) + + with col3: if st.button( ":material/download: Issue Report", use_container_width=True, @@ -275,35 +304,53 @@ def open_download_dialog(data: pd.DataFrame | None = None) -> None: ) download_dialog(dialog_title=dialog_title, file_content_func=zip_func) - cached_functions = [get_anomaly_disposition, get_profiling_anomaly_summary] - # Clear the list cache if the list is sorted by disposition/action - if "r.disposition" in dict(sorting_columns): - cached_functions.append(get_profiling_anomalies) - - disposition_actions = [ - { "icon": "✓", "help": "Confirm this issue as relevant for this run", "status": "Confirmed" }, - { "icon": "✘", "help": "Dismiss this issue as not relevant for this run", "status": "Dismissed" }, - { "icon": "🔇", "help": "Mute this test to deactivate it for future runs", "status": "Inactive" }, - { "icon": "↩ī¸Ž", "help": "Clear action", "status": "No Decision" }, - ] - - if user_session_service.user_can_disposition(): - # Need to render toolbar buttons after grid, so selection status is maintained - for action in disposition_actions: - action["button"] = actions_column.button(action["icon"], help=action["help"], disabled=not selected) - - # This has to be done as a second loop - otherwise, the rest of the buttons after the clicked one are not displayed briefly while refreshing - for action in disposition_actions: - if action["button"]: - fm.reset_post_updates( - do_disposition_update(selected, action["status"]), - as_toast=True, - clear_cache=True, - lst_cached_functions=cached_functions, - ) + fm.render_html_list( + selected_row, + [ + "anomaly_name", + "table_name", + "column_name", + "column_type", + "anomaly_description", + "detail", + "likelihood_explanation", + "suggested_action", + ], + "Hygiene Issue Detail", + int_data_width=700, + ) else: st.markdown(":green[**No Hygiene Issues Found**]") + cached_functions = [get_anomaly_disposition, get_profiling_anomaly_summary, get_profiling_anomalies] + + disposition_actions = [ + { "icon": "✓", "help": "Confirm this issue as relevant for this run", "status": "Confirmed" }, + { "icon": "✘", "help": "Dismiss this issue as not relevant for this run", "status": "Dismissed" }, + { "icon": "🔇", "help": "Mute this test to deactivate it for future runs", "status": "Inactive" }, + { "icon": "↩ī¸Ž", "help": "Clear action", "status": "No Decision" }, + ] + + if user_session_service.user_can_disposition(): + disposition_translator = {"No Decision": None} + # Need to render toolbar buttons after grid, so selection status is maintained + for d_action in disposition_actions: + disable_action=not selected or all( + sel["disposition"] == disposition_translator.get(d_action["status"], d_action["status"]) + for sel in selected + ) + d_action["button"] = actions_column.button(d_action["icon"], help=d_action["help"], disabled=disable_action) + + # This has to be done as a second loop - otherwise, the rest of the buttons after the clicked one are not displayed briefly while refreshing + for d_action in disposition_actions: + if d_action["button"]: + fm.reset_post_updates( + do_disposition_update(selected, d_action["status"]), + as_toast=True, + clear_cache=True, + lst_cached_functions=cached_functions, + ) + # Help Links st.markdown( "[Help on Hygiene Issues](https://docs.datakitchen.io/article/dataops-testgen-help/data-hygiene-issues)" @@ -346,10 +393,11 @@ def refresh_score(project_code: str, run_id: str, table_group_id: str | None) -> def get_profiling_run_columns(profiling_run_id: str) -> pd.DataFrame: schema: str = st.session_state["dbschema"] sql = f""" - SELECT table_name, column_name - FROM {schema}.profile_anomaly_results - WHERE profile_run_id = '{profiling_run_id}' - ORDER BY table_name, column_name; + SELECT r.table_name table_name, r.column_name column_name, r.anomaly_id anomaly_id, t.anomaly_name anomaly_name + FROM {schema}.profile_anomaly_results r + LEFT JOIN {schema}.profile_anomaly_types t on t.id = r.anomaly_id + WHERE r.profile_run_id = '{profiling_run_id}' + ORDER BY r.table_name, r.column_name; """ return db.retrieve_data(sql) @@ -361,6 +409,7 @@ def get_profiling_anomalies( issue_type_id: str | None = None, table_name: str | None = None, column_name: str | None = None, + action: str | None = None, sorting_columns: list[str] | None = None, ): db_session = get_current_session() @@ -380,6 +429,14 @@ def get_profiling_anomalies( if column_name: criteria += " AND r.column_name ILIKE :column_name" params["column_name"] = column_name + if action: + action = action.split(" ", 1)[1] + if action == "No Action": + criteria += " AND r.disposition IS NULL" + else: + action_disposition_converter = {"Muted": "Inactive"} + criteria += " AND r.disposition = :disposition_name" + params["disposition_name"] = action_disposition_converter.get(action, action) if sorting_columns: order_by = "ORDER BY " + (", ".join(" ".join(col) for col in sorting_columns)) @@ -476,13 +533,6 @@ def get_anomaly_disposition(str_profile_run_id): return df[["id", "action"]] -@st.cache_data(show_spinner=False) -def get_issue_types(): - schema = st.session_state["dbschema"] - df = db.retrieve_data(f"SELECT id, anomaly_name FROM {schema}.profile_anomaly_types") - return df - - @st.cache_data(show_spinner=False) def get_profiling_anomaly_summary(str_profile_run_id): str_schema = st.session_state["dbschema"] @@ -537,7 +587,7 @@ def get_excel_report_data( "schema_name": {"header": "Schema"}, "table_name": {"header": "Table"}, "column_name": {"header": "Column"}, - "anomaly_name": {"header": "Issue name"}, + "anomaly_name": {"header": "Issue Type"}, "issue_likelihood": {"header": "Likelihood"}, "anomaly_description": {"header": "Description", "wrap": True}, "action": {}, diff --git a/testgen/ui/views/profiling_results.py b/testgen/ui/views/profiling_results.py index dec8b4ab..9b4f2142 100644 --- a/testgen/ui/views/profiling_results.py +++ b/testgen/ui/views/profiling_results.py @@ -69,7 +69,7 @@ def render(self, run_id: str, table_name: str | None = None, column_name: str | value_column="table_name", default_value=table_name, bind_to_query="table_name", - label="Table Name", + label="Table", ) with column_filter_column: @@ -80,17 +80,17 @@ def render(self, run_id: str, table_name: str | None = None, column_name: str | value_column="column_name", default_value=column_name, bind_to_query="column_name", - label="Column Name", + label="Column", disabled=not table_name, accept_new_options=bool(table_name), ) with sort_column: sortable_columns = ( - ("Schema Name", "schema_name"), - ("Table Name", "table_name"), - ("Column Name", "column_name"), - ("Column Type", "column_type"), + ("Schema", "schema_name"), + ("Table", "table_name"), + ("Column", "column_name"), + ("Data Type", "column_type"), ("Semantic Data Type", "semantic_data_type"), ("Hygiene Issues", "hygiene_issues"), ) @@ -106,7 +106,7 @@ def render(self, run_id: str, table_name: str | None = None, column_name: str | column_name=column_name, sorting_columns=sorting_columns, ) - + show_columns = [ "schema_name", "table_name", @@ -115,17 +115,21 @@ def render(self, run_id: str, table_name: str | None = None, column_name: str | "semantic_data_type", "hygiene_issues", ] - - # Show CREATE script button - if len(df) > 0 and table_name != "%%": - with st.expander("📜 **Table CREATE script with suggested datatypes**"): - st.code(generate_create_script(df), "sql") + show_column_headers = [ + "Schema", + "Table", + "Column", + "Data Type", + "Semantic Data Type", + "Hygiene Issues", + ] selected_row = fm.render_grid_select( df, show_columns, bind_to_query_name="selected", bind_to_query_prop="id", + show_column_headers=show_column_headers, ) popover_container = export_button_column.empty() @@ -276,28 +280,6 @@ def get_excel_report_data( ) -def generate_create_script(df): - ddf = df[["schema_name", "table_name", "column_name", "column_type", "datatype_suggestion"]].copy() - ddf.fillna("", inplace=True) - - ddf["comment"] = ddf.apply( - lambda row: "-- WAS " + row["column_type"] if row["column_type"] != row["datatype_suggestion"] else "", axis=1 - ) - max_len_name = ddf.apply(lambda row: len(row["column_name"]), axis=1).max() + 3 - max_len_type = ddf.apply(lambda row: len(row["datatype_suggestion"]), axis=1).max() + 3 - - str_header = f"CREATE TABLE {df.at[0, 'schema_name']}.{ddf.at[0, 'table_name']} ( " - col_defs = ddf.apply( - lambda row: f" {row['column_name']:<{max_len_name}} {row['datatype_suggestion']:<{max_len_type}}, {row['comment']}", - axis=1, - ).tolist() - str_footer = ");" - # Drop final comma in column definitions - col_defs[-1] = col_defs[-1].replace(", --", " --") - - return "\n".join([str_header, *list(col_defs), str_footer]) - - @st.cache_data(show_spinner=False) def get_profiling_run_tables(profiling_run_id: str): schema: str = st.session_state["dbschema"] diff --git a/testgen/ui/views/project_dashboard.py b/testgen/ui/views/project_dashboard.py index 68c5ef9b..8e28f5db 100644 --- a/testgen/ui/views/project_dashboard.py +++ b/testgen/ui/views/project_dashboard.py @@ -181,7 +181,8 @@ def get_table_groups_summary(project_code: str) -> pd.DataFrame: latest_profile.dismissed_ct as latest_anomalies_dismissed_ct FROM {schema}.table_groups as groups LEFT JOIN latest_profile ON (groups.id = latest_profile.table_groups_id) - WHERE groups.project_code = '{project_code}'; + WHERE groups.project_code = '{project_code}' + AND groups.include_in_dashboard IS TRUE; """ return db.retrieve_data(sql) diff --git a/testgen/ui/views/table_groups.py b/testgen/ui/views/table_groups.py index aff424c7..daecc088 100644 --- a/testgen/ui/views/table_groups.py +++ b/testgen/ui/views/table_groups.py @@ -1,3 +1,4 @@ +import logging import typing from dataclasses import asdict from functools import partial @@ -17,6 +18,7 @@ from testgen.ui.views.connections import FLAVOR_OPTIONS, format_connection from testgen.ui.views.profiling_runs import ProfilingScheduleDialog +LOG = logging.getLogger("testgen") PAGE_TITLE = "Table Groups" @@ -58,7 +60,7 @@ def render(self, project_code: str, connection_id: str | None = None, **_kwargs) }, on_change_handlers={ "RunSchedulesClicked": lambda *_: ProfilingScheduleDialog().open(project_code), - "AddTableGroupClicked": partial(self.add_table_group_dialog, project_code), + "AddTableGroupClicked": partial(self.add_table_group_dialog, project_code, connection_id), "EditTableGroupClicked": partial(self.edit_table_group_dialog, project_code), "DeleteTableGroupClicked": partial(self.delete_table_group_dialog, project_code), "RunProfilingClicked": partial(self.run_profiling_dialog, project_code), @@ -71,115 +73,157 @@ def render(self, project_code: str, connection_id: str | None = None, **_kwargs) @st.dialog(title="Add Table Group") @with_database_session - def add_table_group_dialog(self, project_code, *_args): + def add_table_group_dialog(self, project_code: str, connection_id: str | None, *_args): + return self._table_group_wizard( + project_code, + save_table_group_fn=table_group_service.add, + connection_id=connection_id, + steps=[ + "tableGroup", + "testTableGroup", + "runProfiling", + ], + ) + + @st.dialog(title="Edit Table Group") + def edit_table_group_dialog(self, project_code: str, table_group_id: str): + return self._table_group_wizard( + project_code, + save_table_group_fn=table_group_service.edit, + table_group_id=table_group_id, + steps=[ + "tableGroup", + "testTableGroup", + ], + ) + + def _table_group_wizard( + self, + project_code: str, + *, + save_table_group_fn: typing.Callable[[dict], str], + steps: list[str] | None = None, + connection_id: str | None = None, + table_group_id: str | None = None, + ): def on_preview_table_group_clicked(table_group: dict): mark_for_preview(True) set_table_group(table_group) - def on_save_table_group_clicked(table_group: dict): + def on_save_table_group_clicked(payload: dict): + table_group: dict = payload["table_group"] + table_group_verified: bool = payload.get("table_group_verified", False) + run_profiling: bool = payload.get("run_profiling", False) + set_save(True) set_table_group(table_group) + set_table_group_verified(table_group_verified) + set_run_profiling(run_profiling) - should_preview, mark_for_preview = temp_value("table_groups:preview:new", default=False) - should_save, set_save = temp_value("table_groups:save:new", default=False) - get_table_group, set_table_group = temp_value("table_groups:updated:new", default={}) - - connections = self._get_connections(project_code) - table_group = { - "project_code": project_code, - **get_table_group(), - } - table_group_preview = {} - result = None - - if len(connections) == 1: - table_group["connection_id"] = connections[0]["connection_id"] + def on_go_to_profiling_runs(params: dict) -> None: + set_navigation_params({ **params, "project_code": project_code }) - if should_save(): - try: - table_group_service.add(table_group) - st.rerun() - except IntegrityError: - result = {"success": False, "message": "A Table Group with the same name already exists."} - - if should_preview(): - table_group_preview = self._get_table_group_preview(project_code, table_group["connection_id"], {"id": "temp", **table_group}) - - return testgen.testgen_component( - "table_group", - props={ - "project_code": project_code, - "connections": connections, - "table_group": table_group, - "table_group_preview": table_group_preview, - "result": result, - }, - on_change_handlers={ - "PreviewTableGroupClicked": on_preview_table_group_clicked, - "TableGroupSaveClicked": on_save_table_group_clicked, - }, + get_navigation_params, set_navigation_params = temp_value( + "connections:new_table_group:go_to_profiling_run", + default=None, ) + if (params := get_navigation_params()): + self.router.navigate(to="profiling-runs", with_args=params) - @st.dialog(title="Edit Table Group") - def edit_table_group_dialog(self, project_code: str, table_group_id: str): - def on_preview_table_group_clicked(table_group: dict): - mark_for_preview(True) - set_updated_table_group(table_group) - - def on_save_table_group_clicked(table_group: dict): - set_update(True) - set_updated_table_group(table_group) - - should_preview, mark_for_preview = temp_value( - f"table_groups:preview:{table_group_id}", + should_preview, mark_for_preview = temp_value("table_groups:preview:new", default=False) + should_save, set_save = temp_value("table_groups:save:new", default=False) + get_table_group, set_table_group = temp_value("table_groups:updated:new", default={}) + is_table_group_verified, set_table_group_verified = temp_value( + "table_groups:new:verified", default=False, ) - should_update, set_update = temp_value( - f"table_groups:save:{table_group_id}", + should_run_profiling, set_run_profiling = temp_value( + "table_groups:new:run_profiling", default=False, ) - get_updated_table_group, set_updated_table_group = temp_value( - f"table_groups:updated:{table_group_id}", - default={}, - ) - original_table_group = table_group_service.get_by_id(table_group_id=table_group_id).to_dict() - is_table_group_used = table_group_service.is_table_group_used(table_group_id) + is_table_group_used = False + connections = self._get_connections(project_code) + original_table_group = {"project_code": project_code} + if table_group_id: + original_table_group = table_group_service.get_by_id(table_group_id=table_group_id).to_dict() + is_table_group_used = table_group_service.is_table_group_used(table_group_id) + table_group = { **original_table_group, - **get_updated_table_group(), + **get_table_group(), } + table_group_preview = None + if is_table_group_used: table_group["table_group_schema"] = original_table_group["table_group_schema"] - table_group_preview = { - "schema": table_group["table_group_schema"], - } - result = None + if len(connections) == 1: + table_group["connection_id"] = connections[0]["connection_id"] - if should_update(): - try: - table_group_service.edit(table_group) - st.rerun() - except IntegrityError: - result = {"success": False, "message": "A Table Group with the same name already exists."} + if not table_group.get("connection_id"): + if connection_id: + table_group["connection_id"] = int(connection_id) + elif len(connections) == 1: + table_group["connection_id"] = connections[0]["connection_id"] + elif table_group.get("id"): + connections = [ + conn for conn in connections + if int(conn["connection_id"]) == int(table_group.get("connection_id")) + ] if should_preview(): - table_group_preview = self._get_table_group_preview(project_code, table_group["connection_id"], table_group) + connection = connection_service.get_by_id(table_group["connection_id"], hide_passwords=False) + table_group_preview = table_group_service.get_table_group_preview( + project_code, + connection, + {"id": table_group.get("id") or "temp", **table_group}, + ) + + success = None + message = "" + table_group_id = None + if should_save(): + success = True + if is_table_group_verified(): + try: + table_group_id = save_table_group_fn(table_group) + if should_run_profiling(): + try: + run_profiling_in_background(table_group_id) + message = f"Profiling run started for table group {table_group['table_groups_name']}." + except Exception: + success = False + message = "Profiling run encountered errors" + LOG.exception(message) + else: + st.rerun() + except IntegrityError: + success = False + message = "A Table Group with the same name already exists." + else: + success = False + message = "Verify the table group before saving" return testgen.testgen_component( - "table_group", + "table_group_wizard", props={ "project_code": project_code, - "connections": self._get_connections(project_code, connection_id=table_group["connection_id"]), + "connections": connections, "table_group": table_group, - "in_used": is_table_group_used, + "is_in_use": is_table_group_used, "table_group_preview": table_group_preview, - "result": result, + "steps": steps, + "results": { + "success": success, + "message": message, + "table_group_id": table_group_id, + } if success is not None else None, }, on_change_handlers={ "PreviewTableGroupClicked": on_preview_table_group_clicked, - "TableGroupSaveClicked": on_save_table_group_clicked, + "SaveTableGroupClicked": on_save_table_group_clicked, + "GoToProfilingRunsClicked": on_go_to_profiling_runs, }, ) @@ -204,38 +248,6 @@ def _format_table_group_list(self, table_groups: list[dict]) -> list[dict]: } return table_groups - def _get_table_group_preview(self, project_code: str, connection_id: str | None, table_group: dict) -> dict: - table_group_preview = { - "schema": table_group["table_group_schema"], - "tables": set(), - "column_count": 0, - "success": True, - "message": None, - } - if connection_id: - try: - table_group_results = table_group_service.test_table_group(table_group, connection_id, project_code) - - for column in table_group_results: - table_group_preview["schema"] = column["table_schema"] - table_group_preview["tables"].add(column["table_name"]) - table_group_preview["column_count"] += 1 - - if len(table_group_results) <= 0: - table_group_preview["success"] = False - table_group_preview["message"] = ( - "No tables found matching the criteria. Please check the Table Group configuration." - ) - except Exception as error: - table_group_preview["success"] = False - table_group_preview["message"] = error.args[0] - else: - table_group_preview["success"] = False - table_group_preview["message"] = "No connection selected. Please select a connection to preview the Table Group." - - table_group_preview["tables"] = list(table_group_preview["tables"]) - return table_group_preview - @st.dialog(title="Run Profiling") def run_profiling_dialog(self, project_code: str, table_group_id: str) -> None: def on_go_to_profiling_runs_clicked(table_group_id: str) -> None: @@ -305,7 +317,7 @@ def on_delete_confirmed(*_args): st.rerun() else: message = "This Table Group is in use by a running process and cannot be deleted." - result = {"success": success, "message": message}, + result = {"success": success, "message": message} testgen.testgen_component( "table_group_delete", diff --git a/testgen/ui/views/test_definitions.py b/testgen/ui/views/test_definitions.py index 15157e92..18ddfa39 100644 --- a/testgen/ui/views/test_definitions.py +++ b/testgen/ui/views/test_definitions.py @@ -29,6 +29,7 @@ from testgen.ui.services.string_service import empty_if_null, snake_case_to_title_case from testgen.ui.session import session, temp_value from testgen.ui.views.dialogs.profiling_results_dialog import view_profiling_button +from testgen.ui.views.dialogs.run_tests_dialog import run_tests_dialog LOG = logging.getLogger("testgen") @@ -64,7 +65,7 @@ def render(self, test_suite_id: str, table_name: str | None = None, column_name: ], ) - table_filter_column, column_filter_column, table_actions_column = st.columns([.3, .3, .4], vertical_alignment="bottom") + table_filter_column, column_filter_column, test_filter_column, table_actions_column = st.columns([.3, .3, .3, .4], vertical_alignment="bottom") testgen.flex_row_end(table_actions_column) actions_column, disposition_column = st.columns([.5, .5]) @@ -80,7 +81,7 @@ def render(self, test_suite_id: str, table_name: str | None = None, column_name: default_value=table_name or (table_options[0] if table_options else None), bind_to_query="table_name", required=True, - label="Table Name", + label="Table", ) with column_filter_column: column_options = columns_df.loc[columns_df["table_name"] == table_name]["column_name"].dropna().unique().tolist() @@ -88,10 +89,20 @@ def render(self, test_suite_id: str, table_name: str | None = None, column_name: options=column_options, default_value=column_name, bind_to_query="column_name", - label="Column Name", + label="Column", disabled=not table_name, accept_new_options=True, ) + with test_filter_column: + test_options = columns_df.groupby("test_type").first().reset_index().sort_values("test_name_short") + test_type = testgen.select( + options=test_options, + value_column="test_type", + display_column="test_name_short", + default_value=None, + bind_to_query="test_type", + label="Test Type", + ) with disposition_column: str_help = "Toggle on to perform actions on multiple test definitions" @@ -102,8 +113,14 @@ def render(self, test_suite_id: str, table_name: str | None = None, column_name: ): add_test_dialog(project_code, table_group, test_suite, table_name, column_name) + if user_can_edit and table_actions_column.button( + ":material/play_arrow: Run Tests", + help="Run test suite's tests", + ): + run_tests_dialog(project_code, test_suite) + selected = show_test_defs_grid( - project_code, test_suite["test_suite"], table_name, column_name, do_multi_select, table_actions_column, + project_code, test_suite["test_suite"], table_name, column_name, test_type, do_multi_select, table_actions_column, table_group["id"] ) fm.render_refresh_button(table_actions_column) @@ -121,7 +138,8 @@ def render(self, test_suite_id: str, table_name: str | None = None, column_name: ]) for action in disposition_actions: - action["button"] = disposition_column.button(action["icon"], help=action["help"], disabled=not selected) + action_disabled = not selected or all(sel[action["attribute"]] == ("Y" if action["value"] else "N") for sel in selected) + action["button"] = disposition_column.button(action["icon"], help=action["help"], disabled=action_disabled) # This has to be done as a second loop - otherwise, the rest of the buttons after the clicked one are not displayed briefly while refreshing for action in disposition_actions: @@ -265,12 +283,12 @@ def show_test_form( test_action = empty_if_null(selected_test_def["test_action"]) if mode == "edit" else "" schema_name = selected_test_def["schema_name"] if mode == "edit" else table_group["table_group_schema"] table_name = empty_if_null(selected_test_def["table_name"]) if mode == "edit" else empty_if_null(str_table_name) - skip_errors = selected_test_def["skip_errors"] if mode == "edit" else 0 + skip_errors = selected_test_def["skip_errors"] or 0 if mode == "edit" else 0 test_active = selected_test_def["test_active"] == "Y" if mode == "edit" else True lock_refresh = selected_test_def["lock_refresh"] == "Y" if mode == "edit" else False test_definition_status = selected_test_def["test_definition_status"] if mode == "edit" else "" check_result = selected_test_def["check_result"] if mode == "edit" else None - column_name = empty_if_null(selected_test_def["column_name"]) if mode == "edit" else "" + column_name = empty_if_null(selected_test_def["column_name"]) if mode == "edit" else empty_if_null(str_column_name) last_auto_gen_date = empty_if_null(selected_test_def["last_auto_gen_date"]) if mode == "edit" else "" profiling_as_of_date = empty_if_null(selected_test_def["profiling_as_of_date"]) if mode == "edit" else "" profile_run_id = empty_if_null(selected_test_def["profile_run_id"]) if mode == "edit" else "" @@ -281,12 +299,12 @@ def show_test_form( baseline_unique_ct = empty_if_null(selected_test_def["baseline_unique_ct"]) if mode == "edit" else "" baseline_value = empty_if_null(selected_test_def["baseline_value"]) if mode == "edit" else "" baseline_value_ct = empty_if_null(selected_test_def["baseline_value_ct"]) if mode == "edit" else "" - threshold_value = empty_if_null(selected_test_def["threshold_value"]) if mode == "edit" else 0 + threshold_value = selected_test_def["threshold_value"] or 0 if mode == "edit" else 0 baseline_sum = empty_if_null(selected_test_def["baseline_sum"]) if mode == "edit" else "" baseline_avg = empty_if_null(selected_test_def["baseline_avg"]) if mode == "edit" else "" baseline_sd = empty_if_null(selected_test_def["baseline_sd"]) if mode == "edit" else "" - lower_tolerance = empty_if_null(selected_test_def["lower_tolerance"]) if mode == "edit" else 0 - upper_tolerance = empty_if_null(selected_test_def["upper_tolerance"]) if mode == "edit" else 0 + lower_tolerance = selected_test_def["lower_tolerance"] or 0 if mode == "edit" else 0 + upper_tolerance = selected_test_def["upper_tolerance"] or 0 if mode == "edit" else 0 subset_condition = empty_if_null(selected_test_def["subset_condition"]) if mode == "edit" else "" groupby_names = empty_if_null(selected_test_def["groupby_names"]) if mode == "edit" else "" having_condition = empty_if_null(selected_test_def["having_condition"]) if mode == "edit" else "" @@ -297,7 +315,7 @@ def show_test_form( match_subset_condition = empty_if_null(selected_test_def["match_subset_condition"]) if mode == "edit" else "" match_groupby_names = empty_if_null(selected_test_def["match_groupby_names"]) if mode == "edit" else "" match_having_condition = empty_if_null(selected_test_def["match_having_condition"]) if mode == "edit" else "" - window_days = selected_test_def["window_days"] if mode == "edit" and selected_test_def["window_days"] else 0 + window_days = selected_test_def["window_days"] or 0 if mode == "edit" else 0 test_mode = empty_if_null(selected_test_def["test_mode"]) if mode == "edit" else "" # export_to_observability @@ -490,49 +508,20 @@ def show_test_form( test_definition["column_name"] = None column_name_label = None elif test_scope == "referential": - column_name_disabled = False test_definition["column_name"] = left_column.text_input( label=column_name_label, value=column_name, max_chars=500, help=column_name_help, - disabled=column_name_disabled, ) elif test_scope == "custom": - if str_column_name: - if mode == "add": # query add present - column_name_disabled = False - column_name = str_column_name - else: # query edit present - column_name_disabled = False - column_name = str_column_name - else: - if mode == "add": # query add not-present - column_name_disabled = False - else: # query edit not-present - column_name_disabled = False - test_definition["column_name"] = left_column.text_input( label=column_name_label, value=column_name, max_chars=100, help=column_name_help, - disabled=column_name_disabled, ) elif test_scope == "column": # CAT column test - if str_column_name: - column_name_disabled = True - if mode == "add": - column_name = str_column_name # CAT add present - else: - pass # CAT edit present - else: - column_name_disabled = False - if mode == "add": - pass # CAT add not-present - else: - pass # CAT edit not-present - column_name_label = "Column Name" column_name_options = get_column_names(table_groups_id, test_definition["table_name"]) column_name_help = "Select the column to test" @@ -543,7 +532,6 @@ def show_test_form( index=column_name_index, help=column_name_help, key="column-name-form", - disabled=column_name_disabled, ) leftover_attributes = dynamic_attributes.copy() @@ -551,11 +539,11 @@ def show_test_form( def render_dynamic_attribute(attribute: str, container: DeltaGenerator): if not attribute in dynamic_attributes: return - + numeric_attributes = ["threshold_value", "lower_tolerance", "upper_tolerance"] default_value = 0 if attribute in numeric_attributes else "" - value = empty_if_null(selected_test_def[attribute]) if mode == "edit" else default_value + value = selected_test_def[attribute] if mode == "edit" and selected_test_def[attribute] is not None else default_value index = dynamic_attributes.index(attribute) leftover_attributes.remove(attribute) @@ -577,7 +565,7 @@ def render_dynamic_attribute(attribute: str, container: DeltaGenerator): custom_query_placeholder = "EXAMPLE: status = 'SHIPPED' and qty_shipped = 0" elif test_type == "CUSTOM": custom_query_placeholder = "EXAMPLE: SELECT product, SUM(qty_sold) as sum_sold, SUM(qty_shipped) as qty_shipped \n FROM {DATA_SCHEMA}.sales_history \n GROUP BY product \n HAVING SUM(qty_shipped) > SUM(qty_sold)" - + test_definition[attribute] = st.text_area( label=label_text, value=custom_query, @@ -870,13 +858,13 @@ def update_test_definition(selected, attribute, value, message): def show_test_defs_grid( - str_project_code, str_test_suite, str_table_name, str_column_name, do_multi_select, export_container, + str_project_code, str_test_suite, str_table_name, str_column_name, str_test_type, do_multi_select, export_container, str_table_groups_id ): with st.container(): with st.spinner("Loading data ..."): df = test_definition_service.get_test_definitions( - str_project_code, str_test_suite, str_table_name, str_column_name + str_project_code, str_test_suite, str_table_name, str_column_name, str_test_type ) date_service.accommodate_dataframe_to_timezone(df, st.session_state) @@ -899,7 +887,7 @@ def show_test_defs_grid( "Schema", "Table", "Columns / Focus", - "Test Name", + "Test Type", "Active", "Locked", "Urgency", @@ -1097,8 +1085,9 @@ def run_table_groups_lookup_query(str_project_code, str_connection_id=None, tabl def get_test_suite_columns(test_suite_id: str) -> pd.DataFrame: schema: str = st.session_state["dbschema"] sql = f""" - SELECT table_name, column_name - FROM {schema}.test_definitions + SELECT d.table_name, d.column_name, t.test_name_short, d.test_type + FROM {schema}.test_definitions d + LEFT JOIN {schema}.test_types as t on t.test_type = d.test_type WHERE test_suite_id = '{test_suite_id}' ORDER BY table_name, column_name; """ diff --git a/testgen/ui/views/test_results.py b/testgen/ui/views/test_results.py index 8053babf..67335b4e 100644 --- a/testgen/ui/views/test_results.py +++ b/testgen/ui/views/test_results.py @@ -76,7 +76,7 @@ def render( ) summary_column, score_column, actions_column = st.columns([.4, .2, .4], vertical_alignment="bottom") - status_filter_column, test_type_filter_column, table_filter_column, column_filter_column, sort_column, export_button_column = st.columns( + table_filter_column, column_filter_column, test_type_filter_column, status_filter_column, sort_column, export_button_column = st.columns( [.175, .175, .2, .2, .1, .15], vertical_alignment="bottom" ) @@ -90,61 +90,64 @@ def render( with score_column: render_score(run_df["project_code"], run_id) - with status_filter_column: - status_options = [ - "Failed + Warning", - "Failed", - "Warning", - "Passed", - "Error", - ] - status = testgen.select( - options=status_options, - default_value=status or "Failed + Warning", - bind_to_query="status", - bind_empty_value=True, - label="Result Status", - ) - - with test_type_filter_column: - test_type = testgen.select( - options=get_test_types(), - value_column="test_type", - display_column="test_name_short", - default_value=test_type, - bind_to_query="test_type", - label="Test Type", - ) - run_columns_df = get_test_run_columns(run_id) with table_filter_column: table_name = testgen.select( options=list(run_columns_df["table_name"].unique()), default_value=table_name, bind_to_query="table_name", - label="Table Name", + label="Table", ) with column_filter_column: - column_options = run_columns_df.loc[ - run_columns_df["table_name"] == table_name - ]["column_name"].dropna().unique().tolist() + if table_name: + column_options = run_columns_df.loc[ + run_columns_df["table_name"] == table_name + ]["column_name"].dropna().unique().tolist() + else: + column_options = run_columns_df.groupby("column_name").first().reset_index().sort_values("column_name") column_name = testgen.select( options=column_options, value_column="column_name", default_value=column_name, bind_to_query="column_name", - label="Column Name", - disabled=not table_name, + label="Column", accept_new_options=True, ) + with test_type_filter_column: + test_type = testgen.select( + options=run_columns_df.groupby("test_type").first().reset_index().sort_values("test_name_short"), + value_column="test_type", + display_column="test_name_short", + default_value=test_type, + required=False, + bind_to_query="test_type", + label="Test Type", + ) + + with status_filter_column: + status_options = [ + "Failed + Warning", + "Failed", + "Warning", + "Passed", + "Error", + ] + status = testgen.select( + options=status_options, + default_value=status or "Failed + Warning", + bind_to_query="status", + bind_empty_value=True, + label="Status", + ) + with sort_column: sortable_columns = ( - ("Table Name", "r.table_name"), + ("Table", "r.table_name"), ("Columns/Focus", "r.column_names"), ("Test Type", "r.test_type"), - ("UOM", "tt.measure_uom"), + ("Unit of Measure", "tt.measure_uom"), ("Result Measure", "result_measure"), ("Status", "result_status"), ("Action", "r.disposition"), @@ -183,11 +186,7 @@ def render( ) # Need to render toolbar buttons after grid, so selection status is maintained - disable_dispo = True if not selected or status == "'Passed'" else False - - affected_cached_functions = [get_test_disposition] - if "r.disposition" in dict(sorting_columns): - affected_cached_functions.append(get_test_results) + affected_cached_functions = [get_test_disposition, get_test_results] disposition_actions = [ { "icon": "✓", "help": "Confirm this issue as relevant for this run", "status": "Confirmed" }, @@ -197,7 +196,14 @@ def render( ] if user_session_service.user_can_disposition(): + disable_all_dispo = not selected or status == "'Passed'" or all(sel["result_status"] == "Passed" for sel in selected) + disposition_translator = {"No Decision": None} for action in disposition_actions: + disable_dispo = disable_all_dispo or all( + sel["disposition"] == disposition_translator.get(action["status"], action["status"]) + or sel["result_status"] == "Passed" + for sel in selected + ) action["button"] = actions_column.button(action["icon"], help=action["help"], disabled=disable_dispo) # This has to be done as a second loop - otherwise, the rest of the buttons after the clicked one are not displayed briefly while refreshing @@ -253,7 +259,7 @@ def refresh_score(project_code: str, run_id: str, table_group_id: str | None) -> def get_run_by_id(test_run_id: str) -> pd.Series: if not is_uuid4(test_run_id): return pd.Series() - + schema: str = st.session_state["dbschema"] sql = f""" SELECT tr.test_starttime, @@ -273,19 +279,13 @@ def get_run_by_id(test_run_id: str) -> pd.Series: return pd.Series() -@st.cache_data(show_spinner=False) -def get_test_types(): - schema = st.session_state["dbschema"] - df = db.retrieve_data(f"SELECT test_type, test_name_short FROM {schema}.test_types") - return df - - @st.cache_data(show_spinner=False) def get_test_run_columns(test_run_id: str) -> pd.DataFrame: schema: str = st.session_state["dbschema"] sql = f""" - SELECT table_name, column_names AS column_name - FROM {schema}.test_results + SELECT r.table_name as table_name, r.column_names AS column_name, t.test_name_short as test_name_short, t.test_type as test_type + FROM {schema}.test_results r + LEFT JOIN {schema}.test_types t ON t.test_type = r.test_type WHERE test_run_id = '{test_run_id}' ORDER BY table_name, column_names; """ @@ -408,7 +408,7 @@ def readable_boolean(v: typing.Literal["Y", "N"]): if not test_def_id: st.warning("Test definition no longer exists.") return - + df = get_test_definition(test_def_id) specs = [] @@ -506,11 +506,11 @@ def show_result_detail( ] lst_show_headers = [ - "Table Name", + "Table", "Columns/Focus", "Test Type", "Result Measure", - "UOM", + "Unit of Measure", "Status", "Action", "Details", diff --git a/testgen/ui/views/test_suites.py b/testgen/ui/views/test_suites.py index 13250493..0ae9b654 100644 --- a/testgen/ui/views/test_suites.py +++ b/testgen/ui/views/test_suites.py @@ -182,7 +182,7 @@ def show_test_suite(mode, project_code, table_groups_df, selected=None): help="Overrides the default severity in 'Test Definition' and/or 'Test Run'.", ), "test_suite_schema": test_suite_schema, - "export_to_observability": left_column.toggle( + "export_to_observability": left_column.checkbox( "Export to Observability", value=export_to_observability, help="Fields below are only required when overriding the Table Group defaults.",