diff --git a/deploy/charts/testgen-app/templates/_environment.yaml b/deploy/charts/testgen-app/templates/_environment.yaml index 01a08f38..5d8bed73 100644 --- a/deploy/charts/testgen-app/templates/_environment.yaml +++ b/deploy/charts/testgen-app/templates/_environment.yaml @@ -2,12 +2,12 @@ - name: TG_DECRYPT_SALT valueFrom: secretKeyRef: - name: {{ .Values.testgen.databaseDecryptSecret.name | quote }} + name: {{ .Values.testgen.authSecrets.name | quote }} key: "decrypt-salt" - name: TG_DECRYPT_PASSWORD valueFrom: secretKeyRef: - name: {{ .Values.testgen.databaseDecryptSecret.name | quote }} + name: {{ .Values.testgen.authSecrets.name | quote }} key: "decrypt-password" - name: TG_METADATA_DB_HOST value: {{ .Values.testgen.databaseHost | quote }} diff --git a/deploy/charts/testgen-app/templates/_helpers.tpl b/deploy/charts/testgen-app/templates/_helpers.tpl index 844fadb1..9a1aab55 100644 --- a/deploy/charts/testgen-app/templates/_helpers.tpl +++ b/deploy/charts/testgen-app/templates/_helpers.tpl @@ -34,6 +34,9 @@ Create chart name and version as used by the chart label. Common labels */}} {{- define "testgen.labels" -}} +{{- with .Values.testgen.labels }} +{{- . | toYaml }} +{{- end }} helm.sh/chart: {{ include "testgen.chart" . }} {{ include "testgen.selectorLabels" . }} {{- if .Chart.AppVersion }} diff --git a/deploy/charts/testgen-app/templates/deployment.yaml b/deploy/charts/testgen-app/templates/deployment.yaml index 2a98c24f..489e18bd 100644 --- a/deploy/charts/testgen-app/templates/deployment.yaml +++ b/deploy/charts/testgen-app/templates/deployment.yaml @@ -44,14 +44,6 @@ spec: {{- toYaml .Values.resources | nindent 12 }} env: {{- include "testgen.environment" . | nindent 12 }} - {{- with .Values.extraVolumeMounts }} - volumeMounts: - {{ toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.extraVolumes }} - volumes: - {{ toYaml . | nindent 8 }} - {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/deploy/charts/testgen-app/templates/job.yaml b/deploy/charts/testgen-app/templates/job.yaml index b40f19f4..f19c44c5 100644 --- a/deploy/charts/testgen-app/templates/job.yaml +++ b/deploy/charts/testgen-app/templates/job.yaml @@ -40,15 +40,7 @@ spec: env: {{- include "testgen.environment" $ | nindent 12 }} {{- include "testgen.hookEnvironment" $ | nindent 12 }} - {{- with .Values.extraVolumeMounts }} - volumeMounts: - {{ toYaml . | nindent 12 }} - {{- end }} command: {{ toJson .command }} - {{- with .Values.extraVolumes }} - volumes: - {{ toYaml . | nindent 8 }} - {{- end }} {{- with $.Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/deploy/charts/testgen-app/templates/secrets.yaml b/deploy/charts/testgen-app/templates/secrets.yaml index fdf4fa00..17227a69 100644 --- a/deploy/charts/testgen-app/templates/secrets.yaml +++ b/deploy/charts/testgen-app/templates/secrets.yaml @@ -1,12 +1,13 @@ -{{- if .Values.testgen.databaseDecryptSecret.create -}} +{{- if .Values.testgen.authSecrets.create -}} apiVersion: v1 kind: Secret metadata: - name: {{ .Values.testgen.databaseDecryptSecret.name | quote }} + name: {{ .Values.testgen.authSecrets.name | quote }} labels: {{- include "testgen.labels" . | nindent 4 }} annotations: "helm.sh/hook": "pre-install" + "helm.sh/hook-weight": "-1" type: Opaque data: decrypt-salt: {{ randAlphaNum 32 | b64enc | quote }} diff --git a/deploy/charts/testgen-app/values.yaml b/deploy/charts/testgen-app/values.yaml index 1a60ccf9..5d8d89db 100644 --- a/deploy/charts/testgen-app/values.yaml +++ b/deploy/charts/testgen-app/values.yaml @@ -6,16 +6,17 @@ testgen: databaseSchema: "tgapp" databaseUser: "postgres" databasePasswordSecret: - name: "testgen-services-postgresql" + name: "postgresql" key: "postgres-password" - databaseDecryptSecret: + authSecrets: create: true - name: testgen-database-decrypt + name: testgen-auth uiUser: "admin" uiPassword: trustTargetDatabaseCertificate: false observabilityVerifySsl: true releaseCheck: true + labels: cliHooks: - name: "setup-db" @@ -32,7 +33,7 @@ image: pullPolicy: IfNotPresent tag: "v3" -imagePullSecrets: [] +imagePullSecrets: nameOverride: "" fullnameOverride: "" @@ -64,7 +65,7 @@ securityContext: {} service: type: NodePort port: 8501 - nodePort: 8501 + nodePort: resources: {} # limits: diff --git a/deploy/docker-bake.hcl b/deploy/docker-bake.hcl index 8ee620ef..2518cfc4 100644 --- a/deploy/docker-bake.hcl +++ b/deploy/docker-bake.hcl @@ -15,6 +15,15 @@ target "testgen-release" { dockerfile = "deploy/testgen.dockerfile" platforms = ["linux/amd64", "linux/arm64"] tags = formatlist("datakitchen/dataops-testgen:%s", split(" ", TESTGEN_LABELS)) + attest = [ + { + type = "provenance", + mode = "max", + }, + { + type = "sbom", + } + ] } target "testgen-qa" { diff --git a/deploy/testgen.dockerfile b/deploy/testgen.dockerfile index be800940..a3f1039b 100644 --- a/deploy/testgen.dockerfile +++ b/deploy/testgen.dockerfile @@ -1,4 +1,4 @@ -ARG TESTGEN_BASE_LABEL=v4 +ARG TESTGEN_BASE_LABEL=v5 FROM datakitchen/dataops-testgen-base:${TESTGEN_BASE_LABEL} AS release-image @@ -9,6 +9,8 @@ ARG TESTGEN_DOCKER_HUB_REPO ENV PYTHONPATH=/dk/lib/python3.12/site-packages ENV PATH=$PATH:/dk/bin +RUN apk upgrade + # Now install everything COPY . /tmp/dk/ RUN python3 -m pip install --prefix=/dk /tmp/dk diff --git a/pyproject.toml b/pyproject.toml index 58c7f37b..56321182 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta" [project] name = "dataops-testgen" -version = "3.7.6" +version = "3.7.8" description = "DataKitchen's Data Quality DataOps TestGen" authors = [ { "name" = "DataKitchen, Inc.", "email" = "info@datakitchen.io" }, diff --git a/testgen/commands/queries/profiling_query.py b/testgen/commands/queries/profiling_query.py index b940fce0..6aba1c8c 100644 --- a/testgen/commands/queries/profiling_query.py +++ b/testgen/commands/queries/profiling_query.py @@ -85,15 +85,15 @@ def _get_data_chars_sql(self) -> CRefreshDataCharsSQL: "profiling_table_set": self.parm_table_set, "profiling_include_mask": self.parm_table_include_mask, "profiling_exclude_mask": self.parm_table_exclude_mask, - } + } self._data_chars_sql = CRefreshDataCharsSQL(params, self.run_date, "v_latest_profile_results") - + return self._data_chars_sql - + def _get_rollup_scores_sql(self) -> CRollupScoresSQL: if not self._rollup_scores_sql: self._rollup_scores_sql = CRollupScoresSQL(self.profile_run_id, self.table_groups_id) - + return self._rollup_scores_sql def ReplaceParms(self, strInputString): @@ -106,7 +106,7 @@ def ReplaceParms(self, strInputString): strInputString = strInputString.replace("{COL_NAME}", self.col_name) strInputString = strInputString.replace("{COL_NAME_SANITIZED}", self.col_name.replace("'", "''")) strInputString = strInputString.replace("{COL_GEN_TYPE}", self.col_gen_type) - strInputString = strInputString.replace("{COL_TYPE}", self.col_type) + strInputString = strInputString.replace("{COL_TYPE}", self.col_type or "") strInputString = strInputString.replace("{COL_POS}", str(self.col_ordinal_position)) strInputString = strInputString.replace("{TOP_FREQ}", self.col_top_freq_update) strInputString = strInputString.replace("{PROFILE_RUN_ID}", self.profile_run_id) @@ -177,7 +177,7 @@ def GetAnomalyStatsRefreshQuery(self): def GetAnomalyScoringRollupRunQuery(self): # Runs on DK Postgres Server return self._get_rollup_scores_sql().GetRollupScoresProfileRunQuery() - + def GetAnomalyScoringRollupTableGroupQuery(self): # Runs on DK Postgres Server return self._get_rollup_scores_sql().GetRollupScoresProfileTableGroupQuery() diff --git a/testgen/commands/run_execute_tests.py b/testgen/commands/run_execute_tests.py index 2ee93ba9..321f59f1 100644 --- a/testgen/commands/run_execute_tests.py +++ b/testgen/commands/run_execute_tests.py @@ -169,9 +169,7 @@ def run_execution_steps(project_code: str, test_suite: str, minutes_offset: int= ): has_errors = True - if has_errors: - error_status = "with errors. Check log for details." - else: - error_status = "successfully." - message = f"Test Execution completed {error_status}" - return message + return f""" + Test execution completed {"with errors. Check log for details." if has_errors else "successfully."} + Run ID: {test_run_id} + """ diff --git a/testgen/commands/run_profiling_bridge.py b/testgen/commands/run_profiling_bridge.py index 3bafb1d4..d72423c3 100644 --- a/testgen/commands/run_profiling_bridge.py +++ b/testgen/commands/run_profiling_bridge.py @@ -246,12 +246,12 @@ def run_profiling_queries(strTableGroupsID, spinner=None): if strTableGroupsID is None: raise ValueError("Table Group ID was not specified") - booErrors = False + has_errors = False LOG.info("CurrentStep: Retrieving Parameters") # Generate UUID for Profile Run ID - strProfileRunID = str(uuid.uuid4()) + profiling_run_id = str(uuid.uuid4()) dctParms = RetrieveProfilingParms(strTableGroupsID) @@ -288,7 +288,7 @@ def run_profiling_queries(strTableGroupsID, spinner=None): clsProfiling.parm_max_freq_length = 25 clsProfiling.parm_do_patterns = "Y" clsProfiling.parm_max_pattern_length = 25 - clsProfiling.profile_run_id = strProfileRunID + clsProfiling.profile_run_id = profiling_run_id clsProfiling.data_schema = dctParms["table_group_schema"] clsProfiling.parm_table_set = dctParms["profiling_table_set"] clsProfiling.parm_table_include_mask = dctParms["profiling_include_mask"] @@ -307,7 +307,7 @@ def run_profiling_queries(strTableGroupsID, spinner=None): RunActionQueryList("DKTG", lstProfileRunQuery) if spinner: spinner.next() - message = "Profiling completed " + try: # Retrieve Column Metadata LOG.info("CurrentStep: Getting DDF from project") @@ -342,7 +342,7 @@ def run_profiling_queries(strTableGroupsID, spinner=None): ) dctSampleTables = {x[0]: [x[1], x[2]] for x in lstSampleTables} if intErrors > 0: - booErrors = True + has_errors = True LOG.warning( f"Errors were encountered retrieving sampling table counts. ({intErrors} errors occurred.) Please check log." ) @@ -358,7 +358,7 @@ def run_profiling_queries(strTableGroupsID, spinner=None): clsProfiling.data_table = dctColumnRecord["table_name"] clsProfiling.col_name = dctColumnRecord["column_name"] clsProfiling.col_type = dctColumnRecord["data_type"] - clsProfiling.profile_run_id = strProfileRunID + clsProfiling.profile_run_id = profiling_run_id clsProfiling.col_is_decimal = dctColumnRecord["is_decimal"] clsProfiling.col_ordinal_position = dctColumnRecord["ordinal_position"] clsProfiling.col_max_char_length = dctColumnRecord["character_maximum_length"] @@ -389,7 +389,7 @@ def run_profiling_queries(strTableGroupsID, spinner=None): "PROJECT", lstQueries, dctParms["max_threads"], spinner ) if intErrors > 0: - booErrors = True + has_errors = True LOG.warning( f"Errors were encountered executing profiling queries. ({intErrors} errors occurred.) Please check log." ) @@ -432,7 +432,7 @@ def run_profiling_queries(strTableGroupsID, spinner=None): "PROJECT", lstQueries, dctParms["max_threads"], spinner ) if intErrors > 0: - booErrors = True + has_errors = True LOG.warning( f"Errors were encountered executing frequency queries. ({intErrors} errors occurred.) Please check log." ) @@ -487,7 +487,7 @@ def run_profiling_queries(strTableGroupsID, spinner=None): else: LOG.info("No columns were selected to profile.") except Exception as e: - booErrors = True + has_errors = True sqlsplit = e.args[0].split("[SQL", 1) errorline = sqlsplit[0].replace("'", "''") if len(sqlsplit) > 0 else "unknown error" clsProfiling.exception_message = f"{type(e).__name__}: {errorline}" @@ -506,9 +506,7 @@ def run_profiling_queries(strTableGroupsID, spinner=None): refresh_date=date_service.parse_now(clsProfiling.run_date), ) - if booErrors: - str_error_status = "with errors. Check log for details." - else: - str_error_status = "successfully." - message += str_error_status - return message + return f""" + Profiling completed {"with errors. Check log for details." if has_errors else "successfully."} + Run ID: {profiling_run_id} + """ diff --git a/testgen/ui/queries/profiling_queries.py b/testgen/ui/queries/profiling_queries.py index f58edf0c..ec42541a 100644 --- a/testgen/ui/queries/profiling_queries.py +++ b/testgen/ui/queries/profiling_queries.py @@ -299,6 +299,7 @@ def get_column_by_condition( """ if include_tags else ""} LEFT JOIN {schema}.profile_results ON ( column_chars.last_complete_profile_run_id = profile_results.profile_run_id + AND column_chars.table_name = profile_results.table_name AND column_chars.column_name = profile_results.column_name ) WHERE {filter_condition};