Release/3.7.8

deploy/charts/testgen-app/templates/_environment.yaml

@@ -2,12 +2,12 @@
 - name: TG_DECRYPT_SALT
   valueFrom:
     secretKeyRef:
-      name: {{ .Values.testgen.databaseDecryptSecret.name | quote }}
+      name: {{ .Values.testgen.authSecrets.name | quote }}
       key: "decrypt-salt"
 - name: TG_DECRYPT_PASSWORD
   valueFrom:
     secretKeyRef:
-      name: {{ .Values.testgen.databaseDecryptSecret.name | quote }}
+      name: {{ .Values.testgen.authSecrets.name | quote }}
       key: "decrypt-password"
 - name: TG_METADATA_DB_HOST
   value: {{ .Values.testgen.databaseHost | quote }}

deploy/charts/testgen-app/templates/_helpers.tpl

@@ -34,6 +34,9 @@ Create chart name and version as used by the chart label.
 Common labels
 */}}
 {{- define "testgen.labels" -}}
+{{- with .Values.testgen.labels }}
+{{- . | toYaml }}
+{{- end }}
 helm.sh/chart: {{ include "testgen.chart" . }}
 {{ include "testgen.selectorLabels" . }}
 {{- if .Chart.AppVersion }}

deploy/charts/testgen-app/templates/deployment.yaml

@@ -44,14 +44,6 @@ spec:
             {{- toYaml .Values.resources | nindent 12 }}
           env:
             {{- include "testgen.environment" . | nindent 12 }}
-          {{- with .Values.extraVolumeMounts }}
-          volumeMounts:
-            {{ toYaml . | nindent 12 }}
-          {{- end }}
-      {{- with .Values.extraVolumes }}
-      volumes:
-        {{ toYaml . | nindent 8 }}
-      {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

deploy/charts/testgen-app/templates/job.yaml

@@ -40,15 +40,7 @@ spec:
           env:
             {{- include "testgen.environment" $ | nindent 12 }}
             {{- include "testgen.hookEnvironment" $ | nindent 12 }}
-          {{- with .Values.extraVolumeMounts }}
-          volumeMounts:
-            {{ toYaml . | nindent 12 }}
-          {{- end }}
           command: {{ toJson .command }}
-      {{- with .Values.extraVolumes }}
-      volumes:
-        {{ toYaml . | nindent 8 }}
-      {{- end }}
       {{- with $.Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

deploy/charts/testgen-app/templates/secrets.yaml

@@ -1,12 +1,13 @@
-{{- if .Values.testgen.databaseDecryptSecret.create -}}
+{{- if .Values.testgen.authSecrets.create -}}
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ .Values.testgen.databaseDecryptSecret.name | quote }}
+  name: {{ .Values.testgen.authSecrets.name | quote }}
   labels:
     {{- include "testgen.labels" . | nindent 4 }}
   annotations:
     "helm.sh/hook": "pre-install"
+    "helm.sh/hook-weight": "-1"
 type: Opaque
 data:
   decrypt-salt: {{ randAlphaNum 32 | b64enc | quote }}

deploy/charts/testgen-app/values.yaml

@@ -6,16 +6,17 @@ testgen:
   databaseSchema: "tgapp"
   databaseUser: "postgres"
   databasePasswordSecret:
-    name: "testgen-services-postgresql"
+    name: "postgresql"
     key: "postgres-password"
-  databaseDecryptSecret:
+  authSecrets:
     create: true
-    name: testgen-database-decrypt
+    name: testgen-auth
   uiUser: "admin"
   uiPassword:
   trustTargetDatabaseCertificate: false
   observabilityVerifySsl: true
   releaseCheck: true
+  labels:
 
 cliHooks:
   - name: "setup-db"
@@ -32,7 +33,7 @@ image:
   pullPolicy: IfNotPresent
   tag: "v3"
 
-imagePullSecrets: []
+imagePullSecrets:
 nameOverride: ""
 fullnameOverride: ""
 
@@ -64,7 +65,7 @@ securityContext: {}
 service:
   type: NodePort
   port: 8501
-  nodePort: 8501
+  nodePort:
 
 resources: {}
   # limits:

deploy/docker-bake.hcl

@@ -15,6 +15,15 @@ target "testgen-release" {
   dockerfile = "deploy/testgen.dockerfile"
   platforms = ["linux/amd64", "linux/arm64"]
   tags = formatlist("datakitchen/dataops-testgen:%s", split(" ", TESTGEN_LABELS))
+  attest = [
+    {
+      type = "provenance",
+      mode = "max",
+    },
+    {
+      type = "sbom",
+    }
+  ]
 }
 
 target "testgen-qa" {

deploy/testgen.dockerfile

@@ -1,4 +1,4 @@
-ARG TESTGEN_BASE_LABEL=v4
+ARG TESTGEN_BASE_LABEL=v5
 
 FROM datakitchen/dataops-testgen-base:${TESTGEN_BASE_LABEL} AS release-image
 
@@ -9,6 +9,8 @@ ARG TESTGEN_DOCKER_HUB_REPO
 ENV PYTHONPATH=/dk/lib/python3.12/site-packages
 ENV PATH=$PATH:/dk/bin
 
+RUN apk upgrade
+
 # Now install everything
 COPY . /tmp/dk/
 RUN python3 -m pip install --prefix=/dk /tmp/dk

pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dataops-testgen"
-version = "3.7.6"
+version = "3.7.8"
 description = "DataKitchen's Data Quality DataOps TestGen"
 authors = [
     { "name" = "DataKitchen, Inc.", "email" = "info@datakitchen.io" },

testgen/commands/queries/profiling_query.py

@@ -85,15 +85,15 @@ def _get_data_chars_sql(self) -> CRefreshDataCharsSQL:
                 "profiling_table_set": self.parm_table_set,
                 "profiling_include_mask": self.parm_table_include_mask,
                 "profiling_exclude_mask": self.parm_table_exclude_mask,
-            } 
+            }
             self._data_chars_sql = CRefreshDataCharsSQL(params, self.run_date, "v_latest_profile_results")
-    
+
         return self._data_chars_sql
-    
+
     def _get_rollup_scores_sql(self) -> CRollupScoresSQL:
         if not self._rollup_scores_sql:
             self._rollup_scores_sql = CRollupScoresSQL(self.profile_run_id, self.table_groups_id)
-    
+
         return self._rollup_scores_sql
 
     def ReplaceParms(self, strInputString):
@@ -106,7 +106,7 @@ def ReplaceParms(self, strInputString):
         strInputString = strInputString.replace("{COL_NAME}", self.col_name)
         strInputString = strInputString.replace("{COL_NAME_SANITIZED}", self.col_name.replace("'", "''"))
         strInputString = strInputString.replace("{COL_GEN_TYPE}", self.col_gen_type)
-        strInputString = strInputString.replace("{COL_TYPE}", self.col_type)
+        strInputString = strInputString.replace("{COL_TYPE}", self.col_type or "")
         strInputString = strInputString.replace("{COL_POS}", str(self.col_ordinal_position))
         strInputString = strInputString.replace("{TOP_FREQ}", self.col_top_freq_update)
         strInputString = strInputString.replace("{PROFILE_RUN_ID}", self.profile_run_id)
@@ -177,7 +177,7 @@ def GetAnomalyStatsRefreshQuery(self):
     def GetAnomalyScoringRollupRunQuery(self):
         # Runs on DK Postgres Server
         return self._get_rollup_scores_sql().GetRollupScoresProfileRunQuery()
-    
+
     def GetAnomalyScoringRollupTableGroupQuery(self):
         # Runs on DK Postgres Server
         return self._get_rollup_scores_sql().GetRollupScoresProfileTableGroupQuery()

testgen/commands/run_execute_tests.py

@@ -169,9 +169,7 @@ def run_execution_steps(project_code: str, test_suite: str, minutes_offset: int=
     ):
         has_errors = True
 
-    if has_errors:
-        error_status = "with errors. Check log for details."
-    else:
-        error_status = "successfully."
-    message = f"Test Execution completed {error_status}"
-    return message
+    return f"""
+        Test execution completed {"with errors. Check log for details." if has_errors else "successfully."}
+        Run ID: {test_run_id}
+    """

testgen/commands/run_profiling_bridge.py

@@ -246,12 +246,12 @@ def run_profiling_queries(strTableGroupsID, spinner=None):
     if strTableGroupsID is None:
         raise ValueError("Table Group ID was not specified")
 
-    booErrors = False
+    has_errors = False
 
     LOG.info("CurrentStep: Retrieving Parameters")
 
     # Generate UUID for Profile Run ID
-    strProfileRunID = str(uuid.uuid4())
+    profiling_run_id = str(uuid.uuid4())
 
     dctParms = RetrieveProfilingParms(strTableGroupsID)
 
@@ -288,7 +288,7 @@ def run_profiling_queries(strTableGroupsID, spinner=None):
     clsProfiling.parm_max_freq_length = 25
     clsProfiling.parm_do_patterns = "Y"
     clsProfiling.parm_max_pattern_length = 25
-    clsProfiling.profile_run_id = strProfileRunID
+    clsProfiling.profile_run_id = profiling_run_id
     clsProfiling.data_schema = dctParms["table_group_schema"]
     clsProfiling.parm_table_set = dctParms["profiling_table_set"]
     clsProfiling.parm_table_include_mask = dctParms["profiling_include_mask"]
@@ -307,7 +307,7 @@ def run_profiling_queries(strTableGroupsID, spinner=None):
     RunActionQueryList("DKTG", lstProfileRunQuery)
     if spinner:
         spinner.next()
-    message = "Profiling completed "
+
     try:
         # Retrieve Column Metadata
         LOG.info("CurrentStep: Getting DDF from project")
@@ -342,7 +342,7 @@ def run_profiling_queries(strTableGroupsID, spinner=None):
                 )
                 dctSampleTables = {x[0]: [x[1], x[2]] for x in lstSampleTables}
                 if intErrors > 0:
-                    booErrors = True
+                    has_errors = True
                     LOG.warning(
                         f"Errors were encountered retrieving sampling table counts. ({intErrors} errors occurred.) Please check log."
                     )
@@ -358,7 +358,7 @@ def run_profiling_queries(strTableGroupsID, spinner=None):
                 clsProfiling.data_table = dctColumnRecord["table_name"]
                 clsProfiling.col_name = dctColumnRecord["column_name"]
                 clsProfiling.col_type = dctColumnRecord["data_type"]
-                clsProfiling.profile_run_id = strProfileRunID
+                clsProfiling.profile_run_id = profiling_run_id
                 clsProfiling.col_is_decimal = dctColumnRecord["is_decimal"]
                 clsProfiling.col_ordinal_position = dctColumnRecord["ordinal_position"]
                 clsProfiling.col_max_char_length = dctColumnRecord["character_maximum_length"]
@@ -389,7 +389,7 @@ def run_profiling_queries(strTableGroupsID, spinner=None):
                 "PROJECT", lstQueries, dctParms["max_threads"], spinner
             )
             if intErrors > 0:
-                booErrors = True
+                has_errors = True
                 LOG.warning(
                     f"Errors were encountered executing profiling queries. ({intErrors} errors occurred.) Please check log."
                 )
@@ -432,7 +432,7 @@ def run_profiling_queries(strTableGroupsID, spinner=None):
                         "PROJECT", lstQueries, dctParms["max_threads"], spinner
                     )
                     if intErrors > 0:
-                        booErrors = True
+                        has_errors = True
                         LOG.warning(
                             f"Errors were encountered executing frequency queries. ({intErrors} errors occurred.) Please check log."
                         )
@@ -487,7 +487,7 @@ def run_profiling_queries(strTableGroupsID, spinner=None):
         else:
             LOG.info("No columns were selected to profile.")
     except Exception as e:
-        booErrors = True
+        has_errors = True
         sqlsplit = e.args[0].split("[SQL", 1)
         errorline = sqlsplit[0].replace("'", "''") if len(sqlsplit) > 0 else "unknown error"
         clsProfiling.exception_message = f"{type(e).__name__}: {errorline}"
@@ -506,9 +506,7 @@ def run_profiling_queries(strTableGroupsID, spinner=None):
             refresh_date=date_service.parse_now(clsProfiling.run_date),
         )
 
-        if booErrors:
-            str_error_status = "with errors. Check log for details."
-        else:
-            str_error_status = "successfully."
-        message += str_error_status
-    return message
+    return f"""
+        Profiling completed {"with errors. Check log for details." if has_errors else "successfully."}
+        Run ID: {profiling_run_id}
+    """

testgen/ui/queries/profiling_queries.py

@@ -299,6 +299,7 @@ def get_column_by_condition(
         """ if include_tags else ""}
         LEFT JOIN {schema}.profile_results ON (
             column_chars.last_complete_profile_run_id = profile_results.profile_run_id
+            AND column_chars.table_name = profile_results.table_name
             AND column_chars.column_name = profile_results.column_name
         )
     WHERE {filter_condition};