From 54ae6b612a93085142da2872a8df496f603f4e45 Mon Sep 17 00:00:00 2001 From: Ricardo Boni Date: Wed, 2 Apr 2025 14:49:19 -0400 Subject: [PATCH 1/3] fix: ODBC driver was missing --- deploy/install_linuxodbc.sh | 36 ++++++++++++++++++++++++++++++++++ deploy/testgen-base.dockerfile | 10 ++++++++++ deploy/testgen.dockerfile | 10 +++++----- invocations/dev.py | 22 ++++++++++----------- 4 files changed, 61 insertions(+), 17 deletions(-) create mode 100755 deploy/install_linuxodbc.sh diff --git a/deploy/install_linuxodbc.sh b/deploy/install_linuxodbc.sh new file mode 100755 index 00000000..9f585221 --- /dev/null +++ b/deploy/install_linuxodbc.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env sh + +# From: https://learn.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server +# modifications: Added --non-interactive and --no-cache flags, removed sudo, added aarch64 as an alias for arm64 + +architecture="unsupported" + +case "$(uname -m)" in + x86_64) + architecture="amd64" + ;; + arm64|aarch64) + architecture="arm64" + ;; +esac + +if [ "unsupported" = "$architecture" ]; then + echo "Alpine architecture $(uname -m) is not currently supported." + exit 1 +fi + +#Download the desired package(s) +curl -O https://download.microsoft.com/download/7/6/d/76de322a-d860-4894-9945-f0cc5d6a45f8/msodbcsql18_18.4.1.1-1_$architecture.apk +curl -O https://download.microsoft.com/download/7/6/d/76de322a-d860-4894-9945-f0cc5d6a45f8/mssql-tools18_18.4.1.1-1_$architecture.apk + +#(Optional) Verify signature, if 'gpg' is missing install it using 'apk add gnupg': +curl -O https://download.microsoft.com/download/7/6/d/76de322a-d860-4894-9945-f0cc5d6a45f8/msodbcsql18_18.4.1.1-1_$architecture.sig +curl -O https://download.microsoft.com/download/7/6/d/76de322a-d860-4894-9945-f0cc5d6a45f8/mssql-tools18_18.4.1.1-1_$architecture.sig + +curl https://packages.microsoft.com/keys/microsoft.asc | gpg --import - +gpg --verify msodbcsql18_18.4.1.1-1_$architecture.sig msodbcsql18_18.4.1.1-1_$architecture.apk +gpg --verify mssql-tools18_18.4.1.1-1_$architecture.sig mssql-tools18_18.4.1.1-1_$architecture.apk + +#Install the package(s) +apk add --no-cache --non-interactive --allow-untrusted msodbcsql18_18.4.1.1-1_$architecture.apk +apk add --no-cache --non-interactive --allow-untrusted mssql-tools18_18.4.1.1-1_$architecture.apk diff --git a/deploy/testgen-base.dockerfile b/deploy/testgen-base.dockerfile index 9b5f03f6..1c7d7ea7 100644 --- a/deploy/testgen-base.dockerfile +++ b/deploy/testgen-base.dockerfile @@ -15,6 +15,9 @@ RUN apk update && apk upgrade && apk add --no-cache \ musl-dev \ gfortran \ linux-headers=6.6-r0 \ + # Tools needed for installing the MSSQL ODBC drivers \ + curl \ + gpg \ # Additional libraries needed and their dev counterparts. We add both so that we can remove # the *-dev later, keeping the libraries openblas=0.3.28-r0 \ @@ -28,6 +31,9 @@ RUN apk add --no-cache \ libarrow=18.1.0-r0 \ apache-arrow-dev=18.1.0-r0 +COPY --chmod=775 ./deploy/install_linuxodbc.sh /tmp/dk/install_linuxodbc.sh +RUN /tmp/dk/install_linuxodbc.sh + # Install TestGen's main project empty pyproject.toml to install (and cache) the dependencies first COPY ./pyproject.toml /tmp/dk/pyproject.toml RUN mkdir /dk @@ -40,7 +46,11 @@ RUN apk del \ cmake \ musl-dev \ gfortran \ + curl \ + gpg \ linux-headers \ openblas-dev \ unixodbc-dev \ apache-arrow-dev + +RUN rm /tmp/dk/install_linuxodbc.sh diff --git a/deploy/testgen.dockerfile b/deploy/testgen.dockerfile index 840867a1..be800940 100644 --- a/deploy/testgen.dockerfile +++ b/deploy/testgen.dockerfile @@ -1,4 +1,4 @@ -ARG TESTGEN_BASE_LABEL=v3 +ARG TESTGEN_BASE_LABEL=v4 FROM datakitchen/dataops-testgen-base:${TESTGEN_BASE_LABEL} AS release-image @@ -6,6 +6,9 @@ FROM datakitchen/dataops-testgen-base:${TESTGEN_BASE_LABEL} AS release-image ARG TESTGEN_VERSION ARG TESTGEN_DOCKER_HUB_REPO +ENV PYTHONPATH=/dk/lib/python3.12/site-packages +ENV PATH=$PATH:/dk/bin + # Now install everything COPY . /tmp/dk/ RUN python3 -m pip install --prefix=/dk /tmp/dk @@ -17,12 +20,9 @@ RUN addgroup -S testgen && adduser -S testgen -G testgen RUN mkdir /var/lib/testgen RUN chown -R testgen:testgen /var/lib/testgen /dk/lib/python3.12/site-packages/streamlit/static -ENV PYTHONPATH=/dk/lib/python3.12/site-packages -ENV PATH=$PATH:/dk/bin - ENV TESTGEN_VERSION=${TESTGEN_VERSION} -ENV TG_RELEASE_CHECK=docker ENV TESTGEN_DOCKER_HUB_REPO=${TESTGEN_DOCKER_HUB_REPO} +ENV TG_RELEASE_CHECK=docker ENV STREAMLIT_SERVER_MAX_UPLOAD_SIZE=200 USER testgen diff --git a/invocations/dev.py b/invocations/dev.py index 1bfb6e27..ba5cefdc 100644 --- a/invocations/dev.py +++ b/invocations/dev.py @@ -14,8 +14,6 @@ DOCKER_BUILDER_NAME = "dk-builder" DOCKER_BUILDER_PLATFORMS = "linux/amd64,linux/arm64" -TESTGEN_DEFAULT_BASE_LABEL = "v3" - @task def required_tools(ctx: Context) -> None: @@ -91,7 +89,7 @@ def build_public_image( version: str = "", load: bool = False, - base_label: str = TESTGEN_DEFAULT_BASE_LABEL, + base_label: str = "", debug: bool = False ) -> None: """Builds and pushes the TestGen image""" @@ -114,12 +112,12 @@ def build_public_image( if debug: extra_args.append("--print") - ctx.run( - f"docker buildx bake -f deploy/docker-bake.hcl testgen-{target} {' '.join(extra_args)} ", - env={ - "TESTGEN_LABELS": " ".join(label), - "TESTGEN_BASE_LABEL": base_label, - "TESTGEN_VERSION": version, - }, - echo=True, - ) + env={ + "TESTGEN_LABELS": " ".join(label), + "TESTGEN_VERSION": version, + } + if base_label: + env["TESTGEN_BASE_LABEL"] = base_label + + cmd = f"docker buildx bake -f deploy/docker-bake.hcl testgen-{target} {' '.join(extra_args)} " + ctx.run(cmd, env=env, echo=True) From eb4ce5d3a6710f4293e8aaa4acdab0fa83954d05 Mon Sep 17 00:00:00 2001 From: Ricardo Boni Date: Wed, 2 Apr 2025 15:47:01 -0400 Subject: [PATCH 2/3] fix: Fix the custom test validation --- testgen/ui/services/test_definition_service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testgen/ui/services/test_definition_service.py b/testgen/ui/services/test_definition_service.py index c57f7789..7f224f71 100644 --- a/testgen/ui/services/test_definition_service.py +++ b/testgen/ui/services/test_definition_service.py @@ -134,6 +134,7 @@ def validate_test(test_definition): connection["connect_by_key"], connection["private_key"], connection["private_key_passphrase"], + connection["http_path"], sql_query, ) From cd9646d94c9e599158747bbff56f2ead48c2c8f8 Mon Sep 17 00:00:00 2001 From: Ricardo Boni Date: Wed, 2 Apr 2025 17:59:38 -0400 Subject: [PATCH 3/3] release: 3.7.3 -> 3.7.6 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 43c18c60..58c7f37b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta" [project] name = "dataops-testgen" -version = "3.7.1" +version = "3.7.6" description = "DataKitchen's Data Quality DataOps TestGen" authors = [ { "name" = "DataKitchen, Inc.", "email" = "info@datakitchen.io" },