Add parquet converter

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## master / unreleased
 
+* [FEATURE] Add a parquet converter. #579
 * [DEPENDENCY] Update quay.io/cortexproject/cortex Docker tag to v1.20.1 #577
 
 ## 3.0.0 / 2025-11-27

README.md

@@ -543,6 +543,50 @@ Kubernetes: `^1.19.0-0`
 | overrides_exporter.​terminationGracePeriodSeconds | int | `180` |  |
 | overrides_exporter.​tolerations | list | `[]` |  |
 | overrides_exporter.​topologySpreadConstraints | list | `[]` |  |
+| parquet_converter.​affinity.​podAntiAffinity.​preferredDuringSchedulingIgnoredDuringExecution[0].​podAffinityTerm.​labelSelector.​matchExpressions[0].​key | string | `"app.kubernetes.io/component"` |  |
+| parquet_converter.​affinity.​podAntiAffinity.​preferredDuringSchedulingIgnoredDuringExecution[0].​podAffinityTerm.​labelSelector.​matchExpressions[0].​operator | string | `"In"` |  |
+| parquet_converter.​affinity.​podAntiAffinity.​preferredDuringSchedulingIgnoredDuringExecution[0].​podAffinityTerm.​labelSelector.​matchExpressions[0].​values[0] | string | `"parquet-converter"` |  |
+| parquet_converter.​affinity.​podAntiAffinity.​preferredDuringSchedulingIgnoredDuringExecution[0].​podAffinityTerm.​topologyKey | string | `"kubernetes.io/hostname"` |  |
+| parquet_converter.​affinity.​podAntiAffinity.​preferredDuringSchedulingIgnoredDuringExecution[0].​weight | int | `100` |  |
+| parquet_converter.​annotations | object | `{}` |  |
+| parquet_converter.​containerSecurityContext.​enabled | bool | `true` |  |
+| parquet_converter.​containerSecurityContext.​readOnlyRootFilesystem | bool | `true` |  |
+| parquet_converter.​enabled | bool | `false` | https://cortexmetrics.io/docs/guides/parquet-mode/ |
+| parquet_converter.​env | list | `[]` |  |
+| parquet_converter.​extraArgs | object | `{}` | Additional Cortex container arguments, e.g. log.level (debug, info, warn, error) |
+| parquet_converter.​extraContainers | list | `[]` |  |
+| parquet_converter.​extraPorts | list | `[]` |  |
+| parquet_converter.​extraVolumeMounts | list | `[]` |  |
+| parquet_converter.​extraVolumes | list | `[]` |  |
+| parquet_converter.​initContainers | list | `[]` |  |
+| parquet_converter.​lifecycle | object | `{}` |  |
+| parquet_converter.​livenessProbe.​httpGet.​path | string | `"/ready"` |  |
+| parquet_converter.​livenessProbe.​httpGet.​port | string | `"http-metrics"` |  |
+| parquet_converter.​nodeSelector | object | `{}` |  |
+| parquet_converter.​podAnnotations | object | `{"prometheus.io/port":"http-metrics","prometheus.io/scrape":"true"}` | Pod Annotations |
+| parquet_converter.​podLabels | object | `{}` | Pod Labels |
+| parquet_converter.​readinessProbe.​httpGet.​path | string | `"/ready"` |  |
+| parquet_converter.​readinessProbe.​httpGet.​port | string | `"http-metrics"` |  |
+| parquet_converter.​replicas | int | `1` |  |
+| parquet_converter.​resources | object | `{}` |  |
+| parquet_converter.​securityContext | object | `{}` |  |
+| parquet_converter.​service.​annotations | object | `{}` |  |
+| parquet_converter.​service.​labels | object | `{}` |  |
+| parquet_converter.​serviceMonitor.​additionalLabels | object | `{}` |  |
+| parquet_converter.​serviceMonitor.​enabled | bool | `false` |  |
+| parquet_converter.​serviceMonitor.​extraEndpointSpec | object | `{}` | Additional endpoint configuration https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint |
+| parquet_converter.​serviceMonitor.​metricRelabelings | list | `[]` |  |
+| parquet_converter.​serviceMonitor.​podTargetLabels | list | `[]` |  |
+| parquet_converter.​serviceMonitor.​relabelings | list | `[]` |  |
+| parquet_converter.​startupProbe.​failureThreshold | int | `10` |  |
+| parquet_converter.​startupProbe.​httpGet.​path | string | `"/ready"` |  |
+| parquet_converter.​startupProbe.​httpGet.​port | string | `"http-metrics"` |  |
+| parquet_converter.​strategy.​rollingUpdate.​maxSurge | int | `0` |  |
+| parquet_converter.​strategy.​rollingUpdate.​maxUnavailable | int | `1` |  |
+| parquet_converter.​strategy.​type | string | `"RollingUpdate"` |  |
+| parquet_converter.​terminationGracePeriodSeconds | int | `180` |  |
+| parquet_converter.​tolerations | list | `[]` |  |
+| parquet_converter.​topologySpreadConstraints | list | `[]` |  |
 | purger.​affinity | object | `{}` |  |
 | purger.​annotations | object | `{}` |  |
 | purger.​containerSecurityContext.​enabled | bool | `true` |  |

templates/parquet-converter/_helpers-parquet-converter.tpl

@@ -0,0 +1,23 @@
+
+{{/*
+parquet-converter fullname
+*/}}
+{{- define "cortex.parquetConverterFullname" -}}
+{{ include "cortex.fullname" . }}-parquet-converter
+{{- end }}
+
+{{/*
+parquet-converter common labels
+*/}}
+{{- define "cortex.parquetConverterLabels" -}}
+{{ include "cortex.labels" . }}
+app.kubernetes.io/component: parquet-converter
+{{- end }}
+
+{{/*
+parquet-converter selector labels
+*/}}
+{{- define "cortex.parquetConverterSelectorLabels" -}}
+{{ include "cortex.selectorLabels" . }}
+app.kubernetes.io/component: parquet-converter
+{{- end }}

templates/parquet-converter/parquet-converter-dep.yaml

@@ -0,0 +1,113 @@
+{{- if .Values.parquet_converter.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "cortex.parquetConverterFullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "cortex.parquetConverterLabels" . | nindent 4 }}
+  annotations:
+    {{- toYaml .Values.parquet_converter.annotations | nindent 4 }}
+spec:
+  replicas: {{ .Values.parquet_converter.replicas }}
+  selector:
+    matchLabels:
+      {{- include "cortex.parquetConverterSelectorLabels" . | nindent 6 }}
+  strategy:
+    {{- toYaml .Values.parquet_converter.strategy | nindent 4 }}
+  template:
+    metadata:
+      labels:
+        {{- include "cortex.parquetConverterLabels" . | nindent 8 }}
+        {{- with .Values.parquet_converter.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      annotations:
+      {{- if .Values.useExternalConfig }}
+        checksum/config: {{ .Values.externalConfigVersion }}
+      {{- else }}
+        checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
+      {{- end }}
+      {{- with .Values.parquet_converter.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    spec:
+      serviceAccountName: {{ template "cortex.serviceAccountName" . }}
+      {{- if .Values.parquet_converter.priorityClassName }}
+      priorityClassName: {{ .Values.parquet_converter.priorityClassName }}
+      {{- end }}
+      {{- if .Values.parquet_converter.securityContext.enabled }}
+      securityContext: {{- omit .Values.parquet_converter.securityContext "enabled" | toYaml | nindent 8 }}
+      {{- end }}
+      initContainers:
+        {{- toYaml .Values.parquet_converter.initContainers | nindent 8 }}
+      {{- if .Values.image.pullSecrets }}
+      imagePullSecrets:
+      {{- range .Values.image.pullSecrets }}
+        - name: {{ . }}
+      {{- end }}
+      {{- end }}
+      containers:
+        - name: parquet-converter
+          image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args:
+            - "-target=parquet-converter"
+            - "-config.file=/etc/cortex/cortex.yaml"
+          {{- range $key, $value := .Values.parquet_converter.extraArgs }}
+            - "-{{ $key }}={{ $value }}"
+          {{- end }}
+          volumeMounts:
+            {{- if .Values.parquet_converter.extraVolumeMounts }}
+            {{- toYaml .Values.parquet_converter.extraVolumeMounts | nindent 12 }}
+            {{- end }}
+            - name: config
+              mountPath: /etc/cortex
+            - name: runtime-config
+              mountPath: /etc/cortex-runtime-config
+          ports:
+            - name: http-metrics
+              containerPort: {{ .Values.config.server.http_listen_port }}
+              protocol: TCP
+          startupProbe:
+            {{- toYaml .Values.parquet_converter.startupProbe | nindent 12 }}
+          livenessProbe:
+            {{- toYaml .Values.parquet_converter.livenessProbe | nindent 12 }}
+          readinessProbe:
+            {{- toYaml .Values.parquet_converter.readinessProbe | nindent 12 }}
+          resources:
+            {{- toYaml .Values.parquet_converter.resources | nindent 12 }}
+          {{- if .Values.parquet_converter.containerSecurityContext.enabled }}
+          securityContext: {{- omit .Values.parquet_converter.containerSecurityContext "enabled" | toYaml | nindent 12 }}
+          {{- end }}
+          {{- if .Values.parquet_converter.env }}
+          env:
+            {{- toYaml .Values.parquet_converter.env | nindent 12 }}
+          {{- end }}
+          {{- with .Values.parquet_converter.lifecycle }}
+          lifecycle:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        {{- if .Values.parquet_converter.extraContainers }}
+        {{- toYaml .Values.parquet_converter.extraContainers | nindent 8 }}
+        {{- end }}
+      nodeSelector:
+        {{- toYaml .Values.parquet_converter.nodeSelector | nindent 8 }}
+      {{- if .Values.parquet_converter.topologySpreadConstraints }}
+      topologySpreadConstraints:
+      {{- toYaml .Values.parquet_converter.topologySpreadConstraints | nindent 8}}
+      {{- end }}
+      affinity:
+        {{- toYaml .Values.parquet_converter.affinity | nindent 8 }}
+      tolerations:
+        {{- toYaml .Values.parquet_converter.tolerations | nindent 8 }}
+      terminationGracePeriodSeconds: {{ .Values.parquet_converter.terminationGracePeriodSeconds }}
+      volumes:
+        {{- include "cortex.configVolume" . | nindent 8 }}
+        - name: runtime-config
+          configMap:
+            name: {{ template "cortex.fullname" . }}-runtime-config
+        {{- if .Values.parquet_converter.extraVolumes }}
+        {{- toYaml .Values.parquet_converter.extraVolumes | nindent 8 }}
+        {{- end }}
+{{- end }}

templates/parquet-converter/parquet-converter-servicemonitor.yaml

@@ -0,0 +1,48 @@
+{{- if and .Values.parquet_converter.enabled .Values.parquet_converter.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "cortex.parquetConverterFullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "cortex.parquetConverterLabels" . | nindent 4 }}
+    {{- if .Values.parquet_converter.serviceMonitor.additionalLabels }}
+{{ toYaml .Values.parquet_converter.serviceMonitor.additionalLabels | indent 4 }}
+    {{- end }}
+  {{- if .Values.parquet_converter.serviceMonitor.annotations }}
+  annotations:
+{{ toYaml .Values.parquet_converter.serviceMonitor.annotations | indent 4 }}
+  {{- end }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "cortex.parquetConverterSelectorLabels" . | nindent 6 }}
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace | quote }}
+  {{- if .Values.parquet_converter.serviceMonitor.podTargetLabels }}
+  podTargetLabels:
+  {{- range .Values.parquet_converter.serviceMonitor.podTargetLabels }}
+    - {{ . }}
+  {{- end }}
+  {{- end }}
+  endpoints:
+    - port: http-metrics
+    {{- if .Values.parquet_converter.serviceMonitor.interval }}
+      interval: {{ .Values.parquet_converter.serviceMonitor.interval }}
+    {{- end }}
+    {{- if .Values.parquet_converter.serviceMonitor.scrapeTimeout }}
+      scrapeTimeout: {{ .Values.parquet_converter.serviceMonitor.scrapeTimeout }}
+    {{- end }}
+    {{- if .Values.parquet_converter.serviceMonitor.relabelings }}
+      relabelings:
+    {{- toYaml .Values.parquet_converter.serviceMonitor.relabelings | nindent 4 }}
+    {{- end }}
+    {{- if .Values.parquet_converter.serviceMonitor.metricRelabelings }}
+      metricRelabelings:
+    {{- toYaml .Values.parquet_converter.serviceMonitor.metricRelabelings | nindent 4 }}
+    {{- end }}
+    {{- with .Values.parquet_converter.serviceMonitor.extraEndpointSpec }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+{{- end }}

templates/parquet-converter/parquet-converter-svc.yaml

@@ -0,0 +1,23 @@
+{{- if .Values.parquet_converter.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "cortex.parquetConverterFullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "cortex.parquetConverterLabels" . | nindent 4 }}
+    {{- with .Values.parquet_converter.service.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  annotations:
+    {{- toYaml .Values.parquet_converter.service.annotations | nindent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+    - port: {{ .Values.config.server.http_listen_port }}
+      protocol: TCP
+      name: http-metrics
+      targetPort: http-metrics
+  selector:
+    {{- include "cortex.parquetConverterSelectorLabels" . | nindent 4 }}
+{{- end }}

values.yaml

@@ -1114,6 +1114,91 @@ overrides_exporter:
   env: []
   lifecycle: {}
 
+parquet_converter:
+  # -- https://cortexmetrics.io/docs/guides/parquet-mode/
+  enabled: false
+  replicas: 1
+
+  service:
+    annotations: {}
+    labels: {}
+
+  serviceMonitor:
+    enabled: false
+    additionalLabels: {}
+    relabelings: []
+    metricRelabelings: []
+    # -- Additional endpoint configuration https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint
+    extraEndpointSpec: {}
+    podTargetLabels: []
+
+  resources: {}
+
+  # -- Additional Cortex container arguments, e.g. log.level (debug, info, warn, error)
+  extraArgs: {}
+
+  # -- Pod Labels
+  podLabels: {}
+
+  # -- Pod Annotations
+  podAnnotations:
+    prometheus.io/scrape: 'true'
+    prometheus.io/port: 'http-metrics'
+
+  nodeSelector: {}
+  topologySpreadConstraints: []
+  affinity:
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+        - weight: 100
+          podAffinityTerm:
+            labelSelector:
+              matchExpressions:
+                - key: app.kubernetes.io/component
+                  operator: In
+                  values:
+                    - parquet-converter
+            topologyKey: 'kubernetes.io/hostname'
+
+  annotations: {}
+
+  startupProbe:
+    httpGet:
+      path: /ready
+      port: http-metrics
+    failureThreshold: 10
+  livenessProbe:
+    httpGet:
+      path: /ready
+      port: http-metrics
+  readinessProbe:
+    httpGet:
+      path: /ready
+      port: http-metrics
+
+  securityContext: {}
+  containerSecurityContext:
+    enabled: true
+    readOnlyRootFilesystem: true
+
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+
+  terminationGracePeriodSeconds: 180
+
+  tolerations: []
+
+  initContainers: []
+  extraContainers: []
+  extraVolumes: []
+  extraVolumeMounts: []
+  extraPorts: []
+  env: []
+  lifecycle: {}
+
 purger:
   enabled: false
   replicas: 1