Embed Nginx into Prometheus Pod

charts/rancher-monitoring/v0.0.2/charts/prometheus/templates/_helpers.tpl

@@ -21,7 +21,7 @@
   {{- $_ := set $yamls (.Values.additionalScrapeConfigs | toYaml) "" -}}
 {{- end -}}
 {{- if $yamls -}}
-  {{- keys $yamls | join "\n" | b64enc | quote -}}
+  {{- keys $yamls | join "\n" | quote -}}
 {{- end -}}
 {{- end -}}
 
@@ -47,7 +47,7 @@
   {{- $_ := set $yamls (.Values.additionalAlertManagerConfigs | toYaml) "" -}}
 {{- end -}}
 {{- if $yamls -}}
-  {{- keys $yamls | join "\n" | b64enc | quote -}}
+  {{- keys $yamls | join "\n" | quote -}}
 {{- end -}}
 {{- end -}}
 

charts/rancher-monitoring/v0.0.2/charts/prometheus/templates/hooks-configmap.yaml

-apiVersion: v1
-kind: ConfigMap
-metadata:
-  labels:
-    app: {{ template "app.name" . }}
-    chart: {{ template "app.version" . }}
-    heritage: {{ .Release.Service }}
-    release: {{ .Release.Name }}
-  name: {{ template "app.hooks.fullname" . }}
-data:
-  replace-config-by-auth.sh: |-
-    #!/bin/sh
-
-    srcpath="/template/nginx.conf"
-    dstpath="/host/nginx.conf"
-
-    if [ -f $srcpath ] && [ -d /host ]; then
-      token=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
-
-      sed "s/REPLACE_PARAM_AUTHORIZATION/Bearer ${token}/g" $srcpath > $dstpath
-
-      exit 0
-    fi
-
-    exit 1
-
-

charts/rancher-monitoring/v0.0.2/charts/prometheus/templates/nginx-comfigmap.yaml → charts/rancher-monitoring/v0.0.2/charts/prometheus/templates/nginx-configmap.yaml

@@ -7,12 +7,27 @@ metadata:
     chart: {{ template "app.version" . }}
     heritage: {{ .Release.Service }}
     release: {{ .Release.Name }}
-    component: nginx
 data:
-  nginx.conf: |-
+  run-sh.tmpl: |-
+    #!/bin/sh
+    set -e
+
+    srcpath="/nginx/nginx-conf.tmpl"
+    dstpath="/var/run/nginx.conf"
+
+    if ![ -f $srcpath ]; then
+      exit 1
+    fi
+
+    token=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
+    sed "s/REPLACE_PARAM_AUTHORIZATION/Bearer ${token}/g" $srcpath | sed "s/REPLACE_PARAM_IP/${POD_IP}/g" > $dstpath
+    
+    nginx -g "daemon off;" -c /var/run/nginx.conf
+
+  nginx-conf.tmpl: |-
     user                  nginx;
     worker_processes      auto;
-    error_log             /dev/null warn;
+    error_log             /dev/stdout warn;
     pid                   /var/run/nginx.pid;
 
     events {
@@ -57,7 +72,7 @@ data:
           add_header          X-Proxy-Cache $upstream_cache_status;
           add_header          Cache-Control "public";
 
-          proxy_pass          http://prometheus-operated:9090/;
+          proxy_pass          http://REPLACE_PARAM_IP:9090/;
 
           sub_filter_types    text/html;
           sub_filter_once     off;

charts/rancher-monitoring/v0.0.2/charts/prometheus/templates/nginx-deployment.yaml

-apiVersion: {{ template "deployment_api_version" . }}
-kind: Deployment
-metadata:
-  name: {{ template "app.nginx.fullname" . }}
-  labels:
-    app: {{ template "app.name" . }}
-    chart: {{ template "app.version" . }}
-    heritage: {{ .Release.Service }}
-    release: {{ .Release.Name }}
-    component: nginx
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: {{ template "app.name" . }}
-      chart: {{ template "app.version" . }}
-      release: {{ .Release.Name }}
-      component: nginx
-  template:
-    metadata:
-      labels:
-        app: {{ template "app.name" . }}
-        chart: {{ template "app.version" . }}
-        release: {{ .Release.Name }}
-        component: nginx
-    spec:
-      initContainers:
-      - name: nginx-init-auth-add
-        image: {{ template "system_default_registry" . }}{{ .Values.image.proxy.repository }}:{{ .Values.image.proxy.tag }}
-        command:
-        - /usr/bin/replace-config-by-auth.sh
-        volumeMounts:
-        - name: prometheus-static-hooks
-          mountPath: /usr/bin/replace-config-by-auth.sh
-          subPath: replace-config-by-auth.sh
-        - name: prometheus-static-contents
-          mountPath: /host
-        - name: prometheus-nginx-template
-          mountPath: /template
-        {{- if and .Values.resources .Values.resources.inits }}
-        resources:
-{{ toYaml .Values.resources.inits | indent 10 }}
-        {{- end }}
-      containers:
-      - name: prometheus-proxy
-        image: {{ template "system_default_registry" . }}{{ .Values.image.proxy.repository }}:{{ .Values.image.proxy.tag }}
-        args:
-        - nginx
-        - -g
-        - daemon off;
-        - -c
-        - /nginx/nginx.conf
-        volumeMounts:
-        - mountPath: /nginx
-          name: prometheus-static-contents
-        {{- if and .Values.resources .Values.resources.proxy }}
-        resources:
-{{ toYaml .Values.resources.proxy | indent 10 }}
-        {{- end }}
-        ports:
-        - name: http
-          containerPort: 80
-      {{- if .Values.enabledRBAC }}
-      serviceAccountName: {{ default (default (include "app.fullname" .) .Values.serviceAccountName) .Values.serviceAccountNameOverride }}
-      {{- end }}
-      {{- if .Values.tolerations }}
-      tolerations:
-{{ toYaml .Values.tolerations | indent 8 }}
-      {{- end }}
-      nodeSelector:
-        beta.kubernetes.io/os: linux
-        {{- range .Values.nodeSelectors }}
-        {{- $pair := regexSplit "=" . 2 }}
-        {{- if eq 2 (len $pair) }}
-        {{ (index $pair 0) }}: {{ (index $pair 1) }}
-        {{- else }}
-        {{ (index $pair 0) }}: ""
-        {{- end }}
-        {{- end }}
-      volumes:
-      - name: prometheus-static-hooks
-        configMap:
-          name: {{ template "app.hooks.fullname" . }}
-          defaultMode: 0777
-      - name: prometheus-static-contents
-        emptyDir: {}
-      - name: prometheus-nginx-template
-        configMap:
-          name: {{ template "app.nginx.fullname" . }}
-          defaultMode: 438
-          items:
-          - key: nginx.conf
-            mode: 438
-            path: nginx.conf

charts/rancher-monitoring/v0.0.2/charts/prometheus/templates/prometheus.yaml

@@ -12,9 +12,33 @@ spec:
   securityContext:
 {{ toYaml .Values.securityContext | indent 4 }}
 {{- end }}
-{{- if eq .Values.level "cluster" }}
-  listenLocal: true
   containers:
+  - name: prometheus-proxy
+    command:
+    - /bin/sh
+    - -c 
+    - cp /nginx/run-sh.tmpl /var/run/nginx-start.sh; chmod +x /var/run/nginx-start.sh; /var/run/nginx-start.sh
+    env:
+    - name: POD_IP
+      valueFrom:
+        fieldRef:
+          fieldPath: status.podIP
+    image: {{ template "system_default_registry" . }}{{ .Values.image.proxy.repository }}:{{ .Values.image.proxy.tag }}
+    ports:
+    - containerPort: 80
+      name: http
+      protocol: TCP
+    {{- if and .Values.resources .Values.resources.proxy }}
+    resources:
+{{ toYaml .Values.resources.proxy | indent 6 }}
+    {{- end }}
+    securityContext:
+      runAsNonRoot: false
+      runAsUser: 0
+    volumeMounts:
+    - mountPath: /nginx
+      name: configmap-{{ template "app.nginx.fullname" . }}
+{{- if eq .Values.level "cluster" }}
   - name: prometheus-agent
     command:
     - prometheus-auth
@@ -31,9 +55,6 @@ spec:
     - containerPort: 9090
       name: web
       protocol: TCP
-    - containerPort: 9190
-      name: profile
-      protocol: TCP
     livenessProbe:
       failureThreshold: 6
       httpGet:
@@ -56,14 +77,7 @@ spec:
     resources:
 {{ toYaml .Values.resources.auth | indent 6 }}
     {{- end }}
-  {{- if .Values.sidecarsSpec }}
-{{ toYaml .Values.sidecarsSpec | indent 2 }}
-  {{- end }}
-{{- else }} 
-  {{- if .Values.sidecarsSpec }}
-  containers:
-{{ toYaml .Values.sidecarsSpec | indent 2 }}
-  {{- end }}
+  listenLocal: true
 {{- end }}
   podMetadata:
     labels:
@@ -104,9 +118,11 @@ spec:
 {{ toYaml .Values.resources.core | indent 4 }}
 {{- end }}
   retention: "{{ .Values.retention }}"
+  configMaps:
+  - {{ template "app.nginx.fullname" . }}
 {{- if .Values.secrets }}
   secrets:
-{{ toYaml .Values.secrets | indent 4 }}
+{{ toYaml .Values.secrets | indent 2 }}
 {{- end }}
 {{- if .Values.enabledRBAC }}
   serviceAccountName: {{ default (default (include "app.fullname" .) .Values.serviceAccountName) .Values.serviceAccountNameOverride }}

charts/rancher-monitoring/v0.0.2/charts/prometheus/templates/secrets.yaml

@@ -8,7 +8,7 @@ metadata:
     heritage: {{ .Release.Service }}
     release: {{ .Release.Name }}
   name: {{ template "app.fullname" . }}-additional-scrape-configs
-data:
+stringData:
   additional-scrape-configs.yaml: {{ template "additional-scrape-configs.yaml" . }}
 {{- end }}
 
@@ -23,6 +23,6 @@ metadata:
     heritage: {{ .Release.Service }}
     release: {{ .Release.Name }}
   name: {{ template "app.fullname" . }}-additional-alertmanager-configs
-data:
+stringData:
   additional-alertmanager-configs.yaml: {{ template "additional-alertmanager-configs.yaml" . }}
 {{- end }}
\ No newline at end of file

charts/rancher-monitoring/v0.0.2/charts/prometheus/templates/service.yaml

@@ -10,11 +10,11 @@ metadata:
     kubernetes.io/cluster-service: "true"
 spec:
   type: ClusterIP
+  sessionAffinity: ClientIP
   selector:
     app: {{ template "app.name" . }}
     chart: {{ template "app.version" . }}
     release: {{ .Release.Name }}
-    component: nginx
   ports:
     - name: http
       port: 80

charts/rancher-monitoring/v0.0.2/charts/prometheus/values.yaml

@@ -68,7 +68,3 @@ additionalAlertManagerConfigs: []
 # - static_configs:
 #   - targets:
 #     - "localhost:9093"
-
-sidecarsSpec: []
-# - name: sidecar
-#   image: registry/name:tag

charts/rancher-monitoring/v0.0.2/values.yaml

@@ -283,14 +283,10 @@ prometheus:
   level: cluster
   auth:
     args:
-    - --proxy-url
-    - http://localhost:9090
-    - --listen-address
-    - $(POD_IP):9090
-    - --filter-reader-labels
-    - prometheus
-    - --filter-reader-labels
-    - prometheus_replica
+    - --proxy-url=http://localhost:9090
+    - --listen-address=$(POD_IP):9090
+    - --filter-reader-labels=prometheus
+    - --filter-reader-labels=prometheus_replica
     env:
     - name: POD_IP
       valueFrom:
@@ -308,13 +304,6 @@ prometheus:
       tag: 1.15.8-alpine
   nodeSelectors: []
   resources:
-    inits:
-      limits:
-        memory: 50Mi
-        cpu: 50m
-      requests:
-        memory: 50Mi
-        cpu: 50m
     core:
       limits:
         memory: 500Mi