Commit f2739b92 by Aiwantaozi Committed by Alena Prokharchyk

copy logging chart from 0.0.1

parent 70d482a0
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*~
# Various IDEs
.project
.idea/
*.tmproj
apiVersion: v1
description: Rancher logging helm chart to support logging function in rancher
icon: https://raw.githubusercontent.com/fluent/fluentd-docs/master/public/logo/Fluentd_square.png
name: rancher-logging
version: 0.1.0
appVersion: v1.1.3
home: https://www.fluentd.org/
sources:
- https://www.fluentd.org/
maintainers:
- name: Michelia
email: support@rancher.com
# Rancher Logging
* Installs [Fluentd](https://www.fluentd.org/) and flexvolume log driver to collect container logs in Rancher
## Introduction
This chart bootstraps a [Fluentd](https://www.fluentd.org/) daemonset and a [Log-Aggregator](https://github.com/rancher/log-aggregator) flexvolume on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager.
It's use for sends logs to log target config in rancher.
## Prerequisites
- Rancher 2.1+
\ No newline at end of file
name: fluentd-tester
version: 0.0.1
appVersion: 1.1.3
home: https://www.fluentd.org/
description: A Fluentd Test Helm chart for validate fluentd config
icon: https://raw.githubusercontent.com/fluent/fluentd-docs/master/public/logo/Fluentd_square.png
keywords:
- fluentd
- rancher
- logging
sources:
- https://github.com/helm/charts/stable/fluentd-elasticsearch
maintainers:
- name: michelia
email: support@rancher.com
engine: gotpl
{{/* vim: set filetype=mustache: */}}
{{/*
Expand the name of the chart.
*/}}
{{- define "fluentd-tester.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "fluentd-tester.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- if contains $name .Release.Name -}}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "fluentd-tester.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "fluentd-tester.version" -}}
{{- $name := include "fluentd-tester.name" . -}}
{{- $version := .Chart.Version | replace "+" "_" -}}
{{- printf "%s-%s" $name $version -}}
{{- end -}}
{{/*
Create the name of the service account to use
*/}}
{{- define "fluentd-tester.serviceAccountName" -}}
{{- if .Values.serviceAccount.create -}}
{{ default (include "fluentd-tester.fullname" .) .Values.serviceAccount.name }}
{{- else -}}
{{ default "default" .Values.serviceAccount.name }}
{{- end -}}
{{- end -}}
{{- if .Values.rbac.create -}}
kind: ClusterRole
apiVersion: {{ template "rbac_api_version" . }}
metadata:
name: {{ template "fluentd-tester.fullname" . }}
labels:
app: {{ template "fluentd-tester.name" . }}
chart: {{ template "fluentd-tester.version" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
rules:
- apiGroups:
- ""
resources:
- "namespaces"
- "pods"
verbs:
- "get"
- "watch"
- "list"
{{- end -}}
\ No newline at end of file
{{- if .Values.rbac.create -}}
kind: ClusterRoleBinding
apiVersion: {{ template "rbac_api_version" . }}
metadata:
name: {{ template "fluentd-tester.fullname" . }}
labels:
app: {{ template "fluentd-tester.name" . }}
chart: {{ template "fluentd-tester.version" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
subjects:
- kind: ServiceAccount
name: {{ template "fluentd-tester.serviceAccountName" . }}
namespace: {{ .Release.Namespace }}
roleRef:
kind: ClusterRole
name: {{ template "fluentd-tester.fullname" . }}
apiGroup: rbac.authorization.k8s.io
{{- end -}}
apiVersion: {{ template "deployment_api_version" . }}
kind: Deployment
metadata:
name: {{ template "fluentd-tester.fullname" . }}
labels:
app: {{ template "fluentd-tester.name" . }}
chart: {{ template "fluentd-tester.version" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
{{- if .Values.labels }}
{{ toYaml .Values.labels | indent 4 }}
{{- end }}
spec:
{{- if .Values.updateStrategy }}
updateStrategy:
{{ toYaml .Values.updateStrategy | indent 4 }}
{{- end }}
selector:
matchLabels:
app: {{ template "fluentd-tester.name" . }}
chart: {{ template "fluentd-tester.version" . }}
release: {{ .Release.Name }}
template:
metadata:
labels:
app: {{ template "fluentd-tester.name" . }}
chart: {{ template "fluentd-tester.version" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
{{- if .Values.labels }}
{{ toYaml .Values.labels | indent 4 }}
{{- end }}
{{- if .Values.annotations }}
{{ toYaml .Values.annotations | indent 8 }}
{{- end }}
spec:
serviceAccountName: {{ template "fluentd-tester.fullname" . }}
containers:
- name: "dry-run"
image: {{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}
imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
{{- if .Values.command }}
command: {{ .Values.command }}
{{ end }}
env:
{{- range $key, $value := .Values.env }}
- name: {{ $key }}
value: {{ $value | quote }}
{{- end }}
- name: K8S_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
resources:
{{ toYaml .Values.resources | indent 10 }}
{{- if .Values.extraVolumeMounts }}
{{ toYaml .Values.extraVolumeMounts | indent 8 }}
{{- end }}
{{- if .Values.service }}
ports:
{{- range $port := .Values.service.ports }}
- name: {{ $port.name }}
containerPort: {{ $port.port }}
{{- if $port.protocol }}
protocol: {{ $port.protocol }}
{{- end }}
{{- end }}
{{- end }}
{{- if .Values.extraVolumes }}
{{ toYaml .Values.extraVolumes | indent 6 }}
{{- end }}
{{- if .Values.nodeSelector }}
nodeSelector:
{{ toYaml .Values.nodeSelector | indent 8 }}
{{- end }}
{{- if .Values.tolerations }}
tolerations:
{{ toYaml .Values.tolerations | indent 6 }}
{{- end }}
{{- if .Values.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ template "fluentd-tester.fullname" . }}
labels:
app: {{ template "fluentd-tester.name" . }}
chart: {{ template "fluentd-tester.version" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
{{- end -}}
labels: {}
image:
repository: rancher/fluentd
tag: v0.1.12
pullPolicy: IfNotPresent
resources: {}
# limits:
# cpu: 100m
# memory: 500Mi
# requests:
# cpu: 100m
# memory: 200Mi
# env:
command: '["sh", "-c", "tail -f /dev/null"]'
rbac:
create: true
serviceAccount:
create: true
annotations: {}
# updateStrategy:
# type: RollingUpdate
tolerations:
- effect: NoSchedule
operator: Exists
nodeSelector: {}
\ No newline at end of file
name: fluentd
version: 0.0.1
appVersion: 1.1.3
home: https://www.fluentd.org/
description: A Fluentd Helm chart for Rancher system logging
icon: https://raw.githubusercontent.com/fluent/fluentd-docs/master/public/logo/Fluentd_square.png
keywords:
- fluentd
- rancher
- logging
sources:
- https://github.com/helm/charts/stable/fluentd-elasticsearch
maintainers:
- name: michelia
email: support@rancher.com
engine: gotpl
{{/* vim: set filetype=mustache: */}}
{{/*
Expand the name of the chart.
*/}}
{{- define "fluentd.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "fluentd.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- if contains $name .Release.Name -}}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "fluentd.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "fluentd.version" -}}
{{- $name := include "fluentd.name" . -}}
{{- $version := .Chart.Version | replace "+" "_" -}}
{{- printf "%s-%s" $name $version -}}
{{- end -}}
{{/*
Create the name of the service account to use
*/}}
{{- define "fluentd.serviceAccountName" -}}
{{- if .Values.serviceAccount.create -}}
{{ default (include "fluentd.fullname" .) .Values.serviceAccount.name }}
{{- else -}}
{{ default "default" .Values.serviceAccount.name }}
{{- end -}}
{{- end -}}
{{- if .Values.rbac.create -}}
kind: ClusterRole
apiVersion: {{ template "rbac_api_version" . }}
metadata:
name: {{ template "fluentd.fullname" . }}
labels:
app: {{ template "fluentd.name" . }}
chart: {{ template "fluentd.version" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
rules:
- apiGroups:
- ""
resources:
- "namespaces"
- "pods"
verbs:
- "get"
- "watch"
- "list"
{{- end -}}
\ No newline at end of file
{{- if .Values.rbac.create -}}
kind: ClusterRoleBinding
apiVersion: {{ template "rbac_api_version" . }}
metadata:
name: {{ template "fluentd.fullname" . }}
labels:
app: {{ template "fluentd.name" . }}
chart: {{ template "fluentd.version" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
subjects:
- kind: ServiceAccount
name: {{ template "fluentd.serviceAccountName" . }}
namespace: {{ .Release.Namespace }}
roleRef:
kind: ClusterRole
name: {{ template "fluentd.fullname" . }}
apiGroup: rbac.authorization.k8s.io
{{- end -}}
apiVersion: {{ template "daemonset_api_version" . }}
kind: DaemonSet
metadata:
name: {{ template "fluentd.fullname" . }}
labels:
app: {{ template "fluentd.name" . }}
chart: {{ template "fluentd.version" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
{{- if .Values.labels }}
{{ toYaml .Values.labels | indent 4 }}
{{- end }}
spec:
{{- if .Values.updateStrategy }}
updateStrategy:
{{ toYaml .Values.updateStrategy | indent 4 }}
{{- end }}
selector:
matchLabels:
app: {{ template "fluentd.name" . }}
chart: {{ template "fluentd.version" . }}
release: {{ .Release.Name }}
template:
metadata:
labels:
app: {{ template "fluentd.name" . }}
chart: {{ template "fluentd.version" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
{{- if .Values.labels }}
{{ toYaml .Values.labels | indent 4 }}
{{- end }}
{{- if .Values.annotations }}
{{ toYaml .Values.annotations | indent 8 }}
{{- end }}
spec:
serviceAccountName: {{ template "fluentd.fullname" . }}
containers:
- name: {{ template "fluentd.fullname" . }}
image: {{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}
imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
{{- if .Values.command }}
command: {{ .Values.command }}
{{ end }}
env:
{{- range $key, $value := .Values.env }}
- name: {{ $key }}
value: {{ $value | quote }}
{{- end }}
- name: K8S_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
resources:
{{ toYaml .Values.resources | indent 10 }}
volumeMounts:
- mountPath: /fluentd/etc/config/custom
name: custom
- mountPath: /fluentd/etc/config/precan
name: config
- mountPath: /fluentd/etc/config/entry
name: entry
- mountPath: /fluentd/etc/config/ssl
name: ssl
- mountPath: {{ .Values.cluster.dockerRoot }}
name: dockerroot
- mountPath: /var/log/containers
name: varlogcontainers
- mountPath: /var/log/pods
name: varlogpods
- mountPath: /var/lib/rancher/rke/log
name: rkelog
- mountPath: /var/lib/rancher/log-volumes
name: customlog
- mountPath: /fluentd/log
name: fluentdlog
- name: libsystemddir
mountPath: /host/lib
readOnly: true
{{- if .Values.extraVolumeMounts }}
{{ toYaml .Values.extraVolumeMounts | indent 8 }}
{{- end }}
ports:
{{- range $port := .Values.service.ports }}
- name: {{ $port.name }}
containerPort: {{ $port.port }}
{{- if $port.protocol }}
protocol: {{ $port.protocol }}
{{- end }}
{{- end }}
{{- if .Values.livenessProbe.enabled }}
# Liveness probe is aimed to help in situarions where fluentd
# silently hangs for no apparent reasons until manual restart.
# The idea of this probe is that if fluentd is not queueing or
# flushing chunks for 5 minutes, something is not right. If
# you want to change the fluentd configuration, reducing amount of
# logs fluentd collects, consider changing the threshold or turning
# liveness probe off completely.
livenessProbe:
initialDelaySeconds: 600
periodSeconds: 60
exec:
command:
- '/bin/sh'
- '-c'
- >
LIVENESS_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-300};
STUCK_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-900};
if [ ! -e /fluentd/log/buffer ];
then
exit 1;
fi;
touch -d "${STUCK_THRESHOLD_SECONDS} seconds ago" /tmp/marker-stuck;
if [[ -z "$(find /fluentd/log/buffer -type f -newer /tmp/marker-stuck -print -quit)" ]];
then
rm -rf /fluentd/log/buffer;
exit 1;
fi;
touch -d "${LIVENESS_THRESHOLD_SECONDS} seconds ago" /tmp/marker-liveness;
if [[ -z "$(find /fluentd/log/buffer -type f -newer /tmp/marker-liveness -print -quit)" ]];
then
exit 1;
fi;
{{- end }}
- name: {{ template "fluentd.fullname" . }}-{{ .Values.configmapReload.name }}
image: {{ template "system_default_registry" . }}{{ .Values.configmapReload.image.repository }}:{{ .Values.configmapReload.image.tag }}
imagePullPolicy: "{{ .Values.configmapReload.image.pullPolicy }}"
args:
- --volume-dir=/fluentd/etc/config/custom
- --volume-dir=/fluentd/etc/config/precan
- --volume-dir=/fluentd/etc/config/ssl
- --volume-dir=/fluentd/etc/config/entry
- --webhook-method=GET
- --webhook-url=http://127.0.0.1:24444/api/config.reload
resources:
{{ toYaml .Values.configmapReload.resources | indent 12 }}
volumeMounts:
- mountPath: /fluentd/etc/config/custom
name: custom
- mountPath: /fluentd/etc/config/precan
name: config
- mountPath: /fluentd/etc/config/entry
name: entry
- mountPath: /fluentd/etc/config/ssl
name: ssl
terminationGracePeriodSeconds: 30
volumes:
- hostPath:
path: /var/lib/rancher/fluentd/etc/config/custom
name: custom
- hostPath:
path: {{ .Values.cluster.dockerRoot }}
name: dockerroot
- hostPath:
path: /var/log/containers
name: varlogcontainers
- hostPath:
path: /var/log/pods
name: varlogpods
- hostPath:
path: /var/lib/rancher/rke/log
name: rkelog
- hostPath:
path: /var/lib/rancher/log-volumes
name: customlog
- hostPath:
path: /var/lib/rancher/fluentd/log
name: fluentdlog
- name: config
secret:
secretName: {{ template "fluentd.fullname" . }}
- name: entry
secret:
secretName: {{ template "fluentd.fullname" . }}-entry
- name: ssl
secret:
secretName: {{ template "fluentd.fullname" . }}-ssl
- name: libsystemddir
hostPath:
path: /usr/lib64
{{- if .Values.extraVolumes }}
{{ toYaml .Values.extraVolumes | indent 6 }}
{{- end }}
{{- if .Values.nodeSelector }}
nodeSelector:
{{ toYaml .Values.nodeSelector | indent 8 }}
{{- end }}
{{- if .Values.tolerations }}
tolerations:
{{ toYaml .Values.tolerations | indent 6 }}
{{- end }}
apiVersion: v1
kind: Secret
metadata:
name: {{ template "fluentd.fullname" . }}
labels:
app: {{ template "fluentd.name" . }}
chart: {{ template "fluentd.version" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
type: Opaque
data:
{{- range $key, $value := .Values.secrets.config }}
{{ $key }}: {{ $value | b64enc | quote }}
{{- end }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ template "fluentd.fullname" . }}-entry
labels:
app: {{ template "fluentd.name" . }}-entry
chart: {{ template "fluentd.version" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
kubernetes.io/cluster-service: "true"
type: Opaque
data:
{{- range $key, $value := .Values.secrets.entry }}
{{ $key }}: {{ $value | b64enc | quote }}
{{- end }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ template "fluentd.fullname" . }}-ssl
labels:
app: {{ template "fluentd.name" . }}-ssl
chart: {{ template "fluentd.version" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
kubernetes.io/cluster-service: "true"
type: Opaque
\ No newline at end of file
{{- if .Values.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ template "fluentd.fullname" . }}
labels:
app: {{ template "fluentd.name" . }}
chart: {{ template "fluentd.version" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
{{- end -}}
{{- if .Values.service }}
apiVersion: v1
kind: Service
metadata:
name: {{ template "fluentd.fullname" . }}
labels:
app: {{ template "fluentd.name" . }}
chart: {{ template "fluentd.version" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
spec:
type: {{ .Values.service.type }}
ports:
{{- range $port := .Values.service.ports }}
- name: {{ $port.name }}
port: {{ $port.port }}
targetPort: {{ $port.port }}
{{- if $port.nodePort }}
nodePort: {{ $port.nodePort }}
{{- end }}
{{- if $port.protocol }}
protocol: {{ $port.protocol }}
{{- end }}
{{- end }}
selector:
app: {{ template "fluentd.name" . }}
chart: {{ template "fluentd.version" . }}
{{- end }}
labels: {}
image:
repository: rancher/fluentd
tag: v0.1.13
pullPolicy: IfNotPresent
resources: {}
# limits:
# cpu: 100m
# memory: 500Mi
# requests:
# cpu: 100m
# memory: 200Mi
# env:
command: '["fluentd", "-c", "/fluentd/etc/config/entry/fluent.conf"]'
rbac:
create: true
serviceAccount:
create: true
livenessProbe:
enabled: true
annotations: {}
# updateStrategy:
# type: RollingUpdate
tolerations:
- effect: NoSchedule
operator: Exists
nodeSelector: {}
service:
type: ClusterIP
ports:
- name: metric
port: 24231
targetPort: metric
cluster:
dockerRoot: /var/lib/docker
configmapReload:
name: reloader
image:
repository: jimmidyson/configmap-reload
tag: v0.2.2
pullPolicy: IfNotPresent
resources: {}
secrets:
entry:
fluent.conf: |-
@include /fluentd/etc/config/precan/*.conf
@include /fluentd/etc/config/custom/project/*.conf
@include /fluentd/etc/config/custom/cluster/*.conf
config:
system.conf: |-
<source>
@type prometheus
bind 0.0.0.0
port 24231
metrics_path /metrics
</source>
<source>
@type prometheus_output_monitor
interval 10
<labels>
pod_name ${hostname}
</labels>
</source>
<system>
rpc_endpoint 127.0.0.1:24444
</system>
custom_cluster.conf: |-
<source>
@type tail
path /var/lib/rancher/log-volumes/*/apache2/*/*
pos_file /fluentd/log/custom_cluster_apache2_new.log.pos
tag tmp-cluster-custom.*
format /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)?$/
time_format %d/%b/%Y:%H:%M:%S %z
</source>
<source>
@type tail
path /var/lib/rancher/log-volumes/*/nginx/*/*
pos_file /fluentd/log/custom_cluster_nginx_new.log.pos
tag tmp-cluster-custom.*
format /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)") (?<gzip_ratio>[^ ]*)?$/
time_format %d/%b/%Y:%H:%M:%S %z
</source>
<source>
@type tail
path /var/lib/rancher/log-volumes/*/rfc3164/*/*
pos_file /fluentd/log/custom_cluster_rfc3164_new.log.pos
tag tmp-cluster-custom.*
format /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
time_format "%b %d %H:%M:%S"
</source>
<source>
@type tail
path /var/lib/rancher/log-volumes/*/rfc5424/*/*
pos_file /fluentd/log/custom_cluster_rfc5424_new.log.pos
tag tmp-cluster-custom.*
format /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
time_format "%Y-%m-%dT%H:%M:%S.%L%z"
</source>
<source>
@type tail
path /var/lib/rancher/log-volumes/*/json/*/*
pos_file /fluentd/log/custom_cluster_json_new.log.pos
tag tmp-cluster-custom.*
format json
</source>
<source>
@type tail
path /var/lib/rancher/log-volumes/*/none/*/*
pos_file /fluentd/log/custom_cluster_none_new.log.pos
tag tmp-cluster-custom.*
<parse>
@type none
message_key log
</parse>
</source>
<filter tmp-cluster-custom.**>
@type record_transformer
enable_ruby true
<record>
tag ${tag}
cluster_id ${component = tag.match(Regexp.compile('tmp-cluster-custom\.var\.lib\.rancher\.log-volumes\.(?<pod_id>[^_]+)_(?<volume_name>[^_]+)\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])+)_(?<container_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)')); component['cluster_id']}
cluster_name ${component = tag.match(Regexp.compile('tmp-cluster-custom\.var\.lib\.rancher\.log-volumes\.(?<pod_id>[^_]+)_(?<volume_name>[^_]+)\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])+)_(?<container_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)')); component['cluster_name']}
project_id ${component = tag.match(Regexp.compile('tmp-cluster-custom\.var\.lib\.rancher\.log-volumes\.(?<pod_id>[^_]+)_(?<volume_name>[^_]+)\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])+)_(?<container_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)')); component['project_id']}
project_name ${component = tag.match(Regexp.compile('tmp-cluster-custom\.var\.lib\.rancher\.log-volumes\.(?<pod_id>[^_]+)_(?<volume_name>[^_]+)\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])+)_(?<container_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)')); component['project_name'].gsub('~', '_')}
workload_name ${component = tag.match(Regexp.compile('tmp-cluster-custom\.var\.lib\.rancher\.log-volumes\.(?<pod_id>[^_]+)_(?<volume_name>[^_]+)\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])+)_(?<container_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)')); component['workload_name']}
volume_name ${component = tag.match(Regexp.compile('tmp-cluster-custom\.var\.lib\.rancher\.log-volumes\.(?<pod_id>[^_]+)_(?<volume_name>[^_]+)\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])+)_(?<container_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)')); component['volume_name']}
filename ${component = tag.match(Regexp.compile('tmp-cluster-custom\.var\.lib\.rancher\.log-volumes\.(?<pod_id>[^_]+)_(?<volume_name>[^_]+)\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])+)_(?<container_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)')); component['filename']}
format ${component = tag.match(Regexp.compile('tmp-cluster-custom\.var\.lib\.rancher\.log-volumes\.(?<pod_id>[^_]+)_(?<volume_name>[^_]+)\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])+)_(?<container_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)')); component['format']}
</record>
</filter>
<match tmp-cluster-custom.**>
@type rewrite_tag_filter
<rule>
key tag
pattern tmp-cluster-custom\.var\.lib\.rancher\.log-volumes\.((?<pod_id>[^_]+)_(?<volume_name>[^_]+))\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])*)_(?<container_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)
tag tmp1.cluster-custom.$10_$6_$10_$11
</rule>
</match>
<filter tmp1.cluster-custom.**>
@type kubernetes_metadata
tag_to_kubernetes_name_regexp tmp1\.cluster-custom\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<docker_id>.+)_(?<container_name>.+)$
</filter>
<match tmp1.cluster-custom.**>
@type rewrite_tag_filter
remove_tag_prefix tmp1
<rule>
key tag
pattern tmp-cluster-custom\.var\.lib\.rancher\.log-volumes\.((?<pod_id>[^_]+)_(?<volume_name>[^_]+))\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])*)_(?<container_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)
tag cluster-custom
</rule>
</match>
<filter cluster-custom.**>
@type record_transformer
enable_ruby
remove_keys tag
</filter>
custom_project.conf: |-
<source>
@type tail
path /var/lib/rancher/log-volumes/*/apache2/*/*
pos_file /fluentd/log/custom_project_apache2_new.log.pos
tag tmp-project-custom.*
format /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)?$/
time_format %d/%b/%Y:%H:%M:%S %z
</source>
<source>
@type tail
path /var/lib/rancher/log-volumes/*/nginx/*/*
pos_file /fluentd/log/custom_project_nginx_new.log.pos
tag tmp-project-custom.*
format /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)") (?<gzip_ratio>[^ ]*)?$/
time_format %d/%b/%Y:%H:%M:%S %z
</source>
<source>
@type tail
path /var/lib/rancher/log-volumes/*/rfc3164/*/*
pos_file /fluentd/log/custom_project_rfc3164_new.log.pos
tag tmp-project-custom.*
format /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
time_format "%b %d %H:%M:%S"
</source>
<source>
@type tail
path /var/lib/rancher/log-volumes/*/rfc5424/*/*
pos_file /fluentd/log/custom_project_rfc5424_new.log.pos
tag tmp-project-custom.*
format /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
time_format "%Y-%m-%dT%H:%M:%S.%L%z"
</source>
<source>
@type tail
path /var/lib/rancher/log-volumes/*/json/*/*
pos_file /fluentd/log/custom_project_json_new.log.pos
tag tmp-project-custom.*
format json
</source>
<source>
@type tail
path /var/lib/rancher/log-volumes/*/none/*/*
pos_file /fluentd/log/custom_project_none_new.log.pos
tag tmp-project-custom.*
<parse>
@type none
message_key log
</parse>
</source>
<filter tmp-project-custom.**>
@type record_transformer
enable_ruby true
<record>
tag ${tag}
cluster_id ${component = tag.match(Regexp.compile('tmp-project-custom\.var\.lib\.rancher\.log-volumes\.((?<pod_id>[^_]+)_(?<volume_name>[^_]+))\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])*)_(?<container_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)')); component['cluster_id']}
cluster_name ${component = tag.match(Regexp.compile('tmp-project-custom\.var\.lib\.rancher\.log-volumes\.((?<pod_id>[^_]+)_(?<volume_name>[^_]+))\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])*)_(?<container_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)')); component['cluster_name']}
project_id ${component = tag.match(Regexp.compile('tmp-project-custom\.var\.lib\.rancher\.log-volumes\.((?<pod_id>[^_]+)_(?<volume_name>[^_]+))\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])*)_(?<container_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)')); component['project_id']}
project_name ${component = tag.match(Regexp.compile('tmp-project-custom\.var\.lib\.rancher\.log-volumes\.((?<pod_id>[^_]+)_(?<volume_name>[^_]+))\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])*)_(?<container_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)')); component['project_name'].gsub('~', '_')}
workload_name ${component = tag.match(Regexp.compile('tmp-project-custom\.var\.lib\.rancher\.log-volumes\.((?<pod_id>[^_]+)_(?<volume_name>[^_]+))\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])*)_(?<container_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)')); component['workload_name']}
volume_name ${component = tag.match(Regexp.compile('tmp-project-custom\.var\.lib\.rancher\.log-volumes\.((?<pod_id>[^_]+)_(?<volume_name>[^_]+))\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])*)_(?<container_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)')); component['volume_name']}
filename ${component = tag.match(Regexp.compile('tmp-project-custom\.var\.lib\.rancher\.log-volumes\.((?<pod_id>[^_]+)_(?<volume_name>[^_]+))\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])*)_(?<container_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)')); component['filename']}
format ${component = tag.match(Regexp.compile('tmp-project-custom\.var\.lib\.rancher\.log-volumes\.((?<pod_id>[^_]+)_(?<volume_name>[^_]+))\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])*)_(?<container_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)')); component['format']}
cluster_project ${component = tag.match(Regexp.compile('tmp-project-custom\.var\.lib\.rancher\.log-volumes\.((?<pod_id>[^_]+)_(?<volume_name>[^_]+))\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])*)_(?<container_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)')); "#{component['cluster_id']}:#{component['project_id']}"}
</record>
</filter>
<match tmp-project-custom.**>
@type rewrite_tag_filter
<rule>
key tag
pattern tmp-project-custom\.var\.lib\.rancher\.log-volumes\.((?<pod_id>[^_]+)_(?<volume_name>[^_]+))\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])*)_(?<container_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)
tag tmp1.tmp2-project-custom.$10_$6_$10_$11
</rule>
</match>
<filter tmp1.tmp2-project-custom.**>
@type kubernetes_metadata
tag_to_kubernetes_name_regexp tmp1\.tmp2-project-custom\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<docker_id>.+)_(?<container_name>.+)$
</filter>
<match tmp1.tmp2-project-custom.**>
@type rewrite_tag_filter
remove_tag_prefix tmp1
<rule>
key tag
pattern tmp-project-custom\.var\.lib\.rancher\.log-volumes\.((?<pod_id>[^_]+)_(?<volume_name>[^_]+))\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])*)_(?<container_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)
tag tmp2-project-custom
</rule>
</match>
<match tmp2-project-custom.**>
@type rewrite_tag_filter
<rule>
key cluster_project
pattern (?<cluster_project>.+\:.+)
tag project-custom.$1
</rule>
</match>
<filter project-custom.**>
@type record_transformer
enable_ruby
remove_keys docker,tag
</filter>
legacy_custom_cluster.conf: |-
<source>
@type tail
path /var/lib/rancher/log-volumes/apache2/*/*
pos_file /fluentd/log/custom_cluster_apache2.log.pos
tag tmp2.cluster-custom.*
format /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)?$/
time_format %d/%b/%Y:%H:%M:%S %z
</source>
<source>
@type tail
path /var/lib/rancher/log-volumes/nginx/*/*
pos_file /fluentd/log/custom_cluster_nginx.log.pos
tag tmp2.cluster-custom.*
format /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)") (?<gzip_ratio>[^ ]*)?$/
time_format %d/%b/%Y:%H:%M:%S %z
</source>
<source>
@type tail
path /var/lib/rancher/log-volumes/rfc3164/*/*
pos_file /fluentd/log/custom_cluster_rfc3164.log.pos
tag tmp2.cluster-custom.*
format /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
time_format "%b %d %H:%M:%S"
</source>
<source>
@type tail
path /var/lib/rancher/log-volumes/rfc5424/*/*
pos_file /fluentd/log/custom_cluster_rfc5424.log.pos
tag tmp2.cluster-custom.*
format /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
time_format "%Y-%m-%dT%H:%M:%S.%L%z"
</source>
<source>
@type tail
path /var/lib/rancher/log-volumes/json/*/*
pos_file /fluentd/log/custom_cluster_json.log.pos
tag tmp2.cluster-custom.*
format json
</source>
<filter tmp2.cluster-custom.**>
@type record_transformer
enable_ruby true
<record>
tag ${tag}
cluster_id ${component = tag.match(Regexp.compile('tmp2\.cluster-custom\.var\.lib\.rancher\.log-volumes\.(?<format>[a-z0-9]+)\.(?<cluster_id>.+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<container_name>[^_]+)_(?<volume_name>[^_^.]+)\.(?<filename>[a-z0-9]+\.[a-z0-9]+)$')); component['cluster_id']}
cluster_name ${component = tag.match(Regexp.compile('tmp2\.cluster-custom\.var\.lib\.rancher\.log-volumes\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<container_name>[^_]+)_(?<volume_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)$')); component['cluster_name']}
project_id ${component = tag.match(Regexp.compile('tmp2\.cluster-custom\.var\.lib\.rancher\.log-volumes\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<container_name>[^_]+)_(?<volume_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)$')); component['project_id']}
project_name ${component = tag.match(Regexp.compile('tmp2\.cluster-custom\.var\.lib\.rancher\.log-volumes\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<container_name>[^_]+)_(?<volume_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)$')); component['project_name'].gsub('~', '_')}
workload_name ${component = tag.match(Regexp.compile('tmp2\.cluster-custom\.var\.lib\.rancher\.log-volumes\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<container_name>[^_]+)_(?<volume_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)$')); component['workload_name']}
volume_name ${component = tag.match(Regexp.compile('tmp2\.cluster-custom\.var\.lib\.rancher\.log-volumes\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<container_name>[^_]+)_(?<volume_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)$')); component['volume_name']}
filename ${component = tag.match(Regexp.compile('tmp2\.cluster-custom\.var\.lib\.rancher\.log-volumes\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<container_name>[^_]+)_(?<volume_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)$')); component['filename']}
format ${component = tag.match(Regexp.compile('tmp2\.cluster-custom\.var\.lib\.rancher\.log-volumes\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<container_name>[^_]+)_(?<volume_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)$')); component['format']}
</record>
</filter>
<match tmp2.cluster-custom.**>
@type rewrite_tag_filter
remove_tag_prefix tmp2
<rule>
key tag
pattern tmp2\.cluster-custom\.var\.lib\.rancher\.log-volumes\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<container_name>[^_]+)_(?<volume_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)$
tag cluster-custom
</rule>
</match>
legacy_custom_project.conf: |-
<source>
@type tail
path /var/lib/rancher/log-volumes/apache2/*/*
pos_file /fluentd/log/custom_project_apache2.log.pos
tag tmp2.tmp2-project-custom.*
format /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)?$/
time_format %d/%b/%Y:%H:%M:%S %z
</source>
<source>
@type tail
path /var/lib/rancher/log-volumes/nginx/*/*
pos_file /fluentd/log/custom_project_nginx.log.pos
tag tmp2.tmp2-project-custom.*
format /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)") (?<gzip_ratio>[^ ]*)?$/
time_format %d/%b/%Y:%H:%M:%S %z
</source>
<source>
@type tail
path /var/lib/rancher/log-volumes/rfc3164/*/*
pos_file /fluentd/log/custom_project_rfc3164.log.pos
tag tmp2.tmp2-project-custom.*
format /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
time_format "%b %d %H:%M:%S"
</source>
<source>
@type tail
path /var/lib/rancher/log-volumes/rfc5424/*/*
pos_file /fluentd/log/custom_project_rfc5424.log.pos
tag tmp2.tmp2-project-custom.*
format /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
time_format "%Y-%m-%dT%H:%M:%S.%L%z"
</source>
<source>
@type tail
path /var/lib/rancher/log-volumes/json/*/*
pos_file /fluentd/log/custom_project_json.log.pos
tag tmp2.tmp2-project-custom.*
format json
</source>
<filter tmp2.tmp2-project-custom.**>
@type record_transformer
enable_ruby true
<record>
tag ${tag}
cluster_id ${component = tag.match(Regexp.compile('tmp2\.tmp2-project-custom\.var\.lib\.rancher\.log-volumes\.(?<format>[a-z0-9]+)\.(?<cluster_id>.+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<container_name>[^_]+)_(?<volume_name>[^_^.]+)\.(?<filename>[a-z0-9]+\.[a-z0-9]+)$')); component['cluster_id']}
cluster_name ${component = tag.match(Regexp.compile('tmp2\.tmp2-project-custom\.var\.lib\.rancher\.log-volumes\.(?<format>[a-z0-9]+)\.(?<cluster_id>.+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<container_name>[^_]+)_(?<volume_name>[^_^.]+)\.(?<filename>[a-z0-9]+\.[a-z0-9]+)$')); component['cluster_name']}
project_id ${component = tag.match(Regexp.compile('tmp2\.tmp2-project-custom\.var\.lib\.rancher\.log-volumes\.(?<format>[a-z0-9]+)\.(?<cluster_id>.+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<container_name>[^_]+)_(?<volume_name>[^_^.]+)\.(?<filename>[a-z0-9]+\.[a-z0-9]+)$')); component['project_id']}
project_name ${component = tag.match(Regexp.compile('tmp2\.tmp2-project-custom\.var\.lib\.rancher\.log-volumes\.(?<format>[a-z0-9]+)\.(?<cluster_id>.+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<container_name>[^_]+)_(?<volume_name>[^_^.]+)\.(?<filename>[a-z0-9]+\.[a-z0-9]+)$')); component['project_name']}
workload_name ${component = tag.match(Regexp.compile('tmp2\.tmp2-project-custom\.var\.lib\.rancher\.log-volumes\.(?<format>[a-z0-9]+)\.(?<cluster_id>.+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<container_name>[^_]+)_(?<volume_name>[^_^.]+)\.(?<filename>[a-z0-9]+\.[a-z0-9]+)$')); component['workload_name']}
volume_name ${component = tag.match(Regexp.compile('tmp2\.tmp2-project-custom\.var\.lib\.rancher\.log-volumes\.(?<format>[a-z0-9]+)\.(?<cluster_id>.+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<container_name>[^_]+)_(?<volume_name>[^_^.]+)\.(?<filename>[a-z0-9]+\.[a-z0-9]+)$')); component['volume_name']}
filename ${component = tag.match(Regexp.compile('tmp2\.tmp2-project-custom\.var\.lib\.rancher\.log-volumes\.(?<format>[a-z0-9]+)\.(?<cluster_id>.+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<container_name>[^_]+)_(?<volume_name>[^_^.]+)\.(?<filename>[a-z0-9]+\.[a-z0-9]+)$')); component['filename']}
format ${component = tag.match(Regexp.compile('tmp2\.tmp2-project-custom\.var\.lib\.rancher\.log-volumes\.(?<format>[a-z0-9]+)\.(?<cluster_id>.+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<container_name>[^_]+)_(?<volume_name>[^_^.]+)\.(?<filename>[a-z0-9]+\.[a-z0-9]+)$')); component['format']}
cluster_project ${component = tag.match(Regexp.compile('tmp2\.tmp2-project-custom\.var\.lib\.rancher\.log-volumes\.(?<format>[a-z0-9]+)\.(?<cluster_id>.+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<container_name>[^_]+)_(?<volume_name>[^_^.]+)\.(?<filename>[a-z0-9]+\.[a-z0-9]+)$')); "#{component['cluster_id']}:#{component['project_id']}"}
</record>
</filter>
<match tmp2.tmp2-project-custom.**>
@type rewrite_tag_filter
remove_tag_prefix tmp2
<rule>
key tag
pattern tmp2\.tmp2-project-custom\.var\.lib\.rancher\.log-volumes\.(?<format>[a-z0-9]+)\.(?<cluster_id>[^_]+)_(?<cluster_name>[^_]+)_(?<namespace>[^_]+)_(?<project_id>[^_]+)_(?<project_name>[^_]+)_(?<workload_name>[^_]+)_(?<container_name>[^_]+)_(?<volume_name>[^_^.]+)\.(?<filename>[a-z0-9A-Z]+\.[a-zA-Z0-9]+)$
tag tmp2-project-custom
</rule>
</match>
\ No newline at end of file
name: log-aggregator
version: 0.0.1
appVersion: 0.1.4
home: https://github.com/rancher/log-aggregator
description: Deploy flexvolume driver log-aggregator to collect log.
icon: https://raw.githubusercontent.com/fluent/fluentd-docs/master/public/logo/Fluentd_square.png
keywords:
- fluentd
- flexvolume driver
- logging
maintainers:
- name: michelia
email: support@rancher.com
engine: gotpl
{{/* vim: set filetype=mustache: */}}
{{/*
Expand the name of the chart.
*/}}
{{- define "log-aggregator.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "log-aggregator.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- if contains $name .Release.Name -}}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "log-aggregator.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "log-aggregator.version" -}}
{{- $name := include "log-aggregator.name" . -}}
{{- $version := .Chart.Version | replace "+" "_" -}}
{{- printf "%s-%s" $name $version -}}
{{- end -}}
{{/*
Create the name of the service account to use
*/}}
{{- define "log-aggregator.serviceAccountName" -}}
{{- if .Values.serviceAccount.create -}}
{{ default (include "log-aggregator.fullname" .) .Values.serviceAccount.name }}
{{- else -}}
{{ default "default" .Values.serviceAccount.name }}
{{- end -}}
{{- end -}}
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: {{ template "log-aggregator.fullname" . }}
labels:
app: {{ template "log-aggregator.name" . }}
chart: {{ template "log-aggregator.version" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
{{- if .Values.labels }}
{{ toYaml .Values.labels | indent 4 }}
{{- end }}
spec:
{{- if .Values.updateStrategy }}
updateStrategy:
{{ toYaml .Values.updateStrategy | indent 4 }}
{{- end }}
selector:
matchLabels:
app: {{ template "log-aggregator.name" . }}
chart: {{ template "log-aggregator.version" . }}
release: {{ .Release.Name }}
template:
metadata:
labels:
app: {{ template "log-aggregator.name" . }}
chart: {{ template "log-aggregator.version" . }}
release: {{ .Release.Name }}
spec:
serviceAccountName: {{ template "log-aggregator.fullname" . }}
containers:
- name: log-aggregator
image: {{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}
imagePullPolicy: "{{ .Values.image.pullPolicy }}"
securityContext:
privileged: true
volumeMounts:
- name: flexvolume-driver
mountPath: /flexmnt
{{- if .Values.env }}
env:
{{ toYaml .Values.env | indent 10 }}
{{- end }}
resources:
{{ toYaml .Values.resources | indent 10 }}
terminationGracePeriodSeconds: 10
{{- if .Values.nodeSelector }}
nodeSelector:
{{ toYaml .Values.nodeSelector | indent 8 }}
{{- end }}
tolerations:
{{- if .Values.tolerations }}
{{ toYaml .Values.tolerations | indent 8 }}
{{- end }}
volumes:
- name: flexvolume-driver
hostPath:
path: {{ .Values.flexVolumeDir }}
{{- if .Values.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ template "log-aggregator.fullname" . }}
labels:
app: {{ template "log-aggregator.name" . }}
chart: {{ template "log-aggregator.version" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
kubernetes.io/cluster-service: "true"
{{- end -}}
enabledRBAC: true
labels: {}
## Log Aggregator container image
##
image:
repository: rancher/log-aggregator
tag: v0.1.5
nodeSelector: {}
# updateStrategy:
# type: RollingUpdate
tolerations:
- effect: NoSchedule
operator: Exists
## Resource limits & requests
## Ref: https://kubernetes.io/docs/user-guide/compute-resources/
##
resources: {}
# requests:
# memory: 400Mi
serviceAccount:
# Specifies whether a ServiceAccount should be created
create: true
## flexvolume dir
flexVolumeDir: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
categories:
- Logging
questions:
dependencies:
- name: fluentd
version: 0.0.1
condition: fluentd.enabled
repository: "file://./charts/fluentd/"
- name: log-aggregator
version: 0.0.1
condition: log-aggregator.enabled
repository: "file://./charts/log-aggregator/"
- name: fluentd-tester
version: 0.0.1
condition: fluentd-tester.enabled
repository: "file://./charts/fluentd-tester/"
\ No newline at end of file
{{/* vim: set filetype=mustache: */}}
{{- define "deployment_api_version" -}}
{{- if .Capabilities.APIVersions.Has "apps/v1" -}}
{{- "apps/v1" -}}
{{- else if .Capabilities.APIVersions.Has "apps/v1beta2" -}}
{{- "apps/v1beta1" -}}
{{- else if .Capabilities.APIVersions.Has "apps/v1beta1" -}}
{{- "apps/v1beta1" -}}
{{- else -}}
{{- "extensions/v1beta1" -}}
{{- end -}}
{{- end -}}
{{- define "daemonset_api_version" -}}
{{- if .Capabilities.APIVersions.Has "apps/v1" -}}
{{- "apps/v1" -}}
{{- else if .Capabilities.APIVersions.Has "apps/v1beta2" -}}
{{- "apps/v1beta2" -}}
{{- else -}}
{{- "extensions/v1beta1" -}}
{{- end -}}
{{- end -}}
{{- define "rbac_api_version" -}}
{{- if .Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" -}}
{{- "rbac.authorization.k8s.io/v1" -}}
{{- else if .Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1beta1" -}}
{{- "rbac.authorization.k8s.io/v1beta1" -}}
{{- else -}}
{{- "rbac.authorization.k8s.io/v1alpha1" -}}
{{- end -}}
{{- end -}}
{{- define "system_default_registry" -}}
{{- if .Values.global.systemDefaultRegistry -}}
{{- printf "%s/" .Values.global.systemDefaultRegistry -}}
{{- else -}}
{{- "" -}}
{{- end -}}
{{- end -}}
fluentd:
enabled: false
fluentd-tester:
enabled: false
log-aggregator:
enabled: false
flexVolumeDir: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
global:
systemDefaultRegistry: ""
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment