|
|
@@ -1,87 +1,137 @@
|
|
|
-apiVersion: v1
|
|
|
-kind: ConfigMap
|
|
|
-metadata:
|
|
|
- name: vector-config
|
|
|
- namespace: {{ .Release.Namespace }}
|
|
|
- labels:
|
|
|
- app: vector
|
|
|
-data:
|
|
|
- vector.toml: |
|
|
|
- [sources.kubernetes_logs]
|
|
|
- type = "kubernetes_logs"
|
|
|
- read_from = "beginning"
|
|
|
- fingerprint_lines = 0
|
|
|
- max_read_bytes = 262144
|
|
|
- glob_minimum_cooldown_ms = 100
|
|
|
- max_line_bytes = 102400
|
|
|
- ignore_older_secs = 0
|
|
|
- data_dir = "/var/lib/vector"
|
|
|
-
|
|
|
- [transforms.parse_and_enrich]
|
|
|
- type = "remap"
|
|
|
- inputs = ["kubernetes_logs"]
|
|
|
- drop_on_abort = false
|
|
|
- source = """
|
|
|
- # Extract Kubernetes metadata
|
|
|
- .pod_name = .kubernetes.pod_name
|
|
|
- .namespace = .kubernetes.namespace_name
|
|
|
- .container = .kubernetes.container_name
|
|
|
-
|
|
|
- # Initialize app with unknown
|
|
|
- .app = "unknown"
|
|
|
-
|
|
|
- # Debug trace fields
|
|
|
- .debug_pod = .kubernetes.pod_name
|
|
|
- .debug_k8s_label_app = .kubernetes.labels.app
|
|
|
-
|
|
|
- # Check if this is a JSON log (Java/Logstash format with nested JSON)
|
|
|
- if exists(.log) {
|
|
|
- log_str = string!(.log)
|
|
|
- .debug_log_length = length(log_str)
|
|
|
- .debug_has_timestamp = contains(log_str, "@timestamp")
|
|
|
-
|
|
|
- if contains(log_str, "@timestamp") {
|
|
|
- # Try to parse the nested JSON
|
|
|
- parsed = parse_json(log_str) ?? null
|
|
|
- if parsed != null {
|
|
|
- .debug_parse_success = true
|
|
|
- if exists(parsed.app) {
|
|
|
- .app = parsed.app
|
|
|
- .debug_source = "parsed_json"
|
|
|
- .debug_parsed_app = parsed.app
|
|
|
- } else {
|
|
|
- .debug_parse_no_app = "app_field_missing_in_parsed_json"
|
|
|
- }
|
|
|
- } else {
|
|
|
- .debug_parse_success = false
|
|
|
- .debug_parse_error = "parse_json_returned_null"
|
|
|
- }
|
|
|
- } else {
|
|
|
- .debug_no_timestamp = "log_does_not_contain_@timestamp"
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- # Fallback to Kubernetes labels if we still don't have app name
|
|
|
- if .app == "unknown" && exists(.kubernetes.labels.app) {
|
|
|
- .app = .kubernetes.labels.app
|
|
|
- .debug_source = "k8s_labels"
|
|
|
- .debug_fallback_app = .kubernetes.labels.app
|
|
|
- }
|
|
|
-
|
|
|
- .debug_final_app = .app
|
|
|
- .env = "kubernetes"
|
|
|
- """
|
|
|
-
|
|
|
- [sinks.loki]
|
|
|
- type = "loki"
|
|
|
- inputs = ["parse_and_enrich"]
|
|
|
- endpoint = "{{ .Values.vector.loki.endpoint }}"
|
|
|
- encoding.codec = "json"
|
|
|
- buffer.type = "memory"
|
|
|
- buffer.max_events = 10000
|
|
|
- # Use only templating for labels - Vector Loki sink needs raw field access
|
|
|
- [sinks.loki.labels]
|
|
|
- namespace = "{{ namespace }}"
|
|
|
- app = "{{ app }}"
|
|
|
- pod = "{{ pod_name }}"
|
|
|
- container = "{{ container }}"
|
|
|
+apiVersion: v1
|
|
|
+kind: ConfigMap
|
|
|
+metadata:
|
|
|
+ name: vector-config
|
|
|
+ namespace: {{ .Release.Namespace }}
|
|
|
+ labels:
|
|
|
+ app: vector
|
|
|
+data:
|
|
|
+ vector.toml: |
|
|
|
+ data_dir = "/var/lib/vector"
|
|
|
+
|
|
|
+ [sources.kubernetes_logs]
|
|
|
+ type = "kubernetes_logs"
|
|
|
+ read_from = "end"
|
|
|
+
|
|
|
+ [transforms.parse_and_enrich]
|
|
|
+ type = "remap"
|
|
|
+ inputs = ["kubernetes_logs"]
|
|
|
+ drop_on_abort = false
|
|
|
+ source = """
|
|
|
+ log_line = .log
|
|
|
+ if log_line == null { log_line = .message }
|
|
|
+ if log_line == null { log_line = .msg }
|
|
|
+ log_line = to_string(log_line) ?? ""
|
|
|
+
|
|
|
+ parsed = null
|
|
|
+ if starts_with(log_line, "{") {
|
|
|
+ parsed = parse_json(log_line) ?? null
|
|
|
+ }
|
|
|
+
|
|
|
+ if parsed != null && is_object(parsed) {
|
|
|
+ msg_val = parsed.message
|
|
|
+ if msg_val == null { msg_val = parsed.msg }
|
|
|
+ if msg_val != null { .message = msg_val }
|
|
|
+
|
|
|
+ if parsed.level != null { .level = parsed.level }
|
|
|
+ if parsed.app != null { .app = parsed.app }
|
|
|
+ if parsed.env != null { .env = parsed.env }
|
|
|
+ if parsed.event_class != null { .event_class = parsed.event_class }
|
|
|
+ if parsed.event != null { .event_class = parsed.event }
|
|
|
+ if parsed.uri != null { .uri = parsed.uri }
|
|
|
+ if parsed.uri_group != null { .uri_group = parsed.uri_group }
|
|
|
+ if parsed.status != null { .status = parsed.status }
|
|
|
+ if parsed.duration != null { .duration = parsed.duration }
|
|
|
+ if parsed.traceId != null { .traceId = parsed.traceId }
|
|
|
+ if parsed.userId != null { .userId = parsed.userId }
|
|
|
+ if parsed.orderId != null { .orderId = parsed.orderId }
|
|
|
+ if parsed.error != null { .error = parsed.error }
|
|
|
+ if parsed.logger_name != null { .logger = parsed.logger_name }
|
|
|
+ if parsed.thread_name != null { .thread = parsed.thread_name }
|
|
|
+ }
|
|
|
+
|
|
|
+ if .message == null { .message = log_line }
|
|
|
+
|
|
|
+ app_val = .app
|
|
|
+ if app_val == null { app_val = .kubernetes.labels.app }
|
|
|
+ if app_val == null { app_val = .kubernetes.pod_labels.app }
|
|
|
+ if app_val == null { app_val = .kubernetes.container_name }
|
|
|
+ if app_val == null { app_val = .kubernetes.pod_name }
|
|
|
+ if app_val == null { app_val = "unknown" }
|
|
|
+ .app = app_val
|
|
|
+
|
|
|
+ env_str = to_string(.env) ?? ""
|
|
|
+ if contains(env_str, "prod") {
|
|
|
+ .env = "prod"
|
|
|
+ } else if contains(env_str, "stage") {
|
|
|
+ .env = "stage"
|
|
|
+ } else if contains(env_str, "dev") {
|
|
|
+ .env = "dev"
|
|
|
+ } else if contains(env_str, "local") {
|
|
|
+ .env = "local"
|
|
|
+ } else if env_str != "" {
|
|
|
+ .env = env_str
|
|
|
+ } else {
|
|
|
+ .env = "unknown"
|
|
|
+ }
|
|
|
+
|
|
|
+ uri_str = to_string(.uri) ?? ""
|
|
|
+ if uri_str != "" { .uri = uri_str }
|
|
|
+
|
|
|
+ if !exists(.uri_group) || .uri_group == "" {
|
|
|
+ if uri_str != "" {
|
|
|
+ if starts_with(uri_str, "/order/") {
|
|
|
+ .uri_group = "/order/*"
|
|
|
+ } else if starts_with(uri_str, "/payment/") {
|
|
|
+ .uri_group = "/payment/*"
|
|
|
+ } else if starts_with(uri_str, "/gateway/") {
|
|
|
+ .uri_group = "/gateway/*"
|
|
|
+ } else {
|
|
|
+ .uri_group = "/other"
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ .uri_group = "/other"
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ if !exists(.event_class) || .event_class == "" {
|
|
|
+ if uri_str != "" {
|
|
|
+ if contains(uri_str, "order") {
|
|
|
+ .event_class = "order"
|
|
|
+ } else if contains(uri_str, "payment") {
|
|
|
+ .event_class = "payment"
|
|
|
+ } else if contains(uri_str, "login") {
|
|
|
+ .event_class = "auth"
|
|
|
+ } else {
|
|
|
+ .event_class = "api"
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ .event_class = "api"
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ level_str = to_string(.level) ?? ""
|
|
|
+ if level_str != "" { .level = level_str } else { .level = "INFO" }
|
|
|
+
|
|
|
+ status_str = to_string(.status) ?? ""
|
|
|
+ if status_str != "" { .status = status_str } else { .status = "unknown" }
|
|
|
+
|
|
|
+ duration_str = to_string(.duration) ?? ""
|
|
|
+ duration_str = replace!(duration_str, "ms", "")
|
|
|
+ .duration_ms = to_int(duration_str) ?? null
|
|
|
+ """
|
|
|
+
|
|
|
+ [sinks.loki]
|
|
|
+ type = "loki"
|
|
|
+ inputs = ["parse_and_enrich"]
|
|
|
+ endpoint = "{{ .Values.vector.loki.endpoint }}"
|
|
|
+ encoding.codec = "json"
|
|
|
+ buffer.type = "memory"
|
|
|
+ buffer.max_events = 10000
|
|
|
+ [sinks.loki.labels]
|
|
|
+ env = '{{ "{{ env }}" }}'
|
|
|
+ app = '{{ "{{ app }}" }}'
|
|
|
+ level = '{{ "{{ level }}" }}'
|
|
|
+ event_class = '{{ "{{ event_class }}" }}'
|
|
|
+ uri_group = '{{ "{{ uri_group }}" }}'
|
|
|
+ status = '{{ "{{ status }}" }}'
|
demo-user