|
|
本文 ingress 用的是 Nginx-ingress，普通的 Nginx 原理是一样的。容器运行时是 Docker。
ingress 配置
ingress 需要调整一下日志格式，json 日志更有利于分析与处理。
1
2
3
4
5
6
7
8
9
10
| # Source: ingress-nginx/templates/controller-configmap.yaml
apiVersion: v1
kind: ConfigMap
...
name: ingress-nginx-controller
namespace: ingress-nginx
data:
...
# annotation: nginx.ingress.kubernetes.io/client-body-buffer-size: 1M
log-format-upstream: '{"@timestamp": "$time_iso8601","server_addr":"$server_addr","remote_addr":"$remote_addr","scheme":"$scheme","request_method":"$request_method","request_uri": "$request_uri","request_length": "$request_length","uri": "$uri","request_time":$request_time,"body_bytes_sent":$body_bytes_sent,"bytes_sent":$bytes_sent,"status":"$status","upstream_host":"$upstream_addr","domain":"$host","http_referer":"$http_referer","http_user_agent":"$http_user_agent","http_app_id":"$http_app_id","x_forwarded":"$http_x_forwarded_for","up_r_time":"$upstream_response_time","up_status":"$upstream_status","listen_port":"$server_port"}'
|
filebeat 配置
在 Linux 中，一切皆文件，那么 Pod 的控制台日志也是存储在文件里的，如果容器运行时用的是 Docker，默认使用的是 json 日志文件格式 "log-driver": "json-file" 日志在主机上的 /var/lib/docker/containers/xx/xx-json.log, 那么我们可以把这个目录挂载到 filebeat 容器中即可使用 filebeat 来搜集日志，但是这样不方便分辨是哪个容器的日志，Kubernetes 给我们创建了一个目录 /var/log/pods 这里文件目录都是以固定格式来显示的，可以根据文件名来分析是哪个 namespace 的哪个 pod 的日志了。还有个目录是 /var/log/containers/ 这个目录里面的文件是链接到 /var/log/pods/ 里面的日志文件。所以收集日志可以直接从 /var/log/containers 来收集。所以 filebeat 需要只读访问 3 个目录：
- /var/lib/docker/containers/
- /var/log/containers/
- /var/log/pods/
首先在 Kubernetes 环境下，我们认为 Pod 是并不固定的，可能会漂移到任意的节点，那么收集日志需要每台节点上都部署 filebeat 来搜集，那么使用 daemonset 是最合适的。
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
| apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: filebeat
subjects:
- kind: ServiceAccount
name: filebeat
namespace: ops
roleRef:
kind: ClusterRole
name: filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: filebeat
labels:
app: filebeat
rules:
- apiGroups: [""] # "" indicates the core API group
resources:
- namespaces
- pods
verbs:
- get
- watch
- list
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: filebeat
namespace: ops
labels:
app: filebeat
---
kind: ConfigMap
apiVersion: v1
metadata:
name: filebeat-config
namespace: ops
labels:
app: filebeat
data:
filebeat.yml: |-
###################### Filebeat Configuration Example #########################
filebeat.inputs:
- type: log
symlinks: true
enabled: true
json.keys_under_root: true
json.overwrite_keys: true
json.add_error_key: true
tail_files: true
paths:
- /var/log/containers/ingress-nginx-controller-*.log
processors:
- decode_json_fields:
fields: ['log'] # 这里针对的是docker转换后的json日志，原始日志在log字段内。
target: ""
overwrite_keys: false
process_array: false
max_depth: 1
- drop_event:
when:
or:
- regexp:
http_user_agent: 'Go-http-client'
- regexp:
request_uri: '/actuator/info'
fields:
log_topic: "ingress-k8s"
type: "ingress"
# ============================= Filebeat modules ===============================
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
#==================== Elasticsearch template setting ==========================
setup.template.settings:
index.number_of_shards: 1
#============================== Kibana =====================================
# Uncomment and add the necessary Kibana configuration if needed
# setup.kibana:
# host: "http://your-kibana-host:5601"
output.kafka:
enabled: true
hosts: ["10.0.0.3:9092", "10.0.0.4:9092", "10.0.0.5:9092"]
topic: '%{[fields.log_topic]}'
partition.round_robin:
reachable_only: false
required_acks: 1
compression: none
max_message_bytes: 100000
#================================ Processors =====================================
processors:
- add_host_metadata: ~
- add_cloud_metadata: ~
---
kind: DaemonSet
apiVersion: apps/v1
metadata:
name: filebeat
namespace: ops
labels:
app: filebeat
spec:
selector:
matchLabels:
app: filebeat
template:
metadata:
labels:
app: filebeat
spec:
volumes:
- name: config
configMap:
name: filebeat-config
defaultMode: 420
- name: varlibdockercontainers
hostPath:
path: /opt/docker/containers
type: ''
- name: varlog
hostPath:
path: /var/log
type: ''
- name: data
hostPath:
path: /var/lib/filebeat-data
type: DirectoryOrCreate
containers:
- name: filebeat
image: registry.cn-hangzhou.aliyuncs.com/yaml/images:filebeat-7.3.2
args:
- '-c'
- /etc/filebeat.yml
- '-e'
resources:
limits:
cpu: 300m
memory: 200Mi
requests:
cpu: 200m
memory: 100Mi
volumeMounts:
- name: config
readOnly: true
mountPath: /etc/filebeat.yml
subPath: filebeat.yml
- name: data
mountPath: /usr/share/filebeat/data
- name: varlibdockercontainers
readOnly: true
mountPath: /opt/docker/containers
- name: varlog
readOnly: true
mountPath: /var/log
imagePullPolicy: IfNotPresent
securityContext:
runAsUser: 0
restartPolicy: Always
terminationGracePeriodSeconds: 30
dnsPolicy: ClusterFirst
serviceAccountName: filebeat
serviceAccount: filebeat
hostNetwork: true
|
这里的 filebeat 把收集到的日志写入到 Kafka 集群中。如果容器运行时是 containerd，需要做相应的修改。
logstash 配置
logstash 消费 kafka 数据，进行处理后 写入到 Elasticsearch
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
| apiVersion: apps/v1
kind: Deployment
metadata:
name: logstash-ingress-logs
namespace: ops
spec:
replicas: 1
selector:
matchLabels:
app: logstash-ingress-logs
template:
metadata:
labels:
app: logstash-ingress-logs
spec:
containers:
- image: logstash:7.5.1
name: logstash
volumeMounts:
- mountPath: /usr/share/logstash/config/logstash.yml
name: logstash-ingress-logs-conf
readOnly: true
subPath: logstash.yml
- mountPath: /usr/share/logstash/pipeline/logstash.conf
name: logstash-ingress-logs-conf
readOnly: true
subPath: logstash.conf
- mountPath: /usr/share/logstash/config/jvm.options
name: logstash-ingress-logs-conf
readOnly: true
subPath: jvm.options
volumes:
- configMap:
defaultMode: 420
name: logstash-ingress-logs-conf
name: logstash-ingress-logs-conf
---
apiVersion: v1
kind: ConfigMap
metadata:
name: logstash-ingress-logs-conf
namespace: ops
data:
jvm.options: |
-Xms2g
-Xmx3g
-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
-XX:+UseCMSInitiatingOccupancyOnly
-Djava.awt.headless=true
-Dfile.encoding=UTF-8
-Djruby.compile.invokedynamic=true
-Djruby.jit.threshold=0
-Djruby.regexp.interruptible=true
-XX:+HeapDumpOnOutOfMemoryError
-Djava.security.egd=file:/dev/urandom
logstash.yml: 'config.reload.automatic: true'
logstash.conf: |
input {
kafka {
bootstrap_servers => "10.0.0.3:9092,10.0.0.4:9092,10.0.0.5:9092"
topics => ["ingress-k8s"]
codec => "json"
consumer_threads => 3
group_id => "k8s_group"
decorate_events => true
type => "logstash_mixins"
}
}
filter{
mutate{
rename => ["[host][name]", "hostname"]
remove_field => ["ecs","@version","input","host","agent","log"]
convert => {
"status" => "integer"
}
}
}
output {
if [type] == "logstash_mixins" {
elasticsearch {
action => "index"
hosts => ["http://10.0.0.13:9200","http://10.0.0.14:9200","http://10.0.0.15:9200"]
index => "%{[fields][type]}-%{+YYYY.MM.dd}"
user => "elastic"
password => "elastic的密码"
}
}
}
|
写入到 Elasticsearch 后，在 Elasticsearch 就可以看到对应的索引了。 |
|
English
简体中文
繁體中文
한국 사람
日本語
Deutsch
русский
بالعربية
TÜRKÇE
português
คนไทย
french
