filebeat的部署及基本使用

1安装filebeat

rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch
yum -y install filebeat

查看启动配置

 systemctl cat filebeat.service 
# /usr/lib/systemd/system/filebeat.service
[Unit]
Description=Filebeat sends log files to Logstash or directly to Elasticsearch.
Documentation=https://www.elastic.co/beats/filebeat
Wants=network-online.target
After=network-online.target

[Service]

Environment="GODEBUG='madvdontneed=1'"
Environment="BEAT_LOG_OPTS="
Environment="BEAT_CONFIG_OPTS=-c /etc/filebeat/filebeat.yml"
Environment="BEAT_PATH_OPTS=--path.home /usr/share/filebeat --path.config /etc/filebeat --path.data /var/lib/filebeat --path.logs /var/log/filebeat"
ExecStart=/usr/share/filebeat/bin/filebeat --environment systemd BEAT_LOG_OPTSBEAT_CONFIG_OPTS $BEAT_PATH_OPTS
Restart=always

2 修改filebeat的配置⽂件

(1)编写测试的配置⽂件
mkdir /etc/filebeat/config
cat > /etc/filebeat/config/01-stdin-to-console.yml <<'EOF'
# 指定输⼊的类型
filebeat.inputs:
# 指定输⼊的类型为"stdin",表示标准输⼊
- type: stdin
# 指定输出的类型(输出到终端)
output.console:
 # 打印漂亮的格式
 pretty: true
EOF
(2)运⾏filebeat实例
filebeat -e -c /etc/filebeat/config/01-stdin-to-console.yml 
(3)测试在终端输入内容

[root@filebeat filebeat[]# filebeat -e -c /etc/filebeat/config/01-stdin-to-console.yml 

1111111111
{
  "@timestamp": "2023-05-07T13:52:30.005Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "_doc",
    "version": "7.17.10"
  },
  "ecs": {
    "version": "1.12.0"
  },
  "host": {
    "name": "filebeat"
  },
  "agent": {
    "id": "733682a1-ac88-444e-ae43-4191ce8260bf",
    "name": "filebeat",
    "type": "filebeat",
    "version": "7.17.10",
    "hostname": "filebeat",
    "ephemeral_id": "3942a421-2905-4646-b040-4cc3bcadca70"
  },
  "log": {
    "offset": 0,
    "file": {
      "path": ""
    }
  },
  "message": "1111111111",
  "input": {
    "type": "stdin"
  }
}
2023-05-07T21:52:31.025+0800    ERROR   file/states.go:125  State for  should have been dropped, but couldn't as state is not finished.

3 input的log类型

log类型的收集默认是按行收集,filebeat会在收集之后记录文件的偏移量,已保证下次不会从头开始读。
[root@filebeat config[]# cat 02-log-to-console.yml 
filebeat.inputs:
- type: log
 paths:
 - /tmp/test.log
output.console:
 pretty: true

记录文件位置
[root@filebeat config[]# ll /var/lib/filebeat/registry/filebeat
总用量 8
-rw-------. 1 root root 632 5月   7 22:03 log.json
-rw-------. 1 root root  15 5月   7 21:52 meta.json

{"k":"filebeat::logs::native::67688859-64768","v":{"type":"log","FileStateOS":{"inode":67688859,"device":64768},"identifier_name":"native","id":"native::67688859-64768","source":"/tmp/test.log","timestamp":[2062526270975,1683469777],"ttl":-1,"prev_id":"","offset":31}}

source:表示读取的文件名
offset: 读取的位置,下次从这个位置开始接着读。

4 filebeat配置多个log输入实例

filebeat.inputs:
- type: log
  paths:
  - /tmp/test.log
  - /tmp/*.txt
  - /tmp/test/*/*.log

output.console:
 pretty: true

5 input的通用字段

5.1 enabled

默认是true,如果设置成false就是不使用。

filebeat.inputs:
 - type: log
   enabled: false #设置false不收集下面日志
   paths:
   - /tmp/test.log
   - /tmp/*.txt
output.console:
 pretty: true

5.2 tag

标记区分不同日志,给当前的输入内容打上标签。
filebeat.inputs:
 - type: log
   enabled: false #设置false不收集下面日志
   paths:
   - /tmp/test.log
   tags: ["项目1","前端日志"]

 - type: log
   enabled: false #设置false不收集下面日志
   paths:
   - /tmp/test2.log
   tags: ["项目二","后端日志","阿里云机房"]

output.console:
 pretty: true

5.3 fields

自定义字段

[root@filebeat config[]# cat 03-log-fields-to-console.yml 
filebeat.inputs:
 - type: log
   paths:
   - /tmp/test.log
   - /tmp/*.txt
   fields:
    log_topics: test
    log_host: 127.0.0.1

output.console:
 pretty: true

测试如下

{
  "@timestamp": "2023-05-07T15:04:13.385Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "_doc",
    "version": "7.17.10"
  },
  "ecs": {
    "version": "1.12.0"
  },
  "host": {
    "name": "filebeat"
  },
  "agent": {
    "name": "filebeat",
    "type": "filebeat",
    "version": "7.17.10",
    "hostname": "filebeat",
    "ephemeral_id": "1d14df6b-f685-4351-a8e2-d396a5e16105",
    "id": "c44cb67e-b0fc-4c34-92a9-44edacd78f96"
  },
  "message": "111122222333333",
  "log": {
    "offset": 15,
    "file": {
      "path": "/tmp/test.log"
    }
  },
  "input": {
    "type": "log"
  },
  "fields": {
    "log_topics": "test",
    "log_host": "127.0.0.1"
  }
}

5.4 fields_under_root

[root@filebeat config[]# cat 03-log-fields-to-console.yml 
filebeat.inputs:
 - type: log
   paths:
   - /tmp/test.log
   - /tmp/*.txt
   fields:
    log_topics: test
    log_host: 127.0.0.1
    # 将⾃定义字段的key-value放到顶级字段.
    # 默认值为false,会将数据放在⼀个叫"fields"字段的下⾯.
   fields_under_root: false
output.console:
 pretty: true

6 日志过滤

filebeat.inputs:
- type: log
  enabled: true
  paths:
  - /tmp/test/*.log
  # 注意,⿊⽩名单均⽀持通配符,⽣产环节中不建议同时使⽤,
  # 指定⽩名单,包含指定的内容才会采集,且区分⼤⼩写!
  include_lines: ['^ERR', '^WARN','devopstack.cn']
  # 指定⿊名单,排除指定的内容
  exclude_lines: ['^DBG',"linux","dev"]
  # 排除以.gz结尾的文件
  exclude_files: ['\.gz$']
output.console:
 pretty: true

7 将数据写入至es

cat 04-log-fields-to-es.yml 
filebeat.inputs:
 - type: log
   paths:
   - /tmp/test.log
   - /tmp/*.txt
   fields:
    log_topics: test
    log_host: 127.0.0.1

output.elasticsearch:
 hosts: ["http://192.168.10.31:9200","http://192.168.10.32:9200","http://192.168.10.33:9200"]
 username: "elastic"
 password: "aka2velfFJGz178SExyi"

启动 
[root@filebeat config[]# filebeat -e -c /etc/filebeat/config/04-log-fields-to-es.yml 

通过查看 kibana,可以看到索引已经在es中创建了,数据也过去了

filebeat的部署及基本使用

8 自定义索引名称

[root@filebeat 22[]# cat /etc/filebeat/config/05-log-fields-to-es.yml 
filebeat.inputs:
 - type: log
   paths:
   - /tmp/test.log
   - /tmp/*.txt
   tags: ["test1"]
   fields:
    name: "app1"
    log_topics: test
    log_host: 127.0.0.1

 - type: log
   enabled: true
   paths:
   - /tmp/test/*/*.log
   tags: ["test2","devopstack"]
   fields:
    name: "app2" 
    hobby: "test111" 
   fields_under_root: true

output.elasticsearch:
 hosts: ["http://192.168.10.31:9200","http://192.168.10.32:9200","http://192.168.10.33:9200"]
 username: "elastic"
 password: "aka2velfFJGz178SExyi"
 index: "devopstack-linux-elk-%{+yyyy.MM.dd}"
 # 禁⽤索引⽣命周期管理
setup.ilm.enabled: false
 # # 设置索引模板的名称
setup.template.name: "devopstack-linux"
 # # 设置索引模板的匹配模式
setup.template.pattern: "devopstack-linux*"

如图所示,已创建我们指定的索引模板
filebeat的部署及基本使用

9 根据tag字段写入不同的索引

[root@filebeat config[]# cat 06-log-when-to-es.yml 
filebeat.inputs:
 - type: log
   paths:
   - /tmp/test.log
   - /tmp/*.txt
   tags: ["test1"]
   fields:
    name: "app1"
    log_topics: test
    log_host: 127.0.0.1

 - type: log
   enabled: true
   paths:
   - /tmp/test/*/*.log
   tags: ["test2","devopstack"]
   fields:
    name: "app2" 
    hobby: "test111" 
   fields_under_root: true

output.elasticsearch:
 hosts: ["http://192.168.10.31:9200","http://192.168.10.32:9200","http://192.168.10.33:9200"]
 username: "elastic"
 password: "aka2velfFJGz178SExyi"
 indices: 
   - index: "devopstack-linux-elk-%{+yyyy.MM.dd}"
     when.contains:
       tags: "test1"
   - index: "devopstack-linux-python-%{+yyyy.MM.dd}"
     when.contains:
       tags: "test2"
 # 禁⽤索引⽣命周期管理
setup.ilm.enabled: false
 # # 设置索引模板的名称
setup.template.name: "devopstack-linux"
 # # 设置索引模板的匹配模式
setup.template.pattern: "devopstack-linux*"

启动之后,查看kibana

filebeat的部署及基本使用filebeat的部署及基本使用

10 ES自定义分片和副本

output.elasticsearch:
 hosts: ["http://192.168.10.31:9200","http://192.168.10.32:9200","http://192.168.10.33:9200"]
 username: "elastic"
 password: "aka2velfFJGz178SExyi"
 indices:
   - index: "devopstack-linux-elk-%{+yyyy.MM.dd}"
     when.contains:
       tags: "test1"
   - index: "devopstack-linux-python-%{+yyyy.MM.dd}"
     when.contains:
       tags: "test2"
 # 禁⽤索引⽣命周期管理
setup.ilm.enabled: false
 # # 设置索引模板的名称
setup.template.name: "devopstack-linux"
 # # 设置索引模板的匹配模式
setup.template.pattern: "devopstack-linux*"
# 覆盖已有的索引模板
setup.template.overwrite: false
# 配置索引模板
setup.template.settings:
# 设置分⽚数量
  index.number_of_shards: 3
# 设置副本数量,要求⼩于集群的数量
  index.number_of_replicas: 2

filebeat的部署及基本使用

11 日志聚合到本地

filebeat.inputs:
- type: tcp
 host: "0.0.0.0:9000"
output.file:
 path: "/tmp/filebeat"
 filename: linux-test
# 指定⽂件的滚动⼤⼩,默认值为20MB
 rotate_every_kb: 102400
# 指定保存的⽂件个数,默认是7个,有效值为2-1024个
 number_of_files: 300
# 指定⽂件的权限,默认权限是0600
 permissions: 0600

12 基于log类型收集nginx原⽣⽇志

收集nginx日志配置如下

[root@filebeat config[]# cat 07-nginx-when-to-es.yml 
filebeat.inputs:
 - type: log
   paths:
    - /var/log/nginx/access.log*
   tags: ["nginx"]

output.elasticsearch:
 hosts: ["http://192.168.10.31:9200","http://192.168.10.32:9200","http://192.168.10.33:9200"]
 username: "elastic"
 password: "aka2velfFJGz178SExyi"
 indices: 
   - index: "devopstack-linux-nginx-%{+yyyy.MM.dd}"
 # 禁⽤索引⽣命周期管理
setup.ilm.enabled: false
 # # 设置索引模板的名称
setup.template.name: "devopstack-linux"
 # # 设置索引模板的匹配模式
setup.template.pattern: "devopstack-linux*"
# 覆盖已有的索引模板
setup.template.overwrite: false
# 配置索引模板
setup.template.settings:
# 设置分⽚数量
 index.number_of_shards: 3
# 设置副本数量,要求⼩于集群的数量
 index.number_of_replicas: 2

13 基于log类型收集nginx的json⽇志

修改nginx配置,支持json格式,vim /etc/nginx/nginx.conf
http {
            log_format oldboyedu_nginx_json '{"@timestamp":"time_iso8601",'
        '"host":"server_addr",'
        '"clientip":"remote_addr",'
        '"SendBytes":body_bytes_sent,'
        '"responsetime":request_time,'
        '"upstreamtime":"upstream_response_time",'
        '"upstreamhost":"upstream_addr",'
        '"http_host":"host",'
        '"uri":"uri",'
        '"domain":"host",'
        '"xff":"http_x_forwarded_for",'
        '"referer":"http_referer",'
        '"tcp_xff":"proxy_protocol_addr",'
        '"http_user_agent":"http_user_agent",'
        '"status":"$status"}';
     access_log /var/log/nginx/access.log oldboyedu_nginx_json;

修改filebeat配置

filebeat.inputs:
 - type: log
   paths:
    - /var/log/nginx/access.log*
   tags: ["nginx"]
   # 以JSON格式解析message字段的内容
   json.keys_under_root: true

然后就能收集json格式得nginx日志了。就把nginx message字段拆分成多个字段了,如下图
filebeat的部署及基本使用

14 基于modules收集nginx日志

查看模块
filebeat modules list
启动模块
filebeat modules enable nginx tomcat

禁⽤模块
filebeat modules disable nginx tomcat

filebeat配置⽂件(需要启⽤nginx模块)
filebeat.config.modules:
# 指定模块的配置⽂件路径,如果是yum⽅式安装,在7.17.3版本中不能使⽤如下的默认值。
# path: ${path.config}/modules.d/*.yml
# 经过实际测试,推荐⼤家使⽤如下的配置,此处写绝对路径即可!⽽对于⼆进制部署⽆需做此
操作.
 path: /etc/filebeat/modules.d/*.yml
# 开启热加载功能
 reload.enabled: true

/etc/filebeat/modules.d/nginx.yml⽂件内容
- module: nginx
 access:
 enabled: true
 var.paths: ["/var/log/nginx/access.log*"]
 error:
 enabled: false
 var.paths: ["/var/log/nginx/error.log"]
 ingress_controller:
 enabled: false

15 基于log类型收集tomcat的json⽇志

(1)⾃定义tomcat的⽇志格式
 cp /oldboyedu/softwares/apache-tomcat10.0.20/conf/{server.xml,server.xml-`date +%F`}
 ...(切换到⾏尾修改,⼤概是在133-149之间)
 <Host name="tomcat.oldboyedu.com" appBase="webapps"
 unpackWARs="true" autoDeploy="true">
 <Valve className="org.apache.catalina.valves.AccessLogValve"
directory="logs"
 prefix="tomcat.oldboyedu.com_access_log" suffix=".txt"
pattern="
{&quot;clientip&quot;:&quot;%h&quot;,&quot;ClientUser&quot;:&quot;%l&quo
t;,&quot;authenticated&quot;:&quot;%u&quot;,&quot;AccessTime&quot;:&quot
;%t&quot;,&quot;request&quot;:&quot;%r&quot;,&quot;status&quot;:&quot;%s
&quot;,&quot;SendBytes&quot;:&quot;%b&quot;,&quot;Query?
string&quot;:&quot;%q&quot;,&quot;partner&quot;:&quot;%
{Referer}i&quot;,&quot;http_user_agent&quot;:&quot;%{UserAgent}i&quot;}"/>
 </Host>
 (2)修改filebeat的配置⽂件
filebeat.inputs:
- type: log
 enabled: true
 paths:
 - /oldboyedu/softwares/apache-tomcat-10.0.20/logs/*.txt
# 解析message字段的json格式,并放在顶级字段中
 json.keys_under_root: true
output.elasticsearch:
 enabled: true
  hosts: ["http://192.168.10.31:9200","http://192.168.10.32:9200","http://192.168.10.33:9200"]
 username: "elastic"
 password: "aka2velfFJGz178SExyi"
 index: "oldboyedu-linux-tomcat-access-%{+yyyy.MM.dd}"
# 禁⽤索引⽣命周期管理
setup.ilm.enabled: false
# 设置索引模板的名称
setup.template.name: "oldboyedu-linux"
# 设置索引模板的匹配模式
setup.template.pattern: "oldboyedu-linux*"
# 覆盖已有的索引模板,如果为true,则会直接覆盖现有的索引模板,如果为false则不覆盖!
setup.template.overwrite: true
# 配置索引模板
setup.template.settings:
# 设置分⽚数量
 index.number_of_shards: 3
# 设置副本数量,要求⼩于集群的数量
 index.number_of_replicas: 0

16 多⾏匹配-收集tomcat的错误⽇志

filebeat.inputs:
- type: log
 enabled: true
 paths:
 - /oldboyedu/softwares/apache-tomcat-10.0.20/logs/*.out
# 指定多⾏匹配的类型,可选值为"pattern","count"
 multiline.type: pattern
# 指定匹配模式
 multiline.pattern: '^\d{2}'
# 下⾯2个参数参考官⽅架构图即可,如上图所示。
 multiline.negate: true
 multiline.match: after
output.elasticsearch:
 enabled: true
  hosts: ["http://192.168.10.31:9200","http://192.168.10.32:9200","http://192.168.10.33:9200"]
 username: "elastic"
 password: "aka2velfFJGz178SExyi"
 index: "oldboyedu-linux-tomcat-error-%{+yyyy.MM.dd}"
# 禁⽤索引⽣命周期管理
setup.ilm.enabled: false
# 设置索引模板的名称
setup.template.name: "oldboyedu-linux"
# 设置索引模板的匹配模式
setup.template.pattern: "oldboyedu-linux*"
# 覆盖已有的索引模板,如果为true,则会直接覆盖现有的索引模板,如果为false则不覆盖!
setup.template.overwrite: true
# 配置索引模板
setup.template.settings:
# 设置分⽚数量
 index.number_of_shards: 3
# 设置副本数量,要求⼩于集群的数量
 index.number_of_replicas: 0

17 多⾏匹配M收集elasticsearch的错误⽇志

filebeat.inputs:
- type: log
 enabled: true
 paths:
 - /var/log/elasticsearch/oldboyedu-elk-2022.log*
# 指定多⾏匹配的类型,可选值为"pattern","count"
 multiline.type: pattern
# 指定匹配模式
 multiline.pattern: '^\['
# 下⾯2个参数参考官⽅架构图即可
 multiline.negate: true
 multiline.match: after
output.elasticsearch:
 enabled: true
 hosts: ["http://192.168.10.31:9200","http://192.168.10.32:9200","http://192.168.10.33:9200"]
 username: "elastic"
 password: "aka2velfFJGz178SExyi"
 index: "oldboyedu-linux-es-error-%{+yyyy.MM.dd}"
# 禁⽤索引⽣命周期管理
setup.ilm.enabled: false
# 设置索引模板的名称
setup.template.name: "oldboyedu-linux"
# 设置索引模板的匹配模式
setup.template.pattern: "oldboyedu-linux*"
# 覆盖已有的索引模板,如果为true,则会直接覆盖现有的索引模板,如果为false则不覆盖!
setup.template.overwrite: true
# 配置索引模板
setup.template.settings:
# 设置分⽚数量
 index.number_of_shards: 3
# 设置副本数量,要求⼩于集群的数量
 index.number_of_replicas: 0

18 nginx错误⽇志过滤

 filebeat.inputs:
- type: log
 enabled: true
 paths:
 - /var/log/nginx/access.log*
 tags: ["access"]
# 解析message字段的json格式,并放在顶级字段中
 json.keys_under_root: true
- type: log
 enabled: true
 paths:
 - /var/log/nginx/error.log*
 tags: ["error"]
 include_lines: ['\[error\]']
output.elasticsearch:
 enabled: true
hosts: ["http://192.168.10.31:9200","http://192.168.10.32:9200","http://192.168.10.33:9200"]
 username: "elastic"
 password: "aka2velfFJGz178SExyi"
 indices:
 - index: "oldboyedu-linux-web-nginx-access-%{+yyyy.MM.dd}"
# 匹配指定字段包含的内容
 when.contains:
 tags: "access"
 - index: "oldboyedu-linux-web-nginx-error-%{+yyyy.MM.dd}"
 when.contains:
 tags: "error"
# 禁⽤索引⽣命周期管理
setup.ilm.enabled: false
# 设置索引模板的名称
setup.template.name: "oldboyedu-linux"
# 设置索引模板的匹配模式
setup.template.pattern: "oldboyedu-linux*"
# 覆盖已有的索引模板
setup.template.overwrite: true
# 配置索引模板
setup.template.settings:
# 设置分⽚数量
 index.number_of_shards: 3
# 设置副本数量,要求⼩于集群的数量
 index.number_of_replicas: 0

19 log类型切换filestream类型注意事项

19.1 filestream类型json解析配置

filebeat.inputs:
- type: filestream
 enabled: true
 paths:
 - /var/log/nginx/access.log*
 tags: ["access"]
 # 对于filestream类型⽽⾔,不能直接配置json解析,⽽是需要借助解析器实现
 # json.keys_under_root: true 
 # 综上所述,我们就需要使⽤以下的写法实现.
 parsers:
 # 使 Filebeat能够解码结构化为JSON消息的⽇志。
 # Filebeat逐⾏处理⽇志,因此JSON解码仅在每条消息有⼀个JSON对象时才有效。
 - ndjson:
 # 对message字段进⾏JSON格式解析,并将key放在顶级字段。
 keys_under_root: true
output.elasticsearch:
 enabled: true
  hosts: ["http://192.168.10.31:9200","http://192.168.10.32:9200","http://192.168.10.33:9200"]
 username: "elastic"
 password: "aka2velfFJGz178SExyi"
 index: "oldboyedu-linux-nginx-%{+yyyy.MM.dd}"
setup.ilm.enabled: false
setup.template.name: "oldboyedu-linux"
setup.template.pattern: "oldboyedu-linux*"
setup.template.overwrite: true
setup.template.settings:
 index.number_of_shards: 3
 index.number_of_replicas: 0

19.2 filestream类型多⾏匹配

filebeat.inputs:
- type: filestream
  enabled: true
  paths:
  - /oldboyedu/softwares/apache-tomcat-10.0.20/logs/*.txt
 tags: ["access"]
 parsers:
 - ndjson:
 keys_under_root: true
- type: filestream
 enabled: true
 paths:
 - /oldboyedu/softwares/apache-tomcat-10.0.20/logs/*.out
 tags: ["error"]
 parsers:
 - multiline:
 type: pattern
 pattern: '^\d{2}'
 negate: true
 match: after 
output.elasticsearch:
 enabled: true
 hosts: ["http://192.168.10.31:9200","http://192.168.10.32:9200","http://192.168.10.33:9200"]
 username: "elastic"
 password: "aka2velfFJGz178SExyi"
 indices:
 - index: "oldboyedu-linux-web-tomcat-access-%{+yyyy.MM.dd}"
 when.contains:
 tags: "access"
 - index: "oldboyedu-linux-web-tomcat-error-%{+yyyy.MM.dd}"
 when.contains:
 tags: "error"
setup.ilm.enabled: false
setup.template.name: "oldboyedu-linux"
setup.template.pattern: "oldboyedu-linux*"
setup.template.overwrite: true
setup.template.settings:
 index.number_of_shards: 3
 index.number_of_replicas: 0

20 将filebeat数据写⼊到redis环境

filebeat.inputs:
- type: tcp
 host: "0.0.0.0:9000"
output.redis:
# 写⼊redis的主机地址
 hosts: ["10.0.0.101:6379"]
# 指定redis的认证⼝令
 password: "oldboyedu"
# 指定连接数据库的编号
 db: 5
# 指定的key值
 key: "oldboyedu-linux80-filebeat"
# 规定超时时间.
 timeout: 3
  • 我的微信
  • 这是我的微信扫一扫
  • weinxin
  • 我的微信公众号
  • 我的微信公众号扫一扫
  • weinxin
avatar

发表评论

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen: