ELK学习实验018：filebeat收集docker日志

时间:2020-01-19 梦中泪人气:0

Filebeat收集Docker日志

1 安装docker

[root@node4 ~]# yum install -y yum-utils device-mapper-persistent-data lvm2

[root@node4 ~]# yum update

[root@node4 ~]# yum-config-manager --add-repo http://mirrors.aliyun.comhttps://img.qb5200.com/download-x/docker-ce/linux/centoshttps://img.qb5200.com/download-x/docker-ce.repo

[root@node4 ~]# yum makecache fast

[root@node4 ~]# yum -y install docker-ce

[root@node4 ~]# systemctl restart docker

[root@node4 ~]# systemctl enable docker

2 运行一个nginx容器

[root@node4 ~]# docker run --name nginx -p 8081:80 -d nginx

Unable to find image 'nginx:latest' locally
latest: Pulling from library/nginx
8ec398bc0356: Pull complete 
dfb2a46f8c2c: Pull complete 
b65031b6a2a5: Pull complete 
Digest: sha256:8aa7f6a9585d908a63e5e418dc5d14ae7467d2e36e1ab4f0d8f9d059a3d071ce
Status: Downloaded newer image for nginx:latest
9c29964182697e55e7ca0fd793f1e243a9e404c84868bee814afdb700760ba5a

[root@node4 ~]# docker ps -a

CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS                  NAMES
9c2996418269        nginx               "nginx -g 'daemon of…"   52 seconds ago      Up 51 seconds       0.0.0.0:8081->80/tcp   nginx

访问http://192.168.132.134:8081/

[root@node4 ~]# docker exec -it 9c2996418269 /bin/bash

3 查看docker日志

[root@node4 ~]# docker logs -f nginx

192.168.132.1 - - [19/Jan/2020:11:11:55 +0000] "GET / HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" "-"
2020/01/19 11:11:55 [error] 6#6: *1 open() "/usr/share/nginx/html/favicon.ico" failed (2: No such file or directory), client: 192.168.132.1, server: localhost, request: "GET /favicon.ico HTTP/1.1", host: "192.168.132.134:8081", referrer: "http://192.168.132.134:8081/"
192.168.132.1 - - [19/Jan/2020:11:11:55 +0000] "GET /favicon.ico HTTP/1.1" 404 555 "http://192.168.132.134:8081/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" "-"

本地查看

[root@node4 ~]# tail -f /var/libhttps://img.qb5200.com/download-x/docker/containers/9c29964182697e55e7ca0fd793f1e243a9e404c84868bee814afdb700760ba5a/9c29964182697e55e7ca0fd793f1e243a9e404c84868bee814afdb700760ba5a-json.log

是json格式日志

4 filebeat收集

docker的正确日志

错误日志

错误日志再stream显示的stdeer，正确的是stdout，根据这个规则配置filebeat

5 配置filebeat

filebeat.inputs:
#####################################################
## Nginx log
#####################################################
- type: log
  enabled: true
  paths:
    - /usr/local/nginx/logs/access.log
  json.key_under_root: true
  json.overwrite_keys: true
  tags: ["access"]

- type: log
  enabled: true
  paths:
    - /usr/local/nginx/logs/error.log
  tags: ["error"]


#####################################################
## tomcat  log
#####################################################
- type: log
  enabled: true
  paths:
    - /var/log/tomcat/localhost_access_log.*.txt
  json.key_under_root: true
  json.overwrite_keys: true
  tags: ["tomcat"]


#####################################################
## java  log
#####################################################
- type: log
  enabled: true
  paths:
    - /usr/local/elasticsearch/logs/my-elktest-cluster.log 
  tags: ["es-java"]
  multiline.pattern: '^\['
  multiline.negate: true
  multiline.match: "after"


#####################################################
## docker  log
#####################################################
- type: docker
  containers.ids:
    - '9c29964182697e55e7ca0fd793f1e243a9e404c84868bee814afdb700760ba5a'
  json.key_under_root: true
  json.overwrite_keys: true
  tags: ["docker"]


#####################################################
## Output
#####################################################
setup.kibana:
  host: "192.168.132.131:5601"
output.elasticsearch:
  hosts: ["192.168.132.131:9200","192.168.132.132:9200","192.168.132.133:9200"]
  #index: "nginx-%{[agent.version]}-%{+yyyy.MM.dd}"
  indices:
    - index: "access-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
        tags: "access"
    - index: "error-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
        tags: "error"
    - index: "tomcat-access-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
        tags: "tomcat"
    - index: "javaes-access-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
        tags: "es-java"
    - index: "docker-access-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
         tags: "docker"
         stream: "stdout"
    - index: "docker-error-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
         tags: "docker"
         stream: "stderr"

setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.overwrite: true
setup.template.enabled: true
setup.ilm.enabled: false

查看索引

kibana查看

错误日志

源日志数据

@timestamp    Jan 19, 2020 @ 19:39:11.016
    t_id    wXuZvW8BYiPduFlChbrm
    t_index    docker-error-7.4.2-2020.01.19
    #_score     - 
    t_type    _doc
    tagent.ephemeral_id    66a6dffb-9e49-4914-a6a0-ff1a073eea6a
    tagent.hostname    node4
    tagent.id    bb3818f9-66e2-4eb2-8f0c-3f35b543e025
    tagent.type    filebeat
    tagent.version    7.4.2
    tecs.version    1.1.0
    thost.name    node4
    tinput.type    docker
    tlog.file.path    /var/libhttps://img.qb5200.com/download-x/docker/containers/9c29964182697e55e7ca0fd793f1e243a9e404c84868bee814afdb700760ba5a/9c29964182697e55e7ca0fd793f1e243a9e404c84868bee814afdb700760ba5a-json.log
    #log.offset    7,381
    tmessage    2020/01/19 11:39:11 [error] 6#6: *9 open() "/usr/share/nginx/html/tcp" failed (2: No such file or directory), client: 192.168.132.1, server: localhost, request: "GET /tcp HTTP/1.1", host: "192.168.132.134:8081"
    tstream    stderr
    ttags    docker

正确日志

原日志数据

@timestamp    Jan 19, 2020 @ 19:41:15.401
    t_id    hlGbvW8BOF7DoSFdbG5D
    t_index    docker-access-7.4.2-2020.01.19
    #_score     - 
    t_type    _doc
    tagent.ephemeral_id    66a6dffb-9e49-4914-a6a0-ff1a073eea6a
    tagent.hostname    node4
    tagent.id    bb3818f9-66e2-4eb2-8f0c-3f35b543e025
    tagent.type    filebeat
    tagent.version    7.4.2
    tecs.version    1.1.0
    thost.name    node4
    tinput.type    docker
    tlog.file.path    /var/libhttps://img.qb5200.com/download-x/docker/containers/9c29964182697e55e7ca0fd793f1e243a9e404c84868bee814afdb700760ba5a/9c29964182697e55e7ca0fd793f1e243a9e404c84868bee814afdb700760ba5a-json.log
    #log.offset    8,495
    tmessage    192.168.132.1 - - [19/Jan/2020:11:41:15 +0000] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" "-"
    tstream    stdout
    ttags    docker

6 运行多个容器

[root@node4 ~]# docker run --name nginx-v2 -p 8082:80 -v https://img.qb5200.com/download-x/data:/usr/share/nginx/html -d nginx

[root@node4 ~]# cd https://img.qb5200.com/download-x/data/

[root@node4 data]# echo "this is second container" > index.html

[root@node4 data]# docker ps -a

CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS                  NAMES
7778b091aa01        nginx               "nginx -g 'daemon of…"   30 seconds ago      Up 29 seconds       0.0.0.0:8082->80/tcp   nginx-v2
9c2996418269        nginx               "nginx -g 'daemon of…"   38 minutes ago      Up 38 minutes       0.0.0.0:8081->80/tcp   nginx

访问http://192.168.132.134:8082/

7 配置filebeat收集所有容器

想要收集所有的dokcer日志修改filebeat

filebeat.inputs:
#####################################################
## Nginx log
#####################################################
- type: log
  enabled: true
  paths:
    - /usr/local/nginx/logs/access.log
  json.key_under_root: true
  json.overwrite_keys: true
  tags: ["access"]

- type: log
  enabled: true
  paths:
    - /usr/local/nginx/logs/error.log
  tags: ["error"]


#####################################################
## tomcat  log
#####################################################
- type: log
  enabled: true
  paths:
    - /var/log/tomcat/localhost_access_log.*.txt
  json.key_under_root: true
  json.overwrite_keys: true
  tags: ["tomcat"]


#####################################################
## java  log
#####################################################
- type: log
  enabled: true
  paths:
    - /usr/local/elasticsearch/logs/my-elktest-cluster.log 
  tags: ["es-java"]
  multiline.pattern: '^\['
  multiline.negate: true
  multiline.match: "after"


#####################################################
## docker  log
#####################################################
- type: docker
  containers.ids:
    - '*'
  json.key_under_root: true
  json.overwrite_keys: true
  tags: ["docker"]


#####################################################
## Output
#####################################################
setup.kibana:
  host: "192.168.132.131:5601"
output.elasticsearch:
  hosts: ["192.168.132.131:9200","192.168.132.132:9200","192.168.132.133:9200"]
  #index: "nginx-%{[agent.version]}-%{+yyyy.MM.dd}"
  indices:
    - index: "access-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
        tags: "access"
    - index: "error-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
        tags: "error"
    - index: "tomcat-access-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
        tags: "tomcat"
    - index: "javaes-access-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
        tags: "es-java"
    - index: "docker-access-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
         tags: "docker"
         stream: "stdout"
    - index: "docker-error-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
         tags: "docker"
         stream: "stderr"

setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.overwrite: true
setup.template.enabled: true
setup.ilm.enabled: false

随意访问nginx，查看索引

但是收集到日志以后，所有的容器日志集中在一起，无法分辨，则为每一个容器添加一个标签

使用docker-compose为容器添加新的标签

8 安装docker-compose

参考https://www.cnblogs.com/zyxnhr/p/12158816.html

[root@node4 src]# curl -L https://github.comhttps://img.qb5200.com/download-x/docker/compose/releaseshttps://img.qb5200.com/download-x/download/1.25.0https://img.qb5200.com/download-x/docker-compose-`uname -s`-`uname -m` -o /usr/local/binhttps://img.qb5200.com/download-x/docker-compose

 % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   617    0   617    0     0    810      0 --:--:-- --:--:-- --:--:--   809
100 16.2M  100 16.2M    0     0   529k      0  0:00:31  0:00:31 --:--:--  551k

[root@node4 src]# chmod +x /usr/local/binhttps://img.qb5200.com/download-x/docker-compose

[root@node4 src]# docker-compose --version

docker-compose version 1.25.0, build 0a186604

[root@node4 ~]# vim docker-compose.yaml

version: '3'
services:
  nginx:
    image: nginx
    #设置labels
    labels:
      service: nginx
    #logging设置增加labels.service
    logging:
      options:
        labels: "service"
    ports:
      - "8083:80"
  httpd:
    image: httpd:2.4
    #设置labels
    labels:
      service: httpd
    #logging设置增加labels.service
    logging:
      options:
        labels: "service"
    ports:
      - "8084:80"

10 使用docker-compose发布容器

[root@node4 ~]# docker-compose up

Creating network "root_default" with the default driver
Pulling httpd (httpd:2.4)...
2.4: Pulling from library/httpd
8ec398bc0356: Already exists
354e6904d655: Pull complete
27298e4c749a: Pull complete
10e27104ba69: Pull complete
36412f6b2f6e: Pull complete
Digest: sha256:769018135ba22d3a7a2b91cb89b8de711562cdf51ad6621b2b9b13e95f3798de
Status: Downloaded newer image for httpd:2.4
Creating root_httpd_1 ... done
Creating root_nginx_1 ... done

[root@node4 ~]# docker ps -a

CONTAINER ID        IMAGE               COMMAND                  CREATED              STATUS              PORTS                  NAMES
0c68d79a9a73        nginx               "nginx -g 'daemon of…"   About a minute ago   Up About a minute   0.0.0.0:8083->80/tcp   root_nginx_1
302d59b77fd9        httpd:2.4           "httpd-foreground"       About a minute ago   Up About a minute   0.0.0.0:8084->80/tcp   root_httpd_1
7778b091aa01        nginx               "nginx -g 'daemon of…"   29 minutes ago       Up 29 minutes       0.0.0.0:8082->80/tcp   nginx-v2
9c2996418269        nginx               "nginx -g 'daemon of…"   About an hour ago    Up About an hour    0.0.0.0:8081->80/tcp   nginx

查看索引日志

另一个也有标记

kinban查看

@timestamp    Jan 19, 2020 @ 20:20:49.919
    t_id    nFG_vW8BOF7DoSFdtm7C
    t_index    docker-access-7.4.2-2020.01.19
    #_score     - 
    t_type    _doc
    tagent.ephemeral_id    22c670e2-26fe-459f-8369-36cf36e6aa2f
    tagent.hostname    node4
    tagent.id    bb3818f9-66e2-4eb2-8f0c-3f35b543e025
    tagent.type    filebeat
    tagent.version    7.4.2
    ?docker.attrs.service    httpd     #docker标记
    tecs.version    1.1.0
    thost.name    node4
    tinput.type    docker
    tlog.file.path    /var/libhttps://img.qb5200.com/download-x/docker/containers/302d59b77fd90a5fa664e5e44ff4c774fa66b0850d82a12f8d156463eba3a5dd/302d59b77fd90a5fa664e5e44ff4c774fa66b0850d82a12f8d156463eba3a5dd-json.log
    #log.offset    2,718
    tmessage    192.168.132.1 - - [19/Jan/2020:12:20:49 +0000] "GET /tcp HTTP/1.1" 404 196
    tstream    stdout
    ttags    docker

11 根据容器类别自定义

filebeat.inputs:
#####################################################
## Nginx log
#####################################################
- type: log
  enabled: true
  paths:
    - /usr/local/nginx/logs/access.log
  json.key_under_root: true
  json.overwrite_keys: true
  tags: ["access"]

- type: log
  enabled: true
  paths:
    - /usr/local/nginx/logs/error.log
  tags: ["error"]


#####################################################
## tomcat  log
#####################################################
- type: log
  enabled: true
  paths:
    - /var/log/tomcat/localhost_access_log.*.txt
  json.key_under_root: true
  json.overwrite_keys: true
  tags: ["tomcat"]


#####################################################
## java  log
#####################################################
- type: log
  enabled: true
  paths:
    - /usr/local/elasticsearch/logs/my-elktest-cluster.log 
  tags: ["es-java"]
  multiline.pattern: '^\['
  multiline.negate: true
  multiline.match: "after"


#####################################################
## docker  log
#####################################################
- type: docker
  containers.ids:
    - '*'
  json.key_under_root: true
  json.overwrite_keys: true
  tags: ["docker"]


#####################################################
## Output
#####################################################
setup.kibana:
  host: "192.168.132.131:5601"
output.elasticsearch:
  hosts: ["192.168.132.131:9200","192.168.132.132:9200","192.168.132.133:9200"]
  #index: "nginx-%{[agent.version]}-%{+yyyy.MM.dd}"
  indices:
    - index: "access-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
        tags: "access"
    - index: "error-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
        tags: "error"
    - index: "tomcat-access-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
        tags: "tomcat"
    - index: "javaes-access-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
        tags: "es-java"
    - index: "docker-nginx-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
         tags: "docker"
         docker.attrs.service: "nginx"
    - index: "docker-httpd-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
         tags: "docker"
         docker.attrs.service: "httpd"

setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.overwrite: true
setup.template.enabled: true
setup.ilm.enabled: false

访问后查看索引

12 修改filebeat再细致划分

filebeat.inputs:
#####################################################
## Nginx log
#####################################################
- type: log
  enabled: true
  paths:
    - /usr/local/nginx/logs/access.log
  json.key_under_root: true
  json.overwrite_keys: true
  tags: ["access"]

- type: log
  enabled: true
  paths:
    - /usr/local/nginx/logs/error.log
  tags: ["error"]


#####################################################
## tomcat  log
#####################################################
- type: log
  enabled: true
  paths:
    - /var/log/tomcat/localhost_access_log.*.txt
  json.key_under_root: true
  json.overwrite_keys: true
  tags: ["tomcat"]


#####################################################
## java  log
#####################################################
- type: log
  enabled: true
  paths:
    - /usr/local/elasticsearch/logs/my-elktest-cluster.log 
  tags: ["es-java"]
  multiline.pattern: '^\['
  multiline.negate: true
  multiline.match: "after"


#####################################################
## docker  log
#####################################################
- type: docker
  containers.ids:
    - '*'
  json.key_under_root: true
  json.overwrite_keys: true
  tags: ["docker"]


#####################################################
## Output
#####################################################
setup.kibana:
  host: "192.168.132.131:5601"
output.elasticsearch:
  hosts: ["192.168.132.131:9200","192.168.132.132:9200","192.168.132.133:9200"]
  #index: "nginx-%{[agent.version]}-%{+yyyy.MM.dd}"
  indices:
    - index: "access-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
        tags: "access"
    - index: "error-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
        tags: "error"
    - index: "tomcat-access-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
        tags: "tomcat"
    - index: "javaes-access-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
        tags: "es-java"
    - index: "docker-access-%{[docker.attrs.service]}-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
         tags: "docker"
         stream: "stdout"
    - index: "docker-error-%{[docker.attrs.service]}-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
         tags: "docker"
         stream: "stderr"

setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.overwrite: true
setup.template.enabled: true
setup.ilm.enabled: false

访问后

但是没有docker-error-httpd*

经过日志访问后，发现没有stderr的这个标记

关于Docker的日志收集介绍到这里

加载全部内容