EFK日志系统搭建

目录结构

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
root@ubuntu:~/log-collect/EFK# tree
├── docker-compose.yml
├── .env
├── elasticsearch
│   ├── data
│   └── Dockerfile
├── fluentd
│   ├── Dockerfile
│   └── etc
│       └── fluent.conf
└── kibana
    ├── data
    ├── Dockerfile
    └── kibana.yml

新建目录

1
2
3
mkdir -p EFK/elasticsearch/data
mkdir -p EFK/kibana/data
mkdir -p EFK/fluentd/etc

elasticsearch

Dockerfile

1
2
3
ARG ELK_VERSION

FROM docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}

fluentd

Dockerfile

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
ARG FLUENTD_VERSION

FROM fluent/fluentd:v${FLUENTD_VERSION}

# Use root account to use apk
USER root

# below RUN includes plugin as examples elasticsearch is not required
# you may customize including plugins as you wish

RUN apk add --no-cache --update --virtual .build-deps \
        sudo build-base ruby-dev \
 && sudo gem install fluent-plugin-elasticsearch \
 && sudo gem sources --clear-all \
 && apk del .build-deps \
 && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
# RUN ["gem", "install", "fluent-plugin-elasticsearch", "--no-rdoc", "--no-ri", "--version", "1.9.2"]

USER fluent

fluent.conf

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
<source>
  @type forward
  port 24224
  tag nginx
  <parse>
    @type nginx
    expression /^(?<host>[^ ]*) (?<server_port>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+[^\"])(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*) "(?<referer>[^\"]*)" "(?<agent>[^\"]*)"$/
    time_key time
    time_format %d/%b/%Y:%H:%M:%S %z
  </parse>
</source>

# <filter **>
#   @type parser
#   <parse>
#     @type regexp
    
#     expression /^(?<host>[^ ]*) (?<server_port>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+[^\"])(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*) "(?<referer>[^\"]*)" "(?<agent>[^\"]*)"$/
#     time_key time
#     time_format %d/%b/%Y:%H:%M:%S %z
    
#   </parse>
#   key_name log
# </filter>

<match nginx>
  @type elasticsearch
  host elasticsearch
  port 9200
  logstash_format true
  logstash_prefix fluentd
  logstash_dateformat %Y%m%d
  include_tag_key true
  type_name access_log
  tag_key nginx
  flush_interval 1s
  user elastic
  password elastic
  suppress_type_name true
</match>

kibana

Dockerfile

1
2
3
ARG ELK_VERSION

FROM docker.elastic.co/kibana/kibana:${ELK_VERSION}

kibana.yml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
#
# ** THIS IS AN AUTO-GENERATED FILE **
#

# Default Kibana configuration for docker target
server.name: kibana
server.host: "0"
# server.basePath: "/efk"
elasticsearch.hosts: [ "http://elasticsearch:9200" ]
monitoring.ui.container.elasticsearch.enabled: true
i18n.locale: "zh-CN"
elasticsearch.username: "elastic"
elasticsearch.password: "elastic"

#$ vim nginx.conf
#    location /efk/ {
#          proxy_pass http://192.168.1.11:5601/;
#    }

#配置kibana
#$ vim ./kibana/kibana.yml
#server.basePath: "/efk"

docker-compose.yml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
version: "3.7"
services:
  flog:
      image: mingrammer/flog
      container_name: flog
      command: "-f json -l -d 1"
      depends_on: 
        - fluentd
      links:
        - fluentd
      logging:
        driver: "fluentd"
        options:
          fluentd-address: localhost:24224
          tag: fake.log
      networks:
        - elk-nginx

  fluentd:
      image: fluentd:${FLUENTD_VERSION}
      container_name: fluentd_${FLUENTD_VERSION}
      build:
        context: ./fluentd
        args:
          FLUENTD_VERSION: $FLUENTD_VERSION
      ports:
        - "24224:24224"
        - "24224:24224/udp"
      depends_on: 
        - "elasticsearch"
      networks:
        - elk-nginx
      volumes:
        - ./fluentd/etc:/fluentd/etc

  elasticsearch:
    image: elasticsearch:${ELK_VERSION}
    container_name: elasticsearch_${ELK_VERSION}
    build:
      context: ./elasticsearch
      args:
        ELK_VERSION: $ELK_VERSION
    mem_limit: 1G
    ports:
      - "9200:9200"
    expose:
      - 9200
    volumes:
      - ./elasticsearch/data:/usr/share/elasticsearch/data
    environment:
      - discovery.type=single-node
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - xpack.security.enabled=true
    networks:
      - elk-nginx

  kibana:
    image: kibana:${ELK_VERSION}
    container_name: kibana_${ELK_VERSION}
    build:
      context: kibana/
      args:
        ELK_VERSION: $ELK_VERSION
    mem_limit: 1G
    volumes:
      - ./kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
      - ./kibana/data:/usr/share/kibana/data
    links:
      - "elasticsearch"
    ports:
      - "5601:5601"
    networks:
      - elk-nginx

networks:
  elk-nginx:
    driver: bridge

.env

1
2
ELK_VERSION=7.16.3
FLUENTD_VERSION=1.14-1

修改文件权限

1
chmod 777 -R ./

配置密码访问

Elasticsearch安全认证

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
# 重置密码 auto：随机密码 interactive：手动设置
docker exec -it elasticsearch_7.16.3 bash
./bin/elasticsearch-setup-passwords interactive

--- Please confirm that you would like to continue [y/N]y

Enter password for [elastic]:                  <--输入用户elastic的密码>
Reenter password for [elastic]:                <--再次输入定义的密码>
Enter password for [apm_system]:               <--输入用户apm_system的密码>
Reenter password for [apm_system]:             <--再次输入定义的密码>
Enter password for [kibana_system]:            <--输入用户kibana的密码>
Reenter password for [kibana_system]:          <--再次输入定义的密码>
Enter password for [logstash_system]:          <--输入用户logstash_system的密码>
Reenter password for [logstash_system]:        <--再次输入定义的密码>
Enter password for [beats_system]:             <--输入用户beats_system的密码>
Reenter password for [beats_system]:           <--再次输入定义的密码>
Enter password for [remote_monitoring_user]:   <--输入用户remote_monitoring_user的密码>
Reenter password for [remote_monitoring_user]: <--再次输入定义的密码>
Changed password for user [apm_system]
Changed password for user [kibana_system]
Changed password for user [logstash_system]
Changed password for user [beats_system]
Changed password for user [remote_monitoring_user]
Changed password for user [elastic]

exit
1
curl -H "Content-Type:application/json" -XPOST -u elastic 'http://127.0.0.1:9200/_xpack/security/user/elastic/_password' -d '{ "password" : "123456" }'