ELK基本部署以及使用

评论

ELK 入门学习文章

Dockerfile 以及配置文件

ENV

.env

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
### Drivers ################################################

# All volumes driver
VOLUMES_DRIVER=local

# All Networks driver
NETWORKS_DRIVER=bridge

### ELK Stack ##################################################
ELK_VERSION=7.8.1

### ELASTICSEARCH #########################################
ELASTICSEARCH_HOST_HTTP_PORT=9200
ELASTICSEARCH_HOST_TRANSPORT_PORT=9300

### KIBANA ################################################
KIBANA_HTTP_PORT=5601

elasticsearch

Dockerfile

1
2
3
4
ARG ELK_VERSION=7.6.1
FROM docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}

EXPOSE 9200 9300

logstash

Dockerfile

1
2
ARG ELK_VERSION=7.6.1
FROM logstash:${ELK_VERSION}

logstash.yml

1
2
3
4
5
6
7
http.host: "0.0.0.0"

config.reload.automatic: true
path.config: "/usr/share/logstash/pipeline/"

xpack.monitoring.enabled: true
xpack.monitoring.elasticsearch.hosts: ["elasticsearch:9200"]

kibana

Dockerfile

1
2
3
4
ARG ELK_VERSION=7.6.1
FROM docker.elastic.co/kibana/kibana:${ELK_VERSION}

EXPOSE 5601

kibana.yml

1
2
3
4
5
server.name: kibana
server.host: "0"
elasticsearch.hosts: [ "http://elasticsearch:9200" ]
xpack.monitoring.ui.container.elasticsearch.enabled: true
i18n.locale: "zh-CN"

docker-compose

docker-compose.yml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
version: '3.4'

networks:
  service:
    driver: ${NETWORKS_DRIVER}

volumes:
  elasticsearch:
    driver: ${VOLUMES_DRIVER}
  logstash:
    driver: ${VOLUMES_DRIVER}
  kibana:
    driver: ${VOLUMES_DRIVER}

services:
  ### ElasticSearch ########################################
  elasticsearch:
    build:
      context: ./elasticsearch
      args:
        - ELK_VERSION=${ELK_VERSION}
    volumes:
      - ./elasticsearch/data:/usr/share/elasticsearch/data
    environment:
      - cluster.name=cluster
      - node.name=node
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - cluster.initial_master_nodes=node
    ulimits:
      memlock:
        soft: -1
        hard: -1
    ports:
      - "${ELASTICSEARCH_HOST_HTTP_PORT}:9200"
      - "${ELASTICSEARCH_HOST_TRANSPORT_PORT}:9300"
    restart: always
    networks:
      - service

  ### Logstash ##############################################
  logstash:
    build:
      context: ./logstash
      args:
        - ELK_VERSION=${ELK_VERSION}
    volumes:
      - './logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml'
      - './logstash/pipeline:/usr/share/logstash/pipeline'
      - './logstash/GeoLite2-City:/usr/share/logstash/GeoLite2-City'
    ports:
      - '5001:5001'
      - '5044:5044'
    environment:
      LS_JAVA_OPTS: '-Xmx1g -Xms1g'
    env_file:
      - .env
    networks:
      - service
    restart: always
    depends_on:
      - elasticsearch

  ### Kibana ##############################################
  kibana:
    build:
      context: ./kibana
      args:
        - ELK_VERSION=${ELK_VERSION}
    volumes:
      - ./kibana/config:/usr/share/kibana/config
    ports:
      - "${KIBANA_HTTP_PORT}:5601"
    depends_on:
      - elasticsearch
    restart: always
    networks:
      - service

部署

1
2
3
4
5
## 启动服务
docker-compose up -d

## 查看服务是否正常
docker-compose ps

测试服务是否可用

elasticsearch

1
localhost:9200

kibana

1
localhost:5601

Logstash

测试标准输入输出

1
2
3
4
5
6
7
8
9
10
11
12
13
bash-4.2$ /usr/share/logstash/bin/logstash -e 'input { stdin {} } output { stdout { codec => rubydebug} }'

# 光标闪烁，输入并回车
hello world

# 控制台输出
{
      "@version" => "1", #事件版本号，一个事件就是一个ruby对象
    "@timestamp" => 2021-09-02T07:57:12.277Z, #事件发生时间
          "host" => "e8ff6e2a9658", #事件来源
       "message" => "hello world" #消息内容
}

测试输出到文件

1
2
3
4
5
6
7
8
9
10
11
12
bash-4.2$ /usr/share/logstash/bin/logstash   -e 'input { stdin{} } output { file { path => "/tmp/log-%{+YYYY.MM.dd}messages.log"}}'

# 光标闪烁，输入并回车
hello world


# 控制台输出
Opening file {:path=>"/tmp/log-2021.09.02messages.log"}

bash-4.2$ tail /tmp/log-2021.09.02messages.log
{"@timestamp":"2021-09-02T08:04:06.500Z","host":"e8ff6e2a9658","message":"hello world","@version":"1"}

测试输出到 elasticsearch

1
2
3
4
5
6
7
8
bash-4.2$ /usr/share/logstash/bin/logstash -e 'input {  stdin{} } output { elasticsearch {hosts => ["elasticsearch:9200"] index => "mytest-%{+YYYY.MM.dd}" }}'

# 验证ES是否收到数据
bash-4.2$ curl http://elasticsearch:9200/mytest-2021.09.02
{"mytest-2021.09.02":{"aliases":{},"mappings":{"properties":{"@timestamp":{"type":"date"},"@version":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"host":{"type":"text","fields":{"keyword":{"type":"keywo
rd","ignore_above":256}}},"message":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}},"settings":{"index":{"creation_date":"1630570225131","number_of_shards":"1","number_of_replicas":"1","uuid":"0yt4C-0RRt
2DdG_5aI16UQ","version":{"created":"7070199"},"provided_name":"mytest-2021.09.02"}}}}

常见错误解决

Logstash could not be started because there is already another instance using the configured data directory

查看logstash.yml 中 path.data 路径，若无配置，默认在/usr/share/logstash/data

1
2
3
4
5
6
7
cd /usr/share/logstash/data

# 查看是否存在 .lock 文件，
ls -alh

# 如果存在把它删除
rm .lock

vm.max_map_count [65530] is too low

1
2
3
4
5
6
7
vim /etc/sysctl.conf

# sysctl.conf
vm.max_map_count=262144
# sysctl.conf

sysctl -p