파일구조
./docker_elk
./docker_elk/elasticsearch/config/elasticsearch.yml
./docker_elk/logstash/config/logstash.yml
./docker_elk/logstash/config/pipelines.yml
./docker_elk/logstash/pipeline/0_api_file.conf
./docker_elk/kibana/test/0_api_file.csv
./docker_elk/kibana/config/kibana.yml
./docker_elk/docker-compose.yml
./docker_elk/elasticsearch/config/elasticsearch.yml
---
## Default Elasticsearch configuration from Elasticsearch base image.
## https://github.com/elastic/elasticsearch/blob/master/distribution/docker/src/docker/config/elasticsearch.yml
#
cluster.name: "docker-cluster"
network.host: 0.0.0.0
## X-Pack settings
## see https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-xpack.html
#
# xpack.license.self_generated.type: trial
xpack.security.enabled: false
xpack.monitoring.collection.enabled: false
./docker_elk/logstash/config/logstash.yml
---
## Default Logstash configuration from Logstash base image.
## https://github.com/elastic/logstash/blob/master/docker/data/logstash/config/logstash-full.yml
#
http.host: "0.0.0.0"
# xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ]
## X-Pack security credentials
#
xpack.monitoring.enabled: false
#xpack.monitoring.elasticsearch.username: elastic
#xpack.monitoring.elasticsearch.password: changeme
./docker_elk/logstash/config/pipelines.yml
# This file is where you define your pipelines. You can define multiple.
# For more information on multiple pipelines, see the documentation:
# https://www.elastic.co/guide/en/logstash/current/multiple-pipelines.html
- pipeline.id: 0_api_file
path.config: "/usr/share/logstash/pipeline/0_api_file.conf"
./docker_elk/logstash/pipeline/0_api_file.conf
input
{
http_poller
{
urls =>
{
myurl => "http://api.openweathermap.org/data/2.5/weather?q=Seoul&appid=5af8b7bbeacd98abe5e5e4b84a46d520"
}
schedule =>
{
cron => "*/1 * * * * UTC"
}
codec => "json"
}
}
filter
{
}
output
{
file
{
path => "/usr/share/logstash/test/0_api_file.csv"
# start_position => "beginning"
# sincedb_path => "/dev/null"
}
}
./docker_elk/kibana/test/0_api_file.csv
{"wind":{"deg":140,"speed":2.57},"weather":[{"description":"clear sky","icon":"01d","id":800,"main":"Clear"}],"base":"stations","timezone":32400,"id":1835848,"@version":"1","dt":1626652259,"main":{"feels_like":302.57,"temp_min":298.88,"humidity":71,"pressure":1013,"temp":300.42,"temp_max":300.91},"clouds":{"all":0},"sys":{"country":"KR","sunset":1626691866,"type":1,"id":8105,"sunrise":1626639914},"cod":200,"coord":{"lon":126.9778,"lat":37.5683},"@timestamp":"2021-07-19T00:00:00.894Z","name":"Seoul","visibility":10000}
{"wind":{"deg":127,"gust":2.66,"speed":1.41},"weather":[{"description":"overcast clouds","icon":"04d","id":804,"main":"Clouds"}],"base":"stations","timezone":32400,"id":1835848,"@version":"1","dt":1626652863,"main":{"feels_like":302.4,"temp_min":299.38,"grnd_level":1007,"humidity":66,"pressure":1013,"temp":300.61,"sea_level":1013,"temp_max":300.81},"clouds":{"all":100},"sys":{"country":"KR","sunset":1626691866,"type":1,"id":5509,"sunrise":1626639914},"cod":200,"coord":{"lon":126.9778,"lat":37.5683},"@timestamp":"2021-07-19T00:05:00.630Z","name":"Seoul","visibility":10000}
{"wind":{"deg":127,"gust":2.66,"speed":1.41},"weather":[{"description":"overcast clouds","icon":"04d","id":804,"main":"Clouds"}],"base":"stations","timezone":32400,"id":1835848,"@version":"1","dt":1626652863,"main":{"feels_like":302.4,"temp_min":299.38,"grnd_level":1007,"humidity":66,"pressure":1013,"temp":300.61,"sea_level":1013,"temp_max":300.81},"clouds":{"all":100},"sys":{"country":"KR","sunset":1626691866,"type":1,"id":5509,"sunrise":1626639914},"cod":200,"coord":{"lon":126.9778,"lat":37.5683},"@timestamp":"2021-07-19T00:10:00.175Z","name":"Seoul","visibility":10000}
{"main":{"feels_like":300.2,"temp":300.06,"temp_max":300.81,"humidity":45,"temp_min":298.86,"pressure":1008},"coord":{"lon":126.9778,"lat":37.5683},"name":"Seoul","@version":"1","clouds":{"all":40},"timezone":32400,"cod":200,"base":"stations","@timestamp":"2021-09-20T08:35:00.802Z","dt":1632126360,"id":1835848,"visibility":10000,"wind":{"deg":140,"speed":8.23},"weather":[{"description":"scattered clouds","id":802,"icon":"03d","main":"Clouds"}],"sys":{"id":8105,"sunrise":1632086276,"type":1,"sunset":1632130405,"country":"KR"}}
{"main":{"feels_like":300.2,"temp":300.06,"temp_max":300.81,"humidity":45,"temp_min":298.86,"pressure":1008},"coord":{"lon":126.9778,"lat":37.5683},"name":"Seoul","@version":"1","clouds":{"all":40},"timezone":32400,"cod":200,"base":"stations","@timestamp":"2021-09-20T08:36:00.267Z","dt":1632126360,"id":1835848,"visibility":10000,"wind":{"deg":140,"speed":8.23},"weather":[{"description":"scattered clouds","id":802,"icon":"03d","main":"Clouds"}],"sys":{"id":8105,"sunrise":1632086276,"type":1,"sunset":1632130405,"country":"KR"}}
{"main":{"feels_like":300.2,"temp":300.06,"temp_max":300.81,"humidity":45,"temp_min":298.86,"pressure":1008},"coord":{"lon":126.9778,"lat":37.5683},"name":"Seoul","@version":"1","clouds":{"all":40},"timezone":32400,"cod":200,"base":"stations","@timestamp":"2021-09-20T08:37:00.334Z","dt":1632126900,"id":1835848,"visibility":10000,"wind":{"deg":140,"speed":8.23},"weather":[{"description":"scattered clouds","id":802,"icon":"03d","main":"Clouds"}],"sys":{"id":8105,"sunrise":1632086276,"type":1,"sunset":1632130405,"country":"KR"}}
{"main":{"feels_like":300.2,"temp":300.06,"temp_max":300.81,"humidity":45,"temp_min":298.86,"pressure":1008},"coord":{"lon":126.9778,"lat":37.5683},"name":"Seoul","@version":"1","clouds":{"all":40},"timezone":32400,"cod":200,"base":"stations","@timestamp":"2021-09-20T08:38:00.398Z","dt":1632126900,"id":1835848,"visibility":10000,"wind":{"deg":140,"speed":8.23},"weather":[{"description":"scattered clouds","id":802,"icon":"03d","main":"Clouds"}],"sys":{"id":8105,"sunrise":1632086276,"type":1,"sunset":1632130405,"country":"KR"}}
{"main":{"feels_like":300.2,"temp":300.06,"temp_max":300.81,"humidity":45,"temp_min":298.86,"pressure":1008},"coord":{"lon":126.9778,"lat":37.5683},"name":"Seoul","@version":"1","clouds":{"all":40},"timezone":32400,"cod":200,"base":"stations","@timestamp":"2021-09-20T08:39:00.407Z","dt":1632126900,"id":1835848,"visibility":10000,"wind":{"deg":140,"speed":8.23},"weather":[{"description":"scattered clouds","id":802,"icon":"03d","main":"Clouds"}],"sys":{"id":8105,"sunrise":1632086276,"type":1,"sunset":1632130405,"country":"KR"}}
{"main":{"feels_like":300.2,"temp":300.06,"temp_max":300.81,"humidity":45,"temp_min":298.86,"pressure":1008},"coord":{"lon":126.9778,"lat":37.5683},"name":"Seoul","@version":"1","clouds":{"all":40},"timezone":32400,"cod":200,"base":"stations","@timestamp":"2021-09-20T08:40:00.429Z","dt":1632126900,"id":1835848,"visibility":10000,"wind":{"deg":140,"speed":8.23},"weather":[{"description":"scattered clouds","id":802,"icon":"03d","main":"Clouds"}],"sys":{"id":8105,"sunrise":1632086276,"type":1,"sunset":1632130405,"country":"KR"}}
{"main":{"feels_like":300.2,"temp":300.06,"temp_max":300.81,"humidity":45,"temp_min":298.86,"pressure":1008},"coord":{"lon":126.9778,"lat":37.5683},"name":"Seoul","@version":"1","clouds":{"all":40},"timezone":32400,"cod":200,"base":"stations","@timestamp":"2021-09-20T08:41:00.310Z","dt":1632126900,"id":1835848,"visibility":10000,"wind":{"deg":140,"speed":8.23},"weather":[{"description":"scattered clouds","id":802,"icon":"03d","main":"Clouds"}],"sys":{"id":8105,"sunrise":1632086276,"type":1,"sunset":1632130405,"country":"KR"}}
{"main":{"feels_like":300.2,"temp":300.06,"temp_max":300.81,"humidity":45,"temp_min":298.86,"pressure":1008},"coord":{"lon":126.9778,"lat":37.5683},"name":"Seoul","@version":"1","clouds":{"all":40},"timezone":32400,"cod":200,"base":"stations","@timestamp":"2021-09-20T08:42:00.359Z","dt":1632126900,"id":1835848,"visibility":10000,"wind":{"deg":140,"speed":8.23},"weather":[{"description":"scattered clouds","id":802,"icon":"03d","main":"Clouds"}],"sys":{"id":8105,"sunrise":1632086276,"type":1,"sunset":1632130405,"country":"KR"}}
{"main":{"feels_like":300.2,"temp":300.06,"temp_max":300.81,"humidity":45,"temp_min":298.86,"pressure":1008},"coord":{"lon":126.9778,"lat":37.5683},"name":"Seoul","@version":"1","clouds":{"all":40},"timezone":32400,"cod":200,"base":"stations","@timestamp":"2021-09-20T08:43:00.224Z","dt":1632126900,"id":1835848,"visibility":10000,"wind":{"deg":140,"speed":8.23},"weather":[{"description":"scattered clouds","id":802,"icon":"03d","main":"Clouds"}],"sys":{"id":8105,"sunrise":1632086276,"type":1,"sunset":1632130405,"country":"KR"}}
./docker_elk/kibana/config/kibana.yml
---
## Default Kibana configuration from Kibana base image.
## https://github.com/elastic/kibana/blob/master/src/dev/build/tasks/os_packages/docker_generator/templates/kibana_yml.template.ts
#
server.name: kibana
server.host: 0.0.0.0
elasticsearch.hosts: [ "http://elasticsearch:9200" ]
monitoring.ui.container.elasticsearch.enabled: true
## X-Pack security credentials
#
# elasticsearch.username: elastic
# elasticsearch.password: changeme
./docker_elk/docker-compose.yml
version: '2'
services:
elasticsearch:
hostname: elasticsearch
container_name: elasticsearch
image: docker.io/elasticsearch:7.14.0
volumes:
- type: bind
source: ./elasticsearch/config/elasticsearch.yml
target: /usr/share/elasticsearch/config/elasticsearch.yml
read_only: false
ports:
- "9200:9200"
- "9300:9300"
environment:
ES_JAVA_OPTS: "-Xmx1024m -Xms1024m"
discovery.type: single-node
logstash:
hostname: logstash
container_name: logstash
image: docker.io/logstash:7.14.0
volumes:
- type: bind
source: ./logstash/config/logstash.yml
target: /usr/share/logstash/config/logstash.yml
read_only: false
- type: bind
source: ./logstash/pipeline
target: /usr/share/logstash/pipeline
read_only: false
- type: bind
source: ./logstash/config/pipelines.yml
target: /usr/share/logstash/config/pipelines.yml
read_only: false
- type: bind
source: ./logstash/test
target: /usr/share/logstash/test
read_only: false
ports:
- "5044:5044"
- "5000:5000/tcp"
- "5000:5000/udp"
- "9600:9600"
environment:
LS_JAVA_OPTS: "-Xmx1024m -Xms1024m"
kibana:
hostname: kibana
container_name: kibana
image: docker.io/kibana:7.14.0
volumes:
- type: bind
source: ./kibana/config/kibana.yml
target: /usr/share/kibana/config/kibana.yml
read_only: false
ports:
- "5601:5601"
depends_on:
- elasticsearch
