diff --git a/README.md b/README.md index 4f3d89b..9200331 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ A collection of delicious docker recipes. - [x] h2o - [x] httpbin :+1: - [x] influxdb +- [x] logstash - [x] luigi - [x] mariadb - [x] mariadb-arm diff --git a/logstash/README.md b/logstash/README.md new file mode 100644 index 0000000..9f2a363 --- /dev/null +++ b/logstash/README.md @@ -0,0 +1,81 @@ +logstash +======== + +## How It Works + +![](https://www.elastic.co/assets/blt203883a0718cdc5a/filebeat-diagram.png) + +``` +log files ---> filebeat agent --+ +... | +log files ---> filebeat agent --+> logstash container ---> aliyunâ„¢ log service +... | +log files ---> filebeat agent --+ +``` + +## docker-compose.yml + +```yaml +logstash: + image: docker.elastic.co/logstash/logstash:6.1.1 + ports: + - "5044:5044" + - "9600:9600" + volumes: + - ./data/logstash.yml:/usr/share/logstash/config/logstash.yml + - ./data/pipeline:/usr/share/logstash/pipeline + environment: + LS_JAVA_OPTS: "-Xms1g -Xmx1g" + restart: always +``` + +## Up and Running + +```bash +$ docker-compose up -d +$ docker-compose exec logstash bash +>>> logstash-plugin install logstash-output-logservice +Validating logstash-output-logservice +Installing logstash-output-logservice +Installation successful +>>> logstash-plugin list +logstash-output-logservice +>>> exit +$ vim data/pipeline/logstash.conf +$ docker-compose restart +$ docker-compose logs -f +$ curl http://localhost:9600 +{ + "host": "easypi", + "version": "6.1.1", + "http_address": "0.0.0.0:9600", + "id": "c7c4f9d7-5621-4375-bfc9-96abb0f1b4c3", + "name": "6848fe4c533f", + "build_date": "2017-12-17T21:51:17+00:00", + "build_sha": "d46ca0de31662d29b8c5c94d4162e4c760d3f8fb", + "build_snapshot": false +} +``` + +## Setup Filebeat + +```bash +# https://www.elastic.co/downloads/beats/filebeat +$ wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.1.1-amd64.deb +$ dpkg -i filebeat-6.1.1-amd64.deb +$ vim /etc/filebeat/filebeat.yml +#output.elasticsearch: + # Array of hosts to connect to. + #hosts: ["localhost:9200"] +output.logstash: + # The Logstash hosts + hosts: ["1.2.3.4:5044"] +$ systemctl start filebeat +$ systemctl enable filebeat +``` + +## References + +- https://www.elastic.co/guide/en/logstash/current/getting-started-with-logstash.html +- https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-getting-started.html +- https://www.alibabacloud.com/help/zh/doc-detail/28984.htm diff --git a/logstash/data/logstash.yml b/logstash/data/logstash.yml new file mode 100644 index 0000000..26372e3 --- /dev/null +++ b/logstash/data/logstash.yml @@ -0,0 +1,6 @@ +http.host: "0.0.0.0" +path.config: /usr/share/logstash/pipeline +xpack.monitoring.enabled: false +#xpack.monitoring.elasticsearch.url: http://elasticsearch:9200 +#xpack.monitoring.elasticsearch.username: logstash_system +#xpack.monitoring.elasticsearch.password: changeme diff --git a/logstash/data/pipeline/logstash.conf b/logstash/data/pipeline/logstash.conf new file mode 100644 index 0000000..d5a020d --- /dev/null +++ b/logstash/data/pipeline/logstash.conf @@ -0,0 +1,21 @@ +input { + beats { + port => 5044 + } +} + +output { + stdout { + codec => rubydebug + } +# logservice { +# endpoint => "cn-shanghai.log.aliyuncs.com" +# project => "logging" +# logstore => "logstore" +# source => "default" +# topic => "default" +# access_key_id => "XXXXXXXXXXXXXXXX" +# access_key_secret => "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" +# max_send_retry => 3 +# } +} diff --git a/logstash/docker-compose.yml b/logstash/docker-compose.yml new file mode 100644 index 0000000..5feca43 --- /dev/null +++ b/logstash/docker-compose.yml @@ -0,0 +1,11 @@ +logstash: + image: docker.elastic.co/logstash/logstash:6.1.1 + ports: + - "5044:5044" + - "9600:9600" + volumes: + - ./data/logstash.yml:/usr/share/logstash/config/logstash.yml + - ./data/pipeline:/usr/share/logstash/pipeline + environment: + LS_JAVA_OPTS: "-Xms1g -Xmx1g" + restart: always