From 6034156a6daa4280fc280f71d2de552a38cb56e1 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 30 Dec 2020 20:26:48 +0100 Subject: [PATCH] redesigned elk implementation --- roles/docker-elk/Readme.md | 11 +++ roles/docker-elk/defaults/main.yml | 2 + roles/docker-elk/handlers/main.yml | 7 ++ roles/docker-elk/meta/main.yml | 2 + roles/docker-elk/tasks/main.yml | 67 +++++----------- .../templates/docker-compose.yml.j2 | 77 +++++++++++++++++++ ...elasticsearch.yml => elasticsearch.yml.j2} | 0 .../templates/{kibana.yml => kibana.yml.j2} | 0 .../{logstash.conf => logstash.conf.j2} | 0 .../{logstash.yml => logstash.yml.j2} | 0 10 files changed, 120 insertions(+), 46 deletions(-) create mode 100644 roles/docker-elk/Readme.md create mode 100644 roles/docker-elk/defaults/main.yml create mode 100644 roles/docker-elk/handlers/main.yml create mode 100644 roles/docker-elk/meta/main.yml create mode 100644 roles/docker-elk/templates/docker-compose.yml.j2 rename roles/docker-elk/templates/{elasticsearch.yml => elasticsearch.yml.j2} (100%) rename roles/docker-elk/templates/{kibana.yml => kibana.yml.j2} (100%) rename roles/docker-elk/templates/{logstash.conf => logstash.conf.j2} (100%) rename roles/docker-elk/templates/{logstash.yml => logstash.yml.j2} (100%) diff --git a/roles/docker-elk/Readme.md b/roles/docker-elk/Readme.md new file mode 100644 index 00000000..23c38d2c --- /dev/null +++ b/roles/docker-elk/Readme.md @@ -0,0 +1,11 @@ +# role docker-elk + +## restart all services +```bash +docker restart elk_logstash_1 && docker restart elk_elasticsearch_1 && docker restart elk_kibana_1 +``` + +## see +- https://logz.io/blog/elk-stack-on-docker/ +- https://github.com/kevinveenbirkenbach/docker-elk +- https://logz.io/blog/docker-logging/ diff --git a/roles/docker-elk/defaults/main.yml b/roles/docker-elk/defaults/main.yml new file mode 100644 index 00000000..303afa19 --- /dev/null +++ b/roles/docker-elk/defaults/main.yml @@ -0,0 +1,2 @@ +--- +docker_elk_compose_path: "/srv/github.com/kevinveenbirkenbach/docker-elk/" diff --git a/roles/docker-elk/handlers/main.yml b/roles/docker-elk/handlers/main.yml new file mode 100644 index 00000000..e295b64c --- /dev/null +++ b/roles/docker-elk/handlers/main.yml @@ -0,0 +1,7 @@ +--- +- name: recreate docker-elk + command: + cmd: docker-compose up -d --force-recreate + chdir: "{{docker_elk_compose_path}}" + environment: + COMPOSE_HTTP_TIMEOUT: 600 diff --git a/roles/docker-elk/meta/main.yml b/roles/docker-elk/meta/main.yml new file mode 100644 index 00000000..ce762785 --- /dev/null +++ b/roles/docker-elk/meta/main.yml @@ -0,0 +1,2 @@ +dependencies: +- native-nginx-docker-proxy diff --git a/roles/docker-elk/tasks/main.yml b/roles/docker-elk/tasks/main.yml index d7843708..82660534 100644 --- a/roles/docker-elk/tasks/main.yml +++ b/roles/docker-elk/tasks/main.yml @@ -7,60 +7,35 @@ template: src=roles/native-nginx-docker-proxy/templates/domain.conf.j2 dest=/etc/nginx/conf.d/{{domain}}.conf notify: restart nginx -- name: "create /etc/docker-elk" +- name: "create {{docker_elk_compose_path}}" file: - path: "/etc/docker-elk" + path: "{{docker_elk_compose_path}}" state: directory mode: 0755 +- name: git pull docker-elk + git: + repo: "https://github.com/kevinveenbirkenbach/docker-elk.git" + dest: "{{docker_elk_compose_path}}" + update: yes + notify: recreate docker-elk + +- name: copy docker-compose.yml + template: src=docker-compose.yml.j2 dest={{docker_elk_compose_path}}docker-compose.yml + notify: recreate docker-elk + - name: copy elasticsearch.yml - template: src=elasticsearch.yml dest=/etc/docker-elk/elasticsearch.yml + template: src=elasticsearch.yml.j2 dest={{docker_elk_compose_path}}elasticsearch/config/elasticsearch.yml + notify: recreate docker-elk - name: copy kibana.yml - template: src=kibana.yml dest=/etc/docker-elk/kibana.yml + template: src=kibana.yml.j2 dest={{docker_elk_compose_path}}kibana/config/kibana.yml + notify: recreate docker-elk - name: copy logstash.yml - template: src=logstash.yml dest=/etc/docker-elk/logstash.yml + template: src=logstash.yml.j2 dest={{docker_elk_compose_path}}logstash/config/logstash.yml + notify: recreate docker-elk - name: copy logstash.conf - template: src=logstash.conf dest=/etc/docker-elk/logstash.conf - -- name: "setup elk" - docker_compose: - project_name: elk - definition: - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:{{ELK_VERSION}} - restart: always - volumes: - - /etc/docker-elk/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro - - elasticsearch:/usr/share/elasticsearch/data - ports: - - "9200:9200" - - "9300:9300" - environment: - ES_JAVA_OPTS: "-Xmx256m -Xms256m" - ELASTIC_PASSWORD: changeme - # Use single node discovery in order to disable production mode and avoid bootstrap checks. - # see: https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html - discovery.type: single-node - logstash: - image: docker.elastic.co/logstash/logstash:{{ELK_VERSION}} - restart: always - volumes: - - /etc/docker-elk/logstash.yml:/usr/share/logstash/config/logstash.yml:ro - - /etc/docker-elk/logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro - ports: - - "5044:5044" - - "5000:5000/tcp" - - "5000:5000/udp" - - "9600:9600" - environment: - LS_JAVA_OPTS: "-Xmx256m -Xms256m" - kibana: - image: docker.elastic.co/kibana/kibana:{{ELK_VERSION}} - restart: always - volumes: - - /etc/docker-elk/kibana.yml:/usr/share/kibana/config/kibana.yml:ro - ports: - - "5601:5601" + template: src=logstash.conf.j2 dest={{docker_elk_compose_path}}logstash/pipeline/logstash.conf + notify: recreate docker-elk diff --git a/roles/docker-elk/templates/docker-compose.yml.j2 b/roles/docker-elk/templates/docker-compose.yml.j2 new file mode 100644 index 00000000..669e3373 --- /dev/null +++ b/roles/docker-elk/templates/docker-compose.yml.j2 @@ -0,0 +1,77 @@ +version: '3.2' + +services: + elasticsearch: + build: + context: elasticsearch/ + args: + ELK_VERSION: $ELK_VERSION + volumes: + - type: bind + source: ./elasticsearch/config/elasticsearch.yml + target: /usr/share/elasticsearch/config/elasticsearch.yml + read_only: true + - type: volume + source: elasticsearch + target: /usr/share/elasticsearch/data + ports: + - "9200:9200" + - "9300:9300" + environment: + ES_JAVA_OPTS: "-Xmx256m -Xms256m" + ELASTIC_PASSWORD: changeme + # Use single node discovery in order to disable production mode and avoid bootstrap checks. + # see: https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html + discovery.type: single-node + networks: + - elk + + logstash: + build: + context: logstash/ + args: + ELK_VERSION: $ELK_VERSION + volumes: + - type: bind + source: ./logstash/config/logstash.yml + target: /usr/share/logstash/config/logstash.yml + read_only: true + - type: bind + source: ./logstash/pipeline + target: /usr/share/logstash/pipeline + read_only: true + ports: + - "5044:5044" + - "5000:5000/tcp" + - "5000:5000/udp" + - "9600:9600" + environment: + LS_JAVA_OPTS: "-Xmx256m -Xms256m" + networks: + - elk + depends_on: + - elasticsearch + + kibana: + build: + context: kibana/ + args: + ELK_VERSION: $ELK_VERSION + volumes: + - type: bind + source: ./kibana/config/kibana.yml + target: /usr/share/kibana/config/kibana.yml + read_only: true + ports: + - "5601:5601" + networks: + - elk + depends_on: + - elasticsearch + +networks: + elk: + driver: bridge + +volumes: + elasticsearch: diff --git a/roles/docker-elk/templates/elasticsearch.yml b/roles/docker-elk/templates/elasticsearch.yml.j2 similarity index 100% rename from roles/docker-elk/templates/elasticsearch.yml rename to roles/docker-elk/templates/elasticsearch.yml.j2 diff --git a/roles/docker-elk/templates/kibana.yml b/roles/docker-elk/templates/kibana.yml.j2 similarity index 100% rename from roles/docker-elk/templates/kibana.yml rename to roles/docker-elk/templates/kibana.yml.j2 diff --git a/roles/docker-elk/templates/logstash.conf b/roles/docker-elk/templates/logstash.conf.j2 similarity index 100% rename from roles/docker-elk/templates/logstash.conf rename to roles/docker-elk/templates/logstash.conf.j2 diff --git a/roles/docker-elk/templates/logstash.yml b/roles/docker-elk/templates/logstash.yml.j2 similarity index 100% rename from roles/docker-elk/templates/logstash.yml rename to roles/docker-elk/templates/logstash.yml.j2