sfsdf

docker/elk/extensions/README.md

@@ -0,0 +1,3 @@
+# Extensions
+
+Third-party extensions that enable extra integrations with the Elastic stack.

docker/elk/extensions/apm-server/Dockerfile

@@ -0,0 +1,3 @@
+ARG ELK_VERSION
+
+FROM docker.elastic.co/apm/apm-server:${ELK_VERSION}

docker/elk/extensions/apm-server/README.md

@@ -0,0 +1,56 @@
+# APM Server extension
+
+The APM Server receives data from APM agents and transforms them into Elasticsearch documents that can be visualised in
+Kibana.
+
+## Usage
+
+To include APM Server in the stack, run Docker Compose from the root of the repository with an additional command line
+argument referencing the `apm-server-compose.yml` file:
+
+```console
+$ docker-compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml up
+```
+
+Meanwhile, you can navigate to the **APM** application in Kibana and follow the setup instructions to get started.
+
+## Connecting an agent to APM Server
+
+The most basic configuration to send traces to APM server is to specify the `SERVICE_NAME` and `SERVICE_URL`. Here is an
+example Python Flask configuration:
+
+```python
+import elasticapm
+from elasticapm.contrib.flask import ElasticAPM
+
+from flask import Flask
+
+app = Flask(__name__)
+app.config['ELASTIC_APM'] = {
+    # Set required service name. Allowed characters:
+    # a-z, A-Z, 0-9, -, _, and space
+    'SERVICE_NAME': 'PYTHON_FLASK_TEST_APP',
+
+    # Set custom APM Server URL (default: http://localhost:8200)
+    'SERVER_URL': 'http://apm-server:8200',
+
+    'DEBUG': True,
+}
+```
+
+Configuration settings for each supported language are available in the APM documentation: [APM Agents][apm-agents].
+
+## Checking connectivity and importing default APM dashboards
+
+1. On the Kibana home page, click `Add APM` under the _Observability_ panel.
+1. Click `Check APM Server status` to confirm the server is up and running.
+1. Click `Check agent status` to verify your agent has registered properly.
+1. Click `Load Kibana objects` to create an index pattern for APM.
+1. Click `Launch APM` to be taken to the APM dashboard.
+
+## See also
+
+[Running APM Server on Docker][apm-docker]
+
+[apm-agents]: https://www.elastic.co/guide/en/apm/get-started/current/components.html#_apm_agents
+[apm-docker]: https://www.elastic.co/guide/en/apm/server/current/running-on-docker.html

docker/elk/extensions/apm-server/apm-server-compose.yml

@@ -0,0 +1,23 @@
+version: '3.2'
+
+services:
+  apm-server:
+    build:
+      context: extensions/apm-server/
+      args:
+        ELK_VERSION: $ELK_VERSION
+    command:
+        # Disable strict permission checking on 'apm-server.yml' configuration file
+        # https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html
+      - --strict.perms=false
+    volumes:
+      - type: bind
+        source: ./extensions/apm-server/config/apm-server.yml
+        target: /usr/share/apm-server/apm-server.yml
+        read_only: true
+    ports:
+      - '8200:8200'
+    networks:
+      - elk
+    depends_on:
+      - elasticsearch

docker/elk/extensions/apm-server/config/apm-server.yml

@@ -0,0 +1,8 @@
+apm-server:
+  host: 0.0.0.0:8200
+
+output:
+  elasticsearch:
+    hosts: ['http://elasticsearch:9200']
+    username: elastic
+    password: changeme

docker/elk/extensions/curator/Dockerfile

@@ -0,0 +1,17 @@
+FROM bitnami/elasticsearch-curator:5.8.1
+
+USER root
+
+RUN install_packages cron && \
+    echo \
+        '* * * * *' \
+        root \
+        LC_ALL=C.UTF-8 LANG=C.UTF-8 \
+        /opt/bitnami/python/bin/curator \
+        --config=/usr/share/curator/config/curator.yml \
+        /usr/share/curator/config/delete_log_files_curator.yml \
+        '>/proc/1/fd/1' '2>/proc/1/fd/2' \
+        >>/etc/crontab
+
+ENTRYPOINT ["cron"]
+CMD ["-f", "-L8"]

docker/elk/extensions/curator/README.md

@@ -0,0 +1,20 @@
+# Curator
+
+Elasticsearch Curator helps you curate or manage your indices.
+
+## Usage
+
+If you want to include the Curator extension, run Docker Compose from the root of the repository with an additional
+command line argument referencing the `curator-compose.yml` file:
+
+```bash
+$ docker-compose -f docker-compose.yml -f extensions/curator/curator-compose.yml up
+```
+
+This sample setup demonstrates how to run `curator` every minute using `cron`.
+
+All configuration files are available in the `config/` directory.
+
+## Documentation
+
+[Curator Reference](https://www.elastic.co/guide/en/elasticsearch/client/curator/current/index.html)

docker/elk/extensions/curator/config/curator.yml

@@ -0,0 +1,12 @@
+# Curator configuration
+# https://www.elastic.co/guide/en/elasticsearch/client/curator/current/configfile.html
+
+client:
+  hosts:
+    - elasticsearch
+  port: 9200
+  http_auth: elastic:changeme
+
+logging:
+  loglevel: INFO
+  logformat: default

docker/elk/extensions/curator/config/delete_log_files_curator.yml

@@ -0,0 +1,21 @@
+actions:
+  1:
+    action: delete_indices
+    description: >-
+      Delete indices. Find which to delete by first limiting the list to
+      logstash- prefixed indices. Then further filter those to prevent deletion
+      of anything less than the number of days specified by unit_count.
+      Ignore the error if the filter does not result in an actionable list of
+      indices (ignore_empty_list) and exit cleanly.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: logstash-
+    - filtertype: age
+      source: creation_date
+      direction: older
+      unit: days
+      unit_count: 2

docker/elk/extensions/curator/curator-compose.yml

@@ -0,0 +1,20 @@
+version: '3.2'
+
+services:
+  curator:
+    build:
+      context: extensions/curator/
+    init: true
+    volumes:
+      - type: bind
+        source: ./extensions/curator/config/curator.yml
+        target: /usr/share/curator/config/curator.yml
+        read_only: true
+      - type: bind
+        source: ./extensions/curator/config/delete_log_files_curator.yml
+        target: /usr/share/curator/config/delete_log_files_curator.yml
+        read_only: true
+    networks:
+      - elk
+    depends_on:
+      - elasticsearch

docker/elk/extensions/enterprise-search/Dockerfile

@@ -0,0 +1,4 @@
+ARG ELK_VERSION
+
+# https://www.docker.elastic.co/
+FROM docker.elastic.co/enterprise-search/enterprise-search:${ELK_VERSION}

docker/elk/extensions/enterprise-search/README.md

@@ -0,0 +1,134 @@
+# Enterprise Search extension
+
+Elastic Enterprise Search is a suite of products for search applications backed by the Elastic Stack.
+
+## Requirements
+
+* 2 GB of free RAM, on top of the resources required by the other stack components and extensions.
+
+Enterprise Search exposes the TCP port `3002` for its Web UI and API.
+
+## Usage
+
+### Generate an encryption key
+
+Enterprise Search requires one or more [encryption keys][enterprisesearch-encryption] to be configured before the
+initial startup. Failing to do so prevents the server from starting.
+
+Encryption keys can contain any series of characters. Elastic recommends using 256-bit keys for optimal security.
+
+Those encryption keys must be added manually to the [`config/enterprise-search.yml`][config-enterprisesearch] file. By
+default, the list of encryption keys is empty and must be populated using one of the following formats:
+
+```yaml
+secret_management.encryption_keys:
+  - my_first_encryption_key
+  - my_second_encryption_key
+  - ...
+```
+
+```yaml
+secret_management.encryption_keys: [my_first_encryption_key, my_second_encryption_key, ...]
+```
+
+> :information_source: To generate a strong encryption key, for example using the AES-256 cipher, you can use the
+> OpenSSL utility or any other online/offline tool of your choice:
+>
+> ```console
+> $ openssl enc -aes-256 -P
+>
+> enter aes-256-cbc encryption password: <a strong password>
+> Verifying - enter aes-256-cbc encryption password: <repeat your strong password>
+> ...
+>
+> key=<generated AES key>
+> ```
+
+### Enable Elasticsearch's API key service
+
+Enterprise Search requires Elasticsearch's built-in [API key service][es-security] to be enabled in order to start.
+Unless Elasticsearch is configured to enable TLS on the HTTP interface (disabled by default), this service is disabled
+by default.
+
+To enable it, modify the Elasticsearch configuration file in [`elasticsearch/config/elasticsearch.yml`][config-es] and
+add the following setting:
+
+```yaml
+xpack.security.authc.api_key.enabled: true
+```
+
+### Start the server
+
+To include Enterprise Search in the stack, run Docker Compose from the root of the repository with an additional command
+line argument referencing the `enterprise-search-compose.yml` file:
+
+```console
+$ docker-compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml up
+```
+
+Allow a few minutes for the stack to start, then open your web browser at the address <http://localhost:3002> to see the
+Enterprise Search home page.
+
+Enterprise Search is configured on first boot with the following default credentials:
+
+* user: *enterprise_search*
+* password: *changeme*
+
+## Security
+
+The Enterprise Search password is defined inside the Compose file via the `ENT_SEARCH_DEFAULT_PASSWORD` environment
+variable. We highly recommend choosing a more secure password than the default one for security reasons.
+
+To do so, change the value `ENT_SEARCH_DEFAULT_PASSWORD` environment variable inside the Compose file **before the first
+boot**:
+
+```yaml
+enterprise-search:
+
+  environment:
+    ENT_SEARCH_DEFAULT_PASSWORD: {{some strong password}}
+```
+
+> :warning: The default Enterprise Search password can only be set during the initial boot. Once the password is
+> persisted in Elasticsearch, it can only be changed via the Elasticsearch API.
+
+For more information, please refer to [User Management and Security][enterprisesearch-security].
+
+## Configuring Enterprise Search
+
+The Enterprise Search configuration is stored in [`config/enterprise-search.yml`][config-enterprisesearch]. You can
+modify this file using the [Default Enterprise Search configuration][enterprisesearch-config] as a reference.
+
+You can also specify the options you want to override by setting environment variables inside the Compose file:
+
+```yaml
+enterprise-search:
+
+  environment:
+    ent_search.auth.source: standard
+    worker.threads: '6'
+```
+
+Any change to the Enterprise Search configuration requires a restart of the Enterprise Search container:
+
+```console
+$ docker-compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml restart enterprise-search
+```
+
+Please refer to the following documentation page for more details about how to configure Enterprise Search inside a
+Docker container: [Running Enterprise Search Using Docker][enterprisesearch-docker].
+
+## See also
+
+[Enterprise Search documentation][enterprisesearch-docs]
+
+[config-enterprisesearch]: ./config/enterprise-search.yml
+
+[enterprisesearch-encryption]: https://www.elastic.co/guide/en/enterprise-search/current/encryption-keys.html
+[enterprisesearch-security]: https://www.elastic.co/guide/en/workplace-search/current/workplace-search-security.html
+[enterprisesearch-config]: https://www.elastic.co/guide/en/enterprise-search/current/configuration.html
+[enterprisesearch-docker]: https://www.elastic.co/guide/en/enterprise-search/current/docker.html
+[enterprisesearch-docs]: https://www.elastic.co/guide/en/enterprise-search/current/index.html
+
+[es-security]: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html#api-key-service-settings
+[config-es]: ../../elasticsearch/config/elasticsearch.yml

docker/elk/extensions/enterprise-search/config/enterprise-search.yml

@@ -0,0 +1,27 @@
+---
+## Enterprise Search core configuration
+## https://www.elastic.co/guide/en/enterprise-search/current/configuration.html
+#
+
+## --------------------- REQUIRED ---------------------
+
+# Encryption keys to protect application secrets.
+secret_management.encryption_keys:
+  # add encryption keys below
+  #- add encryption keys here
+
+## ----------------------------------------------------
+
+# IP address Enterprise Search listens on
+ent_search.listen_host: 0.0.0.0
+
+# URL at which users reach Enterprise Search
+ent_search.external_url: http://localhost:3002
+
+# Elasticsearch URL and credentials
+elasticsearch.host: http://elasticsearch:9200
+elasticsearch.username: elastic
+elasticsearch.password: changeme
+
+# Allow Enterprise Search to modify Elasticsearch settings. Used to enable auto-creation of Elasticsearch indexes.
+allow_es_settings_modification: true

docker/elk/extensions/enterprise-search/enterprise-search-compose.yml

@@ -0,0 +1,22 @@
+version: '3.2'
+
+services:
+  enterprise-search:
+    build:
+      context: extensions/enterprise-search/
+      args:
+        ELK_VERSION: $ELK_VERSION
+    volumes:
+      - type: bind
+        source: ./extensions/enterprise-search/config/enterprise-search.yml
+        target: /usr/share/enterprise-search/config/enterprise-search.yml
+        read_only: true
+    environment:
+      JAVA_OPTS: -Xmx2g -Xms2g
+      ENT_SEARCH_DEFAULT_PASSWORD: changeme
+    ports:
+      - '3002:3002'
+    networks:
+      - elk
+    depends_on:
+      - elasticsearch

docker/elk/extensions/logspout/Dockerfile

@@ -0,0 +1,5 @@
+# uses ONBUILD instructions described here:
+# https://github.com/gliderlabs/logspout/tree/master/custom
+
+FROM gliderlabs/logspout:master
+ENV SYSLOG_FORMAT rfc3164

docker/elk/extensions/logspout/README.md

@@ -0,0 +1,28 @@
+# Logspout extension
+
+Logspout collects all Docker logs using the Docker logs API, and forwards them to Logstash without any additional
+configuration.
+
+## Usage
+
+If you want to include the Logspout extension, run Docker Compose from the root of the repository with an additional
+command line argument referencing the `logspout-compose.yml` file:
+
+```bash
+$ docker-compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml up
+```
+
+In your Logstash pipeline configuration, enable the `udp` input and set the input codec to `json`:
+
+```logstash
+input {
+  udp {
+    port  => 5000
+    codec => json
+  }
+}
+```
+
+## Documentation
+
+<https://github.com/looplab/logspout-logstash>

docker/elk/extensions/logspout/build.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+# source: https://github.com/gliderlabs/logspout/blob/621524e/custom/build.sh
+
+set -e
+apk add --update go build-base git mercurial ca-certificates
+cd /src
+go build -ldflags "-X main.Version=$1" -o /bin/logspout
+apk del go git mercurial build-base
+rm -rf /root/go /var/cache/apk/*
+
+# backwards compatibility
+ln -fs /tmp/docker.sock /var/run/docker.sock

docker/elk/extensions/logspout/logspout-compose.yml

@@ -0,0 +1,16 @@
+version: '3.2'
+
+services:
+  logspout:
+    build:
+      context: extensions/logspout
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    environment:
+      ROUTE_URIS: logstash://logstash:5000
+      LOGSTASH_TAGS: docker-elk
+    networks:
+      - elk
+    depends_on:
+      - logstash
+    restart: on-failure

docker/elk/extensions/logspout/modules.go

@@ -0,0 +1,10 @@
+package main
+
+// installs the Logstash adapter for Logspout, and required dependencies
+// https://github.com/looplab/logspout-logstash
+import (
+	_ "github.com/gliderlabs/logspout/healthcheck"
+	_ "github.com/gliderlabs/logspout/transports/tcp"
+	_ "github.com/gliderlabs/logspout/transports/udp"
+	_ "github.com/looplab/logspout-logstash"
+)

docker/elk/extensions/metricbeat/Dockerfile

@@ -0,0 +1,3 @@
+ARG ELK_VERSION
+
+FROM docker.elastic.co/beats/metricbeat:${ELK_VERSION}

docker/elk/extensions/metricbeat/README.md

@@ -0,0 +1,36 @@
+# Metricbeat
+
+Metricbeat is a lightweight shipper that you can install on your servers to periodically collect metrics from the
+operating system and from services running on the server. Metricbeat takes the metrics and statistics that it collects
+and ships them to the output that you specify, such as Elasticsearch or Logstash.
+
+## Usage
+
+To include Metricbeat in the stack, run Docker Compose from the root of the repository with an additional command line
+argument referencing the `metricbeat-compose.yml` file:
+
+```console
+$ docker-compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml up
+```
+
+## Configuring Metricbeat
+
+The Metricbeat configuration is stored in [`config/metricbeat.yml`](./config/metricbeat.yml). You can modify this file
+with the help of the [Configuration reference][metricbeat-config].
+
+Any change to the Metricbeat configuration requires a restart of the Metricbeat container:
+
+```console
+$ docker-compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml restart metricbeat
+```
+
+Please refer to the following documentation page for more details about how to configure Metricbeat inside a
+Docker container: [Run Metricbeat on Docker][metricbeat-docker].
+
+## See also
+
+[Metricbeat documentation][metricbeat-doc]
+
+[metricbeat-config]: https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-reference-yml.html
+[metricbeat-docker]: https://www.elastic.co/guide/en/beats/metricbeat/current/running-on-docker.html
+[metricbeat-doc]: https://www.elastic.co/guide/en/beats/metricbeat/current/index.html

docker/elk/extensions/metricbeat/config/metricbeat.yml

@@ -0,0 +1,44 @@
+## Metricbeat configuration
+## https://github.com/elastic/beats/blob/master/deploy/docker/metricbeat.docker.yml
+#
+
+metricbeat.config:
+  modules:
+    path: ${path.config}/modules.d/*.yml
+    # Reload module configs as they change:
+    reload.enabled: false
+
+metricbeat.autodiscover:
+  providers:
+    - type: docker
+      hints.enabled: true
+
+metricbeat.modules:
+- module: docker
+  metricsets:
+    - container
+    - cpu
+    - diskio
+    - healthcheck
+    - info
+    #- image
+    - memory
+    - network
+  hosts: ['unix:///var/run/docker.sock']
+  period: 10s
+  enabled: true
+
+processors:
+  - add_cloud_metadata: ~
+
+output.elasticsearch:
+  hosts: ['http://elasticsearch:9200']
+  username: elastic
+  password: changeme
+
+## HTTP endpoint for health checking
+## https://www.elastic.co/guide/en/beats/metricbeat/current/http-endpoint.html
+#
+
+http.enabled: true
+http.host: 0.0.0.0

docker/elk/extensions/metricbeat/metricbeat-compose.yml

@@ -0,0 +1,46 @@
+version: '3.2'
+
+services:
+  metricbeat:
+    build:
+      context: extensions/metricbeat/
+      args:
+        ELK_VERSION: $ELK_VERSION
+    # Run as 'root' instead of 'metricbeat' (uid 1000) to allow reading
+    # 'docker.sock' and the host's filesystem.
+    user: root
+    command:
+        # Log to stderr.
+      - -e
+        # Disable config file permissions checks. Allows mounting
+        # 'config/metricbeat.yml' even if it's not owned by root.
+        # see: https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html
+      - --strict.perms=false
+        # Mount point of the host’s filesystem. Required to monitor the host
+        # from within a container.
+      - --system.hostfs=/hostfs
+    volumes:
+      - type: bind
+        source: ./extensions/metricbeat/config/metricbeat.yml
+        target: /usr/share/metricbeat/metricbeat.yml
+        read_only: true
+      - type: bind
+        source: /
+        target: /hostfs
+        read_only: true
+      - type: bind
+        source: /sys/fs/cgroup
+        target: /hostfs/sys/fs/cgroup
+        read_only: true
+      - type: bind
+        source: /proc
+        target: /hostfs/proc
+        read_only: true
+      - type: bind
+        source: /var/run/docker.sock
+        target: /var/run/docker.sock
+        read_only: true
+    networks:
+      - elk
+    depends_on:
+      - elasticsearch