Merge pull request #15412 from marefr/ha_loki

devenv: loki support in ha test

devenv/docker/ha_test/docker-compose.yaml

@@ -7,6 +7,13 @@ services:
       - "80:80"
       - "80:80"
     volumes:
     volumes:
       - /var/run/docker.sock:/tmp/docker.sock:ro
       - /var/run/docker.sock:/tmp/docker.sock:ro
+    depends_on:
+      db:
+        condition: service_healthy
+    logging:
+      driver: "fluentd"
+      options:
+        tag: nginx
 
 
   db:
   db:
     image: mysql:5.6
     image: mysql:5.6
@@ -66,6 +73,8 @@ services:
       # - GF_DATABASE_SSL_MODE=disable
       # - GF_DATABASE_SSL_MODE=disable
       # - GF_SESSION_PROVIDER=postgres
       # - GF_SESSION_PROVIDER=postgres
       # - GF_SESSION_PROVIDER_CONFIG=user=grafana password=password host=db port=5432 dbname=grafana sslmode=disable
       # - GF_SESSION_PROVIDER_CONFIG=user=grafana password=password host=db port=5432 dbname=grafana sslmode=disable
+      - GF_SERVER_ROUTER_LOGGING=true
+      - GF_LOG_CONSOLE_FORMAT=json
       - GF_LOG_FILTERS=alerting.notifier:debug,alerting.notifier.slack:debug,auth:debug
       - GF_LOG_FILTERS=alerting.notifier:debug,alerting.notifier.slack:debug,auth:debug
       - GF_AUTH_TOKEN_ROTATION_INTERVAL_MINUTES=2
       - GF_AUTH_TOKEN_ROTATION_INTERVAL_MINUTES=2
     ports:
     ports:
@@ -73,6 +82,10 @@ services:
     depends_on:
     depends_on:
       db:
       db:
         condition: service_healthy
         condition: service_healthy
+    logging:
+      driver: "fluentd"
+      options:
+        tag: grafana
 
 
   prometheus:
   prometheus:
     image: prom/prometheus:v2.4.2
     image: prom/prometheus:v2.4.2
@@ -82,3 +95,21 @@ services:
       - VIRTUAL_HOST=prometheus.loc
       - VIRTUAL_HOST=prometheus.loc
     ports:
     ports:
       - 9090
       - 9090
+
+  loki:
+    image: grafana/loki:master
+    environment:
+      - VIRTUAL_HOST=loki.loc
+    ports:
+      - 3100
+    command: -config.file=/etc/loki/local-config.yaml
+
+  fluentd:
+    image: grafana/fluent-plugin-loki:master
+    volumes:
+      - ./fluentd/fluentd.conf:/fluentd/etc/fluentd.conf
+    links:
+      - loki
+    ports:
+      - "24224:24224"
+      - "24224:24224/udp"

devenv/docker/ha_test/fluentd/fluentd.conf

+<source>
+  @type forward
+  port 24224
+  bind 0.0.0.0
+</source>
+
+<filter grafana>
+  @type parser
+  <parse>
+    @type json
+    json_parser json
+  </parse>
+  replace_invalid_sequence true
+  emit_invalid_record_to_error false
+  key_name log
+  reserve_data true
+  remove_key_name_field true
+</filter>
+
+<filter nginx>
+  @type parser
+  format /^.*m(?<host>[^ ]*) (?<remote>[^ ]*) \- \- \[(?<time>[^\]]*)\] \"(?<method>\S+)(?: +(?<path>[^\"]*) +\S*)?\" (?<code>[^ ]*) (?<size>[^ ]*)(?: \"(?<referer>[^\"]*)\" \"(?<agent>[^\"]*)\")?$/
+  time_format %d/%b/%Y:%H:%M:%S %z
+  key_name log
+  reserve_data true
+  remove_key_name_field true
+</filter>
+
+<filter **>
+  @type record_transformer
+  remove_keys "source,t"
+</filter>
+
+<match grafana>
+  @type copy
+  <store>
+    @type stdout
+    output_type json
+  </store>
+  <store>
+    @type loki
+    url "http://loki:3100"
+    extra_labels {"app":"grafana"}
+    label_keys "container_name,container_id,logger"
+    flush_interval 10s
+    flush_at_shutdown true
+    buffer_chunk_limit 1m
+  </store>
+</match>
+
+<filter nginx>
+  @type record_transformer
+  <record>
+    lvl "info"
+  </record>
+</filter>
+
+<match nginx>
+  @type copy
+  <store>
+    @type stdout
+    output_type json
+  </store>
+  <store>
+    @type loki
+    url "http://loki:3100"
+    extra_labels {"app":"nginx"}
+    label_keys "container_name,container_id"
+    flush_interval 10s
+    flush_at_shutdown true
+    buffer_chunk_limit 1m
+  </store>
+</match>
\ No newline at end of file

devenv/docker/ha_test/grafana/provisioning/datasources/datasources.yaml

@@ -8,4 +8,9 @@ datasources:
     jsonData:
     jsonData:
       timeInterval: 10s
       timeInterval: 10s
       queryTimeout: 30s
       queryTimeout: 30s
-      httpMethod: POST
+      httpMethod: POST
\ No newline at end of file
+
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100

devenv/docker/ha_test/prometheus/prometheus.yml

@@ -36,4 +36,12 @@ scrape_configs:
         - 'mysqld-exporter'
         - 'mysqld-exporter'
         type: 'A'
         type: 'A'
         port: 9104
         port: 9104
+        refresh_interval: 10s
+
+  - job_name: 'loki'
+    dns_sd_configs:
+      - names:
+        - 'loki'
+        type: 'A'
+        port: 3100
         refresh_interval: 10s
         refresh_interval: 10s
\ No newline at end of file