Initial Adjustments for Graylog4 and Elasticsearch 7

Docker/docker-compose.yml

@@ -4,13 +4,13 @@ services:
 
   # MongoDB: https://hub.docker.com/_/mongo/
   mongodb:
-    image: mongo:3
+    image: mongo:4.2
     volumes:
       - mongo_data:/data/db
 
   # Elasticsearch: https://www.elastic.co/guide/en/elasticsearch/reference/6.x/docker.html
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.5
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.11.1
     mem_limit: 4g
     restart: always
     volumes:
@@ -33,9 +33,11 @@ services:
       - ./graylog/service-names-port-numbers.csv:/etc/graylog/server/service-names-port-numbers.csv
     env_file:
       - ./graylog.env
+    entrypoint: /usr/bin/tini -- wait-for-it elasticsearch:9200 --  /docker-entrypoint.sh
     links:
       - mongodb:mongo
       - elasticsearch
+    restart: always
     depends_on:
       - mongodb
       - elasticsearch
@@ -57,7 +59,7 @@ services:
 
   # Kibana : https://www.elastic.co/guide/en/kibana/6.8/index.html
   kibana:
-    image: docker.elastic.co/kibana/kibana-oss:6.8.5
+    image: docker.elastic.co/kibana/kibana:7.11.1
     env_file:
       - kibana.env
     depends_on:

Docker/elasticsearch.env

@@ -1,5 +1,5 @@
 http.host=0.0.0.0
-transport.host=0.0.0.0
+transport.host=localhost
 network.host=0.0.0.0
 "ES_JAVA_OPTS=-Xms1g -Xmx1g"
 ES_HEAP_SIZE=2g

Docker/graylog.env

@@ -5,3 +5,4 @@ GRAYLOG_ROOT_PASSWORD_SHA2=8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6
 GRAYLOG_HTTP_EXTERNAL_URI=http://localhost:9000/
 # TZ List - https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
 GRAYLOG_TIMEZONE=Europe/Berlin
+GRAYLOG_HTTP_EXTERNAL_URI=http://pfanalytics.home:9000/

Docker/graylog/Dockerfile

@@ -1,4 +1,4 @@
-FROM graylog/graylog:3.1
+FROM graylog/graylog:4.0
 # Probably a bad idea, but it works for now
 USER root
 RUN mkdir -pv /etc/graylog/server/

Docker/graylog/getGeo.sh

@@ -1,2 +1,2 @@
-curl --output /etc/graylog/server/mm.tar.gz https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz
+curl --output /etc/graylog/server/mm.tar.gz "https://download.maxmind.com/app/geoip_download_by_token?edition_id=GeoLite2-City&date=20210216&suffix=tar.gz&token=v2.local.jM7J0O4PMocBknBIc2Hkh1gO4VKQ9sBPM72EOg5i9KVuJL_rOchpeHh7uA9k0cc752E1lj9pWMQsOofvbSqFWW7GcJdsWXXqDONgiyW7_Zxg6UVvREHEa7g9pd7tne5oZG-KZOZx-VjCM_g6CNb2ccblHVnEiAjD9jzSZdY8QcNNMu7qYBMfXvlMQKHlrJvTM0oJgg"
 tar zxvf /etc/graylog/server/mm.tar.gz -C /etc/graylog/server/ --strip-components=1

Elasticsearch_pfsense_custom_template/pfsense_custom_template_es7.json

@@ -0,0 +1,92 @@
+{
+    "order": -1,
+    "index_patterns": [
+      "pfsense_*"
+    ],
+    "settings": {
+      "index": {
+        "analysis": {
+          "analyzer": {
+            "analyzer_keyword": {
+              "filter": "lowercase",
+              "tokenizer": "keyword"
+            }
+          }
+        }
+      }
+    },
+    "mappings": {
+      "_source": {
+        "enabled": true
+      },
+      "dynamic_templates": [
+        {
+          "internal_fields": {
+            "mapping": {
+              "type": "keyword"
+            },
+            "match_mapping_type": "string",
+            "match": "gl2_*"
+          }
+        },
+        {
+          "store_generic": {
+            "mapping": {
+              "type": "keyword"
+            },
+            "match_mapping_type": "string"
+          }
+        }
+      ],
+      "properties": {
+        "src_location": {
+          "type": "geo_point"
+        },
+        "gl2_processing_timestamp": {
+          "format": "uuuu-MM-dd HH:mm:ss.SSS",
+          "type": "date"
+        },
+        "gl2_accounted_message_size": {
+          "type": "long"
+        },
+        "dst_location": {
+          "type": "geo_point"
+        },
+        "gl2_receive_timestamp": {
+          "format": "uuuu-MM-dd HH:mm:ss.SSS",
+          "type": "date"
+        },
+        "full_message": {
+          "fielddata": false,
+          "analyzer": "standard",
+          "type": "text"
+        },
+        "streams": {
+          "type": "keyword"
+        },
+        "dest_ip_geolocation": {
+          "copy_to": "dst_location",
+          "type": "text"
+        },
+        "src_ip_geolocation": {
+          "copy_to": "src_location",
+          "type": "text"
+        },
+        "source": {
+          "fielddata": true,
+          "analyzer": "analyzer_keyword",
+          "type": "text"
+        },
+        "message": {
+          "fielddata": false,
+          "analyzer": "standard",
+          "type": "text"
+        },
+        "timestamp": {
+          "format": "uuuu-MM-dd HH:mm:ss.SSS",
+          "type": "date"
+        }
+      }
+    },
+    "aliases": {}
+  }

README.md

@@ -70,6 +70,8 @@ A salt for encrypting your graylog passwords
 - GRAYLOG_PASSWORD_SECRET (Change that _now_)
 
 
+Edit Docker/graylog/getGeo.sh and insert _your_ tokenized Downloadlink of the Maxmind GeoIP Database. Create an account on https://www.maxmind.com/en/account/login and go to "My Account -> Download Files -> GeoLite2 City" and copy the Link "Download GZIP" to your getGeo.sh File. If you don't do that the geographic lookup of IP Addresses won't work.
+
 Finally, spin up the stack with:
 
 ```
@@ -77,7 +79,7 @@ cd ./Docker
 sudo docker-compose up -d
 ```
 
-Note: graylog will be built the first time you run docker-compose.  The below step is only for updating the GeiLite DB.
+Note: graylog will be built the first time you run docker-compose.  The below step is only for updating the GeoLite DB.
 To update the geolite.maxmind.com GeoLite2-City database, simply run:
 ```
 cd ./Docker