Adjusted Documentation, removed redundant Variable

Docker/docker-compose.yml

@@ -4,13 +4,13 @@ services:
 
   # MongoDB: https://hub.docker.com/_/mongo/
   mongodb:
-    image: mongo:3
+    image: mongo:4.2
     volumes:
       - mongo_data:/data/db
 
   # Elasticsearch: https://www.elastic.co/guide/en/elasticsearch/reference/6.x/docker.html
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.5
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.11.1
     mem_limit: 4g
     restart: always
     volumes:
@@ -30,12 +30,14 @@ services:
       context: ./graylog/.
     volumes:
       - graylog_journal:/usr/share/graylog/data/journal
-      - ./service-names-port-numbers.csv:/etc/graylog/server/service-names-port-numbers.csv
+      - ./graylog/service-names-port-numbers.csv:/etc/graylog/server/service-names-port-numbers.csv
     env_file:
       - ./graylog.env
+    entrypoint: /usr/bin/tini -- wait-for-it elasticsearch:9200 --  /docker-entrypoint.sh
     links:
       - mongodb:mongo
       - elasticsearch
+    restart: always
     depends_on:
       - mongodb
       - elasticsearch
@@ -57,7 +59,7 @@ services:
 
   # Kibana : https://www.elastic.co/guide/en/kibana/6.8/index.html
   kibana:
-    image: docker.elastic.co/kibana/kibana-oss:6.8.5
+    image: docker.elastic.co/kibana/kibana:7.11.1
     env_file:
       - kibana.env
     depends_on:

Docker/elasticsearch.env

@@ -1,5 +1,5 @@
 http.host=0.0.0.0
-transport.host=0.0.0.0
+transport.host=localhost
 network.host=0.0.0.0
 "ES_JAVA_OPTS=-Xms1g -Xmx1g"
 ES_HEAP_SIZE=2g

Docker/graylog.env

@@ -4,4 +4,4 @@ GRAYLOG_PASSWORD_SECRET=somepasswordpepperzzz
 GRAYLOG_ROOT_PASSWORD_SHA2=8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918
 GRAYLOG_HTTP_EXTERNAL_URI=http://localhost:9000/
 # TZ List - https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
-GRAYLOG_TIMEZONE=Europe/Berlin
+GRAYLOG_TIMEZONE=Europe/Berlin

Docker/graylog/Dockerfile

@@ -1,4 +1,4 @@
-FROM graylog/graylog:3.1
+FROM graylog/graylog:4.0
 # Probably a bad idea, but it works for now
 USER root
 RUN mkdir -pv /etc/graylog/server/

Docker/graylog/getGeo.sh

@@ -1,2 +1,2 @@
-curl --output /etc/graylog/server/mm.tar.gz https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz
+curl --output /etc/graylog/server/mm.tar.gz "https://download.maxmind.com/app/geoip_download_by_token?edition_id=GeoLite2-City&date=20210216&suffix=tar.gz&token=v2.local.jM7J0O4PMocBknBIc2Hkh1gO4VKQ9sBPM72EOg5i9KVuJL_rOchpeHh7uA9k0cc752E1lj9pWMQsOofvbSqFWW7GcJdsWXXqDONgiyW7_Zxg6UVvREHEa7g9pd7tne5oZG-KZOZx-VjCM_g6CNb2ccblHVnEiAjD9jzSZdY8QcNNMu7qYBMfXvlMQKHlrJvTM0oJgg"
 tar zxvf /etc/graylog/server/mm.tar.gz -C /etc/graylog/server/ --strip-components=1

Elasticsearch_pfsense_custom_template/pfsense_custom_template_es7.json

@@ -0,0 +1,92 @@
+{
+    "order": -1,
+    "index_patterns": [
+      "pfsense_*"
+    ],
+    "settings": {
+      "index": {
+        "analysis": {
+          "analyzer": {
+            "analyzer_keyword": {
+              "filter": "lowercase",
+              "tokenizer": "keyword"
+            }
+          }
+        }
+      }
+    },
+    "mappings": {
+      "_source": {
+        "enabled": true
+      },
+      "dynamic_templates": [
+        {
+          "internal_fields": {
+            "mapping": {
+              "type": "keyword"
+            },
+            "match_mapping_type": "string",
+            "match": "gl2_*"
+          }
+        },
+        {
+          "store_generic": {
+            "mapping": {
+              "type": "keyword"
+            },
+            "match_mapping_type": "string"
+          }
+        }
+      ],
+      "properties": {
+        "src_location": {
+          "type": "geo_point"
+        },
+        "gl2_processing_timestamp": {
+          "format": "uuuu-MM-dd HH:mm:ss.SSS",
+          "type": "date"
+        },
+        "gl2_accounted_message_size": {
+          "type": "long"
+        },
+        "dst_location": {
+          "type": "geo_point"
+        },
+        "gl2_receive_timestamp": {
+          "format": "uuuu-MM-dd HH:mm:ss.SSS",
+          "type": "date"
+        },
+        "full_message": {
+          "fielddata": false,
+          "analyzer": "standard",
+          "type": "text"
+        },
+        "streams": {
+          "type": "keyword"
+        },
+        "dest_ip_geolocation": {
+          "copy_to": "dst_location",
+          "type": "text"
+        },
+        "src_ip_geolocation": {
+          "copy_to": "src_location",
+          "type": "text"
+        },
+        "source": {
+          "fielddata": true,
+          "analyzer": "analyzer_keyword",
+          "type": "text"
+        },
+        "message": {
+          "fielddata": false,
+          "analyzer": "standard",
+          "type": "text"
+        },
+        "timestamp": {
+          "format": "uuuu-MM-dd HH:mm:ss.SSS",
+          "type": "date"
+        }
+      }
+    },
+    "aliases": {}
+  }

README.md

@@ -1,5 +1,19 @@
 This Project aims to give you better insight of what's going on your pfSense Firewall. It's based on some heavylifting alrerady done by devopstales and opc40772. Since it still was a bit clumsy and outdated I wrapped some docker-compose glue around it, to make it a little bit easier to get up and running. It should work hasslefree with a current Linux that has docker and docker-compose. Thanks as well to MatthewJSalerno for some Streamlining of the Graylog provisioning Process.
 
+I have recently updated the whole stack to utilize Graylog 4 and Elasticsearch 7 and Grafana 7. I don't include any directions for Upgrading GL3/ES6 to GL4/ES7. 
+
+This doc has been tested with the following Versions:
+
+| Component           | Version                | 
+| -------------         |:---------------------: 
+| Elasticsearch     | 7.11.1 |
+| Grafana | 7.4.2 |
+| Graylog | 4.0.3 |
+| Cerebro | 0.9.3 | 
+
+
+If it's easier for you, you can find a video guide here:  https://youtu.be/uOfPzueH6MA (Still the Guide for GL3/ES6, will make a new one some day.)
+
 The whole metric approach is split into several subtopics.
 
 | Metric type           | Stored via                | stored in       | Visualisation  |
@@ -27,7 +41,7 @@ This walkthrough has been made with a fresh install of Ubuntu 18.04 Bionic but s
 
 # 0. System requirements
 
-Since this involves Elasticsearch a few GB of RAM will be required. I'm not sure if an old Raspi will do. Give me feedback :)
+Since this involves Elasticsearch 7 a few GB of RAM will be required. Don't bother with less than 8GB. It just won't run.
 
 Please install docker, docker-compose and git as basic prerequisite.
 
@@ -68,6 +82,8 @@ A salt for encrypting your graylog passwords
 - GRAYLOG_PASSWORD_SECRET (Change that _now_)
 
 
+Edit Docker/graylog/getGeo.sh and insert _your_ tokenized Downloadlink of the Maxmind GeoIP Database. Create an account on https://www.maxmind.com/en/account/login and go to "My Account -> Download Files -> GeoLite2 City" and copy the Link "Download GZIP" to your getGeo.sh File. If you don't do that the geolookup feature for IP Addresses won't work.
+
 Finally, spin up the stack with:
 
 ```
@@ -75,20 +91,20 @@ cd ./Docker
 sudo docker-compose up -d
 ```
 
-Note: graylog will be built the first time you run docker-compose.  The below step is only for updating the GeiLite DB.
+Note: graylog will be built the first time you run docker-compose.  The below step is only for updating the GeoLite DB.
 To update the geolite.maxmind.com GeoLite2-City database, simply run:
 ```
 cd ./Docker
 sudo docker-compose up -d --no-deps --build graylog
 ```
 
-
 This should expose you the following services externally:
 
 | Service       | URL                   | Default Login  | Purpose |
 | ------------- |:---------------------:| --------------:| --------------:|
 | Graylog       | http://localhost:9000 | admin/admin |  Configure Data Ingestions and Extractors for Log Inforation |
 | Grafana       | http://localhost:3000 | admin/admin | Draw nice Graphs |
+| Kibana        | http://localhost:5601/ | none | Default Elastic Data exploratiopn tool. Not required.|
 | Cerebro       | http://localhost:9001 |    none - provide with ES API: http://elasticsearch:9200 | ES Admin tool. Only required for setting up the Index.|
 
 Depending on your hardware a few minutes later you should be able to connect to

misc/jvm.options

@@ -1,71 +0,0 @@
-## You should always set the min and max JVM heap
-## size to the same value. For example, to set
-## the heap to 4 GB, set:
-##
-## -Xms4g
-## -Xmx4g
-##
-## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html
-## for more information
-##
-################################################################
-
-# Xms represents the initial size of total heap space
-# Xmx represents the maximum size of total heap space
-
--Xms2g
--Xmx2g
-
-################################################################
-## Expert settings
-################################################################
-##
-## All settings below this section are considered
-## expert settings. Don't tamper with them unless
-## you understand what you are doing
-##
-################################################################
-
-## GC configuration
--XX:+UseConcMarkSweepGC
--XX:CMSInitiatingOccupancyFraction=75
--XX:+UseCMSInitiatingOccupancyOnly
-
-## optimizations
-
-# pre-touch memory pages used by the JVM during initialization
--XX:+AlwaysPreTouch
-
-## basic
-
-# force the server VM (remove on 32-bit client JVMs)
--server
-# explicitly set the stack size (reduce to 320k on 32-bit client JVMs)
--Xss1m
-
-# set to headless, just in case
--Djava.awt.headless=true
-
-# ensure UTF-8 encoding by default (e.g. filenames)
--Dfile.encoding=UTF-8
-
-# use our provided JNA always versus the system one
--Djna.nosys=true
-
-# use old-style file permissions on JDK9
--Djdk.io.permissionsUseCanonicalPath=true
-
-# flags to configure Netty
--Dio.netty.noUnsafe=true
--Dio.netty.noKeySetOptimization=true
--Dio.netty.recycler.maxCapacityPerThread=0
-
-# log4j 2
--Dlog4j.shutdownHookEnabled=false
--Dlog4j2.disable.jmx=true
--Dlog4j.skipJansi=true
-## heap dumps
-# generate a heap dump when an allocation from the Java heap fails
-# heap dumps are created in the working directory of the JVM
-
--XX:+HeapDumpOnOutOfMemoryError