Restructured entire repo

Moved volume mounted files to their own directory
Split docker envs into their own files
This commit is contained in:
MatthewJSalerno
2019-11-22 22:04:41 -05:00
parent 9c5b76f97e
commit c3d4f73505
14 changed files with 124 additions and 117 deletions

110
Docker/docker-compose.yml Normal file
View File

@@ -0,0 +1,110 @@
version: '2'
services:
# MongoDB: https://hub.docker.com/_/mongo/
mongodb:
image: 'mongo:3'
volumes:
- 'mongo_data:/data/db'
# Elasticsearch: https://www.elastic.co/guide/en/elasticsearch/reference/6.x/docker.html
elasticsearch:
image: 'docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.4'
mem_limit: 4g
restart: always
volumes:
- 'es_data:/usr/share/elasticsearch/data'
env_file:
- ./elasticsearch.env
ulimits:
memlock:
soft: -1
hard: -1
ports:
- '9200:9200'
# Graylog: https://hub.docker.com/r/graylog/graylog/
graylog:
image: 'graylog/graylog:3.1'
volumes:
- 'graylog_journal:/usr/share/graylog/data/journal'
- './service-names-port-numbers.csv:/etc/graylog/server/service-names-port-numbers.csv'
- './GeoLite2-City.mmdb:/etc/graylog/server/GeoLite2-City.mmdb'
env_file:
- ./graylog.env
links:
- 'mongodb:mongo'
- elasticsearch
depends_on:
- mongodb
- elasticsearch
ports:
# Netflow
- '2055:2055/udp'
# Syslog Feed
- '5442:5442/udp'
# Graylog web interface and REST API
- '9000:9000'
# Syslog TCP
- '1514:1514'
# Syslog UDP
- '1514:1514/udp'
# GELF TCP
- '12201:12201'
# GELF UDP
- '12201:12201/udp'
# Kibana : https://www.elastic.co/guide/en/kibana/6.8/index.html
kibana:
image: 'docker.elastic.co/kibana/kibana-oss:6.8.4'
env_file:
- kibana.env
depends_on:
- elasticsearch
ports:
- '5601:5601'
cerebro:
image: lmenezes/cerebro
ports:
- '9001:9000'
links:
- elasticsearch
depends_on:
- elasticsearch
influxdb:
image: 'influxdb:latest'
env_file:
- ./influxdb.env
ports:
- '8086:8086'
volumes:
- 'influxdb:/var/lib/influxdb'
grafana:
image: 'grafana/grafana:latest'
env_file:
- ./grafana.env
ports:
- '3000:3000'
volumes:
- 'grafana:/var/lib/grafana'
- './provisioning/:/etc/grafana/provisioning'
links:
- elasticsearch
- influxdb
depends_on:
- elasticsearch
- influxdb
# Volumes for persisting data, see https://docs.docker.com/engine/admin/volumes/volumes/
volumes:
mongo_data:
driver: local
es_data:
driver: local
graylog_journal:
driver: local
grafana:
driver: local
influxdb:
driver: local

5
Docker/elasticsearch.env Normal file
View File

@@ -0,0 +1,5 @@
http.host=0.0.0.0
transport.host=0.0.0.0
network.host=0.0.0.0
ES_JAVA_OPTS="-Xms1g -Xmx1g"
ES_HEAP_SIZE=2g

1
Docker/grafana.env Normal file
View File

@@ -0,0 +1 @@
GF_INSTALL_PLUGINS=grafana-piechart-panel,grafana-worldmap-panel,savantly-heatmap-panel

View File

@@ -0,0 +1,11 @@
apiVersion: 1
providers:
- name: 'pfSense Analytics'
orgId: 1
folder: ''
type: file
disableDeletion: false
editable: true
options:
path: /etc/grafana/provisioning/dashboards

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,863 @@
{
"__inputs": [
{
"name": "DS_NTOP-CLUSTER",
"label": "ntop-cluster",
"description": "",
"type": "datasource",
"pluginId": "influxdb",
"pluginName": "InfluxDB"
}
],
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "6.4.3"
},
{
"type": "panel",
"id": "grafana-piechart-panel",
"name": "Pie Chart",
"version": "1.3.9"
},
{
"type": "panel",
"id": "grafana-worldmap-panel",
"name": "Worldmap Panel",
"version": "0.2.1"
},
{
"type": "panel",
"id": "graph",
"name": "Graph",
"version": ""
},
{
"type": "datasource",
"id": "influxdb",
"name": "InfluxDB",
"version": "1.0.0"
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 1,
"id": null,
"iteration": 1573140126928,
"links": [],
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "ntop",
"fill": 1,
"fillGradient": 2,
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 0
},
"id": 16,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"alias": "IPV4 $tag_host",
"groupBy": [
{
"params": [
"$__interval"
],
"type": "time"
},
{
"params": [
"host"
],
"type": "tag"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"measurement": "host:traffic",
"orderByTime": "ASC",
"policy": "default",
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"bytes_rcvd"
],
"type": "field"
},
{
"params": [],
"type": "mean"
},
{
"params": [
"1s"
],
"type": "non_negative_derivative"
},
{
"params": [
"*8"
],
"type": "math"
}
]
],
"tags": []
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "IPv4 Traffic by Host",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "bps",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"collapsed": false,
"datasource": "ntop",
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 7
},
"id": 12,
"panels": [],
"title": "Interface igb1",
"type": "row"
},
{
"aliasColors": {},
"breakPoint": "50%",
"cacheTimeout": null,
"combine": {
"label": "Others",
"threshold": 0
},
"datasource": "ntop",
"fontSize": "110%",
"format": "decbytes",
"gridPos": {
"h": 8,
"w": 6,
"x": 0,
"y": 8
},
"id": 6,
"interval": null,
"legend": {
"header": "",
"percentage": true,
"show": true,
"sort": "total",
"sortDesc": true,
"values": false
},
"legendType": "On graph",
"links": [],
"maxDataPoints": 3,
"nullPointMode": "connected",
"options": {},
"pieType": "donut",
"strokeWidth": "3",
"targets": [
{
"alias": "IPv4",
"groupBy": [
{
"params": [
"host"
],
"type": "tag"
}
],
"measurement": "host:traffic",
"orderByTime": "ASC",
"policy": "default",
"query": "select sum(sumtx_sumrx) from (select sumtx+sumrx from (select sum(tx) as sumtx, sum(rx) as sumrx from (SELECT non_negative_difference(\"bytes_rcvd\") as rx,non_negative_difference(\"bytes_sent\") as tx FROM \"host:traffic\" WHERE (\"host\" =~ /^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}$/) AND $timeFilter GROUP BY \"host\") group by *))",
"rawQuery": true,
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"bytes_rcvd"
],
"type": "field"
},
{
"params": [],
"type": "non_negative_difference"
}
]
],
"tags": [
{
"key": "host",
"operator": "=~",
"value": "/^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}$/"
}
]
},
{
"alias": "IPv6",
"groupBy": [
{
"params": [
"host"
],
"type": "tag"
}
],
"measurement": "host:traffic",
"orderByTime": "ASC",
"policy": "default",
"query": "select sum(sumtx_sumrx) from (select sumtx+sumrx from (select sum(tx) as sumtx, sum(rx) as sumrx from (SELECT non_negative_difference(\"bytes_rcvd\") as rx,non_negative_difference(\"bytes_sent\") as tx FROM \"host:traffic\" WHERE (\"host\" !~ /^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}/) AND $timeFilter GROUP BY \"host\") group by *))",
"rawQuery": true,
"refId": "B",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"bytes_rcvd"
],
"type": "field"
},
{
"params": [],
"type": "non_negative_difference"
}
]
],
"tags": [
{
"key": "host",
"operator": "=~",
"value": "/^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}$/"
}
]
}
],
"timeFrom": null,
"timeShift": null,
"title": "Traffix Distribution V4/V6",
"type": "grafana-piechart-panel",
"valueName": "total"
},
{
"aliasColors": {},
"breakPoint": "50%",
"cacheTimeout": null,
"combine": {
"label": "Others",
"threshold": 0
},
"datasource": "ntop",
"fontSize": "80%",
"format": "decbytes",
"gridPos": {
"h": 8,
"w": 8,
"x": 6,
"y": 8
},
"id": 2,
"interval": null,
"legend": {
"percentage": true,
"percentageDecimals": 2,
"show": true,
"sort": "total",
"sortDesc": true,
"values": true
},
"legendType": "Right side",
"links": [],
"maxDataPoints": 3,
"nullPointMode": "connected",
"options": {},
"pieType": "donut",
"pluginVersion": "6.4.3",
"strokeWidth": "1",
"targets": [
{
"alias": "$tag_protocol",
"groupBy": [
{
"params": [
"protocol"
],
"type": "tag"
}
],
"limit": "",
"measurement": "iface:ndpi",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT \"bytes\" as bytes FROM \"iface:ndpi\" WHERE $timeFilter GROUP BY \"protocol\"",
"rawQuery": false,
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"bytes"
],
"type": "field"
},
{
"params": [],
"type": "non_negative_difference"
}
]
],
"slimit": "",
"tags": [
{
"key": "protocol",
"operator": "=~",
"value": "/^$ndpicat$/"
}
]
}
],
"timeFrom": null,
"timeShift": null,
"title": "NDPI Interface",
"type": "grafana-piechart-panel",
"valueName": "total"
},
{
"circleMaxSize": "15",
"circleMinSize": "3",
"colors": [
"#73BF69",
"rgba(237, 129, 40, 0.89)",
"#8F3BB8"
],
"datasource": "ntop",
"decimals": 0,
"esMetric": "Count",
"gridPos": {
"h": 15,
"w": 10,
"x": 14,
"y": 8
},
"hideEmpty": false,
"hideZero": false,
"id": 8,
"initialZoom": "2",
"locationData": "countries",
"mapCenter": "Europe",
"mapCenterLatitude": 46,
"mapCenterLongitude": 14,
"maxDataPoints": 1,
"mouseWheelZoom": true,
"options": {},
"showLegend": true,
"stickyLabels": false,
"tableQueryOptions": {
"geohashField": "geohash",
"latitudeField": "latitude",
"longitudeField": "longitude",
"metricField": "metric",
"queryType": "geohash"
},
"targets": [
{
"alias": "$tag_country",
"groupBy": [
{
"params": [
"country"
],
"type": "tag"
}
],
"measurement": "country:traffic",
"orderByTime": "ASC",
"policy": "default",
"query": "select sum(inout)/1024/1024/1024 from (SELECT \"bytes_ingress\"+\"bytes_egress\" as inout FROM \"country:traffic\" WHERE time >= now() - 3h GROUP BY \"country\") group by *",
"rawQuery": true,
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"bytes_egress"
],
"type": "field"
},
{
"params": [],
"type": "sum"
}
]
],
"tags": []
}
],
"thresholds": "4,10",
"timeFrom": null,
"timeShift": null,
"title": "Traffic egress+ingress per Country",
"type": "grafana-worldmap-panel",
"unitPlural": "GBytes",
"unitSingle": "",
"unitSingular": "GByte",
"valueName": "current"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "ntop",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 14,
"x": 0,
"y": 16
},
"id": 4,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"alias": "local2remote",
"groupBy": [
{
"params": [
"$interval"
],
"type": "time"
}
],
"measurement": "iface:local2remote",
"orderByTime": "ASC",
"policy": "default",
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"bytes"
],
"type": "field"
},
{
"params": [],
"type": "mean"
},
{
"params": [
"1s"
],
"type": "non_negative_derivative"
},
{
"params": [
"*8"
],
"type": "math"
}
]
],
"tags": []
},
{
"alias": "remote2local",
"groupBy": [
{
"params": [
"$interval"
],
"type": "time"
}
],
"measurement": "iface:remote2local",
"orderByTime": "ASC",
"policy": "default",
"refId": "B",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"bytes"
],
"type": "field"
},
{
"params": [],
"type": "mean"
},
{
"params": [
"1s"
],
"type": "non_negative_derivative"
},
{
"params": [
"*8"
],
"type": "math"
}
]
],
"tags": []
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Flowthroughput",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "bps",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "ntop",
"fill": 1,
"fillGradient": 1,
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 23
},
"id": 10,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"max": true,
"min": false,
"rightSide": true,
"show": true,
"sort": "current",
"sortDesc": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"alias": "$tag_protocol",
"groupBy": [
{
"params": [
"$interval"
],
"type": "time"
},
{
"params": [
"protocol"
],
"type": "tag"
}
],
"measurement": "iface:ndpi",
"orderByTime": "ASC",
"policy": "default",
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"bytes"
],
"type": "field"
},
{
"params": [],
"type": "mean"
},
{
"params": [
"1s"
],
"type": "non_negative_derivative"
},
{
"params": [
"*8"
],
"type": "math"
}
]
],
"tags": [
{
"key": "protocol",
"operator": "=~",
"value": "/^$ndpicat$/"
}
]
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Traffic Distribution",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "bps",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"refresh": "30s",
"schemaVersion": 20,
"style": "dark",
"tags": [
"pfSense",
"NTOP"
],
"templating": {
"list": [
{
"allValue": null,
"current": {},
"datasource": "ntop",
"definition": "SHOW TAG VALUES FROM \"iface:ndpi\" WITH KEY=protocol",
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "ndpicat",
"options": [],
"query": "SHOW TAG VALUES FROM \"iface:ndpi\" WITH KEY=protocol",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "apps.$tag.*",
"tags": [],
"tagsQuery": "SHOW TAG VALUES FROM \"iface:ndpi\" WITH KEY=protocol WHERE protocol =~ /^RTP$/",
"type": "query",
"useTags": false
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
]
},
"timezone": "",
"title": "DPI",
"uid": "_LW0mbAZk",
"version": 52
}

View File

@@ -0,0 +1,39 @@
apiVersion: 1
deleteDatasources:
- name: ntop
orgId: 1
- name: pfsensefw
orgId: 1
datasources:
- name: ntop
type: influxdb
access: proxy
url: http://influxdb:8086
password: ntop
user: ntop
database: ndpi
basicAuth: false
basicAuthUser:
basicAuthPassword:
withCredentials:
isDefault: true
jsonData:
tlsAuth: false
tlsAuthWithCACert: false
secureJsonData:
tlsCACert: ""
tlsClientCert: ""
tlsClientKey: ""
version: 1
editable: true
- name: pfsensefw
type: elasticsearch
access: proxy
database: "pfsense_*"
url: http://elasticsearch:9200
jsonData:
esVersion: 60
timeField: "utc_timestamp"

6
Docker/graylog.env Normal file
View File

@@ -0,0 +1,6 @@
# CHANGE ME (must be at least 16 characters)!
GRAYLOG_PASSWORD_SECRET=somepasswordpepperzzz
# Password: admin
GRAYLOG_ROOT_PASSWORD_SHA2=8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918
GRAYLOG_HTTP_EXTERNAL_URI=http://localhost:9000/
GRAYLOG_TIMEZONE=Europe/Berlin

File diff suppressed because it is too large Load Diff

1
Docker/influxdb.env Normal file
View File

@@ -0,0 +1 @@
INFLUXDB_DB="ndpi"

1
Docker/kibana.env Normal file
View File

@@ -0,0 +1 @@
ELASTICSEARCH_URL=http://elasticsearch:9200