Skip to content

Commit a992809

Browse files
AHarmlessPyroAHarmlessPyro
authored
Add better data capture from suricata unix streams (#45)
1 parent 31a8e05 commit a992809

File tree

6 files changed

+159
-54
lines changed

6 files changed

+159
-54
lines changed

‎backend/src/suricata_setup/generics/scripts/suricata.yaml

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ default-log-dir: /etc/suricata-logs
6262

6363
# Global stats configuration
6464
stats:
65-
enabled: yes
65+
enabled: no
6666
# The interval field (in seconds) controls the interval at
6767
# which stats are updated in the log.
6868
interval: 8
@@ -153,12 +153,12 @@ outputs:
153153

154154
types:
155155
- alert:
156-
payload: yes # enable dumping payload in Base64
157-
payload-buffer-size: 4kb # max size of payload buffer to output in eve-log
158-
payload-printable: yes # enable dumping payload in printable (lossy) format
159-
packet: yes # enable dumping of packet (without stream segments)
156+
payload: no # enable dumping payload in Base64
157+
payload-buffer-size: 1000kb # max size of payload buffer to output in eve-log
158+
payload-printable: no # enable dumping payload in printable (lossy) format
159+
packet: no # enable dumping of packet (without stream segments)
160160
# metadata: yes # enable inclusion of app layer metadata with alert. Default yes
161-
http-body: yes # Requires metadata; enable dumping of HTTP body in Base64
161+
http-body: no # Requires metadata; enable dumping of HTTP body in Base64
162162
http-body-printable: yes # Requires metadata; enable dumping of HTTP body in printable format
163163

164164
# Enable the logging of tagged packets for rules using the
@@ -221,7 +221,7 @@ outputs:
221221
# to dump all HTTP headers for every HTTP request and/or response
222222
dump-all-headers: both
223223
payload: yes # enable dumping payload in Base64
224-
payload-buffer-size: 4kb # max size of payload buffer to output in eve-log
224+
payload-buffer-size: 1000kb # max size of payload buffer to output in eve-log
225225
payload-printable: yes # enable dumping payload in printable (lossy) format
226226
packet: yes # enable dumping of packet (without stream segments)
227227
# metadata: yes # enable inclusion of app layer metadata with alert. Default yes

‎ingestors/suricata/index.ts

Lines changed: 64 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,19 @@
11
import net from "net"
22
import fs from "fs"
33
import process from "process"
4-
import { conns, EVENTS, ALERT } from "./interface"
4+
import { conns, EVENTS, ALERT, RecordHolderWithTimestamp } from "./interface"
55
import { program } from "commander"
66
import { prepareResponse, compileHost, pushAlert } from "./utils"
7+
import ndjson from "ndjson"
78

89
var server: net.Server
910
var connections: Record<number, net.Socket> = {}
10-
var http_meta: Record<string, conns> = {}
11+
var http_meta: Record<string, RecordHolderWithTimestamp<conns>> = {}
12+
var alerts: Record<string, RecordHolderWithTimestamp<ALERT>> = {}
13+
const msSendTimeout = 10
14+
// Just offset by enough to not conflict with timeout
15+
const msCleanupTimeout = 10011
16+
1117

1218
var url = ""
1319
var api_key = ""
@@ -29,40 +35,33 @@ function createServer(socket: string) {
2935
delete connections[self]
3036
})
3137

32-
// Messages are buffers. use toString
33-
stream.on("data", function (msg) {
34-
msg
35-
.toString()
36-
.split("\n")
37-
.filter(_msg => !!_msg)
38-
.forEach((_msg, i, a) => {
39-
try {
40-
const jsonmsg = JSON.parse(_msg)
41-
if (EVENTS.HTTP === (jsonmsg["event_type"] as string)) {
42-
compileHost(jsonmsg, http_meta)
43-
}
44-
if (EVENTS.ALERT === (jsonmsg["event_type"] as string)) {
45-
// compileAlert(jsonmsg);
46-
const alert: ALERT = jsonmsg
47-
// Get first metadata for the given connection.
48-
let meta = http_meta[alert.flow_id].metas.shift()
49-
if (meta) {
50-
let resp = prepareResponse(alert, meta)
51-
pushAlert(resp, url, api_key)
52-
}
53-
}
54-
} catch (err) {
55-
console.log(
56-
`/////////////////// ERROR ///////////////////`,
57-
)
58-
console.log(err)
59-
console.log(
60-
`/////////////////// MESSAGE ///////////////////`,
61-
)
62-
console.log(_msg)
38+
stream.pipe(ndjson.parse())
39+
.on('data', function (obj) {
40+
// obj is a javascript object
41+
try {
42+
43+
const jsonmsg = obj
44+
const flow_id = jsonmsg.flow_id
45+
46+
if (EVENTS.HTTP === (jsonmsg["event_type"] as string)) {
47+
compileHost(jsonmsg, http_meta)
6348
}
64-
})
65-
})
49+
if (EVENTS.ALERT === (jsonmsg["event_type"] as string)) {
50+
alerts[flow_id] = { value: jsonmsg, timestamp: Date.now() }
51+
}
52+
} catch (err) {
53+
console.log(
54+
`/////////////////// ERROR ///////////////////`,
55+
)
56+
console.log(err)
57+
console.log(
58+
`/////////////////// MESSAGE ///////////////////`,
59+
)
60+
console.log(JSON.stringify(obj))
61+
}
62+
})
63+
64+
6665
})
6766
.listen(socket)
6867
.on("connection", function (socket) {
@@ -128,6 +127,36 @@ function main() {
128127
process.on("SIGINT", cleanup)
129128
}
130129

130+
function processAlerts() {
131+
Object.entries(http_meta).forEach(([k, v]) => {
132+
const flow_id = k
133+
// Find if both events are present in their respective things
134+
if (flow_id in alerts) {
135+
136+
let curr_alert = alerts[flow_id].value
137+
delete alerts[flow_id]
138+
let curr_http = v.value.metas.shift()
139+
140+
// Get first metadata for the given connection.
141+
let resp = prepareResponse(curr_alert, curr_http)
142+
pushAlert(resp, url, api_key)
143+
}
144+
})
145+
}
146+
147+
function cleanup() {
148+
let new_meta = {}
149+
Object.entries(http_meta).filter((([k, v]) => ((Date.now() - v.timestamp) > msCleanupTimeout))).forEach(([k, v]) => { new_meta[k] = v })
150+
http_meta = new_meta
151+
152+
let new_alerts = {}
153+
Object.entries(alerts).filter((([k, v]) => ((Date.now() - v.timestamp) > msCleanupTimeout))).forEach(([k, v]) => { new_meta[k] = v })
154+
alerts = new_alerts
155+
}
156+
157+
setInterval(processAlerts, msSendTimeout)
158+
setInterval(cleanup, msCleanupTimeout)
159+
131160
process.title = "METLO"
132161

133162
main()

‎ingestors/suricata/interface.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,3 +121,8 @@ export interface RESPONSE {
121121
destinationPort: number
122122
}
123123
}
124+
125+
export interface RecordHolderWithTimestamp<T> {
126+
value: T
127+
timestamp: number
128+
}

‎ingestors/suricata/package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
"@types/node": "^18.6.5",
99
"axios": "^0.27.2",
1010
"commander": "^9.4.0",
11+
"ndjson": "^2.0.0",
1112
"ts-node": "^10.9.1",
1213
"tsc": "^2.0.4",
1314
"typescript": "^4.7.4"
@@ -20,4 +21,4 @@
2021
"build": "tsc",
2122
"format": "prettier --write '**/*.{ts,tsx}'"
2223
}
23-
}
24+
}

‎ingestors/suricata/utils.ts

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
import { ALERT, RESPONSE, HOST, conns } from "./interface"
1+
import { ALERT, RESPONSE, HOST, conns, RecordHolderWithTimestamp } from "./interface"
22
import axios from "axios"
33

4-
export function compileHost(jsonmsg: any, http_meta: Record<string, conns>) {
4+
export function compileHost(jsonmsg: any, http_meta: Record<string, RecordHolderWithTimestamp<conns>>) {
55
// Get Response and Request headers from json blob
66
const resp_headers = jsonmsg.http.response_headers
77
const req_headers = jsonmsg.http.request_headers
@@ -13,24 +13,28 @@ export function compileHost(jsonmsg: any, http_meta: Record<string, conns>) {
1313
// store flow metadata. Each connection in a pipeline comes sequentially
1414
// Store them sequentially
1515
if (host.flow_id in http_meta) {
16-
http_meta[host.flow_id].metas.push({
16+
http_meta[host.flow_id].value.metas.push({
1717
timestamp: host.timestamp,
1818
metadata: host,
1919
})
20+
http_meta[host.flow_id].timestamp = Date.now()
2021
} else {
2122
http_meta[host.flow_id] = {
22-
flowId: host.flow_id,
23-
metas: [
24-
{
25-
timestamp: host.timestamp,
26-
metadata: host,
27-
},
28-
],
23+
timestamp: Date.now(),
24+
value: {
25+
flowId: host.flow_id,
26+
metas: [
27+
{
28+
timestamp: host.timestamp,
29+
metadata: host,
30+
},
31+
]
32+
},
2933
}
3034
}
3135
}
3236

33-
export function pushAlert(resp: RESPONSE, url: string, api_key: string) {
37+
export async function pushAlert(resp: RESPONSE, url: string, api_key: string) {
3438
axios
3539
.post(url, {
3640
...resp,

‎ingestors/suricata/yarn.lock

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,16 @@ form-data@^4.0.0:
121121
combined-stream "^1.0.8"
122122
mime-types "^2.1.12"
123123

124+
inherits@^2.0.3:
125+
version "2.0.4"
126+
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
127+
integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
128+
129+
json-stringify-safe@^5.0.1:
130+
version "5.0.1"
131+
resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"
132+
integrity sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==
133+
124134
make-error@^1.1.1:
125135
version "1.3.6"
126136
resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2"
@@ -138,11 +148,62 @@ mime-types@^2.1.12:
138148
dependencies:
139149
mime-db "1.52.0"
140150

151+
minimist@^1.2.5:
152+
version "1.2.7"
153+
resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.7.tgz#daa1c4d91f507390437c6a8bc01078e7000c4d18"
154+
integrity sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==
155+
156+
ndjson@^2.0.0:
157+
version "2.0.0"
158+
resolved "https://registry.yarnpkg.com/ndjson/-/ndjson-2.0.0.tgz#320ac86f6fe53f5681897349b86ac6f43bfa3a19"
159+
integrity sha512-nGl7LRGrzugTtaFcJMhLbpzJM6XdivmbkdlaGcrk/LXg2KL/YBC6z1g70xh0/al+oFuVFP8N8kiWRucmeEH/qQ==
160+
dependencies:
161+
json-stringify-safe "^5.0.1"
162+
minimist "^1.2.5"
163+
readable-stream "^3.6.0"
164+
split2 "^3.0.0"
165+
through2 "^4.0.0"
166+
141167
prettier@^2.7.1:
142168
version "2.7.1"
143169
resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.7.1.tgz#e235806850d057f97bb08368a4f7d899f7760c64"
144170
integrity sha512-ujppO+MkdPqoVINuDFDRLClm7D78qbDt0/NR+wp5FqEZOoTNAjPHWj17QRhu7geIHJfcNhRk1XVQmF8Bp3ye+g==
145171

172+
readable-stream@3, readable-stream@^3.0.0, readable-stream@^3.6.0:
173+
version "3.6.0"
174+
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
175+
integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
176+
dependencies:
177+
inherits "^2.0.3"
178+
string_decoder "^1.1.1"
179+
util-deprecate "^1.0.1"
180+
181+
safe-buffer@~5.2.0:
182+
version "5.2.1"
183+
resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
184+
integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
185+
186+
split2@^3.0.0:
187+
version "3.2.2"
188+
resolved "https://registry.yarnpkg.com/split2/-/split2-3.2.2.tgz#bf2cf2a37d838312c249c89206fd7a17dd12365f"
189+
integrity sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==
190+
dependencies:
191+
readable-stream "^3.0.0"
192+
193+
string_decoder@^1.1.1:
194+
version "1.3.0"
195+
resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e"
196+
integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==
197+
dependencies:
198+
safe-buffer "~5.2.0"
199+
200+
through2@^4.0.0:
201+
version "4.0.2"
202+
resolved "https://registry.yarnpkg.com/through2/-/through2-4.0.2.tgz#a7ce3ac2a7a8b0b966c80e7c49f0484c3b239764"
203+
integrity sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==
204+
dependencies:
205+
readable-stream "3"
206+
146207
ts-node@^10.9.1:
147208
version "10.9.1"
148209
resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.9.1.tgz#e73de9102958af9e1f0b168a6ff320e25adcff4b"
@@ -172,6 +233,11 @@ typescript@^4.7.4:
172233
resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.7.4.tgz#1a88596d1cf47d59507a1bcdfb5b9dfe4d488235"
173234
integrity sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==
174235

236+
util-deprecate@^1.0.1:
237+
version "1.0.2"
238+
resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
239+
integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==
240+
175241
v8-compile-cache-lib@^3.0.1:
176242
version "3.0.1"
177243
resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf"

0 commit comments

Comments
 (0)