Skip to content

Commit 786710f

Browse files
AHarmlessPyroAHarmlessPyro
authored
Improvements to node/python ingestor(s) (#51)
1 parent f6c3ce4 commit 786710f

File tree

10 files changed

+191
-68
lines changed

10 files changed

+191
-68
lines changed

‎ingestors/node/package.json

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
"format": "prettier --write './src/**/*.{ts,tsx}'"
1313
},
1414
"dependencies": {
15+
"axios": "^1.1.3",
1516
"require-in-the-middle": "^5.2.0"
1617
},
1718
"repository": {
@@ -24,9 +25,9 @@
2425
"homepage": "www.metlo.com",
2526
"devDependencies": {
2627
"@types/node": "^18.8.4",
27-
"typescript": "^4.8.4",
2828
"nodemon": "^2.0.20",
2929
"ts-node": "^10.9.1",
30-
"tslib": "^2.4.0"
30+
"tslib": "^2.4.0",
31+
"typescript": "^4.8.4"
3132
}
3233
}

‎ingestors/node/src/index.js

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ const WorkerPool = require("./pool");
44
const path = require("path")
55

66
const pool = new WorkerPool(os.cpus().length, './workerTarget.js');
7+
const endpoint = "api/v1/log-request/single"
78

89
function exit() {
910
pool.close()
@@ -13,5 +14,16 @@ process.on('exit', exit);
1314
process.on('SIGTERM', exit);
1415

1516
module.exports = function (key, host) {
16-
Modules({ host, key, pool })
17+
try {
18+
new URL(host)
19+
} catch (err) {
20+
console.error(err)
21+
throw new Error(`Couldn't load metlo. Host is not a proper url : ${host}`)
22+
}
23+
let metlo_host = host
24+
if (metlo_host[metlo_host.length - 1] != "/") {
25+
metlo_host += "/"
26+
}
27+
metlo_host += endpoint
28+
Modules({ host: metlo_host, key, pool })
1729
}

‎ingestors/node/src/modules/express.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ function initialize({ key, host, pool }) {
1616
METLO_DETAILS.host = host
1717
METLO_DETAILS.pool = pool
1818

19-
2019
function compileInformation(_req, _res, responseBody) {
2120
const data = JSON.stringify({
2221
request: {
@@ -42,6 +41,7 @@ function initialize({ key, host, pool }) {
4241
sourcePort: _req.socket.remotePort,
4342
destination: _res.socket.localAddress,
4443
destinationPort: _res.socket.localPort,
44+
metloSource: "node/express"
4545
}
4646
})
4747

@@ -77,7 +77,7 @@ function initialize({ key, host, pool }) {
7777

7878
function modifiedSendFile() {
7979
const resp = original_sendFile.apply(this, arguments)
80-
compileInformation(this.req, resp, arguments[arguments.length - 1])
80+
compileInformation(this.req, this, arguments[0])
8181
return resp
8282
};
8383

‎ingestors/node/src/modules/fastify.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ function initialize({ key, host, pool }) {
4343
sourcePort: request.raw.socket.remotePort,
4444
destination: response.raw.socket.localAddress,
4545
destinationPort: response.raw.socket.localPort,
46+
metloSource: "node/fastify"
4647
}
4748
}
4849
)

‎ingestors/node/src/modules/koa.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ function initialize({ key, host, pool }) {
4444
sourcePort: ctx.request.socket.remotePort,
4545
destination: ctx.request.socket.localAddress,
4646
destinationPort: ctx.request.socket.localPort,
47+
metloSource: "node/koa"
4748
}
4849
}
4950
)

‎ingestors/node/src/pool/index.js

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,10 +55,16 @@ class WorkerPool extends EventEmitter {
5555
worker.on('error', (err) => {
5656
// In case of an uncaught exception: Call the callback that was passed to
5757
// `runTask` with the error.
58-
if (worker[kTaskInfo])
59-
worker[kTaskInfo].done(err, null);
60-
else
58+
if (worker[kTaskInfo]) {
59+
console.warn(err)
60+
try {
61+
worker[kTaskInfo].done(err, null);
62+
} catch (err) {
63+
//pass
64+
}
65+
} else {
6166
this.emit('error', err);
67+
}
6268
// Remove the worker from the list and start a new Worker to replace the
6369
// current one.
6470
this.workers.splice(this.workers.indexOf(worker), 1);

‎ingestors/node/src/pool/workerTarget.js

Lines changed: 12 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,18 @@
11
const { parentPort } = require("node:worker_threads")
2-
const https = require("https");
2+
// const { http } = require('follow-redirects');
3+
const axios = require("axios")
34

4-
parentPort.on('message', ({ data: postData, host, key }) => {
5-
var _resp = https.request(host, {
6-
method: "POST", headers: {
5+
parentPort.on('message', ({ data: postData, host, key }) => {
6+
axios({
7+
method: 'post',
8+
url: host,
9+
headers: {
710
'Content-Type': 'application/json',
811
'Content-Length': Buffer.byteLength(postData),
912
'Authorization': key
10-
}
11-
},
12-
// (res) => {
13-
// let data = '';
14-
15-
// console.log('Status Code:', res.statusCode);
16-
17-
// res.on('data', (chunk) => {
18-
// data += chunk;
19-
// });
20-
21-
// res.on('end', () => {
22-
// console.log('Body sent');
23-
// });
24-
25-
// }
26-
).on("error", (err) => {
27-
console.log("Error: ", err.message);
28-
})
29-
30-
_resp.write(postData)
31-
_resp.end();
13+
},
14+
data: postData
15+
}).catch((error) => {
16+
console.warn("Encountered an error with metlo ingestor.\nError: ", error.message);
17+
});
3218
});

‎ingestors/node/yarn.lock

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,20 @@ arg@^4.1.0:
8080
resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089"
8181
integrity sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==
8282

83+
asynckit@^0.4.0:
84+
version "0.4.0"
85+
resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
86+
integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==
87+
88+
axios@^1.1.3:
89+
version "1.1.3"
90+
resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35"
91+
integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA==
92+
dependencies:
93+
follow-redirects "^1.15.0"
94+
form-data "^4.0.0"
95+
proxy-from-env "^1.1.0"
96+
8397
balanced-match@^1.0.0:
8498
version "1.0.2"
8599
resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
@@ -120,6 +134,13 @@ chokidar@^3.5.2:
120134
optionalDependencies:
121135
fsevents "~2.3.2"
122136

137+
combined-stream@^1.0.8:
138+
version "1.0.8"
139+
resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f"
140+
integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==
141+
dependencies:
142+
delayed-stream "~1.0.0"
143+
123144
[email protected]:
124145
version "0.0.1"
125146
resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
@@ -144,6 +165,11 @@ debug@^4.1.1:
144165
dependencies:
145166
ms "2.1.2"
146167

168+
delayed-stream@~1.0.0:
169+
version "1.0.0"
170+
resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619"
171+
integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==
172+
147173
diff@^4.0.1:
148174
version "4.0.2"
149175
resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d"
@@ -156,6 +182,20 @@ fill-range@^7.0.1:
156182
dependencies:
157183
to-regex-range "^5.0.1"
158184

185+
follow-redirects@^1.15.0:
186+
version "1.15.2"
187+
resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13"
188+
integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==
189+
190+
form-data@^4.0.0:
191+
version "4.0.0"
192+
resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452"
193+
integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==
194+
dependencies:
195+
asynckit "^0.4.0"
196+
combined-stream "^1.0.8"
197+
mime-types "^2.1.12"
198+
159199
fsevents@~2.3.2:
160200
version "2.3.2"
161201
resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
@@ -226,6 +266,18 @@ make-error@^1.1.1:
226266
resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2"
227267
integrity sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==
228268

269+
[email protected]:
270+
version "1.52.0"
271+
resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70"
272+
integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==
273+
274+
mime-types@^2.1.12:
275+
version "2.1.35"
276+
resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a"
277+
integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==
278+
dependencies:
279+
mime-db "1.52.0"
280+
229281
minimatch@^3.1.2:
230282
version "3.1.2"
231283
resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b"
@@ -286,6 +338,11 @@ picomatch@^2.0.4, picomatch@^2.2.1:
286338
resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42"
287339
integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==
288340

341+
proxy-from-env@^1.1.0:
342+
version "1.1.0"
343+
resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2"
344+
integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==
345+
289346
pstree.remy@^1.1.8:
290347
version "1.1.8"
291348
resolved "https://registry.yarnpkg.com/pstree.remy/-/pstree.remy-1.1.8.tgz#c242224f4a67c21f686839bbdb4ac282b8373d3a"

‎ingestors/python/metlo/django.py

Lines changed: 49 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,64 +1,93 @@
11
import json
22
from concurrent.futures import ThreadPoolExecutor
33
from urllib.request import Request, urlopen
4+
from urllib.parse import urlparse
45

56
from django.conf import settings
67

8+
endpoint = "api/v1/log-request/single"
79

8-
class MetloDjango(object):
910

11+
class MetloDjango(object):
1012
def perform_request(self, data):
11-
urlopen(
12-
url=self.saved_request,
13-
data=json.dumps(data).encode('utf-8')
14-
)
13+
urlopen(url=self.saved_request, data=json.dumps(data).encode("utf-8"))
1514

1615
def __init__(self, get_response):
1716
"""
1817
Middleware for Django to communicate with METLO
1918
:param get_response: Automatically populated by django
2019
"""
2120
self.get_response = get_response
22-
self.pool = ThreadPoolExecutor(max_workers=settings.METLO_CONFIG.get("workers", 4))
21+
self.pool = ThreadPoolExecutor(
22+
max_workers=settings.METLO_CONFIG.get("workers", 4)
23+
)
2324

24-
assert settings.METLO_CONFIG.get("METLO_HOST") is not None, "METLO_CONFIG is missing METLO_HOST attribute"
25-
assert settings.METLO_CONFIG.get("API_KEY") is not None, "METLO_CONFIG is missing API_KEY attribute"
25+
assert (
26+
settings.METLO_CONFIG.get("METLO_HOST") is not None
27+
), "METLO_CONFIG is missing METLO_HOST attribute"
28+
assert (
29+
settings.METLO_CONFIG.get("API_KEY") is not None
30+
), "METLO_CONFIG is missing API_KEY attribute"
31+
assert urlparse(settings.METLO_CONFIG.get("METLO_HOST")).scheme in [
32+
"http",
33+
"https",
34+
], f"Metlo for Django has invalid host scheme. Host must be in format http[s]://example.com"
2635

2736
self.host = settings.METLO_CONFIG["METLO_HOST"]
37+
self.host += endpoint if self.host[-1] == "/" else f"/{endpoint}"
2838
self.key = settings.METLO_CONFIG["API_KEY"]
2939
self.saved_request = Request(
3040
url=self.host,
3141
headers={
3242
"Content-Type": "application/json; charset=utf-8",
3343
"Authorization": self.key,
3444
},
35-
method="POST"
45+
method="POST",
3646
)
3747

3848
def __call__(self, request):
3949
response = self.get_response(request)
4050
params = request.GET if request.method == "GET" else request.POST
41-
dest_ip = request.META.get("SERVER_NAME") if \
42-
"1.0.0.127.in-addr.arpa" not in request.META.get("SERVER_NAME") else "localhost"
43-
src_ip = request.META.get("REMOTE_ADDR") if \
44-
"1.0.0.127.in-addr.arpa" not in request.META.get("REMOTE_ADDR") else "localhost"
51+
dest_ip = (
52+
request.META.get("SERVER_NAME")
53+
if "1.0.0.127.in-addr.arpa" not in request.META.get("SERVER_NAME")
54+
else "localhost"
55+
)
56+
src_ip = (
57+
request.META.get("REMOTE_ADDR")
58+
if "1.0.0.127.in-addr.arpa" not in request.META.get("REMOTE_ADDR")
59+
else "localhost"
60+
)
4561
source_port = request.environ["wsgi.input"].stream.raw._sock.getpeername()[1]
4662
res_body = response.content.decode("utf-8")
4763
data = {
4864
"request": {
4965
"url": {
50-
"host": request._current_scheme_host if request._current_scheme_host else src_ip,
66+
"host": request._current_scheme_host
67+
if request._current_scheme_host
68+
else src_ip,
5169
"path": request.path,
52-
"parameters": list(map(lambda x: {"name": x[0], "value": x[1]}, params.items())),
70+
"parameters": list(
71+
map(lambda x: {"name": x[0], "value": x[1]}, params.items())
72+
),
5373
},
54-
"headers": list(map(lambda x: {"name": x[0], "value": x[1]}, request.headers.items())),
74+
"headers": list(
75+
map(
76+
lambda x: {"name": x[0], "value": x[1]}, request.headers.items()
77+
)
78+
),
5579
"body": request.body.decode("utf-8"),
5680
"method": request.method,
5781
},
5882
"response": {
5983
"url": f"{dest_ip}:{request.META.get('SERVER_PORT')}",
6084
"status": response.status_code,
61-
"headers": list(map(lambda x: {"name": x[0], "value": x[1]}, response.headers.items())),
85+
"headers": list(
86+
map(
87+
lambda x: {"name": x[0], "value": x[1]},
88+
response.headers.items(),
89+
)
90+
),
6291
"body": res_body,
6392
},
6493
"meta": {
@@ -67,8 +96,9 @@ def __call__(self, request):
6796
"source": src_ip,
6897
"sourcePort": source_port,
6998
"destination": dest_ip,
70-
"destinationPort": request.META.get('SERVER_PORT'),
71-
}
99+
"destinationPort": request.META.get("SERVER_PORT"),
100+
"metloSource": "python/django",
101+
},
72102
}
73103
self.pool.submit(self.perform_request, data=data)
74104
return response

0 commit comments

Comments
 (0)