switch node agent to typescript

ingestors/node/src/index.ts

@@ -0,0 +1,30 @@
+import os from "node:os"
+import SetupMiddleware from "./middleware"
+import WorkerPool from "./pool"
+
+const pool = new WorkerPool(os.cpus().length, "./workerTarget.js")
+const endpoint = "api/v1/log-request/single"
+
+function exit() {
+  pool.close()
+}
+
+process.on("exit", exit)
+process.on("SIGTERM", exit)
+
+const init = (key: string, host: string) => {
+  try {
+    new URL(host)
+  } catch (err) {
+    console.error(err)
+    throw new Error(`Couldn't load metlo. Host is not a proper url : ${host}`)
+  }
+  let metlo_host = host
+  if (metlo_host[metlo_host.length - 1] != "/") {
+    metlo_host += "/"
+  }
+  metlo_host += endpoint
+  SetupMiddleware({ host: metlo_host, key, pool })
+}
+
+export default init

ingestors/node/src/middleware/express.ts

@@ -0,0 +1,100 @@
+import { METLO_POOL } from "."
+
+const ritm = require("require-in-the-middle")
+
+function versionCheck() {
+  return true
+}
+
+const initialize = () => {
+  if (!versionCheck()) {
+    return
+  }
+
+  function compileInformation(_req, _res, responseBody) {
+    const data = JSON.stringify({
+      request: {
+        url: {
+          host: _req.socket.remoteAddress,
+          path: _req.baseUrl + _req.url,
+          parameters: Object.entries(_req.query).map(([k, v]) => ({
+            name: k,
+            value: v,
+          })),
+        },
+        headers: Object.entries(_req.headers).map(([k, v]) => ({
+          name: k,
+          value: v,
+        })),
+        body: _req.body || "No Body",
+        method: _req.method,
+      },
+      response: {
+        url: `${_req.socket.localAddress}:${_req.socket.localPort}`,
+        status: _res.statusCode,
+        headers: Object.entries(_res.getHeaders()).map(([k, v]) => ({
+          name: k,
+          value: v,
+        })),
+        body: responseBody,
+      },
+      meta: {
+        environment: process.env.NODE_ENV,
+        incoming: true,
+        source: _req.socket.remoteAddress,
+        sourcePort: _req.socket.remotePort,
+        destination: _res.socket.localAddress,
+        destinationPort: _res.socket.localPort,
+        metloSource: "node/express",
+      },
+    })
+
+    METLO_POOL.pool.runTask({
+      host: METLO_POOL.host,
+      key: METLO_POOL.key,
+      data,
+    })
+  }
+
+  ritm(["express"], function (exports, name, basedir) {
+    const original_send = exports.response.send
+    const original_json = exports.response.json
+    const original_sendFile = exports.response.sendFile
+
+    function modifiedSend() {
+      const resp = original_send.apply(this, arguments)
+      compileInformation(this.req, resp, arguments[arguments.length - 1])
+      return resp
+    }
+
+    function modifiedJSON() {
+      this.send = original_send
+      const resp = original_json.apply(this, arguments)
+      const stack = new Error().stack
+      if (stack.includes("ServerResponse.modifiedSend")) {
+        // JSON was triggered by patched send
+        // Let send handle the complexities of communicating with metlo
+      } else {
+        // JSON wasn't triggered by patched send.
+        // Feel free to compile Information here.
+        compileInformation(this.req, resp, arguments[arguments.length - 1])
+      }
+      return resp
+    }
+
+    function modifiedSendFile() {
+      const resp = original_sendFile.apply(this, arguments)
+      compileInformation(this.req, this, arguments[0])
+      return resp
+    }
+
+    exports.response.send = modifiedSend
+    exports.response.json = modifiedJSON
+    exports.response.jsonp = modifiedJSON
+    exports.response.sendFile = modifiedSendFile
+
+    return exports
+  })
+}
+
+export default initialize

ingestors/node/src/middleware/fastify.ts

@@ -0,0 +1,75 @@
+import { METLO_POOL } from "."
+
+const ritm = require("require-in-the-middle")
+
+function versionCheck() {
+  return true
+}
+
+const initialize = () => {
+  if (!versionCheck()) {
+    return
+  }
+
+  async function compileInformation(request, response, response_body) {
+    const data = JSON.stringify({
+      request: {
+        url: {
+          host: request.raw.socket.remoteAddress,
+          path: request.url.split("?")[0],
+          parameters: Object.entries(request.query).map(([k, v]) => ({
+            name: k,
+            value: v,
+          })),
+        },
+        headers: Object.entries(request.headers).map(([k, v]) => ({
+          name: k,
+          value: v,
+        })),
+        body: request.body || "",
+        method: request.method,
+      },
+      response: {
+        url: `${response.raw.socket.localAddress}:${response.raw.socket.localPort}`,
+        status: response.statusCode,
+        headers: Object.entries(response.headers).map(([k, v]) => ({
+          name: k,
+          value: v,
+        })),
+        body: response_body,
+      },
+      meta: {
+        environment: process.env.NODE_ENV,
+        incoming: true,
+        source: request.raw.socket.remoteAddress,
+        sourcePort: request.raw.socket.remotePort,
+        destination: response.raw.socket.localAddress,
+        destinationPort: response.raw.socket.localPort,
+        metloSource: "node/fastify",
+      },
+    })
+
+    METLO_POOL.pool.runTask({
+      host: METLO_POOL.host,
+      key: METLO_POOL.key,
+      data,
+    })
+  }
+
+  ritm(["fastify"], function (exports, name, basedir) {
+    const originalFastify = exports
+    function modifiedFastify() {
+      let fastifyInst = originalFastify.apply(this, arguments)
+      fastifyInst.addHook("onSend", async (request, reply, payload) => {
+        compileInformation(request, reply, payload)
+        return payload
+      })
+      return fastifyInst
+    }
+    exports = modifiedFastify
+
+    return exports
+  })
+}
+
+export default initialize

ingestors/node/src/middleware/index.ts

@@ -0,0 +1,39 @@
+import WorkerPool from "../pool"
+import { MetloConfig } from "../types"
+
+export interface InitMiddlewareParams extends MetloConfig {
+  pool: WorkerPool
+}
+
+export let METLO_POOL: InitMiddlewareParams = {
+  key: "",
+  host: "",
+  pool: null,
+}
+
+import ExpressModule from "./express"
+import KoaModule from "./koa"
+import FastifyModule from "./fastify"
+import { getDependencies } from "../utils"
+
+const nameToModule = {
+  express: ExpressModule,
+  koa: KoaModule,
+  fastify: FastifyModule,
+}
+
+const InitMiddleware = (params: InitMiddlewareParams) => {
+  METLO_POOL.key = params.key
+  METLO_POOL.host = params.host
+  METLO_POOL.pool = params.pool
+  let dependencies = getDependencies()
+  Object.entries(nameToModule).forEach(([name, module]) => {
+    try {
+      if (dependencies.includes(name)) module()
+    } catch (err) {
+      // pass
+    }
+  })
+}
+
+export default InitMiddleware

ingestors/node/src/middleware/koa.ts

@@ -0,0 +1,96 @@
+import { METLO_POOL } from "."
+
+const ritm = require("require-in-the-middle")
+
+function versionCheck() {
+  return true
+}
+
+function initialize() {
+  if (!versionCheck()) {
+    return
+  }
+
+  async function compileInformation(ctx, next) {
+    await next()
+    const data = JSON.stringify({
+      request: {
+        url: {
+          host: ctx.response.socket.remoteAddress,
+          path: ctx.path,
+          parameters: Object.entries(ctx.query).map(([k, v]) => ({
+            name: k,
+            value: v,
+          })),
+        },
+        headers: Object.entries(ctx.request.headers).map(([k, v]) => ({
+          name: k,
+          value: v,
+        })),
+        body: ctx.request.body || "",
+        method: ctx.request.method,
+      },
+      response: {
+        url: `${ctx.response.socket.localAddress}:${ctx.response.socket.localPort}`,
+        status: ctx.response.statusCode,
+        headers: Object.entries(ctx.response.headers).map(([k, v]) => ({
+          name: k,
+          value: v,
+        })),
+        body: ctx.body,
+      },
+      meta: {
+        environment: process.env.NODE_ENV,
+        incoming: true,
+        source: ctx.request.socket.remoteAddress,
+        sourcePort: ctx.request.socket.remotePort,
+        destination: ctx.request.socket.localAddress,
+        destinationPort: ctx.request.socket.localPort,
+        metloSource: "node/koa",
+      },
+    })
+
+    METLO_POOL.pool.runTask({
+      host: METLO_POOL.host,
+      key: METLO_POOL.key,
+      data,
+    })
+  }
+
+  compileInformation._meta_ = Symbol("metlo-koa")
+
+  ritm(["koa"], function (exports, name, basedir) {
+    // const module = exports.middlewares.find((fn, idx) => fn.has)
+    //Find if our function exists
+    // If it doesn't, then add it at 0th position
+    // If it does, then take it out and move it down to our preffered location
+
+    const original_use = exports.prototype.use
+
+    function modifiedUse() {
+      const idx = this.middleware.findIndex(fn => fn["_meta_"] !== undefined)
+      if (idx != -1) {
+        if (idx !== 0) {
+          this.middleware = this.middleware
+            // Remove function at same index where metlo middleware was found
+            .filter((_, _idx) => _idx !== idx)
+            // Set Metlo middleware at idx 0
+            .unshift(compileInformation)
+        }
+      } else {
+        if (this.middleware.length === 0) {
+          this.middleware = [compileInformation]
+        } else {
+          Array.from(this.middleware).unshift([compileInformation])
+        }
+      }
+      let resp = original_use.apply(this, arguments)
+      return resp
+    }
+    exports.prototype.use = modifiedUse
+
+    return exports
+  })
+}
+
+export default initialize

ingestors/node/src/middleware/types.ts

@@ -0,0 +1 @@
+export interface MiddlewareParams {}