feature: add default redacted fields and value

backend/src/collector.ts

@@ -44,11 +44,27 @@ const addToBlockFields = (
   pathRegex: string,
   disabledPaths: string[],
 ) => {
+  const disabledPathsObj = {
+    reqQuery: [],
+    reqHeaders: [],
+    reqBody: [],
+    resHeaders: [],
+    resBody: [],
+  }
+  disabledPaths.forEach(path => {
+    if (path.includes("req.query")) disabledPathsObj.reqQuery.push(path)
+    else if (path.includes("req.headers"))
+      disabledPathsObj.reqHeaders.push(path)
+    else if (path.includes("req.body")) disabledPathsObj.reqBody.push(path)
+    else if (path.includes("res.headers"))
+      disabledPathsObj.resHeaders.push(path)
+    else if (path.includes("res.body")) disabledPathsObj.resBody.push(path)
+  })
   const entry = {
     method,
     path,
     pathRegex,
-    disabledPaths,
+    disabledPaths: disabledPathsObj,
     numberParams: BlockFieldsService.getNumberParams(pathRegex, method, path),
   }
   if (blockFieldsEntries[host]) {

backend/src/services/block-fields/index.ts

@@ -3,6 +3,7 @@ import { getDataType, isParameter, parsedJson, parsedJsonNonNull } from "utils"
 import { BlockFieldEntry, PairObject, QueuedApiTrace } from "@common/types"
 import { getPathTokens } from "@common/utils"
 import { BLOCK_FIELDS_ALL_REGEX } from "~/constants"
+import { isSensitiveDataKey } from "./utils"
 
 export class BlockFieldsService {
   static entries: Record<string, BlockFieldEntry[]> = {}
@@ -50,23 +51,15 @@ export class BlockFieldsService {
     return entry
   }
 
-  static isContained(
-    arr: string[],
-    str: string,
-  ): { fully: boolean; partially: boolean } {
-    let res = { fully: false, partially: false }
+  static isContained(arr: string[], str: string): boolean {
     const strLower = str.toLowerCase()
-    arr.forEach(e => {
-      const entryLower = e.toLowerCase()
+    for (const e of arr) {
+      const entryLower = e.toLowerCase().trim()
       if (entryLower === strLower) {
-        res["fully"] = true
-        res["partially"] = true
-        return res
-      } else if (entryLower.includes(strLower)) {
-        res["partially"] = true
+        return true
       }
-    })
-    return res
+    }
+    return false
   }
 
   static recursiveParseBody(
@@ -81,15 +74,15 @@ export class BlockFieldsService {
     if (dataType === DataType.OBJECT) {
       for (const key in jsonBody) {
         const contained = this.isContained(disabledPaths, `${path}.${key}`)
-        if (redacted || contained.fully) {
+        if (redacted || contained || isSensitiveDataKey(key)) {
           jsonBody[key] = this.recursiveParseBody(
             `${dataPath}.${key}`,
             dataSection,
             jsonBody[key],
             disabledPaths,
             true,
           )
-        } else if (contained.partially) {
+        } else {
           jsonBody[key] = this.recursiveParseBody(
             `${dataPath}.${key}`,
             dataSection,
@@ -126,13 +119,11 @@ export class BlockFieldsService {
     if (!body) {
       return
     }
-    if (!disabledPaths || disabledPaths?.length === 0) {
-      return body
-    }
     let redacted = false
-    if (this.isContained(disabledPaths, dataSection).fully) {
+    if (this.isContained(disabledPaths, dataSection)) {
       redacted = true
     }
+
     let jsonBody = parsedJson(body)
     if (jsonBody) {
       const dataType = getDataType(jsonBody)
@@ -142,15 +133,15 @@ export class BlockFieldsService {
             disabledPaths,
             `${dataSection}.${key}`,
           )
-          if (redacted || contained.fully) {
+          if (redacted || contained || isSensitiveDataKey(key)) {
             jsonBody[key] = this.recursiveParseBody(
               key,
               dataSection,
               jsonBody[key],
               disabledPaths,
               true,
             )
-          } else if (contained.partially) {
+          } else {
             jsonBody[key] = this.recursiveParseBody(
               key,
               dataSection,
@@ -170,14 +161,10 @@ export class BlockFieldsService {
             redacted,
           )
         })
-      } else {
-        if (redacted) {
-          jsonBody = "[REDACTED]"
-        }
       }
     } else {
       if (redacted) {
-        jsonBody = "[REDACTED]"
+        body = "[REDACTED]"
       }
     }
     return jsonBody ?? body
@@ -191,21 +178,15 @@ export class BlockFieldsService {
     if (!data) {
       return data
     }
-    if (!disabledPaths || disabledPaths?.length === 0) {
-      return data
-    }
     let redacted = false
-    if (this.isContained(disabledPaths, dataSection).fully) {
+    if (this.isContained(disabledPaths, dataSection)) {
       redacted = true
     }
     return data.map(item => {
       const contained = this.isContained(
         disabledPaths,
         `${dataSection}.${item.name}`,
       )
-      if (!redacted && !contained.fully && !contained.partially) {
-        return item
-      }
 
       return {
         name: item.name,
@@ -214,47 +195,52 @@ export class BlockFieldsService {
           dataSection,
           parsedJsonNonNull(item.value, true),
           disabledPaths,
-          redacted || contained.fully,
+          redacted || contained || isSensitiveDataKey(item.name),
         ),
       }
     })
   }
 
   static async redactBlockedFields(apiTrace: QueuedApiTrace) {
     const blockFieldEntry = this.getBlockFieldsEntry(apiTrace)
-    if (blockFieldEntry) {
-      const disabledPaths = blockFieldEntry.disabledPaths
-      const validRequestParams = this.redactBlockedFieldsPairObject(
-        apiTrace.requestParameters,
-        "req.query",
-        disabledPaths.filter(e => e.includes("req.query")),
-      )
-      const validRequestHeaders = this.redactBlockedFieldsPairObject(
-        apiTrace.requestHeaders,
-        "req.headers",
-        disabledPaths.filter(e => e.includes("req.headers")),
-      )
-      const validRequestBody = this.redactBlockedFieldsBodyData(
-        apiTrace.requestBody,
-        "req.body",
-        disabledPaths.filter(e => e.includes("req.body")),
-      )
-      const validResponseHeaders = this.redactBlockedFieldsPairObject(
-        apiTrace.responseHeaders,
-        "res.headers",
-        disabledPaths.filter(e => e.includes("res.headers")),
-      )
-      const validResponseBody = this.redactBlockedFieldsBodyData(
-        apiTrace.responseBody,
-        "res.body",
-        disabledPaths.filter(e => e.includes("res.body")),
-      )
-
-      apiTrace.requestParameters = validRequestParams
-      apiTrace.requestHeaders = validRequestHeaders
-      apiTrace.requestBody = validRequestBody
-      apiTrace.responseHeaders = validResponseHeaders
-      apiTrace.responseBody = validResponseBody
+    const disabledPaths = blockFieldEntry?.disabledPaths ?? {
+      reqQuery: [],
+      reqHeaders: [],
+      reqBody: [],
+      resHeaders: [],
+      resBody: [],
     }
+
+    const validRequestParams = this.redactBlockedFieldsPairObject(
+      apiTrace.requestParameters,
+      "req.query",
+      disabledPaths.reqQuery,
+    )
+    const validRequestHeaders = this.redactBlockedFieldsPairObject(
+      apiTrace.requestHeaders,
+      "req.headers",
+      disabledPaths.reqHeaders,
+    )
+    const validRequestBody = this.redactBlockedFieldsBodyData(
+      apiTrace.requestBody,
+      "req.body",
+      disabledPaths.reqBody,
+    )
+    const validResponseHeaders = this.redactBlockedFieldsPairObject(
+      apiTrace.responseHeaders,
+      "res.headers",
+      disabledPaths.resHeaders,
+    )
+    const validResponseBody = this.redactBlockedFieldsBodyData(
+      apiTrace.responseBody,
+      "res.body",
+      disabledPaths.resBody,
+    )
+
+    apiTrace.requestParameters = validRequestParams
+    apiTrace.requestHeaders = validRequestHeaders
+    apiTrace.requestBody = validRequestBody
+    apiTrace.responseHeaders = validResponseHeaders
+    apiTrace.responseBody = validResponseBody
   }
 }

backend/src/services/block-fields/utils.ts

@@ -0,0 +1,17 @@
+const scrubKeys = [
+  "authorization",
+  "api_key",
+  "apikey",
+  "password",
+  "secret",
+  "passwd",
+  "access_token",
+  "x-api-key",
+  "x_api_key",
+  "api-key",
+]
+
+export const isSensitiveDataKey = (key: string) => {
+  const keyLowerCase = key?.toLowerCase() ?? null
+  return key && scrubKeys.includes(keyLowerCase)
+}

backend/src/services/spec/index.ts

@@ -489,7 +489,7 @@ export class SpecService {
       const respErrorItems = generateAlertMessageFromRespErrors(
         responseErrors as AjvError[],
         responses?.path,
-        blockFieldEntry?.disabledPaths ?? [],
+        blockFieldEntry?.disabledPaths?.resBody ?? [],
       )
 
       const errorItems = { ...respErrorItems }

backend/src/services/spec/utils.ts

@@ -243,9 +243,7 @@ export const generateAlertMessageFromRespErrors = (
         errorMessage = `Required property '${path}' is missing from response body.`
         break
       case "type":
-        if (isDisabledPath) {
-          ignoreError = true
-        }
+        ignoreError = true
         errorMessage = `Property '${path}' ${error.message} in response body.`
         break
       case "additionalProperties":
@@ -269,9 +267,7 @@ export const generateAlertMessageFromRespErrors = (
           `Property '${path}' is present in response body without matching any schemas/definitions in the OpenAPI Spec.`
         break
       case "format":
-        if (isDisabledPath) {
-          ignoreError = true
-        }
+        ignoreError = true
         errorMessage = `Property '${path}' ${error.message} in response body.`
         break
       default:

backend/src/utils/index.ts

@@ -116,7 +116,12 @@ export const getDataType = (data: any): DataType => {
 
 export const parsedJson = (jsonString: string): any => {
   try {
-    return JSON.parse(jsonString)
+    if (typeof jsonString === "object" || Array.isArray(jsonString)) {
+      return jsonString
+    }
+    const parsed = JSON.parse(jsonString)
+    const isNonScalar = typeof parsed === "object" || Array.isArray(parsed)
+    return isNonScalar ? parsed : null
   } catch (err) {
     return null
   }
@@ -127,7 +132,12 @@ export const parsedJsonNonNull = (
   returnString?: boolean,
 ): any => {
   try {
-    return JSON.parse(jsonString)
+    if (typeof jsonString === "object" || Array.isArray(jsonString)) {
+      return jsonString
+    }
+    const parsed = JSON.parse(jsonString)
+    const isNonScalar = typeof parsed === "object" || Array.isArray(parsed)
+    return isNonScalar ? parsed : jsonString
   } catch (err) {
     if (returnString) {
       return jsonString

common/src/enums.ts

@@ -189,4 +189,4 @@ export enum API_KEY_TYPE {
 export enum OperationType {
   QUERY = "query",
   MUTATION = "mutation",
-}
+}

common/src/types.ts

@@ -510,10 +510,18 @@ export interface AuthenticationConfig {
   cookieName: string
 }
 
+interface DisabledPathSection {
+  reqQuery: string[]
+  reqHeaders: string[]
+  reqBody: string[]
+  resHeaders: string[]
+  resBody: string[]
+}
+
 export interface BlockFieldEntry {
   path: string
   pathRegex: string
   method: DisableRestMethod,
   numberParams: number
-  disabledPaths: string[]
+  disabledPaths: DisabledPathSection
 }