diff --git a/backend/src/collector.ts b/backend/src/collector.ts index 0773aed0..05333037 100644 --- a/backend/src/collector.ts +++ b/backend/src/collector.ts @@ -44,11 +44,27 @@ const addToBlockFields = ( pathRegex: string, disabledPaths: string[], ) => { + const disabledPathsObj = { + reqQuery: [], + reqHeaders: [], + reqBody: [], + resHeaders: [], + resBody: [], + } + disabledPaths.forEach(path => { + if (path.includes("req.query")) disabledPathsObj.reqQuery.push(path) + else if (path.includes("req.headers")) + disabledPathsObj.reqHeaders.push(path) + else if (path.includes("req.body")) disabledPathsObj.reqBody.push(path) + else if (path.includes("res.headers")) + disabledPathsObj.resHeaders.push(path) + else if (path.includes("res.body")) disabledPathsObj.resBody.push(path) + }) const entry = { method, path, pathRegex, - disabledPaths, + disabledPaths: disabledPathsObj, numberParams: BlockFieldsService.getNumberParams(pathRegex, method, path), } if (blockFieldsEntries[host]) { diff --git a/backend/src/services/block-fields/index.ts b/backend/src/services/block-fields/index.ts index 6eb66c2d..2a2d36e1 100644 --- a/backend/src/services/block-fields/index.ts +++ b/backend/src/services/block-fields/index.ts @@ -3,6 +3,7 @@ import { getDataType, isParameter, parsedJson, parsedJsonNonNull } from "utils" import { BlockFieldEntry, PairObject, QueuedApiTrace } from "@common/types" import { getPathTokens } from "@common/utils" import { BLOCK_FIELDS_ALL_REGEX } from "~/constants" +import { isSensitiveDataKey } from "./utils" export class BlockFieldsService { static entries: Record = {} @@ -50,23 +51,15 @@ export class BlockFieldsService { return entry } - static isContained( - arr: string[], - str: string, - ): { fully: boolean; partially: boolean } { - let res = { fully: false, partially: false } + static isContained(arr: string[], str: string): boolean { const strLower = str.toLowerCase() - arr.forEach(e => { - const entryLower = e.toLowerCase() + for (const e of arr) { + const entryLower = e.toLowerCase().trim() if (entryLower === strLower) { - res["fully"] = true - res["partially"] = true - return res - } else if (entryLower.includes(strLower)) { - res["partially"] = true + return true } - }) - return res + } + return false } static recursiveParseBody( @@ -81,7 +74,7 @@ export class BlockFieldsService { if (dataType === DataType.OBJECT) { for (const key in jsonBody) { const contained = this.isContained(disabledPaths, `${path}.${key}`) - if (redacted || contained.fully) { + if (redacted || contained || isSensitiveDataKey(key)) { jsonBody[key] = this.recursiveParseBody( `${dataPath}.${key}`, dataSection, @@ -89,7 +82,7 @@ export class BlockFieldsService { disabledPaths, true, ) - } else if (contained.partially) { + } else { jsonBody[key] = this.recursiveParseBody( `${dataPath}.${key}`, dataSection, @@ -126,13 +119,11 @@ export class BlockFieldsService { if (!body) { return } - if (!disabledPaths || disabledPaths?.length === 0) { - return body - } let redacted = false - if (this.isContained(disabledPaths, dataSection).fully) { + if (this.isContained(disabledPaths, dataSection)) { redacted = true } + let jsonBody = parsedJson(body) if (jsonBody) { const dataType = getDataType(jsonBody) @@ -142,7 +133,7 @@ export class BlockFieldsService { disabledPaths, `${dataSection}.${key}`, ) - if (redacted || contained.fully) { + if (redacted || contained || isSensitiveDataKey(key)) { jsonBody[key] = this.recursiveParseBody( key, dataSection, @@ -150,7 +141,7 @@ export class BlockFieldsService { disabledPaths, true, ) - } else if (contained.partially) { + } else { jsonBody[key] = this.recursiveParseBody( key, dataSection, @@ -170,14 +161,10 @@ export class BlockFieldsService { redacted, ) }) - } else { - if (redacted) { - jsonBody = "[REDACTED]" - } } } else { if (redacted) { - jsonBody = "[REDACTED]" + body = "[REDACTED]" } } return jsonBody ?? body @@ -191,11 +178,8 @@ export class BlockFieldsService { if (!data) { return data } - if (!disabledPaths || disabledPaths?.length === 0) { - return data - } let redacted = false - if (this.isContained(disabledPaths, dataSection).fully) { + if (this.isContained(disabledPaths, dataSection)) { redacted = true } return data.map(item => { @@ -203,9 +187,6 @@ export class BlockFieldsService { disabledPaths, `${dataSection}.${item.name}`, ) - if (!redacted && !contained.fully && !contained.partially) { - return item - } return { name: item.name, @@ -214,7 +195,7 @@ export class BlockFieldsService { dataSection, parsedJsonNonNull(item.value, true), disabledPaths, - redacted || contained.fully, + redacted || contained || isSensitiveDataKey(item.name), ), } }) @@ -222,39 +203,44 @@ export class BlockFieldsService { static async redactBlockedFields(apiTrace: QueuedApiTrace) { const blockFieldEntry = this.getBlockFieldsEntry(apiTrace) - if (blockFieldEntry) { - const disabledPaths = blockFieldEntry.disabledPaths - const validRequestParams = this.redactBlockedFieldsPairObject( - apiTrace.requestParameters, - "req.query", - disabledPaths.filter(e => e.includes("req.query")), - ) - const validRequestHeaders = this.redactBlockedFieldsPairObject( - apiTrace.requestHeaders, - "req.headers", - disabledPaths.filter(e => e.includes("req.headers")), - ) - const validRequestBody = this.redactBlockedFieldsBodyData( - apiTrace.requestBody, - "req.body", - disabledPaths.filter(e => e.includes("req.body")), - ) - const validResponseHeaders = this.redactBlockedFieldsPairObject( - apiTrace.responseHeaders, - "res.headers", - disabledPaths.filter(e => e.includes("res.headers")), - ) - const validResponseBody = this.redactBlockedFieldsBodyData( - apiTrace.responseBody, - "res.body", - disabledPaths.filter(e => e.includes("res.body")), - ) - - apiTrace.requestParameters = validRequestParams - apiTrace.requestHeaders = validRequestHeaders - apiTrace.requestBody = validRequestBody - apiTrace.responseHeaders = validResponseHeaders - apiTrace.responseBody = validResponseBody + const disabledPaths = blockFieldEntry?.disabledPaths ?? { + reqQuery: [], + reqHeaders: [], + reqBody: [], + resHeaders: [], + resBody: [], } + + const validRequestParams = this.redactBlockedFieldsPairObject( + apiTrace.requestParameters, + "req.query", + disabledPaths.reqQuery, + ) + const validRequestHeaders = this.redactBlockedFieldsPairObject( + apiTrace.requestHeaders, + "req.headers", + disabledPaths.reqHeaders, + ) + const validRequestBody = this.redactBlockedFieldsBodyData( + apiTrace.requestBody, + "req.body", + disabledPaths.reqBody, + ) + const validResponseHeaders = this.redactBlockedFieldsPairObject( + apiTrace.responseHeaders, + "res.headers", + disabledPaths.resHeaders, + ) + const validResponseBody = this.redactBlockedFieldsBodyData( + apiTrace.responseBody, + "res.body", + disabledPaths.resBody, + ) + + apiTrace.requestParameters = validRequestParams + apiTrace.requestHeaders = validRequestHeaders + apiTrace.requestBody = validRequestBody + apiTrace.responseHeaders = validResponseHeaders + apiTrace.responseBody = validResponseBody } } diff --git a/backend/src/services/block-fields/utils.ts b/backend/src/services/block-fields/utils.ts new file mode 100644 index 00000000..f7e538e3 --- /dev/null +++ b/backend/src/services/block-fields/utils.ts @@ -0,0 +1,17 @@ +const scrubKeys = [ + "authorization", + "api_key", + "apikey", + "password", + "secret", + "passwd", + "access_token", + "x-api-key", + "x_api_key", + "api-key", +] + +export const isSensitiveDataKey = (key: string) => { + const keyLowerCase = key?.toLowerCase() ?? null + return key && scrubKeys.includes(keyLowerCase) +} diff --git a/backend/src/services/spec/index.ts b/backend/src/services/spec/index.ts index 41f81cdd..89f492f6 100644 --- a/backend/src/services/spec/index.ts +++ b/backend/src/services/spec/index.ts @@ -489,7 +489,7 @@ export class SpecService { const respErrorItems = generateAlertMessageFromRespErrors( responseErrors as AjvError[], responses?.path, - blockFieldEntry?.disabledPaths ?? [], + blockFieldEntry?.disabledPaths?.resBody ?? [], ) const errorItems = { ...respErrorItems } diff --git a/backend/src/services/spec/utils.ts b/backend/src/services/spec/utils.ts index 101d59db..f9ef3292 100644 --- a/backend/src/services/spec/utils.ts +++ b/backend/src/services/spec/utils.ts @@ -243,9 +243,7 @@ export const generateAlertMessageFromRespErrors = ( errorMessage = `Required property '${path}' is missing from response body.` break case "type": - if (isDisabledPath) { - ignoreError = true - } + ignoreError = true errorMessage = `Property '${path}' ${error.message} in response body.` break case "additionalProperties": @@ -269,9 +267,7 @@ export const generateAlertMessageFromRespErrors = ( `Property '${path}' is present in response body without matching any schemas/definitions in the OpenAPI Spec.` break case "format": - if (isDisabledPath) { - ignoreError = true - } + ignoreError = true errorMessage = `Property '${path}' ${error.message} in response body.` break default: diff --git a/backend/src/utils/index.ts b/backend/src/utils/index.ts index 2ae4246f..2f19e0f4 100644 --- a/backend/src/utils/index.ts +++ b/backend/src/utils/index.ts @@ -116,7 +116,12 @@ export const getDataType = (data: any): DataType => { export const parsedJson = (jsonString: string): any => { try { - return JSON.parse(jsonString) + if (typeof jsonString === "object" || Array.isArray(jsonString)) { + return jsonString + } + const parsed = JSON.parse(jsonString) + const isNonScalar = typeof parsed === "object" || Array.isArray(parsed) + return isNonScalar ? parsed : null } catch (err) { return null } @@ -127,7 +132,12 @@ export const parsedJsonNonNull = ( returnString?: boolean, ): any => { try { - return JSON.parse(jsonString) + if (typeof jsonString === "object" || Array.isArray(jsonString)) { + return jsonString + } + const parsed = JSON.parse(jsonString) + const isNonScalar = typeof parsed === "object" || Array.isArray(parsed) + return isNonScalar ? parsed : jsonString } catch (err) { if (returnString) { return jsonString diff --git a/common/src/enums.ts b/common/src/enums.ts index ebf1eb3f..ecfe0475 100644 --- a/common/src/enums.ts +++ b/common/src/enums.ts @@ -189,4 +189,4 @@ export enum API_KEY_TYPE { export enum OperationType { QUERY = "query", MUTATION = "mutation", -} \ No newline at end of file +} diff --git a/common/src/types.ts b/common/src/types.ts index 1e1ee010..2a2883e6 100644 --- a/common/src/types.ts +++ b/common/src/types.ts @@ -510,10 +510,18 @@ export interface AuthenticationConfig { cookieName: string } +interface DisabledPathSection { + reqQuery: string[] + reqHeaders: string[] + reqBody: string[] + resHeaders: string[] + resBody: string[] +} + export interface BlockFieldEntry { path: string pathRegex: string method: DisableRestMethod, numberParams: number - disabledPaths: string[] + disabledPaths: DisabledPathSection }