(cleanup) remove minute level agg

Author: akshay288

backend/src/data-source.ts

@@ -12,7 +12,6 @@ import {
   ApiKey,
   BlockFields,
   InstanceSettings,
-  AggregateTraceDataMinutely,
   AuthenticationConfig,
   AggregateTraceDataHourly,
   Attack,
@@ -34,7 +33,6 @@ export const AppDataSource: DataSource = new DataSource({
     ApiKey,
     BlockFields,
     InstanceSettings,
-    AggregateTraceDataMinutely,
     AuthenticationConfig,
     AggregateTraceDataHourly,
     Attack,

backend/src/models/aggregate-trace-data-minutely.ts

@@ -9,7 +9,6 @@ import { ApiEndpointTest } from "./api-endpoint-test"
 import { ApiKey } from "./keys"
 import { BlockFields } from "./block-fields"
 import { InstanceSettings } from "./instance-settings"
-import { AggregateTraceDataMinutely } from "./aggregate-trace-data-minutely"
 import { AuthenticationConfig } from "./authentication-config"
 import { AggregateTraceDataHourly } from "./aggregate-trace-data-hourly"
 import { Attack } from "./attack"
@@ -26,7 +25,6 @@ export type DatabaseModel =
   | ApiKey
   | BlockFields
   | InstanceSettings
-  | AggregateTraceDataMinutely
   | AuthenticationConfig
   | AggregateTraceDataHourly
   | Attack
@@ -43,7 +41,6 @@ export {
   ApiKey,
   BlockFields,
   InstanceSettings,
-  AggregateTraceDataMinutely,
   AuthenticationConfig,
   AggregateTraceDataHourly,
   Attack,

backend/src/models/index.ts

@@ -1,16 +1,13 @@
 import { DateTime } from "luxon"
 import { ApiTrace } from "models"
 import { AppDataSource } from "data-source"
-import {
-  aggregateTracesDataHourlyQuery,
-  aggregateTracesDataMinutelyQuery,
-} from "./queries"
+import { aggregateTracesDataHourlyQuery } from "./queries"
 
 const clearApiTraces = async (): Promise<void> => {
   const queryRunner = AppDataSource.createQueryRunner()
   await queryRunner.connect()
   try {
-    const now = DateTime.now().startOf("hour")
+    const now = DateTime.now()
     const oneHourAgo = now.minus({ hours: 1 }).toJSDate()
 
     const maxTimeRes = await queryRunner.manager
@@ -25,7 +22,6 @@ const clearApiTraces = async (): Promise<void> => {
 
     if (maxTime) {
       await queryRunner.startTransaction()
-      await queryRunner.query(aggregateTracesDataMinutelyQuery, [maxTime])
       await queryRunner.query(aggregateTracesDataHourlyQuery, [maxTime])
       await queryRunner.manager
         .createQueryBuilder()

backend/src/services/jobs/clearApiTraces.ts

@@ -1,137 +1,3 @@
-const tracesBySecondStatus = `
-  WITH traces_by_second_status AS (
-    SELECT
-      "apiEndpointUuid",
-      DATE_TRUNC('second', traces."createdAt") as second,
-      "responseStatus" as status,
-      COUNT(*) as "numTraces"
-    FROM api_trace traces
-    WHERE
-      "apiEndpointUuid" IS NOT NULL
-      AND analyzed = TRUE
-      AND "createdAt" <= $1
-    GROUP BY 1, 2, 3
-  )
-`
-const newTracesByMinuteStatus = `
-  SELECT
-    "apiEndpointUuid",
-    DATE_TRUNC('minute', second) as minute,
-    status,
-    SUM("numTraces") as "numTraces"
-  FROM traces_by_second_status
-  GROUP BY 1, 2, 3
-`
-const savedTracesByMinuteStatus = `
-  SELECT
-    "apiEndpointUuid",
-    minute,
-    CAST(d.key AS INTEGER) as "status",
-    SUM(CAST(d.value AS BIGINT)) as "numTraces"
-  FROM "aggregate_trace_data_minutely" tmp
-  JOIN jsonb_each_text(tmp."countByStatusCode") d ON TRUE
-  WHERE "minute" <= $1
-  GROUP BY 1, 2, 3
-`
-const tracesByMinuteStatus = `
-  traces_by_minute_status_union AS (
-    ${newTracesByMinuteStatus}
-    UNION ALL
-    ${savedTracesByMinuteStatus}
-  ),
-  traces_by_minute_status AS (
-    SELECT
-      "apiEndpointUuid",
-      minute,
-      status,
-      SUM("numTraces") as "numTraces"
-    FROM traces_by_minute_status_union
-    GROUP BY 1, 2, 3
-  )
-`
-const newTracesByMinute = `
-  SELECT
-    "apiEndpointUuid",
-    DATE_TRUNC('minute', second) as minute,
-    MAX("numTraces") as "maxRPS",
-    MIN("numTraces") as "minRPS",
-    SUM("numTraces"::numeric)/60.0 as "meanRPS",
-    SUM("numTraces") as "numTraces",
-    COUNT(*) as num_secs_with_data
-  FROM traces_by_second_status
-  GROUP BY 1, 2
-`
-const savedTracesByMinute = `
-  SELECT
-    "apiEndpointUuid",
-    "minute",
-    "maxRPS",
-    "minRPS",
-    "meanRPS",
-    "numCalls" as "numTraces",
-    CASE WHEN "minRPS" > 0 THEN 60 ELSE 0 END as num_secs_with_data
-  FROM aggregate_trace_data_minutely
-  WHERE minute <= $1
-`
-const tracesByMinute = `
-  traces_by_minute_union AS (
-    ${newTracesByMinute}
-    UNION ALL
-    ${savedTracesByMinute}
-  ),
-  traces_by_minute AS (
-    SELECT
-      "apiEndpointUuid",
-      minute,
-      MAX("maxRPS") as "maxRPS",
-      MIN("minRPS") as "minRPS",
-      SUM("numTraces"::numeric)/60.0 as "meanRPS",
-      SUM("numTraces") as "numTraces",
-      MAX("num_secs_with_data") as num_secs_with_data
-    FROM traces_by_minute_union
-    GROUP BY 1, 2
-  )
-`
-const minuteCountByStatusCode = `
-  minute_count_by_status_code AS (
-    SELECT
-      "apiEndpointUuid",
-      minute,
-      replace(
-        array_to_string(array_agg(json_build_object(status, "numTraces")), ''),
-        '}{',
-        ', '
-      )::json AS "countByStatusCode"
-    FROM traces_by_minute_status
-    GROUP BY 1, 2
-  )
-`
-export const aggregateTracesDataMinutelyQuery = `
-  ${tracesBySecondStatus},
-  ${tracesByMinuteStatus},
-  ${tracesByMinute},
-  ${minuteCountByStatusCode}
-  INSERT INTO aggregate_trace_data_minutely ("apiEndpointUuid", "minute", "maxRPS", "minRPS", "meanRPS", "numCalls", "countByStatusCode")
-  SELECT
-    traces."apiEndpointUuid",
-    traces.minute,
-    traces."maxRPS",
-    CASE WHEN traces.num_secs_with_data < 60 THEN 0 ELSE traces."minRPS" END as "minRPS",
-    traces."meanRPS",
-    traces."numTraces" as "numCalls",
-    status_code_map."countByStatusCode"
-  FROM traces_by_minute traces
-  JOIN minute_count_by_status_code status_code_map ON
-    traces.minute = status_code_map.minute AND traces."apiEndpointUuid" = status_code_map."apiEndpointUuid"
-  ON CONFLICT ("apiEndpointUuid", "minute")
-  DO UPDATE SET
-    "maxRPS" = EXCLUDED."maxRPS",
-    "minRPS" = EXCLUDED."minRPS",
-    "meanRPS" = EXCLUDED."meanRPS",
-    "numCalls" = EXCLUDED."numCalls", 
-    "countByStatusCode" = EXCLUDED."countByStatusCode"
-`
-
 export const aggregateTracesDataHourlyQuery = `
   INSERT INTO aggregate_trace_data_hourly ("apiEndpointUuid", "hour", "numCalls")
   SELECT

backend/src/services/jobs/queries.ts

@@ -15,7 +15,6 @@ import {
   DataField,
   OpenApiSpec,
   Alert,
-  AggregateTraceDataMinutely,
   AggregateTraceDataHourly,
 } from "models"
 import { JSONValue, OpenApiSpec as OpenApiSpecResponse } from "@common/types"
@@ -39,15 +38,13 @@ import {
   getServersV3,
 } from "./utils"
 import { AlertService } from "services/alert"
-import { DatabaseService } from "services/database"
 import Error404NotFound from "errors/error-404-not-found"
 import { BlockFieldsService } from "services/block-fields"
 import Error500InternalServer from "errors/error-500-internal-server"
 import { RISK_SCORE_ORDER } from "~/constants"
 import {
   insertDataFieldQuery,
   insertAggregateHourlyQuery,
-  insertAggregateMinutelyQuery,
   deleteOpenAPISpecDiffAlerts,
 } from "./queries"
 
@@ -385,13 +382,6 @@ export class SpecService {
             .andWhere(`type IN(:...types)`, {
               types: [AlertType.NEW_ENDPOINT, AlertType.OPEN_API_SPEC_DIFF],
             })
-          const deleteAggregateMinutelyQb = queryRunner.manager
-            .createQueryBuilder()
-            .delete()
-            .from(AggregateTraceDataMinutely)
-            .where(`"apiEndpointUuid" IN(:...ids)`, {
-              ids: similarEndpointUuids,
-            })
           const deleteAggregateHourlyQb = queryRunner.manager
             .createQueryBuilder()
             .delete()
@@ -408,15 +398,10 @@ export class SpecService {
           await deleteDataFieldsQb.execute()
           await updateAlertsQb.execute()
           await deleteAlertsQb.execute()
-          await queryRunner.query(insertAggregateMinutelyQuery, [
-            similarEndpointUuids,
-            item.endpoint.uuid,
-          ])
           await queryRunner.query(insertAggregateHourlyQuery, [
             item.endpoint.uuid,
             similarEndpointUuids,
           ])
-          await deleteAggregateMinutelyQb.execute()
           await deleteAggregateHourlyQb.execute()
           await queryRunner.manager
             .createQueryBuilder()

backend/src/services/spec/index.ts

@@ -54,49 +54,3 @@ export const insertAggregateHourlyQuery = `
   WHERE "apiEndpointUuid" = ANY($2)
   GROUP BY hour
 `
-
-export const insertAggregateMinutelyQuery = `
-  WITH traces_by_minute AS (
-    SELECT
-        minute,
-        SUM("numCalls") as "numCalls",
-        SUM("maxRPS") as "maxRPS",
-        SUM("minRPS") as "minRPS",
-        SUM("meanRPS") as "meanRPS"
-    FROM "aggregate_trace_data_minutely"
-    WHERE "apiEndpointUuid" = ANY($1)
-    GROUP BY minute
-  ),
-  traces_by_minute_status AS (
-    SELECT
-        minute,
-        CAST(d.key AS INTEGER) as "status",
-        SUM(CAST(d.value AS BIGINT)) as "numTraces"
-    FROM "aggregate_trace_data_minutely" tmp
-    JOIN jsonb_each_text(tmp."countByStatusCode") d ON TRUE
-    WHERE tmp."apiEndpointUuid" = ANY($1)
-    GROUP BY 1, 2
-  ),
-  minute_count_by_status_code AS (
-    SELECT
-        minute,
-        replace(
-            array_to_string(array_agg(json_build_object(status, "numTraces")), ''),
-            '}{',
-            ', '
-        )::json AS "countByStatusCode"
-    FROM traces_by_minute_status
-    GROUP BY 1
-  )
-  INSERT INTO aggregate_trace_data_minutely ("minute", "numCalls", "maxRPS", "minRPS", "meanRPS", "countByStatusCode", "apiEndpointUuid")
-  SELECT
-    traces.minute,
-    traces."numCalls",
-    traces."maxRPS",
-    traces."minRPS",
-    traces."meanRPS",
-    status_code_map."countByStatusCode",
-    $2 as "apiEndpointUuid"
-  FROM traces_by_minute traces
-  JOIN minute_count_by_status_code status_code_map ON traces.minute = status_code_map.minute
-`