feat: define summary tables

Author: fstagni

diracx-db/src/diracx/db/sql/job/schema.py

@@ -8,6 +8,7 @@
     Integer,
     String,
     Text,
+    UniqueConstraint,
 )
 from sqlalchemy.orm import declarative_base
 
@@ -156,3 +157,129 @@ class JobCommands(JobDBBase):
     status = Column("Status", String(32), default="Received")
     reception_time = Column("ReceptionTime", DateTime, primary_key=True)
     execution_time = NullColumn("ExecutionTime", DateTime)
+
+
+class JobsHistorySummary(JobDBBase):
+    __tablename__ = "JobsHistorySummary"
+    id = Column("ID", Integer, primary_key=True, autoincrement=True)
+    status = Column("Status", String(32))
+    site = Column("Site", String(100))
+    owner = Column("Owner", String(64))
+    owner_group = Column("OwnerGroup", String(128))
+    vo = Column("VO", String(64))
+    job_group = Column("JobGroup", String(32))
+    job_type = Column("JobType", String(32))
+    application_status = Column("ApplicationStatus", String(255))
+    minor_status = Column("MinorStatus", String(128))
+    job_count = Column("JobCount", Integer, default=0)
+    reschedule_sum = Column("RescheduleSum", Integer, default=0)
+
+    __table_args__ = (
+        UniqueConstraint(
+            "Status",
+            "Site",
+            "Owner",
+            "OwnerGroup",  # TODO: OwnerGroup(32)
+            "VO",
+            "JobGroup",
+            "JobType",
+            "ApplicationStatus",  # TODO: ApplicationStatus(128)
+            "MinorStatus",
+            name="uq_summary",
+        ),
+    )
+
+
+# TODO:
+
+# DELIMITER //
+
+# CREATE TRIGGER trg_Jobs_insert
+# AFTER INSERT ON Jobs
+# FOR EACH ROW
+# BEGIN
+#   INSERT INTO JobsHistorySummary (
+#       Status, Site, Owner, OwnerGroup, VO, JobGroup, JobType,
+#       ApplicationStatus, MinorStatus, JobCount, RescheduleSum
+#   )
+#   VALUES (
+#       NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.VO,
+#       NEW.JobGroup, NEW.JobType, NEW.ApplicationStatus,
+#       NEW.MinorStatus, 1, NEW.RescheduleCounter
+#   )
+#   ON DUPLICATE KEY UPDATE JobCount = JobCount + 1,
+#       RescheduleSum = RescheduleSum + NEW.RescheduleCounter;
+# END;
+# //
+
+# CREATE TRIGGER trg_Jobs_delete
+# AFTER DELETE ON Jobs
+# FOR EACH ROW
+# BEGIN
+#   UPDATE JobsHistorySummary
+#   SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter
+#   WHERE Status = OLD.Status
+#     AND Site = OLD.Site
+#     AND Owner = OLD.Owner
+#     AND OwnerGroup = OLD.OwnerGroup
+#     AND VO = OLD.VO
+#     AND JobGroup = OLD.JobGroup
+#     AND JobType = OLD.JobType
+#     AND ApplicationStatus = OLD.ApplicationStatus
+#     AND MinorStatus = OLD.MinorStatus;
+
+#   -- Remove zero rows
+#   DELETE FROM JobsHistorySummary
+#   WHERE JobCount = 0
+#     AND Status = OLD.Status
+#     AND Site = OLD.Site
+#     AND Owner = OLD.Owner
+#     AND OwnerGroup = OLD.OwnerGroup
+#     AND VO = OLD.VO
+#     AND JobGroup = OLD.JobGroup
+#     AND JobType = OLD.JobType
+#     AND ApplicationStatus = OLD.ApplicationStatus
+#     AND MinorStatus = OLD.MinorStatus;
+# END;
+# //
+
+# CREATE TRIGGER trg_Jobs_update_status
+# AFTER UPDATE ON Jobs
+# FOR EACH ROW
+# BEGIN
+#   IF OLD.Status != NEW.Status THEN
+
+#     -- Decrease count from old status
+#     UPDATE JobsHistorySummary
+#     SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter
+#     WHERE Status = OLD.Status
+#       AND Site = OLD.Site
+#       AND Owner = OLD.Owner
+#       AND OwnerGroup = OLD.OwnerGroup
+#       AND VO = OLD.VO
+#       AND JobGroup = OLD.JobGroup
+#       AND JobType = OLD.JobType
+#       AND ApplicationStatus = OLD.ApplicationStatus
+#       AND MinorStatus = OLD.MinorStatus;
+
+#     -- Delete row if count drops to zero
+#     DELETE FROM JobsHistorySummary WHERE JobCount = 0;
+
+#     -- Increase count for new status
+#     INSERT INTO JobsHistorySummary (
+#           Status, Site, Owner, OwnerGroup, JobGroup, VO,
+#           JobType, ApplicationStatus, MinorStatus, JobCount, RescheduleSum
+#     )
+#     VALUES (
+#           NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.JobGroup,
+#           NEW.VO, NEW.JobType, NEW.ApplicationStatus, NEW.MinorStatus,
+#           1, NEW.RescheduleCounter
+#     )
+#     ON DUPLICATE KEY UPDATE JobCount = JobCount + 1,
+#           RescheduleSum = RescheduleSum + NEW.RescheduleCounter;
+
+#   END IF;
+# END;
+# //
+
+# DELIMITER ;

diracx-db/src/diracx/db/sql/pilot_agents/schema.py

@@ -58,3 +58,76 @@ class PilotOutput(PilotAgentsDBBase):
     pilot_id = Column("PilotID", Integer, primary_key=True)
     std_output = Column("StdOutput", Text)
     std_error = Column("StdError", Text)
+
+
+class PilotsHistorySummary(PilotAgentsDBBase):
+    __tablename__ = "PilotsHistorySummary"
+    grid_site = Column("GridSite", String(128), primary_key=True)
+    destination_site = Column("DestinationSite", String(128), primary_key=True)
+    status = Column("Status", String(32), primary_key=True)
+    vo = Column("VO", String(64), primary_key=True)
+    pilot_count = Column("PilotCount", Integer, default=0)
+
+
+# TODO:
+
+# DELIMITER //
+
+# CREATE TRIGGER trg_PilotAgents_insert
+# AFTER INSERT ON PilotAgents
+# FOR EACH ROW
+# BEGIN
+#   INSERT INTO PilotsHistorySummary (GridSite, DestinationSite, Status, VO, PilotCount)
+#   VALUES (NEW.GridSite, NEW.DestinationSite, NEW.Status, NEW.VO, 1)
+#   ON DUPLICATE KEY UPDATE PilotCount = PilotCount + 1;
+# END;
+# //
+
+# CREATE TRIGGER trg_PilotAgents_delete
+# AFTER DELETE ON PilotAgents
+# FOR EACH ROW
+# BEGIN
+#   UPDATE PilotsHistorySummary
+#   SET PilotCount = PilotCount - 1
+#   WHERE GridSite = OLD.GridSite
+#     AND DestinationSite = OLD.DestinationSite
+#     AND Status = OLD.Status
+#     AND VO = OLD.VO;
+
+#   -- Remove zero rows
+#   DELETE FROM PilotsHistorySummary
+#   WHERE PilotCount = 0
+#     AND GridSite = OLD.GridSite
+#     AND DestinationSite = OLD.DestinationSite
+#     AND Status = OLD.Status
+#     AND VO = OLD.VO;
+# END;
+# //
+
+# CREATE TRIGGER trg_PilotAgents_update_status
+# AFTER UPDATE ON PilotAgents
+# FOR EACH ROW
+# BEGIN
+#   IF OLD.Status != NEW.Status THEN
+
+#     -- Decrease count from old status
+#     UPDATE PilotsHistorySummary
+#     SET PilotCount = PilotCount - 1
+#     WHERE GridSite = OLD.GridSite
+#       AND DestinationSite = OLD.DestinationSite
+#       AND Status = OLD.Status
+#       AND VO = OLD.VO;
+
+#     -- Delete row if count drops to zero
+#     DELETE FROM PilotsHistorySummary WHERE PilotCount = 0;
+
+#     -- Increase count for new status
+#     INSERT INTO PilotsHistorySummary (GridSite, DestinationSite, Status, VO, PilotCount)
+#     VALUES (NEW.GridSite, NEW.DestinationSite, NEW.Status, NEW.VO, 1)
+#     ON DUPLICATE KEY UPDATE PilotCount = PilotCount + 1;
+
+#   END IF;
+# END;
+# //
+
+# DELIMITER ;