feat: Migrating pilot functionalities from DIRAC, and add DiracX pilot auth

Squashed commits:
[33cfe17] test: Changing from sleep to freezegun #537
[3e5d677] refactor: Refactorized the exceptions
[d84f6fe] doc: Documenting the database
[1c0db02] test: Better testing: Tested search engine
[cd1caf4] refactor: Refactoring from the auth router to the pilot router
[9cc99f1] feat: Adding indexes to the database
[de0b580] refactor: Moved the pilot_to_secret_mapping into the pilotagents table itself
[14396f6] refactoring: Lots of refactoring of the code
[56daac5] refactor: Moving db parts to the logic, and some fixes
[79a11cf] fix: Fixed dependency and client
[8c45582] test: Tested interval deletion for the db.
[dd07bca] feat: We can clear pilots that lived more than n days
[7e93153] feat: We can search for pilots
[9da8033] feat: DIRAC can associate a pilot with a job
[77a59b0] feat: We can modify pilot fields (statuses, benchmarks, ...)
[ca0e727] fix: Added pilot management
[521164f] fix: Fixed text
[8e6e881] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
[adadb68] fix: Semantic
[15d9fe6] feat: We can add a number of use max for the secret in pilot registration
[4549203] fix: Fixed cli
[a9ece0a] feat: Adding a command to add a pilot to the CS, and some fixes
[42e4f0b] fix: Changed secret expiration from 600 to 3600s
[1dcb4b8] fix: Regenerated client
[02a5e31] feat: A user can create secrets, and associate them to a pilot
[8072227] feat: Ability to add only secrets, and lots of refactoring in the tests
[10205d6] feat: Autodeletion of secrets after full use or expiration.
[3f013df] feat: Restricting a secret for a certain VO
[c2181aa] fix: Chunky client
[d374ec6] feat: Changing the pilot login db, and moved from references to stamps
[aa67bda] fix: Regenerating client
[135235f] feat: Using pilot user, model for pilot creds, and documentation
[0e0a117] feat: Added last_time_used for a pilot secret, and a max value
[62977ee] fix: Few fixes with refactoring, and tests (+4 squashed commits)
Squashed commits:
[5169ee2] refactor: Moved login for pilots into token file
[12da5c0] fix: Fixed according the the new commits from main branch
[a8e06d8] fix: Fixed client
[93b83b4] fix: Fixed the tests and autorest
[a78c028] feat: A user can register pilots and get their credentials
[1e9f907] fix: Removing the pilot login cli
[3f7c632] feat: Adding duration to secrets
[8aff5d9] fix: Regenerating client, type ignore for the cli
Squashed commits:
[06a969b] fix: Type ignore for mypy...
[28e1ee1] fix: Generated code broke everything
[339ab4f] fix: Fixed cli and pilot endpoints
[31af99e] fix: Regenerated client
[a171d01] feat: Added pilot login to the cli
[54951a6] fix: Correcting client generation with the error patch
Squashed commits:
[65db1ae] fix: Regenerating clients with autorest
[5a4f370] feat: Better handling of refresh tokens for pilots
[6da7bb4] fix: Generating client with autorest
[2c4d2fb] feat: Adding pilot registrations

Author: Robin-Van-de-Merghel

diracx-cli/src/diracx/cli/internal/config.py

@@ -59,7 +59,7 @@ def generate_cs(config_repo: str):
     update_config_and_commit(
         repo_path=repo_path, config=config, message="Initial commit"
     )
-    typer.echo(f"Successfully created repo in {config_repo}", err=True)
+    typer.echo(f"Successfully created repo in {config_repo}")
 
 
 @app.command()
@@ -93,13 +93,14 @@ def add_vo(
         raise typer.Exit(1)
 
     config.Registry[vo] = new_registry
+    config.Operations[vo] = OperationsConfig(Pilot={})
 
     update_config_and_commit(
         repo_path=repo_path,
         config=config,
         message=f"Added vo {vo} registry (default group {default_group} and idp {idp_url})",
     )
-    typer.echo(f"Successfully added vo to {config_repo}", err=True)
+    typer.echo(f"Successfully added vo to {config_repo}")
 
 
 @app.command()
@@ -130,7 +131,7 @@ def add_group(
     update_config_and_commit(
         repo_path=repo_path, config=config, message=f"Added group {group} in {vo}"
     )
-    typer.echo(f"Successfully added group to {config_repo}", err=True)
+    typer.echo(f"Successfully added group to {config_repo}")
 
 
 @app.command()
@@ -177,7 +178,103 @@ def add_user(
         config=config,
         message=f"Added user {sub} ({preferred_username}) to vo {vo} and groups {groups}",
     )
-    typer.echo(f"Successfully added user to {config_repo}", err=True)
+    typer.echo(f"Successfully added user to {config_repo}")
+
+
+@app.command()
+def set_user_as_pilot_user(
+    config_repo: str,
+    *,
+    vo: Annotated[str, typer.Option()],
+    pilot_group: Annotated[str, typer.Option()],
+    sub: Annotated[str, typer.Option()],
+    pilot_preferred_username: Annotated[str, typer.Option()],
+):
+    """Set a user as a pilot.
+
+    *Warning*: This requires an existing VO, and also a group and a pilot dedicated to the pilot.
+    """
+    config_repo = TypeAdapter(ConfigSourceUrl).validate_python(config_repo)
+    if config_repo.scheme != "git+file" or config_repo.path is None:
+        raise NotImplementedError("Only git+file:// URLs are supported")
+
+    repo_path = Path(config_repo.path)
+
+    config = ConfigSource.create_from_url(backend_url=repo_path).read_config()
+
+    # The VO has to exist
+    if vo not in config.Registry:
+        typer.secho(
+            f"ERROR: Virtual Organization {vo} does not exist", err=True, fg="red"
+        )
+        raise typer.Exit(1)
+
+    # The sub has to exist
+    if sub not in config.Registry[vo].Users:
+        typer.secho(
+            f"ERROR: User {sub} does not exist. You have to create it first before setting it as a pilot user",
+            err=True,
+            fg="red",
+        )
+        raise typer.Exit(1)
+
+    # The preferred_username has to match the sub
+    if not config.Registry[vo].Users[sub].PreferedUsername == pilot_preferred_username:
+        typer.secho(
+            f"ERROR: User {sub} does not match pilot_preferred_username {pilot_preferred_username}",
+            err=True,
+            fg="red",
+        )
+        raise typer.Exit(1)
+
+    # The pilot group has to exist
+    groups = config.Registry[vo].Groups
+    if pilot_group not in groups:
+        typer.secho(
+            (
+                f"ERROR: Group {pilot_group} does not exist."
+                "You have to create it first before setting it as a pilot group"
+            ),
+            err=True,
+            fg="red",
+        )
+        raise typer.Exit(1)
+
+    # The sub has to be in the pilot group
+    group_users = groups[pilot_group].Users
+    if sub not in group_users:
+        typer.secho(
+            (
+                f"ERROR: User {sub} is not set into the pilot group."
+                "You have to add it before setting this group as a pilot group"
+            ),
+            err=True,
+            fg="red",
+        )
+        raise typer.Exit(1)
+
+    # Now we know that our pilot is in the pilot_group
+    # We need to define the Operations
+    # Normally, if we have a group, vo, ..., we do have Operations[vo]
+
+    # If the current pilot_group is different, just change it
+    # No error is raised. We could change the "official" pilot group
+    pilot = config.Operations[vo].Pilot
+
+    if pilot is None:
+        typer.secho("Pilot operation must not be None", err=True, fg="red")
+        raise typer.Exit(1)
+
+    pilot["GenericPilotGroup"] = pilot_group
+    pilot["GenericPilotUser"] = pilot_preferred_username
+
+    update_config_and_commit(
+        repo_path=repo_path,
+        config=config,
+        message=f"Setting user {sub} ({pilot_preferred_username}) as {vo}'s pilot user in group {pilot_group}",
+    )
+    typer.secho(f"Successfully set user as pilot to {config_repo}", fg="green")
+    typer.secho(f"New config :\n{config}", fg="green")
 
 
 def update_config_and_commit(repo_path: Path, config: Config, message: str):

diracx-cli/tests/test_internal.py

@@ -222,3 +222,64 @@ def test_add_user(cs_repo, vo, user_group):
     assert sub in config.Registry[vo].Users
     for group in user_group or [TEST_USER_GROUP]:
         assert config.Registry[vo].Groups[group].Users == {sub}
+
+
+def test_add_pilot_user(cs_repo):
+
+    sub = "lhcb:rvandeme"
+
+    pilot_group = "pilot_group"
+    pilot_user = "pilot_user"
+
+    config = ConfigSource.create_from_url(backend_url=cs_repo).read_config()
+
+    # Add a second group to it
+    result = runner.invoke(
+        app,
+        [
+            "internal",
+            "add-group",
+            cs_repo,
+            f"--vo={TEST_VO}",
+            f"--group={pilot_group}",
+            "--properties",
+            "GenericPilot",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+
+    # Add a user to it
+    result = runner.invoke(
+        app,
+        [
+            "internal",
+            "add-user",
+            cs_repo,
+            f"--vo={TEST_VO}",
+            f"--sub={sub}",
+            f"--preferred-username={pilot_user}",
+            f"--group={pilot_group}",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+
+    # Set this user as a pilot
+    result = runner.invoke(
+        app,
+        [
+            "internal",
+            "set-user-as-pilot-user",
+            cs_repo,
+            f"--vo={TEST_VO}",
+            f"--sub={sub}",
+            f"--pilot-preferred-username={pilot_user}",
+            f"--pilot-group={pilot_group}",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+
+    config = ConfigSource.create_from_url(backend_url=cs_repo).read_config()
+
+    # User pilot exists, same for pilot group
+    assert config.Operations[TEST_VO].Pilot["GenericPilotUser"] == pilot_user
+    assert config.Operations[TEST_VO].Pilot["GenericPilotGroup"] == pilot_group

diracx-client/src/diracx/client/_generated/_client.py

@@ -19,6 +19,7 @@
     AuthOperations,
     ConfigOperations,
     JobsOperations,
+    PilotsOperations,
     WellKnownOperations,
 )
 
@@ -34,6 +35,8 @@ class Dirac:  # pylint: disable=client-accepts-api-version-keyword
     :vartype config: _generated.operations.ConfigOperations
     :ivar jobs: JobsOperations operations
     :vartype jobs: _generated.operations.JobsOperations
+    :ivar pilots: PilotsOperations operations
+    :vartype pilots: _generated.operations.PilotsOperations
     :keyword endpoint: Service URL. Required. Default value is "".
     :paramtype endpoint: str
     """
@@ -86,6 +89,9 @@ def __init__(  # pylint: disable=missing-client-constructor-parameter-credential
         self.jobs = JobsOperations(
             self._client, self._config, self._serialize, self._deserialize
         )
+        self.pilots = PilotsOperations(
+            self._client, self._config, self._serialize, self._deserialize
+        )
 
     def send_request(
         self, request: HttpRequest, *, stream: bool = False, **kwargs: Any

diracx-client/src/diracx/client/_generated/aio/_client.py

@@ -19,6 +19,7 @@
     AuthOperations,
     ConfigOperations,
     JobsOperations,
+    PilotsOperations,
     WellKnownOperations,
 )
 
@@ -34,6 +35,8 @@ class Dirac:  # pylint: disable=client-accepts-api-version-keyword
     :vartype config: _generated.aio.operations.ConfigOperations
     :ivar jobs: JobsOperations operations
     :vartype jobs: _generated.aio.operations.JobsOperations
+    :ivar pilots: PilotsOperations operations
+    :vartype pilots: _generated.aio.operations.PilotsOperations
     :keyword endpoint: Service URL. Required. Default value is "".
     :paramtype endpoint: str
     """
@@ -86,6 +89,9 @@ def __init__(  # pylint: disable=missing-client-constructor-parameter-credential
         self.jobs = JobsOperations(
             self._client, self._config, self._serialize, self._deserialize
         )
+        self.pilots = PilotsOperations(
+            self._client, self._config, self._serialize, self._deserialize
+        )
 
     def send_request(
         self, request: HttpRequest, *, stream: bool = False, **kwargs: Any

diracx-client/src/diracx/client/_generated/aio/operations/__init__.py

@@ -14,6 +14,7 @@
 from ._operations import AuthOperations  # type: ignore
 from ._operations import ConfigOperations  # type: ignore
 from ._operations import JobsOperations  # type: ignore
+from ._operations import PilotsOperations  # type: ignore
 
 from ._patch import __all__ as _patch_all
 from ._patch import *
@@ -24,6 +25,7 @@
     "AuthOperations",
     "ConfigOperations",
     "JobsOperations",
+    "PilotsOperations",
 ]
 __all__.extend([p for p in _patch_all if p not in __all__])  # pyright: ignore
 _patch_sdk()