Merge branch 'main' into feature/python-3-14

Author: gchqdev227

.github/workflows/documentation.yaml

@@ -1,15 +1,12 @@
 name: Build Documentation
-
 on:
   pull_request:
     branches: [main]
-
 jobs:
   build:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
-
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python 3.12

.github/workflows/pre-commit.yaml

@@ -1,15 +1,12 @@
 name: Run Pre-commit
-
 on:
   pull_request:
     branches: [main]
-
 jobs:
   build:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
-
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python 3.12

.github/workflows/unit-tests.yaml

@@ -1,17 +1,14 @@
 name: Unit Tests
-
 on:
   pull_request:
     branches: [main]
-
 jobs:
   build:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14.0-beta.1"]
-
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}

.pre-commit-config.yaml

@@ -7,8 +7,17 @@ repos:
       - id: check-yaml
       - id: check-added-large-files
         args: [--enforce-all, --maxkb=100]
+  - repo: https://github.com/google/yamlfmt
+    rev: v0.16.0
+    hooks:
+      - id: yamlfmt
+  - repo: https://github.com/hhatto/autopep8
+    rev: v2.3.2
+    hooks:
+      - id: autopep8
+        exclude: docs/source/conf.py
   - repo: https://github.com/pycqa/isort
-    rev: 5.13.2
+    rev: 6.0.1
     hooks:
       - id: isort
   - repo: https://github.com/jumanjihouse/pre-commit-hooks
@@ -24,12 +33,12 @@ repos:
       - id: forbid-crlf
       - id: forbid-tabs
   - repo: https://github.com/streetsidesoftware/cspell-cli
-    rev: v8.17.0
+    rev: v9.0.1
     hooks:
       - id: cspell
         exclude: \.gitignore|.*\.properties
   - repo: https://github.com/pycqa/flake8
-    rev: 7.1.1
+    rev: 7.2.0
     hooks:
       - id: flake8
         args: [--max-line-length=150]
@@ -39,14 +48,14 @@ repos:
     hooks:
       - id: pydocstyle
         additional_dependencies: ["tomli"]
-        exclude: examples|tests|conf.py
+        exclude: examples|tests|.pre-commit-hooks|conf.py
   - repo: https://github.com/regebro/pyroma
-    rev: f4ef6995f6c054586fe6d5d33d180abe71ebabd0
+    rev: "4.2"
     hooks:
-    - id: pyroma
-      args: ["-d", "--min=10", ".", "--skip-tests", "ValidREST"]
+      - id: pyroma
+        args: ["-d", "--min=10", "."]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.14.1
+    rev: v1.15.0
     hooks:
       - id: mypy
         additional_dependencies: ["types-docutils", "types-python-dateutil", "types-requests"]
@@ -59,3 +68,11 @@ repos:
           - --template
           - (C) Crown Copyright [OWNER]
           - --owner=GCHQ
+  - repo: local
+    hooks:
+      - id: double-string-fixer
+        name: string fixer
+        description: Replace single-quoted strings with double-quoted strings
+        types: [python]
+        language: python
+        entry: python3 .pre-commit-hooks/double_string_fixer.py

.pre-commit-hooks/double_string_fixer.py

@@ -0,0 +1,117 @@
+"""
+A copy of https://github.com/pre-commit/pre-commit-hooks/tree/main#double-quote-string-fixer,
+except it replaces single-quoted strings with double-quoted strings.
+
+Copyright (c) 2014 pre-commit dev team: Anthony Sottile, Ken Struys
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+"""
+from __future__ import annotations
+
+import argparse
+import io
+import re
+import sys
+import tokenize
+from typing import Sequence
+
+if sys.version_info >= (3, 12):  # pragma: >=3.12 cover
+    FSTRING_START = tokenize.FSTRING_START
+    FSTRING_END = tokenize.FSTRING_END
+else:  # pragma: <3.12 cover
+    FSTRING_START = FSTRING_END = -1
+
+START_QUOTE_RE = re.compile("^[a-zA-Z]*'")
+
+
+def handle_match(token_text: str) -> str:
+    if '"""' in token_text or "'''" in token_text:
+        return token_text
+
+    match = START_QUOTE_RE.match(token_text)
+    if match is not None:
+        meat = token_text[match.end():-1]
+        if '"' in meat or "'" in meat:
+            return token_text
+        else:
+            return match.group().replace("'", '"') + meat + '"'
+    else:
+        return token_text
+
+
+def get_line_offsets_by_line_no(src: str) -> list[int]:
+    # Padded so we can index with line number
+    offsets = [-1, 0]
+    for line in src.splitlines(True):
+        offsets.append(offsets[-1] + len(line))
+    return offsets
+
+
+def fix_strings(filename: str) -> int:
+    with open(filename, encoding="UTF-8", newline="") as f:
+        contents = f.read()
+    line_offsets = get_line_offsets_by_line_no(contents)
+
+    # Basically a mutable string
+    splitcontents = list(contents)
+
+    fstring_depth = 0
+
+    # Iterate in reverse so the offsets are always correct
+    tokens_l = list(tokenize.generate_tokens(io.StringIO(contents).readline))
+    tokens = reversed(tokens_l)
+    for token_type, token_text, (srow, scol), (erow, ecol), _ in tokens:
+        if token_type == FSTRING_START:  # pragma: >=3.12 cover
+            fstring_depth += 1
+        elif token_type == FSTRING_END:  # pragma: >=3.12 cover
+            fstring_depth -= 1
+        elif fstring_depth == 0 and token_type == tokenize.STRING:
+            new_text = handle_match(token_text)
+            splitcontents[
+                line_offsets[srow] + scol:
+                line_offsets[erow] + ecol
+            ] = new_text
+
+    new_contents = "".join(splitcontents)
+    if contents != new_contents:
+        with open(filename, "w", encoding="UTF-8", newline="") as f:
+            f.write(new_contents)
+        return 1
+    else:
+        return 0
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("filenames", nargs="*", help="Filenames to fix")
+    args = parser.parse_args(argv)
+
+    retv = 0
+
+    for filename in args.filenames:
+        return_value = fix_strings(filename)
+        if return_value != 0:
+            print(f"Fixing strings in {filename}")
+        retv |= return_value
+
+    return retv
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.readthedocs.yaml

@@ -1,19 +1,14 @@
-
 # Read the Docs configuration file for Sphinx projects
 # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
-
 version: 2
-
 build:
   os: ubuntu-22.04
   tools:
     python: "3.12"
-
 sphinx:
   configuration: docs/source/conf.py
   builder: "html"
   fail_on_warning: true
-
 python:
-   install:
+  install:
     - requirements: docs/requirements.txt

CONTRIBUTING.md

@@ -31,6 +31,8 @@ To run the checks at any time:
 $ pre-commit run --all-files
 ```
 
+Please do not add any new pre-commit checks yourself and instead raise a ticket.
+
 ### Tests
 
 Before opening a pull request, please ensure that the tests pass. To do this, run the following:
@@ -80,11 +82,20 @@ $ python3 -m sphinx -b linkcheck docs/source docs/build  # check that all links
 
 ## Coding Standards and Conventions
 
-Concourse Tools is a fully-typed library, so please ensure all functions, methods and classes are fully typed. Although
-we tend to make use of future annotations (`from __future__ import annotations`) please continue using the `typing`
-module for all types to ensure compatibility with our documentation.
+Concourse Tools is a fully-typed library, so please ensure all functions, methods and classes are fully typed.
 
 Concourse Tools uses [Sphinx-style docstrings](https://sphinx-rtd-tutorial.readthedocs.io/en/latest/docstrings.html).
 
 This project aims to depend only on the standard library, so contributions which add additional dependencies outside of
 the standard library are likely to be rejected unless absolutely necessary.
+
+
+### Style Guide
+
+Concourse Tools does not have an explicit style guide outside of the pre-commit checks for enforcing PEP8 and double-quote strings.
+However, please ensure your code is as readable and clear as possible. Reviewers will highlight any code changes they feel
+is inconsistent or difficult to parse.
+
+Please refrain from running Black over the code, as it can cause readability issues for nested statements.
+
+Finally, please do not refactor "other people's" code when submitting a pull request.

concoursetools/cli/docstring.py

@@ -76,6 +76,6 @@ def _pair_up(data: list[str]) -> Generator[tuple[str, str], None, None]:
     """
     for i in range(0, len(data), 2):
         try:
-            yield data[i], data[i+1]
+            yield data[i], data[i + 1]
         except IndexError:
             raise ValueError(f"Needed an even number of values, got {len(data)}")

concoursetools/mocking.py

@@ -202,7 +202,7 @@ def _get_folder_as_dict(self, folder_path: Path, max_depth: int = 2, encoding: s
                         first_chunk = rf.read(byte_limit)
                     folder_dict[item.name] = first_chunk
             elif item.is_dir():
-                folder_dict[item.name] = self._get_folder_as_dict(item, max_depth=max_depth-1, encoding=encoding)
+                folder_dict[item.name] = self._get_folder_as_dict(item, max_depth=max_depth - 1, encoding=encoding)
         return folder_dict
 
     def _set_folder_from_dict(self, folder_path: Path, folder_dict: FolderDict, encoding: str | None = None) -> None:

cspell.config.yaml

@@ -1,10 +1,11 @@
 version: "0.2"
 language: en-GB
-useGitignore: true,
+useGitignore: true
 ignorePaths:
   - tests
   - requirements*.txt
   - .pre-commit-config.yaml
+  - .pre-commit-hooks/*
 dictionaries:
   - python
 words: