feat: Add pilot pilot logging (legacy and DiracX)

.github/workflows/basic.yml

@@ -3,17 +3,16 @@ name: Basic tests
 on: [push, pull_request]
 
 jobs:
-
   shellcheck:
     runs-on: ubuntu-24.04
     if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC'
     timeout-minutes: 10
 
     steps:
-    - uses: actions/checkout@v3
-    - name: Run shellcheck
-      run: |
-        find tests/CI -name '*.sh' -print0 | xargs -0 -n1 shellcheck --external-sources;
+      - uses: actions/checkout@v3
+      - name: Run shellcheck
+        run: |
+          find tests/CI -name '*.sh' -print0 | xargs -0 -n1 shellcheck --external-sources;
 
   pycodestyle:
     runs-on: ubuntu-24.04
@@ -23,28 +22,26 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: 
-          - 2.7.18
+        python:
           - 3.6.15
           - 3.9.17
 
     container: python:${{ matrix.python }}-slim
     steps:
-    - uses: actions/checkout@v3
-    - name: Installing dependencies
-      run: |
-        python -m pip install pycodestyle
-    - name: Run pycodestyle
-      run: |
-        if [[ "${REFERENCE_BRANCH}" != "" ]]; then
-            git remote add upstream https://github.com/DIRACGrid/Pilot.git
-            git fetch --no-tags upstream "${REFERENCE_BRANCH}"
-            git branch -vv
-            git diff -U0 "upstream/${REFERENCE_BRANCH}" | pycodestyle --diff
-        fi
-      env:
-        REFERENCE_BRANCH: ${{ github['base_ref'] || github['head_ref'] }}
-
+      - uses: actions/checkout@v3
+      - name: Installing dependencies
+        run: |
+          python -m pip install pycodestyle
+      - name: Run pycodestyle
+        run: |
+          if [[ "${REFERENCE_BRANCH}" != "" ]]; then
+              git remote add upstream https://github.com/DIRACGrid/Pilot.git
+              git fetch --no-tags upstream "${REFERENCE_BRANCH}"
+              git branch -vv
+              git diff -U0 "upstream/${REFERENCE_BRANCH}" | pycodestyle --diff
+          fi
+        env:
+          REFERENCE_BRANCH: ${{ github['base_ref'] || github['head_ref'] }}
 
   pytest:
     runs-on: ubuntu-24.04
@@ -54,24 +51,22 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: 
-          - 2.7.18
+        python:
           - 3.6.15
           - 3.9.17
 
     container: python:${{ matrix.python }}-slim
     steps:
-    - uses: actions/checkout@v3
-    - name: Installing dependencies
-      run: |
-        echo 'deb http://archive.debian.org/debian stretch main' > /etc/apt/sources.list
-        echo 'deb http://archive.debian.org/debian-security stretch/updates main' >> /etc/apt/sources.list
-        apt-get update || true
-        python -m pip install pytest mock
-        apt install -y voms-clients
-    - name: Run pytest
-      run: pytest
-
+      - uses: actions/checkout@v3
+      - name: Installing dependencies
+        run: |
+          echo 'deb http://archive.debian.org/debian stretch main' > /etc/apt/sources.list
+          echo 'deb http://archive.debian.org/debian-security stretch/updates main' >> /etc/apt/sources.list
+          apt-get update || true
+          python -m pip install pytest mock
+          apt install -y voms-clients
+      - name: Run pytest
+        run: pytest
 
   pylint:
     runs-on: ubuntu-24.04
@@ -81,16 +76,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: 
-          - 2.7.18
+        python:
           - 3.6.15
           - 3.9.17
 
     container: python:${{ matrix.python }}-slim
     steps:
-    - uses: actions/checkout@v3
-    - name: Installing dependencies
-      run: |
-        python -m pip install pylint
-    - name: Run pylint
-      run: pylint -E Pilot/
+      - uses: actions/checkout@v3
+      - name: Installing dependencies
+        run: |
+          python -m pip install pylint
+      - name: Run pylint
+        run: pylint -E Pilot/

.pylintrc

@@ -18,7 +18,3 @@ dummy-variables=_
 disable=
     invalid-name,
     line-too-long, # would be nice to remove this one
-    consider-using-f-string, # python2/3 support
-    unspecified-encoding, # python2/3 support
-    super-with-arguments, # python2/3 support
-    redefined-builtin, # python2/3 support

Pilot/dirac-pilot.py

@@ -19,36 +19,20 @@
 But, as said, all the actions are actually configurable.
 """
 
-from __future__ import absolute_import, division, print_function
-
 import os
 import sys
 import time
+from io import StringIO
+
+from pilotTools import (
+    Logger,
+    PilotParams,
+    RemoteLogger,
+    getCommand,
+    pythonPathCheck,
+)
+from proxyTools import revokePilotToken
 
-############################
-# python 2 -> 3 "hacks"
-
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from io import StringIO
-
-try:
-    from Pilot.pilotTools import (
-        Logger,
-        PilotParams,
-        RemoteLogger,
-        getCommand,
-        pythonPathCheck,
-    )
-except ImportError:
-    from pilotTools import (
-        Logger,
-        PilotParams,
-        RemoteLogger,
-        getCommand,
-        pythonPathCheck,
-    )
 ############################
 
 if __name__ == "__main__":
@@ -63,25 +47,34 @@
     # print the buffer, so we have a "classic' logger back in sync.
     sys.stdout.write(bufContent)
     # now the remote logger.
-    remote = pilotParams.pilotLogging and (pilotParams.loggerURL is not None)
-    if remote:
+    remote = pilotParams.pilotLogging and pilotParams.diracXServer
+    if remote and pilotParams.jwt != {}:
         # In a remote logger enabled Dirac version we would have some classic logger content from a wrapper,
         # which we passed in:
         receivedContent = ""
         if not sys.stdin.isatty():
             receivedContent = sys.stdin.read()
+
         log = RemoteLogger(
-            pilotParams.loggerURL,
+            pilotParams.diracXServer,
             "Pilot",
             bufsize=pilotParams.loggerBufsize,
             pilotUUID=pilotParams.pilotUUID,
             debugFlag=pilotParams.debugFlag,
-            wnVO=pilotParams.wnVO,
+            jwt=pilotParams.jwt,
+            legacy_logging=pilotParams.isLegacyPilot,
+            clientID=pilotParams.clientID
         )
         log.info("Remote logger activated")
-        log.buffer.write(receivedContent)
+        log.buffer.write(log.format_to_json(
+            "INFO",
+            receivedContent,
+        ))
         log.buffer.flush()
-        log.buffer.write(bufContent)
+        log.buffer.write(log.format_to_json(
+            "INFO",
+            bufContent,
+        ))
     else:
         log = Logger("Pilot", debugFlag=pilotParams.debugFlag)
 
@@ -100,11 +93,13 @@
     log.debug("PARAMETER [%s]" % ", ".join(map(str, pilotParams.optList)))
 
     if pilotParams.commandExtensions:
-        log.info("Requested command extensions: %s" % str(pilotParams.commandExtensions))
+        log.info(
+            "Requested command extensions: %s" % str(pilotParams.commandExtensions)
+        )
 
     log.info("Executing commands: %s" % str(pilotParams.commands))
 
-    if remote:
+    if remote and pilotParams.jwt:
         # It's safer to cancel the timer here. Each command has got its own logger object with a timer cancelled by the
         # finaliser. No need for a timer in the "else" code segment below.
         try:
@@ -122,5 +117,20 @@
             log.error("Command %s could not be instantiated" % commandName)
             # send the last message and abandon ship.
             if remote:
-                log.buffer.flush()
+                log.buffer.flush(force=True)
             sys.exit(-1)
+
+    log.info("Pilot tasks finished.")
+
+    if pilotParams.jwt:
+        if remote:
+            log.buffer.flush(force=True)
+
+        if not pilotParams.isLegacyPilot:
+            log.info("Revoking pilot token.")
+            revokePilotToken(
+                pilotParams.diracXServer,
+                pilotParams.pilotUUID,
+                pilotParams.jwt,
+                pilotParams.clientID,
+            )