From 0162aac4eedc59a0ef51bee837aed2ce361a2c38 Mon Sep 17 00:00:00 2001 From: LonelyCat124 <3043914+LonelyCat124@users.noreply.github.com.> Date: Wed, 15 Oct 2025 15:33:41 +0100 Subject: [PATCH 01/25] First changes to use new Directives from fparser - dependent on upstream changes --- src/psyclone/psyir/frontend/fortran.py | 14 +++++-- src/psyclone/psyir/frontend/fparser2.py | 49 ++++++++++++++++++++----- 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/src/psyclone/psyir/frontend/fortran.py b/src/psyclone/psyir/frontend/fortran.py index 00e81e08e6..9c540b7048 100644 --- a/src/psyclone/psyir/frontend/fortran.py +++ b/src/psyclone/psyir/frontend/fortran.py @@ -94,6 +94,7 @@ def __init__(self, free_form: bool = True, ignore_comments: bool = True, " only have an effect if ignore_comments is also set to False." ) self._ignore_comments = ignore_comments + self._ignore_directives = ignore_directives self._processor = Fparser2Reader(ignore_directives, last_comments_as_codeblocks, resolve_modules) @@ -132,7 +133,8 @@ def psyir_from_source(self, source_code: str) -> Node: SYMBOL_TABLES.clear() string_reader = FortranStringReader( source_code, include_dirs=Config.get().include_paths, - ignore_comments=self._ignore_comments) + ignore_comments=self._ignore_comments, + process_directives=not self._ignore_directives) # Set reader to free format. string_reader.set_format(FortranFormat(self._free_form, False)) @@ -258,9 +260,12 @@ def psyir_from_file(self, file_path): # Using the FortranFileReader instead of manually open the file allows # fparser to keep the filename information in the tree - reader = FortranFileReader(file_path, - include_dirs=Config.get().include_paths, - ignore_comments=self._ignore_comments) + reader = FortranFileReader( + file_path, + include_dirs=Config.get().include_paths, + ignore_comments=self._ignore_comments, + process_directives=not self._ignore_directives + ) reader.set_format(FortranFormat(self._free_form, False)) try: parse_tree = self._parser(reader) @@ -272,6 +277,7 @@ def psyir_from_file(self, file_path): _, filename = os.path.split(file_path) psyir = self._processor.generate_psyir(parse_tree, filename) + print(psyir.view()) return psyir diff --git a/src/psyclone/psyir/frontend/fparser2.py b/src/psyclone/psyir/frontend/fparser2.py index 6421615386..416db98378 100644 --- a/src/psyclone/psyir/frontend/fparser2.py +++ b/src/psyclone/psyir/frontend/fparser2.py @@ -44,7 +44,7 @@ from dataclasses import dataclass, field import os import sys -from typing import Iterable, Optional +from typing import Iterable, Optional, Tuple from fparser.common.readfortran import FortranStringReader from fparser.two import C99Preprocessor, Fortran2003, utils @@ -1040,6 +1040,7 @@ def __init__(self, ignore_directives: bool = True, Fortran2003.Module: self._module_handler, Fortran2003.Main_Program: self._main_program_handler, Fortran2003.Program: self._program_handler, + Fortran2003.Directive: self._directive_handler, } # Used to attach inline comments to the PSyIR symbols and nodes self._last_psyir_parsed_and_span = None @@ -3370,7 +3371,7 @@ def _do_construct_handler(self, node, parent): raise NotImplementedError("Unsupported Loop") # Process loop body (ignore 'do' and 'end do' statements) - # Keep track of the comments before the 'do' statement + # Keep track of the comments and directives before the 'do' statement loop_body_nodes = [] preceding_comments = [] found_do_stmt = False @@ -3378,6 +3379,12 @@ def _do_construct_handler(self, node, parent): if isinstance(child, Fortran2003.Comment) and not found_do_stmt: self.process_comment(child, preceding_comments) continue + if isinstance(child, Fortran2003.Directive) and not found_do_stmt: + directive = self._directive_handler(child, None) + # Add the directive before the loop. + loop.parent.addchild(directive) + continue + # TODO if isinstance(child, Fortran2003.Nonlabel_Do_Stmt): found_do_stmt = True continue @@ -5394,9 +5401,9 @@ def _process_args(self, node, call, canonicalise=None): return call - def _get_lost_declaration_comments(self, decl_list, - attach_trailing_symbol: bool = True)\ - -> list[Fortran2003.Comment]: + def _get_lost_declaration_comments_and_directives( + self, decl_list, attach_trailing_symbol: bool = True + ) -> list[Union[Fortran2003.Comment, Fortran2003.Directive]]: '''Finds comments from the variable declaration that the default declaration handler doesn't keep. Any comments that appear after the final declaration but before the first PSyIR node created are @@ -5410,14 +5417,16 @@ def _get_lost_declaration_comments(self, decl_list, the last symbol to the tree or not. Defaults to True - :returns: a list of comments that have been missed. + :returns: a list of comments and directives that have been + missed. ''' lost_comments = [] if len(decl_list) != 0 and isinstance(decl_list[-1], Fortran2003.Implicit_Part): # fparser puts all comments after the end of the last declaration # in the tree of the last declaration. - for comment in walk(decl_list[-1], Fortran2003.Comment): + for comment in walk(decl_list[-1], (Fortran2003.Comment, + Fortran2003.Directive)): if len(comment.tostr()) == 0: continue if self._last_psyir_parsed_and_span is not None: @@ -5552,7 +5561,8 @@ def _subroutine_handler(self, node, parent): # declarations as part of the declarations, but in PSyclone # they need to be a preceding_comment unless it's an inline # comment on the last declaration. - lost_comments = self._get_lost_declaration_comments(decl_list) + lost_comments = \ + self._get_lost_declaration_comments_and_directives(decl_list) # Check whether the function-stmt has a prefix specifying the # return type (other prefixes are handled in @@ -5707,7 +5717,9 @@ def _main_program_handler(self, node, parent): # fparser puts comments at the end of the declarations # whereas as preceding comments they belong in the execution part # except if it's an inline comment on the last declaration. - lost_comments = self._get_lost_declaration_comments(decl_list, False) + lost_comments = \ + self._get_lost_declaration_comments_and_directives(decl_list, + False) try: prog_exec = _first_type_match(node.content, @@ -5851,6 +5863,25 @@ def process_comment(self, comment, preceding_comments): preceding_comments.append(comment) + def _directive_handler( + self, node: Fortran2003.Directive, parent: Node + ) -> CodeBlock: + ''' + Process a directive and add it to the tree. The current behaviour + places the directive into a CodeBlock. + + :param node: Directive to process. + :param parent: The parent to add the PSyIR node to. + + :returns: a CodeBlock containing the input Directive. + ''' + code_block = CodeBlock( + [node], + CodeBlock.Structure.STATEMENT, + parent=parent + ) + return code_block + # For Sphinx AutoAPI documentation generation __all__ = ["Fparser2Reader"] From 797a57c058410dae999de79a42798b658ce9ca5d Mon Sep 17 00:00:00 2001 From: LonelyCat124 <3043914+LonelyCat124@users.noreply.github.com.> Date: Wed, 15 Oct 2025 15:34:19 +0100 Subject: [PATCH 02/25] linting fixes --- src/psyclone/psyir/frontend/fparser2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/psyclone/psyir/frontend/fparser2.py b/src/psyclone/psyir/frontend/fparser2.py index 416db98378..4a08cdeb3d 100644 --- a/src/psyclone/psyir/frontend/fparser2.py +++ b/src/psyclone/psyir/frontend/fparser2.py @@ -44,7 +44,7 @@ from dataclasses import dataclass, field import os import sys -from typing import Iterable, Optional, Tuple +from typing import Iterable, Optional, Union from fparser.common.readfortran import FortranStringReader from fparser.two import C99Preprocessor, Fortran2003, utils @@ -5866,7 +5866,7 @@ def process_comment(self, comment, preceding_comments): def _directive_handler( self, node: Fortran2003.Directive, parent: Node ) -> CodeBlock: - ''' + ''' Process a directive and add it to the tree. The current behaviour places the directive into a CodeBlock. From 0b9a6272fd830db9488bf9ee77fbd31ec04ef9d3 Mon Sep 17 00:00:00 2001 From: LonelyCat124 <3043914+LonelyCat124@users.noreply.github.com.> Date: Thu, 16 Oct 2025 09:51:39 +0100 Subject: [PATCH 03/25] Tests --- .../psyir/frontend/fparser2_comment_test.py | 17 --- .../psyir/frontend/fparser2_directive_test.py | 108 ++++++++++++++++++ 2 files changed, 108 insertions(+), 17 deletions(-) create mode 100644 src/psyclone/tests/psyir/frontend/fparser2_directive_test.py diff --git a/src/psyclone/tests/psyir/frontend/fparser2_comment_test.py b/src/psyclone/tests/psyir/frontend/fparser2_comment_test.py index bd82cd51da..8d1d22ca32 100644 --- a/src/psyclone/tests/psyir/frontend/fparser2_comment_test.py +++ b/src/psyclone/tests/psyir/frontend/fparser2_comment_test.py @@ -604,20 +604,3 @@ def test_lost_program_comments(): "inline here") assignment = psyir.walk(Assignment)[0] assert assignment.preceding_comment == "Comment here" - - -def test_directive_at_end(): - """Test that the FortranReader stores a directive after all - other code in a subroutine.""" - - code = """subroutine x - integer :: i - i = i + 1 - !$omp barrier - end subroutine""" - reader = FortranReader(ignore_comments=False, ignore_directives=False) - psyir = reader.psyir_from_source(code) - routine = psyir.children[0] - # The directive is a codeblock - assert isinstance(routine.children[-1], CodeBlock) - assert routine.children[-1].debug_string() == "!$omp barrier\n" diff --git a/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py b/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py new file mode 100644 index 0000000000..9e0329dc8a --- /dev/null +++ b/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py @@ -0,0 +1,108 @@ +# ----------------------------------------------------------------------------- +# BSD 3-Clause License +# +# Copyright (c) 2021-2025, Science and Technology Facilities Council. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# ----------------------------------------------------------------------------- +# Author A. B. G. Chalk, STFC Daresbury Lab + +"""Performs pytest tests on the support for directives in the fparser2 +PSyIR front-end""" + +import pytest + +from fparser.two import Fortran2003 + +from psyclone.psyir.frontend.fortran import FortranReader +from psyclone.psyir.nodes import ( + Container, + Routine, + Assignment, + Loop, + IfBlock, + Call, + CodeBlock, +) +from psyclone.psyir.commentable_mixin import CommentableMixin +from psyclone.psyir.symbols import DataTypeSymbol, StructureType + +from psyclone.psyir.backend.fortran import FortranWriter + + +def test_directive_after_decls(): + """Test that the FortranReader correctly finds a directive immediately + after the declarations""" + + code = """subroutine x() + integer :: i + !$omp barrier + i = 3 + end subroutine""" + reader = FortranReader(ignore_comments=False, ignore_directives=False) + psyir = reader.psyir_from_source(code) + routine = psyir.children[0] + # The directive is a codeblock + assert isinstance(routine.children[0], CodeBlock) + assert routine.children[0].debug_string() == "!$omp barrier\n" + + +def test_directive_in_decls(): + """Test that the FortranReader can handle a directive inside the + declarations""" + code = """subroutine x() + !$omp firstprivate + integer, dimension(100) :: i !dir$ aligned + i = 1 + end subroutine x""" + reader = FortranReader(ignore_comments=False, ignore_directives=False) + psyir = reader.psyir_from_source(code) + routine = psyir.children[0] + out = routine.debug_string() + assert """!$omp firstprivate +integer, dimension(100) :: i !dir$ aligned""" in out + + +def test_directive_at_end(): + """Test that the FortranReader stores a directive after all + other code in a subroutine.""" + + code = """subroutine x + integer :: i + i = i + 1 + !$omp barrier + end subroutine""" + reader = FortranReader(ignore_comments=False, ignore_directives=False) + psyir = reader.psyir_from_source(code) + routine = psyir.children[0] + # The directive is a codeblock + assert isinstance(routine.children[-1], CodeBlock) + assert routine.children[-1].debug_string() == "!$omp barrier\n" + + From dc64caaffab3f0865d2382e4840317f45aff06b3 Mon Sep 17 00:00:00 2001 From: LonelyCat124 <3043914+LonelyCat124@users.noreply.github.com.> Date: Fri, 17 Oct 2025 10:35:34 +0100 Subject: [PATCH 04/25] More improvements --- src/psyclone/psyir/frontend/fparser2.py | 14 ++- .../psyir/frontend/fparser2_directive_test.py | 110 ++++++++++++++++++ 2 files changed, 121 insertions(+), 3 deletions(-) diff --git a/src/psyclone/psyir/frontend/fparser2.py b/src/psyclone/psyir/frontend/fparser2.py index 4a08cdeb3d..612ce2ec2a 100644 --- a/src/psyclone/psyir/frontend/fparser2.py +++ b/src/psyclone/psyir/frontend/fparser2.py @@ -2433,7 +2433,8 @@ def process_declarations(self, parent, nodes, arg_list, preceding_comments = [] for node in nodes: if isinstance(node, Fortran2003.Implicit_Part): - for comment in walk(node, Fortran2003.Comment): + for comment in walk(node, (Fortran2003.Comment, + Fortran2003.Directive)): self.process_comment(comment, preceding_comments) elif isinstance(node, Fortran2003.Derived_Type_Def): sym = self._process_derived_type_decln(parent, node, @@ -2461,7 +2462,8 @@ def process_declarations(self, parent, nodes, arg_list, for node in nodes: if isinstance(node, Fortran2003.Implicit_Part): - for comment in walk(node, Fortran2003.Comment): + for comment in walk(node, (Fortran2003.Comment, + Fortran2003.Directive)): self.process_comment(comment, preceding_comments) continue # Anything other than a PARAMETER statement or an @@ -3019,7 +3021,9 @@ def _create_child(self, child, parent=None): # there), so we have to examine the first statement within it. We # must allow for the case where the block is empty though. if (child.content and child.content[0] and - (not isinstance(child.content[0], Fortran2003.Comment)) and + (not isinstance(child.content[0], + (Fortran2003.Comment, + Fortran2003.Directive))) and child.content[0].item and child.content[0].item.label): raise NotImplementedError("Unsupported labelled statement") elif isinstance(child, StmtBase): @@ -3437,6 +3441,10 @@ def _if_construct_handler(self, node, parent): for child in node.content[:clause_indices[0]]: if isinstance(child, Fortran2003.Comment): self.process_comment(child, preceding_comments) + if isinstance(child, Fortran2003.Directive): + direc = self._directive_handler(child, None) + parent.addchild(direc) + # NOTE: The comments are added to the IfBlock node. # NOTE: Comments before the 'else[if]' statements are not handled. diff --git a/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py b/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py index 9e0329dc8a..b828db99bf 100644 --- a/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py +++ b/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py @@ -85,6 +85,7 @@ def test_directive_in_decls(): psyir = reader.psyir_from_source(code) routine = psyir.children[0] out = routine.debug_string() + print(out) assert """!$omp firstprivate integer, dimension(100) :: i !dir$ aligned""" in out @@ -106,3 +107,112 @@ def test_directive_at_end(): assert routine.children[-1].debug_string() == "!$omp barrier\n" +def test_directive_before_loop(): + """Test that the FortranReader stores a directive before a loop as a + CodeBlock.""" + code = """subroutine x + integer :: i, j + i = 1 + !$omp target + do i = 1, 100 + j = i + end do + end subroutine x""" + reader = FortranReader(ignore_comments=False, ignore_directives=False) + psyir = reader.psyir_from_source(code) + routine = psyir.children[0] + # The directive is a codeblock + assert isinstance(routine.children[1], CodeBlock) + assert routine.children[1].debug_string() == "!$omp target\n" + + +def test_directive_before_if(): + """Test that the FortranReader stores a directive before an if as a + CodeBlock.""" + code = """subroutine x + integer :: i, j + i = 1 + !$omp target + if(i == 1 )then + j = i + end if + end subroutine x""" + reader = FortranReader(ignore_comments=False, ignore_directives=False) + psyir = reader.psyir_from_source(code) + routine = psyir.children[0] + # The directive is a codeblock + assert isinstance(routine.children[1], CodeBlock) + assert routine.children[1].debug_string() == "!$omp target\n" + + +def test_directive_before_else(): + """Test that the FortranReader stores a directive before an else as a + CodeBlock.""" + code = """subroutine x + integer :: i, j + i = 1 + if( i == 1 )then + j = i + !$omp barrier + else + j = 2 + end if + end subroutine x""" + reader = FortranReader(ignore_comments=False, ignore_directives=False) + psyir = reader.psyir_from_source(code) + routine = psyir.children[0] + ifblock = routine.children[1] + # The directive is a codeblock + assert isinstance(ifblock.if_body.children[1], CodeBlock) + assert ifblock.if_body.children[1].debug_string() == "!$omp barrier\n" + + +def test_directive_before_module(): + """Test that the FortranReader stores a directive before a module as a + CodeBlock.""" + code = """!$mydir test + module mymod + integer :: i + end module mymod + """ + reader = FortranReader(ignore_comments=False, ignore_directives=False) + psyir = reader.psyir_from_source(code) + # The directive is a codeblock + assert isinstance(psyir.children[0], CodeBlock) + assert psyir.children[0].debug_string() == "!$mydir test\n" + + +def test_directive_before_while(): + """Test that the FortranReader stores a directive before a while loop as a + CodeBlock.""" + code = """subroutine x + integer :: i + !$omp barrier + do while(i < 1) + i = i + 1 + end do + end subroutine x""" + reader = FortranReader(ignore_comments=False, ignore_directives=False) + psyir = reader.psyir_from_source(code) + routine = psyir.children[0] + # The directive is a codeblock + assert isinstance(routine.children[0], CodeBlock) + assert routine.children[0].debug_string() == "!$omp barrier\n" + + +def test_directive_before_allocate(): + """Test that the FortranReader stored a directive before an allocate as a + CodeBlock.""" + code = """subroutine x + integer :: j + integer, dimension(:), allocatable :: i + j = 4 + !dir$ aligned + allocate(i(1:j)) + end subroutine""" + reader = FortranReader(ignore_comments=False, ignore_directives=False) + psyir = reader.psyir_from_source(code) + routine = psyir.children[0] + # The directive is a codeblock. + assert isinstance(routine.children[1], CodeBlock) + assert routine.children[1].debug_string() == "!dir$ aligned\n" From 9ae4ff08438fad7fe9e14ba06299b551c575c5d8 Mon Sep 17 00:00:00 2001 From: LonelyCat124 <3043914+LonelyCat124@users.noreply.github.com.> Date: Mon, 20 Oct 2025 14:08:10 +0100 Subject: [PATCH 05/25] First draft finished --- src/psyclone/psyir/frontend/fortran.py | 1 - src/psyclone/psyir/nodes/codeblock.py | 2 +- .../psyir/frontend/fparser2_directive_test.py | 15 --------------- 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/src/psyclone/psyir/frontend/fortran.py b/src/psyclone/psyir/frontend/fortran.py index 9c540b7048..99f9ea56bd 100644 --- a/src/psyclone/psyir/frontend/fortran.py +++ b/src/psyclone/psyir/frontend/fortran.py @@ -277,7 +277,6 @@ def psyir_from_file(self, file_path): _, filename = os.path.split(file_path) psyir = self._processor.generate_psyir(parse_tree, filename) - print(psyir.view()) return psyir diff --git a/src/psyclone/psyir/nodes/codeblock.py b/src/psyclone/psyir/nodes/codeblock.py index 115ef085de..b03cee05b4 100644 --- a/src/psyclone/psyir/nodes/codeblock.py +++ b/src/psyclone/psyir/nodes/codeblock.py @@ -211,7 +211,7 @@ def get_symbol_names(self) -> List[str]: # For directives, we need to analyse all alphanumeric* parts of the # comment string and return any names that match a symbol in the # symbol table. - for node in walk(parse_tree, Fortran2003.Comment): + for node in walk(parse_tree, Fortran2003.Directive): string_rep = node.tostr() # Directives start with a $ if string_rep.lstrip()[0:2] != "!$": diff --git a/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py b/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py index b828db99bf..c0a41a3d8c 100644 --- a/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py +++ b/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py @@ -36,24 +36,10 @@ """Performs pytest tests on the support for directives in the fparser2 PSyIR front-end""" -import pytest - -from fparser.two import Fortran2003 - from psyclone.psyir.frontend.fortran import FortranReader from psyclone.psyir.nodes import ( - Container, - Routine, - Assignment, - Loop, - IfBlock, - Call, CodeBlock, ) -from psyclone.psyir.commentable_mixin import CommentableMixin -from psyclone.psyir.symbols import DataTypeSymbol, StructureType - -from psyclone.psyir.backend.fortran import FortranWriter def test_directive_after_decls(): @@ -85,7 +71,6 @@ def test_directive_in_decls(): psyir = reader.psyir_from_source(code) routine = psyir.children[0] out = routine.debug_string() - print(out) assert """!$omp firstprivate integer, dimension(100) :: i !dir$ aligned""" in out From 99cdc9fae39be7f5003dd1672b82f4661167e81c Mon Sep 17 00:00:00 2001 From: LonelyCat124 <3043914+LonelyCat124@users.noreply.github.com.> Date: Tue, 21 Oct 2025 10:25:17 +0100 Subject: [PATCH 06/25] Update submodule --- .gitmodules | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index b93fc86163..568efa8a5b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,7 @@ [submodule "external/fparser"] path = external/fparser - url = https://github.com/stfc/fparser.git + url = https://github.com/stfc/fparser.git + branch = 483_generic_directives [submodule "external/dl_esm_inf"] path = external/dl_esm_inf url = https://github.com/stfc/dl_esm_inf.git From 906fd524cb33976b7424c6a963ff6dca6992b701 Mon Sep 17 00:00:00 2001 From: LonelyCat124 <3043914+LonelyCat124@users.noreply.github.com.> Date: Fri, 31 Oct 2025 12:13:00 +0000 Subject: [PATCH 07/25] Update fparser module --- external/fparser | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/fparser b/external/fparser index 00d276a6d5..11a9075eb6 160000 --- a/external/fparser +++ b/external/fparser @@ -1 +1 @@ -Subproject commit 00d276a6d575d7c79411ba8ace0dc95cd3b505ab +Subproject commit 11a9075eb6c24a0ebe6af770acac0e762389c9bb From 3c5737c1cdcf7123fcf870580f5a96fc99375d76 Mon Sep 17 00:00:00 2001 From: LonelyCat124 <3043914+LonelyCat124@users.noreply.github.com.> Date: Mon, 3 Nov 2025 14:29:24 +0000 Subject: [PATCH 08/25] Fixes for the new directive and to handle comment only mode --- src/psyclone/psyir/frontend/fparser2.py | 17 +++++++++-- .../psyir/frontend/fparser2_comment_test.py | 30 +++++++++++++++++++ .../psyir/frontend/fparser2_directive_test.py | 7 +++-- 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/src/psyclone/psyir/frontend/fparser2.py b/src/psyclone/psyir/frontend/fparser2.py index 612ce2ec2a..855b9937f7 100644 --- a/src/psyclone/psyir/frontend/fparser2.py +++ b/src/psyclone/psyir/frontend/fparser2.py @@ -42,6 +42,7 @@ from collections import OrderedDict from dataclasses import dataclass, field +import re import os import sys from typing import Iterable, Optional, Union @@ -5858,10 +5859,22 @@ def process_comment(self, comment, preceding_comments): :type preceding_comments: List[:py:class:`fparser.two.utils.Comment`] ''' + _directive_formats = [ + r"\!\$[a-z]", # Generic directive + r"c\$[a-z]", # Generic directive + r"\*\$[a-z]", # Generic directive + r"\!dir\$", # flang, ifx, ifort directives. + r"cdir\$", # flang, ifx, ifort fixed format directive. + r"\!gcc\$", # GCC compiler directive + ] if len(comment.tostr()) == 0: return - if self._ignore_directives and comment.tostr().startswith("!$"): - return + if self._ignore_directives: + comment_str = comment.tostr().lower() + directive = False + for dir_form in _directive_formats: + if re.match(dir_form, comment_str): + return if self._last_psyir_parsed_and_span is not None: last_psyir, last_span = self._last_psyir_parsed_and_span if (last_span[1] is not None diff --git a/src/psyclone/tests/psyir/frontend/fparser2_comment_test.py b/src/psyclone/tests/psyir/frontend/fparser2_comment_test.py index 8d1d22ca32..f33ad15a4b 100644 --- a/src/psyclone/tests/psyir/frontend/fparser2_comment_test.py +++ b/src/psyclone/tests/psyir/frontend/fparser2_comment_test.py @@ -604,3 +604,33 @@ def test_lost_program_comments(): "inline here") assignment = psyir.walk(Assignment)[0] assert assignment.preceding_comment == "Comment here" + + +@pytest.mark.parametrize("directive", ["$omp target", + "$acc kernels", + "dir$ vector", + "DIR$ VECTOR"]) +def test_directives_not_comments(directive): + """Test that the FortranReader doesn't keep directives when only + comments are requested.""" + code = f"""module A + implicit none + integer, public :: a + public + + contains + subroutine test() + + !$ a = 0 + & + !$& 0 + !{directive} + a = 1 + a = 2 + a = 3 + + end subroutine test + +end module A""" + reader = FortranReader(ignore_comments=False) + psyir = reader.psyir_from_source(code) + assert directive not in psyir.debug_string() diff --git a/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py b/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py index c0a41a3d8c..eedca847b0 100644 --- a/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py +++ b/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py @@ -71,8 +71,11 @@ def test_directive_in_decls(): psyir = reader.psyir_from_source(code) routine = psyir.children[0] out = routine.debug_string() - assert """!$omp firstprivate -integer, dimension(100) :: i !dir$ aligned""" in out + assert """!$ omp firstprivate +integer, dimension(100) :: i ! dir$ aligned""" in out + + pytest.xfail(reason="TODO #3178 PSyclone can't store directives in " + "declrations as directives.") def test_directive_at_end(): From f34519025a3911a0e4035ad02fd2b511b5ee76a8 Mon Sep 17 00:00:00 2001 From: LonelyCat124 <3043914+LonelyCat124@users.noreply.github.com.> Date: Mon, 3 Nov 2025 14:37:02 +0000 Subject: [PATCH 09/25] linting and test fixes --- src/psyclone/psyir/frontend/fparser2.py | 1 - .../tests/psyir/frontend/fparser2_directive_test.py | 7 +++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/psyclone/psyir/frontend/fparser2.py b/src/psyclone/psyir/frontend/fparser2.py index 855b9937f7..af1f7e3387 100644 --- a/src/psyclone/psyir/frontend/fparser2.py +++ b/src/psyclone/psyir/frontend/fparser2.py @@ -5871,7 +5871,6 @@ def process_comment(self, comment, preceding_comments): return if self._ignore_directives: comment_str = comment.tostr().lower() - directive = False for dir_form in _directive_formats: if re.match(dir_form, comment_str): return diff --git a/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py b/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py index eedca847b0..a725ea1424 100644 --- a/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py +++ b/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py @@ -36,6 +36,8 @@ """Performs pytest tests on the support for directives in the fparser2 PSyIR front-end""" +import pytest + from psyclone.psyir.frontend.fortran import FortranReader from psyclone.psyir.nodes import ( CodeBlock, @@ -71,8 +73,9 @@ def test_directive_in_decls(): psyir = reader.psyir_from_source(code) routine = psyir.children[0] out = routine.debug_string() - assert """!$ omp firstprivate -integer, dimension(100) :: i ! dir$ aligned""" in out + print(out) + assert """ ! $omp firstprivate + integer, dimension(100) :: i ! dir$ aligned""" in out pytest.xfail(reason="TODO #3178 PSyclone can't store directives in " "declrations as directives.") From be28e30420c385bc1aedf035e1bf671574158958 Mon Sep 17 00:00:00 2001 From: LonelyCat124 <3043914+LonelyCat124@users.noreply.github.com.> Date: Mon, 3 Nov 2025 16:05:19 +0000 Subject: [PATCH 10/25] Removed dead code, updated to new firective handling --- src/psyclone/psyir/frontend/fparser2.py | 27 +++++++++++++++---------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/psyclone/psyir/frontend/fparser2.py b/src/psyclone/psyir/frontend/fparser2.py index af1f7e3387..93043f0ce1 100644 --- a/src/psyclone/psyir/frontend/fparser2.py +++ b/src/psyclone/psyir/frontend/fparser2.py @@ -2895,24 +2895,29 @@ def _add_comments_to_tree(self, parent: Node, preceding_comments, :type preceding_comments: list[:py:class:`fparser.two.utils.Base`] :param psy_child: The current PSyIR node being constructed. ''' + _directive_formats = [ + r"\!\$[a-z]", # Generic directive + r"c\$[a-z]", # Generic directive + r"\*\$[a-z]", # Generic directive + r"\!dir\$", # flang, ifx, ifort directives. + r"cdir\$", # flang, ifx, ifort fixed format directive. + r"\!gcc\$", # GCC compiler directive + ] for comment in preceding_comments[:]: # If the comment is a directive and we # keep_directives then create a CodeBlock for # the directive. - # TODO: fparser #469. This only captures some free-form - # directives. - if (not self._ignore_directives and - comment.tostr().startswith("!$")): - block = self.nodes_to_code_block(parent, [comment]) + if not self._ignore_directives: + comment_str = comment.tostr().lower() + is_directive = False + for dir_form in _directive_formats: + if re.match(dir_form, comment_str): + is_directive = True # Attach any comments that came before this directive to this # CodeBlock node. - if comment is not preceding_comments[0]: - index = preceding_comments.index(comment) - block.preceding_comment += self._comments_list_to_string( - preceding_comments[0:index]) - preceding_comments = preceding_comments[index:] - preceding_comments.remove(comment) + if is_directive: + preceding_comments.remove(comment) # Leftover comments are added to the provided PSyIR node. psy_child.preceding_comment += self._comments_list_to_string( preceding_comments From 076e48060c59f0ea32facffb27a1045e1f28bf98 Mon Sep 17 00:00:00 2001 From: LonelyCat124 <3043914+LonelyCat124@users.noreply.github.com.> Date: Tue, 4 Nov 2025 10:25:54 +0000 Subject: [PATCH 11/25] Fix failing test with upstream changes --- src/psyclone/tests/psyir/frontend/fparser2_directive_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py b/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py index a725ea1424..fabedaaf0d 100644 --- a/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py +++ b/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py @@ -161,7 +161,7 @@ def test_directive_before_else(): def test_directive_before_module(): """Test that the FortranReader stores a directive before a module as a CodeBlock.""" - code = """!$mydir test + code = """!dir$ test module mymod integer :: i end module mymod @@ -170,7 +170,7 @@ def test_directive_before_module(): psyir = reader.psyir_from_source(code) # The directive is a codeblock assert isinstance(psyir.children[0], CodeBlock) - assert psyir.children[0].debug_string() == "!$mydir test\n" + assert psyir.children[0].debug_string() == "!dir$ test\n" def test_directive_before_while(): From b4522f6998087907b516440a53d651826f4ff6b5 Mon Sep 17 00:00:00 2001 From: LonelyCat124 <3043914+LonelyCat124@users.noreply.github.com.> Date: Tue, 4 Nov 2025 10:48:50 +0000 Subject: [PATCH 12/25] Moved fparser to latest commit --- external/fparser | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/fparser b/external/fparser index 11a9075eb6..669e2b1190 160000 --- a/external/fparser +++ b/external/fparser @@ -1 +1 @@ -Subproject commit 11a9075eb6c24a0ebe6af770acac0e762389c9bb +Subproject commit 669e2b119018e4fa1cacbd613d360dba77727f07 From f9a557a917a6a97297483fb3e878fb61971dbb3b Mon Sep 17 00:00:00 2001 From: LonelyCat124 <3043914+LonelyCat124@users.noreply.github.com.> Date: Tue, 4 Nov 2025 10:58:11 +0000 Subject: [PATCH 13/25] Removed no longer necessary code --- src/psyclone/psyir/frontend/fparser2.py | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/src/psyclone/psyir/frontend/fparser2.py b/src/psyclone/psyir/frontend/fparser2.py index 93043f0ce1..4df8edf079 100644 --- a/src/psyclone/psyir/frontend/fparser2.py +++ b/src/psyclone/psyir/frontend/fparser2.py @@ -2895,29 +2895,6 @@ def _add_comments_to_tree(self, parent: Node, preceding_comments, :type preceding_comments: list[:py:class:`fparser.two.utils.Base`] :param psy_child: The current PSyIR node being constructed. ''' - _directive_formats = [ - r"\!\$[a-z]", # Generic directive - r"c\$[a-z]", # Generic directive - r"\*\$[a-z]", # Generic directive - r"\!dir\$", # flang, ifx, ifort directives. - r"cdir\$", # flang, ifx, ifort fixed format directive. - r"\!gcc\$", # GCC compiler directive - ] - for comment in preceding_comments[:]: - # If the comment is a directive and we - # keep_directives then create a CodeBlock for - # the directive. - - if not self._ignore_directives: - comment_str = comment.tostr().lower() - is_directive = False - for dir_form in _directive_formats: - if re.match(dir_form, comment_str): - is_directive = True - # Attach any comments that came before this directive to this - # CodeBlock node. - if is_directive: - preceding_comments.remove(comment) # Leftover comments are added to the provided PSyIR node. psy_child.preceding_comment += self._comments_list_to_string( preceding_comments @@ -2965,6 +2942,7 @@ def process_nodes(self, parent, nodes): # Add the comments to nodes that support it and reset the # list of comments if isinstance(psy_child, CommentableMixin): + print("Adding comments to tree", preceding_comments) self._add_comments_to_tree(parent, preceding_comments, psy_child) preceding_comments = [] From e480d3ef4a1d7ce3fe1ffac9407112891ceee8a1 Mon Sep 17 00:00:00 2001 From: LonelyCat124 <3043914+LonelyCat124@users.noreply.github.com.> Date: Tue, 4 Nov 2025 11:23:02 +0000 Subject: [PATCH 14/25] Removed leftover print --- src/psyclone/psyir/frontend/fparser2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/psyclone/psyir/frontend/fparser2.py b/src/psyclone/psyir/frontend/fparser2.py index 4df8edf079..27182e80e4 100644 --- a/src/psyclone/psyir/frontend/fparser2.py +++ b/src/psyclone/psyir/frontend/fparser2.py @@ -2942,7 +2942,6 @@ def process_nodes(self, parent, nodes): # Add the comments to nodes that support it and reset the # list of comments if isinstance(psy_child, CommentableMixin): - print("Adding comments to tree", preceding_comments) self._add_comments_to_tree(parent, preceding_comments, psy_child) preceding_comments = [] From 7d558ce1071fda99e2982b9cd6e72ba7ecc400c8 Mon Sep 17 00:00:00 2001 From: LonelyCat124 <3043914+LonelyCat124@users.noreply.github.com.> Date: Tue, 4 Nov 2025 15:29:28 +0000 Subject: [PATCH 15/25] Added documentation on limitation re: directives --- doc/user_guide/psyclone_command.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/user_guide/psyclone_command.rst b/doc/user_guide/psyclone_command.rst index 314c0e13e5..b7e92465d1 100644 --- a/doc/user_guide/psyclone_command.rst +++ b/doc/user_guide/psyclone_command.rst @@ -520,6 +520,10 @@ some limitations: nodes. Also PSyclone will not know any details about these nodes (including that they contain directives) but this functionality will be improved over time. + 3. Directives that appear before or inside the declaration section of a + Subroutine, Program or Module will be converted to comments if directives + are kept, and the default output for comments will add a space after the + ``!``. Note that using the ``keep-comments`` option alone means that any comments that PSyclone interprets as directives will be lost from the input. From 54e58a38836d4be2d1f5c239827e4239aab6c5b1 Mon Sep 17 00:00:00 2001 From: LonelyCat124 <3043914+LonelyCat124@users.noreply.github.com.> Date: Tue, 11 Nov 2025 14:01:21 +0000 Subject: [PATCH 16/25] Latest changes --- doc/user_guide/psyclone_command.rst | 4 +- src/psyclone/psyir/frontend/fparser2.py | 1 - .../psyir/frontend/fparser2_comment_test.py | 5 +- .../psyir/frontend/fparser2_directive_test.py | 53 +++++++++++++++++++ 4 files changed, 57 insertions(+), 6 deletions(-) diff --git a/doc/user_guide/psyclone_command.rst b/doc/user_guide/psyclone_command.rst index b7e92465d1..bc96b39187 100644 --- a/doc/user_guide/psyclone_command.rst +++ b/doc/user_guide/psyclone_command.rst @@ -525,5 +525,5 @@ some limitations: are kept, and the default output for comments will add a space after the ``!``. -Note that using the ``keep-comments`` option alone means that any comments -that PSyclone interprets as directives will be lost from the input. +Note that using the ``--keep-comments`` option without the ``--keep-directives`` +option means that any comments that PSyclone interprets as directives will be excluded. diff --git a/src/psyclone/psyir/frontend/fparser2.py b/src/psyclone/psyir/frontend/fparser2.py index 27182e80e4..5cf36b8be8 100644 --- a/src/psyclone/psyir/frontend/fparser2.py +++ b/src/psyclone/psyir/frontend/fparser2.py @@ -3371,7 +3371,6 @@ def _do_construct_handler(self, node, parent): # Add the directive before the loop. loop.parent.addchild(directive) continue - # TODO if isinstance(child, Fortran2003.Nonlabel_Do_Stmt): found_do_stmt = True continue diff --git a/src/psyclone/tests/psyir/frontend/fparser2_comment_test.py b/src/psyclone/tests/psyir/frontend/fparser2_comment_test.py index f33ad15a4b..00d49c17c1 100644 --- a/src/psyclone/tests/psyir/frontend/fparser2_comment_test.py +++ b/src/psyclone/tests/psyir/frontend/fparser2_comment_test.py @@ -609,7 +609,8 @@ def test_lost_program_comments(): @pytest.mark.parametrize("directive", ["$omp target", "$acc kernels", "dir$ vector", - "DIR$ VECTOR"]) + "DIR$ VECTOR", + "$pos dir"]) def test_directives_not_comments(directive): """Test that the FortranReader doesn't keep directives when only comments are requested.""" @@ -625,8 +626,6 @@ def test_directives_not_comments(directive): !$& 0 !{directive} a = 1 - a = 2 - a = 3 end subroutine test diff --git a/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py b/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py index fabedaaf0d..880be78b28 100644 --- a/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py +++ b/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py @@ -207,3 +207,56 @@ def test_directive_before_allocate(): # The directive is a codeblock. assert isinstance(routine.children[1], CodeBlock) assert routine.children[1].debug_string() == "!dir$ aligned\n" + + +def test_multiple_directives(): + """Test that we get the correct directives when we have multiple + directive regions (including their end directives).""" + code = """subroutine x + integer :: i + !$omp parallel + i = 1 + !$omp end parallel + end subroutine x + """ + reader = FortranReader(ignore_comments=False, ignore_directives=False) + psyir = reader.psyir_from_source(code) + routine = psyir.children[0] + cbs = routine.walk(CodeBlock) + assert len(cbs) == 2 + assert cbs[0].debug_string() == "!$omp parallel\n" + assert cbs[1].debug_string() == "!$omp end parallel\n" + code = """subroutine x + integer :: i + !$omp parallel + i = 1 + !$omp end parallel + !$omp parallel + i = 2 + !$omp end parallel + end subroutine x + """ + reader = FortranReader(ignore_comments=False, ignore_directives=False) + psyir = reader.psyir_from_source(code) + routine = psyir.children[0] + cbs = routine.walk(CodeBlock) + assert len(cbs) == 4 + assert cbs[0].debug_string() == "!$omp parallel\n" + assert cbs[1].debug_string() == "!$omp end parallel\n" + assert cbs[2].debug_string() == "!$omp parallel\n" + assert cbs[3].debug_string() == "!$omp end parallel\n" + + +def test_inline_comment(): + """Test that the FortranReaer doesn't create a CodeBlock for an inlined + comment that looks like a directive.""" + code = """subroutine x + integer :: j + j = 4 !$omp atomic + end subroutine""" + reader = FortranReader(ignore_comments=False, ignore_directives=False) + psyir = reader.psyir_from_source(code) + routine = psyir.children[0] + print(routine.debug_string()) + print(routine.walk(CodeBlock)[0].debug_string()) + assert len(routine.walk(CodeBlock)) == 0 From 87dfcf4ff5726f1ee9d3f0f479ed9f3970be9a7a Mon Sep 17 00:00:00 2001 From: LonelyCat124 <3043914+LonelyCat124@users.noreply.github.com.> Date: Tue, 11 Nov 2025 14:01:28 +0000 Subject: [PATCH 17/25] Latest changes --- .gitmodules | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 568efa8a5b..bb4e813184 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,6 @@ [submodule "external/fparser"] path = external/fparser url = https://github.com/stfc/fparser.git - branch = 483_generic_directives [submodule "external/dl_esm_inf"] path = external/dl_esm_inf url = https://github.com/stfc/dl_esm_inf.git From fb172c3ffadb6b2d4a99736884cd512e2d6c3773 Mon Sep 17 00:00:00 2001 From: LonelyCat124 <3043914+LonelyCat124@users.noreply.github.com.> Date: Tue, 25 Nov 2025 13:18:57 +0000 Subject: [PATCH 18/25] Test fixes for new directive changes --- .../tests/psyir/frontend/fparser2_directive_test.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py b/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py index 880be78b28..258a4423f3 100644 --- a/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py +++ b/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py @@ -247,8 +247,8 @@ def test_multiple_directives(): assert cbs[3].debug_string() == "!$omp end parallel\n" -def test_inline_comment(): - """Test that the FortranReaer doesn't create a CodeBlock for an inlined +def test_inline_comment(fortran_writer): + """Test that the FortranReader doesn't create a CodeBlock for an inlined comment that looks like a directive.""" code = """subroutine x integer :: j @@ -257,6 +257,7 @@ def test_inline_comment(): reader = FortranReader(ignore_comments=False, ignore_directives=False) psyir = reader.psyir_from_source(code) routine = psyir.children[0] - print(routine.debug_string()) - print(routine.walk(CodeBlock)[0].debug_string()) + # We shouldn't have a directive (i.e. No CodeBlock) assert len(routine.walk(CodeBlock)) == 0 + # The comment should still be inline + assert "j = 4 ! $omp atomic" in fortran_writer(psyir) From 2d7a18ae702a2187386f70395382a0216b9bfad8 Mon Sep 17 00:00:00 2001 From: LonelyCat124 <3043914+LonelyCat124@users.noreply.github.com.> Date: Tue, 25 Nov 2025 13:19:20 +0000 Subject: [PATCH 19/25] linting --- src/psyclone/tests/psyir/frontend/fparser2_directive_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py b/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py index 258a4423f3..10157f2ad1 100644 --- a/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py +++ b/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py @@ -232,7 +232,7 @@ def test_multiple_directives(): i = 1 !$omp end parallel !$omp parallel - i = 2 + i = 2 !$omp end parallel end subroutine x """ From c0fd1e01b4e41177863957d7e112180c65a94085 Mon Sep 17 00:00:00 2001 From: LonelyCat124 <3043914+LonelyCat124@users.noreply.github.com.> Date: Tue, 25 Nov 2025 13:19:39 +0000 Subject: [PATCH 20/25] Fparser updates --- external/fparser | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/fparser b/external/fparser index 669e2b1190..044f28da46 160000 --- a/external/fparser +++ b/external/fparser @@ -1 +1 @@ -Subproject commit 669e2b119018e4fa1cacbd613d360dba77727f07 +Subproject commit 044f28da46cf29d37ee4d8bd5a7099dcff2a5978 From 5c89fe2c26f2a8722624588426bba742e42bf540 Mon Sep 17 00:00:00 2001 From: LonelyCat124 <3043914+LonelyCat124@users.noreply.github.com.> Date: Tue, 25 Nov 2025 14:02:41 +0000 Subject: [PATCH 21/25] Updated fparser installation to use the upstream --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 93e5123e6a..4601f44c8a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -103,7 +103,7 @@ jobs: # We need to install sphinx to get correct doc testing # Uncomment the below to use the submodule version of fparser rather # than the latest release from pypi. - # pip install external/fparser + pip install external/fparser pip install .[doc] pip install .[test] - name: Lint with flake8 From cedb527b0e631ccc5783b6955222df669458c1bf Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Fri, 28 Nov 2025 10:10:14 +0000 Subject: [PATCH 22/25] #3178 Update documentation regarding directive exclusions --- doc/user_guide/psyclone_command.rst | 5 ++--- src/psyclone/psyir/frontend/fparser2.py | 22 ++++++++++++++-------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/doc/user_guide/psyclone_command.rst b/doc/user_guide/psyclone_command.rst index bc96b39187..99a536c687 100644 --- a/doc/user_guide/psyclone_command.rst +++ b/doc/user_guide/psyclone_command.rst @@ -521,9 +521,8 @@ some limitations: (including that they contain directives) but this functionality will be improved over time. 3. Directives that appear before or inside the declaration section of a - Subroutine, Program or Module will be converted to comments if directives - are kept, and the default output for comments will add a space after the - ``!``. + Subroutine, Program or Module are converted to comments (the default + output add a space after the ``!``, effectively disabling the directive. Note that using the ``--keep-comments`` option without the ``--keep-directives`` option means that any comments that PSyclone interprets as directives will be excluded. diff --git a/src/psyclone/psyir/frontend/fparser2.py b/src/psyclone/psyir/frontend/fparser2.py index 5cf36b8be8..8bb380d2c9 100644 --- a/src/psyclone/psyir/frontend/fparser2.py +++ b/src/psyclone/psyir/frontend/fparser2.py @@ -5840,17 +5840,23 @@ def process_comment(self, comment, preceding_comments): :type preceding_comments: List[:py:class:`fparser.two.utils.Comment`] ''' - _directive_formats = [ - r"\!\$[a-z]", # Generic directive - r"c\$[a-z]", # Generic directive - r"\*\$[a-z]", # Generic directive - r"\!dir\$", # flang, ifx, ifort directives. - r"cdir\$", # flang, ifx, ifort fixed format directive. - r"\!gcc\$", # GCC compiler directive - ] if len(comment.tostr()) == 0: return if self._ignore_directives: + # When directive detection is disabled in fparser, but we still + # request comments the directive will be part of the comments. This + # is typically not an issue because current psyir backends add an + # space between the comment character and the start of the comment, + # effectively disabling the directive. However, for clarity, we + # still try to clean them up using the regex below. + _directive_formats = [ + r"\!\$[a-z]", # Generic directive + r"c\$[a-z]", # Generic directive + r"\*\$[a-z]", # Generic directive + r"\!dir\$", # flang, ifx, ifort directives. + r"cdir\$", # flang, ifx, ifort fixed format directive. + r"\!gcc\$", # GCC compiler directive + ] comment_str = comment.tostr().lower() for dir_form in _directive_formats: if re.match(dir_form, comment_str): From 950fd4d020f142eeaa5088e2d68ffeae280c943e Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Fri, 28 Nov 2025 10:22:51 +0000 Subject: [PATCH 23/25] #3178 Update changelog and make integration tests use the external fparser --- .github/workflows/compilation.yml | 2 +- .github/workflows/extraction_test.yml | 2 +- .github/workflows/lfric_test.yml | 2 +- .github/workflows/nemo_tests.yml | 2 +- .github/workflows/nemo_v5_tests.yml | 2 +- changelog | 2 ++ setup.py | 2 +- 7 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/compilation.yml b/.github/workflows/compilation.yml index fdee4781f7..243da9000a 100644 --- a/.github/workflows/compilation.yml +++ b/.github/workflows/compilation.yml @@ -87,7 +87,7 @@ jobs: python -m pip install --upgrade pip # Uncomment the below to use the submodule version of fparser rather # than the latest release from pypi. - # pip install external/fparser + pip install external/fparser pip install .[test,psydata,doc] - name: Unit tests with compilation - gfortran run: | diff --git a/.github/workflows/extraction_test.yml b/.github/workflows/extraction_test.yml index 7f3d32f909..3da7654dbd 100644 --- a/.github/workflows/extraction_test.yml +++ b/.github/workflows/extraction_test.yml @@ -74,7 +74,7 @@ jobs: python -m pip install --upgrade pip # Uncomment the below to use the submodule version of fparser rather # than the latest release from pypi. - # pip install external/fparser + pip install external/fparser pip install .[test] # Install BAF, and its version of Fab: # ------------------------------------ diff --git a/.github/workflows/lfric_test.yml b/.github/workflows/lfric_test.yml index 4cce0d2bde..b4e1da5df9 100644 --- a/.github/workflows/lfric_test.yml +++ b/.github/workflows/lfric_test.yml @@ -86,7 +86,7 @@ jobs: python -m pip install --upgrade pip # Uncomment the below to use the submodule version of fparser rather # than the latest release from pypi. - # pip install external/fparser + pip install external/fparser pip install .[test] # Fetch the specified version of LFRic apps source /archive/psyclone-spack/psyclone-spack-Jun25/spack-repo/share/spack/setup-env.sh diff --git a/.github/workflows/nemo_tests.yml b/.github/workflows/nemo_tests.yml index 75b5ccde30..1efafd8417 100644 --- a/.github/workflows/nemo_tests.yml +++ b/.github/workflows/nemo_tests.yml @@ -85,7 +85,7 @@ jobs: python -m pip install --upgrade pip # Uncomment the below to use the submodule version of fparser rather # than the latest release from pypi. - # pip install external/fparser + pip install external/fparser pip install .[test] # Compile nvidia profiling tools module load nvidia-hpcsdk/${NVFORTRAN_VERSION} diff --git a/.github/workflows/nemo_v5_tests.yml b/.github/workflows/nemo_v5_tests.yml index da6bc0b4d9..39b502bee8 100644 --- a/.github/workflows/nemo_v5_tests.yml +++ b/.github/workflows/nemo_v5_tests.yml @@ -82,7 +82,7 @@ jobs: python -m pip install --upgrade pip # Uncomment the below to use the submodule version of fparser rather # than the latest release from pypi. - # pip install external/fparser + pip install external/fparser pip install . - name: Reset working directory run: | diff --git a/changelog b/changelog index 40048eed1e..50ea88d0cc 100644 --- a/changelog +++ b/changelog @@ -1,3 +1,5 @@ + 4) PR #3178 for #3196. Improves fparser2 reader directive handling. + 3) PR #3210 for #3203. Removes outstanding references to the old, pared-down LFRic infrastructure. diff --git a/setup.py b/setup.py index 12e88df19d..747aad367f 100644 --- a/setup.py +++ b/setup.py @@ -169,7 +169,7 @@ def get_files(directory, install_path, valid_suffixes): classifiers=CLASSIFIERS, packages=PACKAGES, package_dir={"": "src"}, - install_requires=['pyparsing', 'fparser==0.2.1', 'configparser', + install_requires=['pyparsing', 'fparser>=0.2.1', 'configparser', 'sympy', "Jinja2", 'termcolor', 'graphviz'], extras_require={ 'doc': ["sphinx", "sphinxcontrib.bibtex", "sphinx_design", From 39125ee2c7f7db33011bf90dd0a5ba2f699ca8a6 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Fri, 28 Nov 2025 11:18:45 +0000 Subject: [PATCH 24/25] #3178 Remove print statement --- src/psyclone/tests/psyir/frontend/fparser2_directive_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py b/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py index 10157f2ad1..4a6d315a18 100644 --- a/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py +++ b/src/psyclone/tests/psyir/frontend/fparser2_directive_test.py @@ -73,7 +73,6 @@ def test_directive_in_decls(): psyir = reader.psyir_from_source(code) routine = psyir.children[0] out = routine.debug_string() - print(out) assert """ ! $omp firstprivate integer, dimension(100) :: i ! dir$ aligned""" in out From 8cdbf128e0af59296b4f0af33a3693f871107ce9 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Fri, 28 Nov 2025 11:20:58 +0000 Subject: [PATCH 25/25] Ignore khronos linkcheck --- doc/conf.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/conf.py b/doc/conf.py index f9694e6395..9ca3c58cae 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -413,7 +413,9 @@ # the links to anchors to the main README. r'^https://github.com/stfc/PSyclone#', # Requires authentication. - r'^https://code.metoffice.gov.uk/trac' + r'^https://code.metoffice.gov.uk/trac', + # This often fails, but the link exists + r'^https://www.khronos.org' ] # -- Autodoc configuration ---------------------------------------------------