Merge pull request #855 from nipype/copyfile-workflow-fix

Copyfile_workflow conflicting filenames bugfix

Author: tclose

pydra/engine/result.py

@@ -233,12 +233,16 @@ def copyfile_workflow(
 ) -> workflow.Outputs:
     """if file in the wf results, the file will be copied to the workflow directory"""
 
+    clashes_to_avoid: set[Path] = set()
     for field in attrs_fields(outputs):
         value = getattr(outputs, field.name)
         # if the field is a path or it can contain a path _copyfile_single_value is run
         # to move all files and directories to the workflow directory
         new_value = copy_nested_files(
-            value, wf_path, mode=FileSet.CopyMode.hardlink_or_copy
+            value,
+            wf_path,
+            mode=FileSet.CopyMode.hardlink_or_copy,
+            clashes_to_avoid=clashes_to_avoid,
         )
         setattr(outputs, field.name, new_value)
     return outputs

pydra/engine/tests/test_result.py

@@ -1,4 +1,14 @@
-from pydra.engine.result import Result, Runtime
+from pathlib import Path
+from fileformats.text import TextFile
+from pydra.compose import python
+from pydra.engine.result import Result, Runtime, copyfile_workflow
+
+
+@python.define(outputs=["d", "e", "f"])
+def MockTask(
+    a: TextFile, b: TextFile, c: TextFile
+) -> tuple[TextFile, TextFile, TextFile]:
+    return a, b, c
 
 
 def test_runtime():
@@ -8,9 +18,30 @@ def test_runtime():
     assert hasattr(runtime, "cpu_peak_percent")
 
 
-def test_result(tmp_path):
+def test_result(tmp_path: Path):
     result = Result(cache_dir=tmp_path)
     assert hasattr(result, "runtime")
     assert hasattr(result, "outputs")
     assert hasattr(result, "errored")
     assert getattr(result, "errored") is False
+
+
+def test_copyfile_workflow_conflicting_filenames(tmp_path: Path) -> None:
+    """Copy outputs to the workflow output directory with conflicting filenames.
+    The filenames should be disambiguated to avoid clashes"""
+    file1 = TextFile.sample(stem="out")
+    file2 = TextFile.sample(stem="out")
+    file3 = TextFile.sample(stem="out")
+
+    workflow_dir = tmp_path / "output"
+    mock = MockTask(a=file1, b=file2, c=file3)
+    outputs = mock()
+    workflow_dir.mkdir()
+
+    copyfile_workflow(workflow_dir, outputs)
+
+    assert sorted(p.stem for p in workflow_dir.iterdir()) == [
+        "out",
+        "out (1)",
+        "out (2)",
+    ]

pydra/utils/typing.py

@@ -1206,6 +1206,7 @@ def copy_nested_files(
     value: ty.Any,
     dest_dir: os.PathLike,
     supported_modes: generic.FileSet.CopyMode = generic.FileSet.CopyMode.any,
+    clashes_to_avoid: set[Path] | None = None,
     **kwargs,
 ) -> ty.Any:
     """Copies all "file-sets" found within the nested value (e.g. dict, list,...) into the
@@ -1229,7 +1230,8 @@ def copy_nested_files(
 
     # Set to keep track of file paths that have already been copied
     # to allow FileSet.copy to avoid name clashes
-    clashes_to_avoid = set()
+    if clashes_to_avoid is None:
+        clashes_to_avoid = set()
 
     def copy_fileset(fileset: generic.FileSet):
         try: