Pass a bash script to sbatch rather than the command directly to circumvent SLURM command-line character limit

Author: dmwnz

snakemake_executor_plugin_slurm/__init__.py

@@ -500,19 +500,21 @@ def run_job(self, job: JobExecutorInterface):
 
         exec_job = self.format_job_exec(job)
 
-        # and finally the job to execute with all the snakemake parameters
-        call += f' --wrap="{exec_job}"'
+        # format the job to execute with all the snakemake parameters into a shell script
+        sbatch_script = "\n".join(["#!/bin/sh", exec_job])
 
         self.logger.debug(f"sbatch call: {call}")
+        self.logger.debug(f"sbatch script:\n{sbatch_script}")
         try:
             process = subprocess.Popen(
-                call,
+                f"{call} /dev/stdin",  # tell sbatch to read the sbatch script from stdin
                 shell=True,
                 text=True,
+                stdin=subprocess.PIPE,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
             )
-            out, err = process.communicate()
+            out, err = process.communicate(input=sbatch_script)  # feed the sbatch shell script through stdin
             if process.returncode != 0:
                 raise subprocess.CalledProcessError(
                     process.returncode, call, output=err
@@ -524,6 +526,7 @@ def run_job(self, job: JobExecutorInterface):
                     "SLURM sbatch failed. "
                     f"The error message was '{e.output.strip()}'.\n"
                     f"    sbatch call:\n        {call}\n"
+                    f"    sbatch script:\n{sbatch_script}\n"
                 ),
             )
             return