Mercurial > repos > mbernt > singularity_scriptrunner

changeset 2:2eb0532a911c draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload for repository https://github.com/bernt-matthias/mb-galaxy-tools/tree/master/tools/scripting/ commit 8c309a4891e29a0fd273f9edc37292e1f3965317
author ufz
date Fri, 11 Apr 2025 15:20:00 +0000
parents 0da37b889932
children
files general.xml test-data/scripting_images.loc
diffstat 2 files changed, 75 insertions(+), 50 deletions(-) [+]
line wrap: on
line diff
--- a/general.xml	Sun Aug 06 16:01:44 2023 +0000
+++ b/general.xml	Fri Apr 11 15:20:00 2025 +0000
@@ -1,4 +1,4 @@
-<tool id="singularity_scriptrunner" name="scriptrunner" version="0.1" profile="22.05">
+<tool id="singularity_scriptrunner" name="scriptrunner" version="0.2" profile="22.05">
     <description>singularity</description>
     <creator>
         <person givenName="Matthias" familyName="Bernt" email="m.bernt@ufz.de" />
@@ -49,22 +49,11 @@
         <configfile name="script">$code</configfile>
     </configfiles>
     <inputs>
-        <param name="interpreter" type="select" label="Interpreter">
-            <options from_data_table="scripting_images">
-                <column name="name" index="3"/>
-                <column name="value" index="3"/>
-                <filter type="regexp" column="4" value="singularity|apptainer"/>
-            </options>
-            <validator type="no_options" message="No interpreter available. Contact you Galaxy administrator." />
+        <param name="image" type="select" label="Image" >
+            <options from_data_table="scripting_images"/>
+            <validator type="no_options" message="No image is available. Contact you Galaxy administrator." />
         </param>
-        <param name="image" type="select" label="Image" >
-            <options from_data_table="scripting_images">
-                <filter type="regexp" column="4" value="singularity|apptainer"/>
-                <filter type="param_value" column="3" ref="interpreter"/>
-            </options>
-            <validator type="no_options" message="No interpreter / image is available. Contact you Galaxy administrator." />
-        </param>
-        <repeat name="parameters" title="Parameters" min="1" default="1" help="Supply one or more parameters">
+        <repeat name="parameters" title="Parameters" min="0" default="1" help="Supply parameters">
             <conditional name="type_cond">
                 <param name="type_sel" type="select" label="Parameter type">
                     <option value="data">Dataset</option>
@@ -104,15 +93,14 @@
     <tests>
         <!-- read tsv write csv -->
         <test>
-            <param name="interpreter" value="python"/>
-            <param name="image" value="python_continuumio_anaconda"/>
+            <param name="image" value="python_python"/>
             <repeat name="parameters">
                 <conditional name="type_cond">
                     <param name="type_sel" value="data"/>
                     <param name="param" value="test.tsv" ftype="tabular"/>
                 </conditional>
             </repeat>
-            <param name="code" value='import sys; import pandas as pd; df = pd.read_csv(sys.argv[1], sep="\t"); df.to_csv("data.csv", index=False, sep=",");'/>
+            <param name="code" value='import sys, subprocess; subprocess.check_call([sys.executable, "-m", "pip", "install", "pandas", "matplotlib"]); import pandas as pd; df = pd.read_csv(sys.argv[1], sep="\t"); df.to_csv("data.csv", index=False, sep=",");'/>
             <output_collection name="output" type="list" count="1">
                 <element name="data" ftype="csv">
                     <assert_contents>
@@ -130,15 +118,14 @@
         </test>
         <!-- plot w matplotlib -->
         <test>
-            <param name="interpreter" value="python"/>
-            <param name="image" value="python_continuumio_anaconda"/>
+            <param name="image" value="python_python"/>
             <repeat name="parameters">
                 <conditional name="type_cond">
                     <param name="param" value="test.tsv" ftype="tabular"/>
                     <param name="filename" value="custom_name.tsv"/>
                 </conditional>
             </repeat>
-            <param name="code" value='import sys; import pandas as pd; from matplotlib.backends.backend_pdf import PdfPages;  df = pd.read_csv(sys.argv[1], sep="\t"); fh = PdfPages("points.pdf"); plt = df.plot(); fh.savefig(); fh.close(); print(f"plotted {sys.argv[1]}")'/>
+            <param name="code" value='import sys, subprocess; subprocess.check_call([sys.executable, "-m", "pip", "install", "pandas", "matplotlib"]); import pandas as pd; from matplotlib.backends.backend_pdf import PdfPages;  df = pd.read_csv(sys.argv[1], sep="\t"); fh = PdfPages("points.pdf"); plt = df.plot(); fh.savefig(); fh.close(); print(f"plotted {sys.argv[1]}")'/>
             <output_collection name="output" type="list" count="1">
                 <element name="points" ftype="pdf">
                     <assert_contents>
@@ -152,20 +139,26 @@
         </test>
         <!-- install libraries ("forbidden") -->
         <test expect_failure="true">
-            <param name="interpreter" value="python"/>
-            <param name="image" value="python_continuumio_anaconda"/>
+            <param name="image" value="python_python_noinstall"/>
+            <repeat name="parameters">
+                <!-- mocking test here .. can't leave the repeat empty (in the test)
+                     since Galaxy will expect the data parameter anyway 
+                     https://github.com/galaxyproject/galaxy/pull/19472 -->
+                <conditional name="type_cond">
+                    <param name="type_sel" value="text"/>
+                </conditional>
+            </repeat>
             <param name="code" value='import pip; pip.main(["install", "biopython"]); import Bio'/>
         </test>
         <!-- read binary files (eg feather)  -->
         <test>
-            <param name="interpreter" value="python"/>
-            <param name="image" value="python_continuumio_anaconda"/>
+            <param name="image" value="python_python"/>
             <repeat name="parameters">
                 <conditional name="type_cond">
                     <param name="param" value="test.h5" ftype="h5"/>
                 </conditional>
             </repeat>
-            <param name="code" value='import sys; import os; import pandas as pd; df = pd.read_hdf(sys.argv[1]); df.to_csv("data.csv", index=False, sep=",");'/>
+            <param name="code" value='import sys, subprocess; subprocess.check_call([sys.executable, "-m", "pip", "install", "pandas", "matplotlib", "tables"]); import os; import pandas as pd; df = pd.read_hdf(sys.argv[1]); df.to_csv("data.csv", index=False, sep=",");'/>
             <output_collection name="output" type="list" count="1">
                 <element name="data" ftype="csv">
                     <assert_contents>
@@ -178,8 +171,7 @@
         </test>
         <!-- parameters -->
         <test>
-            <param name="interpreter" value="python"/>
-            <param name="image" value="python_continuumio_anaconda"/>
+            <param name="image" value="python_python"/>
             <repeat name="parameters">
                 <conditional name="type_cond">
                     <param name="type_sel" value="text"/>
@@ -192,7 +184,7 @@
                     <param name="param" value="some value"/>
                 </conditional>
             </repeat>
-            <param name="code" value='import sys; fh = open(sys.argv[1], "w"); fh.write("Hello,world\n"); fh.write("Bye,world\n"); fh.close(); print(sys.argv[2]);'/>
+            <param name="code" value='import sys, subprocess; subprocess.check_call([sys.executable, "-m", "pip", "install", "pandas", "matplotlib"]);  fh = open(sys.argv[1], "w"); fh.write("Hello,world\n"); fh.write("Bye,world\n"); fh.close(); print(sys.argv[2]);'/>
             <output_collection name="output" type="list" count="1">
                 <element name="filename" ftype="csv">
                     <assert_contents>
@@ -209,14 +201,13 @@
 
         <!-- read tsv write csv -->
         <test>
-            <param name="interpreter" value="Rscript"/>
             <param name="image" value="r_rocker_tidyverse"/>
             <repeat name="parameters">
                 <conditional name="type_cond">
                     <param name="param" value="test.tsv" ftype="tabular"/>
                 </conditional>
             </repeat>
-            <param name="code" value='args = commandArgs(trailingOnly = TRUE); data = read.delim(args[1]); write.csv(data, "data.csv", row.names=FALSE)'/>
+            <param name="code" value='args = commandArgs(trailingOnly = TRUE); install.packages("data.table", lib=Sys.getenv("R_LIBS_USER")); library(data.table); data = read.delim(args[1]); write.csv(data, "data.csv", row.names=FALSE)'/>
             <output_collection name="output" type="list" count="1">
                 <element name="data" ftype="csv">
                     <assert_contents>
@@ -228,13 +219,12 @@
             </output_collection>
             <!-- but the data table this should use apptainer and not use any additional parameters to it -->
             <assert_command>
-                <has_text text="apptainer"/>
-                <has_text text="--cleanenv" negate="true"/>
+                <has_text_matching expression="apptainer|singularity"/>
+                <has_text text="--cleanenv"/>
             </assert_command>
         </test>
         <!-- use a tidyverse library -->
         <test>
-            <param name="interpreter" value="Rscript"/>
             <param name="image" value="r_rocker_tidyverse"/>
             <repeat name="parameters">
                 <conditional name="type_cond">
@@ -256,18 +246,31 @@
         </test>
         <!-- install libraries fails -->
         <test expect_failure="true">
-            <param name="interpreter" value="Rscript"/>
             <param name="image" value="r_rocker_tidyverse"/>
+            <repeat name="parameters">
+                <!-- mocking test here .. can't leave the repeat empty (in the test)
+                     since Galaxy will expect the data parameter anyway 
+                     https://github.com/galaxyproject/galaxy/pull/19472 -->
+                <conditional name="type_cond">
+                    <param name="type_sel" value="text"/>
+                </conditional>
+            </repeat>
             <param name="code" value='install.packages("maybe"); library(maybe); print("success")'/>
         </test>
         <test expect_failure="true">
-            <param name="interpreter" value="Rscript"/>
             <param name="image" value="r_rocker_tidyverse"/>
+            <repeat name="parameters">
+                <!-- mocking test here .. can't leave the repeat empty (in the test)
+                     since Galaxy will expect the data parameter anyway 
+                     https://github.com/galaxyproject/galaxy/pull/19472 -->
+                <conditional name="type_cond">
+                    <param name="type_sel" value="text"/>
+                </conditional>
+            </repeat>
             <param name="code" value='install.packages("BiocManager"); BiocManager::install("multtest"); print("success")'/>
         </test>
         <!-- read binary files (eg rds) -->
         <test>
-            <param name="interpreter" value="Rscript"/>
             <param name="image" value="r_rocker_tidyverse"/>
             <repeat name="parameters">
                 <conditional name="type_cond">
@@ -287,7 +290,6 @@
         </test>
         <!-- optional input and parameters -->
         <test>
-            <param name="interpreter" value="Rscript"/>
             <param name="image" value="r_rocker_tidyverse"/>
             <repeat name="parameters">
                 <conditional name="type_cond"> 
@@ -318,8 +320,7 @@
 
         <!-- some tests with bash-->
         <test>
-            <param name="interpreter" value="bash"/>
-            <param name="image" value="bash_continuumio_anaconda"/>
+            <param name="image" value="bash_python"/>
             <repeat name="parameters">
                 <conditional name="type_cond">
                     <param name="param" value="test.tsv" ftype="tabular"/>
@@ -338,15 +339,14 @@
         </test>
 
         <!-- check that we can turn off networking -->
-        <test expect_failure="true">
-            <param name="interpreter" value="bash"/>
-            <param name="image" value="bash_continuumio_anaconda"/>
+        <test expect_failure="false">
+            <param name="image" value="bash_rocker_tidyverse"/>
             <repeat name="parameters">
                 <conditional name="type_cond">
                     <param name="param" value="test.tsv" ftype="tabular"/>
                 </conditional>
             </repeat>
-            <param name="code" value="curl -iL https://www.galaxyproject.org"/>
+            <param name="code" value="wget https://www.galaxyproject.org"/>
         </test>
     </tests>
     <help><![CDATA[
@@ -411,5 +411,23 @@
 single collection.  The collection elements will be named as the file names
 (without the extension). The file extension determines the datatype of the
 datasets (or Galaxy will try to autodetect the data type).
+
+**Custom 3rd party packages or libraries**
+
+Installation of 3rd party software might work depending on the container
+and the 3rd party software. For python and R the following can work. In case
+of problems contact your Galaxy administrator.
+
+To install two packages ``PACKAGE1`` and ``PACKAGE2`` in python:
+
+::
+    import sys, subprocess
+    subprocess.check_call([sys.executable, "-m", "pip", "install", "PACKAGE1", "PACKAGE2"]);
+
+For R scripts the following will install two libraries ``LIBRARY1`` and ``LIBRARY2``:
+
+::
+    install.packages(c("LIBRARY1", "LIBRARY2"), lib=Sys.getenv("R_LIBS_USER"));
+
     ]]></help>
-</tool>
\ No newline at end of file
+</tool>
--- a/test-data/scripting_images.loc	Sun Aug 06 16:01:44 2023 +0000
+++ b/test-data/scripting_images.loc	Fri Apr 11 15:20:00 2025 +0000
@@ -1,10 +1,17 @@
 # <id>	<name>	<image_name>	<interpreter>	<container_type>	<container_params>
 
-# docker://continuumio/anaconda3
-python_continuumio_anaconda	python (continuumio/anaconda)	/tmp/anaconda3_latest.sif	python	singularity	--cleanenv --network none --no-mount bind-paths
+# docker://python_3.10
+python_python	python (python)	/tmp/python_3.10-slim.sif	python	mkdir -p "$HOME/.local/lib/python3.10/site-packages" && singularity	--no-home --cleanenv --home $HOME --env PYTHONUSERBASE="$HOME/.local"
+
+# docker://python_3.10
+python_python_noinstall	python (python no install)	/tmp/python_3.10-slim.sif	python	singularity	--no-home --cleanenv
+
 
 # docker://rocker/tidyverse
-r_rocker_tidyverse	R (tidyverse)	/tmp/tidyverse_latest.sif	Rscript	apptainer	
+r_rocker_tidyverse	R (tidyverse)	/tmp/tidyverse_latest.sif	Rscript	mkdir -p "$HOME/custom_R_library" && singularity	--no-home --cleanenv --home $HOME --env R_LIBS_USER="$HOME/custom_R_library" 
+
+bash_rocker_tidyverse	bash (R tidyverse)	/tmp/tidyverse_latest.sif	bash	singularity	--no-home --cleanenv --home $HOME 
+
 
 # docker://rocker/tidyverse
-bash_continuumio_anaconda	bash (continuumio/anaconda)	/tmp/anaconda3_latest.sif	bash	apptainer	--cleanenv --network none --no-mount bind-paths
\ No newline at end of file
+bash_python	bash (python)	/tmp/python_3.10-slim.sif	bash	singularity	--no-home --cleanenv --home $HOME