diff gcp_batch_netcat.py @ 11:fe0bf22037a5 draft

planemo upload for repository https://github.com/afgane/gcp_batch_netcat commit f730cbb207e028a5d4fd982fe65ece7345af4879
author enis
date Thu, 14 Aug 2025 16:39:36 +0000
parents cdfa2e1a7ef4
children
line wrap: on
line diff
--- a/gcp_batch_netcat.py	Tue Aug 12 16:22:41 2025 +0000
+++ b/gcp_batch_netcat.py	Thu Aug 14 16:39:36 2025 +0000
@@ -88,7 +88,7 @@
 echo "Target: {target_host}:{target_port}"
 echo "Timestamp: $(date)"
 echo "Container hostname: $(hostname)"
-echo "Host VM Image: galaxy-k8s-boot-v2025-08-10"
+echo "Host VM Image: galaxy-k8s-boot-v2025-08-12"
 echo "Container Image: afgane/gcp-batch-netcat:0.3.0"
 echo ""
 
@@ -153,65 +153,149 @@
 ip route
 echo ""
 
-# NFS Mount Test
-echo "=== NFS Mount Test ==="
-MOUNT_POINT="/tmp/nfs_test_mount"
-echo "Creating mount point: $MOUNT_POINT"
-mkdir -p "$MOUNT_POINT"
-
-echo "Attempting to mount NFS share..."
-echo "Command: mount -t nfs -o vers=3,tcp {target_host}:/ $MOUNT_POINT"
-
-# Try mounting the NFS share
+# NFS Mount Test - Check if Batch mounted it for us
+echo "=== NFS Mount Test (via Batch Volume) ==="
+NFS_MOUNT_POINT="/mnt/nfs"
 mount_result=1
-if mount -t nfs -o vers=3,tcp {target_host}:/ "$MOUNT_POINT" 2>&1; then
-    mount_result=0
-    echo "✓ NFS mount successful!"
+
+echo "Checking if NFS is mounted by Batch at $NFS_MOUNT_POINT..."
+if [ -d "$NFS_MOUNT_POINT" ]; then
+    echo "✓ NFS mount point exists"
+
+    # Check if it's actually mounted
+    if mount | grep "$NFS_MOUNT_POINT"; then
+        mount_result=0
+        echo "✓ NFS mounted by Batch successfully!"
+
+        echo ""
+        echo "=== NFS Share Contents ==="
+        echo "Long listing of NFS share:"
+        ls -la "$NFS_MOUNT_POINT" 2>/dev/null || echo "Could not list directory contents"
+
+        echo ""
+        echo "Disk usage of NFS share:"
+        df -h "$NFS_MOUNT_POINT" 2>/dev/null || echo "Could not get disk usage"
+
+        # Look for export subdirectories
+        echo ""
+        echo "=== Looking for export directories ==="
+        if [ -d "$NFS_MOUNT_POINT/export" ]; then
+            echo "✓ Found: export directory"
+            ls -la "$NFS_MOUNT_POINT/export" | head -10 2>/dev/null || echo "Could not list export contents"
+
+            # Look for PVC subdirectories
+            echo "Looking for PVC directories in export..."
+            find "$NFS_MOUNT_POINT/export" -name "pvc-*" -type d | head -5 2>/dev/null || echo "No PVC directories found"
+        else
+            echo "✗ No export directory found"
+        fi
 
-    echo ""
-    echo "=== NFS Share Contents ==="
-    echo "Long listing of NFS share root:"
-    ls -la "$MOUNT_POINT" 2>/dev/null || echo "Could not list directory contents"
+        # Try to find common Galaxy directories
+        echo ""
+        echo "=== Looking for Galaxy directories ==="
+
+        # First check if they exist directly in the NFS root
+        galaxy_dirs_in_root=0
+        for dir in "jobs_directory" "shed_tools" "objects" "tools" "cache" "config"; do
+            if [ -d "$NFS_MOUNT_POINT/$dir" ]; then
+                echo "✓ Found in root: $dir"
+                ls -la "$NFS_MOUNT_POINT/$dir" | head -5
+                galaxy_dirs_in_root=$((galaxy_dirs_in_root + 1))
+            fi
+        done
 
-    echo ""
-    echo "Disk usage of NFS share:"
-    df -h "$MOUNT_POINT" 2>/dev/null || echo "Could not get disk usage"
+        if [ $galaxy_dirs_in_root -eq 0 ]; then
+            echo "✗ No Galaxy directories found in NFS root"
+        else
+            echo "✓ Found $galaxy_dirs_in_root Galaxy directories in NFS root"
+        fi
 
-    echo ""
-    echo "Mount information:"
-    mount | grep "$MOUNT_POINT" || echo "Mount info not found"
+        # Then check inside any PVC directories under export
+        if [ -d "$NFS_MOUNT_POINT/export" ]; then
+            echo ""
+            echo "=== Checking PVC directories for Galaxy structure ==="
+
+            # Find all PVC directories
+            pvc_count=0
+            for pvc_dir in $(find "$NFS_MOUNT_POINT/export" -name "pvc-*" -type d 2>/dev/null); do
+                pvc_count=$((pvc_count + 1))
+                echo ""
+                echo "Checking PVC ($pvc_count): $(basename $pvc_dir)"
+                echo "  Full path: $pvc_dir"
 
-    # Try to find common Galaxy directories
-    echo ""
-    echo "=== Looking for Galaxy directories ==="
-    for dir in "database" "database/files" "database/objects" "tools" "shed_tools"; do
-        if [ -d "$MOUNT_POINT/$dir" ]; then
-            echo "✓ Found: $dir"
-            ls -la "$MOUNT_POINT/$dir" | head -10
+                # Show directory listing of PVC
+                echo "  Contents:"
+                ls -la "$pvc_dir" | head -10 | sed 's/^/    /'
+
+                # Check for Galaxy directories inside this PVC
+                galaxy_dirs_found=0
+                for dir in "jobs_directory" "shed_tools" "objects" "tools" "cache" "config" "deps" "tmp"; do
+                    if [ -d "$pvc_dir/$dir" ]; then
+                        echo "  ✓ Found Galaxy directory: $dir"
+                        # Show a sample of contents
+                        ls -la "$pvc_dir/$dir" 2>/dev/null | head -3 | sed 's/^/      /'
+                        galaxy_dirs_found=$((galaxy_dirs_found + 1))
+                    fi
+                done
+
+                # Check for Galaxy-specific files
+                galaxy_files_found=0
+                for file in "galaxy.yml" "universe_wsgi.ini" "config/galaxy.yml" "results.sqlite" "celery-beat-schedule"; do
+                    if [ -f "$pvc_dir/$file" ]; then
+                        echo "  ✓ Found Galaxy file: $file"
+                        galaxy_files_found=$((galaxy_files_found + 1))
+                    fi
+                done
+
+                total_indicators=$((galaxy_dirs_found + galaxy_files_found))
+                if [ $total_indicators -gt 0 ]; then
+                    echo "  🎯 This PVC contains $galaxy_dirs_found Galaxy directories and $galaxy_files_found Galaxy files"
+
+                    # Test write access
+                    test_file="$pvc_dir/.batch_test_file_$(date +%s)"
+                    if echo "test" > "$test_file" 2>/dev/null; then
+                        echo "  ✓ Write access confirmed"
+                        rm -f "$test_file" 2>/dev/null
+                    else
+                        echo "  ✗ No write access"
+                    fi
+
+                    # Test specific Galaxy directories access
+                    if [ -d "$pvc_dir/jobs_directory" ]; then
+                        echo "  � Jobs directory details:"
+                        du -sh "$pvc_dir/jobs_directory" 2>/dev/null | sed 's/^/      /' || echo "      Could not get size"
+                        job_count=$(find "$pvc_dir/jobs_directory" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l)
+                        echo "      Job subdirectories: $job_count"
+                    fi
+
+                    if [ -d "$pvc_dir/shed_tools" ]; then
+                        echo "  🔧 Shed tools directory details:"
+                        du -sh "$pvc_dir/shed_tools" 2>/dev/null | sed 's/^/      /' || echo "      Could not get size"
+                        tool_count=$(find "$pvc_dir/shed_tools" -name "*.py" -o -name "*.xml" 2>/dev/null | wc -l)
+                        echo "      Tool files (py/xml): $tool_count"
+                    fi
+                else
+                    echo "  ✗ No Galaxy directories or files found in this PVC"
+                fi
+            done
+
+            if [ $pvc_count -eq 0 ]; then
+                echo "✗ No PVC directories found in export"
+            else
+                echo ""
+                echo "📊 Summary: Found $pvc_count PVC directories in export"
+            fi
         else
-            echo "✗ Not found: $dir"
+            echo ""
+            echo "✗ No export directory found in NFS mount"
         fi
-    done
-
-    echo ""
-    echo "Unmounting NFS share..."
-    umount "$MOUNT_POINT" 2>/dev/null && echo "✓ Unmount successful" || echo "✗ Unmount failed"
+    else
+        echo "✗ NFS mount point exists but is not mounted"
+        echo "This suggests Batch volume configuration may be incorrect"
+    fi
 else
-    echo "✗ NFS mount failed"
-    echo "Mount error details above"
-
-    # Try alternative mount options
-    echo ""
-    echo "Trying alternative NFS mount options..."
-    echo "Command: mount -t nfs -o vers=4,tcp {target_host}:/ $MOUNT_POINT"
-    if mount -t nfs -o vers=4,tcp {target_host}:/ "$MOUNT_POINT" 2>&1; then
-        mount_result=0
-        echo "✓ NFS v4 mount successful!"
-        ls -la "$MOUNT_POINT" 2>/dev/null || echo "Could not list directory contents"
-        umount "$MOUNT_POINT" 2>/dev/null && echo "✓ Unmount successful" || echo "✗ Unmount failed"
-    else
-        echo "✗ NFS v4 mount also failed"
-    fi
+    echo "✗ NFS mount point $NFS_MOUNT_POINT does not exist"
+    echo "This suggests Batch volume was not configured"
 fi
 
 # CVMFS Mount Test
@@ -235,8 +319,8 @@
         ls -la "/cvmfs/data.galaxyproject.org" 2>/dev/null | head -10 || echo "Could not list directory contents"
 
         echo ""
-        echo "Checking for Galaxy reference data directories:"
-        for dir in "byhand" "location" "tool-data" "genomes"; do
+        echo "Listing Galaxy reference data directories:"
+        for dir in "byhand" "managed"; do
             if [ -d "/cvmfs/data.galaxyproject.org/$dir" ]; then
                 echo "✓ Found CVMFS directory: $dir"
                 ls "/cvmfs/data.galaxyproject.org/$dir" | head -5 2>/dev/null || echo "Could not list contents"
@@ -246,6 +330,22 @@
         done
 
         echo ""
+        echo "=== CVMFS File Access Test ==="
+        echo "Testing access to specific Galaxy reference file..."
+        echo "File: /cvmfs/data.galaxyproject.org/byhand/Arabidopsis_thaliana_TAIR10/seq/Arabidopsis_thaliana_TAIR10.fa.fai"
+
+        CVMFS_TEST_FILE="/cvmfs/data.galaxyproject.org/byhand/Arabidopsis_thaliana_TAIR10/seq/Arabidopsis_thaliana_TAIR10.fa.fai"
+        if [ -f "$CVMFS_TEST_FILE" ]; then
+            echo "✓ File exists, reading first 10 lines:"
+            head "$CVMFS_TEST_FILE" 2>/dev/null || echo "Could not read file contents"
+        else
+            echo "✗ File not found"
+            echo "Checking if parent directories exist:"
+            [ -d "/cvmfs/data.galaxyproject.org/byhand/Arabidopsis_thaliana_TAIR10" ] && echo "  ✓ Arabidopsis_thaliana_TAIR10 directory exists" || echo "  ✗ Arabidopsis_thaliana_TAIR10 directory missing"
+            [ -d "/cvmfs/data.galaxyproject.org/byhand/Arabidopsis_thaliana_TAIR10/seq" ] && echo "  ✓ seq directory exists" || echo "  ✗ seq directory missing"
+        fi
+
+        echo ""
         echo "CVMFS mount information from host:"
         mount | grep cvmfs || echo "CVMFS mount info not visible from container"
     else
@@ -261,6 +361,7 @@
     echo "Expected: /cvmfs from host VM bind-mounted into container"
 fi
 
+
 echo ""
 echo "=== Final Result ==="
 if [ $nc_result -eq 0 ] && [ $mount_result -eq 0 ]; then
@@ -277,7 +378,6 @@
     echo "This suggests:"
     echo "- NFS server is reachable but may not be properly configured"
     echo "- NFS export permissions may be incorrect"
-    echo "- NFS version mismatch (tried NFSv3 and NFSv4)"
     echo "- Firewall may allow port 2049 but block other NFS ports (111, 20048)"
     if [ $cvmfs_result -eq 0 ]; then
         echo "✓ CVMFS repository mount was successful"
@@ -305,17 +405,56 @@
 
     # Define the job using the Python client library objects
     logger.info("Building job specification...")
-    runnable = batch_v1.Runnable()
-    runnable.container = batch_v1.Runnable.Container()
-    runnable.container.image_uri = "afgane/gcp-batch-netcat:0.3.0"
+
+    # Escape the test script for use in docker command (outside f-string to avoid backslash issues)
+    escaped_test_script = test_script.replace("'", "'\"'\"'")
+
+    # Create a host script that triggers CVMFS mount and then runs the container
+    host_script = f'''#!/bin/bash
+set -e
+echo "=== Pre-Container Host Script ==="
+echo "Timestamp: $(date)"
+echo "Host VM Image: galaxy-k8s-boot-v2025-08-12"
+echo "Running on host before container starts..."
+echo ""
+
+echo "=== Triggering CVMFS Mount ==="
+echo "Checking CVMFS autofs status:"
+mount | grep cvmfs || echo "No CVMFS mounts yet"
+
+echo ""
+echo "Triggering CVMFS mount by accessing repository:"
+ls /cvmfs/data.galaxyproject.org/ || echo "Could not access CVMFS repository"
+
+echo ""
+echo "After access - checking CVMFS mounts:"
+mount | grep cvmfs || echo "Still no CVMFS mounts visible"
 
-    # Bind mount /cvmfs from the host VM (which has CVMFS client) into the container
-    # Use the docker-style volume syntax for bind mounting host paths
-    runnable.container.volumes = ["/cvmfs:/cvmfs:ro"]
+echo ""
+echo "Testing specific file access from host:"
+if [ -f "/cvmfs/data.galaxyproject.org/byhand/Arabidopsis_thaliana_TAIR10/seq/Arabidopsis_thaliana_TAIR10.fa.fai" ]; then
+    echo "✓ CVMFS file accessible from host"
+    head -3 "/cvmfs/data.galaxyproject.org/byhand/Arabidopsis_thaliana_TAIR10/seq/Arabidopsis_thaliana_TAIR10.fa.fai"
+else
+    echo "✗ CVMFS file not accessible from host"
+fi
+
+echo ""
+echo "=== Starting Container ==="
+echo "Running container with bind-mounted CVMFS and NFS..."
 
-    runnable.container.entrypoint = "/bin/bash"
-    runnable.container.commands = ["-c", test_script]
-    logger.debug(f"Container config: image={runnable.container.image_uri}, with /cvmfs bind mount from custom VM")
+# Run the container with the test script and volume mounts
+docker run --rm \\
+    -v /cvmfs:/cvmfs:ro \\
+    -v /mnt/nfs:/mnt/nfs:rw \\
+    afgane/gcp-batch-netcat:0.3.0 \\
+    /bin/bash -c '{escaped_test_script}'
+'''
+
+    runnable = batch_v1.Runnable()
+    runnable.script = batch_v1.Runnable.Script()
+    runnable.script.text = host_script
+    logger.debug(f"Host script configured to trigger CVMFS mount and run container")
 
     task = batch_v1.TaskSpec()
     task.runnables = [runnable]
@@ -324,6 +463,16 @@
     task.compute_resource.memory_mib = 1024
     logger.debug(f"Compute resources: CPU={task.compute_resource.cpu_milli}m, Memory={task.compute_resource.memory_mib}MiB")
 
+    # Configure NFS volume in the task
+    volume = batch_v1.Volume()
+    volume.nfs = batch_v1.NFS()
+    volume.nfs.server = target_host
+    volume.nfs.remote_path = "/"  # Root of the NFS export
+    volume.mount_path = "/mnt/nfs"
+
+    task.volumes = [volume]
+    logger.debug(f"NFS volume configured: {target_host}:/ -> /mnt/nfs")
+
     task_group = batch_v1.TaskGroup()
     task_group.task_count = 1
     task_group.parallelism = 1
@@ -344,7 +493,7 @@
     instance_policy = batch_v1.AllocationPolicy.InstancePolicy()
     instance_policy.machine_type = "e2-medium"  # Specify machine type for custom image
     instance_policy.boot_disk = batch_v1.AllocationPolicy.Disk()
-    instance_policy.boot_disk.image = f"projects/{project_id}/global/images/galaxy-k8s-boot-v2025-08-10"
+    instance_policy.boot_disk.image = f"projects/{project_id}/global/images/galaxy-k8s-boot-v2025-08-12"
     instance_policy.boot_disk.size_gb = 99
     logger.debug(f"Using custom VM image: {instance_policy.boot_disk.image}")