comparison gcp_batch_netcat.py @ 11:fe0bf22037a5 draft

planemo upload for repository https://github.com/afgane/gcp_batch_netcat commit f730cbb207e028a5d4fd982fe65ece7345af4879

10:cdfa2e1a7ef4

set -e
echo "=== GCP Batch NFS Connectivity Test ==="
echo "Target: {target_host}:{target_port}"
echo "Timestamp: $(date)"
echo "Container hostname: $(hostname)"
echo "Host VM Image: galaxy-k8s-boot-v2025-08-10"
echo "Container Image: afgane/gcp-batch-netcat:0.3.0"
echo ""

# Basic system info
echo "=== System Information ==="

echo "=== Network Troubleshooting ==="
echo "Route table:"
ip route
echo ""

# NFS Mount Test
echo "=== NFS Mount Test ==="
MOUNT_POINT="/tmp/nfs_test_mount"
echo "Creating mount point: $MOUNT_POINT"
mkdir -p "$MOUNT_POINT"

echo "Attempting to mount NFS share..."
echo "Command: mount -t nfs -o vers=3,tcp {target_host}:/ $MOUNT_POINT"

# Try mounting the NFS share
mount_result=1
if mount -t nfs -o vers=3,tcp {target_host}:/ "$MOUNT_POINT" 2>&1; then
    mount_result=0
    echo "✓ NFS mount successful!"

    echo ""
    echo "=== NFS Share Contents ==="
    echo "Long listing of NFS share root:"
    ls -la "$MOUNT_POINT" 2>/dev/null || echo "Could not list directory contents"

    echo ""
    echo "Disk usage of NFS share:"
    df -h "$MOUNT_POINT" 2>/dev/null || echo "Could not get disk usage"

    echo ""
    echo "Mount information:"
    mount | grep "$MOUNT_POINT" || echo "Mount info not found"

    # Try to find common Galaxy directories
    echo ""
    echo "=== Looking for Galaxy directories ==="
    for dir in "database" "database/files" "database/objects" "tools" "shed_tools"; do
        if [ -d "$MOUNT_POINT/$dir" ]; then
            echo "✓ Found: $dir"
            ls -la "$MOUNT_POINT/$dir" | head -10
        else
            echo "✗ Not found: $dir"
        fi
    done

    echo ""
    echo "Unmounting NFS share..."
    umount "$MOUNT_POINT" 2>/dev/null && echo "✓ Unmount successful" || echo "✗ Unmount failed"
else
    echo "✗ NFS mount failed"
    echo "Mount error details above"

    # Try alternative mount options
    echo ""
    echo "Trying alternative NFS mount options..."
    echo "Command: mount -t nfs -o vers=4,tcp {target_host}:/ $MOUNT_POINT"
    if mount -t nfs -o vers=4,tcp {target_host}:/ "$MOUNT_POINT" 2>&1; then
        mount_result=0
        echo "✓ NFS v4 mount successful!"
        ls -la "$MOUNT_POINT" 2>/dev/null || echo "Could not list directory contents"
        umount "$MOUNT_POINT" 2>/dev/null && echo "✓ Unmount successful" || echo "✗ Unmount failed"
    else
        echo "✗ NFS v4 mount also failed"
    fi
fi

# CVMFS Mount Test
echo ""
echo "=== CVMFS Access Test ==="

        echo "=== CVMFS Repository Contents ==="
        echo "Long listing of CVMFS repository root:"
        ls -la "/cvmfs/data.galaxyproject.org" 2>/dev/null | head -10 || echo "Could not list directory contents"

        echo ""
        echo "Checking for Galaxy reference data directories:"
        for dir in "byhand" "location" "tool-data" "genomes"; do
            if [ -d "/cvmfs/data.galaxyproject.org/$dir" ]; then
                echo "✓ Found CVMFS directory: $dir"
                ls "/cvmfs/data.galaxyproject.org/$dir" | head -5 2>/dev/null || echo "Could not list contents"
            else
                echo "✗ Not found: $dir"
            fi
        done

        echo ""
        echo "CVMFS mount information from host:"
        mount | grep cvmfs || echo "CVMFS mount info not visible from container"
    else

else
    echo "✗ /cvmfs directory not found"
    echo "This indicates the bind mount from host VM failed"
    echo "Expected: /cvmfs from host VM bind-mounted into container"
fi

echo ""
echo "=== Final Result ==="
if [ $nc_result -eq 0 ] && [ $mount_result -eq 0 ]; then
    echo "✓ SUCCESS: Both network connectivity and NFS mount to {target_host}:{target_port} successful"

    echo "⚠ PARTIAL SUCCESS: Network connectivity successful but NFS mount failed"
    echo "Network connection to {target_host}:{target_port} works, but NFS mounting failed."
    echo "This suggests:"
    echo "- NFS server is reachable but may not be properly configured"
    echo "- NFS export permissions may be incorrect"
    echo "- NFS version mismatch (tried NFSv3 and NFSv4)"
    echo "- Firewall may allow port 2049 but block other NFS ports (111, 20048)"
    if [ $cvmfs_result -eq 0 ]; then
        echo "✓ CVMFS repository mount was successful"
    fi
    exit 1

fi
'''

    # Define the job using the Python client library objects
    logger.info("Building job specification...")
    runnable = batch_v1.Runnable()
    runnable.container = batch_v1.Runnable.Container()
    runnable.container.image_uri = "afgane/gcp-batch-netcat:0.3.0"

    # Bind mount /cvmfs from the host VM (which has CVMFS client) into the container
    # Use the docker-style volume syntax for bind mounting host paths
    runnable.container.volumes = ["/cvmfs:/cvmfs:ro"]

    runnable.container.entrypoint = "/bin/bash"
    runnable.container.commands = ["-c", test_script]
    logger.debug(f"Container config: image={runnable.container.image_uri}, with /cvmfs bind mount from custom VM")

    task = batch_v1.TaskSpec()
    task.runnables = [runnable]
    task.compute_resource = batch_v1.ComputeResource()
    task.compute_resource.cpu_milli = 1000
    task.compute_resource.memory_mib = 1024
    logger.debug(f"Compute resources: CPU={task.compute_resource.cpu_milli}m, Memory={task.compute_resource.memory_mib}MiB")

    task_group = batch_v1.TaskGroup()
    task_group.task_count = 1
    task_group.parallelism = 1
    task_group.task_spec = task
    logger.debug(f"Task group: count={task_group.task_count}, parallelism={task_group.parallelism}")

    # Instance policy with custom VM image
    instance_policy = batch_v1.AllocationPolicy.InstancePolicy()
    instance_policy.machine_type = "e2-medium"  # Specify machine type for custom image
    instance_policy.boot_disk = batch_v1.AllocationPolicy.Disk()
    instance_policy.boot_disk.image = f"projects/{project_id}/global/images/galaxy-k8s-boot-v2025-08-10"
    instance_policy.boot_disk.size_gb = 99
    logger.debug(f"Using custom VM image: {instance_policy.boot_disk.image}")

    # Wrap the instance policy in InstancePolicyOrTemplate
    instance_policy_or_template = batch_v1.AllocationPolicy.InstancePolicyOrTemplate()

11:fe0bf22037a5

set -e
echo "=== GCP Batch NFS Connectivity Test ==="
echo "Target: {target_host}:{target_port}"
echo "Timestamp: $(date)"
echo "Container hostname: $(hostname)"
echo "Host VM Image: galaxy-k8s-boot-v2025-08-12"
echo "Container Image: afgane/gcp-batch-netcat:0.3.0"
echo ""

# Basic system info
echo "=== System Information ==="

echo "=== Network Troubleshooting ==="
echo "Route table:"
ip route
echo ""

# NFS Mount Test - Check if Batch mounted it for us
echo "=== NFS Mount Test (via Batch Volume) ==="
NFS_MOUNT_POINT="/mnt/nfs"
mount_result=1

echo "Checking if NFS is mounted by Batch at $NFS_MOUNT_POINT..."
if [ -d "$NFS_MOUNT_POINT" ]; then
    echo "✓ NFS mount point exists"

    # Check if it's actually mounted
    if mount | grep "$NFS_MOUNT_POINT"; then
        mount_result=0
        echo "✓ NFS mounted by Batch successfully!"

        echo ""
        echo "=== NFS Share Contents ==="
        echo "Long listing of NFS share:"
        ls -la "$NFS_MOUNT_POINT" 2>/dev/null || echo "Could not list directory contents"

        echo ""
        echo "Disk usage of NFS share:"
        df -h "$NFS_MOUNT_POINT" 2>/dev/null || echo "Could not get disk usage"

        # Look for export subdirectories
        echo ""
        echo "=== Looking for export directories ==="
        if [ -d "$NFS_MOUNT_POINT/export" ]; then
            echo "✓ Found: export directory"
            ls -la "$NFS_MOUNT_POINT/export" | head -10 2>/dev/null || echo "Could not list export contents"

            # Look for PVC subdirectories
            echo "Looking for PVC directories in export..."
            find "$NFS_MOUNT_POINT/export" -name "pvc-*" -type d | head -5 2>/dev/null || echo "No PVC directories found"
        else
            echo "✗ No export directory found"
        fi

        # Try to find common Galaxy directories
        echo ""
        echo "=== Looking for Galaxy directories ==="

        # First check if they exist directly in the NFS root
        galaxy_dirs_in_root=0
        for dir in "jobs_directory" "shed_tools" "objects" "tools" "cache" "config"; do
            if [ -d "$NFS_MOUNT_POINT/$dir" ]; then
                echo "✓ Found in root: $dir"
                ls -la "$NFS_MOUNT_POINT/$dir" | head -5
                galaxy_dirs_in_root=$((galaxy_dirs_in_root + 1))
            fi
        done

        if [ $galaxy_dirs_in_root -eq 0 ]; then
            echo "✗ No Galaxy directories found in NFS root"
        else
            echo "✓ Found $galaxy_dirs_in_root Galaxy directories in NFS root"
        fi

        # Then check inside any PVC directories under export
        if [ -d "$NFS_MOUNT_POINT/export" ]; then
            echo ""
            echo "=== Checking PVC directories for Galaxy structure ==="

            # Find all PVC directories
            pvc_count=0
            for pvc_dir in $(find "$NFS_MOUNT_POINT/export" -name "pvc-*" -type d 2>/dev/null); do
                pvc_count=$((pvc_count + 1))
                echo ""
                echo "Checking PVC ($pvc_count): $(basename $pvc_dir)"
                echo "  Full path: $pvc_dir"

                # Show directory listing of PVC
                echo "  Contents:"
                ls -la "$pvc_dir" | head -10 | sed 's/^/    /'

                # Check for Galaxy directories inside this PVC
                galaxy_dirs_found=0
                for dir in "jobs_directory" "shed_tools" "objects" "tools" "cache" "config" "deps" "tmp"; do
                    if [ -d "$pvc_dir/$dir" ]; then
                        echo "  ✓ Found Galaxy directory: $dir"
                        # Show a sample of contents
                        ls -la "$pvc_dir/$dir" 2>/dev/null | head -3 | sed 's/^/      /'
                        galaxy_dirs_found=$((galaxy_dirs_found + 1))
                    fi
                done

                # Check for Galaxy-specific files
                galaxy_files_found=0
                for file in "galaxy.yml" "universe_wsgi.ini" "config/galaxy.yml" "results.sqlite" "celery-beat-schedule"; do
                    if [ -f "$pvc_dir/$file" ]; then
                        echo "  ✓ Found Galaxy file: $file"
                        galaxy_files_found=$((galaxy_files_found + 1))
                    fi
                done

                total_indicators=$((galaxy_dirs_found + galaxy_files_found))
                if [ $total_indicators -gt 0 ]; then
                    echo "  🎯 This PVC contains $galaxy_dirs_found Galaxy directories and $galaxy_files_found Galaxy files"

                    # Test write access
                    test_file="$pvc_dir/.batch_test_file_$(date +%s)"
                    if echo "test" > "$test_file" 2>/dev/null; then
                        echo "  ✓ Write access confirmed"
                        rm -f "$test_file" 2>/dev/null
                    else
                        echo "  ✗ No write access"
                    fi

                    # Test specific Galaxy directories access
                    if [ -d "$pvc_dir/jobs_directory" ]; then
                        echo "  � Jobs directory details:"
                        du -sh "$pvc_dir/jobs_directory" 2>/dev/null | sed 's/^/      /' || echo "      Could not get size"
                        job_count=$(find "$pvc_dir/jobs_directory" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l)
                        echo "      Job subdirectories: $job_count"
                    fi

                    if [ -d "$pvc_dir/shed_tools" ]; then
                        echo "  🔧 Shed tools directory details:"
                        du -sh "$pvc_dir/shed_tools" 2>/dev/null | sed 's/^/      /' || echo "      Could not get size"
                        tool_count=$(find "$pvc_dir/shed_tools" -name "*.py" -o -name "*.xml" 2>/dev/null | wc -l)
                        echo "      Tool files (py/xml): $tool_count"
                    fi
                else
                    echo "  ✗ No Galaxy directories or files found in this PVC"
                fi
            done

            if [ $pvc_count -eq 0 ]; then
                echo "✗ No PVC directories found in export"
            else
                echo ""
                echo "📊 Summary: Found $pvc_count PVC directories in export"
            fi
        else
            echo ""
            echo "✗ No export directory found in NFS mount"
        fi
    else
        echo "✗ NFS mount point exists but is not mounted"
        echo "This suggests Batch volume configuration may be incorrect"
    fi
else
    echo "✗ NFS mount point $NFS_MOUNT_POINT does not exist"
    echo "This suggests Batch volume was not configured"
fi

# CVMFS Mount Test
echo ""
echo "=== CVMFS Access Test ==="

        echo "=== CVMFS Repository Contents ==="
        echo "Long listing of CVMFS repository root:"
        ls -la "/cvmfs/data.galaxyproject.org" 2>/dev/null | head -10 || echo "Could not list directory contents"

        echo ""
        echo "Listing Galaxy reference data directories:"
        for dir in "byhand" "managed"; do
            if [ -d "/cvmfs/data.galaxyproject.org/$dir" ]; then
                echo "✓ Found CVMFS directory: $dir"
                ls "/cvmfs/data.galaxyproject.org/$dir" | head -5 2>/dev/null || echo "Could not list contents"
            else
                echo "✗ Not found: $dir"
            fi
        done

        echo ""
        echo "=== CVMFS File Access Test ==="
        echo "Testing access to specific Galaxy reference file..."
        echo "File: /cvmfs/data.galaxyproject.org/byhand/Arabidopsis_thaliana_TAIR10/seq/Arabidopsis_thaliana_TAIR10.fa.fai"

        CVMFS_TEST_FILE="/cvmfs/data.galaxyproject.org/byhand/Arabidopsis_thaliana_TAIR10/seq/Arabidopsis_thaliana_TAIR10.fa.fai"
        if [ -f "$CVMFS_TEST_FILE" ]; then
            echo "✓ File exists, reading first 10 lines:"
            head "$CVMFS_TEST_FILE" 2>/dev/null || echo "Could not read file contents"
        else
            echo "✗ File not found"
            echo "Checking if parent directories exist:"
            [ -d "/cvmfs/data.galaxyproject.org/byhand/Arabidopsis_thaliana_TAIR10" ] && echo "  ✓ Arabidopsis_thaliana_TAIR10 directory exists" || echo "  ✗ Arabidopsis_thaliana_TAIR10 directory missing"
            [ -d "/cvmfs/data.galaxyproject.org/byhand/Arabidopsis_thaliana_TAIR10/seq" ] && echo "  ✓ seq directory exists" || echo "  ✗ seq directory missing"
        fi

        echo ""
        echo "CVMFS mount information from host:"
        mount | grep cvmfs || echo "CVMFS mount info not visible from container"
    else

else
    echo "✗ /cvmfs directory not found"
    echo "This indicates the bind mount from host VM failed"
    echo "Expected: /cvmfs from host VM bind-mounted into container"
fi


echo ""
echo "=== Final Result ==="
if [ $nc_result -eq 0 ] && [ $mount_result -eq 0 ]; then
    echo "✓ SUCCESS: Both network connectivity and NFS mount to {target_host}:{target_port} successful"

    echo "⚠ PARTIAL SUCCESS: Network connectivity successful but NFS mount failed"
    echo "Network connection to {target_host}:{target_port} works, but NFS mounting failed."
    echo "This suggests:"
    echo "- NFS server is reachable but may not be properly configured"
    echo "- NFS export permissions may be incorrect"
    echo "- Firewall may allow port 2049 but block other NFS ports (111, 20048)"
    if [ $cvmfs_result -eq 0 ]; then
        echo "✓ CVMFS repository mount was successful"
    fi
    exit 1

fi
'''

    # Define the job using the Python client library objects
    logger.info("Building job specification...")

    # Escape the test script for use in docker command (outside f-string to avoid backslash issues)
    escaped_test_script = test_script.replace("'", "'\"'\"'")

    # Create a host script that triggers CVMFS mount and then runs the container
    host_script = f'''#!/bin/bash
set -e
echo "=== Pre-Container Host Script ==="
echo "Timestamp: $(date)"
echo "Host VM Image: galaxy-k8s-boot-v2025-08-12"
echo "Running on host before container starts..."
echo ""

echo "=== Triggering CVMFS Mount ==="
echo "Checking CVMFS autofs status:"
mount | grep cvmfs || echo "No CVMFS mounts yet"

echo ""
echo "Triggering CVMFS mount by accessing repository:"
ls /cvmfs/data.galaxyproject.org/ || echo "Could not access CVMFS repository"

echo ""
echo "After access - checking CVMFS mounts:"
mount | grep cvmfs || echo "Still no CVMFS mounts visible"

echo ""
echo "Testing specific file access from host:"
if [ -f "/cvmfs/data.galaxyproject.org/byhand/Arabidopsis_thaliana_TAIR10/seq/Arabidopsis_thaliana_TAIR10.fa.fai" ]; then
    echo "✓ CVMFS file accessible from host"
    head -3 "/cvmfs/data.galaxyproject.org/byhand/Arabidopsis_thaliana_TAIR10/seq/Arabidopsis_thaliana_TAIR10.fa.fai"
else
    echo "✗ CVMFS file not accessible from host"
fi

echo ""
echo "=== Starting Container ==="
echo "Running container with bind-mounted CVMFS and NFS..."

# Run the container with the test script and volume mounts
docker run --rm \\
    -v /cvmfs:/cvmfs:ro \\
    -v /mnt/nfs:/mnt/nfs:rw \\
    afgane/gcp-batch-netcat:0.3.0 \\
    /bin/bash -c '{escaped_test_script}'
'''

    runnable = batch_v1.Runnable()
    runnable.script = batch_v1.Runnable.Script()
    runnable.script.text = host_script
    logger.debug(f"Host script configured to trigger CVMFS mount and run container")

    task = batch_v1.TaskSpec()
    task.runnables = [runnable]
    task.compute_resource = batch_v1.ComputeResource()
    task.compute_resource.cpu_milli = 1000
    task.compute_resource.memory_mib = 1024
    logger.debug(f"Compute resources: CPU={task.compute_resource.cpu_milli}m, Memory={task.compute_resource.memory_mib}MiB")

    # Configure NFS volume in the task
    volume = batch_v1.Volume()
    volume.nfs = batch_v1.NFS()
    volume.nfs.server = target_host
    volume.nfs.remote_path = "/"  # Root of the NFS export
    volume.mount_path = "/mnt/nfs"

    task.volumes = [volume]
    logger.debug(f"NFS volume configured: {target_host}:/ -> /mnt/nfs")

    task_group = batch_v1.TaskGroup()
    task_group.task_count = 1
    task_group.parallelism = 1
    task_group.task_spec = task
    logger.debug(f"Task group: count={task_group.task_count}, parallelism={task_group.parallelism}")

    # Instance policy with custom VM image
    instance_policy = batch_v1.AllocationPolicy.InstancePolicy()
    instance_policy.machine_type = "e2-medium"  # Specify machine type for custom image
    instance_policy.boot_disk = batch_v1.AllocationPolicy.Disk()
    instance_policy.boot_disk.image = f"projects/{project_id}/global/images/galaxy-k8s-boot-v2025-08-12"
    instance_policy.boot_disk.size_gb = 99
    logger.debug(f"Using custom VM image: {instance_policy.boot_disk.image}")

    # Wrap the instance policy in InstancePolicyOrTemplate
    instance_policy_or_template = batch_v1.AllocationPolicy.InstancePolicyOrTemplate()