Mercurial > repos > enis > gcp_batch_netcat
diff gcp_batch_netcat.py @ 11:fe0bf22037a5 draft
planemo upload for repository https://github.com/afgane/gcp_batch_netcat commit f730cbb207e028a5d4fd982fe65ece7345af4879
author | enis |
---|---|
date | Thu, 14 Aug 2025 16:39:36 +0000 |
parents | cdfa2e1a7ef4 |
children |
line wrap: on
line diff
--- a/gcp_batch_netcat.py Tue Aug 12 16:22:41 2025 +0000 +++ b/gcp_batch_netcat.py Thu Aug 14 16:39:36 2025 +0000 @@ -88,7 +88,7 @@ echo "Target: {target_host}:{target_port}" echo "Timestamp: $(date)" echo "Container hostname: $(hostname)" -echo "Host VM Image: galaxy-k8s-boot-v2025-08-10" +echo "Host VM Image: galaxy-k8s-boot-v2025-08-12" echo "Container Image: afgane/gcp-batch-netcat:0.3.0" echo "" @@ -153,65 +153,149 @@ ip route echo "" -# NFS Mount Test -echo "=== NFS Mount Test ===" -MOUNT_POINT="/tmp/nfs_test_mount" -echo "Creating mount point: $MOUNT_POINT" -mkdir -p "$MOUNT_POINT" - -echo "Attempting to mount NFS share..." -echo "Command: mount -t nfs -o vers=3,tcp {target_host}:/ $MOUNT_POINT" - -# Try mounting the NFS share +# NFS Mount Test - Check if Batch mounted it for us +echo "=== NFS Mount Test (via Batch Volume) ===" +NFS_MOUNT_POINT="/mnt/nfs" mount_result=1 -if mount -t nfs -o vers=3,tcp {target_host}:/ "$MOUNT_POINT" 2>&1; then - mount_result=0 - echo "✓ NFS mount successful!" + +echo "Checking if NFS is mounted by Batch at $NFS_MOUNT_POINT..." +if [ -d "$NFS_MOUNT_POINT" ]; then + echo "✓ NFS mount point exists" + + # Check if it's actually mounted + if mount | grep "$NFS_MOUNT_POINT"; then + mount_result=0 + echo "✓ NFS mounted by Batch successfully!" + + echo "" + echo "=== NFS Share Contents ===" + echo "Long listing of NFS share:" + ls -la "$NFS_MOUNT_POINT" 2>/dev/null || echo "Could not list directory contents" + + echo "" + echo "Disk usage of NFS share:" + df -h "$NFS_MOUNT_POINT" 2>/dev/null || echo "Could not get disk usage" + + # Look for export subdirectories + echo "" + echo "=== Looking for export directories ===" + if [ -d "$NFS_MOUNT_POINT/export" ]; then + echo "✓ Found: export directory" + ls -la "$NFS_MOUNT_POINT/export" | head -10 2>/dev/null || echo "Could not list export contents" + + # Look for PVC subdirectories + echo "Looking for PVC directories in export..." + find "$NFS_MOUNT_POINT/export" -name "pvc-*" -type d | head -5 2>/dev/null || echo "No PVC directories found" + else + echo "✗ No export directory found" + fi - echo "" - echo "=== NFS Share Contents ===" - echo "Long listing of NFS share root:" - ls -la "$MOUNT_POINT" 2>/dev/null || echo "Could not list directory contents" + # Try to find common Galaxy directories + echo "" + echo "=== Looking for Galaxy directories ===" + + # First check if they exist directly in the NFS root + galaxy_dirs_in_root=0 + for dir in "jobs_directory" "shed_tools" "objects" "tools" "cache" "config"; do + if [ -d "$NFS_MOUNT_POINT/$dir" ]; then + echo "✓ Found in root: $dir" + ls -la "$NFS_MOUNT_POINT/$dir" | head -5 + galaxy_dirs_in_root=$((galaxy_dirs_in_root + 1)) + fi + done - echo "" - echo "Disk usage of NFS share:" - df -h "$MOUNT_POINT" 2>/dev/null || echo "Could not get disk usage" + if [ $galaxy_dirs_in_root -eq 0 ]; then + echo "✗ No Galaxy directories found in NFS root" + else + echo "✓ Found $galaxy_dirs_in_root Galaxy directories in NFS root" + fi - echo "" - echo "Mount information:" - mount | grep "$MOUNT_POINT" || echo "Mount info not found" + # Then check inside any PVC directories under export + if [ -d "$NFS_MOUNT_POINT/export" ]; then + echo "" + echo "=== Checking PVC directories for Galaxy structure ===" + + # Find all PVC directories + pvc_count=0 + for pvc_dir in $(find "$NFS_MOUNT_POINT/export" -name "pvc-*" -type d 2>/dev/null); do + pvc_count=$((pvc_count + 1)) + echo "" + echo "Checking PVC ($pvc_count): $(basename $pvc_dir)" + echo " Full path: $pvc_dir" - # Try to find common Galaxy directories - echo "" - echo "=== Looking for Galaxy directories ===" - for dir in "database" "database/files" "database/objects" "tools" "shed_tools"; do - if [ -d "$MOUNT_POINT/$dir" ]; then - echo "✓ Found: $dir" - ls -la "$MOUNT_POINT/$dir" | head -10 + # Show directory listing of PVC + echo " Contents:" + ls -la "$pvc_dir" | head -10 | sed 's/^/ /' + + # Check for Galaxy directories inside this PVC + galaxy_dirs_found=0 + for dir in "jobs_directory" "shed_tools" "objects" "tools" "cache" "config" "deps" "tmp"; do + if [ -d "$pvc_dir/$dir" ]; then + echo " ✓ Found Galaxy directory: $dir" + # Show a sample of contents + ls -la "$pvc_dir/$dir" 2>/dev/null | head -3 | sed 's/^/ /' + galaxy_dirs_found=$((galaxy_dirs_found + 1)) + fi + done + + # Check for Galaxy-specific files + galaxy_files_found=0 + for file in "galaxy.yml" "universe_wsgi.ini" "config/galaxy.yml" "results.sqlite" "celery-beat-schedule"; do + if [ -f "$pvc_dir/$file" ]; then + echo " ✓ Found Galaxy file: $file" + galaxy_files_found=$((galaxy_files_found + 1)) + fi + done + + total_indicators=$((galaxy_dirs_found + galaxy_files_found)) + if [ $total_indicators -gt 0 ]; then + echo " 🎯 This PVC contains $galaxy_dirs_found Galaxy directories and $galaxy_files_found Galaxy files" + + # Test write access + test_file="$pvc_dir/.batch_test_file_$(date +%s)" + if echo "test" > "$test_file" 2>/dev/null; then + echo " ✓ Write access confirmed" + rm -f "$test_file" 2>/dev/null + else + echo " ✗ No write access" + fi + + # Test specific Galaxy directories access + if [ -d "$pvc_dir/jobs_directory" ]; then + echo " � Jobs directory details:" + du -sh "$pvc_dir/jobs_directory" 2>/dev/null | sed 's/^/ /' || echo " Could not get size" + job_count=$(find "$pvc_dir/jobs_directory" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l) + echo " Job subdirectories: $job_count" + fi + + if [ -d "$pvc_dir/shed_tools" ]; then + echo " 🔧 Shed tools directory details:" + du -sh "$pvc_dir/shed_tools" 2>/dev/null | sed 's/^/ /' || echo " Could not get size" + tool_count=$(find "$pvc_dir/shed_tools" -name "*.py" -o -name "*.xml" 2>/dev/null | wc -l) + echo " Tool files (py/xml): $tool_count" + fi + else + echo " ✗ No Galaxy directories or files found in this PVC" + fi + done + + if [ $pvc_count -eq 0 ]; then + echo "✗ No PVC directories found in export" + else + echo "" + echo "📊 Summary: Found $pvc_count PVC directories in export" + fi else - echo "✗ Not found: $dir" + echo "" + echo "✗ No export directory found in NFS mount" fi - done - - echo "" - echo "Unmounting NFS share..." - umount "$MOUNT_POINT" 2>/dev/null && echo "✓ Unmount successful" || echo "✗ Unmount failed" + else + echo "✗ NFS mount point exists but is not mounted" + echo "This suggests Batch volume configuration may be incorrect" + fi else - echo "✗ NFS mount failed" - echo "Mount error details above" - - # Try alternative mount options - echo "" - echo "Trying alternative NFS mount options..." - echo "Command: mount -t nfs -o vers=4,tcp {target_host}:/ $MOUNT_POINT" - if mount -t nfs -o vers=4,tcp {target_host}:/ "$MOUNT_POINT" 2>&1; then - mount_result=0 - echo "✓ NFS v4 mount successful!" - ls -la "$MOUNT_POINT" 2>/dev/null || echo "Could not list directory contents" - umount "$MOUNT_POINT" 2>/dev/null && echo "✓ Unmount successful" || echo "✗ Unmount failed" - else - echo "✗ NFS v4 mount also failed" - fi + echo "✗ NFS mount point $NFS_MOUNT_POINT does not exist" + echo "This suggests Batch volume was not configured" fi # CVMFS Mount Test @@ -235,8 +319,8 @@ ls -la "/cvmfs/data.galaxyproject.org" 2>/dev/null | head -10 || echo "Could not list directory contents" echo "" - echo "Checking for Galaxy reference data directories:" - for dir in "byhand" "location" "tool-data" "genomes"; do + echo "Listing Galaxy reference data directories:" + for dir in "byhand" "managed"; do if [ -d "/cvmfs/data.galaxyproject.org/$dir" ]; then echo "✓ Found CVMFS directory: $dir" ls "/cvmfs/data.galaxyproject.org/$dir" | head -5 2>/dev/null || echo "Could not list contents" @@ -246,6 +330,22 @@ done echo "" + echo "=== CVMFS File Access Test ===" + echo "Testing access to specific Galaxy reference file..." + echo "File: /cvmfs/data.galaxyproject.org/byhand/Arabidopsis_thaliana_TAIR10/seq/Arabidopsis_thaliana_TAIR10.fa.fai" + + CVMFS_TEST_FILE="/cvmfs/data.galaxyproject.org/byhand/Arabidopsis_thaliana_TAIR10/seq/Arabidopsis_thaliana_TAIR10.fa.fai" + if [ -f "$CVMFS_TEST_FILE" ]; then + echo "✓ File exists, reading first 10 lines:" + head "$CVMFS_TEST_FILE" 2>/dev/null || echo "Could not read file contents" + else + echo "✗ File not found" + echo "Checking if parent directories exist:" + [ -d "/cvmfs/data.galaxyproject.org/byhand/Arabidopsis_thaliana_TAIR10" ] && echo " ✓ Arabidopsis_thaliana_TAIR10 directory exists" || echo " ✗ Arabidopsis_thaliana_TAIR10 directory missing" + [ -d "/cvmfs/data.galaxyproject.org/byhand/Arabidopsis_thaliana_TAIR10/seq" ] && echo " ✓ seq directory exists" || echo " ✗ seq directory missing" + fi + + echo "" echo "CVMFS mount information from host:" mount | grep cvmfs || echo "CVMFS mount info not visible from container" else @@ -261,6 +361,7 @@ echo "Expected: /cvmfs from host VM bind-mounted into container" fi + echo "" echo "=== Final Result ===" if [ $nc_result -eq 0 ] && [ $mount_result -eq 0 ]; then @@ -277,7 +378,6 @@ echo "This suggests:" echo "- NFS server is reachable but may not be properly configured" echo "- NFS export permissions may be incorrect" - echo "- NFS version mismatch (tried NFSv3 and NFSv4)" echo "- Firewall may allow port 2049 but block other NFS ports (111, 20048)" if [ $cvmfs_result -eq 0 ]; then echo "✓ CVMFS repository mount was successful" @@ -305,17 +405,56 @@ # Define the job using the Python client library objects logger.info("Building job specification...") - runnable = batch_v1.Runnable() - runnable.container = batch_v1.Runnable.Container() - runnable.container.image_uri = "afgane/gcp-batch-netcat:0.3.0" + + # Escape the test script for use in docker command (outside f-string to avoid backslash issues) + escaped_test_script = test_script.replace("'", "'\"'\"'") + + # Create a host script that triggers CVMFS mount and then runs the container + host_script = f'''#!/bin/bash +set -e +echo "=== Pre-Container Host Script ===" +echo "Timestamp: $(date)" +echo "Host VM Image: galaxy-k8s-boot-v2025-08-12" +echo "Running on host before container starts..." +echo "" + +echo "=== Triggering CVMFS Mount ===" +echo "Checking CVMFS autofs status:" +mount | grep cvmfs || echo "No CVMFS mounts yet" + +echo "" +echo "Triggering CVMFS mount by accessing repository:" +ls /cvmfs/data.galaxyproject.org/ || echo "Could not access CVMFS repository" + +echo "" +echo "After access - checking CVMFS mounts:" +mount | grep cvmfs || echo "Still no CVMFS mounts visible" - # Bind mount /cvmfs from the host VM (which has CVMFS client) into the container - # Use the docker-style volume syntax for bind mounting host paths - runnable.container.volumes = ["/cvmfs:/cvmfs:ro"] +echo "" +echo "Testing specific file access from host:" +if [ -f "/cvmfs/data.galaxyproject.org/byhand/Arabidopsis_thaliana_TAIR10/seq/Arabidopsis_thaliana_TAIR10.fa.fai" ]; then + echo "✓ CVMFS file accessible from host" + head -3 "/cvmfs/data.galaxyproject.org/byhand/Arabidopsis_thaliana_TAIR10/seq/Arabidopsis_thaliana_TAIR10.fa.fai" +else + echo "✗ CVMFS file not accessible from host" +fi + +echo "" +echo "=== Starting Container ===" +echo "Running container with bind-mounted CVMFS and NFS..." - runnable.container.entrypoint = "/bin/bash" - runnable.container.commands = ["-c", test_script] - logger.debug(f"Container config: image={runnable.container.image_uri}, with /cvmfs bind mount from custom VM") +# Run the container with the test script and volume mounts +docker run --rm \\ + -v /cvmfs:/cvmfs:ro \\ + -v /mnt/nfs:/mnt/nfs:rw \\ + afgane/gcp-batch-netcat:0.3.0 \\ + /bin/bash -c '{escaped_test_script}' +''' + + runnable = batch_v1.Runnable() + runnable.script = batch_v1.Runnable.Script() + runnable.script.text = host_script + logger.debug(f"Host script configured to trigger CVMFS mount and run container") task = batch_v1.TaskSpec() task.runnables = [runnable] @@ -324,6 +463,16 @@ task.compute_resource.memory_mib = 1024 logger.debug(f"Compute resources: CPU={task.compute_resource.cpu_milli}m, Memory={task.compute_resource.memory_mib}MiB") + # Configure NFS volume in the task + volume = batch_v1.Volume() + volume.nfs = batch_v1.NFS() + volume.nfs.server = target_host + volume.nfs.remote_path = "/" # Root of the NFS export + volume.mount_path = "/mnt/nfs" + + task.volumes = [volume] + logger.debug(f"NFS volume configured: {target_host}:/ -> /mnt/nfs") + task_group = batch_v1.TaskGroup() task_group.task_count = 1 task_group.parallelism = 1 @@ -344,7 +493,7 @@ instance_policy = batch_v1.AllocationPolicy.InstancePolicy() instance_policy.machine_type = "e2-medium" # Specify machine type for custom image instance_policy.boot_disk = batch_v1.AllocationPolicy.Disk() - instance_policy.boot_disk.image = f"projects/{project_id}/global/images/galaxy-k8s-boot-v2025-08-10" + instance_policy.boot_disk.image = f"projects/{project_id}/global/images/galaxy-k8s-boot-v2025-08-12" instance_policy.boot_disk.size_gb = 99 logger.debug(f"Using custom VM image: {instance_policy.boot_disk.image}")