Mercurial > repos > ketan > swift
view swift-hello @ 1:7aa6ee21028c draft
Uploaded
author | ketan |
---|---|
date | Tue, 15 Oct 2013 16:22:01 -0400 |
parents | |
children |
line wrap: on
line source
#!/bin/bash site=$1 n=$2 outdir=$3 logfile=$4 htmlfile=$5 mkdir -p $outdir echo $@ >> /tmp/args.txt swift=`\which swift` # use "here" document for configs cat << EOF > /tmp/hello.swift type file; app (file o) helloapp (string msg) { echo msg stdout=@o; } file out[]<simple_mapper; location="$outdir", prefix="f.",suffix=".txt">; foreach j in [1:@toInt(@arg("n","2"))] { out[j] = helloapp ("Hello World!"); } EOF case "$site" in cluster ) cat << 'EOF' > /tmp/sites.xml <?xml version="1.0" encoding="UTF-8"?> <config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites"> <pool handle="cluster"> <execution provider="coaster" jobmanager="local:pbs"/> <profile namespace="globus" key="queue">default</profile> <profile namespace="globus" key="jobsPerNode">8</profile> <profile namespace="globus" key="maxWalltime">02:00:00</profile> <profile namespace="globus" key="maxTime">7500</profile> <profile namespace="globus" key="highOverAllocation">100</profile> <profile namespace="globus" key="lowOverAllocation">100</profile> <profile namespace="globus" key="slots">1</profile> <profile namespace="globus" key="maxNodes">1</profile> <profile namespace="globus" key="nodeGranularity">1</profile> <profile namespace="karajan" key="jobThrottle">.31</profile> <profile namespace="karajan" key="initialScore">10000</profile> <workdirectory>/tmp/swift.workdir</workdirectory> </pool> </config> EOF cat << 'EOF' > /tmp/tc cluster echo echo cluster cat cat EOF cat << 'EOF' > /tmp/cf use.provider.staging=true wrapperlog.always.transfer=true execution.retries=1 provider.staging.pin.swiftfiles=false sitedir.keep=true lazy.errors=false EOF ;; localhost ) cat << 'EOF' > /tmp/sites.xml <?xml version="1.0" encoding="UTF-8"?> <config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites"> <pool handle="localhost"> <execution provider="local" /> <profile namespace="karajan" key="jobThrottle">0.01</profile> <profile namespace="karajan" key="initialScore">10000</profile> <filesystem provider="local"/> <workdirectory>/tmp/swift.work</workdirectory> <profile namespace="swift" key="stagingMethod">local</profile> </pool> </config> EOF cat << 'EOF' > /tmp/tc localhost echo echo localhost cat cat EOF cat << 'EOF' > /tmp/cf use.provider.staging=false wrapperlog.always.transfer=true execution.retries=0 provider.staging.pin.swiftfiles=false sitedir.keep=true lazy.errors=false EOF ;; * ) echo "defaulting to localhost" cat << 'EOF' > /tmp/sites.xml <?xml version="1.0" encoding="UTF-8"?> <config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites"> <pool handle="localhost"> <execution provider="local" /> <profile namespace="karajan" key="jobThrottle">0.01</profile> <profile namespace="karajan" key="initialScore">10000</profile> <filesystem provider="local"/> <workdirectory>/tmp/swift.work</workdirectory> <profile namespace="swift" key="stagingMethod">local</profile> </pool> </config> EOF cat << 'EOF' > /tmp/tc localhost echo echo localhost cat cat EOF cat << 'EOF' > /tmp/cf use.provider.staging=false wrapperlog.always.transfer=true execution.retries=0 provider.staging.pin.swiftfiles=false sitedir.keep=true lazy.errors=false EOF ;; esac $swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/hello.swift -n=$n -data=$data 2> std.err EXITCODE=$? swiftlog=`\ls -t *.log | head -1` cat << 'EOF' > /tmp/plotswiftlogs #!/bin/bash #usage: ./extract4plots <swift.logfile> SWIFTLOGFILE=$1 #TMPDIR=`mktemp -d plotlog.XXX` grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out SWIFTOUTFILE=swiftoutfile.out #extract start time TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '` START_TIME=`date +%s -d "$TMPDATE"` #extract end time TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '` END_TIME=`date +%s -d "$TMPDATE"` #duration DIFFTIME=$((END_TIME - START_TIME)) #echo 0 > active.txt #echo 0 > cumulative.txt #extract active runs in a file (grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > active.txt) #extract successful completions in a file (grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt) #prepare tics activelines=`wc -l active.txt | awk '{print $1}'` cumulines=`wc -l cumulative.txt | awk '{print $1}'` if [ $activelines -ne 0 ] then activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc` fi seq 0 $activelinespertic $DIFFTIME > activetics.txt if [ $cumulines -ne 0 ] then cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc` fi seq 0 $cumulinespertic $DIFFTIME > cumultics.txt #final plot data paste activetics.txt active.txt > plot_active.txt paste cumultics.txt cumulative.txt > plot_cumulative.txt cat << EOF1 > plotit.gp set terminal png enhanced set output "activeplot.png" set nokey set xlabel "Time in sec" set ylabel "number of active jobs" set title "Active jobs" plot "plot_active.txt" using 1:2 with line set output "cumulativeplot.png" set ylabel "number of completed jobs" set title "Cumulative jobs" plot "plot_cumulative.txt" using 1:2 with lines EOF1 gnuplot plotit.gp 2>/dev/null #cp *.png ../ echo `pwd`>/tmp/curdir echo `ls`>/tmp/dirlist #cd .. EOF cat << EOF > $htmlfile <!DOCTYPE HTML> <html xmlns="http://www.w3.org/1999/xhtml"> <head> <title>Hello output</title> </head> <body> <h2>Browse output of hello application</h2> `for i in \`find $outdir -type f\`; do echo "<a href=\`basename $i\`>Output: \`basename $i\`</a><br/>"; done` </body> </html> EOF if [ "$EXITCODE" -ne "0" ]; then cat std.err >&2 fi cp std.err $logfile rm std.err exit $EXITCODE