Mercurial > repos > ketan > swift
diff swift-hello @ 1:7aa6ee21028c draft
Uploaded
author | ketan |
---|---|
date | Tue, 15 Oct 2013 16:22:01 -0400 |
parents | |
children |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/swift-hello Tue Oct 15 16:22:01 2013 -0400 @@ -0,0 +1,224 @@ +#!/bin/bash + +site=$1 +n=$2 +outdir=$3 +logfile=$4 +htmlfile=$5 +mkdir -p $outdir +echo $@ >> /tmp/args.txt + +swift=`\which swift` + +# use "here" document for configs +cat << EOF > /tmp/hello.swift +type file; + +app (file o) helloapp (string msg) { + echo msg stdout=@o; +} + +file out[]<simple_mapper; location="$outdir", prefix="f.",suffix=".txt">; + +foreach j in [1:@toInt(@arg("n","2"))] { + out[j] = helloapp ("Hello World!"); +} +EOF + +case "$site" in +cluster ) +cat << 'EOF' > /tmp/sites.xml +<?xml version="1.0" encoding="UTF-8"?> +<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites"> + <pool handle="cluster"> + <execution provider="coaster" jobmanager="local:pbs"/> + <profile namespace="globus" key="queue">default</profile> + <profile namespace="globus" key="jobsPerNode">8</profile> + <profile namespace="globus" key="maxWalltime">02:00:00</profile> + <profile namespace="globus" key="maxTime">7500</profile> + <profile namespace="globus" key="highOverAllocation">100</profile> + <profile namespace="globus" key="lowOverAllocation">100</profile> + <profile namespace="globus" key="slots">1</profile> + <profile namespace="globus" key="maxNodes">1</profile> + <profile namespace="globus" key="nodeGranularity">1</profile> + <profile namespace="karajan" key="jobThrottle">.31</profile> + <profile namespace="karajan" key="initialScore">10000</profile> + <workdirectory>/tmp/swift.workdir</workdirectory> + </pool> +</config> +EOF + +cat << 'EOF' > /tmp/tc +cluster echo echo +cluster cat cat +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=true +wrapperlog.always.transfer=true +execution.retries=1 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; + +localhost ) +cat << 'EOF' > /tmp/sites.xml +<?xml version="1.0" encoding="UTF-8"?> +<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites"> + <pool handle="localhost"> + <execution provider="local" /> + <profile namespace="karajan" key="jobThrottle">0.01</profile> + <profile namespace="karajan" key="initialScore">10000</profile> + <filesystem provider="local"/> + <workdirectory>/tmp/swift.work</workdirectory> + <profile namespace="swift" key="stagingMethod">local</profile> + </pool> +</config> +EOF + +cat << 'EOF' > /tmp/tc +localhost echo echo +localhost cat cat +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=false +wrapperlog.always.transfer=true +execution.retries=0 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; + +* ) +echo "defaulting to localhost" + +cat << 'EOF' > /tmp/sites.xml +<?xml version="1.0" encoding="UTF-8"?> +<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites"> + <pool handle="localhost"> + <execution provider="local" /> + <profile namespace="karajan" key="jobThrottle">0.01</profile> + <profile namespace="karajan" key="initialScore">10000</profile> + <filesystem provider="local"/> + <workdirectory>/tmp/swift.work</workdirectory> + <profile namespace="swift" key="stagingMethod">local</profile> + </pool> +</config> +EOF + +cat << 'EOF' > /tmp/tc +localhost echo echo +localhost cat cat +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=false +wrapperlog.always.transfer=true +execution.retries=0 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; +esac + +$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/hello.swift -n=$n -data=$data 2> std.err +EXITCODE=$? + +swiftlog=`\ls -t *.log | head -1` + +cat << 'EOF' > /tmp/plotswiftlogs +#!/bin/bash + +#usage: ./extract4plots <swift.logfile> + +SWIFTLOGFILE=$1 +#TMPDIR=`mktemp -d plotlog.XXX` +grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out +SWIFTOUTFILE=swiftoutfile.out +#extract start time +TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '` +START_TIME=`date +%s -d "$TMPDATE"` + +#extract end time +TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '` +END_TIME=`date +%s -d "$TMPDATE"` + +#duration +DIFFTIME=$((END_TIME - START_TIME)) +#echo 0 > active.txt +#echo 0 > cumulative.txt +#extract active runs in a file +(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > active.txt) + +#extract successful completions in a file +(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt) + +#prepare tics +activelines=`wc -l active.txt | awk '{print $1}'` +cumulines=`wc -l cumulative.txt | awk '{print $1}'` + +if [ $activelines -ne 0 ] +then + activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc` +fi +seq 0 $activelinespertic $DIFFTIME > activetics.txt + +if [ $cumulines -ne 0 ] +then + cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc` +fi + +seq 0 $cumulinespertic $DIFFTIME > cumultics.txt + +#final plot data +paste activetics.txt active.txt > plot_active.txt +paste cumultics.txt cumulative.txt > plot_cumulative.txt + +cat << EOF1 > plotit.gp +set terminal png enhanced +set output "activeplot.png" +set nokey +set xlabel "Time in sec" +set ylabel "number of active jobs" +set title "Active jobs" +plot "plot_active.txt" using 1:2 with line + +set output "cumulativeplot.png" +set ylabel "number of completed jobs" +set title "Cumulative jobs" +plot "plot_cumulative.txt" using 1:2 with lines +EOF1 + +gnuplot plotit.gp 2>/dev/null + +#cp *.png ../ +echo `pwd`>/tmp/curdir +echo `ls`>/tmp/dirlist +#cd .. +EOF + +cat << EOF > $htmlfile +<!DOCTYPE HTML> + +<html xmlns="http://www.w3.org/1999/xhtml"> +<head> + <title>Hello output</title> +</head> +<body> +<h2>Browse output of hello application</h2> +`for i in \`find $outdir -type f\`; do echo "<a href=\`basename $i\`>Output: \`basename $i\`</a><br/>"; done` +</body> +</html> +EOF + +if [ "$EXITCODE" -ne "0" ]; then + cat std.err >&2 +fi +cp std.err $logfile +rm std.err +exit $EXITCODE