comparison swift-hello @ 1:7aa6ee21028c draft

Uploaded

1:7aa6ee21028c

#!/bin/bash

site=$1
n=$2
outdir=$3
logfile=$4
htmlfile=$5
mkdir -p $outdir
echo $@ >> /tmp/args.txt

swift=`\which swift`

# use "here" document for configs
cat << EOF > /tmp/hello.swift
type file;

app (file o) helloapp (string msg) {
  echo msg stdout=@o;
}

file out[]<simple_mapper; location="$outdir", prefix="f.",suffix=".txt">;

foreach j in [1:@toInt(@arg("n","2"))] {
 out[j] = helloapp ("Hello World!");
}
EOF

case "$site" in
cluster )
cat << 'EOF' > /tmp/sites.xml
<?xml version="1.0" encoding="UTF-8"?>
<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
  <pool handle="cluster">
    <execution provider="coaster" jobmanager="local:pbs"/>
    <profile namespace="globus" key="queue">default</profile>
    <profile namespace="globus" key="jobsPerNode">8</profile>
    <profile namespace="globus" key="maxWalltime">02:00:00</profile>
    <profile namespace="globus" key="maxTime">7500</profile>
    <profile namespace="globus" key="highOverAllocation">100</profile>
    <profile namespace="globus" key="lowOverAllocation">100</profile>
    <profile namespace="globus" key="slots">1</profile>
    <profile namespace="globus" key="maxNodes">1</profile>
    <profile namespace="globus" key="nodeGranularity">1</profile>
    <profile namespace="karajan" key="jobThrottle">.31</profile>
    <profile namespace="karajan" key="initialScore">10000</profile>
    <workdirectory>/tmp/swift.workdir</workdirectory>
  </pool>
</config>
EOF

cat << 'EOF' > /tmp/tc
cluster echo  echo
cluster cat   cat
EOF

cat << 'EOF' > /tmp/cf
use.provider.staging=true
wrapperlog.always.transfer=true
execution.retries=1
provider.staging.pin.swiftfiles=false
sitedir.keep=true
lazy.errors=false
EOF
;;

localhost )
cat << 'EOF' > /tmp/sites.xml
<?xml version="1.0" encoding="UTF-8"?>
<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
   <pool handle="localhost">
      <execution provider="local" />
      <profile namespace="karajan" key="jobThrottle">0.01</profile>
      <profile namespace="karajan" key="initialScore">10000</profile>
      <filesystem provider="local"/>
      <workdirectory>/tmp/swift.work</workdirectory>
      <profile namespace="swift" key="stagingMethod">local</profile>
   </pool>
</config>
EOF

cat << 'EOF' > /tmp/tc
localhost echo echo
localhost cat cat
EOF

cat << 'EOF' > /tmp/cf
use.provider.staging=false
wrapperlog.always.transfer=true
execution.retries=0
provider.staging.pin.swiftfiles=false
sitedir.keep=true
lazy.errors=false
EOF
;;

* )
echo "defaulting to localhost"

cat << 'EOF' > /tmp/sites.xml
<?xml version="1.0" encoding="UTF-8"?>
<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
   <pool handle="localhost">
      <execution provider="local" />
      <profile namespace="karajan" key="jobThrottle">0.01</profile>
      <profile namespace="karajan" key="initialScore">10000</profile>
      <filesystem provider="local"/>
      <workdirectory>/tmp/swift.work</workdirectory>
      <profile namespace="swift" key="stagingMethod">local</profile>
   </pool>
</config>
EOF

cat << 'EOF' > /tmp/tc
localhost echo echo
localhost cat cat
EOF

cat << 'EOF' > /tmp/cf
use.provider.staging=false
wrapperlog.always.transfer=true
execution.retries=0
provider.staging.pin.swiftfiles=false
sitedir.keep=true
lazy.errors=false
EOF
;;
esac

$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/hello.swift -n=$n -data=$data 2> std.err
EXITCODE=$?

swiftlog=`\ls -t *.log | head -1`

cat << 'EOF' > /tmp/plotswiftlogs
#!/bin/bash

#usage: ./extract4plots <swift.logfile>

SWIFTLOGFILE=$1
#TMPDIR=`mktemp -d plotlog.XXX`
grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out
SWIFTOUTFILE=swiftoutfile.out
#extract start time
TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '`
START_TIME=`date +%s -d "$TMPDATE"`

#extract end time
TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '`
END_TIME=`date +%s -d "$TMPDATE"`

#duration
DIFFTIME=$((END_TIME - START_TIME))
#echo 0 > active.txt
#echo 0 > cumulative.txt
#extract active runs in a file
(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > active.txt)

#extract successful completions in a file
(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt)

#prepare tics
activelines=`wc -l active.txt | awk '{print $1}'`
cumulines=`wc -l cumulative.txt | awk '{print $1}'`

if [ $activelines -ne 0 ]
then
  activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc`
fi
seq 0 $activelinespertic $DIFFTIME > activetics.txt

if [ $cumulines -ne 0 ]
then
    cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc`
fi

seq 0 $cumulinespertic $DIFFTIME > cumultics.txt

#final plot data
paste activetics.txt active.txt > plot_active.txt
paste cumultics.txt cumulative.txt > plot_cumulative.txt

cat << EOF1 > plotit.gp
set terminal png enhanced
set output "activeplot.png"
set nokey
set xlabel "Time in sec"
set ylabel "number of active jobs"
set title "Active jobs"
plot "plot_active.txt" using 1:2 with line

set output "cumulativeplot.png"
set ylabel "number of completed jobs"
set title "Cumulative jobs"
plot "plot_cumulative.txt" using 1:2 with lines
EOF1

gnuplot plotit.gp 2>/dev/null

#cp *.png ../
echo `pwd`>/tmp/curdir
echo `ls`>/tmp/dirlist
#cd ..
EOF

cat << EOF > $htmlfile
<!DOCTYPE HTML>

<html xmlns="http://www.w3.org/1999/xhtml">
<head>
  <title>Hello output</title>
</head>
<body>
<h2>Browse output of hello application</h2>
`for i in \`find $outdir -type f\`; do echo "<a href=\`basename $i\`>Output: \`basename $i\`</a><br/>"; done`
</body>
</html>
EOF

if [ "$EXITCODE" -ne "0" ]; then
       cat std.err >&2
fi
cp std.err $logfile
rm std.err
exit $EXITCODE