comparison swift-hello @ 1:7aa6ee21028c draft

Uploaded
author ketan
date Tue, 15 Oct 2013 16:22:01 -0400
parents
children
comparison
equal deleted inserted replaced
0:a015252417ed 1:7aa6ee21028c
1 #!/bin/bash
2
3 site=$1
4 n=$2
5 outdir=$3
6 logfile=$4
7 htmlfile=$5
8 mkdir -p $outdir
9 echo $@ >> /tmp/args.txt
10
11 swift=`\which swift`
12
13 # use "here" document for configs
14 cat << EOF > /tmp/hello.swift
15 type file;
16
17 app (file o) helloapp (string msg) {
18 echo msg stdout=@o;
19 }
20
21 file out[]<simple_mapper; location="$outdir", prefix="f.",suffix=".txt">;
22
23 foreach j in [1:@toInt(@arg("n","2"))] {
24 out[j] = helloapp ("Hello World!");
25 }
26 EOF
27
28 case "$site" in
29 cluster )
30 cat << 'EOF' > /tmp/sites.xml
31 <?xml version="1.0" encoding="UTF-8"?>
32 <config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
33 <pool handle="cluster">
34 <execution provider="coaster" jobmanager="local:pbs"/>
35 <profile namespace="globus" key="queue">default</profile>
36 <profile namespace="globus" key="jobsPerNode">8</profile>
37 <profile namespace="globus" key="maxWalltime">02:00:00</profile>
38 <profile namespace="globus" key="maxTime">7500</profile>
39 <profile namespace="globus" key="highOverAllocation">100</profile>
40 <profile namespace="globus" key="lowOverAllocation">100</profile>
41 <profile namespace="globus" key="slots">1</profile>
42 <profile namespace="globus" key="maxNodes">1</profile>
43 <profile namespace="globus" key="nodeGranularity">1</profile>
44 <profile namespace="karajan" key="jobThrottle">.31</profile>
45 <profile namespace="karajan" key="initialScore">10000</profile>
46 <workdirectory>/tmp/swift.workdir</workdirectory>
47 </pool>
48 </config>
49 EOF
50
51 cat << 'EOF' > /tmp/tc
52 cluster echo echo
53 cluster cat cat
54 EOF
55
56 cat << 'EOF' > /tmp/cf
57 use.provider.staging=true
58 wrapperlog.always.transfer=true
59 execution.retries=1
60 provider.staging.pin.swiftfiles=false
61 sitedir.keep=true
62 lazy.errors=false
63 EOF
64 ;;
65
66 localhost )
67 cat << 'EOF' > /tmp/sites.xml
68 <?xml version="1.0" encoding="UTF-8"?>
69 <config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
70 <pool handle="localhost">
71 <execution provider="local" />
72 <profile namespace="karajan" key="jobThrottle">0.01</profile>
73 <profile namespace="karajan" key="initialScore">10000</profile>
74 <filesystem provider="local"/>
75 <workdirectory>/tmp/swift.work</workdirectory>
76 <profile namespace="swift" key="stagingMethod">local</profile>
77 </pool>
78 </config>
79 EOF
80
81 cat << 'EOF' > /tmp/tc
82 localhost echo echo
83 localhost cat cat
84 EOF
85
86 cat << 'EOF' > /tmp/cf
87 use.provider.staging=false
88 wrapperlog.always.transfer=true
89 execution.retries=0
90 provider.staging.pin.swiftfiles=false
91 sitedir.keep=true
92 lazy.errors=false
93 EOF
94 ;;
95
96 * )
97 echo "defaulting to localhost"
98
99 cat << 'EOF' > /tmp/sites.xml
100 <?xml version="1.0" encoding="UTF-8"?>
101 <config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
102 <pool handle="localhost">
103 <execution provider="local" />
104 <profile namespace="karajan" key="jobThrottle">0.01</profile>
105 <profile namespace="karajan" key="initialScore">10000</profile>
106 <filesystem provider="local"/>
107 <workdirectory>/tmp/swift.work</workdirectory>
108 <profile namespace="swift" key="stagingMethod">local</profile>
109 </pool>
110 </config>
111 EOF
112
113 cat << 'EOF' > /tmp/tc
114 localhost echo echo
115 localhost cat cat
116 EOF
117
118 cat << 'EOF' > /tmp/cf
119 use.provider.staging=false
120 wrapperlog.always.transfer=true
121 execution.retries=0
122 provider.staging.pin.swiftfiles=false
123 sitedir.keep=true
124 lazy.errors=false
125 EOF
126 ;;
127 esac
128
129 $swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/hello.swift -n=$n -data=$data 2> std.err
130 EXITCODE=$?
131
132 swiftlog=`\ls -t *.log | head -1`
133
134 cat << 'EOF' > /tmp/plotswiftlogs
135 #!/bin/bash
136
137 #usage: ./extract4plots <swift.logfile>
138
139 SWIFTLOGFILE=$1
140 #TMPDIR=`mktemp -d plotlog.XXX`
141 grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out
142 SWIFTOUTFILE=swiftoutfile.out
143 #extract start time
144 TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '`
145 START_TIME=`date +%s -d "$TMPDATE"`
146
147 #extract end time
148 TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '`
149 END_TIME=`date +%s -d "$TMPDATE"`
150
151 #duration
152 DIFFTIME=$((END_TIME - START_TIME))
153 #echo 0 > active.txt
154 #echo 0 > cumulative.txt
155 #extract active runs in a file
156 (grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > active.txt)
157
158 #extract successful completions in a file
159 (grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt)
160
161 #prepare tics
162 activelines=`wc -l active.txt | awk '{print $1}'`
163 cumulines=`wc -l cumulative.txt | awk '{print $1}'`
164
165 if [ $activelines -ne 0 ]
166 then
167 activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc`
168 fi
169 seq 0 $activelinespertic $DIFFTIME > activetics.txt
170
171 if [ $cumulines -ne 0 ]
172 then
173 cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc`
174 fi
175
176 seq 0 $cumulinespertic $DIFFTIME > cumultics.txt
177
178 #final plot data
179 paste activetics.txt active.txt > plot_active.txt
180 paste cumultics.txt cumulative.txt > plot_cumulative.txt
181
182 cat << EOF1 > plotit.gp
183 set terminal png enhanced
184 set output "activeplot.png"
185 set nokey
186 set xlabel "Time in sec"
187 set ylabel "number of active jobs"
188 set title "Active jobs"
189 plot "plot_active.txt" using 1:2 with line
190
191 set output "cumulativeplot.png"
192 set ylabel "number of completed jobs"
193 set title "Cumulative jobs"
194 plot "plot_cumulative.txt" using 1:2 with lines
195 EOF1
196
197 gnuplot plotit.gp 2>/dev/null
198
199 #cp *.png ../
200 echo `pwd`>/tmp/curdir
201 echo `ls`>/tmp/dirlist
202 #cd ..
203 EOF
204
205 cat << EOF > $htmlfile
206 <!DOCTYPE HTML>
207
208 <html xmlns="http://www.w3.org/1999/xhtml">
209 <head>
210 <title>Hello output</title>
211 </head>
212 <body>
213 <h2>Browse output of hello application</h2>
214 `for i in \`find $outdir -type f\`; do echo "<a href=\`basename $i\`>Output: \`basename $i\`</a><br/>"; done`
215 </body>
216 </html>
217 EOF
218
219 if [ "$EXITCODE" -ne "0" ]; then
220 cat std.err >&2
221 fi
222 cp std.err $logfile
223 rm std.err
224 exit $EXITCODE