Mercurial > repos > ketan > swift
comparison swift-hello @ 1:7aa6ee21028c draft
Uploaded
author | ketan |
---|---|
date | Tue, 15 Oct 2013 16:22:01 -0400 |
parents | |
children |
comparison
equal
deleted
inserted
replaced
0:a015252417ed | 1:7aa6ee21028c |
---|---|
1 #!/bin/bash | |
2 | |
3 site=$1 | |
4 n=$2 | |
5 outdir=$3 | |
6 logfile=$4 | |
7 htmlfile=$5 | |
8 mkdir -p $outdir | |
9 echo $@ >> /tmp/args.txt | |
10 | |
11 swift=`\which swift` | |
12 | |
13 # use "here" document for configs | |
14 cat << EOF > /tmp/hello.swift | |
15 type file; | |
16 | |
17 app (file o) helloapp (string msg) { | |
18 echo msg stdout=@o; | |
19 } | |
20 | |
21 file out[]<simple_mapper; location="$outdir", prefix="f.",suffix=".txt">; | |
22 | |
23 foreach j in [1:@toInt(@arg("n","2"))] { | |
24 out[j] = helloapp ("Hello World!"); | |
25 } | |
26 EOF | |
27 | |
28 case "$site" in | |
29 cluster ) | |
30 cat << 'EOF' > /tmp/sites.xml | |
31 <?xml version="1.0" encoding="UTF-8"?> | |
32 <config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites"> | |
33 <pool handle="cluster"> | |
34 <execution provider="coaster" jobmanager="local:pbs"/> | |
35 <profile namespace="globus" key="queue">default</profile> | |
36 <profile namespace="globus" key="jobsPerNode">8</profile> | |
37 <profile namespace="globus" key="maxWalltime">02:00:00</profile> | |
38 <profile namespace="globus" key="maxTime">7500</profile> | |
39 <profile namespace="globus" key="highOverAllocation">100</profile> | |
40 <profile namespace="globus" key="lowOverAllocation">100</profile> | |
41 <profile namespace="globus" key="slots">1</profile> | |
42 <profile namespace="globus" key="maxNodes">1</profile> | |
43 <profile namespace="globus" key="nodeGranularity">1</profile> | |
44 <profile namespace="karajan" key="jobThrottle">.31</profile> | |
45 <profile namespace="karajan" key="initialScore">10000</profile> | |
46 <workdirectory>/tmp/swift.workdir</workdirectory> | |
47 </pool> | |
48 </config> | |
49 EOF | |
50 | |
51 cat << 'EOF' > /tmp/tc | |
52 cluster echo echo | |
53 cluster cat cat | |
54 EOF | |
55 | |
56 cat << 'EOF' > /tmp/cf | |
57 use.provider.staging=true | |
58 wrapperlog.always.transfer=true | |
59 execution.retries=1 | |
60 provider.staging.pin.swiftfiles=false | |
61 sitedir.keep=true | |
62 lazy.errors=false | |
63 EOF | |
64 ;; | |
65 | |
66 localhost ) | |
67 cat << 'EOF' > /tmp/sites.xml | |
68 <?xml version="1.0" encoding="UTF-8"?> | |
69 <config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites"> | |
70 <pool handle="localhost"> | |
71 <execution provider="local" /> | |
72 <profile namespace="karajan" key="jobThrottle">0.01</profile> | |
73 <profile namespace="karajan" key="initialScore">10000</profile> | |
74 <filesystem provider="local"/> | |
75 <workdirectory>/tmp/swift.work</workdirectory> | |
76 <profile namespace="swift" key="stagingMethod">local</profile> | |
77 </pool> | |
78 </config> | |
79 EOF | |
80 | |
81 cat << 'EOF' > /tmp/tc | |
82 localhost echo echo | |
83 localhost cat cat | |
84 EOF | |
85 | |
86 cat << 'EOF' > /tmp/cf | |
87 use.provider.staging=false | |
88 wrapperlog.always.transfer=true | |
89 execution.retries=0 | |
90 provider.staging.pin.swiftfiles=false | |
91 sitedir.keep=true | |
92 lazy.errors=false | |
93 EOF | |
94 ;; | |
95 | |
96 * ) | |
97 echo "defaulting to localhost" | |
98 | |
99 cat << 'EOF' > /tmp/sites.xml | |
100 <?xml version="1.0" encoding="UTF-8"?> | |
101 <config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites"> | |
102 <pool handle="localhost"> | |
103 <execution provider="local" /> | |
104 <profile namespace="karajan" key="jobThrottle">0.01</profile> | |
105 <profile namespace="karajan" key="initialScore">10000</profile> | |
106 <filesystem provider="local"/> | |
107 <workdirectory>/tmp/swift.work</workdirectory> | |
108 <profile namespace="swift" key="stagingMethod">local</profile> | |
109 </pool> | |
110 </config> | |
111 EOF | |
112 | |
113 cat << 'EOF' > /tmp/tc | |
114 localhost echo echo | |
115 localhost cat cat | |
116 EOF | |
117 | |
118 cat << 'EOF' > /tmp/cf | |
119 use.provider.staging=false | |
120 wrapperlog.always.transfer=true | |
121 execution.retries=0 | |
122 provider.staging.pin.swiftfiles=false | |
123 sitedir.keep=true | |
124 lazy.errors=false | |
125 EOF | |
126 ;; | |
127 esac | |
128 | |
129 $swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/hello.swift -n=$n -data=$data 2> std.err | |
130 EXITCODE=$? | |
131 | |
132 swiftlog=`\ls -t *.log | head -1` | |
133 | |
134 cat << 'EOF' > /tmp/plotswiftlogs | |
135 #!/bin/bash | |
136 | |
137 #usage: ./extract4plots <swift.logfile> | |
138 | |
139 SWIFTLOGFILE=$1 | |
140 #TMPDIR=`mktemp -d plotlog.XXX` | |
141 grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out | |
142 SWIFTOUTFILE=swiftoutfile.out | |
143 #extract start time | |
144 TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '` | |
145 START_TIME=`date +%s -d "$TMPDATE"` | |
146 | |
147 #extract end time | |
148 TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '` | |
149 END_TIME=`date +%s -d "$TMPDATE"` | |
150 | |
151 #duration | |
152 DIFFTIME=$((END_TIME - START_TIME)) | |
153 #echo 0 > active.txt | |
154 #echo 0 > cumulative.txt | |
155 #extract active runs in a file | |
156 (grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > active.txt) | |
157 | |
158 #extract successful completions in a file | |
159 (grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt) | |
160 | |
161 #prepare tics | |
162 activelines=`wc -l active.txt | awk '{print $1}'` | |
163 cumulines=`wc -l cumulative.txt | awk '{print $1}'` | |
164 | |
165 if [ $activelines -ne 0 ] | |
166 then | |
167 activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc` | |
168 fi | |
169 seq 0 $activelinespertic $DIFFTIME > activetics.txt | |
170 | |
171 if [ $cumulines -ne 0 ] | |
172 then | |
173 cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc` | |
174 fi | |
175 | |
176 seq 0 $cumulinespertic $DIFFTIME > cumultics.txt | |
177 | |
178 #final plot data | |
179 paste activetics.txt active.txt > plot_active.txt | |
180 paste cumultics.txt cumulative.txt > plot_cumulative.txt | |
181 | |
182 cat << EOF1 > plotit.gp | |
183 set terminal png enhanced | |
184 set output "activeplot.png" | |
185 set nokey | |
186 set xlabel "Time in sec" | |
187 set ylabel "number of active jobs" | |
188 set title "Active jobs" | |
189 plot "plot_active.txt" using 1:2 with line | |
190 | |
191 set output "cumulativeplot.png" | |
192 set ylabel "number of completed jobs" | |
193 set title "Cumulative jobs" | |
194 plot "plot_cumulative.txt" using 1:2 with lines | |
195 EOF1 | |
196 | |
197 gnuplot plotit.gp 2>/dev/null | |
198 | |
199 #cp *.png ../ | |
200 echo `pwd`>/tmp/curdir | |
201 echo `ls`>/tmp/dirlist | |
202 #cd .. | |
203 EOF | |
204 | |
205 cat << EOF > $htmlfile | |
206 <!DOCTYPE HTML> | |
207 | |
208 <html xmlns="http://www.w3.org/1999/xhtml"> | |
209 <head> | |
210 <title>Hello output</title> | |
211 </head> | |
212 <body> | |
213 <h2>Browse output of hello application</h2> | |
214 `for i in \`find $outdir -type f\`; do echo "<a href=\`basename $i\`>Output: \`basename $i\`</a><br/>"; done` | |
215 </body> | |
216 </html> | |
217 EOF | |
218 | |
219 if [ "$EXITCODE" -ne "0" ]; then | |
220 cat std.err >&2 | |
221 fi | |
222 cp std.err $logfile | |
223 rm std.err | |
224 exit $EXITCODE |