postscript chapter 4

• Messages sorted by: [ date ][ thread ][ subject ][ author ]
• Next message: Rolf Rabenseifner: "Re: new version of chapter four"
• Previous message: Marc Snir: "new version of chapter four"

end
save
-108.0 495.0 translate
1 -1 scale

/clp {closepath} bind def
/ef {eofill} bind def
/gr {grestore} bind def
/gs {gsave} bind def
/l {lineto} bind def
/m {moveto} bind def
/n {newpath} bind def
/s {stroke} bind def
/slc {setlinecap} bind def
/slj {setlinejoin} bind def
/slw {setlinewidth} bind def
/srgb {setrgbcolor} bind def
/rot {rotate} bind def
/sc {scale} bind def
/tr {translate} bind def
/tnt {dup dup currentrgbcolor
4 -2 roll dup 1 exch sub 3 -1 roll mul add
4 -2 roll dup 1 exch sub 3 -1 roll mul add
4 -2 roll dup 1 exch sub 3 -1 roll mul add srgb}
bind def
/shd {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul
4 -2 roll mul srgb} bind def
/$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def
/$F2psEnd {$F2psEnteredState restore end} def

$F2psBegin
10 setmiterlimit
0.06000 0.06000 sc
15.000 slw
[100.0] 0 setdash
n 4950 2625 m 3300 3150 l gs col-1 s gr [] 0 setdash
n 3546.89 3134.41 m 3300.00 3150.00 l 3510.51 3020.06 l 3529.20 3077.73 l 3546.89 3134.41 l clp gs 0.00 setgray ef gr gs col-1 s gr
[100.0] 0 setdash
n 6450 2625 m 8025 3150 l gs col-1 s gr [] 0 setdash
n 7816.29 3017.18 m 8025.00 3150.00 l 7778.34 3131.03 l 7797.82 3074.61 l 7816.29 3017.18 l clp gs 0.00 setgray ef gr gs col-1 s gr
[100.0] 0 setdash
n 8025 5175 m 6600 6000 l gs col-1 s gr [] 0 setdash
n 6837.76 5931.68 m 6600.00 6000.00 l 6777.64 5827.83 l 6808.20 5880.25 l 6837.76 5931.68 l clp gs 0.00 setgray ef gr gs col-1 s gr
[100.0] 0 setdash
n 3300 5325 m 4950 6000 l gs col-1 s gr [] 0 setdash
n 4750.59 5853.60 m 4950.00 6000.00 l 4705.15 5964.66 l 4728.37 5909.63 l 4750.59 5853.60 l clp gs 0.00 setgray ef gr gs col-1 s gr
n 7875 4050 m 5700 7200 l gs col-1 s gr
n 5885.74 7036.60 m 5700.00 7200.00 l 5786.99 6968.41 l 5836.87 7003.00 l 5885.74 7036.60 l clp gs 0.00 setgray ef gr gs col-1 s gr
n 3075 4125 m 5025 6750 l gs col-1 s gr
n 4930.05 6521.56 m 5025.00 6750.00 l 4833.72 6593.12 l 4882.38 6557.84 l 4930.05 6521.56 l clp gs 0.00 setgray ef gr gs col-1 s gr
/Times-Bold findfont 270.00 scalefont setfont
1800 3930 m
gs 1 -1 sc (put) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
1800 4560 m
gs 1 -1 sc (put) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
5100 1380 m
gs 1 -1 sc (load) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
5100 2010 m
gs 1 -1 sc (load) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
8100 3930 m
gs 1 -1 sc (put) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
8100 4560 m
gs 1 -1 sc (put) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
5100 6930 m
gs 1 -1 sc (load) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
5100 7560 m
gs 1 -1 sc (load) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
5100 750 m
gs 1 -1 sc (RMA_wait) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
8100 3300 m
gs 1 -1 sc (RMA_start) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
5100 6300 m
gs 1 -1 sc (RMA_wait) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
1800 3300 m
gs 1 -1 sc (RMA_start) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
5100 2640 m
gs 1 -1 sc (RMA_post) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
5100 8190 m
gs 1 -1 sc (RMA_post) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
1800 5190 m
gs 1 -1 sc (RMA_complete) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
8100 5190 m
gs 1 -1 sc (RMA_complete) col-1 show gr
$F2psEnd
restore
%%EndDocument
75 4 a
endTexFig
75 1218 a Fu(Figure)19 b(1.1:)27 b(2-part)o(y)18 b(comm)o(unication.)31
b(Dashed)19 b(arro)o(ws)f(represen)o(t)h(sync)o(hronizations)h(and)f(solid)75
1274 y(arro)o(ws)14 b(represen)o(t)h(comm)o(unication)h(\(data)e(dep)q
(endencies\).)75 1414 y(In)20 b(a)e(similar)i(manner,)g(w)o(e)e(can)h
(replace)h(in)g(Example)f(1.2,)g(page)f(9,)i(the)f(call)h(to)e(get)g(b)o(y)h
(a)g(call)h(to)75 1471 y(accum)o(ulate,)e(th)o(us)f(p)q(erforming)h(the)g
(computation)f(with)h(only)g(one)f(comm)o(unication)i(b)q(et)o(w)o(een)e(an)o
(y)75 1527 y(t)o(w)o(o)d(pro)q(cesses.)75 1670 y Fr(1.4)59
b(Synchronization)20 b(calls)75 1772 y Fu(Sync)o(hronization)e(calls)g
(accept)f(as)f(parameter)g(a)h(windo)o(w)g(iden)o(ti\014er)h(\(comm)o
(unicator,)e(or)g(comm)o(u-)75 1828 y(nicator)g(and)g(rank\).)21
b(These)16 b(calls)h(are)f(used)g(for)g(ordering)g(correctly)g(con\015icting)
h(accesses)g(to)e(RMA)75 1885 y(windo)o(ws,)h(and)h(making)f(sure)h(that)e
(they)h(are)g(not)g(concurren)o(t.)23 b(These)17 b(sync)o(hronization)g
(calls)h(ma)o(y)75 1941 y(not,)12 b(in)g(general,)g(b)q(e)h(used)f(to)f(sync)
o(hronize)i(op)q(erations)e(other)h(than)f(con\015icting)i(accesses)f(to)f
(an)h(RMA)75 1997 y(windo)o(w.)166 2054 y(Figure)k(1.1)e(illustrates)j(the)f
(general)g(sync)o(hronization)g(pattern,)f(for)g(2-part)o(y)g(comm)o
(unication,)75 2110 y(i.e.,)h(for)g(con\015icts)i(b)q(et)o(w)o(een)f(RMA)f
(accesses)h(and)g(lo)q(cal)g(accesses.)24 b(The)17 b(\014gure)g(illustrates)g
(the)g(case)75 2167 y(of)22 b(m)o(ultiple)j(pro)q(ducers)e(to)f(a)h(windo)o
(w,)h(that)e(use)h(put)g(calls;)k(the)c(other)f(RMA)h(calls)h(are)f(dealt)75
2223 y(similarly)l(.)37 b(RMA)20 b(accesses)h(to)e(a)h(windo)o(w)h(is)f
(preceded)i(and)e(follo)o(w)o(ed)h(b)o(y)f(sync)o(hronization)h(calls)75
2280 y(on)d(this)g(windo)o(w.)27 b(The)18 b(preceding)h(sync)o(hronization)g
(call)f(\()p Fo(MPI)p 1242 2280 14 2 v 16 w(RMA)p 1357 2280
V 17 w(ST)l(ART)p Fu(\))g(ensures)g(that)f(the)75 2336 y(windo)o(w)11
b(access)g(do)q(es)g(not)f(start)g(b)q(efore)h(other)f(con\015icting)i
(accesses)f(to)f(the)h(windo)o(w)g(ha)o(v)o(e)f(completed;)75
2393 y(the)18 b(ensuing)g(sync)o(hronization)h(call)g(\()p
Fo(MPI)p 840 2393 V 15 w(RMA)p 954 2393 V 17 w(COMPLETE)p Fu(\))e(ensures)h
(that)f(the)h(windo)o(w)g(access)75 2449 y(completes)e(b)q(efore)f(other)g
(con\015icting)i(accesses)e(to)g(the)g(windo)o(w)h(are)e(started.)166
2506 y(Sync)o(hronization)i(calls)h(that)d(protect)h(RMA)g(accesses)h(need)g
(a\013ect)f(only)g(the)h(brac)o(k)o(eted)f(RMA)75 2562 y(calls)h(to)e(the)h
(relev)m(an)o(t)g(windo)o(w\(s\).)20 b(The)15 b(preceding)h(sync)o
(hronization)g(has)e(to)g(b)q(e)i(completed)g(b)q(efore)75
2618 y(the)f(RMA)h(call)g(a\013ects)e(the)h(target)f(memory;)h(and)g(the)g
(data)g(transfers)f(e\013ected)h(b)o(y)g(the)h(RMA)f(calls)75
2675 y(m)o(ust)k(complete)h(b)q(efore)g(the)f(ensuing)i(sync)o(hronization)f
(call)g(starts.)31 b(On)20 b(the)g(other)f(hand,)h(other)-32
46 y Fj(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32
328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40
611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40
893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119
y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40
1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627
y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40
1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135
y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40
2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643
y(47)-40 2699 y(48)p eop
%%Page: 14 16
bop 75 -100 a Fu(14)723 b Fl(CHAPTER)15 b(1.)35 b(ONE)16 b(SIDED)f(COMMUNICA)
l(TIONS)75 4 y
18945146 18566234 0 0 30851645 30391173 startTexFig
75 4 a
%%BeginDocument: sync2.eps
/$F2psDict 200 dict def
$F2psDict begin
$F2psDict /mtrx matrix put
/col-1 {} def
/col0 {0.000 0.000 0.000 srgb} bind def
/col1 {0.000 0.000 1.000 srgb} bind def
/col2 {0.000 1.000 0.000 srgb} bind def
/col3 {0.000 1.000 1.000 srgb} bind def
/col4 {1.000 0.000 0.000 srgb} bind def
/col5 {1.000 0.000 1.000 srgb} bind def
/col6 {1.000 1.000 0.000 srgb} bind def
/col7 {1.000 1.000 1.000 srgb} bind def
/col8 {0.000 0.000 0.560 srgb} bind def
/col9 {0.000 0.000 0.690 srgb} bind def
/col10 {0.000 0.000 0.820 srgb} bind def
/col11 {0.530 0.810 1.000 srgb} bind def
/col12 {0.000 0.560 0.000 srgb} bind def
/col13 {0.000 0.690 0.000 srgb} bind def
/col14 {0.000 0.820 0.000 srgb} bind def
/col15 {0.000 0.560 0.560 srgb} bind def
/col16 {0.000 0.690 0.690 srgb} bind def
/col17 {0.000 0.820 0.820 srgb} bind def
/col18 {0.560 0.000 0.000 srgb} bind def
/col19 {0.690 0.000 0.000 srgb} bind def
/col20 {0.820 0.000 0.000 srgb} bind def
/col21 {0.560 0.000 0.560 srgb} bind def
/col22 {0.690 0.000 0.690 srgb} bind def
/col23 {0.820 0.000 0.820 srgb} bind def
/col24 {0.500 0.190 0.000 srgb} bind def
/col25 {0.630 0.250 0.000 srgb} bind def
/col26 {0.750 0.380 0.000 srgb} bind def
/col27 {1.000 0.500 0.500 srgb} bind def
/col28 {1.000 0.630 0.630 srgb} bind def
/col29 {1.000 0.750 0.750 srgb} bind def
/col30 {1.000 0.880 0.880 srgb} bind def
/col31 {1.000 0.840 0.000 srgb} bind def

end
save
-108.0 494.0 translate
1 -1 scale

/clp {closepath} bind def
/ef {eofill} bind def
/gr {grestore} bind def
/gs {gsave} bind def
/l {lineto} bind def
/m {moveto} bind def
/n {newpath} bind def
/s {stroke} bind def
/slc {setlinecap} bind def
/slj {setlinejoin} bind def
/slw {setlinewidth} bind def
/srgb {setrgbcolor} bind def
/rot {rotate} bind def
/sc {scale} bind def
/tr {translate} bind def
/tnt {dup dup currentrgbcolor
4 -2 roll dup 1 exch sub 3 -1 roll mul add
4 -2 roll dup 1 exch sub 3 -1 roll mul add
4 -2 roll dup 1 exch sub 3 -1 roll mul add srgb}
bind def
/shd {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul
4 -2 roll mul srgb} bind def
/$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def
/$F2psEnd {$F2psEnteredState restore end} def

$F2psBegin
10 setmiterlimit
0.06000 0.06000 sc
15.000 slw
[100.0] 0 setdash
n 4950 2625 m 3300 3150 l gs col-1 s gr [] 0 setdash
n 3546.89 3134.41 m 3300.00 3150.00 l 3510.51 3020.06 l 3529.20 3077.73 l 3546.89 3134.41 l clp gs 0.00 setgray ef gr gs col-1 s gr
[100.0] 0 setdash
n 6450 2625 m 8025 3150 l gs col-1 s gr [] 0 setdash
n 7816.29 3017.18 m 8025.00 3150.00 l 7778.34 3131.03 l 7797.82 3074.61 l 7816.29 3017.18 l clp gs 0.00 setgray ef gr gs col-1 s gr
[100.0] 0 setdash
n 8025 5175 m 6600 6000 l gs col-1 s gr [] 0 setdash
n 6837.76 5931.68 m 6600.00 6000.00 l 6777.64 5827.83 l 6808.20 5880.25 l 6837.76 5931.68 l clp gs 0.00 setgray ef gr gs col-1 s gr
[100.0] 0 setdash
n 3300 5325 m 4950 6000 l gs col-1 s gr [] 0 setdash
n 4750.59 5853.60 m 4950.00 6000.00 l 4705.15 5964.66 l 4728.37 5909.63 l 4750.59 5853.60 l clp gs 0.00 setgray ef gr gs col-1 s gr
n 4500 1800 m 2925 4125 l gs col-1 s gr
n 3109.28 3959.95 m 2925.00 4125.00 l 3009.93 3892.65 l 3060.10 3926.80 l 3109.28 3959.95 l clp gs 0.00 setgray ef gr gs col-1 s gr
n 6000 1725 m 7500 4050 l gs col-1 s gr
n 7420.31 3815.80 m 7500.00 4050.00 l 7319.47 3880.86 l 7370.39 3848.83 l 7420.31 3815.80 l clp gs 0.00 setgray ef gr gs col-1 s gr
/Times-Bold findfont 270.00 scalefont setfont
5100 2010 m
gs 1 -1 sc (put) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
5100 1380 m
gs 1 -1 sc (put) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
1800 3930 m
gs 1 -1 sc (get) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
1800 4560 m
gs 1 -1 sc (get) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
8100 3930 m
gs 1 -1 sc (get) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
8100 4560 m
gs 1 -1 sc (get) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
5100 6930 m
gs 1 -1 sc (put) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
5100 7560 m
gs 1 -1 sc (put) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
5100 750 m
gs 1 -1 sc (lock\(exclusive\)) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
5100 2640 m
gs 1 -1 sc (unlock) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
1800 3300 m
gs 1 -1 sc (lock\(shared\)) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
1800 5190 m
gs 1 -1 sc (unlock) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
8100 3300 m
gs 1 -1 sc (lock\(shared\)) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
8100 5190 m
gs 1 -1 sc (unlock) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
5100 6300 m
gs 1 -1 sc (lock\(exclusive\)) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
5100 8190 m
gs 1 -1 sc (unlock) col-1 show gr
$F2psEnd
restore
%%EndDocument
75 4 a
endTexFig
75 1278 a Fu(Figure)k(1.2:)27 b(3-part)o(y)18 b(comm)o(unication.)31
b(Dashed)19 b(arro)o(ws)f(represen)o(t)h(sync)o(hronizations)h(and)f(solid)75
1334 y(arro)o(ws)14 b(represen)o(t)h(comm)o(unication)h(\(data)e(dep)q
(endencies\).)75 1474 y(op)q(erations)i(brac)o(k)o(eted)g(b)o(y)g(these)g(t)o
(w)o(o)f(sync)o(hronization)i(calls)g(ma)o(y)f(not)f(b)q(e)i(dela)o(y)o(ed)g
(un)o(til)g(the)f(\014rst)75 1531 y(sync)o(hronization)i(call)g(completes,)g
(and)f(ma)o(y)f(not)h(complete)g(b)q(efore)h(the)f(ensuing)h(sync)o
(hronization)75 1587 y(completes.)166 1643 y(Figure)e(1.2)e(illustrates)j
(the)f(general)g(sync)o(hronization)g(pattern,)f(for)g(3-part)o(y)g(comm)o
(unication,)75 1700 y(i.e.,)20 b(for)e(con\015icts)i(b)q(et)o(w)o(een)g(RMA)f
(accesses.)32 b(The)19 b(\014gure)g(illustrates)i(m)o(ultiple)g(readers)e
(and)g(one)75 1756 y(writer,)d(whic)o(h)i(access)f(memory)f(at)g(a)g(pro)q
(cess)h(that)f(is)h(not)f(in)o(v)o(olv)o(ed)i(in)f(the)g(comm)o(unication.)25
b(Ac-)75 1813 y(cesses)18 b(are)f(brac)o(k)o(eted)g(b)o(y)h(sync)o
(hronization)g(calls)g(\()p Fo(MPI)p 1093 1813 14 2 v 16 w(LOCK)f
Fu(and)h Fo(MPI)p 1418 1813 V 15 w(UNLOCK)p Fu(\))g(that)e(ensure)75
1869 y(that)e(con\015icts)j(are)d(a)o(v)o(oided.)166 1926 y(MPI)c(pro)o
(vides)h(three)g(sync)o(hronization)h(mec)o(hanisms:)18 b(The)11
b(most)e(elab)q(orate)i(is)g(pro)o(vided)g(b)o(y)g(the)75 1982
y(four)f(functions)h Fo(MPI)p 441 1982 V 15 w(RMA)p 555 1982
V 17 w(ST)l(ART,)g(MPI)p 820 1982 V 16 w(RMA)p 935 1982 V 17
w(COMPLETE,)e(MPI)p 1297 1982 V 16 w(RMA)p 1412 1982 V 17 w(W)l(AIT,)h
Fu(and)g Fo(MPI)p 1730 1982 V 16 w(RMA)p 1845 1982 V 17 w(POST)p
Fu(,)75 2039 y(whic)o(h)17 b(directly)g(supp)q(orts)f(the)g(2-part)o(y)f
(sync)o(hronization)i(pattern)e(illustrated)j(in)e(Figure)h(1.1.)k(The)75
2095 y Fo(MPI)p 160 2095 V 16 w(BARRIER)16 b Fu(collectiv)o(e)h(comm)o
(unication)f(call)g(supp)q(orts)g(a)f(simpli\014ed)i(sync)o(hronization)f
(pattern)75 2152 y(that)j(is)h(often)f(used)i(in)f(parallel)h(computations:)
29 b(namely)20 b(a)g(lo)q(osely-sync)o(hronous)g(mo)q(del,)i(where)75
2208 y(global)g(computation)f(phases)g(alternate)g(with)g(global)h(comm)o
(unication)g(phases.)37 b(Finally)23 b(shared)75 2264 y(and)f(exclusiv)o(e)h
(lo)q(c)o(ks)e(are)h(pro)o(vided)g(b)o(y)f(the)h(t)o(w)o(o)e(function)i
Fo(MPI)p 1276 2264 V 16 w(LOCK)f Fu(and)g Fo(MPI)p 1608 2264
V 16 w(UNLOCK)p Fu(,)g(in)75 2321 y(supp)q(ort)15 b(of)g(3-part)o(y)f(comm)o
(unication)i(sync)o(hronization)g(of)f(the)g(t)o(yp)q(e)h(illustrated)g(in)g
(Figure)g(1.2.)1967 46 y Fj(1)1967 103 y(2)1967 159 y(3)1967
215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967
498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724
y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959
1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959
1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959
1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959
1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959
1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959
2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959
2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959
2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop
%%Page: 15 17
bop 75 -100 a Fl(1.4.)34 b(SYNCHR)o(ONIZA)l(TION)17 b(CALLS)1031
b Fu(15)75 49 y Fe(1.4.1)49 b(General)16 b(2-pa)o(rt)o(y)g(synchronization)75
230 y Fo(MPI)p 160 230 14 2 v 16 w(RMA)p 275 230 V 17 w(ST)l(ART\(comm)m(,)c
(rank,)j(info\))117 307 y Fk(IN)155 b Fo(comm)466 b Fk(comm)o(unicator)11
b(asso)q(ciated)k(with)f(windo)o(w)f(\(handle\))117 382 y(IN)155
b Fo(rank)504 b Fk(rank)14 b(of)f(pro)q(cess)j(\(in)o(teger\))117
457 y(IN)155 b Fo(info)516 b Fk(information)11 b(\015ag)j(\(handle\))75
581 y Fq(int)23 b(MPI)p 245 581 15 2 v 17 w(RMA)p 334 581 V
17 w(start\(MPI)p 567 581 V 16 w(Comm)g(comm,)g(int)h(rank,)f(MPI)p
1156 581 V 17 w(info)g(info\))75 668 y(MPI)p 150 668 V 17 w(RMA)p
239 668 V 17 w(START\(COMM,)f(RANK,)h(INFO,)g(IERROR\))170
724 y(INTEGER)g(COMM,)h(RANK,)f(INFO,)g(IERROR)166 811 y Fu(Starts)15
b(a)g(sequence)i(of)f(RMA)f(op)q(erations)h(targeted)f(to)h(the)f(windo)o(w)h
(asso)q(ciated)g(with)h Fo(\(com)o(m)m(,)75 867 y(rank\))p
Fu(.)i(The)d(succeeding)g(RMA)f(accesses)h(to)e(this)h(windo)o(w)g(will)i(b)q
(e)e(dela)o(y)o(ed,)h(if)f(necessary)l(,)g(un)o(til)h(the)75
924 y(target)e(windo)o(w)h(is)h(p)q(osted)g(b)o(y)f(a)g(call)h(to)e
Fo(MPI)p 890 924 14 2 v 16 w(RMA)p 1005 924 V 17 w(POST)p Fu(.)166
980 y(The)h(follo)o(wing)h(three)g(prede\014ned)g(v)m(alues)h(are)e
(de\014ned)h(for)f(the)g Fo(info)g Fu(argumen)o(t:)75 1074
y Fn(MPI)p 184 1074 16 2 v 18 w(STR)o(ONG)23 b Fu(The)13 b(sync)o
(hronization)h(call)g Fo(MPI)p 1027 1074 14 2 v 16 w(RMA)p
1142 1074 V 17 w(ST)l(ART)g Fu(returns)e(only)i(when)f(the)g(target)189
1130 y(windo)o(w)i(is)h(p)q(osted.)75 1224 y Fn(MPI)p 184 1224
16 2 v 18 w(WEAK)23 b Fu(The)16 b(sync)o(hronization)i(call)f
Fo(MPI)p 982 1224 14 2 v 16 w(RMA)p 1097 1224 V 17 w(ST)l(ART)g
Fu(ma)o(y)f(return)h(earlier,)g(ev)o(en)g(if)g(the)189 1281
y(target)12 b(windo)o(w)i(is)h(not)e(p)q(osted.)20 b(Ho)o(w)o(ev)o(er,)12
b(the)i(actual)g(transfer)f(of)h(data)f(will)i(not)f(o)q(ccur)g(un)o(til)189
1337 y(the)20 b(windo)o(w)h(is)f(p)q(osted.)36 b(This)21 b(ma)o(y)e(sa)o(v)o
(e)h(a)g(comm)o(unication)h(and)g(allo)o(w)f(to)g(com)o(bine)h(the)189
1394 y(sync)o(hronization)16 b(with)f(the)h(data)e(transfer.)75
1487 y Fn(MPI)p 184 1487 16 2 v 18 w(NOCHECK)23 b Fu(The)d(sync)o
(hronization)g(call)h Fo(MPI)p 1095 1487 14 2 v 16 w(RMA)p
1210 1487 V 17 w(ST)l(ART)f Fu(need)h(not)e(c)o(hec)o(k)h(that)f(the)189
1544 y(target)12 b(windo)o(w)h(is)h(p)q(osted:)19 b(the)13
b(user)g(is)h(resp)q(onsible)h(to)e(ensure)h(that)e(that)h(windo)o(w)g(is)h
(already)189 1600 y(p)q(osted)19 b(when)g(the)f(call)i(o)q(ccurs.)31
b(This)19 b(is)g(similar)h(to)e(the)h(optimization)g(of)f(\\ready-receiv)o
(e")189 1657 y(that)c(sa)o(v)o(es)h(a)f(hand-shak)o(e)i(when)g(this)f
(hand-shak)o(e)h(is)g(implicit)h(in)f(the)g(co)q(de,)f(an)o(yho)o(w.)166
1798 y Fh(Missing:)i Fk(Need)d(to)g(de\014ne)h(k)o(eyw)o(ord-v)n(alue)e
(pairs)h(for)g(info)e(argumen)o(t.)189 1951 y Fp(A)n(dvic)n(e)22
b(to)i(implementors.)86 b Fu(The)23 b(info)g(argumen)o(t)g(ma)o(y)f(b)q(e)h
(ignored)h(b)o(y)f(implemen)o(tors;)189 2008 y(it)c(is)g(alw)o(a)o(ys)g
(correct)f(\(alb)q(eit,)j(ine\016cien)o(t\))f(to)f(implemen)o(t)h
Fo(MPI)p 1354 2008 V 16 w(RMA)p 1469 2008 V 17 w(ST)l(ART)g
Fu(as)e(if)i Fo(info)f(=)189 2064 y(MPI)p 274 2064 V 15 w(STRONG)p
Fu(.)e(\()p Fp(End)e(of)i(advic)n(e)f(to)g(implementors.)p
Fu(\))75 2265 y Fo(MPI)p 160 2265 V 16 w(RMA)p 275 2265 V 17
w(COMPLETE\(com)o(m)n(,)11 b(rank\))117 2342 y Fk(IN)155 b
Fo(comm)466 b Fk(comm)o(unicator)11 b(asso)q(ciated)k(with)f(windo)o(w)f
(\(handle\))117 2417 y(IN)155 b Fo(rank)504 b Fk(rank)14 b(of)f(pro)q(cess)j
(\(in)o(teger\))75 2542 y Fq(int)23 b(MPI)p 245 2542 15 2 v
17 w(RMA)p 334 2542 V 17 w(complete\(MPI)p 639 2542 V 16 w(Comm)g(comm,)g
(int)g(rank\))75 2628 y(MPI)p 150 2628 V 17 w(RMA)p 239 2628
V 17 w(COMPLETE\(COMM,)e(RANK,)j(IERROR\))170 2685 y(INTEGER)f(COMM,)h(RANK,)
f(IERROR)-32 46 y Fj(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32
272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40
554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40
836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40
1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345
y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40
1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853
y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40
2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361
y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40
2643 y(47)-40 2699 y(48)p eop
%%Page: 16 18
bop 75 -100 a Fu(16)723 b Fl(CHAPTER)15 b(1.)35 b(ONE)16 b(SIDED)f(COMMUNICA)
l(TIONS)166 49 y Fu(Complete)h(\(at)f(the)h(origin\))g(a)f(sequence)i(of)f
(RMA)g(op)q(erations)g(targeted)f(to)g(the)h(windo)o(w)g(asso-)75
106 y(ciated)g(with)f Fo(\(comm)m(,)d(rank\))p Fu(.)166 162
y(Consider)k(a)f(sequence)h(of)f(calls)h(in)g(the)f(example)h(b)q(elo)o(w.)75
256 y Fn(Example)i(1.5)75 362 y Fq(MPI_RMA_START\(comm,)j(rank,)i(info\))75
419 y(...)75 475 y(MPI_PUT\(...,rank,...,comm\))75 532 y(...)75
588 y(MPI_RMA_COMPLETE\(comm,)e(rank\))75 732 y Fu(The)e(call)h(to)f
Fo(MPI)p 404 732 14 2 v 15 w(RMA)p 518 732 V 17 w(COMPLETE)g
Fu(blo)q(c)o(ks)h(un)o(til)g(the)f(put)g(call)h(has)f(completed)h(at)f(the)g
(origin;)75 788 y(and)g(the)g(target)e(windo)o(w)i(will)h(b)q(e)g(accessed)f
(b)o(y)g(the)f(put)h(op)q(eration)g(only)g(if)g(it)g(is)h(p)q(osted.)30
b(If)19 b Fo(info)75 845 y(=)g(MPI)p 214 845 V 16 w(STRONG)g
Fu(then)g(the)g(call)g(to)f(start)f(blo)q(c)o(ks)i(un)o(til)h(the)e(target)g
(windo)o(w)g(is)h(p)q(osted.)30 b(On)19 b(the)75 901 y(other)14
b(hand,)g(if)h Fo(info)f(=)h(MPI)p 579 901 V 16 w(WEAK)p Fu(,)f(it)g(is)h(up)
g(to)e(the)i(implemen)o(tation)g(whether)g(the)f(\014rst)g(t)o(w)o(o)f(calls)
75 958 y(blo)q(c)o(k.)24 b(An)17 b(implemen)o(tation)g(ma)o(y)f(ha)o(v)o(e)g
(the)g(call)h(to)f Fo(MPI)p 1128 958 V 16 w(RMA)p 1243 958
V 17 w(ST)l(ART)h Fu(blo)q(c)o(k)g(un)o(til)g(the)g(windo)o(w)75
1014 y(is)h(p)q(osted;)h(or)f(it)g(ma)o(y)f(allo)o(w)h(the)g(call)h(to)e
Fo(MPI)p 923 1014 V 16 w(RMA)p 1038 1014 V 16 w(ST)l(ART)i
Fu(to)e(pro)q(ceed,)i(and)f(blo)q(c)o(k)h(the)f(call)h(to)75
1070 y Fo(MPI)p 160 1070 V 16 w(PUT)p Fu(,)c(un)o(til)h(the)f(windo)o(w)g(is)
g(p)q(osted;)g(or,)f(it)i(ma)o(y)e(allo)o(w)h(b)q(oth)g(calls)h(to)e(pro)q
(ceed)i(and)f(dela)o(y)h(the)75 1127 y(access)f(to)f(the)h(target)e(windo)o
(w)i(un)o(til)h(the)f(call)g(to)f Fo(MPI)p 1047 1127 V 16 w(RMA)p
1162 1127 V 17 w(COMPLETE)h Fu(o)q(ccurred)g(and)g(un)o(til)h(the)75
1183 y(target)e(windo)o(w)h(is)h(p)q(osted.)166 1240 y(The)i(lo)q(cal)i(comm)
o(unication)f(bu\013er)f(of)g(the)h(put)f(call)i(should)f(not)f(b)q(e)h
(accessed)g(after)e(the)i(put)75 1296 y(call)d(started,)e(un)o(til)j(the)e
(ensuing)h(call)h(to)d Fo(MPI)p 899 1296 V 16 w(RMA)p 1014
1296 V 17 w(COMPLETE)h Fu(completed.)166 1353 y Fo(MPI)p 251
1353 V 16 w(RMA)p 366 1353 V 17 w(COMPLETE)j Fu(enforces)g(completion)i(of)e
(preceding)h(RMA)g(calls)g(at)f(the)g(origin.)30 b(A)75 1409
y(put)15 b(or)f(accum)o(ulate)h(call)g(ma)o(y)f(not)g(ha)o(v)o(e)g(completed)
i(at)e(the)g(target)f(when)i(it)g(has)g(completed)g(at)f(the)75
1466 y(origin.)166 1522 y(Eac)o(h)h(call)h(to)f Fo(MPI)p 503
1522 V 16 w(RMA)p 618 1522 V 16 w(ST)l(ART)i Fu(has)e(to)f(b)q(e)i(matc)o
(hed)f(b)o(y)g(a)g(unique)i(follo)o(wing)f(call)g(to)75 1579
y Fo(MPI)p 160 1579 V 16 w(RMA)p 275 1579 V 17 w(COMPLETE)p
Fu(,)f(and)g(vice)h(v)o(ersa.)75 1729 y Fo(MPI)p 160 1729 V
16 w(RMA)p 275 1729 V 17 w(W)l(AIT\(com)o(m)m(,)c(count\))117
1807 y Fk(IN)155 b Fo(comm)466 b Fk(comm)o(unicator)11 b(asso)q(ciated)k
(with)f(windo)o(w)f(\(handle\))117 1882 y(IN)155 b Fo(count)482
b Fk(n)o(um)o(b)q(er)13 b(of)h(of)f(RMA)h(op)q(erations)g(to)f(w)o(ait)g(for)
h(\(in)o(teger\))75 2006 y Fq(int)23 b(MPI)p 245 2006 15 2
v 17 w(RMA)p 334 2006 V 17 w(wait\(MPI)p 543 2006 V 16 w(Comm)g(comm,)h(int)f
(count\))75 2093 y(MPI)p 150 2093 V 17 w(RMA)p 239 2093 V 17
w(WAIT\(COMM,)f(COUNT,)h(IERROR\))170 2149 y(INTEGER)g(COMM,)h(COUNT,)f
(IERROR)166 2236 y Fu(Blo)q(c)o(ks)15 b(un)o(til)g Fo(count)g
Fu(calls)g(to)e Fo(MPI)p 775 2236 14 2 v 16 w(RMA)p 890 2236
V 17 w(COMPLETE\(comm)m(,)e(rank\))j Fu(o)q(ccurred,)g(where)g
Fo(rank)g Fu(is)75 2292 y(the)h(rank)h(of)f(the)g(caller.)22
b(I.e.,)14 b Fo(MPI)p 703 2292 V 16 w(RMA)p 818 2292 V 17 w(W)l(AIT)h
Fu(blo)q(c)o(ks)h(un)o(til)h Fo(count)g Fu(pro)q(cesses)e(accessed)h(the)g
(lo)q(cal)75 2349 y(windo)o(w)h(asso)q(ciated)h(with)f Fo(comm)11
b Fu(with)18 b(RMA)f(calls.)26 b(When)18 b(the)f(call)h(returns,)f(all)h(RMA)
g(accesses)75 2405 y(that)e(where)g(started)g(b)q(efore)h(the)f(matc)o(hing)h
(calls)g(to)f Fo(MPI)p 1124 2405 V 16 w(RMA)p 1239 2405 V 17
w(COMPLETE)g Fu(ha)o(v)o(e)g(completed)i(at)75 2461 y(the)d(target)f(windo)o
(w.)1967 46 y Fj(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967
272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959
554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780
y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959
1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959
1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959
1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959
1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959
1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959
2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959
2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959
2643 y(47)1959 2699 y(48)p eop
%%Page: 17 19
bop 75 -100 a Fl(1.4.)34 b(SYNCHR)o(ONIZA)l(TION)17 b(CALLS)1031
b Fu(17)75 49 y Fo(MPI)p 160 49 14 2 v 16 w(RMA)p 275 49 V
17 w(POST\(comm)m(\))117 126 y Fk(IN)155 b Fo(comm)466 b Fk(comm)o(unicator)
11 b(asso)q(ciated)k(with)f(windo)o(w)f(\(handle\))75 251 y
Fq(int)23 b(MPI)p 245 251 15 2 v 17 w(RMA)p 334 251 V 17 w(post\(MPI)p
543 251 V 16 w(Comm)g(comm\))75 337 y(MPI)p 150 337 V 17 w(RMA)p
239 337 V 17 w(POST\(COMM,)f(IERROR\))170 394 y(INTEGER)h(COMM,)h(IERROR)166
480 y Fu(P)o(ost)14 b(the)h(lo)q(cal)h(windo)o(w)g(asso)q(ciated)f(with)h
Fo(comm)m Fu(,)11 b(th)o(us)k(allo)o(wing)h(it)g(to)e(b)q(e)i(accessed)g(b)o
(y)f(RMA)75 537 y(op)q(erations.)166 593 y(Eac)o(h)10 b(call)h(to)f
Fo(MPI)p 488 593 14 2 v 15 w(RMA)p 602 593 V 17 w(W)l(AIT)g
Fu(has)g(to)g(b)q(e)h(matc)o(hed)f(b)o(y)g(a)g(unique)h(follo)o(wing)g(call)g
(to)f Fo(MPI)p 1735 593 V 16 w(RMA)p 1850 593 V 17 w(POST)p
Fu(,)75 650 y(and)15 b(vice)i(v)o(ersa.)166 747 y Fh(Alternativ)o(es)o(:)30
b Fk(One)10 b(ma)o(y)d(w)o(an)o(t)i(a)g(non)o(blo)q(c)o(king)f(v)o(ersion)i
(of)e Fi(MPI)p 1244 747 13 2 v 15 w(RMA)p 1350 747 V 14 w(W)m(AIT)h
Fk(and/or)g Fi(MPI)p 1686 747 V 14 w(RMA)p 1791 747 V 15 w(ST)m(ART)p
Fk(.)166 797 y(A)19 b(prop)q(osal)f(w)o(as)g(made)g(to)g(use)i(only)e(t)o(w)o
(o)g(calls,)h(namely)e Fi(MPI)p 1235 797 V 14 w(RMA)p 1340
797 V 15 w(W)m(AIT)h Fk(and)g Fi(MPI)p 1637 797 V 14 w(RMA)p
1742 797 V 15 w(POST)p Fk(;)75 846 y(the)h(w)o(ait)f(call)g(also)g(functions)
h(as)g(a)f(call)g(to)h Fi(MPI)p 909 846 V 14 w(RMA)p 1014 846
V 15 w(ST)m(ART)p Fk(,)f(and)g(the)h(p)q(ost)h(call)e(also)g(functions)g(as)h
(a)75 896 y(call)c(to)h Fi(MPI)p 283 896 V 14 w(RMA)p 388 896
V 15 w(COMPLETE)p Fk(.)e(A)i(call)g(to)f Fi(MPI)p 903 896 V
15 w(RMA)p 1009 896 V 14 w(W)m(AIT\(comm)m(,)e(count\))k Fk(blo)q(c)o(ks)f
(un)o(til)f Fi(count)i Fk(calls)e(to)75 946 y Fi(MPI)p 152
946 V 14 w(RMA)p 257 946 V 15 w(POST\(comm)m(,)10 b(rank\))15
b Fk(ha)o(v)o(e)f(b)q(een)h(made,)d(targeting)i(the)g(calling)f(pro)q(cess.)
166 1002 y(This)18 b(alternativ)o(e)g(do)q(es)g(not)g(w)o(ork)g(w)o(ell)f
(when)i(a)f(windo)o(w)f(is)h(accessed)i(b)o(y)e(m)o(ultiple)d(pro)q(cesses.)
34 b(The)75 1059 y(windo)o(w)16 b(o)o(wner)g(needs)i(to)e(call)g(p)q(ost)h
(once)g(for)f(eac)o(h)h(pro)q(cess)h(that)e(is)g(gran)o(ted)h(access)h(to)e
(the)h(windo)o(w,)f(and)75 1115 y(needs)f(to)f(sp)q(ecify)g(the)h(ranks)f(of)
f(these)j(pro)q(cesses.)75 1284 y Fe(1.4.2)49 b(Ba)o(rrier)75
1370 y Fu(The)16 b(MPI)f(call)i Fo(MPI)p 442 1370 14 2 v 16
w(BARRIER\(comm)n(\))p Fu(,)12 b(when)k(executed)h(on)e(a)h(comm)o(unicator)f
(that)g(is)h(asso)q(ciated)75 1427 y(with)i(RMA)h(windo)o(ws,)f(has)g(the)g
(additional)i(e\013ect)d(of)h(sync)o(hronizing)h(RMA)g(op)q(erations)f(on)g
(these)75 1483 y(windo)o(ws.)27 b(All)18 b(RMA)g(op)q(erations)g(that)e(w)o
(ere)i(started)e(on)i(a)f(windo)o(w)g(asso)q(ciated)h(with)g
Fo(comm)11 b Fu(will)75 1539 y(b)q(e)19 b(completed)g(b)q(efore)g(the)f
(barrier)h(completes.)30 b(RMA)18 b(comm)o(unications)h(initiated)h(b)o(y)e
(the)h(caller)75 1596 y(b)q(efore)e(the)g(barrier)f(call)i(will)g(complete)g
(at)e(the)g(origin)i(b)q(efore)f(the)f(barrier)h(call)h(returns;)f(they)f
(will)75 1652 y(complete)g(at)e(the)i(target)e(b)q(efore)h(the)g(barrier)h
(call)g(returns)f(at)g(the)g(target)f(pro)q(cess.)75 1774 y
Fe(1.4.3)49 b(Lo)q(ck)75 1954 y Fo(MPI)p 160 1954 V 16 w(RMA)p
275 1954 V 17 w(LOCK\(com)o(m)m(,)12 b(rank,)j(lo)q(ck)p 759
1954 V 16 w(t)o(yp)q(e,)h(info\))117 2032 y Fk(IN)155 b Fo(comm)466
b Fk(comm)o(unicator)11 b(sp)q(ecifying)j(the)h(lo)q(c)o(k)o(ed)e(windo)o(w)g
(\(handle\))117 2107 y(IN)155 b Fo(rank)504 b Fk(rank)14 b(of)f(windo)o(w)g
(o)o(wner)i(\(in)o(teger\))117 2182 y(IN)155 b Fo(lo)q(ck)p
398 2182 V 17 w(t)o(yp)q(e)413 b Fk(either)10 b Fi(MPI)p 1095
2182 13 2 v 15 w(LOCK)p 1219 2182 V 14 w(EX)o(CLUSIVE)d Fk(or)i
Fi(MPI)p 1579 2182 V 15 w(LOCK)p 1703 2182 V 14 w(SHARED)117
2257 y Fk(IN)155 b Fo(info)516 b Fk(info)13 b(argumen)o(t)g(\(handle\))75
2381 y Fq(int)23 b(MPI)p 245 2381 15 2 v 17 w(RMA)p 334 2381
V 17 w(lock\(MPI)p 543 2381 V 16 w(Comm)g(comm,)h(int)f(rank,)g(int)h(lock)p
1252 2381 V 16 w(type,)f(MPI)p 1483 2381 V 17 w(Info)h(info\))75
2468 y(MPI)p 150 2468 V 17 w(RMA)p 239 2468 V 17 w(LOCK\(COMM,)e(RANK,)h
(LOCK)p 757 2468 V 17 w(TYPE,)g(INFO,)g(IERROR\))170 2524 y(INTEGER)g(COMM,)h
(RANK,)f(LOCK)p 747 2524 V 16 w(TYPE,)h(INFO,)f(IERROR)166
2611 y Fu(Starts)14 b(a)i(sequence)g(or)f(RMA)h(accesses)g(to)e(the)i(windo)o
(w)g(asso)q(ciated)f(with)h Fo(\(comm)m(,)c(rank\))p Fu(.)21
b(The)75 2667 y(call)e(also)g(ensures)f(that)g(an)o(y)g(RMA)g(access)h(to)e
(the)i(windo)o(w)f(asso)q(ciated)h(with)f Fo(\(comm)m(,)e(rank\))i
Fu(that)-32 46 y Fj(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32
272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40
554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40
836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40
1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345
y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40
1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853
y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40
2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361
y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40
2643 y(47)-40 2699 y(48)p eop
%%Page: 18 20
bop 75 -100 a Fu(18)723 b Fl(CHAPTER)15 b(1.)35 b(ONE)16 b(SIDED)f(COMMUNICA)
l(TIONS)75 49 y Fu(w)o(as)h(protected)g(b)o(y)h(a)f(previously)j(released,)e
(con\015icting)h(lo)q(c)o(k)f(will)i(ha)o(v)o(e)d(completed)i(at)e(the)h
(target.)75 106 y(As)g(for,)g Fo(MPI)p 314 106 14 2 v 16 w(ST)l(ART)h
Fu(the)g(three)f Fo(info)g Fu(v)m(alues)i Fi(MPI)p 989 106
13 2 v 14 w(STRONG,)c(MPI)p 1271 106 V 14 w(WEAK)i Fu(and)h
Fi(MPI)p 1589 106 V 14 w(NOCHECK)e Fu(are)75 162 y(prede\014ned.)39
b(If)21 b Fo(info)g(=)g(MPI)p 614 162 14 2 v 16 w(STRONG)p
Fu(,)h(then)f(the)g(lo)q(c)o(k)h(call)g(blo)q(c)o(ks)f(un)o(til)h(the)f(lo)q
(c)o(k)h(is)f(actually)75 219 y(acquired.)26 b(If)17 b Fo(info)f(=)h(MPI)p
552 219 V 16 w(WEAK)g Fu(then)g(the)g(call)h(to)e Fo(MPI)p
1126 219 V 16 w(LOCK)g Fu(ma)o(y)g(return,)h(ev)o(en)g(if)g(the)g(target)75
275 y(windo)o(w)f(is)g(lo)q(c)o(k)o(ed.)21 b(Ho)o(w)o(ev)o(er,)14
b(the)h(actual)h(RMA)g(data)e(transfer)h(will)i(b)q(e)f(dela)o(y)o(ed)g(un)o
(til)h(the)f(lo)q(c)o(k)f(is)75 332 y(acquired.)23 b(If)16
b Fo(info)g(=)g(MPI)p 547 332 V 16 w(NOCHECK)p Fu(,)g(then)h(no)e(lo)q(c)o(k)
i(need)g(b)q(e)f(acquired.)23 b(It)16 b(is)h(up)f(to)g(the)g(user)g(to)75
388 y(ensure)g(that)e(no)h(con\015icting)i(lo)q(c)o(k)f(is)g(held)g(or)f
(acquired)h(concurren)o(tly)l(.)189 494 y Fp(A)n(dvic)n(e)22
b(to)i(implementors.)86 b Fu(The)23 b(info)g(argumen)o(t)g(ma)o(y)f(b)q(e)h
(ignored)h(b)o(y)f(implemen)o(tors;)189 551 y(it)c(is)g(alw)o(a)o(ys)g
(correct)f(\(alb)q(eit,)j(ine\016cien)o(t\))f(to)f(implemen)o(t)h
Fo(MPI)p 1354 551 V 16 w(RMA)p 1469 551 V 17 w(ST)l(ART)g Fu(as)e(if)i
Fo(info)f(=)189 607 y(MPI)p 274 607 V 15 w(STRONG)p Fu(.)e(\()p
Fp(End)e(of)i(advic)n(e)f(to)g(implementors.)p Fu(\))75 808
y Fo(MPI)p 160 808 V 16 w(RMA)p 275 808 V 17 w(UNLOCK\(com)o(m)n(,)11
b(rank\))117 885 y Fk(IN)155 b Fo(comm)466 b Fk(comm)o(unication)11
b(sp)q(ecifying)i(the)i(lo)q(c)o(k)o(ed)f(windo)o(w)117 960
y(IN)155 b Fo(rank)504 b Fk(rank)14 b(of)f(windo)o(w)g(o)o(wner)75
1085 y Fq(int)23 b(MPI)p 245 1085 15 2 v 17 w(RMA)p 334 1085
V 17 w(unlock\(MPI)p 591 1085 V 16 w(comm)g(comm,)g(int)h(rank\))75
1171 y(MPI)p 150 1171 V 17 w(RMA)p 239 1171 V 17 w(UNLOCK\()e(COMM,)i(RANK,)f
(IERROR\))170 1228 y(INTEGER)g(COMM,)h(RANK,)f(IERROR)166 1314
y Fu(Completes)c(at)f(the)h(origin)g(a)f(sequence)i(of)e(RMA)h(accesses)g(to)
f(the)h(windo)o(w)g(asso)q(ciated)g(with)75 1371 y Fo(\(comm)m(,)12
b(rank\))p Fu(.)166 1427 y(RMA)19 b(accesses)g(that)f(are)h(protected)f(b)o
(y)h(an)g(exclusiv)o(e)h(lo)q(c)o(k)g(will)g(not)e(b)q(e)i(concurren)o(t)f
(at)f(the)75 1483 y(target)c(with)h(other)f(RMA)h(accesses)g(to)f(the)h(same)
g(windo)o(w.)20 b(RMA)15 b(accesses)g(that)f(are)g(protected)h(b)o(y)75
1540 y(a)h(shared)h(lo)q(c)o(k)g(will)h(not)e(b)q(e)i(concurren)o(t)e(at)g
(the)h(target)e(with)i(accesses)g(protected)f(b)o(y)h(an)f(exclusiv)o(e)75
1596 y(lo)q(c)o(k)g(to)e(the)i(same)e(windo)o(w.)166 1653 y(Consider)i(the)f
(sequence)h(of)f(calls)h(in)g(the)g(example)g(b)q(elo)o(w.)75
1747 y Fn(Example)i(1.6)75 1853 y Fq(MPI_LOCK\(comm,)k(rank,)h
(MPI_EXCLUSIVE,)f(info\))75 1909 y(MPI_PUT\(...,)g(rank,)h(...,)h(comm\))75
1966 y(MPI_UNLOCK\(comm,)e(rank\))75 2109 y Fu(The)13 b(call)h(to)e
Fo(MPI)p 385 2109 14 2 v 16 w(UNLOCK)h Fu(will)i(blo)q(c)o(k)e(un)o(til)i
(the)e(put)g(transfer)f(completed)i(at)e(the)h(origin.)20 b(If)13
b Fo(info)g(=)75 2166 y(MPI)p 160 2166 V 16 w(STRONG)p Fu(,)j(then)g(the)g
(call)h(to)e Fo(MPI)p 790 2166 V 16 w(LOCK)g Fu(will)i(blo)q(c)o(k)f(un)o
(til)h(a)e(lo)q(c)o(k)h(is)g(acquired)h(on)f(the)f(target)75
2222 y(windo)o(w.)34 b(On)20 b(the)g(other)f(hand,)i(if)f Fo(info)g(=)g(MPI)p
967 2222 V 16 w(WEAK)p Fu(,)f(then)h(it)g(is)h(up)f(to)f(the)h(implemen)o
(tation)75 2279 y(whether)g(the)f(call)i(is)f(blo)q(c)o(king.)34
b(An)20 b(implemen)o(tation)g(ma)o(y)f(ha)o(v)o(e)g(the)h(call)g(to)f
Fo(MPI)p 1619 2279 V 16 w(LOCK)g Fu(blo)q(c)o(k)75 2335 y(un)o(til)e(an)f
(exclusiv)o(e)h(lo)q(c)o(k)f(on)g(the)g(windo)o(w)g(is)g(acquired.)23
b(It)16 b(could)h(p)q(ostp)q(one)f(lo)q(c)o(k)g(acquisition)i(un)o(til)75
2392 y(the)g(call)i(to)d Fo(MPI)p 386 2392 V 16 w(PUT)h Fu(o)q(ccurs;)i(or)e
(it)g(could)i(p)q(ostp)q(one)e(lo)q(c)o(k)h(acquisition)h(and)e(the)g(up)q
(date)h(to)f(the)75 2448 y(windo)o(w)d(un)o(til)i(the)e(call)h(to)f
Fo(MPI)p 650 2448 V 16 w(UNLOCK)g Fu(o)q(ccurs.)166 2505 y(Lo)q(c)o(ks)g(can)
h(also)f(b)q(e)h(used)f(to)g(protect)g(lo)q(cal)h(load/store)f(accesses)g(to)
g(a)g(windo)o(w.)1967 46 y Fj(1)1967 103 y(2)1967 159 y(3)1967
215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967
498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724
y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959
1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959
1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959
1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959
1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959
1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959
2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959
2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959
2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop
%%Page: 19 21
bop 75 -100 a Fl(1.5.)34 b(SEMANTICS)16 b(AND)f(CORRECTNESS)896
b Fu(19)75 49 y Fr(1.5)59 b(Semantics)18 b(and)i(Co)n(rrectness)75
151 y Fu(An)c(access)g(b)o(y)f(pro)q(cess)h(A)f(to)g(a)h(windo)o(w)f
Fo(\(comm)m(,)d(rank\))k Fp(pr)n(e)n(c)n(e)n(des)e Fu(an)h(access)h(b)o(y)f
(pro)q(cess)h(B)g(to)f(that)75 207 y(same)g(windo)o(w)g(if)h(one)f(of)g(the)g
(follo)o(wing)h(conditions)h(hold:)131 301 y(1.)22 b(A)10 b(executed)i(a)f
(call)g(to)g Fo(MPI)p 666 301 14 2 v 15 w(BARRIER\(comm)o(\))c
Fu(after)j(its)h(access)g(completed,)h(and)f(B)g(completed)189
357 y(a)k(matc)o(hing)g(call)h(b)q(efore)g(its)f(access)g(started.)131
451 y(2.)22 b(A)12 b(released)h(its)f(lo)q(c)o(k)g(on)g(the)g(windo)o(w,)h
(calling)h Fo(MPI)p 1092 451 V 15 w(WINDO)o(W)p 1303 451 V
17 w(UNLOCK\(com)o(m)m(,)c(rank\))p Fu(,)i(after)189 508 y(its)20
b(access,)i(B)f(acquired)g(that)f(lo)q(c)o(k,)i(completing)g(a)e(call)i(to)e
Fo(MPI)p 1386 508 V 16 w(WINDO)o(W)p 1598 508 V 16 w(LOCK\(com)o(m)m(,)189
564 y(rank,)k(...\))p Fu(,)e(after)g(A)h(released)h(it,)g(and)f(b)q(efore)g
(its)g(o)o(wn)f(access,)i(and)f(at)f(least)h(one)g(of)f(the)189
621 y(pro)q(cesses)15 b(held)i(the)e(lo)q(c)o(k)h(exclusiv)o(ely)l(.)131
714 y(3.)22 b(The)12 b(access)h(b)o(y)g(A)f(is)i(a)e(load)h(or)f(store,)g
(the)h(access)f(b)o(y)h(B)g(is)g(an)f(RMA)h(op)q(eration,)g(A)g(executed)g(a)
189 771 y(call)j(to)e Fo(MPI)p 412 771 V 16 w(RMA)p 527 771
V 16 w(POST)i Fu(after)e(its)h(access,)g(and)g(B)g(completed)h(a)e(call)i(to)
e Fo(MPI)p 1603 771 V 16 w(RMA)p 1718 771 V 17 w(ST)l(ART)189
827 y Fu(after)g(A)h(p)q(osted)h(the)f(windo)o(w)g(and)h(b)q(efore)f(its)h(o)
o(wn)e(access.)131 921 y(4.)22 b(The)16 b(access)h(b)o(y)f(A)g(is)h(an)f(RMA)
h(access,)f(the)g(access)h(b)o(y)f(B)g(is)h(a)f(load)h(or)e(store,)h(A)g
(executed)h(a)189 978 y(call)e(to)f Fo(MPI)p 411 978 V 16 w(RMA)p
526 978 V 17 w(COMPLETE)g Fu(after)g(its)h(access,)f(B)g(completed)i(a)e
(call)i(to)d Fo(MPI)p 1633 978 V 16 w(RMA)p 1748 978 V 17 w(W)l(AIT)189
1034 y Fu(b)q(efore)k(its)g(access,)h(and)f(the)g(complete)h(call)g(w)o(as)f
(one)g(of)g(the)g Fo(count)h Fu(calls)g(that)f(matc)o(hed)g(the)189
1091 y(w)o(ait)d(call.)166 1184 y(W)l(e)h(sa)o(y)f(that)h(u)g
Fp(fol)r(lows)g Fu(v)g(if)g(v)g(precedes)h(u;)f(u)g(and)g(v)g(are)g
Fp(c)n(oncurr)n(ent)f Fu(if)i(neither)g(u)f(precedes)h(v)75
1241 y(nor)f(v)g(precedes)h(u.)166 1297 y(A)f(correct)g(MPI)g(program)f(m)o
(ust)h(ob)q(ey)g(the)h(follo)o(wing)g(constrain)o(ts.)131 1391
y(1.)22 b(An)d(RMA)g(up)q(date)g(\(put)g(or)f(accum)o(ulate\))h(to)f(a)g
(target)g(windo)o(w)h(is)g(not)f(concurren)o(t)h(with)g(a)189
1448 y(store)e(up)q(date)i(to)f(that)f(same)h(windo)o(w.)30
b(Ev)o(en)18 b(up)q(dates)h(to)f(distinct)h(lo)q(cations)g(of)f(the)g(same)
189 1504 y(windo)o(w)d(are)g(prohibited.)131 1598 y(2.)22 b(An)e(RMA)g
(access)g(to)f(a)g(target)g(windo)o(w)h(is)g(not)f(concurren)o(t)h(with)g
(another)g(access)g(to)f(that)189 1654 y(same)f(windo)o(w,)h(if)g(the)f(same)
g(lo)q(cation)h(is)g(accessed)g(b)o(y)g(b)q(oth)f(op)q(erations,)h(and)g(up)q
(dated)g(b)o(y)189 1711 y(at)g(least)h(one)g(of)f(the)h(t)o(w)o(o)e
(accesses.)34 b(The)20 b(only)g(exception)h(to)e(this)h(rule)h(is)f(that)f
(the)h(same)189 1767 y(v)m(ariable)h(can)f(b)q(e)g(up)q(dated)h(b)o(y)f(t)o
(w)o(o)e(concurren)o(t)i Fo(MPI)p 1175 1767 V 16 w(A)o(CCUMULA)l(TE)g
Fu(calls,)i(that)d(use)h(the)189 1824 y(same)15 b(accum)o(ulate)g(op)q
(eration,)g(with)h(the)f(same)g(basic)h(datat)o(yp)q(e.)166
1917 y(In)23 b(addition,)i(the)e(origin)h(bu\013er)e(of)g(an)h(RMA)g(op)q
(eration)g(cannot)f(b)q(e)i(accessed)f(un)o(til)h(that)75 1974
y(op)q(eration)15 b(is)h(kno)o(wn)f(to)g(b)q(e)g(complete)h(at)f(the)g
(origin,)h(at)e(the)i(ensuing)g(sync)o(hronization.)166 2030
y(If)k(a)g(program)e(ob)q(eys)j(these)f(restrictions,)h(then)f(RMA)g(op)q
(erations)g(will)i(ha)o(v)o(e)d(the)h(follo)o(wing)75 2087
y(b)q(eha)o(vior:)166 2143 y(A)c(read)g(r)g(\(get)g(or)g(load\))g(from)f(a)h
(lo)q(cation)h(in)g(the)f(windo)o(w)h(will)h(return)e(the)g(v)m(alue)i
(written)e(b)o(y)75 2200 y(a)f(write)g(w)g(\(put,)g(accum)o(ulate)g(or)g
(store\))f(to)h(that)f(lo)q(cation)i(if)75 2256 y(The)f(write)h(w)f(precedes)
h(the)f(read)g(r,)g(and)75 2313 y(an)o(y)g(other)g(write)g(w')g(to)f(that)h
(lo)q(cation)h(either)g(precedes)g(the)f(write)g(w)g(or)g(follo)o(ws)g(the)g
(read)h(r.)166 2369 y(The)f(v)m(alue)i(returned)f(b)o(y)f(the)g(read)h(is)f
(unde\014ned)i(if)f(no)f(write)h(satis\014es)f(this)h(condition)h(\(e.g.,)c
(if)75 2426 y(no)j(write)g(precedes)g(the)g(read,)g(or)f(a)g(write)h(w)g
(precedes)h(the)f(read)f(r,)h(and)g(another)f(write)h(w')f(neither)75
2482 y(precedes)h(w)f(nor)g(follo)o(ws)g(r\).)166 2586 y Fh(Alternativ)o(es)o
(:)51 b Fk(W)m(e)17 b(can)h(c)o(ho)q(ose)h(to)f(lab)q(el)f(programs)g(that)h
(break)g(one,)h(or)e(b)q(oth)i(rules)f(erroneous,)75 2642 y(rather)d(than)f
(unde\014ned.)-32 46 y Fj(1)-32 103 y(2)-32 159 y(3)-32 215
y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498
y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40
780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40
1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288
y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40
1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796
y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40
2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304
y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40
2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop
%%Page: 20 22
bop 75 -100 a Fu(20)723 b Fl(CHAPTER)15 b(1.)35 b(ONE)16 b(SIDED)f(COMMUNICA)
l(TIONS)189 49 y Fp(R)n(ationale.)57 b Fu(The)18 b(correctness)g(criteria)h
(deal)g(only)f(with)h(con\015icts)g(that)e(in)o(v)o(olv)o(e)i(RMA)f(op-)189
106 y(erations.)33 b(Con\015icts)21 b(b)q(et)o(w)o(een)f(b)q(et)o(w)o(een)g
(load/store)f(accesses)i(to)e(a)g(windo)o(w)h(will)i(b)q(e)f(dealt)189
162 y(according)16 b(to)g(the)g(shared)g(memory)g(seman)o(tics)g(of)g(the)g
(underlying)i(arc)o(hitecture,)e(whic)o(h)h(are)189 219 y(outside)e(the)h
(scop)q(e)f(of)g(MPI.)189 294 y(The)c(\014rst)g(constrain)o(t)g(on)g(correct)
g(RMA)g(accesses)g(ma)o(y)g(seem)g(unduly)i(restrictiv)o(e,)f(as)f(it)h
(forbids)189 350 y(concurren)o(t)22 b(accesses)h(to)f(nono)o(v)o(erlaping)i
(lo)q(cations)f(in)g(a)g(windo)o(w.)42 b(The)23 b(reason)f(for)h(this)189
407 y(constrain)o(t)14 b(is)h(that,)f(on)g(some)h(arc)o(hitectures,)f
(explicit)j(coherence)f(restoring)e(op)q(erations)h(ma)o(y)189
463 y(b)q(e)c(needed)h(at)e(sync)o(hronization)h(p)q(oin)o(ts.)19
b(A)11 b(di\013eren)o(t)f(op)q(eration)h(ma)o(y)f(b)q(e)h(needed)h(for)e(lo)q
(cations)189 520 y(that)g(where)i(lo)q(cally)h(up)q(dated)f(b)o(y)f(stores)f
(and)i(for)f(lo)q(cations)h(that)e(where)i(remotely)f(up)q(dated)h(b)o(y)189
576 y(put)k(or)f(accum)o(ulate)h(op)q(erations.)23 b(Without)15
b(this)i(constrain)o(t,)e(the)h(MPI)g(library)h(will)g(ha)o(v)o(e)f(to)189
632 y(trac)o(k)9 b(precisely)k(whic)o(h)f(lo)q(cations)f(in)h(a)e(windo)o(w)h
(where)g(up)q(dated)h(b)o(y)e(a)h(put)g(or)f(accum)o(ulate)h(call.)189
689 y(The)20 b(additional)h(o)o(v)o(erhead)e(of)g(main)o(taining)i(suc)o(h)f
(information)f(is)i(considered)g(prohibitiv)o(e.)189 745 y(\()p
Fp(End)15 b(of)i(r)n(ationale.)p Fu(\))75 867 y Fe(1.5.1)49
b(A)o(tomicit)o(y)75 953 y Fu(The)14 b(outcome)g(of)f(concurren)o(t)h(accum)o
(ulates)h(to)e(the)h(same)g(lo)q(cation,)h(with)f(the)g(same)g(op)q(eration)g
(and)75 1009 y(basic)k(datat)o(yp)q(e,)f(is)h(as)g(if)g(the)f(accum)o(ulates)
h(where)g(done)g(at)f(that)f(lo)q(cation)j(in)f(some)f(serial)i(order.)75
1066 y(On)g(the)g(other)f(hand,)i(if)f(t)o(w)o(o)e(lo)q(cations)i(are)g(b)q
(oth)f(up)q(dated)i(b)o(y)e(t)o(w)o(o)g(accum)o(ulate)h(calls,)h(then)f(the)
75 1122 y(up)q(dates)12 b(ma)o(y)g(o)q(ccur)g(in)h(rev)o(erse)f(order)f(at)h
(the)g(t)o(w)o(o)e(lo)q(cation.)20 b(Th)o(us,)12 b(there)g(is)h(no)e(guaran)o
(tee)h(that)f(the)75 1179 y(en)o(tire)i(call)h(to)e Fo(MPI)p
419 1179 14 2 v 15 w(A)o(CCUMULA)l(TE)i Fu(is)f(executed)g(atomically)l(.)20
b(The)13 b(e\013ect)f(of)g(this)h(lac)o(k)g(of)f(atomicit)o(y)75
1235 y(is)17 b(limited:)24 b(The)17 b(previous)g(correctness)f(conditions)i
(imply)f(that)f(a)g(lo)q(cation)h(up)q(dated)h(b)o(y)e(a)g(call)i(to)75
1292 y Fo(MPI)p 160 1292 V 16 w(A)o(CCUMULA)l(TE)p Fu(,)g(cannot)g(b)q(e)h
(accessed)g(b)o(y)f(load)g(or)g(an)g(RMA)h(call)g(other)f(than)g(accum)o
(ulate,)75 1348 y(un)o(til)k(the)e Fo(MPI)p 356 1348 V 16 w(A)o(CCUMULA)l(TE)
h Fu(call)g(has)g(completed)g(\(at)e(the)i(target\).)34 b(Di\013eren)o(t)20
b(in)o(terlea)o(vings)75 1405 y(can)h(lead)g(to)f(di\013eren)o(t)h(results)g
(only)g(to)f(the)h(exten)o(t)f(that)g(computer)h(arithmetics)g(are)f(not)h
(truly)75 1461 y(asso)q(ciativ)o(e.)75 1583 y Fe(1.5.2)49 b(Progress)75
1669 y Fu(RMA)16 b(calls)g(should)h(not)e(blo)q(c)o(k,)h(except)h(when)f
(required)g(for)f(sync)o(hronization.)22 b(Consider)17 b(the)e(co)q(de)75
1725 y(fragmen)o(t)g(in)i(Example)f(1.5.)22 b(Some)16 b(of)g(the)g(calls)h
(ma)o(y)e(blo)q(c)o(k)i(if)g(the)f(target)f(windo)o(w)h(is)g(not)g(p)q
(osted.)75 1781 y(Ho)o(w)o(ev)o(er,)e(if)i(the)f(target)f(windo)o(w)h(is)h(p)
q(osted,)f(then)g(the)h(co)q(de)g(fragmen)o(t)e(m)o(ust)g(complete.)166
1838 y(Consider)20 b(the)f(co)q(de)h(fragmen)o(t)e(in)i(Example)g(1.6.)31
b(Some)19 b(of)g(the)h(call)g(ma)o(y)f(blo)q(c)o(k)h(if)f(another)75
1894 y(pro)q(cess)j(holds)h(a)e(con\015icting)i(lo)q(c)o(k.)41
b(Ho)o(w)o(ev)o(er,)22 b(if)g(no)g(con\015icting)h(lo)q(c)o(k)g(is)f(held,)i
(then)e(the)g(co)q(de)75 1951 y(fragmen)o(t)14 b(m)o(ust)h(complete.)166
2007 y(Consider)c(the)g(co)q(de)g(fragmen)o(t)e(in)i(Figure)g(1.3.)17
b(Eac)o(h)11 b(pro)q(cess)f(up)q(dates)h(the)g(windo)o(w)g(of)f(the)g(other)
75 2064 y(pro)q(cess)i(using)g(a)f(put)g(op)q(eration,)h(next)f(accesses)h
(its)f(o)o(wn)g(windo)o(w.)19 b(The)12 b(p)q(ost)f(calls)h(are)f(non)o(blo)q
(c)o(king,)75 2120 y(and)j(should)g(complete.)20 b(Once)15
b(the)e(p)q(ost)h(calls)g(o)q(ccurred,)g(RMA)g(access)g(to)f(the)g(windo)o
(ws)h(is)g(enabled,)75 2177 y(so)i(that)f(eac)o(h)h(pro)q(cess)g(should)h
(complete)f(the)g(sequence)h(of)f(calls)h(start-put-complete.)22
b(Once)17 b(these)75 2233 y(are)e(complete,)i(the)f(w)o(ait)f(calls)i(should)
g(complete)f(at)f(b)q(oth)h(pro)q(cesses.)22 b(Th)o(us,)16
b(this)g(comm)o(unication)75 2290 y(should)g(not)f(deadlo)q(c)o(k,)h(irresp)q
(ectiv)o(e)g(of)f(the)g(amoun)o(t)g(of)g(data)f(transfered.)75
2411 y Fe(1.5.3)49 b(Implem)o(e)o(ntation)14 b(mo)q(del)75
2497 y Fo(Distributed)j(Memo)o(r)o(y)75 2583 y Fu(The)h(data)f(transfer)g
(e\013ected)h(b)o(y)g(a)f(put)h(can)g(b)q(e)g(implemen)o(ted)i(in)e(a)g
(distributed)h(memory)e(system)75 2639 y(b)o(y)h(a)f(matc)o(hing)h(pair)h(of)
e(send)i(and)f(receiv)o(e.)29 b(The)18 b(origin)g(pro)q(cess)g(sends)h(the)f
(put)g(request,)g(whic)o(h)75 2696 y(includes)d(the)d(target)f(displacemen)o
(t,)j(coun)o(t)e(and)h(datat)o(yp)q(e)f(argumen)o(ts,)f(to)h(the)g(target)g
(pro)q(cess.)19 b(The)1967 46 y Fj(1)1967 103 y(2)1967 159
y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967
441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959
724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949
y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175
y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401
y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627
y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853
y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078
y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304
y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530
y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop
%%Page: 21 23
bop 75 -100 a Fl(1.5.)34 b(SEMANTICS)16 b(AND)f(CORRECTNESS)896
b Fu(21)75 4 y
23681433 13735227 0 0 21247508 12498534 startTexFig
75 4 a
%%BeginDocument: symmetric.eps
/$F2psDict 200 dict def
$F2psDict begin
$F2psDict /mtrx matrix put
/col-1 {} def
/col0 {0.000 0.000 0.000 srgb} bind def
/col1 {0.000 0.000 1.000 srgb} bind def
/col2 {0.000 1.000 0.000 srgb} bind def
/col3 {0.000 1.000 1.000 srgb} bind def
/col4 {1.000 0.000 0.000 srgb} bind def
/col5 {1.000 0.000 1.000 srgb} bind def
/col6 {1.000 1.000 0.000 srgb} bind def
/col7 {1.000 1.000 1.000 srgb} bind def
/col8 {0.000 0.000 0.560 srgb} bind def
/col9 {0.000 0.000 0.690 srgb} bind def
/col10 {0.000 0.000 0.820 srgb} bind def
/col11 {0.530 0.810 1.000 srgb} bind def
/col12 {0.000 0.560 0.000 srgb} bind def
/col13 {0.000 0.690 0.000 srgb} bind def
/col14 {0.000 0.820 0.000 srgb} bind def
/col15 {0.000 0.560 0.560 srgb} bind def
/col16 {0.000 0.690 0.690 srgb} bind def
/col17 {0.000 0.820 0.820 srgb} bind def
/col18 {0.560 0.000 0.000 srgb} bind def
/col19 {0.690 0.000 0.000 srgb} bind def
/col20 {0.820 0.000 0.000 srgb} bind def
/col21 {0.560 0.000 0.560 srgb} bind def
/col22 {0.690 0.000 0.690 srgb} bind def
/col23 {0.820 0.000 0.820 srgb} bind def
/col24 {0.500 0.190 0.000 srgb} bind def
/col25 {0.630 0.250 0.000 srgb} bind def
/col26 {0.750 0.380 0.000 srgb} bind def
/col27 {1.000 0.500 0.500 srgb} bind def
/col28 {1.000 0.630 0.630 srgb} bind def
/col29 {1.000 0.750 0.750 srgb} bind def
/col30 {1.000 0.880 0.880 srgb} bind def
/col31 {1.000 0.840 0.000 srgb} bind def

end
save
-90.0 254.0 translate
1 -1 scale

/clp {closepath} bind def
/ef {eofill} bind def
/gr {grestore} bind def
/gs {gsave} bind def
/l {lineto} bind def
/m {moveto} bind def
/n {newpath} bind def
/s {stroke} bind def
/slc {setlinecap} bind def
/slj {setlinejoin} bind def
/slw {setlinewidth} bind def
/srgb {setrgbcolor} bind def
/rot {rotate} bind def
/sc {scale} bind def
/tr {translate} bind def
/tnt {dup dup currentrgbcolor
4 -2 roll dup 1 exch sub 3 -1 roll mul add
4 -2 roll dup 1 exch sub 3 -1 roll mul add
4 -2 roll dup 1 exch sub 3 -1 roll mul add srgb}
bind def
/shd {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul
4 -2 roll mul srgb} bind def
/$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def
/$F2psEnd {$F2psEnteredState restore end} def

$F2psBegin
10 setmiterlimit
0.06000 0.06000 sc
/Times-Bold findfont 210.00 scalefont setfont
1575 1800 m
gs 1 -1 sc (Start\(rank=1\)) col-1 show gr
/Times-Bold findfont 210.00 scalefont setfont
1575 2400 m
gs 1 -1 sc (Put\(rank=1\)) col-1 show gr
/Times-Bold findfont 210.00 scalefont setfont
1575 3000 m
gs 1 -1 sc (Complete\(rank=1\)) col-1 show gr
/Times-Bold findfont 210.00 scalefont setfont
1575 3600 m
gs 1 -1 sc (Wait\(count=1\)) col-1 show gr
/Times-Bold findfont 210.00 scalefont setfont
1575 4200 m
gs 1 -1 sc (Load) col-1 show gr
15.000 slw
n 3075 2325 m 3600 2325 l 4500 4125 l 4950 4125 l gs col-1 s gr
n 4830.00 4095.00 m 4950.00 4125.00 l 4830.00 4155.00 l 4830.50 4125.50 l 4830.00 4095.00 l clp gs 0.00 setgray ef gr gs col-1 s gr
n 4875 2325 m 4350 2325 l 3450 4125 l 3000 4125 l gs col-1 s gr
n 3120.00 4155.00 m 3000.00 4125.00 l 3120.00 4095.00 l 3120.50 4125.50 l 3120.00 4155.00 l clp gs 0.00 setgray ef gr gs col-1 s gr
/Times-Bold findfont 210.00 scalefont setfont
5100 1200 m
gs 1 -1 sc (Post) col-1 show gr
/Times-Bold findfont 210.00 scalefont setfont
5100 1800 m
gs 1 -1 sc (Start\(rank=0\)) col-1 show gr
/Times-Bold findfont 210.00 scalefont setfont
5100 2400 m
gs 1 -1 sc (Put\(rank=0\)) col-1 show gr
/Times-Bold findfont 210.00 scalefont setfont
5100 3000 m
gs 1 -1 sc (Complete\(rank=0\)) col-1 show gr
/Times-Bold findfont 210.00 scalefont setfont
5100 3600 m
gs 1 -1 sc (Wait\(count=1\)) col-1 show gr
/Times-Bold findfont 210.00 scalefont setfont
5100 4200 m
gs 1 -1 sc (Load) col-1 show gr
/Times-Bold findfont 210.00 scalefont setfont
1500 1200 m
gs 1 -1 sc (Post) col-1 show gr
$F2psEnd
restore
%%EndDocument
75 4 a
endTexFig
593 972 a Fu(Figure)15 b(1.3:)k(Symmetric)d(comm)o(unication)75
1109 y(origin)h(next)f(sends)h(the)g(data)e(using)i(a)f(call)i(to)e
Fo(MPI)p 1001 1109 14 2 v 15 w(ISEND)p Fu(,)g(with)h(the)f(origin)h(address,)
g(coun)o(t,)f(and)75 1166 y(datat)o(yp)q(e)d(argumen)o(ts.)18
b(The)13 b(non)o(blo)q(c)o(king)i(sends,)e(is)h(completed)g(b)o(y)f(a)g(w)o
(ait)g(p)q(osted)g(when)h(the)f(RMA)75 1222 y(completion)j(call)h
Fo(MPI)p 476 1222 V 15 w(RMA)p 590 1222 V 17 w(COMPLETE,)e(MPI)p
958 1222 V 16 w(BARRIER)i Fu(or)d Fo(MPI)p 1315 1222 V 16 w(RMA)p
1430 1222 V 17 w(UNLOCK)h Fu(o)q(ccurs.)166 1278 y(The)j(target)f(pro)q(cess)
h(receiv)o(es)h(the)f(put)g(request)g(and)h(p)q(osts)e(a)h(corresp)q(onding)h
(receiv)o(e,)g(using)75 1335 y(the)f(target)f(displacemen)o(t,)k(coun)o(t)d
(and)g(datat)o(yp)q(e)g(argumen)o(ts.)28 b(A)19 b(similar)g(sc)o(heme)g(w)o
(orks)e(for)h(the)75 1391 y(other)d(RMA)g(transfers.)166 1448
y(The)k(only)g(non)o(trivial)h(part)e(of)h(this)g(sc)o(heme)g(is)g(to)f
(ensure)i(that)e(the)h(target)e(pro)q(cess)i(receiv)o(es)75
1504 y(the)c(put)g(request)h(and)f(acts)f(on)h(it,)g(p)q(osting)h(a)f(receiv)
o(e)h(for)e(the)h(data,)f(within)j(a)e(b)q(ounded)h(amoun)o(t)e(of)75
1561 y(time.)166 1617 y(A)22 b(pure)g(p)q(olling)i(implemen)o(tation)f(is)f
(p)q(ossible)i(for)d(2-part)o(y)g(comm)o(unication,)i(to)e(the)h(same)75
1674 y(extend)13 b(that)f(a)h(pure)g(p)q(olling)i(implemen)o(tation)f(is)f(p)
q(ossible)i(for)d(send-receiv)o(e)j(comm)o(unication.)k(This)75
1730 y(requires)14 b(that)f(the)g(target)f(pro)q(cess)i(p)q(olls)h(for)e(p)q
(ending)i(RMA)e(requests)h(when)g(it)f(executes)h(an)o(y)f(other)75
1787 y(MPI)j(call,)g(and)g(that)f(is)h(p)q(olls)h(for)e(p)q(ending)i(RMA)f
(requests)g(while)h(blo)q(c)o(k)o(ed)f(on)g(an)f(MPI)h(call.)22
b(Suc)o(h)75 1843 y(sc)o(heme)16 b(guaran)o(tees)f(that)g(a)g(put)h(request)f
(will)i(b)q(e)g(ev)o(en)o(tually)f(pro)q(cessed,)g(and)g(that)f(deadlo)q(c)o
(ks)h(are)75 1899 y(not)h(in)o(tro)q(duced)h(b)o(y)f(the)g(implemen)o
(tation.)26 b(\(As)16 b(illustrated)j(b)o(y)e(the)g(example)g(in)h(Figure)f
(1.3,)f(it)h(is)75 1956 y(not)12 b(su\016cien)o(t)h(to)e(handle)i(p)q(ending)
h(requests)e(only)h(at)f(a)f(w)o(ait)h(or)g(barrier)g(call)h(that)f(gran)o(t)
f(lo)q(cal)i(access)75 2012 y(to)g(the)h(windo)o(w,)f(since)i(this)f(ma)o(y)f
(lead)i(to)e(deadlo)q(c)o(k.)20 b(A)14 b(pro)q(cess)f(has)h(to)f(p)q(oll)i
(for,)e(or)g(b)q(e)h(in)o(terrupted)75 2069 y(b)o(y)l(,)h(p)q(ending)i(RMA)e
(requests)g(b)q(efore)h(an)o(y)f(MPI)g(call)h(that)f(ma)o(y)f(blo)q(c)o(k,)i
(and)f(while)i(blo)q(c)o(k)o(ed.\))166 2125 y(As)i(for)g(send-receiv)o(e,)i
(suc)o(h)f(pure)g(p)q(olling)h(implemen)o(tation)g(has)e(the)g(defect)h
(that,)f(ev)o(en)h(if)f(a)75 2182 y(comm)o(unication)f(is)h(enabled,)g(the)f
(comm)o(unication)h(ma)o(y)e(not)g(complete)h(un)o(til)h(the)f(target)f(pro)q
(cess)75 2238 y(executes)i(some)e(MPI)h(call.)29 b(Suc)o(h)19
b(call)g(is)f(b)q(ound)h(to)e(o)q(ccur,)i(unless)g(the)f(target)f(pro)q(cess)
h(is)g(caugh)o(t)75 2295 y(in)h(an)e(in\014nite)j(lo)q(op,)f(and)f(is)g(lik)o
(ely)h(to)f(o)q(ccur)g(so)q(on,)g(since)h(the)f(target)e(pro)q(cess)i(is)h
(in)o(v)o(olv)o(ed)f(in)h(the)75 2351 y(comm)o(unication.)25
b(Ho)o(w)o(ev)o(er,)16 b(the)g(dela)o(y)i(un)o(til)f(this)h(call)f(o)q(ccurs)
g(can)g(b)q(e)h(arbitrarily)f(large.)24 b(This)17 b(is)75 2408
y(similar)g(to)e(the)h(case)g(of)g(a)f(pro)q(cess)h(that)g(p)q(osts)f(a)h
(non)o(blo)q(c)o(king)h(receiv)o(e,)g(but)f(do)q(es)g(not)f(execute)i(the)75
2464 y(ensuing)f(w)o(ait)f(or)g(test.)166 2520 y(It)h(is)h(debatable)g
(whether)g(suc)o(h)f(implemen)o(tation)i(is)f(correct)f(for)f(send-receiv)o
(e,)j(as)e(w)o(ell)i(as)e(for)75 2577 y(RMA.)11 b(Go)q(o)q(d)g(qualit)o(y)g
(\(correct)g(?\))19 b(implemen)o(tations)12 b(will)h(ensure)e(that)g(an)g
(enabled)h(comm)o(unication)75 2633 y(will)19 b(complete)g(within)f(a)g(time)
g(b)q(ound)g(that)f(is)h(indep)q(enden)o(t)i(of)d(the)h(state)f(of)g(the)h
(target)e(pro)q(cess.)75 2690 y(This)i(can)f(b)q(e)h(ac)o(hiev)o(ed)g(b)o(y)f
(replacing)i(or)d(supplemen)o(ting)j(p)q(olling)h(b)o(y)d(async)o(hronous)g
(agen)o(ts,)f(e.g.,)-32 46 y Fj(1)-32 103 y(2)-32 159 y(3)-32
215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32
498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40
780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40
1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288
y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40
1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796
y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40
2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304
y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40
2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop
%%Page: 22 24
bop 75 -100 a Fu(22)723 b Fl(CHAPTER)15 b(1.)35 b(ONE)16 b(SIDED)f(COMMUNICA)
l(TIONS)75 49 y Fu(b)o(y)j(using)h(in)o(terrupt)f(receiv)o(es,)i(or)d(b)o(y)h
(using)h(a)f(timer)h(signal)g(to)e(enforce)h(an)h(upp)q(er)g(b)q(ound)g(on)f
(the)75 106 y(w)o(aiting)d(time)h(for)e(p)q(olling.)166 162
y(A)i(pure)g(p)q(olling)i(strategy)d(is)h(clearly)h(unnacceptable)h(for)d
(3-part)o(y)g(comm)o(unication,)i(since)g(the)75 219 y(target)11
b(pro)q(cess)i(ma)o(y)f(nev)o(er)g(execute)h(MPI)g(calls)g(\()p
Fo(MPI)p 1039 219 14 2 v 15 w(FINALIZE)f Fu(excepted\).)20
b(this)12 b(case)h(requires)g(an)75 275 y(async)o(hronous)k(agen)o(t)g
(\(signal)h(handler,)h(p)q(olling)g(thread,)f(etc.\))26 b(that)17
b(pro)q(cesses)h(p)q(ending)h(requests)75 332 y(within)d(a)f(b)q(ounded)i
(amoun)o(t)d(of)h(time,)g(ev)o(en)h(if)f(no)h(MPI)f(calls)h(o)q(ccur)g(on)f
(the)g(target)f(pro)q(cess.)75 452 y Fo(Sha)o(red)i(Memo)n(ry)75
537 y Fu(Put)23 b(can)h(b)q(e)g(implemen)o(ted)h(in)f(a)g(homogeneous)f
(shared)g(memory)g(en)o(vironmen)o(t)h(b)o(y)f(co)q(de)h(exe-)75
594 y(cuted)d(b)o(y)f(the)g(calling)i(pro)q(cess.)35 b(Supp)q(ose)21
b(one)f(has,)h(in)g(this)f(en)o(vironmen)o(t,)i(an)e(implemen)o(tation)75
650 y(of)f(send/receiv)o(e,)i(using)f(memory-to-memory)e(cop)o(y)l(.)32
b(The)20 b(put)f(op)q(eration)h(can)f(b)q(e)h(implemen)o(ted)75
707 y(as)g(a)g(\\send")h(from)e(origin)i(memory)f(to)g(an)g(in)o(termediate)h
(bu\013er,)h(implemen)o(ted)g(as)e(memory-to-)75 763 y(memory)c(cop)o(y)l(,)h
(follo)o(w)o(ed)g(b)o(y)g(a)f(\\receiv)o(e")i(from)e(the)h(in)o(termediate)g
(bu\013er)g(to)f(the)h(target)f(memory)l(,)75 820 y(implemen)o(ted)h(again)f
(using)g(memory-to-memory)f(cop)o(y)l(.)21 b(The)16 b(in)o(termediate)g(cop)o
(y)g(can)f(b)q(e)i(a)o(v)o(oided)75 876 y(\(or)c(pip)q(elined\))j(for)d
(simple)i(\(con)o(tiguous\))e(bu\013ers)g(or)g(with)h(a)f(more)g(elab)q
(orate)h(implemen)o(tation.)20 b(On)75 933 y(the)c(other)g(hand,)h
(additional)h(cop)o(ying)f(ma)o(y)e(b)q(e)i(required)h(to)d(a)o(v)o(oid)i
(alignmen)o(t)g(problems.)23 b(Similar)75 989 y(tec)o(hniques)17
b(can)f(b)q(e)h(used)f(for)f(other)h(systems)f(that)g(supp)q(ort)h(some)g
(form)f(of)g(cop)o(ying)i(from)e(lo)q(cal)i(to)75 1045 y(remote)f(memory)h
(\(no)q(de-to-no)q(de)g(DMA,)f(re\015ectiv)o(e)i(memory)l(,)e(etc.\).)25
b(Progress)16 b(is)h(not)g(an)f(issue)i(in)75 1102 y(suc)o(h)e(implemen)o
(tations.)166 1158 y(An)10 b(additional)i(memory-to-memory)d(cop)o(y)h(b)o(y)
g(the)g(target)f(pro)q(cess)h(is)h(required)g(if)g(b)q(oth)f(pro)q(cesses)75
1215 y(ha)o(v)o(e)i(access)g(to)g(shared)g(memory)l(,)g(but)g(the)h(origin)f
(pro)q(cess)h(cannot)f(access)g(the)g(target)f(windo)o(w.)19
b(This)75 1271 y(cop)o(ying)c(need)g(b)q(e)g(e\013ected)f(in)h(a)f(timely)h
(fashion,)g(using)g(similar)g(mec)o(hanisms)g(as)f(discussed)h(for)f(dis-)75
1328 y(tributed)i(memory)f(implemen)o(tations.)166 1384 y(V)l(arious)g(w)o
(eak)g(memory)g(mo)q(dels)h(in)o(tro)q(duce)g(the)f(follo)o(wing)h
(complexities:)75 1478 y Fn(Ordering)23 b Fu(If)15 b(memory)g(is)h(not)f
(serially)i(consisten)o(t,)e(explicit)i(calls)g(are)e(needed)i(to)d(ensure)i
(that)f(ac-)189 1534 y(cesses)e(b)o(y)h(di\013eren)o(t)f(pro)q(cessors)g(are)
g(ordered)h(correctly)l(,)g(at)e(p)q(oin)o(ts)i(that)f(indicate)i(completion)
189 1591 y(of)f(RMA)i(op)q(erations.)75 1685 y Fn(Coherence)22
b Fu(In)g(man)o(y)e(of)g(the)h(target)e(systems)h(lo)q(cal)i(load/store)e
(accesses)h(are)f(not)h(necessarily)189 1741 y(coheren)o(t)13
b(with)h(remote)f(put/get)g(accesses.)19 b(It)14 b(ma)o(y)e(b)q(e)j(the)e
(case)h(that)e(put/get)h(access)h(shared)189 1798 y(memory)l(,)j(and)h(that)f
(lo)q(cal)h(cac)o(hes)g(are)f(not)g(main)o(tained)i(coheren)o(t)e(with)h
(memory)l(.)27 b(Or,)18 b(that)189 1854 y(a)g(windo)o(w)h(is)h(in)f(priv)m
(ate)h(memory)l(,)f(and)g(a)f(\(noncoheren)o(t\))h(mirror)f(cop)o(y)h(is)g(k)
o(ept)g(in)h(shared)189 1911 y(memory)l(,)14 b(where)i(it)f(can)g(b)q(e)h
(accessed)g(b)o(y)f(other)g(pro)q(cesses.)75 2004 y Fn(A)o(tomicit)o(y)23
b Fu(The)15 b(implemen)o(tation)h(m)o(ust)f(guaran)o(tee)g(that)f(accum)o
(ulates)i(are)f(atomic.)75 2098 y Fn(Gran)o(ularit)o(y)23 b
Fu(Man)o(y)11 b(target)g(systems)g(do)h(not)g(supp)q(ort)g(atomic)g(stores)f
(of)g(arbitrarily)i(sized)g(en)o(tries.)189 2155 y(Load)j(and)h(stores)e(ma)o
(y)h(b)q(e)h(restricted)f(to)g(w)o(ords;)g(and)g(DMA)g(engines)h(ma)o(y)f(b)q
(e)h(restricted)g(to)189 2211 y(cac)o(he)e(line)i(or)e(page)g(transfers.)166
2305 y(The)c(seman)o(tics)f(of)g(RMA)h(op)q(erations)g(allo)o(w)f(us)h(to)f
(asso)q(ciate)g(coherence)i(restoring)e(and)h(memory)75 2361
y(sync)o(hronization)20 b(op)q(erations)e(with)h(RMA)g(sync)o(hronization)g
(calls.)31 b(Th)o(us,)19 b(RMA)g(op)q(erations)g(can)75 2418
y(b)q(e)d(implemen)o(ted)h(e\016cien)o(tly)f(on)f(shared)h(memory)e(systems)h
(with)h(w)o(eak)e(memory)h(seman)o(tics.)166 2474 y(W)l(e)j(\014rst)g
(describ)q(e)i(a)e(generic)h(implemen)o(tation)h(mo)q(del,)f(and)g(then)f
(map)g(this)h(generic)g(imple-)75 2531 y(men)o(tation)c(on)o(to)f(concrete)i
(examples.)166 2587 y(In)d(our)g(generic)h(mo)q(del,)g(there)f(are)f(t)o(w)o
(o)g(copies)h(of)g(eac)o(h)g(RMA)g(windo)o(w:)18 b(a)13 b Fp(private)g
Fu(cop)o(y)g(and)g(an)75 2644 y Fp(RMA)g Fu(cop)o(y)l(.)19
b(A)13 b(load)h(accesses)g(the)f(priv)m(ate)h(cop)o(y;)f(a)g(store)g(up)q
(dates)g(the)h(priv)m(ate)g(cop)o(y)l(,)f(and)g(p)q(ossibly)75
2700 y(the)j(RMA)g(cop)o(y;)g(a)g(get)f(accesses)i(the)f(RMA)g(cop)o(y;)g(a)f
(put)i(up)q(dates)f(the)g(RMA)g(cop)o(y)l(,)g(and)g(p)q(ossibly)1967
46 y Fj(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967
328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959
611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836
y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959
1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959
1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959
1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959
1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959
2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959
2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959
2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959
2699 y(48)p eop
%%Page: 23 25
bop 75 -100 a Fl(1.5.)34 b(SEMANTICS)16 b(AND)f(CORRECTNESS)896
b Fu(23)75 4 y
28417720 14208860 0 0 42692362 21707980 startTexFig
75 4 a
%%BeginDocument: newindow.eps
/$F2psDict 200 dict def
$F2psDict begin
$F2psDict /mtrx matrix put
/col-1 {} def
/col0 {0.000 0.000 0.000 srgb} bind def
/col1 {0.000 0.000 1.000 srgb} bind def
/col2 {0.000 1.000 0.000 srgb} bind def
/col3 {0.000 1.000 1.000 srgb} bind def
/col4 {1.000 0.000 0.000 srgb} bind def
/col5 {1.000 0.000 1.000 srgb} bind def
/col6 {1.000 1.000 0.000 srgb} bind def
/col7 {1.000 1.000 1.000 srgb} bind def
/col8 {0.000 0.000 0.560 srgb} bind def
/col9 {0.000 0.000 0.690 srgb} bind def
/col10 {0.000 0.000 0.820 srgb} bind def
/col11 {0.530 0.810 1.000 srgb} bind def
/col12 {0.000 0.560 0.000 srgb} bind def
/col13 {0.000 0.690 0.000 srgb} bind def
/col14 {0.000 0.820 0.000 srgb} bind def
/col15 {0.000 0.560 0.560 srgb} bind def
/col16 {0.000 0.690 0.690 srgb} bind def
/col17 {0.000 0.820 0.820 srgb} bind def
/col18 {0.560 0.000 0.000 srgb} bind def
/col19 {0.690 0.000 0.000 srgb} bind def
/col20 {0.820 0.000 0.000 srgb} bind def
/col21 {0.560 0.000 0.560 srgb} bind def
/col22 {0.690 0.000 0.690 srgb} bind def
/col23 {0.820 0.000 0.820 srgb} bind def
/col24 {0.500 0.190 0.000 srgb} bind def
/col25 {0.630 0.250 0.000 srgb} bind def
/col26 {0.750 0.380 0.000 srgb} bind def
/col27 {1.000 0.500 0.500 srgb} bind def
/col28 {1.000 0.630 0.630 srgb} bind def
/col29 {1.000 0.750 0.750 srgb} bind def
/col30 {1.000 0.880 0.880 srgb} bind def
/col31 {1.000 0.840 0.000 srgb} bind def

end
save
-90.0 425.0 translate
1 -1 scale

/clp {closepath} bind def
/ef {eofill} bind def
/gr {grestore} bind def
/gs {gsave} bind def
/l {lineto} bind def
/m {moveto} bind def
/n {newpath} bind def
/s {stroke} bind def
/slc {setlinecap} bind def
/slj {setlinejoin} bind def
/slw {setlinewidth} bind def
/srgb {setrgbcolor} bind def
/rot {rotate} bind def
/sc {scale} bind def
/tr {translate} bind def
/tnt {dup dup currentrgbcolor
4 -2 roll dup 1 exch sub 3 -1 roll mul add
4 -2 roll dup 1 exch sub 3 -1 roll mul add
4 -2 roll dup 1 exch sub 3 -1 roll mul add srgb}
bind def
/shd {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul
4 -2 roll mul srgb} bind def
/$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def
/$F2psEnd {$F2psEnteredState restore end} def

$F2psBegin
10 setmiterlimit
0.06000 0.06000 sc
7.500 slw
n 6000 5100 m 7200 5100 l 7200 5400 l 6000 5400 l clp gs col-1 s gr
/Times-Roman findfont 180.00 scalefont setfont
6150 5325 m
gs 1 -1 sc (? ? ? ? ?) col-1 show gr
[1 50.0] 50.000000 setdash
n 8700 4500 m 8700 4200 l 5700 4200 l 5700 4500 l clp gs col-1 s gr [] 0 setdash
30.000 slw
n 6300 3000 m 6600 3900 l gs col-1 s gr
n 6573.91 3513.41 m 6600.00 3900.00 l 6388.92 3575.08 l gs col-1 s gr
7.500 slw
n 6000 4200 m 7200 4200 l 7200 4500 l 6000 4500 l clp gs col7 0.50 shd ef gr gs col-1 s gr
[1 50.0] 50.000000 setdash
n 8700 5400 m 8700 5100 l 5700 5100 l 5700 5400 l clp gs col-1 s gr [] 0 setdash
15.000 slw
[50.0] 0 setdash
n 6600 4575 m 6600 5025 l gs col-1 s gr [] 0 setdash
7.500 slw
n 6630.00 4905.00 m 6600.00 5025.00 l 6570.00 4905.00 l gs col-1 s gr
/Times-Bold findfont 240.00 scalefont setfont
6075 2775 m
gs 1 -1 sc (PUT) col-1 show gr
n 9600 4200 m 10800 4200 l 10800 4500 l 9600 4500 l clp gs col-1 s gr
/Times-Roman findfont 180.00 scalefont setfont
9750 4425 m
gs 1 -1 sc (? ? ? ? ?) col-1 show gr
n 9600 5100 m 10800 5100 l 10800 5400 l 9600 5400 l clp gs col7 0.50 shd ef gr gs col-1 s gr
[1 50.0] 50.000000 setdash
n 12300 5400 m 12300 5100 l 9300 5100 l 9300 5400 l clp gs col-1 s gr [] 0 setdash
[1 50.0] 50.000000 setdash
n 12300 4500 m 12300 4200 l 9300 4200 l 9300 4500 l clp gs col-1 s gr [] 0 setdash
30.000 slw
n 9900 6600 m 10200 5700 l gs col-1 s gr
n 9988.92 6024.92 m 10200.00 5700.00 l 10173.91 6086.59 l gs col-1 s gr
15.000 slw
[50.0] 0 setdash
n 10200 5025 m 10200 4575 l gs col-1 s gr [] 0 setdash
7.500 slw
n 10170.00 4695.00 m 10200.00 4575.00 l 10230.00 4695.00 l gs col-1 s gr
/Times-Bold findfont 240.00 scalefont setfont
9525 6975 m
gs 1 -1 sc (STORE) col-1 show gr
30.000 slw
n 1800 3000 m 2100 3900 l gs col-1 s gr
n 2073.91 3513.41 m 2100.00 3900.00 l 1888.92 3575.08 l gs col-1 s gr
n 4200 3900 m 4500 3000 l gs col-1 s gr
n 4288.92 3324.92 m 4500.00 3000.00 l 4473.91 3386.59 l gs col-1 s gr
n 1800 6600 m 2100 5700 l gs col-1 s gr
n 1888.92 6024.92 m 2100.00 5700.00 l 2073.91 6086.59 l gs col-1 s gr
n 4200 5700 m 4500 6600 l gs col-1 s gr
n 4473.91 6213.41 m 4500.00 6600.00 l 4288.92 6275.08 l gs col-1 s gr
7.500 slw
[1 50.0] 50.000000 setdash
n 4800 4500 m 4800 4200 l 1800 4200 l 1800 4500 l clp gs col-1 s gr [] 0 setdash
[1 50.0] 50.000000 setdash
n 4800 5400 m 4800 5100 l 1800 5100 l 1800 5400 l clp gs col-1 s gr [] 0 setdash
/Times-Bold findfont 240.00 scalefont setfont
1500 2775 m
gs 1 -1 sc (PUT) col-1 show gr
/Times-Bold findfont 240.00 scalefont setfont
4275 2775 m
gs 1 -1 sc (GET) col-1 show gr
/Times-Bold findfont 240.00 scalefont setfont
1500 7050 m
gs 1 -1 sc (STORE) col-1 show gr
/Times-Bold findfont 240.00 scalefont setfont
4200 7050 m
gs 1 -1 sc (LOAD) col-1 show gr
/Times-Bold findfont 240.00 scalefont setfont
2625 5325 m
gs 1 -1 sc (private copy) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
2700 1800 m
gs 1 -1 sc (Window) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
6225 1800 m
gs 1 -1 sc (RMA Update) col-1 show gr
/Times-Bold findfont 270.00 scalefont setfont
9900 1800 m
gs 1 -1 sc (Local Update) col-1 show gr
/Times-Bold findfont 240.00 scalefont setfont
2550 4425 m
gs 1 -1 sc (RMA copy) col-1 show gr
$F2psEnd
restore
%%EndDocument
75 4 a
endTexFig
533 1002 a Fu(Figure)15 b(1.4:)k(Sc)o(hematic)d(description)g(of)f(windo)o(w)
75 1139 y(the)i(priv)m(ate)h(cop)o(y)l(.)26 b(This)17 b(includes)j(puts)d
(and)g(gets)g(issued)h(b)o(y)f(the)h(windo)o(w)f(o)o(wner.)25
b(The)17 b(mo)q(del)h(is)75 1196 y(illustrated)f(in)f(Figure)f(1.4.)75
1316 y Fo(Ordering:)45 b Fu(The)23 b(implemen)o(tation)g(m)o(ust)f(ensure)h
(that)f(RMA)g(accesses)h(ha)o(v)o(e)e(completed)j(in)f(the)75
1372 y(RMA)16 b(windo)o(w)g(cop)o(y)g(at)f(those)g(p)q(oin)o(ts)i(in)f(the)g
(execution)h(where)f(RMA)g(op)q(erations)g(are)g(said)g(to)f(b)q(e)75
1429 y(complete)h(at)e(the)i(target.)75 1549 y Fo(Coherence:)46
b Fu(The)16 b(rules)g(on)g(correct)f(use)h(of)f(RMA)h(calls)h(imply)g(that)e
(accesses)h(to)f(windo)o(ws)h(can)f(b)q(e)75 1605 y(divided)23
b(in)o(to)e Fp(private)g(ep)n(o)n(chs)p Fu(,)h(where)f(the)f(windo)o(w)h(is)h
(accessed)f(b)o(y)g(loads)g(or)f(stores)g(and)h Fp(glob)n(al)75
1661 y(ep)n(o)n(chs)p Fu(,)j(where)f(the)g(windo)o(w)g(is)h(accessed)g(b)o(y)
e(RMA)i(calls.)44 b(A)23 b(priv)m(ate)h(ep)q(o)q(c)o(h)f(starts)f(\(and)h(an)
75 1718 y(RMA)d(ep)q(o)q(c)o(h)g(ends\))g(with)f(a)h(call)g(b)o(y)g(the)f
(windo)o(w)h(o)o(wner)f(to)f Fo(MPI)p 1302 1718 14 2 v 16 w(BARRIER,)j(MPI)p
1621 1718 V 16 w(RMA)p 1736 1718 V 16 w(W)l(AIT)p Fu(,)75 1774
y(or)c Fo(MPI)p 218 1774 V 16 w(RMA)p 333 1774 V 17 w(LOCK)p
Fu(;)g(a)h(priv)m(ate)g(ep)q(o)q(c)o(h)h(ends)g(\(and)e(an)h(RMA)g(ep)q(o)q
(c)o(h)h(starts\))d(with)i(a)g(call)h(b)o(y)f(the)75 1831 y(windo)o(w)d(o)o
(wner)g(to)g Fo(MPI)p 514 1831 V 15 w(BARRIER,)i(MPI)p 828
1831 V 15 w(RMA)p 942 1831 V 17 w(POST)f Fu(or)f Fo(MPI)p 1231
1831 V 16 w(RMA)p 1346 1831 V 16 w(UNLOCK)p Fu(.)166 1887 y(Coherence)h(is)f
(ac)o(hiev)o(ed)h(b)o(y)f(cop)o(ying)h(the)f(RMA)g(cop)o(y)g(in)o(to)g(the)g
(priv)m(ate)h(cop)o(y)f(when)g(a)g(priv)m(ate)75 1944 y(ep)q(o)q(c)o(h)21
b(starts;)g(and)f(cop)o(ying)h(the)f(priv)m(ate)h(cop)o(y)f(in)o(to)h(the)f
(RMA)g(cop)o(y)g(when)h(the)f(priv)m(ate)h(ep)q(o)q(c)o(h)75
2000 y(ends.)f(There)13 b(is)h(one)f(di\016cult)o(y:)20 b(a)13
b(call)h(to)e Fo(MPI)p 911 2000 V 16 w(BARRIER)j Fu(can)e(signify)h(either)g
(the)f(b)q(eginning)i(or)e(the)75 2057 y(end)j(of)f(a)h(priv)m(ate)g(ep)q(o)q
(c)o(h.)21 b(In)c(order)e(to)g(distinguish)i(b)q(et)o(w)o(een)f(the)g(t)o(w)o
(o,)e(one)h(ma)o(y)g(main)o(tain)h(a)f Fp(dirty)75 2113 y(bit)j
Fu(that)g(is)h(set)g(when)g(the)f(windo)o(w)h(is)g(up)q(dated)h(b)o(y)e(a)g
(put)h(or)f(accum)o(ulate)h(call,)h(and)f(reset)f(when)75 2170
y Fo(MPI)p 160 2170 V 16 w(BARRIER)h Fu(is)f(called.)30 b(A)17
b(call)i(to)e Fo(MPI)p 862 2170 V 16 w(BARRIER)i Fu(is)g(assumed)f(to)f
(complete)h(an)g(RMA)g(ep)q(o)q(c)o(h)75 2226 y(if)f(the)f(bit)h(is)f(set,)g
(a)g(priv)m(ate)h(ep)q(o)q(c)o(h,)g(otherwise.)23 b(It)16 b(is)h(not)e
(necessary)i(to)e(cop)o(y)h(the)g(en)o(tire)h(windo)o(w:)75
2282 y(only)f(lo)q(cations)g(that)e(ha)o(v)o(e)h(b)q(een)h(up)q(dated)g(need)
g(b)q(e)g(copied.)75 2403 y Fo(A)o(tomicit)o(y)75 2488 y Fu(The)g(atomicit)o
(y)g(of)g(accum)o(ulate)g(up)q(date)h(can)f(b)q(e)h(guaran)o(teed)f(b)o(y)g
(using)h(an)f(in)o(ternal)h(windo)o(w)f(lo)q(c)o(k.)75 2545
y(Since)j(puts)f(and)g(gets)f(cannot)h(con\015ict)h(with)f(accum)o(ulates,)g
(there)g(is)h(no)e(need)i(to)e(c)o(hec)o(k)h(this)g(lo)q(c)o(k)75
2601 y(for)g(regular)h(put)g(and)g(get)g(accesses.)31 b(Finer)19
b(lo)q(c)o(king)h(is)g(p)q(ossible,)h(with)e(a)g(more)f(complex)i(lo)q(c)o
(king)75 2658 y(proto)q(col.)-32 46 y Fj(1)-32 103 y(2)-32
159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32
441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40
724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40
1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232
y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40
1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740
y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40
2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248
y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40
2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop
%%Page: 24 26
bop 75 -100 a Fu(24)723 b Fl(CHAPTER)15 b(1.)35 b(ONE)16 b(SIDED)f(COMMUNICA)
l(TIONS)75 49 y Fo(Granula)o(rit)o(y)75 135 y Fu(Supp)q(ose)g(that)e(RMA)h
(accesses)g(are)f(restricted)i(to)e(aligned)i(w)o(ords,)e(and)h(up)q(dates)g
(of)g(smaller)g(en)o(tities)75 192 y(require)20 b(a)f(load-mo)q(dify-store)h
(cycle.)33 b(Then,)21 b(correct)d(b)q(eha)o(vior)i(can)g(b)q(e)g(ac)o(hiev)o
(ed)g(b)o(y)f(protecting)75 248 y(partial)12 b(w)o(ord)e(accesses)i(with)f
(an)g(in)o(ternal)h(windo)o(w)g(lo)q(c)o(k)f(\(can)g(b)q(e)h(the)g(same)f(as)
f(for)h(accum)o(ulate\).)19 b(F)l(ull)75 304 y(w)o(ord)c(accesses)g(need)h
(not)f(b)q(e)h(lo)q(c)o(k)g(protected.)k(Finer)c(lo)q(c)o(k)f(gran)o(ularit)o
(y)g(\(and)g(higher)h(concurrency\))75 361 y(is)g(p)q(ossible)i(b)o(y)e
(using)g(di\013eren)o(t)h(lo)q(c)o(ks)f(for)f(di\013eren)o(t)h(parts)f(of)h
(the)g(windo)o(w.)22 b(A)16 b(shared)g(lo)q(c)o(k)g(can)g(b)q(e)75
417 y(used)g(for)e(gets,)h(and)g(an)g(exclusiv)o(e)i(lo)q(c)o(k)f(for)e
(puts.)166 474 y(Indeed,)19 b(con\015icting)g(accesses)f(can)f(only)h(b)q(e)g
(due)g(to)f Fp(false)g(sharing)p Fu(,)g(i.e.)27 b(concurren)o(t)18
b(accesses)75 530 y(to)h(di\013eren)o(t)h(parts)f(of)h(the)g(same)f(w)o(ord.)
34 b(This,)21 b(b)q(ecause)g(RMA)f(accesses)g(cannot)f(b)q(e)i(concurren)o(t)
75 587 y(with)15 b(lo)q(cal)h(accesses,)f(and)h(\(truly\))e(con\015icting)j
(RMA)e(accesses)g(cannot)g(b)q(e)h(concurren)o(t.)k(Th)o(us,)14
b(t)o(w)o(o)75 643 y(con\015icting)20 b(accesses)g(m)o(ust)e(b)q(e)i(partial)
f(w)o(ord)f(accesses,)i(and)f(these)g(comp)q(ete)h(for)e(the)h(same)g(lo)q(c)
o(k.)75 700 y(P)o(artial)g(w)o(ord)f(accum)o(ulates)i(are)e(also)h(protected)
g(b)o(y)g(that)g(lo)q(c)o(k)g(\(this)g(is)h(the)f(case)g(an)o(yho)o(w)f(if)i
(the)75 756 y(same)15 b(lo)q(c)o(k)h(is)f(used)h(for)f(accum)o(ulates)g(and)h
(partial)f(w)o(ord)g(up)q(dates.\))166 813 y(An)20 b(additional)i(problem)f
(o)q(ccurs)f(if)h(the)f(windo)o(w)g(b)q(oundaries)i(are)e(not)f(w)o(ord)h
(aligned.)36 b(If)20 b(a)75 869 y(w)o(ord)f(o)o(v)o(erlaps)h(the)f(windo)o(w)
h(b)q(oundary)h(than)e(a)h(lo)q(cal)h(access)f(to)f(the)h(part)f(of)g(the)h
(w)o(ord)f(that)g(is)75 925 y(outside)e(the)g(windo)o(w)g(can)g(b)q(e)g
(concurren)o(t)g(with)g(an)g(RMA)g(access)f(to)g(the)h(part)f(of)h(that)f(w)o
(ord)g(that)75 982 y(is)f(inside)h(the)f(windo)o(w.)20 b(This)15
b(con\015ict)g(is)g(a)o(v)o(oided)g(if)g(t)o(w)o(o)e(copies)j(\(priv)m(ate)f
(and)f(RMA\))h(of)f(this)h(w)o(ord)75 1038 y(are)g(main)o(tained)h({)f(using)
h(the)f(windo)o(w)h(cop)o(y)f(proto)q(col)g(describ)q(ed)i(ab)q(o)o(v)o(e.)
166 1095 y(An)j(implemen)o(tation)h(that)d(c)o(hec)o(ks)i(for)f(out)g(of)g
(windo)o(w)h(b)q(oundary)g(accesses)g(will)h(not)e(su\013er)75
1151 y(additional)e(o)o(v)o(erheads,)d(as)h(long)g(as)g(the)g(o)o(v)o
(erlapping)h(w)o(ord)f(is)g(not)g(up)q(dated.)166 1208 y(W)l(e)20
b(sp)q(ecialize)i(no)o(w)d(this)h(generic)g(algorithm)g(to)f(sp)q(eci\014c)i
(arc)o(hitectures,)g(with)f(noncoheren)o(t)75 1264 y(cac)o(hes.)g(There)c
(are)e(three)i(cases)f(to)f(consider.)131 1358 y(1.)22 b(RMA)16
b(is)g(implemen)o(ted)h(using)g(non-coheren)o(t)f(shared)g(memory)l(,)f(with)
h(write-through)g(nonco-)189 1414 y(heren)o(t)i(cac)o(hes.)29
b(Both)18 b(lo)q(cal)h(and)f(remote)g(writes)g(go)f(directly)j(to)d(memory)l
(,)h(but)h(b)q(oth)f(lo)q(cal)189 1471 y(and)d(remote)g(reads)g(ma)o(y)f(go)h
(to)f(cac)o(he)i(\(Cra)o(y)e(T90,)g(Cra)o(y)g(T3D?\).)131 1565
y(2.)22 b(RMA)11 b(is)g(implemen)o(ted)h(using)g(non-coheren)o(t)f(shared)g
(memory)l(,)g(with)g(write-bac)o(k)g(noncoheren)o(t)189 1621
y(cac)o(hes.)18 b(Both)10 b(lo)q(cal)i(and)f(remote)f(accesses)g(ma)o(y)g(go)
g(to)g(cac)o(he,)h(rather)f(than)g(memory)g(\(Mercury)189 1678
y(RA)o(CE)15 b(i860?\).)131 1771 y(3.)22 b(RMA)13 b(is)i(implemen)o(ted)g
(using)f(a)f(non-coheren)o(t)h(DMA)f(engine.)21 b(Remote)14
b(accesses)g(go)f(directly)189 1828 y(to)19 b(memory)l(,)g(whereas)h(lo)q
(cal)h(accesses)e(\(b)q(oth)h(reads)f(and)h(writes\))f(ma)o(y)g(go)g(to)g
(cac)o(he)g(\(NEC)189 1884 y(Cenju-3,)c(Cra)o(y)f(T3D?\).)166
1978 y(In)j(all)h(cases,)f(the)g(RMA)h(cop)o(y)e(of)h(the)g(windo)o(w)g(is)h
(in)f(memory)l(,)g(whereas)g(the)g(priv)m(ate)h(cop)o(y)e(of)75
2035 y(the)f(windo)o(w)f(consists)h(of)f(the)h(v)m(alues)g(in)h(the)e(lo)q
(cal)i(pro)q(cessor)e(cac)o(he,)h(or)f(the)h(v)m(alues)g(in)h(memory)l(,)e
(for)75 2091 y(lo)q(cations)i(that)e(are)h(not)g(cac)o(hed.)166
2148 y(Completing)g(an)g(RMA)f(up)q(date)h(is)g(ac)o(hiev)o(ed)h(b)o(y)e
(\015ushing)i(the)f(cac)o(he)g(of)f(the)g(calling)j(pro)q(cessor,)75
2204 y(so)g(that)f(the)h(up)q(date)h(b)q(ecomes)g(visible)h(in)f(the)f(RMA)g
(windo)o(w.)26 b(Suc)o(h)18 b(\015ush)f(call)i(will)f(b)q(e)g(executed)75
2260 y(as)j(part)g(of)g(calls)h(to)f Fo(MPI)p 554 2260 14 2
v 15 w(BARRIER,)i(MPI)p 874 2260 V 16 w(RMA)p 989 2260 V 16
w(COMPLETE)f Fu(and)f Fo(MPI)p 1444 2260 V 16 w(RMA)p 1559
2260 V 17 w(UNLOCK)p Fu(.)g(It)g(is)75 2317 y(su\016cien)o(t)16
b(to)e(\015ush)i(dirt)o(y)g(cac)o(he)f(lines)i(that)d(o)o(v)o(erlap)h(the)h
(windo)o(w.)166 2373 y(Cop)o(ying)f(of)g(the)g(priv)m(ate)h(cop)o(y)f(to)g
(the)g(RMA)g(cop)o(y)g(is)h(ac)o(hiev)o(ed)g(b)o(y)f(\015ushing)h(the)g(cac)o
(he)f(of)g(the)75 2430 y(lo)q(cal)h(pro)q(cessor.)k(It)15 b(is)h(su\016cien)o
(t)g(to)e(\015ush)i(dirt)o(y)f(cac)o(he)h(lines)h(that)d(o)o(v)o(erlap)h
(with)h(the)f(windo)o(w.)166 2486 y(Cop)o(ying)e(of)f(the)h(RMA)g(cop)o(y)f
(in)o(to)h(the)g(priv)m(ate)g(cop)o(y)g(is)g(ac)o(hiev)o(ed)h(b)o(y)f(in)o(v)
m(alidating)i(the)d(cac)o(he)h(of)75 2543 y(the)h(lo)q(cal)g(pro)q(cessor.)20
b(It)13 b(is)h(su\016cien)o(t)h(to)e(in)o(v)m(alidate)i(the)f(lines)h(that)e
(o)o(v)o(erlap)g(with)h(the)g(windo)o(w,)g(and)75 2599 y(are)j(dirt)o(y)g(in)
h(the)f(RMA)h(cop)o(y)l(.)25 b(On)18 b(some)f(systems,)g(\015ush)g(ma)o(y)g
(b)q(e)h(faster)e(than)h(in)o(v)m(alidate.)27 b(Since)75 2656
y(the)16 b(lo)q(cal)h(cop)o(y)f(is)h(clean,)g(all)g(the)f(windo)o(w)g(cac)o
(he)g(lines)i(are)e(clean,)h(and)f(a)g(\015ush)g(call)i(ma)o(y)d(b)q(e)i
(used,)1967 46 y Fj(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967
272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959
554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780
y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959
1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959
1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959
1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959
1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959
1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959
2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959
2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959
2643 y(47)1959 2699 y(48)p eop
%%Page: 25 27
bop 75 -100 a Fl(1.6.)29 b(EXAMPLES)1398 b Fu(25)75 49 y(instead)18
b(of)f(an)h(in)o(v)m(alidate)i(\(assuming)d(that)g(the)h(cac)o(he)g(con)o
(troller)g(is)g(smart)f(enough)h(not)f(to)g(write)75 106 y(bac)o(k)e(clean)h
(cac)o(he)g(lines.\))166 162 y(Tw)o(o)g(windo)o(w)h(copies)g(ma)o(y)f(also)h
(b)q(e)g(used)h(in)f(a)f(shared)h(memory)f(implemen)o(tation)i(where)f(not)75
219 y(all)f(pro)q(cesses)g(can)g(access)g(all)g(memory)l(.)21
b(E.g.,)14 b(in)i(the)g(case)f(where)h(a)f(windo)o(w)h(is)g(in)g(pro)q(cess)g
(priv)m(ate)75 275 y(memory)l(,)22 b(one)f(ma)o(y)f(use)h(an)g(RMA)h(cop)o(y)
e(that)h(is)g(in)h(shared)f(memory)g(and)g(is)g(accessible)i(b)o(y)e(all)75
332 y(pro)q(cesses)16 b(in)h(the)f(comm)o(unication)g(group.)22
b(Or,)15 b(the)h(RMA)g(cop)o(y)g(could)g(b)q(e)h(in)g(nonpaged)f(memory)l(,)
75 388 y(that)j(is)i(accessible)h(to)d(DMA,)g(whereas)h(the)g(priv)m(ate)h
(cop)o(y)f(is)g(in)h(pageable)g(memory)l(.)34 b(Then,)21 b(the)75
444 y(cop)o(ying)13 b(of)g(one)g(cop)o(y)g(to)f(another)h(actually)h
(requires)f(memory)g(to)f(memory)g(cop)o(y)l(,)i(that)e(can)h(b)q(e)h(done)75
501 y(b)o(y)j(the)g(windo)o(w)g(o)o(wning)g(pro)q(cess.)25
b(F)l(ull)18 b(windo)o(w)f(cop)o(ying)g(can)g(b)q(e)h(a)o(v)o(oided)f(b)o(y)g
(main)o(taining)g(\014ner)75 557 y(dirt)o(y)e(bits.)20 b(\(Dirt)o(y)13
b(bits)i(for)f(the)h(RMA)g(cop)o(y)f(are)g(main)o(tained)i(b)o(y)e(trac)o
(king)g(the)h(lo)q(cations)g(accessed)75 614 y(b)o(y)g(put)h(or)f(accum)o
(ulate)h(calls)g({)g(this)f(can)h(b)q(e)g(part)f(of)g(the)h(co)q(de)g(for)f
(these)g(calls.)22 b(Dirt)o(y)15 b(bits)h(for)f(the)75 670
y(lo)q(cal)k(cop)o(y)f(are)g(harder)g(to)g(main)o(tain.)29
b(One)19 b(needs)g(to)f(use)g(tec)o(hniques)h(similar)h(to)d(those)h(used)h
(b)o(y)75 727 y(soft)o(w)o(are)12 b(implemen)o(tations)i(of)f(distributed)i
(shared)f(memory:)19 b(e.g.,)12 b(use)i(page)f(protection)h(to)f(detect)75
783 y(lo)q(cal)j(up)q(dates)g(at)f(page)g(gran)o(ularit)o(y)l(.\))75
902 y Fe(1.5.4)49 b(Registers)15 b(and)i(Compiler)e(Optimizations)75
988 y Fu(Another)e(coherence)g(problem)h(exists)f(b)q(et)o(w)o(een)g(v)m
(ariables)h(k)o(ept)e(in)i(registers)e(and)h(the)g(memory)f(v)m(alue)75
1044 y(of)19 b(these)g(v)m(ariables.)32 b(An)20 b(RMA)f(call)h(ma)o(y)e
(access)h(a)g(v)m(ariable)h(in)g(memory)f(\(or)f(cac)o(he\),)h(while)i(the)75
1101 y(up-to-date)12 b(v)m(alue)h(of)f(this)h(v)m(ariable)g(is)g(in)g
(register.)18 b(A)12 b(get)g(will)i(not)e(return)g(the)g(latest)g(v)m
(ariable)h(v)m(alue,)75 1157 y(and)i(a)g(put)h(ma)o(y)e(b)q(e)i(o)o(v)o
(erwritten)e(when)i(the)f(register)g(is)h(stored)f(bac)o(k)g(in)h(memory)l(.)
166 1213 y(The)f(problem)h(is)g(illustrated)g(b)o(y)g(the)f(follo)o(wing)h
(co)q(de:)110 1297 y(Source)g(of)f(Pro)q(cess)g(1)122 b(Source)16
b(of)f(Pro)q(cess)g(2)122 b(Executed)16 b(in)g(Pro)q(cess)f(2)110
1362 y(bbbb)i(=)e(777)272 b(bu\013)15 b(=)h(999)296 b(reg)p
1190 1362 14 2 v 16 w(A:=999)110 1418 y(call)17 b(MPI)p 286
1418 V 16 w(PUT\(bbbb)611 b(stop)15 b(appl.)8 b(thread)110
1475 y(in)o(to)16 b(bu\013)f(of)g(pro)q(cess)g(2\))580 b(bu\013:=777)15
b(in)h(PUT)f(handler)1126 1531 y(con)o(tin)o(ue)h(appl.)8 b(thread)110
1588 y(call)17 b(MPI)p 286 1588 V 16 w(BARRIER\(...\))32 b(call)17
b(MPI)p 794 1588 V 16 w(BARRIER\(...\))618 1644 y(ccc)f(=)g(bu\013)304
b(ccc:=reg)p 1298 1644 V 17 w(A)166 1728 y(In)18 b(this)f(example,)h(v)m
(ariable)g(bu\013)f(is)h(allo)q(cated)g(in)g(the)f(register)g(reg)p
1380 1728 V 16 w(A)g(and)g(therefore)g(ccc)g(will)75 1784 y(ha)o(v)o(e)e(the)
g(old)h(v)m(alue)g(of)f(bu\013)g(and)h(not)e(the)i(new)f(v)m(alue)i(777.)166
1841 y(This)11 b(problem,)h(whic)o(h)f(also)g(a\017icts)f(in)i(some)e(cases)h
(send/receiv)o(e)g(comm)o(unication,)h(is)f(discussed)75 1897
y(more)k(at)f(length)i(in)g(Section)g(1.7.)166 1954 y(MPI)k(implemen)o
(tations)i(will)f(a)o(v)o(oid)f(this)h(problem)g(for)e(standard)h(conforming)
h(C)f(programs.)75 2010 y(High)13 b(qualit)o(y)h(MPI)f(F)l(ortran)e(implemen)
o(tations)j(will)h(a)o(v)o(oid)d(this)i(problem)f(for)g(standard)f
(conforming)75 2067 y(F)l(ortran)i(programs)h(as)g(w)o(ell,)h(without)g
(disabling)h(compiler)g(optimizations.)22 b(Ho)o(w)o(ev)o(er,)14
b(MPI)h(imple-)75 2123 y(men)o(tations)h(are)h(not)f(required)i(to)e(a)o(v)o
(oid)h(altogether)f(the)h(register)f(coherence)i(problem,)g(since)g(this)75
2180 y(requires)d(compiler)h(supp)q(ort)e(that)g(is)h(not)f(curren)o(tly)h(a)
o(v)m(ailable)h(on)f(most)e(F)l(ortran)g(en)o(vironmen)o(ts.)20
b(In)75 2236 y(order)14 b(to)f(a)o(v)o(oid)g(register)h(coherence)h
(problems,)f(users)g(should)h(restrict)e(their)h(use)h(of)e(RMA)h(windo)o(ws)
75 2293 y(to)h(v)m(ariables)h(stored)f(in)h Fi(COMMON)f Fu(blo)q(c)o(ks,)h
(or)e(to)h(v)m(ariables)i(that)d(w)o(ere)h(declared)i Fi(V)o(OLA)m(TILE)d
Fu(\(while)75 2349 y Fi(V)o(OLA)m(TILE)e Fu(is)h(no)g(a)f(standard)h(F)l
(ortran)f(declaration,)h(it)h(is)f(supp)q(orted)g(b)o(y)g(man)o(y)g(F)l
(ortran)e(compilers\).)75 2405 y(Details,)k(and)h(an)f(additional)h(solution)
g(are)f(discussed)i(in)f(Section)g(1.7)e(on)i(page)f(31.)75
2546 y Fr(1.6)59 b(Examples)75 2647 y Fn(Example)18 b(1.7)k
Fu(The)17 b(follo)o(wing)f(example)h(sho)o(ws)e(a)h(generic)h(lo)q(osely)g
(sync)o(hronous,)f(iterativ)o(e)g(co)q(de,)75 2704 y(using)g(barrier)f(sync)o
(hronization.)21 b(The)16 b(windo)o(w)f(at)g(eac)o(h)g(pro)q(cess)g(consists)
h(of)e(arra)o(y)g Fq(A)p Fu(.)-32 46 y Fj(1)-32 103 y(2)-32
159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32
441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40
724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40
1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232
y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40
1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740
y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40
2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248
y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40
2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop
%%Page: 26 28
bop 75 -100 a Fu(26)728 b Fl(CHAPTER)15 b(1.)30 b(ONE)16 b(SIDED)f(COMMUNICA)
l(TIONS)75 49 y Fq(...)75 106 y(while\(!converged\(A\)\){)147
162 y(update\(A\);)147 219 y(MPI_Barrier\(comm\);)147 275 y(for\(i=0;)22
b(i)i(<)g(toneighbors;)e(i++\))218 332 y(MPI_Put\(frombuf\(i\),)f(1,)j
(fromtype\(i\),)e(toneighbor\(i\),)672 388 y(todisp\(i\),)g(1,)i
(totype\(i\),)e(comm\);)147 444 y(MPI_Barrier\(comm\);)147
501 y(})75 591 y Fu(Same)11 b(co)q(de)h(could)h(b)q(e)f(written)f(with)h
(get,)f(rather)g(than)h(put.)18 b(Note)11 b(that,)g(during)i(the)e(comm)o
(unication)75 648 y(phase,)19 b(eac)o(h)f(windo)o(w)g(is)h(concurren)o(tly)f
(read)g(\(as)g(origin)g(bu\013er)g(of)g(puts\))g(and)g(written)g(\(as)f
(target)75 704 y(bu\013er)e(of)g(puts\).)20 b(This)15 b(is)h(OK,)g(pro)o
(vided)g(that)e(there)h(is)h(no)f(o)o(v)o(erlap)g(b)q(et)o(w)o(een)h
(bu\013ers.)75 806 y Fn(Example)i(1.8)k Fu(Same)13 b(generic)g(example,)h
(with)e(more)g(computation/comm)o(unication)h(o)o(v)o(erlap.)19
b(W)l(e)75 862 y(assume)14 b(that)f(the)g(up)q(date)i(phase)f(is)g(brok)o(en)
g(in)g(t)o(w)o(o)f(subphases:)20 b(the)13 b(\014rst,)h(where)g(the)f(\\b)q
(oundary",)75 919 y(whic)o(h)18 b(is)f(in)o(v)o(olv)o(ed)h(in)f(comm)o
(unication,)h(is)f(up)q(dated,)h(and)f(the)g(second,)g(where)g(the)g
(\\core",)f(whic)o(h)75 975 y(neither)g(use)g(nor)f(pro)o(vide)g(comm)o
(unicated)h(data,)e(is)i(up)q(dated.)75 1066 y Fq(...)75 1122
y(while\(!converged\(A\)\){)147 1179 y(update_boundary\(A\);)147
1235 y(MPI_Barrier\(comm\);)147 1292 y(for\(i=0;)22 b(i)i(<)g(fromneighbors;)
e(i++\))218 1348 y(MPI_Get\(tobuf\(i\),)g(1,)h(totype\(i\),)g
(fromneighbor\(i\),)576 1405 y(fromdisp\(i\),)f(1,)i(fromtype\(i\),)e
(comm\);)218 1461 y(update_core\(comm\);)147 1518 y(MPI_Barrier\(comm\);)147
1574 y(})75 1665 y Fu(The)12 b(get)f(comm)o(unication)h(can)f(b)q(e)h
(concurren)o(t)g(with)g(the)f(core)g(up)q(date,)i(since)f(they)g(do)f(not)g
(access)h(the)75 1721 y(same)j(lo)q(cations,)i(and)f(the)g(lo)q(cal)h(up)q
(date)f(of)f(the)h(origin)h(bu\013er)f(b)o(y)f(the)h(get)g(call)g(can)g(b)q
(e)h(concurren)o(t)75 1777 y(with)g(the)f(lo)q(cal)h(up)q(date)g(of)f(the)g
(core)h(b)o(y)f(the)g Fq(update)p 1041 1777 15 2 v 16 w(core)g
Fu(call.)24 b(In)17 b(order)f(to)g(get)g(similar)h(o)o(v)o(erlap)75
1834 y(with)g(put)h(comm)o(unication)f(w)o(e)g(w)o(ould)h(need)g(to)e(use)i
(separate)e(windo)o(ws)h(for)g(the)g(core)g(and)g(for)g(the)75
1890 y(b)q(oundary)l(.)75 1992 y Fn(Example)h(1.9)k Fu(Same)14
b(co)q(de)g(as)g(in)g(Example)g(1.7,)f(rewritten)h(using)g(w)o(ait-p)q
(ost-start-complete.)19 b(W)l(e)75 2049 y(assume)c(that)g(eac)o(h)g(pro)q
(cess)g(kno)o(ws)g(n)o(um)o(b)q(er)g(of)g(pro)q(cesses)h(that)e(put)i(in)o
(to)f(its)g(windo)o(w.)75 2139 y Fq(...)75 2195 y(while\(!converged\(A\)\){)
147 2252 y(update\(A\);)147 2308 y(MPI_RMA_post\(comm\);)147
2365 y(for\(i=0;)22 b(i)i(<)g(toneighbors;)e(i++\))h({)75 2421
y(/*)h(depending)e(on)i(implementation,)e(it)h(may)h(be)f(more)h(efficient)e
(to)48 b(*/)75 2478 y(/*)24 b(split)f(this)g(loop)g(into)h(two)f(or)h(three)f
(separate)g(loops)238 b(*/)218 2534 y(MPI_RMA_start\(comm,)21
b(toneighbor\(i\),)h(MPI_WEAK\);)218 2591 y(MPI_Put\(frombuf\(i\),)f(1,)j
(fromtype\(i\),)e(toneighbor\(i\),)672 2647 y(todisp\(i\),)g(1,)i
(totype\(i\),)e(comm\);)218 2704 y(MPI_RMA_complete\(comm,)f
(toneighbor\(i\)\);)1967 46 y Fj(1)1967 103 y(2)1967 159 y(3)1967
215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967
498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724
y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959
1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959
1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959
1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959
1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959
1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959
2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959
2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959
2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop
%%Page: 27 29
bop 75 -100 a Fl(1.6.)29 b(EXAMPLES)1398 b Fu(27)218 49 y Fq(})147
106 y(MPI_RMA_wait\(fromneighb)o(ors\)\);)147 162 y(})75 267
y Fn(Example)18 b(1.10)23 b Fu(Same)15 b(example,)g(with)h(split)g(phases,)f
(as)g(in)h(Example)g(1.8.)75 360 y Fq(...)75 417 y(while\(!converged\(A\)\){)
147 473 y(update_boundary\(A\);)147 530 y(MPI_RMA_post\(comm\);)147
586 y(for\(i=0;)22 b(i)i(<)g(fromneighbors;)e(i++\){)218 643
y(MPI_RMA_start\(comm,)f(fromneighbor\(i\),)h(MPI_WEAK\);)218
699 y(MPI_Get\(tobuf\(i\),)g(1,)h(totype\(i\),)g(fromneighbor\(i\),)576
755 y(fromdisp\(i\),)f(1,)i(fromtype\(i\),)e(comm\);)218 812
y(})147 868 y(update_core\(A\);)147 925 y(for\(i=0;)g(i)i(<)g(fromneighbors;)
e(i++\))242 981 y(MPI_RMA_complete\(comm,)f(fromneighbor\(i\)\);)147
1038 y(MPI_RMA_wait\(toneighbor)o(s\);)147 1094 y(})75 1199
y Fn(Example)d(1.11)23 b Fu(A)12 b(c)o(hec)o(k)o(erb)q(oard,)h(or)f(double)h
(bu\013er)g(comm)o(unication)g(pattern,)f(that)g(allo)o(ws)g(more)75
1256 y(computation/comm)o(unication)19 b(o)o(v)o(erlap.)30
b(Arra)o(y)17 b Fq(A0)h Fu(is)h(up)q(dated)h(using)f(v)m(alues)h(of)e(arra)o
(y)f Fq(A1)p Fu(,)i(and)75 1312 y(vice)j(v)o(ersa.)38 b(W)l(e)22
b(assume)f(that)g(comm)o(unication)h(is)g(symmetric:)32 b(if)22
b(pro)q(cess)g(A)f(gets)g(data)g(from)75 1369 y(pro)q(cess)15
b(B,)g(then)h(pro)q(cess)f(B)h(gets)e(data)h(from)f(pro)q(cess)i(A.)f(Windo)o
(w)g Fq(commi)g Fu(consists)g(of)g(arra)o(y)f Fq(Ai)p Fu(.)75
1462 y Fq(...)75 1518 y(MPI_RMA_post\(comm0\);)75 1574 y
(MPI_Barrier\(comm0\);)75 1687 y(while\(!converged\(A0,)21
b(A1\)\){)147 1744 y(/*)i(communication)f(on)i(A0)f(and)h(computation)e(on)i
(A1)f(*/)147 1800 y(update\(A1,)f(A0\))i(/*)f(local)g(update)g(of)h(A1)g
(that)f(depends)g(on)g(A0)h(\(and)f(A1\))h(*/)147 1857 y(for\(i=0;)e(i)i(<)g
(neighbors;)e(i++\){)218 1913 y(MPI_RMA_start\(comm0,)f(neighbor\(i\),)i
(MPI_NOCHECK\);)218 1970 y(MPI_Get\(tobuf0\(i\),)f(1,)h(totype0\(i\),)g
(neighbor\(i\),)528 2026 y(fromdisp0\(i\),)f(1,)i(fromtype0\(i\),)e(comm0\);)
218 2083 y(})147 2139 y(update\(A1\);)g(/*)i(local)f(update)g(of)g(A1)h(that)
f(depends)g(on)h(A1)f(only,)505 2195 y(concurrent)f(with)h(communication)f
(that)i(updates)f(A0)g(*/)147 2252 y(MPI_RMA_post\(comm1\);)147
2308 y(for\(i=0;)f(i)i(<)g(neighbors;)e(i++\))218 2365 y
(MPI_RMA_complete\(comm0,)f(neighbor\(i\)\);)147 2421 y(MPI_RMA_wait\(comm0,)
g(neighbors\);)147 2534 y(/*)i(communication)f(on)i(A1)f(and)h(computation)e
(on)i(A0)f(*/)147 2591 y(update\(A0,)f(A1\))i(/*)f(local)g(update)g(of)h(A0)g
(that)f(depends)g(on)g(A1)h(\(and)f(A0\)*/)147 2647 y(for\(i=0;)f(i)i(<)g
(neighbors;)e(i++\){)218 2704 y(MPI_RMA_start\(comm1,)f(neighbor\(i\),)i
(MPI_NOCHECK\);)-32 46 y Fj(1)-32 103 y(2)-32 159 y(3)-32 215
y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498
y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40
780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40
1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288
y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40
1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796
y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40
2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304
y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40
2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop
%%Page: 28 30
bop 75 -100 a Fu(28)728 b Fl(CHAPTER)15 b(1.)30 b(ONE)16 b(SIDED)f(COMMUNICA)
l(TIONS)218 49 y Fq(MPI_Get\(tobuf1\(i\),)22 b(1,)h(totype1\(i\),)g
(neighbor\(i\),)528 106 y(fromdisp1\(i\),)f(1,)i(fromtype1\(i\),)e(comm1\);)
218 162 y(})147 219 y(update\(A0\);)g(/*)i(local)f(update)g(of)g(A0)h(that)f
(depends)g(on)h(A0)f(only,)505 275 y(concurrent)f(with)h(communication)f
(that)i(updates)f(A1)g(*/)147 332 y(MPI_RMA_post\(comm0\);)147
388 y(for\(i=0;)f(i)i(<)g(neighbors;)e(i++\))218 444 y
(MPI_RMA_complete\(comm1,)f(neighbor\(i\)\);)147 501 y(MPI_RMA_wait\(comm1,)g
(neighbors\);)147 557 y(})75 648 y Fu(A)11 b(pro)q(cess)g(p)q(osts)g(the)g
(lo)q(cal)h(windo)o(w)f(asso)q(ciated)g(with)g Fq(commi)f Fu(b)q(efore)i(it)f
(completes)g(RMA)g(accesses)h(to)75 704 y(the)j(remote)f(windo)o(ws)h(asso)q
(ciated)g(with)g Fq(comm\(1-i\))p Fu(.)j(When)e(the)e Fq(wait\(commi)p
Fu(\))f(call)j(returns,)f(then)75 761 y(all)k(neigh)o(b)q(ors)h(of)e(the)h
(calling)h(pro)q(cess)f(ha)o(v)o(e)f(p)q(osted)h(the)f(windo)o(ws)h(asso)q
(ciated)g(with)g Fq(comm\(1-i\))p Fu(.)75 817 y(Therefore,)c(no)g(c)o(hec)o
(k)g(is)h(needed)h(on)e(the)g(follo)o(wing)h Fo(MPI)p 1095
817 14 2 v 16 w(RMA)p 1210 817 V 16 w(ST)l(ART)h Fu(call.)75
919 y Fn(Example)h(1.12)23 b Fu(This)12 b(example)g(sho)o(ws)f(ho)o(w)h(one)f
(can)h(manipulate)h(a)f(distributed)h(link)o(ed)g(list.)20
b(Lo)q(c)o(k)75 976 y(sync)o(hronization)13 b(is)g(used.)20
b(All)13 b(accesses)g(of)f(the)g(list)h(are)f(protected)h(b)o(y)f(one)h
(master)e(lo)q(c)o(k.)19 b(Therefore,)75 1032 y(no)c(c)o(hec)o(king)h(is)g
(required)g(on)f(lo)q(c)o(ks)h(that)e(con)o(trol)h(access)h(to)e(the)i(link)o
(ed)g(list)g(itself.)75 1123 y Fq(...)75 1179 y(typedef)23
b(struct{)170 1236 y(int)h(rank;)170 1292 y(int)g(disp;)75
1349 y(})g(GPOINTER;)75 1405 y(/*)g(global)f(pointer)g(identifies)f(node)h
(by)h(rank)f(of)h(containing)e(window,)h(*/)75 1462 y(/*)h(and)f
(displacement)f(within)h(window)739 b(*/)75 1574 y(typedef)23
b(struct{)147 1631 y(char)g(info[8];)147 1687 y(GPOINTER)f(next;)75
1744 y(})i(NODE;)75 1857 y(MPI_Datatype)e(GPOINTERTYPE,)g(NODETYPE;)75
1970 y(MPI_Comm)h(control,)g(/*)g(window)g(to)h(control)f(block)g(stored)g
(at)g(process)g(0.)96 b(*/)505 2026 y(/*)23 b(contain)g(global)g(pointer)g
(to)h(dummy)f(node)g(that)g(is)h(at)g(*/)505 2083 y(/*)47 b(head)23
b(of)h(list)f(and)h(global)f(pointer)g(to)g(head)h(of)119 b(*/)505
2139 y(/*)47 b(free)23 b(list.)883 b(*/)290 2195 y(heap;)95
b(/*)23 b(window)g(to)h(heap,)f(distributed)f(on)i(all)f(processes)95
b(*/)290 2252 y(lock;)g(/*)23 b(master)g(lock)883 b(*/)75 2365
y(/*)24 b(initialization)e(code.)47 b(Executed)22 b(collectively)h(by)g(all)h
(processes.)e(*/)75 2421 y(void)h(heap_init\(int)f(heap_size\))75
2478 y({)75 2534 y(...)75 2591 y(MPI_Comm_rank\(MPI_COMM_WOR)o(LD,)f
(&rank\);)75 2647 y(MPI_Comm_size\(MPI_COMM_WOR)o(LD,)g(&commsize\);)1967
46 y Fj(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967
328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959
611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836
y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959
1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959
1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959
1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959
1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959
2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959
2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959
2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959
2699 y(48)p eop
%%Page: 29 31
bop 75 -100 a Fl(1.6.)29 b(EXAMPLES)1398 b Fu(29)75 49 y Fq(/*)24
b(allocate)e(heap)i(*/)75 106 y(MPI_Mem_alloc\(heap_size*si)o(zeof\(NO)o
(DE\),)d(MPI_COMM_WORLD,)h(&heapbase\);)75 162 y(MPI_RMA_init\(heapbase,)f
(heap_size*sizeof\(NODE\),)g(sizeof\(NODE\),)910 219 y(MPI_COMM_WORLD,)h
(heap\);)75 332 y(/*)i(allocate)e(control)h(block)g(*/)75 388
y(if\(rank==0\){)147 444 y(MPI_Mem_alloc\(2*sizeof\()o(GPOINTE)o(R\),)e
(MPI_COMM_WORLD,)h(&cntrlbase\);)147 501 y(MPI_RMA_init\(cntrlbase,)e
(2*sizeof\(GPOINTER\),)i(sizeof\(GPOINTER\),)910 557 y(MPI_COMM_WORLD,)g
(control\);)524 b(})75 614 y(else)147 670 y(MPI_RMA_init\(cntrlbase,)20
b(0,)k(1,)f(MPI_COMM_WORLD,)f(control\);)75 783 y(/*)i(initialize)e(master)h
(lock)g(*/)75 840 y(MPI_RMA_init\(cntrlbase,)e(0,)i(1,)h(MPI_COMM_WORLD,)e
(lock\);)75 953 y(/*)i(initialize)e(free)h(list)h(*/)75 1009
y(...)75 1065 y(/*)g(initialize)e(control)h(block)g(*/)75 1122
y(...)75 1178 y(/*)h(initialize)e(datatypes)h(*/)75 1235 y(...)75
1291 y(})75 1404 y(int)g(delete\(GNODE)g(*previous\))75 1461
y(/*)h(delete)f(node)g(after)g("previous")g(node)g(in)h(list)f(*/)75
1517 y({)75 1574 y(GPOINTER)g(free,)g(temp;)75 1630 y(NODE)g(prevnode,)g
(delnode;)75 1743 y(MPI_RMA_lock\(lock,)e(0,)j(MPI_EXCLUSIVE,)e
(MPI_STRONG\);)75 1856 y(/*)i(get)f(free)g(list)h(head)f(pointer)g(*/)75
1912 y(MPI_RMA_lock\(control,)e(0,)j(MPI_EXCLUSIVE,)e(MPI_NOCHECK\);)75
1969 y(MPI_Get\(&free,)g(1,)i(GPOINTERTYPE,)e(0,)h(1,)h(1,)f(GPOINTERTYPE,)f
(control\);)75 2025 y(MPI_RMA_unlock\(control,)f(0\);)75 2138
y(/*)j(get)f(previous)g(node)g(*/)75 2195 y(MPI_RMA_lock\(heap,)e
(previous.rank,)h(MPI_EXCLUSIVE,)g(MPI_NOCHECK\);)75 2251 y
(MPI_Get\(&prevnode,)f(1,)j(NODETYPE,)f(previous.rank,)f(previous.disp,)g(1,)
1149 2307 y(NODETYPE,)g(heap\);)75 2364 y(MPI_RMA_unlock\(heap,)f
(previous.rank\);)75 2477 y(/*)j(get)f(deleted)g(node)g(*/)75
2533 y(MPI_RMA_lock\(heap,)e(prevnode.next.rank,)h(MPI_EXCLUSIVE,)g
(MPI_NOCHECK\);)75 2590 y(MPI_Get\(&delnode,)g(1,)h(NODETYPE,)g
(prevnode.next.rank,)600 2646 y(prevnode.next.disp,)e(1,)j(NODETYPE,)e
(heap\);)75 2703 y(MPI_RMA_unlock\(heap,)f(prevnode.next.rank\);)-32
46 y Fj(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32
328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40
611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40
893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119
y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40
1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627
y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40
1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135
y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40
2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643
y(47)-40 2699 y(48)p eop
%%Page: 30 32
bop 75 -100 a Fu(30)728 b Fl(CHAPTER)15 b(1.)30 b(ONE)16 b(SIDED)f(COMMUNICA)
l(TIONS)75 106 y Fq(/*)24 b(delete)f(node)g(from)g(list)h(*/)75
162 y(temp.rank)f(=)g(prevnode.next.rank;)75 219 y(temp.disp)g(=)g
(prevnode.next.disp;)75 275 y(prevnode.next.rank)e(=)j(delnode.next.rank;)75
332 y(prevnode.next.disp)d(=)j(delnode.next.disp;)75 388 y
(MPI_RMA_lock\(heap,)d(previous.rank,)h(MPI_EXCLUSIVE,)g(MPI_NOCHECK\);)75
444 y(MPI_Put\(&prevnode,)f(1,)j(NODETYPE,)f(previous.rank,)f(previous.disp,)
g(1,)1149 501 y(NODETYPE,)g(heap\);)75 614 y(/*)i(return)f(deleted)g(node)g
(to)g(free)h(list)f(*/)75 670 y(delnode.next.rank)f(=)h(free.rank;)75
727 y(delnode.next.disp)f(=)h(free.disp;)75 783 y(MPI_RMA_lock\(heap,)e
(temp.rank,)i(MPI_EXCLUSIVE,)f(MPI_NOCHECK\);)75 840 y(MPI_Put\(&delnode,)g
(1,)h(NODETYPE,)g(temp.rank,)f(temp.disp,)h(1,)1149 896 y(NODETYPE,)f
(heap\);)75 1009 y(/*)i(update)f(free)g(list)g(head)h(pointer)e(*/)75
1065 y(MPI_RMA_lock\(control,)f(0,)j(MPI_EXCLUSIVE,)e(MPI_NOCHECK\);)75
1122 y(MPI_Put\(&temp,)g(1,)i(GPOINTERTYPE,)e(0,)h(1,)h(1,)f(GPOINTERTYPE,)f
(control\);)75 1235 y(MPI_RMA_unlock\(heap,)f(previous.rank\);)75
1291 y(MPI_RMA_unlock\(heap,)g(temp.rank\);)75 1348 y
(MPI_RMA_unlock\(control,)g(0\);)75 1404 y(MPI_RMA_unlock\(lock,)g(0\);)75
1461 y(return\(0\);)75 1517 y(})75 1630 y(int)i(insert\(char)g(*info,)g
(GNODE)g(*previous\))75 1686 y(/*)h(insert)f(new)g(node)g(with)h("info")f
(after)g("previous")f(node)i(in)f(list)h(*/)75 1743 y({)75
1799 y(MPI_RMA_lock\(lock,)d(0,)j(MPI_STRONG\);)75 1856 y(...)75
1912 y(MPI_RMA_unlock\(lock,)d(0\);)75 1969 y(})75 2082 y(int)i(search\(info)
g(*char,)g(GPOINTER)g(*previous\);)75 2138 y(/*)h(search)f(node)g(with)g
("info")g(and)h(return)f(pointer)g(to)g(previous)g(node)g(*/)75
2195 y({)75 2251 y(GPOINTER)g(gpointer;)75 2307 y(NODE)g(node;)75
2420 y(MPI_RMA_lock\(lock,)e(0,)j(MPI_SHARED,)e(MPI_STRONG\);)75
2533 y(/*)i(get)f(\(dummy\))g(node)g(at)h(head)f(lof)h(list)f(*/)75
2590 y(MPI_RMA_lock\(control,)e(0,)j(MPI_SHARED,)e(MPI_NOCHECK\);)75
2646 y(MPI_Get\(&gpointer,)f(1,)j(GPOINTERTYPE,)e(0,)i(0,)f(1,)h
(GPOINTERTYPE,)e(control\);)75 2703 y(MPI_RMA_unlock\(control,)f(0\);)1967
46 y Fj(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967
328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959
611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836
y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959
1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959
1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959
1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959
1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959
2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959
2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959
2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959
2699 y(48)p eop
%%Page: 31 33
bop 75 -100 a Fl(1.7.)34 b(A)15 b(F)o(OR)l(TRAN)h(PR)o(OBLEM)g(WITH)f
(REGISTER)i(OPTIMIZA)l(TION)361 b Fu(31)75 49 y Fq(MPI_RMA_lock\(heap,)21
b(gpointer.rank,)h(MPI_SHARED,)h(MPI_NOCHECK\);)75 106 y(MPI_Get\(&node,)f
(1,)i(NODETYPE,)e(gpointer.rank,)g(gpointer.disp,)1006 162
y(1,)h(NODETYPE,)g(heap\);)75 219 y(MPI_RMA_unlock\(heap,)e(gpointer.rank\);)
75 332 y(/*)j(search)f(*/)75 388 y(while\(node.next.rank)e(!=)j(-1\){)147
444 y(MPI_RMA_lock\(heap,)d(node.next.rank,)h(MPI_SHARED,)g(MPI_NOCHECK\);)
147 501 y(MPI_Get\(&node,)g(1,)h(NODETYPE,)g(node.next.rank,)f
(node.next.disp,)1006 557 y(1,)h(NODETYPE,)g(heap\);)147 614
y(MPI_RMA_unlock\(heap,)e(node.next.rank\);)147 670 y(if\(strcmp\(info,)g
(node.info\)==0\){)218 727 y(previous.rank)h(=)i(gpointer.rank;)218
783 y(previous.disp)e(=)i(gpointer.disp;)218 840 y(MPI_RMA_unlock\(lock,)d
(0\);)218 896 y(return\(0\);)218 953 y(})147 1009 y(else{)218
1065 y(gpointer.rank)h(=)i(node.next.rank;)218 1122 y(gpointer.disp)e(=)i
(node.next.disp;)218 1178 y(})147 1235 y(})75 1291 y(MPI_RMA_unlock\(lock,)d
(0\);)75 1348 y(return\(ERR_NOTFOUND\);)75 1404 y(})75 1498
y Fu(The)d(co)q(de)h(is)f(quite)h(\(to)q(o?\))28 b(v)o(erb)q(ose,)19
b(and)f(has)g(man)o(y)f(apparen)o(tly)i(sup)q(er\015uous)g(lo)q(c)o(k)f
(calls.)30 b(This,)75 1554 y(b)q(ecause)16 b(of)f(the)g(design)i(c)o(hoice)f
(of)e(separating)i(sync)o(hronization)g(from)e(comm)o(unication,)i(and)f
(using)75 1611 y(the)k(same)f(put)g(and)h(get)f(calls)h(b)q(oth)g(for)f
(2-part)o(y)f(and)i(for)f(3-part)o(y)f(comm)o(unication.)30
b(In)19 b(order)g(to)75 1667 y(supp)q(ort)12 b(e\016cien)o(tly)i(2-part)o(y)e
(comm)o(unication,)h(it)f(is)h(imp)q(ortan)o(t)f(that)g(2-part)o(y)f(and)i
(3-part)o(y)e(comm)o(u-)75 1724 y(nication)j(b)q(e)f(syn)o(tactically)g
(distinct.)20 b(T)l(o)13 b(do)f(so)g(in)i(a)e(less)h(v)o(erb)q(ose)f(manner,)
h(w)o(e)f(w)o(ould)h(need)h(distinct)75 1780 y(put/get)j(calls)i(\(or)e(same)
h(calls)h(with)f(an)g(additional)h(argumen)o(t\))e(for)g(eac)o(h)h(case.)28
b(The)19 b(curren)o(t)e(de-)75 1837 y(sign)f(facilitates)h(the)f(expression)h
(and)g(e\016cien)o(t)f(implemen)o(tation)i(of)d(2-part)o(y)g(comm)o
(unication,)i(and)75 1893 y(esp)q(ecially)k(of)c(the)h(prev)m(alen)o(t)h(lo)q
(osely)g(sync)o(hronous)f(st)o(yle.)28 b(It)18 b(is)g(less)h(suited)f(to)g
(\\shared)g(memory)75 1950 y(em)o(ulation".)35 b(On)20 b(distributed)i
(memory)d(systems,)h(suc)o(h)g(em)o(ulation)h(en)o(tails,)h(an)o(yho)o(w,)e
(the)g(o)o(v)o(er-)75 2006 y(head)c(of)f(async)o(hronous)g(agen)o(t)g(in)o(v)
o(o)q(cation,)g(so)h(that)e(the)i(additional)h(function)f(calls)g(will)i(not)
d(a\013ect)75 2063 y(p)q(erformance)g(signi\014can)o(tly:)22
b(these)15 b(calls)h(do)g(not)e(force)h(sup)q(er\015uous)i(comm)o(unication.)
75 2206 y Fr(1.7)59 b(A)20 b(F)n(o)n(rtran)g(Problem)e(with)h(Register)h
(Optimization)75 2354 y Fh(Discussion:)15 b Fk(This)f(section)h(should)f(b)q
(e)g(included)g(in)o(to)f(MISC)189 2508 y Fp(A)n(dvic)n(e)i(to)i(users.)189
2583 y Fu(MPI)d(pro)o(vides)g(op)q(erations)g(whic)o(h)h(ma)o(y)e(b)q(e)i
(hidden)h(from)d(the)h(user)g(co)q(de)h(and)f(run)h(in)f(parallel)189
2640 y(with)g(it,)h(accessing)g(the)f(same)g(memory)g(as)g(user)g(co)q(de.)20
b(Examples)15 b(include)i(the)d(data)g(transfer)189 2696 y(for)c(an)h
Fo(MPI)p 398 2696 14 2 v 16 w(RECV)h Fu(The)f(optimizer)i(of)d(a)h(compiler)i
(will)f(assume)f(that)g(it)g(can)h(recognize)g(p)q(erio)q(ds)-32
46 y Fj(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32
328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40
611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40
893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119
y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40
1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627
y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40
1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135
y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40
2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643
y(47)-40 2699 y(48)p eop
%%Page: 32 34
bop 75 -100 a Fu(32)723 b Fl(CHAPTER)15 b(1.)35 b(ONE)16 b(SIDED)f(COMMUNICA)
l(TIONS)189 49 y Fu(when)g(a)g(cop)o(y)g(of)g(a)f(v)m(ariable)j(can)e(b)q(e)h
(k)o(ept)f(in)h(a)e(register)h(without)g(reloading)h(from)f(or)f(storing)189
106 y(to)e(memory)l(.)18 b(When)13 b(the)g(user)g(co)q(de)g(is)g(w)o(orking)f
(with)h(a)f(register)h(cop)o(y)f(of)g(some)h(v)m(ariable)g(while)189
162 y(the)i(hidden)i(op)q(eration)f(reads)f(or)g(writes)g(the)h(memory)f(cop)
o(y)l(,)g(problems)h(o)q(ccur.)k(This)c(section)189 219 y(discusses)g
(register)f(optimization)h(pitfalls.)189 293 y(When)f(a)f(v)m(ariable)i(is)f
(lo)q(cal)h(to)e(a)g(F)l(ortran)f(subroutine)j(\(i.e.)k(not)14
b(in)h(a)g(COMMON)f(blo)q(c)o(k\),)h(the)189 349 y(compiler)g(will)i(assume)d
(that)g(it)h(cannot)f(b)q(e)h(mo)q(di\014ed)h(b)o(y)f(a)f(called)i
(subroutine)f(unless)h(it)f(is)g(an)189 406 y(actual)g(argumen)o(t)g(of)h
(the)f(call.)23 b(In)16 b(the)g(most)e(common)i(link)m(age)h(con)o(v)o(en)o
(tion,)e(the)h(subroutine)189 462 y(is)d(exp)q(ected)i(to)d(sa)o(v)o(e)h(and)
g(restore)g(certain)h(registers.)19 b(Th)o(us,)13 b(the)g(optimizer)i(will)g
(assume)e(that)189 519 y(a)k(register)h(whic)o(h)h(held)g(a)e(v)m(alid)j(cop)
o(y)d(of)h(suc)o(h)g(a)g(v)m(ariable)h(b)q(efore)f(the)g(call)h(will)g(still)
h(hold)e(a)189 575 y(v)m(alid)e(cop)o(y)g(on)f(return.)189
650 y(Normally)22 b(users)g(are)g(not)g(a\017icted)g(with)h(this.)41
b(But)22 b(the)g(user)g(should)h(pa)o(y)f(atten)o(tion)g(to)189
706 y(this)f(section)g(if)g(in)h(his/her)f(program)e(a)i(bu\013er)f(argumen)o
(t)g(to)g(an)h Fo(MPI)p 1498 706 14 2 v 16 w(SEND)p Fu(,)f
Fo(MPI)p 1746 706 V 16 w(RECV)189 763 y Fu(etc.)25 b(uses)18
b(a)f(name)g(whic)o(h)h(hides)g(the)f(actual)h(v)m(ariables)g(in)o(v)o(olv)o
(ed.)27 b Fo(MPI)p 1493 763 V 16 w(BOTTOM)18 b Fu(with)f(an)189
819 y(MPI)p 281 819 V 16 w(Datat)o(yp)q(e)i(con)o(taining)i(absolute)g
(addresses)f(is)h(one)f(example.)36 b(Creating)20 b(a)f(datat)o(yp)q(e)189
876 y(whic)o(h)14 b(uses)f(one)g(v)m(ariable)i(as)e(an)g(anc)o(hor)f(and)i
(brings)f(along)h(others)e(b)o(y)h(using)h Fo(MPI)p 1659 876
V 16 w(ADDRESS)189 932 y Fu(to)h(determine)j(their)f(o\013sets)e(from)g(the)i
(anc)o(hor)f(is)h(another.)22 b(The)17 b(anc)o(hor)f(v)m(ariable)i(w)o(ould)e
(b)q(e)189 988 y(the)g(only)h(one)f(men)o(tioned)h(in)g(the)g(call.)24
b(Also)16 b(atten)o(tion)g(m)o(ust)g(b)q(e)h(pa)o(y)o(ed)f(if)h(MPI)f(op)q
(erations)189 1045 y(are)f(used)g(that)g(run)g(in)h(parallel)h(with)f(the)f
(user's)g(application.)189 1119 y(The)g(follo)o(wing)h(example)g(sho)o(ws,)e
(what)h(F)l(ortran)f(compilers)i(are)f(allo)o(w)o(ed)h(to)e(do:)224
1239 y(This)i(source)f(...)663 b(can)15 b(b)q(e)h(compiled)h(as:)224
1303 y(call)f(MPI)p 399 1303 V 17 w(ADDRESS\(buf,bufaddr,ierr\))189
b(call)16 b(MPI)p 1344 1303 V 17 w(ADDRESS\(buf,...\))224 1360
y(call)g(MPI)p 399 1360 V 17 w(TYPE)p 545 1360 V 16 w(STR)o
(UCT\(1,1,bufaddr,)163 b(call)16 b(MPI)p 1344 1360 V 17 w(TYPE)p
1490 1360 V 16 w(STR)o(UCT\(...\))590 1416 y(MPI)p 682 1416
V 17 w(REAL,t)o(yp,ierr\))224 1473 y(call)g(MPI)p 399 1473
V 17 w(TYPE)p 545 1473 V 16 w(COMMIT\(t)o(yp\))308 b(call)16
b(MPI)p 1344 1473 V 17 w(TYPE)p 1490 1473 V 16 w(COMMIT\(...\))224
1529 y(v)m(al)p 283 1529 V 17 w(old)g(=)g(buf)681 b(register)15
b(=)h(buf)1169 1586 y(v)m(al)p 1228 1586 V 17 w(old)g(=)f(register)224
1642 y(call)h(MPI)p 399 1642 V 17 w(RECV\(MPI)p 654 1642 V
16 w(BOTTOM,1,t)o(yp,...\))107 b(call)16 b(MPI)p 1344 1642
V 17 w(RECV\(MPI)p 1599 1642 V 16 w(BOTTOM,...\))224 1699 y(v)m(al)p
283 1699 V 17 w(new)g(=)f(buf)664 b(v)m(al)p 1228 1699 V 17
w(new)16 b(=)f(register)189 1818 y(The)j(compiler)g(do)q(es)g(not)g(in)o(v)m
(alidate)h(the)f(register)g(b)q(ecause)g(it)g(cannot)f(see)h(that)f
Fo(MPI)p 1746 1818 V 16 w(RECV)189 1874 y Fu(c)o(hanges)h(the)g(v)m(alue)i
(of)e Fo(buf)p Fu(.)30 b(The)19 b(access)f(of)g Fo(buf)i Fu(is)f(hidden)h(b)o
(y)e(the)h(use)f(of)g Fo(MPI)p 1659 1874 V 16 w(ADDRESS)189
1931 y Fu(and)d Fo(MPI)p 362 1931 V 16 w(BOTTOM)p Fu(.)189
2005 y(The)g(next)g(example)h(sho)o(ws)f(extreme,)g(but)g(allo)o(w)o(ed)h(p)q
(ossibilities!)224 2124 y(Source)424 b(compiled)17 b(as)322
b(or)15 b(compiled)i(as)224 2189 y(call)f(MPI)p 399 2189 V
17 w(IRECV\(buf,..req\))i(call)f(MPI)p 955 2189 V 16 w(IRECV\(buf,..req\))h
(call)f(MPI)p 1510 2189 V 16 w(IRECV\(buf,..req\))779 2246
y(register)e(=)h(buf)278 b(b1)15 b(=)h(buf)224 2302 y(call)g(MPI)p
399 2302 V 17 w(W)-5 b(AIT\(req,..\))104 b(call)17 b(MPI)p
955 2302 V 16 w(W)-5 b(AIT\(req,..\))104 b(call)17 b(MPI)p
1510 2302 V 16 w(W)-5 b(AIT\(req,..\))224 2359 y(b1)15 b(=)h(buf)377
b(b1)15 b(:=)g(register)189 2478 y Fo(MPI)p 274 2478 V 15 w(W)l(AIT)e
Fu(or)f(a)g(parallel)i(thread)e(mo)q(di\014es)i Fo(buf)f Fu(b)q(et)o(w)o(een)
g(the)f(in)o(v)o(o)q(cation)h(of)f Fo(MPI)p 1648 2478 V 16
w(IRECV)h Fu(and)189 2534 y(the)j(\014nish)h(of)f Fo(MPI)p
528 2534 V 16 w(W)l(AIT)p Fu(.)g(But)g(the)g(compiler)i(cannot)e(see)g(an)o
(y)g(p)q(ossibilit)o(y)i(that)e Fo(buf)h Fu(can)f(b)q(e)189
2591 y(c)o(hanged)g(after)e Fo(MPI)p 557 2591 V 16 w(IRECV)i
Fu(has)g(\014nished.)23 b(And)16 b(therefore)f(it)h(can)g(sc)o(hedule)h(the)f
(load)g(of)f Fo(buf)189 2647 y Fu(earlier)f(than)g(t)o(yp)q(ed)g(in)h(the)f
(source.)20 b(It)13 b(has)h(no)g(reason)f(to)h(a)o(v)o(oid)f(using)i(a)e
(register)h(to)f(hold)i Fo(buf)189 2704 y Fu(across)f(the)h(call)i(to)d
Fo(MPI)p 625 2704 V 16 w(W)l(AIT)p Fu(.)h(It)g(also)g(can)h(reorder)f(the)g
(instructions)h(as)f(in)h(the)f(left)h(case.)1967 46 y Fj(1)1967
103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967
385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959
667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893
y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119
y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345
y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570
y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796
y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022
y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248
y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474
y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699
y(48)p eop
%%Page: 33 35
bop 75 -100 a Fl(1.7.)34 b(A)15 b(F)o(OR)l(TRAN)h(PR)o(OBLEM)g(WITH)f
(REGISTER)i(OPTIMIZA)l(TION)361 b Fu(33)189 49 y(T)l(o)12 b(prev)o(en)o(t)h
(instruction)h(reordering)g(or)e(the)h(allo)q(cation)h(of)f(a)g(v)m(ariable)h
(or)f(bu\013er)g(in)h(a)e(register,)189 106 y(there)j(are)g(three)g(p)q
(ossibilities)k(in)d(p)q(ortable)f(F)l(ortran)f(co)q(de:)243
200 y Fm(\017)23 b Fu(One)d(can)f(put)h(the)g(v)m(ariables)g(and)g(bu\013ers)
f(in)o(to)h(common)f(blo)q(c)o(ks;)j(then)e(they)f(will)i(b)q(e)289
256 y(allo)q(cated)d(in)g(the)g(memory)e(while)j(an)o(y)e(library)h
(subroutine)h(\(e.g.)25 b(an)18 b(MPI)f(routine\))g(is)289
312 y(called)c(pro)o(vided)f(that)f(the)h(compiler)h(cannot)e(analyze)h(that)
f(the)h(library)g(subroutine)h(do)q(es)289 369 y(not)h(use)i(these)f(common)g
(blo)q(c)o(ks,)h(or)243 442 y Fm(\017)23 b Fu(one)11 b(can)h(declare)g(them)f
(as)g(V)o(OLA)l(TILE,)h(but)g(this)g(is)f(not)g(part)g(of)g(the)g(standards)g
(F)l(ortran)289 498 y(77)f(and)h(90,)g(and)g(y)o(our)f(co)q(de)i(will)h(not)d
(b)q(e)i(p)q(ortable)f(b)q(ecause)h(there)f(are)g(compilers)h(without)289
555 y(V)o(OLA)l(TILE,)k(and)f(normally)h(it)g(prev)o(en)o(ts)e(an)o(y)h
(optimization,)h(or)243 628 y Fm(\017)23 b Fu(one)14 b(can)g(call)h(a)f(dumm)
o(y)g(routine)g(with)g(the)h(v)m(ariable)g(or)e(bu\013er)h(as)g(an)g(actual)g
(argumen)o(t;)289 684 y(if)22 b(the)h(dumm)o(y)f(routine)h(is)f(not)g(part)g
(of)g(the)g(application)i(then)f(the)f(compiler)i(m)o(ust)289
741 y(allo)q(cate)16 b(the)f(v)m(ariable)i(or)e(bu\013er)h(in)g(the)f(memory)
g(while)i(that)e(subroutine)h(is)g(executed;)289 797 y Fo(MPI)p
374 797 14 2 v 15 w(ADDRESS)h Fu(can)e(b)q(e)h(used)g(as)e(dumm)o(y)h
(routine.)189 891 y(High)e(qualit)o(y)g(F)l(ortran)e(implemen)o(tations)j(ma)
o(y)e(pro)o(vide)h(a)f(general)h(solution)h(to)e(this)h(problem.)189
948 y(Ho)o(w)o(ev)o(er,)h(co)q(de)j(that)e(do)q(es)i(not)e(use)i(one)f(of)g
(the)g(three)g(solutions)g(describ)q(ed)i(ab)q(o)o(v)o(e)e(ma)o(y)f(not)189
1004 y(b)q(e)h(p)q(ortable.)189 1079 y(In)h(C,)f(subroutines)i(whic)o(h)g(mo)
q(dify)f(v)m(ariables)h(that)e(are)h(not)f(in)i(the)f(parameter)f(list)i
(will)g(not)189 1136 y(cause)e(register)h(optimization)g(problems.)25
b(This)17 b(is)g(b)q(ecause)g(taking)g(p)q(oin)o(ters)g(to)f(storage)f(ob-)
189 1192 y(jects)g(b)o(y)g(using)h(the)f(&)h(op)q(erator)e(and)h(later)h
(referencing)g(the)f(ob)s(jects)g(b)o(y)g(w)o(a)o(y)f(of)h(the)g(p)q(oin)o
(ter)189 1249 y(is)i(an)g(in)o(tegral)g(part)f(of)g(the)h(language.)25
b(A)17 b(C)g(compiler)h(understands)f(the)g(implications,)i(but)189
1305 y(with)c(aggressiv)o(e)g(optimization)h(compilers)g(can)g(p)q(ossibly)h
(generate)e(incorrect)g(co)q(de.)189 1380 y(\()p Fp(End)g(of)i(advic)n(e)f
(to)g(users.)p Fu(\))p 75 1486 827 2 v 75 1543 a(The)f(follo)o(wing)h(advice)
h(m)o(ust)d(b)q(e)i(added)g(in)75 1611 y(-)f(MPI)g(2,)g(c)o(hapter)g(?.?)21
b(Handlers,)75 1668 y(-)15 b Fo(MPI)p 190 1668 14 2 v 16 w(ADDRESS)h
Fu(\(MPI)f(1.1,)f(page)h(70,)f(line)j(16\),)d(and)h(in)75 1724
y(-)g Fo(MPI)p 190 1724 V 16 w(IRECV)h Fu(\(MPI)f(1.1,)e(\\Non)o(blo)q(c)o
(king)k(comm)o(unication",)e(end)h(of)e(page)h(40\))189 1830
y Fp(A)n(dvic)n(e)i(to)i(users.)53 b Fu(T)l(o)17 b(prev)o(en)o(t)g(troubles)h
(with)g(the)f(register)g(optimization)i(of)e(the)g(F)l(ortran)189
1887 y(compilers)i(please)g(pa)o(y)e(atten)o(tion)h(to)f(the)h(hin)o(ts)g(in)
h(section)g(1.7)e(\\A)g(F)l(ortran)g(Problem)i(with)189 1943
y(register)c(optimization")h(on)f(page)g(31.)k(\()p Fp(End)d(of)g(advic)n(e)g
(to)h(users.)p Fu(\))-32 46 y Fj(1)-32 103 y(2)-32 159 y(3)-32
215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32
498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40
780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40
1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288
y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40
1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796
y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40
2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304
y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40
2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop
%%Page: 34 36
bop 75 381 a Fs(Bibliograph)m(y)952 2828 y Fu(34)p eop
%%Trailer
end
userdict /end-hook known{end-hook}if
%%EOF

• Next message: Rolf Rabenseifner: "Re: new version of chapter four"
• Previous message: Marc Snir: "new version of chapter four"