-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathpickle_display_result.txt
388 lines (339 loc) · 301 KB
/
pickle_display_result.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
Content of prof_database.pkl file:
Key: ('default', (4, 8))
all_gather_cost_dict:
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31),), 'f16'): [(0, 1.7332873539999127e-05), (1, 1.7417012713849545e-05), (2, 1.7361424397677183e-05), (4, 1.7271784599870443e-05), (8, 1.730496296659112e-05), (16, 1.7406331608071923e-05), (32, 3.4812663216143847e-05), (64, 6.962532643228769e-05), (128, 0.00013925065286457539), (256, 0.00027850130572915077), (512, 0.0005570026114583015), (1024, 0.0009007146582007409), (2048, 0.0009413969819433987), (4096, 0.0010115511249750853), (8192, 0.0009628146653994917), (16384, 0.000999337004031986), (32768, 0.0010854350693989545), (65536, 0.0014509196917060763), (131072, 0.0013366117782425135), (262144, 0.0013115187757648528), (524288, 0.0015455494867637753), (1048576, 0.0017563758883625267), (2097152, 0.0028965752571821213), (4194304, 0.00424243463203311), (8388608, 0.007192713208496571), (16777216, 0.013117061927914621), (33554432, 0.026234123855829242), (67108864, 0.052468247711658485), (134217728, 0.10369202494621277), (268435456, 0.18623790740966797), (536870912, 0.36912539005279543), (1073741824, 0.7382507801055909), (2147483648, 1.4765015602111817)]
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31),), 'f32'): [(0, 1.718458952382207e-05), (1, 1.7263722838833928e-05), (2, 1.7092039342969656e-05), (4, 1.714465906843543e-05), (8, 1.7201848095282912e-05), (16, 1.7301528714597225e-05), (32, 3.460305742919445e-05), (64, 6.92061148583889e-05), (128, 0.0001384122297167778), (256, 0.0002768244594335556), (512, 0.0005536489188671112), (1024, 0.0010206055012531579), (2048, 0.0009887842170428485), (4096, 0.0010352478129789233), (8192, 0.0008913195051718503), (16384, 0.0009667194390203803), (32768, 0.0013040972116868943), (65536, 0.0015935226110741496), (131072, 0.0015405000594910234), (262144, 0.0018264591926708817), (524288, 0.001847971579991281), (1048576, 0.0031116963364183903), (2097152, 0.004851556848734617), (4194304, 0.007083970122039318), (8388608, 0.014167940244078636), (16777216, 0.026497863233089444), (33554432, 0.05299572646617889), (67108864, 0.0999959260225296), (134217728, 0.19497160911560057), (268435456, 0.38994321823120115), (536870912, 0.7551776409149169), (1073741824, 1.5103552818298338)]
(((0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30), (1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31)), 'f16'): [(0, 1.7943268176168203e-05), (1, 1.790435635484755e-05), (2, 1.7868471331894398e-05), (4, 1.7902610125020146e-05), (8, 1.797761069610715e-05), (16, 3.59552213922143e-05), (32, 7.19104427844286e-05), (64, 0.0001438208855688572), (128, 0.0002876417711377144), (256, 0.0005752835422754288), (512, 0.0006111701077315956), (1024, 0.0006549967802129686), (2048, 0.0006380010163411498), (4096, 0.0006054977129679173), (8192, 0.0005989798519294709), (16384, 0.0007176542421802878), (32768, 0.0008820141956675799), (65536, 0.0009676805348135531), (131072, 0.0009680570219643415), (262144, 0.0011407407582737505), (524288, 0.0014553942019119859), (1048576, 0.0022020163014531136), (2097152, 0.0035622785799205303), (4194304, 0.0064664538949728), (8388608, 0.012723999097943306), (16777216, 0.025447998195886612), (33554432, 0.050895996391773224), (67108864, 0.10179199278354645), (134217728, 0.2035839855670929), (268435456, 0.3940170764923096), (536870912, 0.7686288356781006), (1073741824, 1.506540870666504), (2147483648, 3.013081741333008)]
(((0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30), (1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31)), 'f32'): [(0, 1.730577787384391e-05), (1, 1.7482525436207652e-05), (2, 1.724570756778121e-05), (4, 1.7405167454853654e-05), (8, 1.922665978781879e-05), (16, 3.845331957563758e-05), (32, 7.690663915127516e-05), (64, 0.00015381327830255032), (128, 0.00030762655660510063), (256, 0.0006152531132102013), (512, 0.0006313803023658693), (1024, 0.0006568469107151031), (2048, 0.0006171247514430434), (4096, 0.0006261954549700022), (8192, 0.0007677132671233267), (16384, 0.0008870720921549947), (32768, 0.0009790566109586507), (65536, 0.0009687251586001367), (131072, 0.001202824030769989), (262144, 0.0015310856397263708), (524288, 0.002096264041028917), (1048576, 0.0036343876272439957), (2097152, 0.0064069293439388275), (4194304, 0.012813858687877655), (8388608, 0.02562771737575531), (16777216, 0.05125543475151062), (33554432, 0.10175205767154692), (67108864, 0.20314782857894897), (134217728, 0.3935680389404297), (268435456, 0.7819717407226563), (536870912, 1.5473366975784302), (1073741824, 3.092773985862732)]
(((0, 4, 8, 12, 16, 20, 24, 28), (1, 5, 9, 13, 17, 21, 25, 29), (2, 6, 10, 14, 18, 22, 26, 30), (3, 7, 11, 15, 19, 23, 27, 31)), 'f16'): [(0, 1.72372383531183e-05), (1, 1.720443833619356e-05), (2, 1.731939846649766e-05), (4, 1.741599407978356e-05), (8, 3.483198815956712e-05), (16, 6.966397631913424e-05), (32, 0.00013932795263826847), (64, 0.00027865590527653694), (128, 0.0004998018557671458), (256, 0.0005164734611753374), (512, 0.0005125142342876643), (1024, 0.0005400250956881791), (2048, 0.0005110916681587696), (4096, 0.0005143657035659999), (8192, 0.000588631140999496), (16384, 0.0007731385121587665), (32768, 0.0008220045419875533), (65536, 0.0008910732867661864), (131072, 0.0010126133274752647), (262144, 0.0012894310639239848), (524288, 0.0019200040842406454), (1048576, 0.0034412973327562217), (2097152, 0.0068825946655124435), (4194304, 0.012717891484498978), (8388608, 0.025435782968997955), (16777216, 0.05087156593799591), (33554432, 0.10174313187599182), (67108864, 0.20348626375198364), (134217728, 0.4025467783212662), (268435456, 0.8050935566425323), (536870912, 1.6101871132850647), (1073741824, 3.1886387348175047), (2147483648, 6.377277469635009)]
(((0, 4, 8, 12, 16, 20, 24, 28), (1, 5, 9, 13, 17, 21, 25, 29), (2, 6, 10, 14, 18, 22, 26, 30), (3, 7, 11, 15, 19, 23, 27, 31)), 'f32'): [(0, 1.730051008053124e-05), (1, 1.721852459013462e-05), (2, 1.7344282241538167e-05), (4, 1.7346057575196028e-05), (8, 3.4692115150392056e-05), (16, 6.938423030078411e-05), (32, 0.00013876846060156822), (64, 0.00027753692120313644), (128, 0.0005000678356736898), (256, 0.0005071016785223037), (512, 0.0005130831268616021), (1024, 0.0005009168235119432), (2048, 0.0005022736149840057), (4096, 0.0005741462809965014), (8192, 0.000777226930949837), (16384, 0.0008149922359734773), (32768, 0.0008781681826803833), (65536, 0.0010139362420886755), (131072, 0.0013022350030951202), (262144, 0.001926723460201174), (524288, 0.003367476165294647), (1048576, 0.006734952330589294), (2097152, 0.013448507990688084), (4194304, 0.026897015981376168), (8388608, 0.053794031962752335), (16777216, 0.10758806392550467), (33554432, 0.20250806212425232), (67108864, 0.40418606996536255), (134217728, 0.8083721399307251), (268435456, 1.6010717153549194), (536870912, 3.202143430709839), (1073741824, 6.356719207763672)]
(((0, 8, 16, 24), (1, 9, 17, 25), (2, 10, 18, 26), (3, 11, 19, 27), (4, 12, 20, 28), (5, 13, 21, 29), (6, 14, 22, 30), (7, 15, 23, 31)), 'f16'): [(0, 1.7680227756500244e-05), (1, 1.759864971973002e-05), (2, 1.77331967279315e-05), (4, 3.5466393455863e-05), (8, 7.0932786911726e-05), (16, 0.000141865573823452), (32, 0.000283731147646904), (64, 0.00038124926504679024), (128, 0.0003919736482203007), (256, 0.000405193684855476), (512, 0.0004138716030865907), (1024, 0.00037790663191117346), (2048, 0.00039169733645394444), (4096, 0.0004108888097107411), (8192, 0.0005060397379565984), (16384, 0.0006787017046008259), (32768, 0.0007283979211933911), (65536, 0.0008243091579061002), (131072, 0.0010314994142390788), (262144, 0.0015222433139570057), (524288, 0.0028416288550943136), (1048576, 0.005683257710188627), (2097152, 0.011366515420377254), (4194304, 0.02273303084075451), (8388608, 0.04546606168150902), (16777216, 0.09093212336301804), (33554432, 0.18186424672603607), (67108864, 0.36372849345207214), (134217728, 0.7274569869041443), (268435456, 1.4549139738082886), (536870912, 2.909827947616577), (1073741824, 5.819655895233154), (2147483648, 11.639311790466309)]
(((0, 8, 16, 24), (1, 9, 17, 25), (2, 10, 18, 26), (3, 11, 19, 27), (4, 12, 20, 28), (5, 13, 21, 29), (6, 14, 22, 30), (7, 15, 23, 31)), 'f32'): [(0, 1.7134443623945117e-05), (1, 1.7527869204059243e-05), (2, 1.715857069939375e-05), (4, 3.43171413987875e-05), (8, 6.8634282797575e-05), (16, 0.00013726856559515), (32, 0.0002745371311903), (64, 0.0004133596376050263), (128, 0.0004089857102371752), (256, 0.0004016145539935679), (512, 0.00039672444108873606), (1024, 0.00040112511487677693), (2048, 0.0004174358618911356), (4096, 0.0005006188584957272), (8192, 0.0006828648038208485), (16384, 0.0007193773344624788), (32768, 0.000829170923680067), (65536, 0.0010162527905777097), (131072, 0.001490441820351407), (262144, 0.0029078054940327998), (524288, 0.0058156109880655995), (1048576, 0.011631221976131199), (2097152, 0.023262443952262398), (4194304, 0.046524887904524796), (8388608, 0.09304977580904959), (16777216, 0.18609955161809918), (33554432, 0.37219910323619837), (67108864, 0.7443982064723967), (134217728, 1.4887964129447935), (268435456, 2.977592825889587), (536870912, 5.955185651779174), (1073741824, 11.910371303558348)]
(((0, 16), (1, 17), (2, 18), (3, 19), (4, 20), (5, 21), (6, 22), (7, 23), (8, 24), (9, 25), (10, 26), (11, 27), (12, 28), (13, 29), (14, 30), (15, 31)), 'f16'): [(0, 1.7349084373563528e-05), (1, 1.7615733668208122e-05), (2, 3.5231467336416245e-05), (4, 7.046293467283249e-05), (8, 0.00014092586934566498), (16, 0.00016400078311562538), (32, 0.00015645322855561972), (64, 0.00015379677643068135), (128, 0.00015829555923119187), (256, 0.0001634931832086295), (512, 0.00015195540618151426), (1024, 0.00015746074495837092), (2048, 0.0001672519720159471), (4096, 0.0001774991978891194), (8192, 0.00022500701015815136), (16384, 0.00029296401771716774), (32768, 0.0003303223929833621), (65536, 0.00039963040035217995), (131072, 0.0005690072721336037), (262144, 0.0009728293516673148), (524288, 0.0017316086450591683), (1048576, 0.0034632172901183367), (2097152, 0.006926434580236673), (4194304, 0.013852869160473347), (8388608, 0.027705738320946693), (16777216, 0.05541147664189339), (33554432, 0.11082295328378677), (67108864, 0.22164590656757355), (134217728, 0.4432918131351471), (268435456, 0.8865836262702942), (536870912, 1.7731672525405884), (1073741824, 3.5463345050811768), (2147483648, 7.0926690101623535)]
(((0, 16), (1, 17), (2, 18), (3, 19), (4, 20), (5, 21), (6, 22), (7, 23), (8, 24), (9, 25), (10, 26), (11, 27), (12, 28), (13, 29), (14, 30), (15, 31)), 'f32'): [(0, 1.7375947209075093e-05), (1, 1.7458369256928563e-05), (2, 3.4916738513857126e-05), (4, 6.983347702771425e-05), (8, 0.0001396669540554285), (16, 0.00016675129882059991), (32, 0.00016766486805863678), (64, 0.00016263409634120762), (128, 0.00016444738139398396), (256, 0.00015062492457218468), (512, 0.00014388674753718078), (1024, 0.00015375870862044394), (2048, 0.00017141317948698997), (4096, 0.00021091874805279073), (8192, 0.00027967229834757745), (16384, 0.0003166335809510201), (32768, 0.00039114058017730713), (65536, 0.0005687440279871225), (131072, 0.0010086434776894748), (262144, 0.00181894184788689), (524288, 0.00363788369577378), (1048576, 0.00727576739154756), (2097152, 0.01455153478309512), (4194304, 0.02910306956619024), (8388608, 0.05820613913238048), (16777216, 0.11641227826476096), (33554432, 0.23282455652952191), (67108864, 0.46564911305904383), (134217728, 0.9312982261180877), (268435456, 1.8625964522361753), (536870912, 3.7251929044723506), (1073741824, 7.450385808944701)]
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), (16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31)), 'f16'): [(0, 1.7404439859092236e-05), (1, 1.776954741217196e-05), (2, 1.7920508980751038e-05), (4, 1.7375656170770526e-05), (8, 1.9762577721849084e-05), (16, 3.952515544369817e-05), (32, 7.905031088739634e-05), (64, 0.00015810062177479267), (128, 0.00031620124354958534), (256, 0.0005986841861158609), (512, 0.0005880231910850853), (1024, 0.0005576793046202511), (2048, 0.0005541205464396626), (4096, 0.0005298584001138806), (8192, 0.0005423797119874507), (16384, 0.0006006949115544558), (32768, 0.0007245952147059143), (65536, 0.0007213969656731933), (131072, 0.0007852241396903992), (262144, 0.0009310944587923586), (524288, 0.001266673905774951), (1048576, 0.0019131965236738326), (2097152, 0.002620682353153825), (4194304, 0.003987868782132864), (8388608, 0.00687621347606182), (16777216, 0.01292714662849903), (33554432, 0.02537575736641884), (67108864, 0.04391656816005707), (134217728, 0.08783313632011414), (268435456, 0.17566627264022827), (536870912, 0.35133254528045654), (1073741824, 0.7026650905609131), (2147483648, 1.4053301811218262)]
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), (16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31)), 'f32'): [(0, 1.7217476852238178e-05), (1, 1.722280285321176e-05), (2, 1.72575528267771e-05), (4, 1.718368730507791e-05), (8, 1.7252255929633975e-05), (16, 3.450451185926795e-05), (32, 6.90090237185359e-05), (64, 0.0001380180474370718), (128, 0.0002760360948741436), (256, 0.0005520721897482872), (512, 0.0005656672001350671), (1024, 0.0005566327308770269), (2048, 0.0005722828791476786), (4096, 0.0005735092563554645), (8192, 0.0006098390149418265), (16384, 0.0007232810312416404), (32768, 0.0007779335719533265), (65536, 0.0009109785023611039), (131072, 0.0009219051571562885), (262144, 0.001081116555724293), (524288, 0.0017576602986082435), (1048576, 0.002488024067133665), (2097152, 0.0038019125349819665), (4194304, 0.006714073009788989), (8388608, 0.013227419927716255), (16777216, 0.02645483985543251), (33554432, 0.044699519872665405), (67108864, 0.08939903974533081), (134217728, 0.17707388401031493), (268435456, 0.353167200088501), (536870912, 0.697830057144165), (1073741824, 1.39566011428833)]
(((0, 1, 2, 3, 4, 5, 6, 7), (8, 9, 10, 11, 12, 13, 14, 15), (16, 17, 18, 19, 20, 21, 22, 23), (24, 25, 26, 27, 28, 29, 30, 31)), 'f16'): [(0, 1.7745100194588304e-05), (1, 1.769801019690931e-05), (2, 1.7792917788028717e-05), (4, 1.786238863132894e-05), (8, 3.572477726265788e-05), (16, 7.144955452531576e-05), (32, 0.00014289910905063152), (64, 0.0002108890621457249), (128, 0.00021262562950141728), (256, 0.0002073821960948408), (512, 0.00020830513676628473), (1024, 0.00021070722141303125), (2048, 0.00021233459119684992), (4096, 0.0002066729066427797), (8192, 0.00020844885148108006), (16384, 0.00021895984536968172), (32768, 0.0002159866853617132), (65536, 0.00022322928998619318), (131072, 0.0002281295892316848), (262144, 0.0002432149776723236), (524288, 0.0002405183622613549), (1048576, 0.00025923785869963467), (2097152, 0.0002749858540482819), (4194304, 0.00029766547959297895), (8388608, 0.00035671563819050783), (16777216, 0.0004662876017391682), (33554432, 0.000691702589392662), (67108864, 0.001119568943977356), (134217728, 0.0020230114459991455), (268435456, 0.0038251727819442753), (536870912, 0.0075957030057907104), (1073741824, 0.014567995071411132), (2147483648, 0.02888376712799072)]
(((0, 1, 2, 3, 4, 5, 6, 7), (8, 9, 10, 11, 12, 13, 14, 15), (16, 17, 18, 19, 20, 21, 22, 23), (24, 25, 26, 27, 28, 29, 30, 31)), 'f32'): [(0, 1.7784303054213524e-05), (1, 1.8191931303590536e-05), (2, 1.932412851601839e-05), (4, 1.7883314285427332e-05), (8, 3.5766628570854664e-05), (16, 7.153325714170933e-05), (32, 0.00014306651428341866), (64, 0.00021314778132364154), (128, 0.00021375989308580756), (256, 0.00021657225443050268), (512, 0.00021713745081797242), (1024, 0.0002178046852350235), (2048, 0.00021642641513608396), (4096, 0.00021421865676529706), (8192, 0.00021912067313678563), (16384, 0.00022212852491065863), (32768, 0.00022422280744649473), (65536, 0.00023372709983959797), (131072, 0.00023926186258904639), (262144, 0.00024292326997965572), (524288, 0.0002561184810474515), (1048576, 0.00027412932831794024), (2097152, 0.0002833206672221422), (4194304, 0.0003548592794686556), (8388608, 0.00043283682316541677), (16777216, 0.0006607351824641228), (33554432, 0.0010902956128120422), (67108864, 0.002005927264690399), (134217728, 0.0037881731986999516), (268435456, 0.0073975473642349234), (536870912, 0.014536142349243164), (1073741824, 0.028910946846008298)]
(((0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11), (12, 13, 14, 15), (16, 17, 18, 19), (20, 21, 22, 23), (24, 25, 26, 27), (28, 29, 30, 31)), 'f16'): [(0, 1.884423545561731e-05), (1, 1.907508703880012e-05), (2, 1.904513919726014e-05), (4, 3.809027839452028e-05), (8, 7.618055678904057e-05), (16, 0.00010441668564453721), (32, 0.00010111890151165426), (64, 0.00010167015716433524), (128, 0.00010139294317923485), (256, 0.00010531133739277719), (512, 0.00010274493251927196), (1024, 0.00010144218686036766), (2048, 0.00010321906302124262), (4096, 0.00010557266068644822), (8192, 0.00010988287976942956), (16384, 0.00011406349949538709), (32768, 0.0001113435428123921), (65536, 0.00012009340571239592), (131072, 0.00011980225099250673), (262144, 0.00012825080193579197), (524288, 0.00013738227426074445), (1048576, 0.00015468071796931326), (2097152, 0.00017451343592256308), (4194304, 0.00021058914717286825), (8388608, 0.0002866904251277447), (16777216, 0.00043463334441185), (33554432, 0.0007170699536800385), (67108864, 0.0013284776359796524), (134217728, 0.002536766231060028), (268435456, 0.00501570850610733), (536870912, 0.009862437844276428), (1073741824, 0.019724875688552856), (2147483648, 0.03944975137710571)]
(((0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11), (12, 13, 14, 15), (16, 17, 18, 19), (20, 21, 22, 23), (24, 25, 26, 27), (28, 29, 30, 31)), 'f32'): [(0, 1.7360987840220332e-05), (1, 1.7259182641282678e-05), (2, 1.7364072846248746e-05), (4, 3.472814569249749e-05), (8, 6.945629138499498e-05), (16, 0.00010112873860634863), (32, 0.00010103662498295307), (64, 0.0001014052832033485), (128, 0.00010246416786685586), (256, 0.00010088563431054354), (512, 0.000102821592008695), (1024, 0.00010249123442918062), (2048, 0.00010234222281724215), (4096, 0.00011121906572952867), (8192, 0.0001086733245756477), (16384, 0.00010619961540214719), (32768, 0.00011722947238013148), (65536, 0.00012074489495716989), (131072, 0.00012375004007481039), (262144, 0.00013175624189898372), (524288, 0.00015503651229664683), (1048576, 0.0001755335833877325), (2097152, 0.00021040055435150862), (4194304, 0.0002868336159735918), (8388608, 0.0004294672980904579), (16777216, 0.0007193209603428841), (33554432, 0.0013358071446418762), (67108864, 0.002544909715652466), (134217728, 0.0049772560596466064), (268435456, 0.00988130271434784), (536870912, 0.01976260542869568), (1073741824, 0.039504170417785645)]
(((0, 1), (2, 3), (4, 5), (6, 7), (8, 9), (10, 11), (12, 13), (14, 15), (16, 17), (18, 19), (20, 21), (22, 23), (24, 25), (26, 27), (28, 29), (30, 31)), 'f16'): [(0, 1.7543701687827706e-05), (1, 1.7812708392739296e-05), (2, 3.562541678547859e-05), (4, 6.862339796498418e-05), (8, 6.75699848216027e-05), (16, 6.811140337958932e-05), (32, 6.781303090974689e-05), (64, 6.707236752845347e-05), (128, 6.781725096516311e-05), (256, 6.790924817323685e-05), (512, 6.705839768983424e-05), (1024, 6.723491242155433e-05), (2048, 6.878943531773984e-05), (4096, 6.76183553878218e-05), (8192, 7.185249705798924e-05), (16384, 7.602819823659956e-05), (32768, 7.637721137143672e-05), (65536, 8.521851850673556e-05), (131072, 8.712135604582727e-05), (262144, 9.513486293144524e-05), (524288, 0.00010425987420603633), (1048576, 0.00012713597971014678), (2097152, 0.00017635332187637687), (4194304, 0.00027569010853767395), (8388608, 0.00046311621554195875), (16777216, 0.0008444557897746563), (33554432, 0.001614738255739212), (67108864, 0.0031472556293010707), (134217728, 0.006229516118764877), (268435456, 0.012361489236354826), (536870912, 0.024722978472709652), (1073741824, 0.0492318868637085), (2147483648, 0.09811146259307861)]
(((0, 1), (2, 3), (4, 5), (6, 7), (8, 9), (10, 11), (12, 13), (14, 15), (16, 17), (18, 19), (20, 21), (22, 23), (24, 25), (26, 27), (28, 29), (30, 31)), 'f32'): [(0, 1.7578800907358527e-05), (1, 1.7720856703817844e-05), (2, 3.544171340763569e-05), (4, 5.417448119260371e-05), (8, 6.0387596022337675e-05), (16, 6.24649110250175e-05), (32, 5.92870928812772e-05), (64, 6.193757872097194e-05), (128, 6.891050725243986e-05), (256, 6.902540917508304e-05), (512, 6.645024404861033e-05), (1024, 6.862255395390093e-05), (2048, 6.849667988717556e-05), (4096, 7.049512350931764e-05), (8192, 7.628844468854368e-05), (16384, 7.893104339018464e-05), (32768, 8.290039841085672e-05), (65536, 8.928505121730268e-05), (131072, 9.48677770793438e-05), (262144, 0.00010411831317469478), (524288, 0.00012688696733675897), (1048576, 0.0001752072130329907), (2097152, 0.0002696835435926914), (4194304, 0.00045944214798510075), (8388608, 0.0008439370431005954), (16777216, 0.0016087144613265991), (33554432, 0.003146616742014885), (67108864, 0.006218336522579193), (134217728, 0.012354254722595215), (268435456, 0.02461583912372589), (536870912, 0.049152064323425296), (1073741824, 0.09810509681701658)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,), (8,), (9,), (10,), (11,), (12,), (13,), (14,), (15,), (16,), (17,), (18,), (19,), (20,), (21,), (22,), (23,), (24,), (25,), (26,), (27,), (28,), (29,), (30,), (31,)), 'f16'): [(0, 1.73145963344723e-05), (1, 2.011191099882126e-05), (2, 2.0733772544190288e-05), (4, 2.1050567738711834e-05), (8, 2.081732964143157e-05), (16, 2.0664359908550978e-05), (32, 2.050056355074048e-05), (64, 2.0462990505620837e-05), (128, 2.050920738838613e-05), (256, 2.0601379219442606e-05), (512, 2.0502484403550625e-05), (1024, 2.0534556824713945e-05), (2048, 2.0544655853882432e-05), (4096, 2.050466719083488e-05), (8192, 2.0046019926667213e-05), (16384, 1.9883213099092245e-05), (32768, 2.0758365280926228e-05), (65536, 2.085554297082126e-05), (131072, 2.06992554012686e-05), (262144, 2.1207728423178196e-05), (524288, 2.1201383788138628e-05), (1048576, 2.0591163774952292e-05), (2097152, 3.328808816149831e-05), (4194304, 4.5070191845297813e-05), (8388608, 6.638141348958015e-05), (16777216, 0.00011712266132235527), (33554432, 0.00019697658717632294), (67108864, 0.00036694668233394623), (134217728, 0.000730372965335846), (268435456, 0.0014284402132034302), (536870912, 0.002731010317802429), (1073741824, 0.00541996955871582), (2147483648, 0.01083993911743164)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,), (8,), (9,), (10,), (11,), (12,), (13,), (14,), (15,), (16,), (17,), (18,), (19,), (20,), (21,), (22,), (23,), (24,), (25,), (26,), (27,), (28,), (29,), (30,), (31,)), 'f32'): [(0, 1.74628512468189e-05), (1, 2.0546169253066182e-05), (2, 2.073062933050096e-05), (4, 2.09067075047642e-05), (8, 2.048647729679942e-05), (16, 2.0765000954270363e-05), (32, 2.1151587134227157e-05), (64, 2.0554609363898635e-05), (128, 2.0567094907164574e-05), (256, 2.0561012206599116e-05), (512, 2.052844502031803e-05), (1024, 2.066290471702814e-05), (2048, 2.0748761016875505e-05), (4096, 2.0625011529773474e-05), (8192, 2.048158785328269e-05), (16384, 2.051866613328457e-05), (32768, 2.0414008758962154e-05), (65536, 2.0494364434853196e-05), (131072, 2.0603503799065948e-05), (262144, 2.0416075130924582e-05), (524288, 2.0425766706466675e-05), (1048576, 3.280665259808302e-05), (2097152, 4.429428372532129e-05), (4194304, 6.608362309634686e-05), (8388608, 0.00010899920016527176), (16777216, 0.00019239168614149094), (33554432, 0.00036197155714035034), (67108864, 0.0006975047290325165), (134217728, 0.001375041902065277), (268435456, 0.002724558115005493), (536870912, 0.005416989326477051), (1073741824, 0.010747075080871582)]
all_reduce_cost_dict:
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31),), 'f16'): [(0, 0.00020907120779156685), (1, 0.0007231955241877586), (2, 0.0007164216658566147), (4, 0.0007582055986858904), (8, 0.0007475263846572489), (16, 0.0007461428176611662), (32, 0.0007648553873877972), (64, 0.0007580947130918504), (128, 0.0007716122199781239), (256, 0.0007641906850039959), (512, 0.0007578672375530005), (1024, 0.0007802714826539159), (2048, 0.000803517468739301), (4096, 0.0008051139302551746), (8192, 0.001570153166539967), (16384, 0.0015875670942477882), (32768, 0.0020940150134265423), (65536, 0.0026446462143212557), (131072, 0.0021997439907863736), (262144, 0.0021840021363459527), (524288, 0.002575485035777092), (1048576, 0.002963828155770898), (2097152, 0.005170824937522411), (4194304, 0.00787976011633873), (8388608, 0.013896984979510307), (16777216, 0.024436209350824356), (33554432, 0.04887241870164871), (67108864, 0.09774483740329742), (134217728, 0.18926610946655273), (268435456, 0.37444689273834236), (536870912, 0.7305773735046387), (1073741824, 1.4611547470092774), (2147483648, 2.9223094940185548)]
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31),), 'f32'): [(0, 0.00020557080279104412), (1, 0.000732959306333214), (2, 0.0007208841561805457), (4, 0.0007169517921283841), (8, 0.0007345675257965922), (16, 0.0007353082473855465), (32, 0.0007489712443202734), (64, 0.0007535686600022018), (128, 0.0007422583003062755), (256, 0.000747906044125557), (512, 0.0007685107702855021), (1024, 0.0007912652508821338), (2048, 0.0007869042747188361), (4096, 0.001537258067401126), (8192, 0.001641288778046146), (16384, 0.0019043181964661928), (32768, 0.0022095819876994938), (65536, 0.0021063157764729112), (131072, 0.0022746396134607494), (262144, 0.0027368083829060197), (524288, 0.0030244691297411914), (1048576, 0.006048938259482383), (2097152, 0.007471310906112194), (4194304, 0.01452857442200184), (8388608, 0.02440258115530014), (16777216, 0.04880516231060028), (33554432, 0.09690128266811371), (67108864, 0.1871286630630493), (134217728, 0.3742573261260986), (268435456, 0.7289063930511475), (536870912, 1.4235233545303345), (1073741824, 2.842148447036743)]
(((0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30), (1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31)), 'f16'): [(0, 0.00010935781756415963), (1, 0.0005865342682227492), (2, 0.0005897395603824407), (4, 0.0005972514627501369), (8, 0.0005993445520289242), (16, 0.0005942163406871259), (32, 0.0005856143252458423), (64, 0.000583335873670876), (128, 0.0005922628042753786), (256, 0.000593814707826823), (512, 0.0006093463744036853), (1024, 0.0006327748706098646), (2048, 0.0008458109805360436), (4096, 0.001056837005307898), (8192, 0.0010807210637722164), (16384, 0.0012582241324707866), (32768, 0.001542574173072353), (65536, 0.0016138835926540196), (131072, 0.0017004286928568035), (262144, 0.0019613899057731032), (524288, 0.002543776063248515), (1048576, 0.0038280028384178877), (2097152, 0.006760891992598772), (4194304, 0.011765558272600174), (8388608, 0.023531116545200348), (16777216, 0.047062233090400696), (33554432, 0.09412446618080139), (67108864, 0.18824893236160278), (134217728, 0.3717639446258545), (268435456, 0.743527889251709), (536870912, 1.473983669281006), (1073741824, 2.947967338562012), (2147483648, 5.895894765853882)]
(((0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30), (1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31)), 'f32'): [(0, 0.00011219744919799268), (1, 0.000578768172999844), (2, 0.0005851590831298381), (4, 0.0005878356460016221), (8, 0.0005872740875929594), (16, 0.0005951032799202949), (32, 0.0005907007434871048), (64, 0.0005887283477932215), (128, 0.0006064136396162212), (256, 0.000606804242124781), (512, 0.0006202167132869363), (1024, 0.0008746892563067378), (2048, 0.0010958426864817739), (4096, 0.0010540069779381156), (8192, 0.001273602683795616), (16384, 0.0015126559883356094), (32768, 0.0016256562084890904), (65536, 0.0016974102472886445), (131072, 0.0020404839888215065), (262144, 0.0026451750891283154), (524288, 0.003936795052140951), (1048576, 0.006701299455016851), (2097152, 0.011794519610702991), (4194304, 0.023589039221405983), (8388608, 0.047178078442811966), (16777216, 0.09435615688562393), (33554432, 0.18871231377124786), (67108864, 0.3735712289810181), (134217728, 0.7471424579620362), (268435456, 1.480526852607727), (536870912, 2.9507985591888426), (1073741824, 5.889850902557373)]
(((0, 4, 8, 12, 16, 20, 24, 28), (1, 5, 9, 13, 17, 21, 25, 29), (2, 6, 10, 14, 18, 22, 26, 30), (3, 7, 11, 15, 19, 23, 27, 31)), 'f16'): [(0, 7.074192399159074e-05), (1, 0.0005066912563052028), (2, 0.0005204228218644857), (4, 0.0005237487494014204), (8, 0.0005459529056679457), (16, 0.0005275715666357428), (32, 0.0005100858106743544), (64, 0.0005168039933778346), (128, 0.0006140382902231067), (256, 0.0005979787965770811), (512, 0.0008164269966073335), (1024, 0.0007931547006592155), (2048, 0.0009234662284143269), (4096, 0.0009500398882664739), (8192, 0.0010857607994694263), (16384, 0.0010770323278848082), (32768, 0.0015229430282488465), (65536, 0.0016145004774443805), (131072, 0.0018483313324395567), (262144, 0.0023947060690261424), (524288, 0.0036241206107661132), (1048576, 0.0065580192022025585), (2097152, 0.013116038404405117), (4194304, 0.025299351662397385), (8388608, 0.05059870332479477), (16777216, 0.1005057767033577), (33554432, 0.1973169967532158), (67108864, 0.38948585093021393), (134217728, 0.7761846303939819), (268435456, 1.5523692607879638), (536870912, 3.067075967788696), (1073741824, 6.134151935577392), (2147483648, 12.268303871154783)]
(((0, 4, 8, 12, 16, 20, 24, 28), (1, 5, 9, 13, 17, 21, 25, 29), (2, 6, 10, 14, 18, 22, 26, 30), (3, 7, 11, 15, 19, 23, 27, 31)), 'f32'): [(0, 6.973207928240299e-05), (1, 0.0005059373797848821), (2, 0.0005021451797802001), (4, 0.0004969426081515849), (8, 0.0005061057454440743), (16, 0.0005135562678333372), (32, 0.0005225029308348894), (64, 0.0006167106621433049), (128, 0.0006205554818734527), (256, 0.0007681018905714154), (512, 0.0008147208427544683), (1024, 0.000882530293893069), (2048, 0.0009119130263570697), (4096, 0.0010451307171024382), (8192, 0.00104833411751315), (16384, 0.0014647433417849243), (32768, 0.0015696579357609153), (65536, 0.001858886011177674), (131072, 0.002451605221722275), (262144, 0.003707593539729715), (524288, 0.006602318026125431), (1048576, 0.012898261193186045), (2097152, 0.024486507289111614), (4194304, 0.04897301457822323), (8388608, 0.09794602915644646), (16777216, 0.1958920583128929), (33554432, 0.3863827884197235), (67108864, 0.772765576839447), (134217728, 1.545531153678894), (268435456, 3.036298322677612), (536870912, 6.072596645355224), (1073741824, 12.145193290710449)]
(((0, 8, 16, 24), (1, 9, 17, 25), (2, 10, 18, 26), (3, 11, 19, 27), (4, 12, 20, 28), (5, 13, 21, 29), (6, 14, 22, 30), (7, 15, 23, 31)), 'f16'): [(0, inf), (1, inf), (2, inf), (4, inf), (8, inf), (16, inf), (32, inf), (64, inf), (128, inf), (256, inf), (512, inf), (1024, inf), (2048, inf), (4096, inf), (8192, inf), (16384, inf), (32768, inf), (65536, inf), (131072, inf), (262144, inf), (524288, inf), (1048576, inf), (2097152, inf), (4194304, inf), (8388608, inf), (16777216, inf), (33554432, inf), (67108864, inf), (134217728, inf), (268435456, inf), (536870912, inf), (1073741824, inf), (2147483648, inf)]
(((0, 8, 16, 24), (1, 9, 17, 25), (2, 10, 18, 26), (3, 11, 19, 27), (4, 12, 20, 28), (5, 13, 21, 29), (6, 14, 22, 30), (7, 15, 23, 31)), 'f32'): [(0, inf), (1, inf), (2, inf), (4, inf), (8, inf), (16, inf), (32, inf), (64, inf), (128, inf), (256, inf), (512, inf), (1024, inf), (2048, inf), (4096, inf), (8192, inf), (16384, inf), (32768, inf), (65536, inf), (131072, inf), (262144, inf), (524288, inf), (1048576, inf), (2097152, inf), (4194304, inf), (8388608, inf), (16777216, inf), (33554432, inf), (67108864, inf), (134217728, inf), (268435456, inf), (536870912, inf), (1073741824, inf)]
(((0, 16), (1, 17), (2, 18), (3, 19), (4, 20), (5, 21), (6, 22), (7, 23), (8, 24), (9, 25), (10, 26), (11, 27), (12, 28), (13, 29), (14, 30), (15, 31)), 'f16'): [(0, 3.070142702199519e-05), (1, 0.00027347737341187894), (2, 0.0002801221562549472), (4, 0.0002538918924983591), (8, 0.00025576495681889355), (16, 0.00027770025189965963), (32, 0.00028401199961081147), (64, 0.0002678932505659759), (128, 0.0002622373285703361), (256, 0.0002647491346579045), (512, 0.00028147181728854775), (1024, 0.00026860437355935574), (2048, 0.0002900336985476315), (4096, 0.00031953066354617476), (8192, 0.00046960750478319824), (16384, 0.0005001848912797868), (32768, 0.0005757595063187182), (65536, 0.0007159721571952105), (131072, 0.0010295001266058534), (262144, 0.0017614919343031943), (524288, 0.003169601084664464), (1048576, 0.006339202169328928), (2097152, 0.012678404338657856), (4194304, 0.025356808677315712), (8388608, 0.050713617354631424), (16777216, 0.10142723470926285), (33554432, 0.2028544694185257), (67108864, 0.4057089388370514), (134217728, 0.8114178776741028), (268435456, 1.6228357553482056), (536870912, 3.245671510696411), (1073741824, 6.491343021392822), (2147483648, 12.982686042785645)]
(((0, 16), (1, 17), (2, 18), (3, 19), (4, 20), (5, 21), (6, 22), (7, 23), (8, 24), (9, 25), (10, 26), (11, 27), (12, 28), (13, 29), (14, 30), (15, 31)), 'f32'): [(0, 3.069874946959317e-05), (1, 0.000257840845733881), (2, 0.00026165152667090297), (4, 0.00023814672022126618), (8, 0.00026081540272571146), (16, 0.00024911054060794413), (32, 0.0002676515723578632), (64, 0.0002760209026746452), (128, 0.00026412642910145223), (256, 0.00027121134917251766), (512, 0.0002786148397717625), (1024, 0.00029705159249715507), (2048, 0.00031501249759458005), (4096, 0.00045581170707009733), (8192, 0.0005081011913716793), (16384, 0.000569127791095525), (32768, 0.000732883985619992), (65536, 0.0010305635805707425), (131072, 0.0017932403716258702), (262144, 0.003187679103575647), (524288, 0.006375358207151294), (1048576, 0.012750716414302588), (2097152, 0.025501432828605175), (4194304, 0.05100286565721035), (8388608, 0.1020057313144207), (16777216, 0.2040114626288414), (33554432, 0.4080229252576828), (67108864, 0.8160458505153656), (134217728, 1.6320917010307312), (268435456, 3.2641834020614624), (536870912, 6.528366804122925), (1073741824, 13.05673360824585)]
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), (16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31)), 'f16'): [(0, 0.00021038210252299905), (1, 0.00044710125075653195), (2, 0.0004695298848673702), (4, 0.0004698075645137578), (8, 0.00047735971747897565), (16, 0.0004675060627050697), (32, 0.0004861790512222797), (64, 0.00048350077122449875), (128, 0.00046559443580918014), (256, 0.0004704190068878234), (512, 0.00047546552377752965), (1024, 0.0004749428771901875), (2048, 0.0004840566252823919), (4096, 0.00048455424257554114), (8192, 0.0004944423271808773), (16384, 0.0009888846543617547), (32768, 0.0011795376485679299), (65536, 0.0012307923170737922), (131072, 0.0012490669323597103), (262144, 0.0013857976300641894), (524288, 0.0017014129552990198), (1048576, 0.0028217968065291643), (2097152, 0.004321993328630924), (4194304, 0.0068623656406998634), (8388608, 0.011913377791643143), (16777216, 0.021069034934043884), (33554432, 0.03938367962837219), (67108864, 0.07420994341373444), (134217728, 0.14575588703155518), (268435456, 0.287961745262146), (536870912, 0.5349471569061279), (1073741824, 1.0698943138122559), (2147483648, 1.7599720001220702)]
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), (16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31)), 'f32'): [(0, 0.00021361996186897159), (1, 0.0004535396583378315), (2, 0.0004553671169560402), (4, 0.0004475478199310601), (8, 0.0004610212636180222), (16, 0.0004689063935074955), (32, 0.00045220053289085627), (64, 0.0004574244085233659), (128, 0.00047186063602566714), (256, 0.0004814976709894836), (512, 0.0005014111229684204), (1024, 0.0005028736195527017), (2048, 0.0004963746760040522), (4096, 0.0004889873671345413), (8192, 0.000945945066632703), (16384, 0.0011789067939389497), (32768, 0.0011491671612020582), (65536, 0.0012395659286994487), (131072, 0.0015049141366034746), (262144, 0.0018018170958384871), (524288, 0.00278978212736547), (1048576, 0.005174475256353617), (2097152, 0.00719090923666954), (4194304, 0.009923679754137993), (8388608, 0.019847359508275986), (16777216, 0.03572673350572586), (33554432, 0.06393291056156158), (67108864, 0.11842727661132811), (134217728, 0.23509304523468014), (268435456, 0.4701860904693603), (536870912, 0.8634168386459351), (1073741824, 1.7268336772918702)]
(((0, 1, 2, 3, 4, 5, 6, 7), (8, 9, 10, 11, 12, 13, 14, 15), (16, 17, 18, 19, 20, 21, 22, 23), (24, 25, 26, 27, 28, 29, 30, 31)), 'f16'): [(0, 0.00019402269390411675), (1, 0.0001898087211884558), (2, 0.00019224901916459203), (4, 0.00018929180805571377), (8, 0.00019694701768457887), (16, 0.00020056139328517017), (32, 0.00020161073189228773), (64, 0.00019208414596505463), (128, 0.00019517805776558814), (256, 0.0002028795424848795), (512, 0.00019880008767358956), (1024, 0.00020277185831218958), (2048, 0.00020384008530527353), (4096, 0.00019645693828351798), (8192, 0.00020190459326840937), (16384, 0.00020878034410998225), (32768, 0.00021478405687958002), (65536, 0.00022668231395073238), (131072, 0.00022903719218447804), (262144, 0.00023339202743954957), (524288, 0.00025010042008943856), (1048576, 0.00025437300791963935), (2097152, 0.0003130671102553606), (4194304, 0.0003975091967731714), (8388608, 0.0005014082416892052), (16777216, 0.000696658156812191), (33554432, 0.001136617735028267), (67108864, 0.0020390860736370087), (134217728, 0.0038376972079277043), (268435456, 0.007393732666969299), (536870912, 0.014531660079956054), (1073741824, 0.029026150703430176), (2147483648, 0.05732266902923584)]
(((0, 1, 2, 3, 4, 5, 6, 7), (8, 9, 10, 11, 12, 13, 14, 15), (16, 17, 18, 19, 20, 21, 22, 23), (24, 25, 26, 27, 28, 29, 30, 31)), 'f32'): [(0, 0.0002033021009992808), (1, 0.00019647396402433515), (2, 0.00019174563931301236), (4, 0.0002061054692603648), (8, 0.00020659188157878816), (16, 0.00020923741976730523), (32, 0.00020057824440300462), (64, 0.00019984427490271628), (128, 0.0002000636304728687), (256, 0.00019912738935090604), (512, 0.0001992539328057319), (1024, 0.00019964916282333434), (2048, 0.00020828098058700562), (4096, 0.00020833141752518713), (8192, 0.0002159379655495286), (16384, 0.00020548354950733483), (32768, 0.00022071815328672528), (65536, 0.00022436259314417842), (131072, 0.00023717651492916048), (262144, 0.00026376062305644155), (524288, 0.0002824802650138736), (1048576, 0.0003257261123508215), (2097152, 0.0004086641129106284), (4194304, 0.000505378469824791), (8388608, 0.0007137004286050797), (16777216, 0.0011429712176322937), (33554432, 0.002036038786172867), (67108864, 0.003822341561317444), (134217728, 0.007387503981590271), (268435456, 0.014525938034057615), (536870912, 0.028682112693786625), (1073741824, 0.05706255435943604)]
(((0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11), (12, 13, 14, 15), (16, 17, 18, 19), (20, 21, 22, 23), (24, 25, 26, 27), (28, 29, 30, 31)), 'f16'): [(0, 9.805790614336729e-05), (1, 9.633859735913575e-05), (2, 9.787853923626244e-05), (4, 9.764669812284411e-05), (8, 9.457787382416427e-05), (16, 9.405409218743443e-05), (32, 9.833989315666258e-05), (64, 9.995038271881639e-05), (128, 9.859175770543516e-05), (256, 9.802647400647402e-05), (512, 9.928664076142013e-05), (1024, 9.86612867563963e-05), (2048, 0.00010092297452501953), (4096, 0.00010578421643003821), (8192, 0.00010705061140470208), (16384, 0.00010676513193175197), (32768, 0.00011689576786011457), (65536, 0.00011837310739792883), (131072, 0.00012892630184069276), (262144, 0.00014669480151496828), (524288, 0.00015806627925485373), (1048576, 0.00018826755695044994), (2097152, 0.00022193940822035074), (4194304, 0.00029463693499565125), (8388608, 0.000456295907497406), (16777216, 0.0007468350231647491), (33554432, 0.001347830519080162), (67108864, 0.0025824233889579773), (134217728, 0.005051597952842712), (268435456, 0.009990721940994263), (536870912, 0.019900083541870117), (1073741824, 0.03958327770233154), (2147483648, 0.07915744781494141)]
(((0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11), (12, 13, 14, 15), (16, 17, 18, 19), (20, 21, 22, 23), (24, 25, 26, 27), (28, 29, 30, 31)), 'f32'): [(0, 0.00010259912232868373), (1, 9.947526268661021e-05), (2, 0.00010099925566464661), (4, 0.00010216017835773528), (8, 9.998746099881828e-05), (16, 0.00010478604235686362), (32, 0.00010528095299378037), (64, 0.00010081633809022607), (128, 9.918454452417791e-05), (256, 0.00010086878319270909), (512, 0.00010293751256540419), (1024, 0.00010516465408727525), (2048, 0.0001099100336432457), (4096, 0.00011322394129820168), (8192, 0.00011295033618807793), (16384, 0.0001182258129119873), (32768, 0.00012394663644954562), (65536, 0.00013057500473223627), (131072, 0.00014324966468848288), (262144, 0.00015274796169251204), (524288, 0.00017777422908693552), (1048576, 0.00021224224474281073), (2097152, 0.0002812196034938097), (4194304, 0.00043558888137340546), (8388608, 0.0007412731647491455), (16777216, 0.0013468284159898758), (33554432, 0.0025492683053016663), (67108864, 0.004979036748409271), (134217728, 0.009894058108329773), (268435456, 0.01961660385131836), (536870912, 0.03923320770263672), (1073741824, 0.07846641540527344)]
(((0, 1), (2, 3), (4, 5), (6, 7), (8, 9), (10, 11), (12, 13), (14, 15), (16, 17), (18, 19), (20, 21), (22, 23), (24, 25), (26, 27), (28, 29), (30, 31)), 'f16'): [(0, 6.701468373648822e-05), (1, 5.9263315051794045e-05), (2, 6.297198706306517e-05), (4, 6.0458871303126216e-05), (8, 5.9486308600753546e-05), (16, 5.9690471971407526e-05), (32, 6.835968815721571e-05), (64, 6.759262760169804e-05), (128, 6.839173147454858e-05), (256, 6.857368862256408e-05), (512, 6.817231769673526e-05), (1024, 6.926650530658662e-05), (2048, 6.86876883264631e-05), (4096, 6.74861075822264e-05), (8192, 7.39474780857563e-05), (16384, 7.722782902419567e-05), (32768, 7.906620157882571e-05), (65536, 8.628761861473322e-05), (131072, 9.256883640773594e-05), (262144, 0.00010259583359584212), (524288, 0.00012316612992435694), (1048576, 0.00017437193309888244), (2097152, 0.0002691722474992275), (4194304, 0.0004527510609477759), (8388608, 0.0008283373899757862), (16777216, 0.0015777861699461935), (33554432, 0.0030789654701948166), (67108864, 0.0060943737626075745), (134217728, 0.012124225497245789), (268435456, 0.024057596921920776), (536870912, 0.047979474067687995), (1073741824, 0.09573488235473633), (2147483648, 0.19136600494384762)]
(((0, 1), (2, 3), (4, 5), (6, 7), (8, 9), (10, 11), (12, 13), (14, 15), (16, 17), (18, 19), (20, 21), (22, 23), (24, 25), (26, 27), (28, 29), (30, 31)), 'f32'): [(0, 7.002998609095812e-05), (1, 6.816268432885408e-05), (2, 6.806888268329203e-05), (4, 6.926702917553484e-05), (8, 6.902290624566376e-05), (16, 6.926819332875311e-05), (32, 6.853058584965765e-05), (64, 6.86219718772918e-05), (128, 6.903748726472259e-05), (256, 6.951147224754095e-05), (512, 6.988042150624096e-05), (1024, 6.9966830778867e-05), (2048, 6.970152026042342e-05), (4096, 7.411182741634548e-05), (8192, 7.81161361373961e-05), (16384, 7.801922038197517e-05), (32768, 8.896100916899742e-05), (65536, 9.513678378425539e-05), (131072, 0.00010458883480168879), (262144, 0.00012513261754065752), (524288, 0.00017437536735087633), (1048576, 0.0002694418653845787), (2097152, 0.0004558726213872432), (4194304, 0.0008303169161081313), (8388608, 0.001574898138642311), (16777216, 0.003075104206800461), (33554432, 0.0060702115297317505), (67108864, 0.012046754360198975), (134217728, 0.02401702105998993), (268435456, 0.04794297218322754), (536870912, 0.09572031497955322), (1073741824, 0.1913147211074829)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,), (8,), (9,), (10,), (11,), (12,), (13,), (14,), (15,), (16,), (17,), (18,), (19,), (20,), (21,), (22,), (23,), (24,), (25,), (26,), (27,), (28,), (29,), (30,), (31,)), 'f16'): [(0, 1.7455138731747866e-05), (1, 2.0410458091646433e-05), (2, 2.0808132831007242e-05), (4, 2.0691659301519394e-05), (8, 2.0842970116063952e-05), (16, 2.084547304548323e-05), (32, 2.0800973288714886e-05), (64, 2.087416942231357e-05), (128, 2.0773208234459162e-05), (256, 2.06819677259773e-05), (512, 2.0820909412577748e-05), (1024, 2.0632462110370398e-05), (2048, 2.0385865354910493e-05), (4096, 2.0276318537071347e-05), (8192, 2.0081119146198034e-05), (16384, 2.0387989934533834e-05), (32768, 2.0227220375090837e-05), (65536, 2.0686566131189466e-05), (131072, 2.0246312487870455e-05), (262144, 2.0348350517451763e-05), (524288, 1.9864586647599936e-05), (1048576, 1.9880186300724745e-05), (2097152, 3.2210489735007286e-05), (4194304, 4.461663775146008e-05), (8388608, 6.672646850347519e-05), (16777216, 0.00010997336357831954), (33554432, 0.0001953374594449997), (67108864, 0.0003670528531074524), (134217728, 0.0007341057062149048), (268435456, 0.0013994872570037842), (536870912, 0.002766132354736328), (1073741824, 0.00541689395904541), (2147483648, 0.010747146606445313)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,), (8,), (9,), (10,), (11,), (12,), (13,), (14,), (15,), (16,), (17,), (18,), (19,), (20,), (21,), (22,), (23,), (24,), (25,), (26,), (27,), (28,), (29,), (30,), (31,)), 'f32'): [(0, 1.7211510566994548e-05), (1, 2.0657695131376386e-05), (2, 2.023280831053853e-05), (4, 2.2405263734981418e-05), (8, 2.022969420067966e-05), (16, 2.0101142581552267e-05), (32, 2.0299426978453994e-05), (64, 2.0360894268378615e-05), (128, 2.023886190727353e-05), (256, 2.0345993107184768e-05), (512, 2.0074163330718875e-05), (1024, 2.0230421796441078e-05), (2048, 2.0621635485440493e-05), (4096, 2.0595965906977654e-05), (8192, 2.0634382963180542e-05), (16384, 2.0583742298185825e-05), (32768, 2.041360130533576e-05), (65536, 2.0526698790490627e-05), (131072, 2.061008126474917e-05), (262144, 2.271067933179438e-05), (524288, 2.8432928957045075e-05), (1048576, 4.001555498689413e-05), (2097152, 5.106162279844284e-05), (4194304, 7.27078877389431e-05), (8388608, 0.00011597014963626863), (16777216, 0.00020405463874340057), (33554432, 0.0003785267472267151), (67108864, 0.0007229074835777283), (134217728, 0.0014147162437438965), (268435456, 0.00276639461517334), (536870912, 0.00553278923034668), (1073741824, 0.01076955795288086)]
all_to_all_cost_dict:
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31),), 'f16'): [(0, 1.7561018466949463e-05), (1, 1.7502257833257318e-05), (2, 1.751532545313239e-05), (4, 1.7164595192298293e-05), (8, 1.7129379557445645e-05), (16, 1.7114391084760427e-05), (32, 1.704771420918405e-05), (64, 1.7313897842541337e-05), (128, 1.7249956727027893e-05), (256, 1.725248876027763e-05), (512, 1.734166289679706e-05), (1024, 3.468332579359412e-05), (2048, 6.936665158718824e-05), (4096, 0.0001387333031743765), (8192, 0.000277466606348753), (16384, 0.000554933212697506), (32768, 0.001109866425395012), (65536, 0.002219732850790024), (131072, 0.0028981030627619475), (262144, 0.0029684113687835634), (524288, 0.003059706068597734), (1048576, 0.0031822958262637258), (2097152, 0.003363211639225483), (4194304, 0.003384992014616728), (8388608, 0.003915905952453613), (16777216, 0.004918154329061508), (33554432, 0.008205477148294449), (67108864, 0.01324722170829773), (134217728, 0.02649444341659546), (268435456, 0.05298888683319092), (536870912, 0.10597777366638184), (1073741824, 0.21195554733276367), (2147483648, 0.42391109466552734)]
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31),), 'f32'): [(0, 1.7750426195561886e-05), (1, 1.7619342543184757e-05), (2, 1.7670390661805868e-05), (4, 1.7672748072072864e-05), (8, 1.77005713339895e-05), (16, 1.712684752419591e-05), (32, 1.7066806321963668e-05), (64, 1.716069527901709e-05), (128, 1.7192884115502238e-05), (256, 1.707722549326718e-05), (512, 1.7219950677827e-05), (1024, 3.4439901355654e-05), (2048, 6.8879802711308e-05), (4096, 0.000137759605422616), (8192, 0.000275519210845232), (16384, 0.000551038421690464), (32768, 0.001102076843380928), (65536, 0.002204153686761856), (131072, 0.0029077893414068967), (262144, 0.003107612079475075), (524288, 0.00314534897916019), (1048576, 0.0032649708446115255), (2097152, 0.003401941154152155), (4194304, 0.003821616992354393), (8388608, 0.004998631775379181), (16777216, 0.008329644799232483), (33554432, 0.014141909778118132), (67108864, 0.028283819556236264), (134217728, 0.05656763911247253), (268435456, 0.11313527822494505), (536870912, 0.2262705564498901), (1073741824, 0.4525411128997802)]
(((0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30), (1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31)), 'f16'): [(0, 1.7862505046650767e-05), (1, 1.7908576410263777e-05), (2, 1.782327308319509e-05), (4, 1.785953645594418e-05), (8, 1.7887185094878078e-05), (16, 1.7890124581754208e-05), (32, 1.795380376279354e-05), (64, 1.7855578335002065e-05), (128, 2.0455190679058433e-05), (256, 4.0910381358116865e-05), (512, 8.182076271623373e-05), (1024, 0.00016364152543246746), (2048, 0.0003272830508649349), (4096, 0.0006545661017298698), (8192, 0.0013091322034597397), (16384, 0.0014746517990715802), (32768, 0.0015375232032965869), (65536, 0.0015372608322650194), (131072, 0.0014650360972154886), (262144, 0.0015167488600127397), (524288, 0.0015904243336990478), (1048576, 0.0017114736838266253), (2097152, 0.0019680217374116182), (4194304, 0.0025023608468472958), (8388608, 0.00403624027967453), (16777216, 0.007795201614499093), (33554432, 0.01171819493174553), (67108864, 0.02343638986349106), (134217728, 0.04687277972698212), (268435456, 0.09374555945396423), (536870912, 0.18749111890792847), (1073741824, 0.37498223781585693), (2147483648, 0.7499644756317139)]
(((0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30), (1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31)), 'f32'): [(0, 1.7710699466988444e-05), (1, 1.764320768415928e-05), (2, 1.7660058801993728e-05), (4, 1.751550007611513e-05), (8, 1.7587823094800115e-05), (16, 1.7637445125728846e-05), (32, 1.7675512935966253e-05), (64, 1.7547805327922106e-05), (128, 1.7609534552320838e-05), (256, 3.5219069104641676e-05), (512, 7.043813820928335e-05), (1024, 0.0001408762764185667), (2048, 0.0002817525528371334), (4096, 0.0005635051056742668), (8192, 0.0011270102113485336), (16384, 0.0015382216661237182), (32768, 0.0015438618138432503), (65536, 0.0014808221021667123), (131072, 0.0014587711193598807), (262144, 0.0015909358626231551), (524288, 0.0016486442182213068), (1048576, 0.001957900123670697), (2097152, 0.002491831313818693), (4194304, 0.0045113759115338326), (8388608, 0.00645807757973671), (16777216, 0.01291615515947342), (33554432, 0.02583231031894684), (67108864, 0.05166462063789368), (134217728, 0.10332924127578735), (268435456, 0.2066584825515747), (536870912, 0.4133169651031494), (1073741824, 0.8266339302062988)]
(((0, 4, 8, 12, 16, 20, 24, 28), (1, 5, 9, 13, 17, 21, 25, 29), (2, 6, 10, 14, 18, 22, 26, 30), (3, 7, 11, 15, 19, 23, 27, 31)), 'f16'): [(0, 1.760132727213204e-05), (1, 1.7429323634132743e-05), (2, 1.74384331330657e-05), (4, 1.743726897984743e-05), (8, 1.742900349199772e-05), (16, 1.7405167454853654e-05), (32, 1.7433834727853537e-05), (64, 3.486766945570707e-05), (128, 6.973533891141415e-05), (256, 0.0001394706778228283), (512, 0.0002789413556456566), (1024, 0.0005578827112913132), (2048, 0.0007615354261361063), (4096, 0.0007872696151025593), (8192, 0.0007947616104502232), (16384, 0.0007802333857398479), (32768, 0.0007723076560068876), (65536, 0.0007530749135185033), (131072, 0.0007938195194583386), (262144, 0.0008438294171355665), (524288, 0.000979712582193315), (1048576, 0.0012855154927819967), (2097152, 0.0020974164362996817), (4194304, 0.0033005313016474247), (8388608, 0.006056752987205982), (16777216, 0.012113505974411964), (33554432, 0.02422701194882393), (67108864, 0.04845402389764786), (134217728, 0.09690804779529572), (268435456, 0.19381609559059143), (536870912, 0.38763219118118286), (1073741824, 0.7752643823623657), (2147483648, 1.4854458093643188)]
(((0, 4, 8, 12, 16, 20, 24, 28), (1, 5, 9, 13, 17, 21, 25, 29), (2, 6, 10, 14, 18, 22, 26, 30), (3, 7, 11, 15, 19, 23, 27, 31)), 'f32'): [(0, 1.7553276848047972e-05), (1, 1.7581041902303696e-05), (2, 1.758287544362247e-05), (4, 1.7428392311558127e-05), (8, 1.7537182429805398e-05), (16, 1.7460784874856472e-05), (32, 1.9588100258260965e-05), (64, 3.917620051652193e-05), (128, 7.835240103304386e-05), (256, 0.00015670480206608772), (512, 0.00031340960413217545), (1024, 0.0006268192082643509), (2048, 0.0007777170394547284), (4096, 0.0008122563012875617), (8192, 0.0007842292834538966), (16384, 0.0007509876741096376), (32768, 0.000770401005866006), (65536, 0.0007853166607674211), (131072, 0.000858392653753981), (262144, 0.0009860435384325683), (524288, 0.0012583053903654218), (1048576, 0.00198547076433897), (2097152, 0.0031566992402076726), (4194304, 0.006156689487397671), (8388608, 0.011384094133973122), (16777216, 0.022768188267946243), (33554432, 0.04553637653589249), (67108864, 0.09107275307178497), (134217728, 0.18214550614356995), (268435456, 0.3642910122871399), (536870912, 0.7285820245742798), (1073741824, 1.4571640491485596)]
(((0, 8, 16, 24), (1, 9, 17, 25), (2, 10, 18, 26), (3, 11, 19, 27), (4, 12, 20, 28), (5, 13, 21, 29), (6, 14, 22, 30), (7, 15, 23, 31)), 'f16'): [(0, 1.7505401046946645e-05), (1, 1.742792665027082e-05), (2, 1.794766285456717e-05), (4, 1.7539539840072393e-05), (8, 1.7551705241203308e-05), (16, 3.5103410482406616e-05), (32, 7.020682096481323e-05), (64, 0.00014041364192962646), (128, 0.00028082728385925293), (256, 0.0003668894642032683), (512, 0.0003829419729299843), (1024, 0.00036773402825929224), (2048, 0.0003822311700787395), (4096, 0.0003815077943727374), (8192, 0.00036478033871389925), (16384, 0.0003623832599259913), (32768, 0.0004045115201734007), (65536, 0.00042561101145111024), (131072, 0.0004693608789239078), (262144, 0.0005863209953531623), (524288, 0.0009522659820504488), (1048576, 0.0015351691981777549), (2097152, 0.0028240361716598272), (4194304, 0.0056480723433196545), (8388608, 0.011296144686639309), (16777216, 0.022592289373278618), (33554432, 0.045184578746557236), (67108864, 0.09036915749311447), (134217728, 0.18073831498622894), (268435456, 0.3614766299724579), (536870912, 0.6727816104888916), (1073741824, 1.2158523082733155), (2147483648, 2.351746344566345)]
(((0, 8, 16, 24), (1, 9, 17, 25), (2, 10, 18, 26), (3, 11, 19, 27), (4, 12, 20, 28), (5, 13, 21, 29), (6, 14, 22, 30), (7, 15, 23, 31)), 'f32'): [(0, 1.727242488414049e-05), (1, 1.8648133845999837e-05), (2, 1.730470103211701e-05), (4, 1.7302692867815495e-05), (8, 1.759117003530264e-05), (16, 3.518234007060528e-05), (32, 7.036468014121056e-05), (64, 0.0001407293602824211), (128, 0.0002814587205648422), (256, 0.00039496284443885093), (512, 0.0003762671258300543), (1024, 0.00040442252065986395), (2048, 0.0003973585844505578), (4096, 0.00037809330387972295), (8192, 0.00038460196810774505), (16384, 0.0003836522519122809), (32768, 0.00040393526433035726), (65536, 0.00047133828047662973), (131072, 0.0005954965017735958), (262144, 0.0009506985661573709), (524288, 0.001535996561869979), (1048576, 0.0028215800411999226), (2097152, 0.005643160082399845), (4194304, 0.01128632016479969), (8388608, 0.02257264032959938), (16777216, 0.04514528065919876), (33554432, 0.09029056131839752), (67108864, 0.18058112263679504), (134217728, 0.3611622452735901), (268435456, 0.6591518878936767), (536870912, 1.2017903804779053), (1073741824, 2.345884394645691)]
(((0, 16), (1, 17), (2, 18), (3, 19), (4, 20), (5, 21), (6, 22), (7, 23), (8, 24), (9, 25), (10, 26), (11, 27), (12, 28), (13, 29), (14, 30), (15, 31)), 'f16'): [(0, 1.7546204617246985e-05), (1, 1.7642800230532885e-05), (2, 1.7564278095960617e-05), (4, 3.5128556191921234e-05), (8, 7.025711238384247e-05), (16, 0.00014051422476768494), (32, 0.00017544321599416435), (64, 0.00016916502499952912), (128, 0.0001586211146786809), (256, 0.0001689771597739309), (512, 0.00016681008855812252), (1024, 0.00017006191774271429), (2048, 0.00016057922039180994), (4096, 0.00015495784464292228), (8192, 0.00016470460104756057), (16384, 0.00018184076179750264), (32768, 0.00021535027190111578), (65536, 0.00025235023349523544), (131072, 0.00039102177834138274), (262144, 0.0005842374812345952), (524288, 0.0010097235208377242), (1048576, 0.0020194470416754484), (2097152, 0.0038393952418118715), (4194304, 0.007678790483623743), (8388608, 0.015233561396598816), (16777216, 0.030467122793197632), (33554432, 0.060934245586395264), (67108864, 0.11911823600530624), (134217728, 0.21320046484470367), (268435456, 0.3777923107147217), (536870912, 0.7113176584243774), (1073741824, 1.4027729988098145), (2147483648, 2.805545997619629)]
(((0, 16), (1, 17), (2, 18), (3, 19), (4, 20), (5, 21), (6, 22), (7, 23), (8, 24), (9, 25), (10, 26), (11, 27), (12, 28), (13, 29), (14, 30), (15, 31)), 'f32'): [(0, 1.7358193872496486e-05), (1, 1.7360900528728962e-05), (2, 1.726235495880246e-05), (4, 3.452470991760492e-05), (8, 6.904941983520985e-05), (16, 0.0001380988396704197), (32, 0.000159181043272838), (64, 0.00016305645112879574), (128, 0.00017090403707697988), (256, 0.00018290331354364753), (512, 0.00016897483146749437), (1024, 0.00015983698540367186), (2048, 0.0001654422958381474), (4096, 0.00016826402861624956), (8192, 0.00018291542073711753), (16384, 0.00020582787692546844), (32768, 0.00027084629982709885), (65536, 0.0003927598008885979), (131072, 0.0005906971346121281), (262144, 0.0010253205546177924), (524288, 0.0020506411092355847), (1048576, 0.003683219663798809), (2097152, 0.007262691389769316), (4194304, 0.014525382779538631), (8388608, 0.029050765559077263), (16777216, 0.058101531118154526), (33554432, 0.11620306223630905), (67108864, 0.2324061244726181), (134217728, 0.3929547548294067), (268435456, 0.712515926361084), (536870912, 1.425031852722168), (1073741824, 2.8073819637298585)]
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), (16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31)), 'f16'): [(0, 1.7368525732308626e-05), (1, 1.7423852114006877e-05), (2, 1.7496960936114192e-05), (4, 1.7518934328109026e-05), (8, 1.7455749912187457e-05), (16, 1.7443933757022023e-05), (32, 1.7551239579916e-05), (64, 1.74060114659369e-05), (128, 1.749047078192234e-05), (256, 3.498094156384468e-05), (512, 6.996188312768936e-05), (1024, 0.00013992376625537872), (2048, 0.00027984753251075745), (4096, 0.0005596950650215149), (8192, 0.0011193901300430298), (16384, 0.001208557398058474), (32768, 0.0012186093663331121), (65536, 0.0012074512778781354), (131072, 0.0011708794045262039), (262144, 0.001188312191516161), (524288, 0.0012259986251592636), (1048576, 0.0013234871439635754), (2097152, 0.0014991613570600748), (4194304, 0.0019651311449706554), (8388608, 0.003125898540019989), (16777216, 0.004791241139173508), (33554432, 0.008920025080442429), (67108864, 0.017840050160884857), (134217728, 0.035680100321769714), (268435456, 0.060274815559387206), (536870912, 0.12054963111877441), (1073741824, 0.24109926223754882), (2147483648, 0.47030642032623293)]
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), (16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31)), 'f32'): [(0, 1.745921326801181e-05), (1, 1.7570971976965666e-05), (2, 1.7466896679252386e-05), (4, 1.7430924344807863e-05), (8, 1.7493293853476644e-05), (16, 1.749902730807662e-05), (32, 1.7463957192376256e-05), (64, 1.7481186660006642e-05), (128, 1.7528946045786142e-05), (256, 3.5057892091572285e-05), (512, 7.011578418314457e-05), (1024, 0.00014023156836628914), (2048, 0.0002804631367325783), (4096, 0.0005609262734651566), (8192, 0.0011218525469303131), (16384, 0.0012310949969105422), (32768, 0.0012252048763912171), (65536, 0.0011741654016077518), (131072, 0.0011843253450933844), (262144, 0.0012504393816925585), (524288, 0.0013239661930128932), (1048576, 0.0015215075109153984), (2097152, 0.00191786652430892), (4194304, 0.0030862903222441673), (8388608, 0.006172580644488335), (16777216, 0.011881832033395767), (33554432, 0.023763664066791534), (67108864, 0.04752732813358307), (134217728, 0.07771856784820556), (268435456, 0.13432247638702394), (536870912, 0.2686449527740479), (1073741824, 0.48264799118041996)]
(((0, 1, 2, 3, 4, 5, 6, 7), (8, 9, 10, 11, 12, 13, 14, 15), (16, 17, 18, 19, 20, 21, 22, 23), (24, 25, 26, 27, 28, 29, 30, 31)), 'f16'): [(0, 1.8254417227581143e-05), (1, 1.819065073505044e-05), (2, 1.8169317627325654e-05), (4, 1.8359278328716755e-05), (8, 1.8718303181231022e-05), (16, 1.8604041542857885e-05), (32, 1.8320162780582905e-05), (64, 3.664032556116581e-05), (128, 7.328065112233162e-05), (256, 0.00014656130224466324), (512, 0.00024724166723899543), (1024, 0.0002489218604750931), (2048, 0.00024663080694153905), (4096, 0.0002449623425491154), (8192, 0.00024528949870727956), (16384, 0.00024372010375373065), (32768, 0.00023839951609261334), (65536, 0.00023900970700196922), (131072, 0.00024484354071319103), (262144, 0.0002448149607516825), (524288, 0.00025125921820290387), (1048576, 0.0002651932300068438), (2097152, 0.0003109078388661146), (4194304, 0.0003943570191040635), (8388608, 0.0005181559827178717), (16777216, 0.0007778513245284557), (33554432, 0.0012554051354527473), (67108864, 0.002203240990638733), (134217728, 0.004082765430212021), (268435456, 0.007968857884407043), (536870912, 0.01542043685913086), (1073741824, 0.03022191524505615), (2147483648, 0.0604438304901123)]
(((0, 1, 2, 3, 4, 5, 6, 7), (8, 9, 10, 11, 12, 13, 14, 15), (16, 17, 18, 19, 20, 21, 22, 23), (24, 25, 26, 27, 28, 29, 30, 31)), 'f32'): [(0, 1.7867045244202018e-05), (1, 1.7821352230384946e-05), (2, 1.7751881387084723e-05), (4, 1.7802754882723093e-05), (8, 1.7895479686558247e-05), (16, 1.7876620404422283e-05), (32, 1.763945329003036e-05), (64, 3.527890658006072e-05), (128, 7.055781316012144e-05), (256, 0.00014111562632024288), (512, 0.00024101106100715697), (1024, 0.00024397845845669505), (2048, 0.00024689544807188213), (4096, 0.0002459950337652117), (8192, 0.00023937138030305508), (16384, 0.00023191099171526727), (32768, 0.00024471827782690525), (65536, 0.0002451619366183877), (131072, 0.00024704294628463686), (262144, 0.00024729277356527746), (524288, 0.00026547364541329443), (1048576, 0.0003125263028778136), (2097152, 0.00039240042679011816), (4194304, 0.0005183164030313492), (8388608, 0.0007685991004109383), (16777216, 0.001255233772099018), (33554432, 0.002207636833190918), (67108864, 0.004137534648180008), (134217728, 0.00787150114774704), (268435456, 0.015481486916542053), (536870912, 0.03043222427368164), (1073741824, 0.06057474613189698)]
(((0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11), (12, 13, 14, 15), (16, 17, 18, 19), (20, 21, 22, 23), (24, 25, 26, 27), (28, 29, 30, 31)), 'f16'): [(0, 1.8267077393829823e-05), (1, 1.813744893297553e-05), (2, 1.821239129640162e-05), (4, 1.829498796723783e-05), (8, 1.8193532014265656e-05), (16, 3.638706402853131e-05), (32, 7.277412805706263e-05), (64, 0.0001295882393606007), (128, 0.00013135315384715796), (256, 0.00013114447938278317), (512, 0.00013136363122612238), (1024, 0.00013439255417324603), (2048, 0.00013519366621039808), (4096, 0.00013403553748503327), (8192, 0.0001315736153628677), (16384, 0.00012927691568620503), (32768, 0.00012697622878476977), (65536, 0.00012202197103761135), (131072, 0.00012284342665225267), (262144, 0.00012301374226808548), (524288, 0.00012560258619487286), (1048576, 0.00013427401427179575), (2097152, 0.00013103248784318566), (4194304, 0.0001509415451437235), (8388608, 0.00019445107318460939), (16777216, 0.00026223016902804375), (33554432, 0.0004145139828324318), (67108864, 0.0007337462157011032), (134217728, 0.0014130212366580963), (268435456, 0.0027186647057533264), (536870912, 0.005382031202316284), (1073741824, 0.010688424110412598), (2147483648, 0.021340060234069824)]
(((0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11), (12, 13, 14, 15), (16, 17, 18, 19), (20, 21, 22, 23), (24, 25, 26, 27), (28, 29, 30, 31)), 'f32'): [(0, 1.778220757842064e-05), (1, 1.7853017197921872e-05), (2, 1.7748039681464434e-05), (4, 1.7813581507652998e-05), (8, 1.787798828445375e-05), (16, 3.57559765689075e-05), (32, 7.1511953137815e-05), (64, 0.0001269588537979871), (128, 0.00013282030704431236), (256, 0.00012983407941646874), (512, 0.00012961059110239148), (1024, 0.00012992197298444808), (2048, 0.0001278255949728191), (4096, 0.00012946073547936976), (8192, 0.00013236221275292337), (16384, 0.00013293736265040934), (32768, 0.00013387182843871415), (65536, 0.00013409889652393758), (131072, 0.00013469328405335546), (262144, 0.0001315574045293033), (524288, 0.00013796312850899994), (1048576, 0.00014499278040602803), (2097152, 0.0001640211557969451), (4194304, 0.0001995624043047428), (8388608, 0.00026996340602636337), (16777216, 0.00043320562690496445), (33554432, 0.0007379110902547836), (67108864, 0.0014060065150260925), (134217728, 0.002763800323009491), (268435456, 0.005468159914016724), (536870912, 0.010897564888000488), (1073741824, 0.02168574333190918)]
(((0, 1), (2, 3), (4, 5), (6, 7), (8, 9), (10, 11), (12, 13), (14, 15), (16, 17), (18, 19), (20, 21), (22, 23), (24, 25), (26, 27), (28, 29), (30, 31)), 'f16'): [(0, 1.7712678527459502e-05), (1, 1.780464663170278e-05), (2, 1.929758582264185e-05), (4, 3.85951716452837e-05), (8, 7.71903432905674e-05), (16, 8.402625098824501e-05), (32, 8.2397396909073e-05), (64, 8.299908949993551e-05), (128, 8.298695320263505e-05), (256, 8.491118205711246e-05), (512, 8.481863187626004e-05), (1024, 8.463999256491661e-05), (2048, 8.346620597876608e-05), (4096, 8.365584653802216e-05), (8192, 8.418707875534892e-05), (16384, 8.375116158276796e-05), (32768, 8.217088179662824e-05), (65536, 8.33454541862011e-05), (131072, 8.37372790556401e-05), (262144, 8.634617552161217e-05), (524288, 9.289555600844324e-05), (1048576, 0.0001082445087376982), (2097152, 0.00013721594586968422), (4194304, 0.0001966505078598857), (8388608, 0.00036859512329101557), (16777216, 0.0006412323564291), (33554432, 0.0012424327433109283), (67108864, 0.002406161278486252), (134217728, 0.004812322556972504), (268435456, 0.00946970283985138), (536870912, 0.018888801336288452), (1073741824, 0.037761878967285153), (2147483648, 0.07543320655822754)]
(((0, 1), (2, 3), (4, 5), (6, 7), (8, 9), (10, 11), (12, 13), (14, 15), (16, 17), (18, 19), (20, 21), (22, 23), (24, 25), (26, 27), (28, 29), (30, 31)), 'f32'): [(0, 1.7423240933567286e-05), (1, 1.736794365569949e-05), (2, 1.740315929055214e-05), (4, 3.480631858110428e-05), (8, 6.961263716220856e-05), (16, 8.171662921085954e-05), (32, 8.51305085234344e-05), (64, 8.395151235163212e-05), (128, 8.394039468839765e-05), (256, 8.470620377920568e-05), (512, 8.406720007769763e-05), (1024, 8.425820851698518e-05), (2048, 8.532500942237675e-05), (4096, 8.307336247526109e-05), (8192, 8.513435022905469e-05), (16384, 8.245464414358139e-05), (32768, 8.518033428117633e-05), (65536, 8.875862113200128e-05), (131072, 8.715188596397638e-05), (262144, 9.581627091392875e-05), (524288, 0.00010910321725532411), (1048576, 0.0001392217818647623), (2097152, 0.0001964106922969222), (4194304, 0.0003641708754003048), (8388608, 0.0006480803713202477), (16777216, 0.0012354515492916107), (33554432, 0.0024180300533771515), (67108864, 0.004785299301147461), (134217728, 0.009509861469268799), (268435456, 0.01897326111793518), (536870912, 0.03790979385375977), (1073741824, 0.0756220817565918)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,), (8,), (9,), (10,), (11,), (12,), (13,), (14,), (15,), (16,), (17,), (18,), (19,), (20,), (21,), (22,), (23,), (24,), (25,), (26,), (27,), (28,), (29,), (30,), (31,)), 'f16'): [(0, 1.773444819264114e-05), (1, 2.1025014575570822e-05), (2, 2.1105457562953234e-05), (4, 2.1319399820640683e-05), (8, 2.1055195247754455e-05), (16, 2.1006271708756685e-05), (32, 2.130170469172299e-05), (64, 2.1304877009242773e-05), (128, 2.1192565327510238e-05), (256, 2.128424239344895e-05), (512, 2.1260057110339403e-05), (1024, 2.117676194757223e-05), (2048, 2.1143874619156122e-05), (4096, 2.0871899323537946e-05), (8192, 2.1169165847823024e-05), (16384, 2.1008774638175964e-05), (32768, 2.088263863697648e-05), (65536, 2.0734500139951706e-05), (131072, 2.0807317923754454e-05), (262144, 2.064998261630535e-05), (524288, 2.0669482182711363e-05), (1048576, 2.0652514649555087e-05), (2097152, 3.2908457797020674e-05), (4194304, 4.5097083784639835e-05), (8388608, 6.630620919167995e-05), (16777216, 0.00010819267481565475), (33554432, 0.00019635073840618134), (67108864, 0.0003651101142168045), (134217728, 0.0006989948451519012), (268435456, 0.0013979896903038025), (536870912, 0.002758115530014038), (1073741824, 0.005405402183532715), (2147483648, 0.010782814025878907)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,), (8,), (9,), (10,), (11,), (12,), (13,), (14,), (15,), (16,), (17,), (18,), (19,), (20,), (21,), (22,), (23,), (24,), (25,), (26,), (27,), (28,), (29,), (30,), (31,)), 'f32'): [(0, 1.6920239431783557e-05), (1, 2.0142266293987632e-05), (2, 2.0470586605370045e-05), (4, 2.0705629140138626e-05), (8, 2.068775938823819e-05), (16, 2.0665553165599704e-05), (32, 2.0560284610837698e-05), (64, 2.0679377485066652e-05), (128, 2.042963751591742e-05), (256, 2.070132177323103e-05), (512, 2.107393811456859e-05), (1024, 2.099768607877195e-05), (2048, 2.070059417746961e-05), (4096, 2.1013867808505893e-05), (8192, 2.043996937572956e-05), (16384, 2.043016138486564e-05), (32768, 2.035373472608626e-05), (65536, 2.079381374642253e-05), (131072, 2.0354811567813158e-05), (262144, 2.0385952666401863e-05), (524288, 2.0337000023573637e-05), (1048576, 3.627827391028404e-05), (2097152, 4.8276153393089764e-05), (4194304, 6.66517298668623e-05), (8388608, 0.0001106131821870804), (16777216, 0.0001972857862710953), (33554432, 0.0003674812614917755), (67108864, 0.0007088072597980499), (134217728, 0.0013839974999427795), (268435456, 0.00272485613822937), (536870912, 0.00544971227645874), (1073741824, 0.010736942291259766)]
reduce_scatter_cost_dict:
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31),), 'f16'): [(0, 1.773977419361472e-05), (1, 1.780982711352408e-05), (2, 1.777693978510797e-05), (4, 1.709593925625086e-05), (8, 1.7156649846583605e-05), (16, 1.716389670036733e-05), (32, 3.432779340073466e-05), (64, 6.865558680146933e-05), (128, 0.00013731117360293865), (256, 0.0002746223472058773), (512, 0.0005492446944117546), (1024, 0.0009431125654373318), (2048, 0.0009110749524552375), (4096, 0.0009506460919510573), (8192, 0.0009630068379919976), (16384, 0.0009726927673909814), (32768, 0.0010486652317922562), (65536, 0.00122546122292988), (131072, 0.0011894852796103805), (262144, 0.0012590619444381446), (524288, 0.001545226317830384), (1048576, 0.0017712310655042531), (2097152, 0.002790206577628851), (4194304, 0.004220800008624792), (8388608, 0.00722063984721899), (16777216, 0.012312978506088257), (33554432, 0.024625957012176514), (67108864, 0.04925191402435303), (134217728, 0.09850382804870605), (268435456, 0.19646685123443602), (536870912, 0.3825624227523804), (1073741824, 0.7432988882064819), (2147483648, 1.4818591594696044)]
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31),), 'f32'): [(0, 1.750446972437203e-05), (1, 1.7451122403144836e-05), (2, 1.740735024213791e-05), (4, 1.752059324644506e-05), (8, 1.7632177332416177e-05), (16, 1.758866710588336e-05), (32, 3.517733421176672e-05), (64, 7.035466842353344e-05), (128, 0.00014070933684706688), (256, 0.00028141867369413376), (512, 0.0005628373473882675), (1024, 0.0009383987344335765), (2048, 0.0009623794467188419), (4096, 0.0008893219637684524), (8192, 0.0009706476121209562), (16384, 0.0010495916940271854), (32768, 0.0011852862080559134), (65536, 0.0012779716635122895), (131072, 0.0013249466428533196), (262144, 0.0016774943796917794), (524288, 0.0018504982581362128), (1048576, 0.0032214568927884098), (2097152, 0.004339246079325676), (4194304, 0.007224856875836848), (8388608, 0.013112185522913931), (16777216, 0.026224371045827862), (33554432, 0.052448742091655724), (67108864, 0.09651464223861694), (134217728, 0.1930292844772339), (268435456, 0.3860585689544678), (536870912, 0.7721171379089355), (1073741824, 1.5055613279342652)]
(((0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30), (1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31)), 'f16'): [(0, 1.7442478565499187e-05), (1, 1.7365760868415236e-05), (2, 1.736768172122538e-05), (4, 1.69191334862262e-05), (8, 1.704040914773941e-05), (16, 3.408081829547882e-05), (32, 6.816163659095764e-05), (64, 0.00013632327318191528), (128, 0.00027264654636383057), (256, 0.0005452930927276611), (512, 0.0006092188414186239), (1024, 0.0006121062906458974), (2048, 0.0006128296663518995), (4096, 0.0006086846115067601), (8192, 0.0005938619433436543), (16384, 0.0007253980438690633), (32768, 0.0009034131362568587), (65536, 0.0009673956083133816), (131072, 0.0009458924469072373), (262144, 0.0011078178358729929), (524288, 0.0014195051626302302), (1048576, 0.0021156903821974993), (2097152, 0.0036571195814758544), (4194304, 0.006408130284398794), (8388608, 0.012816260568797588), (16777216, 0.025632521137595177), (33554432, 0.05126504227519035), (67108864, 0.09991732239723207), (134217728, 0.19983464479446414), (268435456, 0.3892930507659912), (536870912, 0.7690172910690307), (1073741824, 1.5380345821380614), (2147483648, 3.076069164276123)]
(((0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30), (1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31)), 'f32'): [(0, 1.7183978343382478e-05), (1, 1.7237645806744695e-05), (2, 1.7263286281377077e-05), (4, 1.726869959384203e-05), (8, 1.7433252651244402e-05), (16, 3.4866505302488804e-05), (32, 6.973301060497761e-05), (64, 0.00013946602120995522), (128, 0.00027893204241991043), (256, 0.0005578640848398209), (512, 0.0006312235782388598), (1024, 0.0006303322152234614), (2048, 0.0006370155606418848), (4096, 0.0006184426893014461), (8192, 0.000737832218874246), (16384, 0.0008878634544089437), (32768, 0.0009442320442758501), (65536, 0.000993622961686924), (131072, 0.0012158253812231123), (262144, 0.0014413117314688861), (524288, 0.0020945569267496467), (1048576, 0.003625182202085853), (2097152, 0.006557627115398645), (4194304, 0.012159829027950764), (8388608, 0.024319658055901527), (16777216, 0.04840321466326713), (33554432, 0.09680642932653426), (67108864, 0.18865594267845154), (134217728, 0.3726711988449097), (268435456, 0.7325046539306641), (536870912, 1.4650093078613282), (1073741824, 2.9300186157226564)]
(((0, 4, 8, 12, 16, 20, 24, 28), (1, 5, 9, 13, 17, 21, 25, 29), (2, 6, 10, 14, 18, 22, 26, 30), (3, 7, 11, 15, 19, 23, 27, 31)), 'f16'): [(0, 1.709564821794629e-05), (1, 1.7100857803598046e-05), (2, 1.737571437843144e-05), (4, 1.7193728126585484e-05), (8, 3.438745625317097e-05), (16, 6.877491250634193e-05), (32, 0.00013754982501268387), (64, 0.00027509965002536774), (128, 0.00050021568313241), (256, 0.0005021009419579059), (512, 0.0005288345273584127), (1024, 0.0005052358901593834), (2048, 0.00047202585847117007), (4096, 0.00047771079698577523), (8192, 0.000556825747480616), (16384, 0.0007657559763174505), (32768, 0.0007925092650111766), (65536, 0.0008549076155759394), (131072, 0.0010001863411162049), (262144, 0.0012901712907478213), (524288, 0.0019256192026659848), (1048576, 0.0034144666278734808), (2097152, 0.0068289332557469615), (4194304, 0.013162362854927778), (8388608, 0.026324725709855556), (16777216, 0.05264945141971111), (33554432, 0.10529890283942223), (67108864, 0.21059780567884445), (134217728, 0.418261393904686), (268435456, 0.8357519626617431), (536870912, 1.6540821552276612), (1073741824, 3.2992878675460817), (2147483648, 6.5516352891922)]
(((0, 4, 8, 12, 16, 20, 24, 28), (1, 5, 9, 13, 17, 21, 25, 29), (2, 6, 10, 14, 18, 22, 26, 30), (3, 7, 11, 15, 19, 23, 27, 31)), 'f32'): [(0, 1.660201814956963e-05), (1, 1.6616162611171603e-05), (2, 1.6578997019678354e-05), (4, 1.7154496163129807e-05), (8, 3.430899232625961e-05), (16, 6.861798465251923e-05), (32, 0.00013723596930503845), (64, 0.0002744719386100769), (128, 0.0004994682094547898), (256, 0.000496312219183892), (512, 0.0005083027353975922), (1024, 0.000502825336297974), (2048, 0.0005037616065237671), (4096, 0.0005852091999258846), (8192, 0.0007738540007267147), (16384, 0.0007995172054506838), (32768, 0.0008621603192295879), (65536, 0.0010108230344485492), (131072, 0.0013000097242183983), (262144, 0.001886286772787571), (524288, 0.00344702962320298), (1048576, 0.00689405924640596), (2097152, 0.01301252469420433), (4194304, 0.02602504938840866), (8388608, 0.05205009877681732), (16777216, 0.10410019755363464), (33554432, 0.2082003951072693), (67108864, 0.40893495082855225), (134217728, 0.8178699016571045), (268435456, 1.635739803314209), (536870912, 3.271479606628418), (1073741824, 6.542959213256836)]
(((0, 8, 16, 24), (1, 9, 17, 25), (2, 10, 18, 26), (3, 11, 19, 27), (4, 12, 20, 28), (5, 13, 21, 29), (6, 14, 22, 30), (7, 15, 23, 31)), 'f16'): [(0, 1.771110692061484e-05), (1, 1.7690123058855534e-05), (2, 1.7726386431604624e-05), (4, 3.545277286320925e-05), (8, 7.09055457264185e-05), (16, 0.000141811091452837), (32, 0.000283622182905674), (64, 0.00039444045978598297), (128, 0.0003921138704754412), (256, 0.00041422006324864924), (512, 0.0004037275793962181), (1024, 0.00039219282916747034), (2048, 0.00039980252040550113), (4096, 0.0004349445516709238), (8192, 0.000515000632731244), (16384, 0.0006827028992120177), (32768, 0.0007250136404763907), (65536, 0.0008281457412522286), (131072, 0.0010648764437064528), (262144, 0.0015275685000233352), (524288, 0.0029288309742696583), (1048576, 0.005857661948539317), (2097152, 0.011715323897078633), (4194304, 0.023430647794157267), (8388608, 0.04686129558831453), (16777216, 0.09372259117662907), (33554432, 0.18744518235325813), (67108864, 0.37489036470651627), (134217728, 0.7497807294130325), (268435456, 1.499561458826065), (536870912, 2.99912291765213), (1073741824, 5.99824583530426), (2147483648, 11.99649167060852)]
(((0, 8, 16, 24), (1, 9, 17, 25), (2, 10, 18, 26), (3, 11, 19, 27), (4, 12, 20, 28), (5, 13, 21, 29), (6, 14, 22, 30), (7, 15, 23, 31)), 'f32'): [(0, 1.9363535102456808e-05), (1, 1.7714366549625993e-05), (2, 1.7740705516189337e-05), (4, 3.5481411032378674e-05), (8, 7.096282206475735e-05), (16, 0.0001419256441295147), (32, 0.0002838512882590294), (64, 0.00040059888851828873), (128, 0.00040695775533095), (256, 0.00040283132693730295), (512, 0.0003966543299611658), (1024, 0.0004090475849807263), (2048, 0.000422054814407602), (4096, 0.0005003084952477366), (8192, 0.0006877447303850204), (16384, 0.0007368612859863788), (32768, 0.0008441579702775926), (65536, 0.001045476325089112), (131072, 0.001514301169663668), (262144, 0.0029260965529829264), (524288, 0.005852193105965853), (1048576, 0.011704386211931705), (2097152, 0.02340877242386341), (4194304, 0.04681754484772682), (8388608, 0.09363508969545364), (16777216, 0.1872701793909073), (33554432, 0.3745403587818146), (67108864, 0.7490807175636292), (134217728, 1.4981614351272583), (268435456, 2.9963228702545166), (536870912, 5.992645740509033), (1073741824, 11.985291481018066)]
(((0, 16), (1, 17), (2, 18), (3, 19), (4, 20), (5, 21), (6, 22), (7, 23), (8, 24), (9, 25), (10, 26), (11, 27), (12, 28), (13, 29), (14, 30), (15, 31)), 'f16'): [(0, 1.7503421986475587e-05), (1, 1.8240156350657344e-05), (2, 3.648031270131469e-05), (4, 7.296062540262938e-05), (8, 0.00014592125080525875), (16, 0.0001511118607595563), (32, 0.00015748234000056982), (64, 0.0001526256965007633), (128, 0.00016481851343996823), (256, 0.00016661672270856798), (512, 0.00015035839169286191), (1024, 0.0001478901831433177), (2048, 0.0001614198845345527), (4096, 0.00018092439859174192), (8192, 0.00021779266535304487), (16384, 0.000285958667518571), (32768, 0.00032308674417436123), (65536, 0.00040211318992078304), (131072, 0.0005780849023722112), (262144, 0.0009930593660101295), (524288, 0.0018316610949113965), (1048576, 0.003663322189822793), (2097152, 0.007326644379645586), (4194304, 0.014653288759291172), (8388608, 0.029306577518582344), (16777216, 0.05861315503716469), (33554432, 0.11722631007432938), (67108864, 0.23445262014865875), (134217728, 0.4689052402973175), (268435456, 0.937810480594635), (536870912, 1.87562096118927), (1073741824, 3.75124192237854), (2147483648, 7.50248384475708)]
(((0, 16), (1, 17), (2, 18), (3, 19), (4, 20), (5, 21), (6, 22), (7, 23), (8, 24), (9, 25), (10, 26), (11, 27), (12, 28), (13, 29), (14, 30), (15, 31)), 'f32'): [(0, 1.7028592992573977e-05), (1, 1.7062935512512922e-05), (2, 3.4125871025025845e-05), (4, 6.825174205005169e-05), (8, 0.00013650348410010338), (16, 0.00015416787937283516), (32, 0.000149580737343058), (64, 0.00015589199028909206), (128, 0.00016008820966817439), (256, 0.00015518278814852238), (512, 0.0001668885233812034), (1024, 0.00016443649656139314), (2048, 0.00016683535068295896), (4096, 0.00021062607993371787), (8192, 0.00028092885622754693), (16384, 0.0003213812015019357), (32768, 0.00039036781527101994), (65536, 0.0005666537908837199), (131072, 0.0009927813371177763), (262144, 0.0018085020710714161), (524288, 0.0036170041421428323), (1048576, 0.0072340082842856646), (2097152, 0.014468016568571329), (4194304, 0.028936033137142658), (8388608, 0.057872066274285316), (16777216, 0.11574413254857063), (33554432, 0.23148826509714127), (67108864, 0.46297653019428253), (134217728, 0.9259530603885651), (268435456, 1.8519061207771301), (536870912, 3.7038122415542603), (1073741824, 7.4076244831085205)]
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), (16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31)), 'f16'): [(0, 1.715507823973894e-05), (1, 1.7068698070943356e-05), (2, 1.709876232780516e-05), (4, 1.6984617104753852e-05), (8, 1.717996201477945e-05), (16, 3.43599240295589e-05), (32, 6.87198480591178e-05), (64, 0.0001374396961182356), (128, 0.0002748793922364712), (256, 0.0005297008319757879), (512, 0.0005549909546971321), (1024, 0.0005927770398557186), (2048, 0.0005705036455765367), (4096, 0.0005531802016776055), (8192, 0.0005486246082000434), (16384, 0.0006169343541841954), (32768, 0.0006911621312610805), (65536, 0.0007179751701187342), (131072, 0.0007928663690108806), (262144, 0.0009041097946465015), (524288, 0.0011328032705932856), (1048576, 0.001857031020335853), (2097152, 0.0024416286032646894), (4194304, 0.0038046734407544136), (8388608, 0.006383334286510944), (16777216, 0.011894050985574724), (33554432, 0.023788101971149448), (67108864, 0.04416092485189438), (134217728, 0.0881398618221283), (268435456, 0.17370319366455078), (536870912, 0.34726433753967284), (1073741824, 0.6932718515396118), (2147483648, 1.3865437030792236)]
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), (16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31)), 'f32'): [(0, 1.732181408442557e-05), (1, 1.746730413287878e-05), (2, 1.9159138901159167e-05), (4, 1.7171376384794712e-05), (8, 1.7329788533970714e-05), (16, 3.465957706794143e-05), (32, 6.931915413588285e-05), (64, 0.0001386383082717657), (128, 0.0002772766165435314), (256, 0.0005516940727829933), (512, 0.0006083214248064905), (1024, 0.0006182710058055818), (2048, 0.0005277488671708852), (4096, 0.0005385861732065678), (8192, 0.000597501959418878), (16384, 0.0007448436808772385), (32768, 0.0007215157384052873), (65536, 0.0007902109064161776), (131072, 0.0009062970639206469), (262144, 0.0010605307179503143), (524288, 0.001809400855563581), (1048576, 0.0026967786252498627), (2097152, 0.0038623777218163013), (4194304, 0.006652302108705044), (8388608, 0.012114007025957108), (16777216, 0.02300300821661949), (33554432, 0.04476796090602875), (67108864, 0.0895359218120575), (134217728, 0.1779865264892578), (268435456, 0.3450301170349121), (536870912, 0.6900602340698242), (1073741824, 1.3801204681396484)]
(((0, 1, 2, 3, 4, 5, 6, 7), (8, 9, 10, 11, 12, 13, 14, 15), (16, 17, 18, 19, 20, 21, 22, 23), (24, 25, 26, 27, 28, 29, 30, 31)), 'f16'): [(0, 1.712879748083651e-05), (1, 1.6952952137216926e-05), (2, 1.711168442852795e-05), (4, 1.718083512969315e-05), (8, 3.43616702593863e-05), (16, 6.87233405187726e-05), (32, 0.0001374466810375452), (64, 0.0001957295462489128), (128, 0.00019911772687919435), (256, 0.00019475718727335334), (512, 0.00019846987561322751), (1024, 0.00020113078062422574), (2048, 0.00020538602257147431), (4096, 0.00021072325762361288), (8192, 0.00019993033492937684), (16384, 0.00020946748554706573), (32768, 0.00021277356427162886), (65536, 0.00021022206055931747), (131072, 0.00021421379642561078), (262144, 0.00021918880520388484), (524288, 0.00021944305626675484), (1048576, 0.00025305867893621325), (2097152, 0.0002826710697263479), (4194304, 0.0002862764522433281), (8388608, 0.00034924549981951714), (16777216, 0.0004670443013310433), (33554432, 0.0007110936567187309), (67108864, 0.0011910740286111832), (134217728, 0.002199307084083557), (268435456, 0.004110388457775116), (536870912, 0.00806330144405365), (1073741824, 0.0157146692276001), (2147483648, 0.031145572662353516)]
(((0, 1, 2, 3, 4, 5, 6, 7), (8, 9, 10, 11, 12, 13, 14, 15), (16, 17, 18, 19, 20, 21, 22, 23), (24, 25, 26, 27, 28, 29, 30, 31)), 'f32'): [(0, 1.8117105355486274e-05), (1, 1.9834493286907673e-05), (2, 1.8185877706855536e-05), (4, 1.8147053197026253e-05), (8, 3.6294106394052505e-05), (16, 7.258821278810501e-05), (32, 0.00014517642557621002), (64, 0.00017366939573548734), (128, 0.00018790981266647575), (256, 0.00020749375107698143), (512, 0.00020814628805965185), (1024, 0.0002041477127932012), (2048, 0.00020230453810654578), (4096, 0.00020165147725492716), (8192, 0.0001995437196455896), (16384, 0.00021368276793509722), (32768, 0.0002129922213498503), (65536, 0.00021519878646358848), (131072, 0.00022067595273256302), (262144, 0.0002098331751767546), (524288, 0.00025645160349085927), (1048576, 0.00028387625934556127), (2097152, 0.0003110166871920228), (4194304, 0.0003760166000574827), (8388608, 0.00046705454587936396), (16777216, 0.0007020197808742523), (33554432, 0.0011819936335086823), (67108864, 0.0020995289087295532), (134217728, 0.00399499386548996), (268435456, 0.007795870304107665), (536870912, 0.01536240577697754), (1073741824, 0.03042001724243164)]
(((0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11), (12, 13, 14, 15), (16, 17, 18, 19), (20, 21, 22, 23), (24, 25, 26, 27), (28, 29, 30, 31)), 'f16'): [(0, 1.74680317286402e-05), (1, 1.7567595932632685e-05), (2, 1.7483020201325417e-05), (4, 3.496604040265083e-05), (8, 6.993208080530167e-05), (16, 0.0001020632917061448), (32, 0.00010908549302257597), (64, 0.00010413641575723888), (128, 0.00010827137157320978), (256, 0.00010906526586040854), (512, 0.00010752936941571532), (1024, 0.00010026601376011968), (2048, 0.0001001693308353424), (4096, 0.00010080731590278447), (8192, 0.00010579457739368081), (16384, 0.00011465317220427095), (32768, 0.00011117223766632378), (65536, 0.00011697274749167262), (131072, 0.00012283888645470142), (262144, 0.00012115982826799154), (524288, 0.00014297498273663223), (1048576, 0.00015055492985993624), (2097152, 0.000175624736584723), (4194304, 0.00021789479069411755), (8388608, 0.00030819117091596127), (16777216, 0.0004982282407581806), (33554432, 0.0008544884622097015), (67108864, 0.0015481580048799515), (134217728, 0.002924583852291107), (268435456, 0.005666106939315796), (536870912, 0.011332213878631592), (1073741824, 0.022664427757263184), (2147483648, 0.04532885551452637)]
(((0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11), (12, 13, 14, 15), (16, 17, 18, 19), (20, 21, 22, 23), (24, 25, 26, 27), (28, 29, 30, 31)), 'f32'): [(0, 1.745644840411842e-05), (1, 1.8598511815071106e-05), (2, 1.8589780665934086e-05), (4, 3.717956133186817e-05), (8, 7.435912266373634e-05), (16, 0.00010247156023979188), (32, 0.00010166189167648554), (64, 0.00010263579315505923), (128, 0.00010113936150446534), (256, 0.00010168610606342554), (512, 0.00010225965525023639), (1024, 9.871716611087322e-05), (2048, 0.00010676868259906769), (4096, 0.00010559021029621363), (8192, 0.00011222841567359865), (16384, 0.00011303057544864713), (32768, 0.00011533973156474531), (65536, 0.00012235730537213385), (131072, 0.00012213608715683222), (262144, 0.0001406148076057434), (524288, 0.00015315169002860785), (1048576, 0.0001761870225891471), (2097152, 0.0002173284301534295), (4194304, 0.00029628234915435314), (8388608, 0.00046233460307121277), (16777216, 0.0007902281358838081), (33554432, 0.0013912338763475418), (67108864, 0.0027486495673656464), (134217728, 0.005362644791603088), (268435456, 0.010725289583206177), (536870912, 0.021450579166412354), (1073741824, 0.04290115833282471)]
(((0, 1), (2, 3), (4, 5), (6, 7), (8, 9), (10, 11), (12, 13), (14, 15), (16, 17), (18, 19), (20, 21), (22, 23), (24, 25), (26, 27), (28, 29), (30, 31)), 'f16'): [(0, 1.7125479644164443e-05), (1, 1.703374437056482e-05), (2, 3.406748874112964e-05), (4, 5.72382123209536e-05), (8, 6.632311851717532e-05), (16, 6.937005673535168e-05), (32, 6.451882654801011e-05), (64, 5.8724923292174935e-05), (128, 5.934634828008711e-05), (256, 6.213434971868992e-05), (512, 5.721923662349582e-05), (1024, 6.744911661371589e-05), (2048, 6.762082921341062e-05), (4096, 6.718188524246216e-05), (8192, 7.100915536284447e-05), (16384, 7.34281784389168e-05), (32768, 7.796043064445257e-05), (65536, 8.043917478062212e-05), (131072, 8.787092519924045e-05), (262144, 9.3898648628965e-05), (524288, 0.0001089498691726476), (1048576, 0.00014392324374057353), (2097152, 0.00022062979405745864), (4194304, 0.0003643343225121498), (8388608, 0.0006580343469977379), (16777216, 0.0012271702289581299), (33554432, 0.002362431026995182), (67108864, 0.004511019214987755), (134217728, 0.0087461918592453), (268435456, 0.01643119752407074), (536870912, 0.03139737248420715), (1073741824, 0.06027610301971436), (2147483648, 0.12055220603942872)]
(((0, 1), (2, 3), (4, 5), (6, 7), (8, 9), (10, 11), (12, 13), (14, 15), (16, 17), (18, 19), (20, 21), (22, 23), (24, 25), (26, 27), (28, 29), (30, 31)), 'f32'): [(0, 1.8750346498563886e-05), (1, 1.7355079762637615e-05), (2, 3.471015952527523e-05), (4, 6.618935731239617e-05), (8, 6.542162736877799e-05), (16, 6.654762546531856e-05), (32, 6.705935811623931e-05), (64, 6.648758426308632e-05), (128, 6.680551450699568e-05), (256, 6.674931501038373e-05), (512, 6.748916348442435e-05), (1024, 6.880165892653167e-05), (2048, 6.855599349364638e-05), (4096, 7.178515079431236e-05), (8192, 7.121791713871062e-05), (16384, 7.553814793936908e-05), (32768, 7.85196025390178e-05), (65536, 8.657920989207923e-05), (131072, 9.446503827348351e-05), (262144, 0.00011055151117034256), (524288, 0.000143089157063514), (1048576, 0.0002140679280273616), (2097152, 0.000350512913428247), (4194304, 0.0006245076656341553), (8388608, 0.0011608689092099667), (16777216, 0.0022277114912867546), (33554432, 0.00428500771522522), (67108864, 0.00836155191063881), (134217728, 0.01646888256072998), (268435456, 0.032685235142707825), (536870912, 0.06504812240600585), (1073741824, 0.12989568710327148)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,), (8,), (9,), (10,), (11,), (12,), (13,), (14,), (15,), (16,), (17,), (18,), (19,), (20,), (21,), (22,), (23,), (24,), (25,), (26,), (27,), (28,), (29,), (30,), (31,)), 'f16'): [(0, 1.7310056136921048e-05), (1, 2.050492912530899e-05), (2, 2.0884093828499317e-05), (4, 2.0873674657195807e-05), (8, 2.0828767446801066e-05), (16, 2.192621468566358e-05), (32, 2.0853098249062896e-05), (64, 2.083959407173097e-05), (128, 2.0741019397974014e-05), (256, 2.1433457732200623e-05), (512, 2.1324900444597006e-05), (1024, 2.1272629965096712e-05), (2048, 2.1320214727893472e-05), (4096, 2.380495425313711e-05), (8192, 2.128415508195758e-05), (16384, 2.135604154318571e-05), (32768, 2.1743704564869404e-05), (65536, 2.1790590835735202e-05), (131072, 2.1203741198405623e-05), (262144, 2.1479208953678608e-05), (524288, 2.130703069269657e-05), (1048576, 2.1235522581264377e-05), (2097152, 3.379001282155514e-05), (4194304, 4.537298809736967e-05), (8388608, 6.879703141748905e-05), (16777216, 0.00011334661394357681), (33554432, 0.00019311904907226562), (67108864, 0.00036338716745376587), (134217728, 0.0007267743349075317), (268435456, 0.0013803914189338684), (536870912, 0.0027515143156051636), (1073741824, 0.005416607856750489), (2147483648, 0.010740184783935547)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,), (8,), (9,), (10,), (11,), (12,), (13,), (14,), (15,), (16,), (17,), (18,), (19,), (20,), (21,), (22,), (23,), (24,), (25,), (26,), (27,), (28,), (29,), (30,), (31,)), 'f32'): [(0, 1.685650204308331e-05), (1, 2.052899799309671e-05), (2, 2.0634441170841455e-05), (4, 2.0965846488252282e-05), (8, 2.1399289835244417e-05), (16, 2.123255399055779e-05), (32, 2.1260377252474427e-05), (64, 2.1473912056535482e-05), (128, 2.2383377654477954e-05), (256, 2.0894425688311458e-05), (512, 2.08600249607116e-05), (1024, 2.0921259419992566e-05), (2048, 2.090731868520379e-05), (4096, 2.1033076336607337e-05), (8192, 2.0954932551831007e-05), (16384, 2.0931096514686942e-05), (32768, 2.0985491573810577e-05), (65536, 2.1021929569542408e-05), (131072, 2.0843202946707606e-05), (262144, 2.088851761072874e-05), (524288, 2.0851788576692343e-05), (1048576, 3.3031508792191744e-05), (2097152, 4.459626507014036e-05), (4194304, 6.597698666155338e-05), (8388608, 0.00010871980339288713), (16777216, 0.00019322801381349564), (33554432, 0.000367976725101471), (67108864, 0.0007203370332717896), (134217728, 0.0013808086514472961), (268435456, 0.0027297884225845337), (536870912, 0.005426526069641113), (1073741824, 0.010750508308410645)]
available_memory_per_device:
14667979161
dot_cost_dict:
((), 'f16'): [(0, 1.662003342062235e-05), (4194304, 2.9675080440938473e-05), (33554432, 2.9678398277610537e-05), (113246208, 3.9767706766724586e-05), (268435456, 2.968066837638617e-05), (524288000, 3.7421355955302715e-05), (905969664, 3.7816702388226986e-05), (1438646272, 5.177216371521354e-05), (2147483648, 5.5833079386502504e-05), (3057647616, 7.949670873586001e-05), (4194304000, 8.927180897444487e-05), (5582618624, 9.796867379918694e-05), (7247757312, 0.00010636699153110385), (9214885888, 0.00013523627337606203), (11509170176, 0.00016890684303318807), (14155776000, 0.0002077480303341872), (17179869184, 0.0002307849936187267), (20606615552, 0.00024195347214117646), (24461180928, 0.0002872120190367069), (28768731136, 0.00033778930702550974), (33554432000, 0.0003939808217238778), (38843449344, 0.0004220617401016342), (44660948992, 0.000485273016801879), (51032096768, 0.000554500074702147), (57982058496, 0.0006300163584024394), (65536000000, 0.0006726105486760374), (73719087104, 0.0007565953922259221), (82556485632, 0.0008472955794937884), (92073361408, 0.0009154096712284546), (102294880256, 0.0010170338225032243), (113246208000, 0.0011259138631180882), (124952510464, 0.0012299516797065735), (137438953472, 0.0013328195438096415), (150730702848, 0.0013959321501028485), (164852924416, 0.0015267194598225326), (179830784000, 0.0016654309189876625), (195689447424, 0.0018122995908172218), (212454080512, 0.0019675585385301943), (230149849088, 0.0021314408253455632), (248801918976, 0.0023041795144823126), (268435456000, 0.002394599299277029), (289075625984, 0.0025787215360230016), (310747594752, 0.0027720480138255705), (333476528128, 0.0029748032263690427), (357287591936, 0.0031872116673377256), (382205952000, 0.0034094978304159257), (408256774144, 0.003641886209287951), (435465224192, 0.0038497791540154204), (463856467968, 0.004100775128833432), (493455671296, 0.004362450205549447), (524288000000, 0.004635026865453007), (556378619904, 0.004918727589833655), (589752696832, 0.005189138051320815), (624435396608, 0.005494305485224802), (660451885056, 0.005811208708702071), (697827328000, 0.006124526470691174), (736586891264, 0.006464702273039226), (776755740672, 0.006817246765960695), (818359042048, 0.0071823808189301055), (861421961216, 0.007560325301421984), (905969664000, 0.007951301082910856), (952027316224, 0.008355529032871244), (999620083712, 0.008773230020777678), (1048773132288, 0.009177684783935547)]
((), 'f32'): [(0, 1.65026867762208e-05), (4194304, 2.9112619813531634e-05), (33554432, 2.9380840715020895e-05), (113246208, 4.260562127456069e-05), (268435456, 4.5800465159118176e-05), (524288000, 7.021648343652487e-05), (905969664, 9.645731188356875e-05), (1438646272, 0.00013177230721339583), (2147483648, 0.00019034423166885972), (3057647616, 0.0002501418930478394), (4194304000, 0.0003141497145406902), (5582618624, 0.00041813327005365864), (7247757312, 0.0005278292810544372), (9214885888, 0.0006554973078891635), (11509170176, 0.0008187003244642079), (14155776000, 0.0009950751761482904), (17179869184, 0.0011966387430826823), (20606615552, 0.0014353237658118207), (24461180928, 0.0016767866690798048), (28768731136, 0.001972064431278872), (33554432000, 0.0022670427424795678), (38843449344, 0.0026243853547629095), (44660948992, 0.0030174338902403046), (51032096768, 0.0034478886309686125), (57982058496, 0.003917449859004693), (65536000000, 0.004365294311303166), (73719087104, 0.004910362420189725), (82556485632, 0.005457679102243471), (92073361408, 0.006086824754989009), (102294880256, 0.006762553250247219), (113246208000, 0.007437059938772465), (124952510464, 0.008205831579110019), (137438953472, 0.009025836299025782), (150730702848, 0.00989872677850615), (164852924416, 0.010826155697537516), (179830784000, 0.011809775736106273), (195689447424, 0.012851239574198817), (212454080512, 0.01395219989180154), (230149849088, 0.015109097353324363), (248801918976, 0.01621807098388672), (268435456000, 0.01731490960685156), (289075625984, 0.018646263828340878), (310747594752, 0.020044172233631533), (333476528128, 0.02151025809549917), (357287591936, 0.023046144686719422), (382205952000, 0.02465345528006794), (408256774144, 0.026333813148320364), (435465224192, 0.028088841564252332), (463856467968, 0.029920163800639492), (493455671296, 0.031829403130257485), (524288000000, 0.03381818282588195), (556378619904, 0.03588812616028853), (589752696832, 0.03804085640625287), (624435396608, 0.04027799683655062), (660451885056, 0.0426011707239574), (697827328000, 0.04501200134124887), (736586891264, 0.04751211196120067), (776755740672, 0.05010312585658844), (818359042048, 0.05278666630018782), (861421961216, 0.05556435656477446), (905969664000, 0.058437819923123996), (952027316224, 0.061408679648012075), (999620083712, 0.06447855901221433), (1048773132288, 0.06764908128850641)]
conv_cost_dict:
[]
op_cost_dict:
[]
--------------------------------------------------
Key: ('default', (2, 8))
all_gather_cost_dict:
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15),), 'f16'): [(0, 1.7434300389140844e-05), (1, 1.7390615539625287e-05), (2, 1.7592363292351365e-05), (4, 1.7374521121382713e-05), (8, 1.7467624275013804e-05), (16, 3.493524855002761e-05), (32, 6.987049710005522e-05), (64, 0.00013974099420011044), (128, 0.00027948198840022087), (256, 0.0005578311393037438), (512, 0.000543937407201156), (1024, 0.0005629559746012092), (2048, 0.0005692573904525489), (4096, 0.0005746695096604526), (8192, 0.0005735404265578836), (16384, 0.0006374561635311693), (32768, 0.0007235037337522955), (65536, 0.0007574897317681462), (131072, 0.0007695992535445838), (262144, 0.0009032374655362219), (524288, 0.0011071275221183896), (1048576, 0.0018564455676823854), (2097152, 0.0025985692627727985), (4194304, 0.003971142694354057), (8388608, 0.006692989729344845), (16777216, 0.01288990303874016), (33554432, 0.02577980607748032), (67108864, 0.04453050345182419), (134217728, 0.08906100690364838), (268435456, 0.17812201380729675), (536870912, 0.3475048303604126), (1073741824, 0.6950096607208252), (2147483648, 1.3900193214416503)]
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15),), 'f32'): [(0, 1.75797613337636e-05), (1, 1.818983582779765e-05), (2, 1.754786353558302e-05), (4, 1.8145889043807983e-05), (8, 1.7782003851607442e-05), (16, 3.5564007703214884e-05), (32, 7.112801540642977e-05), (64, 0.00014225603081285954), (128, 0.00028451206162571907), (256, 0.0005690241232514381), (512, 0.000608004309469834), (1024, 0.0005964826559647918), (2048, 0.0005771339929196984), (4096, 0.0005624739569611847), (8192, 0.0006249232683330774), (16384, 0.000736755784600973), (32768, 0.0007599778764415532), (65536, 0.0007874499133322388), (131072, 0.0009775339858606458), (262144, 0.001205847191158682), (524288, 0.00193195347674191), (1048576, 0.0025529281701892614), (2097152, 0.0039140903390944), (4194304, 0.006622246466577053), (8388608, 0.011875571683049202), (16777216, 0.023751143366098404), (33554432, 0.04682799428701401), (67108864, 0.09261052310466766), (134217728, 0.1746246337890625), (268435456, 0.349249267578125), (536870912, 0.6948797941207886), (1073741824, 1.3897595882415772)]
(((0, 2, 4, 6, 8, 10, 12, 14), (1, 3, 5, 7, 9, 11, 13, 15)), 'f16'): [(0, 1.9182305550202727e-05), (1, 1.7444253899157047e-05), (2, 1.7653423128649592e-05), (4, 1.9406783394515514e-05), (8, 3.881356678903103e-05), (16, 7.762713357806206e-05), (32, 0.00015525426715612411), (64, 0.00031050853431224823), (128, 0.00035839094198308885), (256, 0.00034418562427163124), (512, 0.00033780589001253247), (1024, 0.0003915357519872487), (2048, 0.0003438893472775817), (4096, 0.00034850029624067247), (8192, 0.00038288044743239874), (16384, 0.0005021546094212681), (32768, 0.0005374988832045346), (65536, 0.0005661315808538347), (131072, 0.0006633821176365018), (262144, 0.0008136381511576473), (524288, 0.0011934252688661218), (1048576, 0.0019106807885691521), (2097152, 0.0031371482182294126), (4194304, 0.0056471507996320724), (8388608, 0.011294301599264145), (16777216, 0.02258860319852829), (33554432, 0.04517720639705658), (67108864, 0.09035441279411316), (134217728, 0.1790209412574768), (268435456, 0.3517500162124634), (536870912, 0.6928546905517579), (1073741824, 1.3811166763305665), (2147483648, 2.762233352661133)]
(((0, 2, 4, 6, 8, 10, 12, 14), (1, 3, 5, 7, 9, 11, 13, 15)), 'f32'): [(0, 1.715624239295721e-05), (1, 1.7207436030730605e-05), (2, 1.7334299627691507e-05), (4, 1.7667538486421108e-05), (8, 3.5335076972842216e-05), (16, 7.067015394568443e-05), (32, 0.00014134030789136887), (64, 0.00028268061578273773), (128, 0.00034685563878156245), (256, 0.0003692322061397135), (512, 0.00035775426658801734), (1024, 0.0003678250068333), (2048, 0.00037160090869292617), (4096, 0.00039826260763220484), (8192, 0.0004970520385541022), (16384, 0.0004992936446797103), (32768, 0.0005384869291447103), (65536, 0.00066832106676884), (131072, 0.0008187878411263229), (262144, 0.0011964453151449561), (524288, 0.0018555513815954325), (1048576, 0.0031262682750821114), (2097152, 0.005697816144675016), (4194304, 0.011395632289350033), (8388608, 0.022791264578700066), (16777216, 0.04558252915740013), (33554432, 0.09116505831480026), (67108864, 0.18089978396892548), (134217728, 0.35755767822265627), (268435456, 0.6983450889587403), (536870912, 1.3966901779174805), (1073741824, 2.793380355834961)]
(((0, 4, 8, 12), (1, 5, 9, 13), (2, 6, 10, 14), (3, 7, 11, 15)), 'f16'): [(0, 1.790552050806582e-05), (1, 1.789769157767296e-05), (2, 1.7855665646493435e-05), (4, 3.571133129298687e-05), (8, 7.142266258597374e-05), (16, 0.00014284532517194748), (32, 0.00027064591995440423), (64, 0.00026401266222819686), (128, 0.0002800218644551933), (256, 0.0002770925057120621), (512, 0.0002965596504509449), (1024, 0.0002739178016781807), (2048, 0.00028851203387603164), (4096, 0.00029033597093075514), (8192, 0.00033483983133919537), (16384, 0.00042344644316472113), (32768, 0.0004442593199200928), (65536, 0.0005226706271059811), (131072, 0.0006383352156262845), (262144, 0.0009101213072426617), (524288, 0.0014534334186464546), (1048576, 0.002571977092884481), (2097152, 0.004923638654872775), (4194304, 0.009547384455800056), (8388608, 0.019094768911600113), (16777216, 0.038189537823200226), (33554432, 0.07637907564640045), (67108864, 0.1527581512928009), (134217728, 0.3054821193218231), (268435456, 0.6109642386436462), (536870912, 1.2219284772872925), (1073741824, 2.443599534034729), (2147483648, 4.882336759567261)]
(((0, 4, 8, 12), (1, 5, 9, 13), (2, 6, 10, 14), (3, 7, 11, 15)), 'f32'): [(0, 1.7444835975766182e-05), (1, 1.746855559758842e-05), (2, 1.7610349459573627e-05), (4, 3.522069891914725e-05), (8, 7.04413978382945e-05), (16, 0.000140882795676589), (32, 0.0002575736434664577), (64, 0.000265585258603096), (128, 0.00028294610092416406), (256, 0.0002692592970561236), (512, 0.0002618648577481508), (1024, 0.0002602064923848957), (2048, 0.0002737118920776993), (4096, 0.00033123555476777256), (8192, 0.000434955582022667), (16384, 0.0004406339139677584), (32768, 0.0005108655022922903), (65536, 0.0006477581046056002), (131072, 0.0009017567790579051), (262144, 0.0015776559594087305), (524288, 0.002597912563942373), (1048576, 0.004933586344122887), (2097152, 0.009246932808309793), (4194304, 0.018493865616619587), (8388608, 0.036987731233239174), (16777216, 0.07397546246647835), (33554432, 0.1479509249329567), (67108864, 0.2959018498659134), (134217728, 0.5918036997318268), (268435456, 1.1836073994636536), (536870912, 2.367214798927307), (1073741824, 4.734429597854614)]
(((0, 8), (1, 9), (2, 10), (3, 11), (4, 12), (5, 13), (6, 14), (7, 15)), 'f16'): [(0, 1.7624726751819253e-05), (1, 1.797449658624828e-05), (2, 3.594899317249656e-05), (4, 7.189798634499311e-05), (8, 0.00014250961248762906), (16, 0.00013785218470729887), (32, 0.0001371793041471392), (64, 0.00014277960872277617), (128, 0.00014341081259772182), (256, 0.00015373594942502677), (512, 0.00015095600974746048), (1024, 0.000157581438543275), (2048, 0.00016977728228084743), (4096, 0.0001790293317753822), (8192, 0.00021405302686616778), (16384, 0.00027946726186200976), (32768, 0.0003163153014611453), (65536, 0.0003815912641584873), (131072, 0.0005601567681878805), (262144, 0.0009671211009845137), (524288, 0.0017449253355152905), (1048576, 0.003489850671030581), (2097152, 0.006979701342061162), (4194304, 0.013959402684122324), (8388608, 0.027918805368244648), (16777216, 0.055837610736489296), (33554432, 0.11167522147297859), (67108864, 0.22335044294595718), (134217728, 0.44670088589191437), (268435456, 0.8934017717838287), (536870912, 1.7868035435676575), (1073741824, 3.573607087135315), (2147483648, 7.14721417427063)]
(((0, 8), (1, 9), (2, 10), (3, 11), (4, 12), (5, 13), (6, 14), (7, 15)), 'f32'): [(0, 1.7276412108913064e-05), (1, 1.7095881048589945e-05), (2, 3.419176209717989e-05), (4, 6.838352419435978e-05), (8, 0.00013676704838871956), (16, 0.0001436314487364143), (32, 0.0001655240193940699), (64, 0.00017339229816570878), (128, 0.00016240458353422582), (256, 0.00015306012937799096), (512, 0.00014351095887832344), (1024, 0.000154908251715824), (2048, 0.00016153717297129333), (4096, 0.0001893829321488738), (8192, 0.0002929262991528958), (16384, 0.0003118962049484253), (32768, 0.0003754380450118333), (65536, 0.0005587605701293796), (131072, 0.0009547821537125856), (262144, 0.0017806717660278082), (524288, 0.0035613435320556164), (1048576, 0.007122687064111233), (2097152, 0.014245374128222466), (4194304, 0.02849074825644493), (8388608, 0.05698149651288986), (16777216, 0.11396299302577972), (33554432, 0.22792598605155945), (67108864, 0.4558519721031189), (134217728, 0.9117039442062378), (268435456, 1.8234078884124756), (536870912, 3.646815776824951), (1073741824, 7.293631553649902)]
(((0, 1, 2, 3, 4, 5, 6, 7), (8, 9, 10, 11, 12, 13, 14, 15)), 'f16'): [(0, 1.7587124602869153e-05), (1, 1.7511832993477583e-05), (2, 1.757816062308848e-05), (4, 1.7628743080422282e-05), (8, 3.5257486160844564e-05), (16, 7.051497232168913e-05), (32, 0.00014102994464337826), (64, 0.0002054695796687156), (128, 0.00020393446902744472), (256, 0.00020905421115458012), (512, 0.0002062937419395894), (1024, 0.0002015617792494595), (2048, 0.00020327829406596718), (4096, 0.00020565534941852093), (8192, 0.00020133896032348275), (16384, 0.00020538942771963772), (32768, 0.00020587537437677383), (65536, 0.0002084918087348342), (131072, 0.00021451630163937807), (262144, 0.00022815645206719637), (524288, 0.00022818541037850085), (1048576, 0.00022800761507824066), (2097152, 0.00026014266768470407), (4194304, 0.00027241394855082035), (8388608, 0.0003325191792100668), (16777216, 0.000434298999607563), (33554432, 0.0006606690585613251), (67108864, 0.0011033937335014343), (134217728, 0.002009764313697815), (268435456, 0.0038747265934944153), (536870912, 0.007405355572700501), (1073741824, 0.01453399658203125), (2147483648, 0.028803682327270506)]
(((0, 1, 2, 3, 4, 5, 6, 7), (8, 9, 10, 11, 12, 13, 14, 15)), 'f32'): [(0, 1.722466549836099e-05), (1, 1.7223763279616833e-05), (2, 1.72363652382046e-05), (4, 1.7274433048442006e-05), (8, 3.454886609688401e-05), (16, 6.909773219376802e-05), (32, 0.00013819546438753605), (64, 0.00020487376605160534), (128, 0.00020259132725186646), (256, 0.00020103721180930734), (512, 0.00019896062440238893), (1024, 0.00021634908625856042), (2048, 0.00020966984448023143), (4096, 0.00020258914446458223), (8192, 0.0001973652106244117), (16384, 0.00021994006237946448), (32768, 0.00022148925927467644), (65536, 0.00022226612782105803), (131072, 0.0002210386737715453), (262144, 0.00023345428053289652), (524288, 0.00025363892200402915), (1048576, 0.0002449792227707803), (2097152, 0.00027838279493153095), (4194304, 0.0003282784018665552), (8388608, 0.0004286505281925201), (16777216, 0.000664043240249157), (33554432, 0.0010892152786254883), (67108864, 0.0019952580332756042), (134217728, 0.003796406090259552), (268435456, 0.007395163178443909), (536870912, 0.014529204368591309), (1073741824, 0.028855967521667476)]
(((0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11), (12, 13, 14, 15)), 'f16'): [(0, 1.7837039195001125e-05), (1, 1.7859041690826416e-05), (2, 1.7901096725836396e-05), (4, 3.580219345167279e-05), (8, 7.160438690334558e-05), (16, 0.00010161325917579234), (32, 0.00010435195872560142), (64, 0.00010162935359403492), (128, 0.00010287357144989073), (256, 0.00010388216469436884), (512, 0.00010592362377792597), (1024, 0.00010780323646031319), (2048, 0.00010475548333488405), (4096, 0.00010495423339307308), (8192, 0.00011114749941043556), (16384, 0.0001134952181018889), (32768, 0.00011416329653002322), (65536, 0.00012158378376625478), (131072, 0.00012327000149525702), (262144, 0.00012995328870601952), (524288, 0.00013825629139319062), (1048576, 0.00015165057266131043), (2097152, 0.00017433910397812724), (4194304, 0.00020797410979866982), (8388608, 0.0002810778096318245), (16777216, 0.00042610382661223406), (33554432, 0.0007119011133909225), (67108864, 0.00131143257021904), (134217728, 0.0025332458317279816), (268435456, 0.005008503794670105), (536870912, 0.00990457832813263), (1073741824, 0.019798779487609865), (2147483648, 0.03959755897521973)]
(((0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11), (12, 13, 14, 15)), 'f32'): [(0, 1.778022851794958e-05), (1, 1.7812912119552493e-05), (2, 1.7832324374467134e-05), (4, 3.566464874893427e-05), (8, 7.132929749786854e-05), (16, 0.00010626445873640479), (32, 0.00010613841004669665), (64, 0.00010653119534254074), (128, 0.00010754735558293758), (256, 0.00010723734158091247), (512, 0.00010862993076443672), (1024, 0.0001087482669390738), (2048, 0.00010682671563699841), (4096, 0.00010996137280017138), (8192, 0.00011255638673901558), (16384, 0.00011809813440777361), (32768, 0.00011992114013992251), (65536, 0.00012763956328853965), (131072, 0.00013114835019223392), (262144, 0.00013404199853539467), (524288, 0.00015300847007893026), (1048576, 0.0001746517955325544), (2097152, 0.0002112877555191517), (4194304, 0.0002853740006685257), (8388608, 0.0004308405332267284), (16777216, 0.0007186196744441987), (33554432, 0.0013285428285598755), (67108864, 0.002546984702348709), (134217728, 0.004979729652404785), (268435456, 0.009900972247123718), (536870912, 0.019725918769836426), (1073741824, 0.03945183753967285)]
(((0, 1), (2, 3), (4, 5), (6, 7), (8, 9), (10, 11), (12, 13), (14, 15)), 'f16'): [(0, 1.797467120923102e-05), (1, 1.7996964743360877e-05), (2, 3.5993929486721754e-05), (4, 6.961121107451618e-05), (8, 6.965044303797185e-05), (16, 6.878862041048706e-05), (32, 6.957983714528382e-05), (64, 5.710317054763436e-05), (128, 6.769414176233113e-05), (256, 6.667623529210687e-05), (512, 6.658706115558743e-05), (1024, 6.733695045113564e-05), (2048, 6.622672663070261e-05), (4096, 6.679943180643022e-05), (8192, 7.238943362608552e-05), (16384, 7.574391202069819e-05), (32768, 7.796630961820483e-05), (65536, 8.492128108628094e-05), (131072, 8.680101018399e-05), (262144, 9.521032916381955e-05), (524288, 0.00010066694812849165), (1048576, 0.00012564740609377623), (2097152, 0.0001770437229424715), (4194304, 0.0002689533866941929), (8388608, 0.0004638920072466135), (16777216, 0.000848572701215744), (33554432, 0.0016135480254888532), (67108864, 0.003156617283821106), (134217728, 0.006214398890733719), (268435456, 0.012365072965621948), (536870912, 0.024655669927597046), (1073741824, 0.049219584465026854), (2147483648, 0.09812426567077635)]
(((0, 1), (2, 3), (4, 5), (6, 7), (8, 9), (10, 11), (12, 13), (14, 15)), 'f32'): [(0, 1.7597252735868096e-05), (1, 1.7525278963148594e-05), (2, 3.505055792629719e-05), (4, 6.714227492921054e-05), (8, 6.716873031109571e-05), (16, 6.776468944735825e-05), (32, 6.653115269728005e-05), (64, 6.651887088082731e-05), (128, 6.565163494087756e-05), (256, 6.560893962159753e-05), (512, 6.59321085549891e-05), (1024, 6.672111339867115e-05), (2048, 6.649366696365178e-05), (4096, 7.416805601678789e-05), (8192, 7.341644959524274e-05), (16384, 7.617531809955835e-05), (32768, 8.355526369996369e-05), (65536, 8.499695104546845e-05), (131072, 9.251691517420113e-05), (262144, 0.00010061633656732738), (524288, 0.00012620151392184198), (1048576, 0.00017670675879344344), (2097152, 0.0002682164777070284), (4194304, 0.0004632361233234405), (8388608, 0.0008502868004143238), (16777216, 0.0016152560710906982), (33554432, 0.0031495261937379837), (67108864, 0.006216071546077728), (134217728, 0.01233762502670288), (268435456, 0.02461393177509308), (536870912, 0.04913022518157959), (1073741824, 0.09810860157012939)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,), (8,), (9,), (10,), (11,), (12,), (13,), (14,), (15,)), 'f16'): [(0, 1.756934216246009e-05), (1, 2.0855077309533954e-05), (2, 2.087693428620696e-05), (4, 2.0832638256251812e-05), (8, 2.0857638446614146e-05), (16, 2.139329444617033e-05), (32, 2.0981475245207548e-05), (64, 2.0926352590322495e-05), (128, 2.0925450371578336e-05), (256, 2.1206593373790383e-05), (512, 2.0888313883915544e-05), (1024, 2.077795215882361e-05), (2048, 2.0776467863470316e-05), (4096, 2.0816311007365584e-05), (8192, 2.1040934370830655e-05), (16384, 2.0896346541121602e-05), (32768, 2.072358620353043e-05), (65536, 2.079978003166616e-05), (131072, 2.116232644766569e-05), (262144, 2.0603183656930923e-05), (524288, 2.0599894924089313e-05), (1048576, 2.0632287487387657e-05), (2097152, 3.2700481824576855e-05), (4194304, 4.499021451920271e-05), (8388608, 6.645731627941132e-05), (16777216, 0.00010866019874811174), (33554432, 0.0001988131552934647), (67108864, 0.0003637243062257766), (134217728, 0.000701267272233963), (268435456, 0.001402534544467926), (536870912, 0.002730250358581543), (1073741824, 0.005449414253234863), (2147483648, 0.010741114616394043)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,), (8,), (9,), (10,), (11,), (12,), (13,), (14,), (15,)), 'f32'): [(0, 1.7370242858305573e-05), (1, 2.0348757971078157e-05), (2, 2.0623847376555204e-05), (4, 2.0660634618252516e-05), (8, 2.0569248590618372e-05), (16, 2.0492298062890768e-05), (32, 2.0716863218694925e-05), (64, 2.0804902305826545e-05), (128, 2.066281740553677e-05), (256, 2.0596198737621307e-05), (512, 2.0764273358508945e-05), (1024, 2.061837585642934e-05), (2048, 2.053985372185707e-05), (4096, 2.0545354345813394e-05), (8192, 2.0964827854186296e-05), (16384, 2.0622333977371454e-05), (32768, 2.0997016690671444e-05), (65536, 2.09819118026644e-05), (131072, 2.0388368284329772e-05), (262144, 2.0792620489373803e-05), (524288, 2.0836014300584793e-05), (1048576, 3.261410165578127e-05), (2097152, 4.467496182769537e-05), (4194304, 6.614881567656994e-05), (8388608, 0.00010865461081266402), (16777216, 0.0001964149996638298), (33554432, 0.00037484243512153625), (67108864, 0.0007018223404884338), (134217728, 0.0013848692178726196), (268435456, 0.002740815281867981), (536870912, 0.005416297912597656), (1073741824, 0.01075742244720459)]
all_reduce_cost_dict:
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15),), 'f16'): [(0, 0.00020883057732135057), (1, 0.0004555860359687358), (2, 0.0004584142589010299), (4, 0.00047359045129269367), (8, 0.0004624565481208265), (16, 0.0004566590941976756), (32, 0.0004603058914653957), (64, 0.00046698012738488615), (128, 0.00046563553041778505), (256, 0.00046503546764142806), (512, 0.000473352411063388), (1024, 0.0004933056188747287), (2048, 0.0004970050067640841), (4096, 0.0005001516838092357), (8192, 0.00051125354366377), (16384, 0.0010159244120586663), (32768, 0.0012624455557670444), (65536, 0.0012680788349825889), (131072, 0.0012256886693648994), (262144, 0.001409389078617096), (524288, 0.001767151872627437), (1048576, 0.003118509892374277), (2097152, 0.0044950940646231174), (4194304, 0.006805797107517719), (8388608, 0.011049332097172737), (16777216, 0.01703067496418953), (33554432, 0.03406134992837906), (67108864, 0.06340043246746063), (134217728, 0.12072043418884278), (268435456, 0.23823473453521732), (536870912, 0.47646946907043464), (1073741824, 0.9529389381408693), (2147483648, 1.7934196233749389)]
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15),), 'f32'): [(0, 0.00020587912877090275), (1, 0.00046610296703875065), (2, 0.0004563349357340485), (4, 0.0004593155754264444), (8, 0.0004599998064804823), (16, 0.00047214428195729846), (32, 0.0004742362361866981), (64, 0.00047589300083927805), (128, 0.0004711807705461979), (256, 0.00046965444926172495), (512, 0.00047798969899304217), (1024, 0.00048741252976469696), (2048, 0.00047065925900824373), (4096, 0.00048664267524145544), (8192, 0.0009732853504829109), (16384, 0.0012022692244499922), (32768, 0.001079241425031796), (65536, 0.0011310360569041222), (131072, 0.001365457836072892), (262144, 0.001735167112201452), (524288, 0.00320145464502275), (1048576, 0.004590007476508617), (2097152, 0.007152009755373001), (4194304, 0.008169708773493767), (8388608, 0.016339417546987534), (16777216, 0.03267883509397507), (33554432, 0.06535767018795013), (67108864, 0.13071534037590027), (134217728, 0.26143068075180054), (268435456, 0.45894207954406735), (536870912, 0.9178841590881347), (1073741824, 1.7439177751541137)]
(((0, 2, 4, 6, 8, 10, 12, 14), (1, 3, 5, 7, 9, 11, 13, 15)), 'f16'): [(0, 0.00010161296813748777), (1, 0.0004086760163772851), (2, 0.000398303905967623), (4, 0.0004012445278931409), (8, 0.00040702748810872436), (16, 0.000380560610210523), (32, 0.0003876216651406139), (64, 0.00038122761179693043), (128, 0.00038748045335523784), (256, 0.0003190487914253026), (512, 0.0003250120207667351), (1024, 0.00033591786632314324), (2048, 0.0003609138948377222), (4096, 0.0003437136474531144), (8192, 0.0006101286271587014), (16384, 0.0007877819880377501), (32768, 0.000854573561809957), (65536, 0.0008681745093781501), (131072, 0.0010699838458094746), (262144, 0.0014023123076185584), (524288, 0.0020497298100963235), (1048576, 0.003445627400651574), (2097152, 0.005753838457167149), (4194304, 0.01064027938991785), (8388608, 0.0212805587798357), (16777216, 0.04230529069900513), (33554432, 0.06681565195322037), (67108864, 0.1223662942647934), (134217728, 0.19435493946075438), (268435456, 0.38870987892150877), (536870912, 0.7395735263824463), (1073741824, 1.4791470527648927), (2147483648, 2.9582941055297853)]
(((0, 2, 4, 6, 8, 10, 12, 14), (1, 3, 5, 7, 9, 11, 13, 15)), 'f32'): [(0, 0.0001034754968713969), (1, 0.00039044566801749175), (2, 0.00038058913196437055), (4, 0.0003841735888272524), (8, 0.0004007164388895035), (16, 0.00038319057784974575), (32, 0.0003890400403179229), (64, 0.0003891937085427344), (128, 0.00032612643553875387), (256, 0.00032486021518707275), (512, 0.00032644448219798505), (1024, 0.0003492894757073373), (2048, 0.00035176146775484085), (4096, 0.0006059384904801846), (8192, 0.0008101487474050372), (16384, 0.0008239145681727676), (32768, 0.000886208115844056), (65536, 0.0010561850212980062), (131072, 0.0013338550343178213), (262144, 0.002026231260970235), (524288, 0.0033500867430120702), (1048576, 0.005754352081567049), (2097152, 0.010702230967581272), (4194304, 0.021404461935162544), (8388608, 0.04258769378066063), (16777216, 0.07706230133771896), (33554432, 0.14274737238883972), (67108864, 0.2548658847808838), (134217728, 0.47188291549682615), (268435456, 0.801437783241272), (536870912, 1.5998589515686035), (1073741824, 2.917495036125183)]
(((0, 4, 8, 12), (1, 5, 9, 13), (2, 6, 10, 14), (3, 7, 11, 15)), 'f16'): [(0, 7.102187373675406e-05), (1, 0.0003048602375201881), (2, 0.00030462886206805706), (4, 0.00030920657445676625), (8, 0.0003110250981990248), (16, 0.0003179804771207273), (32, 0.0003191001305822283), (64, 0.000295147649012506), (128, 0.00031327386386692524), (256, 0.00034180967486463487), (512, 0.00039271262357942755), (1024, 0.00044808685197494924), (2048, 0.0004443627840373665), (4096, 0.0005192828248254955), (8192, 0.0005091829807497561), (16384, 0.0007007367385085672), (32768, 0.0007326821505557746), (65536, 0.0008708325622137637), (131072, 0.0010755056282505393), (262144, 0.001537484407890588), (524288, 0.002654781797900796), (1048576, 0.005216172197833657), (2097152, 0.009895468596369028), (4194304, 0.019667230546474457), (8388608, 0.039334461092948914), (16777216, 0.07866892218589783), (33554432, 0.1420898661017418), (67108864, 0.25615382194519043), (134217728, 0.4375342130661011), (268435456, 0.8129375219345093), (536870912, 1.485215926170349), (1073741824, 2.8336618900299073), (2147483648, 5.667323780059815)]
(((0, 4, 8, 12), (1, 5, 9, 13), (2, 6, 10, 14), (3, 7, 11, 15)), 'f32'): [(0, 7.0533889811486e-05), (1, 0.00031869899248704314), (2, 0.00032249875948764384), (4, 0.0003114475985057652), (8, 0.00030902339494787157), (16, 0.0003082269395235926), (32, 0.0003317376831546426), (64, 0.00032905934494920075), (128, 0.0003279176016803831), (256, 0.00037030302337370813), (512, 0.00042380692320875823), (1024, 0.0004471911524888128), (2048, 0.0005128285847604275), (4096, 0.0004979418881703168), (8192, 0.0006853540544398129), (16384, 0.0007240003906190395), (32768, 0.0008706329681444913), (65536, 0.0010869998950511217), (131072, 0.0015849771443754435), (262144, 0.002677156124264002), (524288, 0.0051768028642982244), (1048576, 0.009838758036494255), (2097152, 0.019290140829980373), (4194304, 0.03858028165996075), (8388608, 0.0771605633199215), (16777216, 0.14485591650009155), (33554432, 0.2676630765199661), (67108864, 0.46125700473785397), (134217728, 0.7964886903762818), (268435456, 1.5227014303207398), (536870912, 2.8367443084716797), (1073741824, 5.673488616943359)]
(((0, 8), (1, 9), (2, 10), (3, 11), (4, 12), (5, 13), (6, 14), (7, 15)), 'f16'): [(0, 2.9445625841617584e-05), (1, 0.0002608791983220726), (2, 0.0002708246174734086), (4, 0.0002668725501280278), (8, 0.00026251800591126084), (16, 0.00026036318740807474), (32, 0.00026991471531800926), (64, 0.00027820566901937127), (128, 0.00027830080944113433), (256, 0.00026547478046268225), (512, 0.0002823387912940234), (1024, 0.0002863601257558912), (2048, 0.00029541426920332015), (4096, 0.00031715136719867587), (8192, 0.00046856451081112027), (16384, 0.0005230451643001288), (32768, 0.0006048170325811952), (65536, 0.0007372617837972939), (131072, 0.0010375259444117546), (262144, 0.0017786950338631868), (524288, 0.0032128958264365797), (1048576, 0.006425791652873159), (2097152, 0.012851583305746319), (4194304, 0.025703166611492637), (8388608, 0.051406333222985275), (16777216, 0.10281266644597055), (33554432, 0.2056253328919411), (67108864, 0.4112506657838822), (134217728, 0.8225013315677644), (268435456, 1.6450026631355288), (536870912, 3.2900053262710576), (1073741824, 6.580010652542115), (2147483648, 13.16002130508423)]
(((0, 8), (1, 9), (2, 10), (3, 11), (4, 12), (5, 13), (6, 14), (7, 15)), 'f32'): [(0, 2.910778857767582e-05), (1, 0.0002677922020666301), (2, 0.00027125515043735504), (4, 0.000273309153271839), (8, 0.00027454373775981367), (16, 0.00028245890280231833), (32, 0.000279801752185449), (64, 0.0002852373290807009), (128, 0.0002690661931410432), (256, 0.0002789002610370517), (512, 0.00028632936300709844), (1024, 0.0003186393005307764), (2048, 0.0003318158851470798), (4096, 0.00048074463848024607), (8192, 0.0005393436294980347), (16384, 0.0006006672338116914), (32768, 0.0007211792399175465), (65536, 0.0010344722541049123), (131072, 0.0017747163074091077), (262144, 0.0032354495488107204), (524288, 0.006470899097621441), (1048576, 0.012941798195242882), (2097152, 0.025883596390485764), (4194304, 0.05176719278097153), (8388608, 0.10353438556194305), (16777216, 0.2070687711238861), (33554432, 0.4141375422477722), (67108864, 0.8282750844955444), (134217728, 1.6565501689910889), (268435456, 3.3131003379821777), (536870912, 6.6262006759643555), (1073741824, 13.252401351928711)]
(((0, 1, 2, 3, 4, 5, 6, 7), (8, 9, 10, 11, 12, 13, 14, 15)), 'f16'): [(0, 0.00021338820806704462), (1, 0.0002188019279856235), (2, 0.00021676954929716885), (4, 0.00021783736883662644), (8, 0.00021303363610059026), (16, 0.00021210458362475038), (32, 0.000218911241972819), (64, 0.00021835675579495728), (128, 0.00021988447406329215), (256, 0.00021725948317907748), (512, 0.0002159720752388239), (1024, 0.00022141056251712143), (2048, 0.000217644585063681), (4096, 0.00021910347277298567), (8192, 0.00022018325398676095), (16384, 0.00022457443992607296), (32768, 0.00022322672884911296), (65536, 0.00020524297724477947), (131072, 0.00023690279340371492), (262144, 0.00024853801005519927), (524288, 0.00027354617486707866), (1048576, 0.00028528005350381136), (2097152, 0.0003268623258918524), (4194304, 0.000415522838011384), (8388608, 0.0005049146711826324), (16777216, 0.0007426682859659195), (33554432, 0.0011542066931724548), (67108864, 0.0020563751459121704), (134217728, 0.003842346370220184), (268435456, 0.007448956370353699), (536870912, 0.014671611785888673), (1073741824, 0.02879838943481445), (2147483648, 0.057282447814941406)]
(((0, 1, 2, 3, 4, 5, 6, 7), (8, 9, 10, 11, 12, 13, 14, 15)), 'f32'): [(0, 0.0002122737350873649), (1, 0.0002062956627923995), (2, 0.00020596481044776735), (4, 0.00020563392899930477), (8, 0.00020562586723826826), (16, 0.00020343161304481328), (32, 0.00020586224854923785), (64, 0.0002073949435725808), (128, 0.0002090416382998228), (256, 0.00020491419127210975), (512, 0.00020506826695054772), (1024, 0.00021548333461396393), (2048, 0.00022222989355213943), (4096, 0.00021563505288213491), (8192, 0.0002206570643465966), (16384, 0.00021760756499134007), (32768, 0.00022241956321522593), (65536, 0.00022872706176713112), (131072, 0.0002374179020989686), (262144, 0.00025993079179897904), (524288, 0.0002771299914456904), (1048576, 0.0003241762751713395), (2097152, 0.0004107183776795864), (4194304, 0.0005016690120100975), (8388608, 0.0007137889042496681), (16777216, 0.0011528506875038147), (33554432, 0.0020685121417045593), (67108864, 0.0038328468799591064), (134217728, 0.007385462522506714), (268435456, 0.01454291343688965), (536870912, 0.028686141967773436), (1073741824, 0.05713868141174317)]
(((0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11), (12, 13, 14, 15)), 'f16'): [(0, 0.00010461278725415468), (1, 0.00010134404874406755), (2, 0.00010795486741699278), (4, 0.0001083112438209355), (8, 0.00010782654862850904), (16, 0.00010726210894063116), (32, 0.00010729936184361578), (64, 0.00010708114132285118), (128, 0.0001068100973498076), (256, 0.00010446127271279693), (512, 0.00010364261106587945), (1024, 0.00010466779349371791), (2048, 0.00011049897875636815), (4096, 0.00011304585495963693), (8192, 0.00011529511539265515), (16384, 0.00011677210568450391), (32768, 0.00011962480493821202), (65536, 0.00012752681504935026), (131072, 0.00013454575673677027), (262144, 0.00014999127597548068), (524288, 0.00016404554480686784), (1048576, 0.00018711801385506988), (2097152, 0.00022086151875555515), (4194304, 0.00029018381610512733), (8388608, 0.0004490739665925503), (16777216, 0.0007559619843959808), (33554432, 0.0013389550149440765), (67108864, 0.002579405903816223), (134217728, 0.005121886730194092), (268435456, 0.01008865237236023), (536870912, 0.019861888885498048), (1073741824, 0.039512038230895996), (2147483648, 0.07902407646179199)]
(((0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11), (12, 13, 14, 15)), 'f32'): [(0, 0.00010447605745866895), (1, 0.00010354522964917122), (2, 0.00010349464719183743), (4, 0.00010398426093161107), (8, 0.00010267080506309867), (16, 0.00010448464308865368), (32, 0.00010421726619824767), (64, 0.00010381685569882393), (128, 0.00010272738290950657), (256, 0.00010357721475884318), (512, 0.00010587708675302565), (1024, 0.00010921192006208001), (2048, 0.00011391658335924149), (4096, 0.00010593858314678072), (8192, 0.00010594117338769137), (16384, 0.00012020839494653045), (32768, 0.0001255083188880235), (65536, 0.00013218639651313424), (131072, 0.00014368127449415624), (262144, 0.00015286140842363238), (524288, 0.000186479592230171), (1048576, 0.00022366270422935489), (2097152, 0.00029291282407939434), (4194304, 0.00044770119711756706), (8388608, 0.0007541552186012269), (16777216, 0.001343660056591034), (33554432, 0.002536628395318985), (67108864, 0.00498625636100769), (134217728, 0.009904682636260986), (268435456, 0.019698166847229005), (536870912, 0.03939633369445801), (1073741824, 0.07873747348785401)]
(((0, 1), (2, 3), (4, 5), (6, 7), (8, 9), (10, 11), (12, 13), (14, 15)), 'f16'): [(0, 6.6927052102983e-05), (1, 6.641639629378915e-05), (2, 6.64594117552042e-05), (4, 6.737830699421465e-05), (8, 6.755645154044032e-05), (16, 6.607320392504334e-05), (32, 6.705941632390022e-05), (64, 6.835028761997819e-05), (128, 6.800083792768419e-05), (256, 6.781244883313775e-05), (512, 6.777993985451758e-05), (1024, 6.76233321428299e-05), (2048, 6.70046720188111e-05), (4096, 7.249318878166378e-05), (8192, 7.450091652572155e-05), (16384, 7.643693243153393e-05), (32768, 7.94637599028647e-05), (65536, 8.731504203751683e-05), (131072, 9.305361891165376e-05), (262144, 0.00010314895189367235), (524288, 0.00012438010890036821), (1048576, 0.00017052545445039868), (2097152, 0.0002690783003345132), (4194304, 0.0004562260583043099), (8388608, 0.0008336333557963371), (16777216, 0.0015784073621034622), (33554432, 0.003084298223257065), (67108864, 0.0060743317008018485), (134217728, 0.01207432895898819), (268435456, 0.024062648415565487), (536870912, 0.047930145263671876), (1073741824, 0.09581396579742432), (2147483648, 0.19139294624328612)]
(((0, 1), (2, 3), (4, 5), (6, 7), (8, 9), (10, 11), (12, 13), (14, 15)), 'f32'): [(0, 6.722306716255844e-05), (1, 6.578312604688108e-05), (2, 6.603918154723942e-05), (4, 6.667518755421042e-05), (8, 6.689847214147449e-05), (16, 6.681759259663522e-05), (32, 6.677699275314808e-05), (64, 6.71934976708144e-05), (128, 6.71831367071718e-05), (256, 6.782691343687475e-05), (512, 6.961394683457911e-05), (1024, 6.740112439729273e-05), (2048, 7.002608617767692e-05), (4096, 7.45860452298075e-05), (8192, 7.684039883315563e-05), (16384, 8.029001764953136e-05), (32768, 8.886438445188105e-05), (65536, 9.507298818789423e-05), (131072, 0.00010206946171820164), (262144, 0.00012310704914852977), (524288, 0.0001710226060822606), (1048576, 0.00026780227199196815), (2097152, 0.0004499214701354504), (4194304, 0.0008307616226375102), (8388608, 0.001576949842274189), (16777216, 0.0030738171190023422), (33554432, 0.006067294627428055), (67108864, 0.01204795390367508), (134217728, 0.024003371596336365), (268435456, 0.04800674319267273), (536870912, 0.09572057723999024), (1073741824, 0.19136548042297363)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,), (8,), (9,), (10,), (11,), (12,), (13,), (14,), (15,)), 'f16'): [(0, 1.6941077774390578e-05), (1, 2.0396779291331768e-05), (2, 2.0826846593990922e-05), (4, 2.0656996639445424e-05), (8, 2.0406179828569293e-05), (16, 2.0586070604622364e-05), (32, 2.060990664176643e-05), (64, 2.034829230979085e-05), (128, 2.1093874238431454e-05), (256, 2.0196166587993503e-05), (512, 2.004799898713827e-05), (1024, 2.0529550965875387e-05), (2048, 2.0874576875939965e-05), (4096, 2.0871433662250638e-05), (8192, 2.048772876150906e-05), (16384, 2.00835638679564e-05), (32768, 2.0273757399991155e-05), (65536, 2.0165607566013932e-05), (131072, 1.9983795937150717e-05), (262144, 2.0511652110144496e-05), (524288, 2.0084145944565535e-05), (1048576, 2.0139908883720636e-05), (2097152, 3.3511314541101456e-05), (4194304, 4.4516054913401604e-05), (8388608, 6.704218685626984e-05), (16777216, 0.00011004600673913957), (33554432, 0.00020912848412990573), (67108864, 0.0003681294620037079), (134217728, 0.0007362589240074158), (268435456, 0.0014676302671432495), (536870912, 0.002765512466430664), (1073741824, 0.005463337898254395), (2147483648, 0.01077284812927246)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,), (8,), (9,), (10,), (11,), (12,), (13,), (14,), (15,)), 'f32'): [(0, 1.7466547433286905e-05), (1, 2.0752253476530313e-05), (2, 2.0600855350494385e-05), (4, 2.00418580789119e-05), (8, 2.0695064449682832e-05), (16, 2.0903069525957108e-05), (32, 2.0916719222441316e-05), (64, 2.0539562683552504e-05), (128, 2.0511943148449063e-05), (256, 2.057343954220414e-05), (512, 2.0480540115386248e-05), (1024, 2.0343257347121835e-05), (2048, 2.0304054487496614e-05), (4096, 2.0127365132793784e-05), (8192, 2.0386709365993738e-05), (16384, 2.0193023374304175e-05), (32768, 2.0169682102277875e-05), (65536, 2.006386057473719e-05), (131072, 2.015859354287386e-05), (262144, 2.0091654732823372e-05), (524288, 2.0221108570694923e-05), (1048576, 3.265647683292627e-05), (2097152, 4.4452957808971405e-05), (4194304, 6.858492270112038e-05), (8388608, 0.00010965485125780107), (16777216, 0.00019918382167816162), (33554432, 0.00037208572030067444), (67108864, 0.0007441714406013489), (134217728, 0.0014170557260513306), (268435456, 0.0027762651443481445), (536870912, 0.00542449951171875), (1073741824, 0.010774898529052734)]
all_to_all_cost_dict:
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15),), 'f16'): [(0, 1.768811489455402e-05), (1, 1.7740647308528423e-05), (2, 1.7749611288309097e-05), (4, 1.771128154359758e-05), (8, 1.7684244085103273e-05), (16, 1.7636280972510576e-05), (32, 1.776847057044506e-05), (64, 2.0549894543364644e-05), (128, 1.766011700965464e-05), (256, 3.532023401930928e-05), (512, 7.064046803861856e-05), (1024, 0.00014128093607723713), (2048, 0.00028256187215447426), (4096, 0.0005651237443089485), (8192, 0.001130247488617897), (16384, 0.0011837465281132609), (32768, 0.001213199837366119), (65536, 0.0012000009592156857), (131072, 0.0011996495886705816), (262144, 0.0011972171487286687), (524288, 0.001268449064809829), (1048576, 0.0013472804566845298), (2097152, 0.0015408031176775694), (4194304, 0.001920292619615793), (8388608, 0.0031225280836224556), (16777216, 0.004817025735974312), (33554432, 0.009186316281557083), (67108864, 0.018372632563114166), (134217728, 0.03674526512622833), (268435456, 0.06748194694519043), (536870912, 0.13496389389038085), (1073741824, 0.2699277877807617), (2147483648, 0.4751358032226563)]
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15),), 'f32'): [(0, 1.7742218915373087e-05), (1, 1.7773068975657225e-05), (2, 1.7774698790162802e-05), (4, 1.778005389496684e-05), (8, 1.8294667825102806e-05), (16, 1.7889018636196852e-05), (32, 1.7748912796378136e-05), (64, 1.7844722606241703e-05), (128, 1.9233208149671555e-05), (256, 3.846641629934311e-05), (512, 7.693283259868622e-05), (1024, 0.00015386566519737244), (2048, 0.0003077313303947449), (4096, 0.0006154626607894897), (8192, 0.0012107417278457433), (16384, 0.001228181121405214), (32768, 0.001211092370795086), (65536, 0.0011842541862279177), (131072, 0.0012077375140506774), (262144, 0.0012387356837280095), (524288, 0.0013265589950606227), (1048576, 0.0015228800475597382), (2097152, 0.0019152653403580189), (4194304, 0.0030726026743650436), (8388608, 0.006145205348730087), (16777216, 0.012290410697460175), (33554432, 0.02458082139492035), (67108864, 0.0491616427898407), (134217728, 0.061289381980896), (268435456, 0.122578763961792), (536870912, 0.245157527923584), (1073741824, 0.490315055847168)]
(((0, 2, 4, 6, 8, 10, 12, 14), (1, 3, 5, 7, 9, 11, 13, 15)), 'f16'): [(0, 1.7548067262396216e-05), (1, 1.7658487195149064e-05), (2, 1.7601385479792953e-05), (4, 1.759204315021634e-05), (8, 1.746168709360063e-05), (16, 1.7581303836777806e-05), (32, 1.7566984752193093e-05), (64, 3.5133969504386187e-05), (128, 7.026793900877237e-05), (256, 0.00014053587801754475), (512, 0.0002810717560350895), (1024, 0.000562143512070179), (2048, 0.0006271630700211972), (4096, 0.0006245644472073764), (8192, 0.0006267843127716333), (16384, 0.000636425509583205), (32768, 0.0006241773662623018), (65536, 0.0006108318630140275), (131072, 0.0006394443335011601), (262144, 0.0007171970792114735), (524288, 0.0008032582118175925), (1048576, 0.0010956887854263186), (2097152, 0.001569921849295497), (4194304, 0.002451015170663595), (8388608, 0.00490203034132719), (16777216, 0.00980406068265438), (33554432, 0.01960812136530876), (67108864, 0.03921624273061752), (134217728, 0.07843248546123505), (268435456, 0.14285733699798583), (536870912, 0.27829775810241697), (1073741824, 0.4878313541412353), (2147483648, 0.7363193273544312)]
(((0, 2, 4, 6, 8, 10, 12, 14), (1, 3, 5, 7, 9, 11, 13, 15)), 'f32'): [(0, 1.7367972759529948e-05), (1, 1.736171543598175e-05), (2, 1.737623824737966e-05), (4, 1.7413636669516563e-05), (8, 1.7356709577143192e-05), (16, 1.7426791600883007e-05), (32, 1.739387516863644e-05), (64, 3.478775033727288e-05), (128, 6.957550067454576e-05), (256, 0.00013915100134909153), (512, 0.00027830200269818306), (1024, 0.0005566040053963661), (2048, 0.0006169928819872439), (4096, 0.0006172523426357657), (8192, 0.0006472197128459811), (16384, 0.0005964382726233453), (32768, 0.0005973548395559192), (65536, 0.0006377657991833985), (131072, 0.0006644238601438701), (262144, 0.0007499690400436521), (524288, 0.0009794952347874641), (1048576, 0.0017283130437135696), (2097152, 0.002414384391158819), (4194304, 0.004828768782317638), (8388608, 0.009657537564635277), (16777216, 0.019315075129270554), (33554432, 0.03046330064535141), (67108864, 0.06092660129070282), (134217728, 0.12185320258140564), (268435456, 0.24370640516281128), (536870912, 0.46795399188995357), (1073741824, 0.7140632152557373)]
(((0, 4, 8, 12), (1, 5, 9, 13), (2, 6, 10, 14), (3, 7, 11, 15)), 'f16'): [(0, 1.7209298675879836e-05), (1, 1.728784991428256e-05), (2, 1.7192825907841325e-05), (4, 1.7226760974153876e-05), (8, 1.7365964595228434e-05), (16, 3.473192919045687e-05), (32, 6.946385838091373e-05), (64, 0.00013892771676182747), (128, 0.00027785543352365494), (256, 0.00035497124190442264), (512, 0.0003536349977366626), (1024, 0.0003773822681978345), (2048, 0.00037454167613759637), (4096, 0.00036030780756846076), (8192, 0.00034281137050129473), (16384, 0.0003518833254929632), (32768, 0.00036172353429719806), (65536, 0.00037859586882404983), (131072, 0.00044539759983308613), (262144, 0.000545648654224351), (524288, 0.0008189292857423425), (1048576, 0.0012842405121773481), (2097152, 0.0021454740781337023), (4194304, 0.0042909481562674046), (8388608, 0.008581896312534809), (16777216, 0.017131805419921875), (33554432, 0.03426361083984375), (67108864, 0.0685272216796875), (134217728, 0.128544420003891), (268435456, 0.257088840007782), (536870912, 0.45775518417358396), (1073741824, 0.7559669494628907), (2147483648, 1.4223337411880492)]
(((0, 4, 8, 12), (1, 5, 9, 13), (2, 6, 10, 14), (3, 7, 11, 15)), 'f32'): [(0, 1.7356622265651822e-05), (1, 1.71732681337744e-05), (2, 1.8481456208974123e-05), (4, 1.7114303773269057e-05), (8, 1.7192942323163152e-05), (16, 3.4385884646326303e-05), (32, 6.877176929265261e-05), (64, 0.00013754353858530521), (128, 0.00027508707717061043), (256, 0.00034086036612279713), (512, 0.00036084357998333866), (1024, 0.000377010932425037), (2048, 0.0003655850887298584), (4096, 0.00034538304316811264), (8192, 0.00034961040364578366), (16384, 0.00036840594839304686), (32768, 0.0003809363697655499), (65536, 0.00044151683687232435), (131072, 0.000541781511856243), (262144, 0.0008101573912426829), (524288, 0.001301489188335836), (1048576, 0.0020926478318870068), (2097152, 0.0041852956637740135), (4194304, 0.008370591327548027), (8388608, 0.016741182655096054), (16777216, 0.03348236531019211), (33554432, 0.06696473062038422), (67108864, 0.13392946124076843), (134217728, 0.26785892248153687), (268435456, 0.47004020214080816), (536870912, 0.7700756311416626), (1073741824, 1.4160184144973755)]
(((0, 8), (1, 9), (2, 10), (3, 11), (4, 12), (5, 13), (6, 14), (7, 15)), 'f16'): [(0, 1.798642915673554e-05), (1, 1.811981201171875e-05), (2, 1.8692633602768183e-05), (4, 3.7385267205536366e-05), (8, 7.477053441107273e-05), (16, 0.00014954106882214546), (32, 0.00016262801364064217), (64, 0.00016417267033830285), (128, 0.00016009435057640076), (256, 0.00016297504771500826), (512, 0.00016740791033953428), (1024, 0.00017267916700802743), (2048, 0.0001631091581657529), (4096, 0.00016569418949075043), (8192, 0.00017459772061556578), (16384, 0.00019066396635025742), (32768, 0.00020703766494989395), (65536, 0.00024813736672513187), (131072, 0.0003814394294749946), (262144, 0.0005890281172469258), (524288, 0.0011177673586644232), (1048576, 0.0022355347173288465), (2097152, 0.004452670458704233), (4194304, 0.007562112063169479), (8388608, 0.01459299586713314), (16777216, 0.028336964547634125), (33554432, 0.05667392909526825), (67108864, 0.1133478581905365), (134217728, 0.2001493275165558), (268435456, 0.37960617542266845), (536870912, 0.7094990968704223), (1073741824, 1.4033711671829223), (2147483648, 2.8067423343658446)]
(((0, 8), (1, 9), (2, 10), (3, 11), (4, 12), (5, 13), (6, 14), (7, 15)), 'f32'): [(0, 1.7559243133291602e-05), (1, 1.7688580555841327e-05), (2, 1.7673650290817022e-05), (4, 3.5347300581634045e-05), (8, 7.069460116326809e-05), (16, 0.00014138920232653618), (32, 0.0001859762705862522), (64, 0.00017632372328080237), (128, 0.00018312837346456945), (256, 0.00018342971452511847), (512, 0.00017104149446822703), (1024, 0.00017654756084084508), (2048, 0.00018236090545542538), (4096, 0.0001894494635052979), (8192, 0.00018317249487154186), (16384, 0.00021202192874625325), (32768, 0.00026099695242010057), (65536, 0.0003704132395796478), (131072, 0.0005869979795534164), (262144, 0.001102500653360039), (524288, 0.002205001306720078), (1048576, 0.004410002613440156), (2097152, 0.007632461376488209), (4194304, 0.014646276831626892), (8388608, 0.029292553663253784), (16777216, 0.05858510732650757), (33554432, 0.11387308686971664), (67108864, 0.20823708176612854), (134217728, 0.3676485061645508), (268435456, 0.7180757522583008), (536870912, 1.4068291425704955), (1073741824, 2.813658285140991)]
(((0, 1, 2, 3, 4, 5, 6, 7), (8, 9, 10, 11, 12, 13, 14, 15)), 'f16'): [(0, 1.769844675436616e-05), (1, 1.8053222447633743e-05), (2, 1.78998161572963e-05), (4, 1.7899437807500362e-05), (8, 1.701180008240044e-05), (16, 1.7921411199495196e-05), (32, 1.8081191228702664e-05), (64, 3.616238245740533e-05), (128, 7.232476491481066e-05), (256, 0.00014464952982962132), (512, 0.000240408320678398), (1024, 0.00024470456992276013), (2048, 0.0002459324605297297), (4096, 0.00024389399914070967), (8192, 0.0002428166917525232), (16384, 0.00024686282267794013), (32768, 0.00024373148335143926), (65536, 0.0002427614526823163), (131072, 0.00023803938529454172), (262144, 0.00024015927920117974), (524288, 0.00022723351139575243), (1048576, 0.0002530739875510335), (2097152, 0.00029754661954939365), (4194304, 0.00037543836515396833), (8388608, 0.0004964100662618876), (16777216, 0.0007911180146038531), (33554432, 0.0013060858473181725), (67108864, 0.0023213550448417664), (134217728, 0.004526320844888687), (268435456, 0.008661694824695587), (536870912, 0.016814693808555603), (1073741824, 0.03320176601409912), (2147483648, 0.0655637264251709)]
(((0, 1, 2, 3, 4, 5, 6, 7), (8, 9, 10, 11, 12, 13, 14, 15)), 'f32'): [(0, 1.708007766865194e-05), (1, 1.7129757907241583e-05), (2, 1.7065060092136264e-05), (4, 1.72301079146564e-05), (8, 1.7244135960936546e-05), (16, 1.719561987556517e-05), (32, 1.7256883438676596e-05), (64, 3.451376687735319e-05), (128, 6.902753375470638e-05), (256, 0.00013805506750941277), (512, 0.00023573939688503742), (1024, 0.00023958642850629985), (2048, 0.00023653070093132553), (4096, 0.00020624074386432767), (8192, 0.00022634092601947486), (16384, 0.0002358142228331417), (32768, 0.00023515577777288854), (65536, 0.00020643763127736745), (131072, 0.000212968559935689), (262144, 0.0002521616406738758), (524288, 0.0002618014987092465), (1048576, 0.0003021201118826866), (2097152, 0.00037278467789292336), (4194304, 0.0005008620209991932), (8388608, 0.0007672822102904319), (16777216, 0.0013209404423832893), (33554432, 0.002351650968194008), (67108864, 0.004555810242891312), (134217728, 0.008716613054275513), (268435456, 0.017138376832008362), (536870912, 0.033510994911193845), (1073741824, 0.06616423130035401)]
(((0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11), (12, 13, 14, 15)), 'f16'): [(0, 1.783110201358795e-05), (1, 1.7931422917172313e-05), (2, 1.743726897984743e-05), (4, 1.727056223899126e-05), (8, 1.777114812284708e-05), (16, 3.554229624569416e-05), (32, 7.108459249138832e-05), (64, 0.0001307884231209755), (128, 0.00013122189557179809), (256, 0.00013126240810379386), (512, 0.0001274479436688125), (1024, 0.00011868006549775602), (2048, 0.0001313630200456828), (4096, 0.00013087468687444925), (8192, 0.00013014610158279538), (16384, 0.0001278102572541684), (32768, 0.00013103100354783237), (65536, 0.00013205455616116524), (131072, 0.0001347229117527604), (262144, 0.00013530193245969713), (524288, 0.0001275970134884119), (1048576, 0.00013219696120359004), (2097152, 0.00013906107051298022), (4194304, 0.00014691532123833895), (8388608, 0.00018083374015986922), (16777216, 0.0002502184361219406), (33554432, 0.00041954405605793005), (67108864, 0.0007227789610624313), (134217728, 0.0013900622725486755), (268435456, 0.0027236193418502808), (536870912, 0.005390062928199768), (1073741824, 0.010686516761779785), (2147483648, 0.021258997917175292)]
(((0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11), (12, 13, 14, 15)), 'f32'): [(0, 1.799248275347054e-05), (1, 1.7991173081099987e-05), (2, 1.8002261640504003e-05), (4, 1.802839688025415e-05), (8, 1.798482844606042e-05), (16, 3.596965689212084e-05), (32, 7.193931378424168e-05), (64, 0.00012742792023345828), (128, 0.0001268957566935569), (256, 0.00012892117956653237), (512, 0.00013128656428307295), (1024, 0.0001276765251532197), (2048, 0.00012759806122630835), (4096, 0.00012666668044403195), (8192, 0.00012794320355169475), (16384, 0.00012839349801652133), (32768, 0.00013011862756684422), (65536, 0.0001265041355509311), (131072, 0.00012892039376311004), (262144, 0.00013239748659543693), (524288, 0.00013796283747069538), (1048576, 0.00014504609862342477), (2097152, 0.0001604873687028885), (4194304, 0.00019431370310485366), (8388608, 0.00026617618277668953), (16777216, 0.00042860303074121475), (33554432, 0.0007354132831096649), (67108864, 0.0014154985547065737), (134217728, 0.0028035417199134827), (268435456, 0.005461350083351135), (536870912, 0.01088104248046875), (1073741824, 0.02152571678161621)]
(((0, 1), (2, 3), (4, 5), (6, 7), (8, 9), (10, 11), (12, 13), (14, 15)), 'f16'): [(0, 1.7423328245058656e-05), (1, 1.7346610547974706e-05), (2, 1.734477700665593e-05), (4, 3.468955401331186e-05), (8, 6.937910802662373e-05), (16, 8.368896669708192e-05), (32, 8.381123188883066e-05), (64, 8.504209108650684e-05), (128, 8.593531674705446e-05), (256, 8.499188697896898e-05), (512, 8.12598445918411e-05), (1024, 8.153310045599937e-05), (2048, 8.21009452920407e-05), (4096, 8.149771019816399e-05), (8192, 8.139931014738977e-05), (16384, 8.272097329609096e-05), (32768, 8.39347776491195e-05), (65536, 8.582562441006303e-05), (131072, 8.475797949358821e-05), (262144, 8.68489732965827e-05), (524288, 8.642219472676516e-05), (1048576, 0.0001064683310687542), (2097152, 0.00013738189591094851), (4194304, 0.00019515864551067352), (8388608, 0.00036314106546342373), (16777216, 0.0006439881399273872), (33554432, 0.001238849014043808), (67108864, 0.00240921787917614), (134217728, 0.004772741347551346), (268435456, 0.00947713851928711), (536870912, 0.018902570009231567), (1073741824, 0.03770427703857422), (2147483648, 0.07532477378845215)]
(((0, 1), (2, 3), (4, 5), (6, 7), (8, 9), (10, 11), (12, 13), (14, 15)), 'f32'): [(0, 1.7844344256445765e-05), (1, 1.8100632587447762e-05), (2, 1.7526414012536407e-05), (4, 3.505282802507281e-05), (8, 7.010565605014563e-05), (16, 8.207804057747126e-05), (32, 8.362787775695324e-05), (64, 8.165341569110751e-05), (128, 8.136805263347924e-05), (256, 8.24015005491674e-05), (512, 8.205531048588455e-05), (1024, 8.304280345328152e-05), (2048, 8.224172051995993e-05), (4096, 8.224675548262894e-05), (8192, 8.385465480387211e-05), (16384, 8.427098509855568e-05), (32768, 8.785142563283443e-05), (65536, 8.770223939791322e-05), (131072, 8.929838077165186e-05), (262144, 9.600751218385994e-05), (524288, 0.00010769118671305478), (1048576, 0.00013327982742339373), (2097152, 0.00019395304843783379), (4194304, 0.00036442256532609463), (8388608, 0.0006417892873287201), (16777216, 0.0012349486351013184), (33554432, 0.002415543422102928), (67108864, 0.004787437617778778), (134217728, 0.009509608149528503), (268435456, 0.018964752554893494), (536870912, 0.03784611225128174), (1073741824, 0.07558202743530273)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,), (8,), (9,), (10,), (11,), (12,), (13,), (14,), (15,)), 'f16'): [(0, 1.7394544556736946e-05), (1, 2.1981395548209548e-05), (2, 2.2055464796721935e-05), (4, 2.076261444017291e-05), (8, 2.0648876670747995e-05), (16, 2.0599225535988808e-05), (32, 2.0495790522545576e-05), (64, 2.0646140910685062e-05), (128, 2.0517705706879497e-05), (256, 2.052722265943885e-05), (512, 2.0698353182524443e-05), (1024, 2.0656269043684006e-05), (2048, 2.070676418952644e-05), (4096, 2.0714622223749757e-05), (8192, 2.0971521735191345e-05), (16384, 2.076110104098916e-05), (32768, 2.1301355445757508e-05), (65536, 2.0608043996617198e-05), (131072, 2.102210419252515e-05), (262144, 2.070685150101781e-05), (524288, 2.1496991394087672e-05), (1048576, 2.1928426576778293e-05), (2097152, 3.392918733879924e-05), (4194304, 5.095009692013264e-05), (8388608, 7.215584628283978e-05), (16777216, 0.00010834541171789169), (33554432, 0.00019581709057092667), (67108864, 0.00036425143480300903), (134217728, 0.000700436532497406), (268435456, 0.0013750195503234863), (536870912, 0.002725541591644287), (1073741824, 0.005410432815551758), (2147483648, 0.01073594093322754)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,), (8,), (9,), (10,), (11,), (12,), (13,), (14,), (15,)), 'f32'): [(0, 1.7620710423216224e-05), (1, 2.1080952137708664e-05), (2, 2.2798747522756454e-05), (4, 2.089110785163939e-05), (8, 2.085877349600196e-05), (16, 2.290983684360981e-05), (32, 2.0946405129507184e-05), (64, 2.0609499188140035e-05), (128, 2.0668288925662637e-05), (256, 2.0719104213640094e-05), (512, 2.103991573676467e-05), (1024, 2.2903026547282934e-05), (2048, 2.0880310330539942e-05), (4096, 2.1086802007630467e-05), (8192, 2.0729668904095888e-05), (16384, 2.1139014279469848e-05), (32768, 2.106273313984275e-05), (65536, 2.111939829774201e-05), (131072, 2.3104483261704445e-05), (262144, 2.1434796508401632e-05), (524288, 2.0714447600767016e-05), (1048576, 3.2823707442730665e-05), (2097152, 4.438147880136967e-05), (4194304, 6.653694435954094e-05), (8388608, 0.00011240923777222633), (16777216, 0.0002034306526184082), (33554432, 0.00036849454045295715), (67108864, 0.0007048696279525757), (134217728, 0.001386396586894989), (268435456, 0.0027282238006591797), (536870912, 0.005406641960144043), (1073741824, 0.010769987106323242)]
reduce_scatter_cost_dict:
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15),), 'f16'): [(0, 1.7075391951948404e-05), (1, 1.7055601347237825e-05), (2, 1.70299899764359e-05), (4, 1.7092301277443767e-05), (8, 1.7180194845423102e-05), (16, 3.4360389690846205e-05), (32, 6.872077938169241e-05), (64, 0.00013744155876338482), (128, 0.00027488311752676964), (256, 0.0005497662350535393), (512, 0.00056453249999322), (1024, 0.0005835940246470273), (2048, 0.0005653866101056337), (4096, 0.0005403385439421982), (8192, 0.0005516578676179051), (16384, 0.0006324340938590467), (32768, 0.0007528328569605947), (65536, 0.0007961377559695393), (131072, 0.0007853221031837164), (262144, 0.0008243096817750485), (524288, 0.0010033916914835572), (1048576, 0.0017302549676969647), (2097152, 0.0025505865924060345), (4194304, 0.004003727808594704), (8388608, 0.006633089855313301), (16777216, 0.012146616354584694), (33554432, 0.024293232709169388), (67108864, 0.044309139251708984), (134217728, 0.08861827850341797), (268435456, 0.17723655700683594), (536870912, 0.34369826316833496), (1073741824, 0.6873965263366699), (2147483648, 1.3747930526733398)]
(((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15),), 'f32'): [(0, 1.754437107592821e-05), (1, 1.7514597857370973e-05), (2, 1.7591053619980812e-05), (4, 1.755080302245915e-05), (8, 1.7629703506827354e-05), (16, 3.525940701365471e-05), (32, 7.051881402730942e-05), (64, 0.00014103762805461884), (128, 0.00028207525610923767), (256, 0.0005641505122184753), (512, 0.0005726454837713391), (1024, 0.0005996421677991748), (2048, 0.0005729364638682455), (4096, 0.0005505851877387613), (8192, 0.0006077388534322381), (16384, 0.0007058225746732205), (32768, 0.0007374890556093304), (65536, 0.0007584432023577392), (131072, 0.0009371616470161824), (262144, 0.0011990231578238308), (524288, 0.002029833383858204), (1048576, 0.002509860787540674), (2097152, 0.003787038847804069), (4194304, 0.006394205614924431), (8388608, 0.01187310554087162), (16777216, 0.02374621108174324), (33554432, 0.045100077986717224), (67108864, 0.0884372740983963), (134217728, 0.17450661659240724), (268435456, 0.34663958549499513), (536870912, 0.6932791709899903), (1073741824, 1.3758246898651123)]
(((0, 2, 4, 6, 8, 10, 12, 14), (1, 3, 5, 7, 9, 11, 13, 15)), 'f16'): [(0, 1.7415295587852597e-05), (1, 1.7496960936114192e-05), (2, 1.746302586980164e-05), (4, 1.7521437257528305e-05), (8, 3.504287451505661e-05), (16, 7.008574903011322e-05), (32, 0.00014017149806022644), (64, 0.0002803429961204529), (128, 0.00035998306702822447), (256, 0.0003543490602169186), (512, 0.0003592422290239483), (1024, 0.0003523663035593927), (2048, 0.0003549547982402146), (4096, 0.00034204605617560446), (8192, 0.0003944921481888741), (16384, 0.0004915759491268545), (32768, 0.0005180453299544752), (65536, 0.0005538091936614364), (131072, 0.0006368725444190204), (262144, 0.000794734776718542), (524288, 0.00113490701187402), (1048576, 0.0019417292205616832), (2097152, 0.0030981760937720537), (4194304, 0.005595980677753687), (8388608, 0.011191961355507374), (16777216, 0.022383922711014748), (33554432, 0.044017352163791656), (67108864, 0.08615188300609589), (134217728, 0.17230376601219177), (268435456, 0.34460753202438354), (536870912, 0.6892150640487671), (1073741824, 1.3729657173156737), (2147483648, 2.7390671968460083)]
(((0, 2, 4, 6, 8, 10, 12, 14), (1, 3, 5, 7, 9, 11, 13, 15)), 'f32'): [(0, 1.691424404270947e-05), (1, 1.6864214558154345e-05), (2, 1.6777950804680586e-05), (4, 1.938061905093491e-05), (8, 3.876123810186982e-05), (16, 7.752247620373964e-05), (32, 0.00015504495240747929), (64, 0.00031008990481495857), (128, 0.0003703430702444166), (256, 0.00037639503716491163), (512, 0.00038283978938125074), (1024, 0.00036200002068653703), (2048, 0.0003778346290346235), (4096, 0.00041715256520546973), (8192, 0.0005073112552054226), (16384, 0.0005075287481304258), (32768, 0.0005672303377650678), (65536, 0.0006558397726621479), (131072, 0.0008132126240525396), (262144, 0.0011025121784768999), (524288, 0.001855041482485831), (1048576, 0.003106350777670741), (2097152, 0.00560550345107913), (4194304, 0.01121100690215826), (8388608, 0.02242201380431652), (16777216, 0.043294332921504974), (33554432, 0.08658866584300995), (67108864, 0.17277781665325165), (134217728, 0.34104151725769044), (268435456, 0.6820830345153809), (536870912, 1.3641660690307618), (1073741824, 2.7283321380615235)]
(((0, 4, 8, 12), (1, 5, 9, 13), (2, 6, 10, 14), (3, 7, 11, 15)), 'f16'): [(0, 1.761806197464466e-05), (1, 1.7724523786455393e-05), (2, 1.781043829396367e-05), (4, 3.562087658792734e-05), (8, 7.124175317585468e-05), (16, 0.00014248350635170937), (32, 0.0002573473902884871), (64, 0.0002589597715996206), (128, 0.00026502867694944143), (256, 0.0002618387225084007), (512, 0.0002579028077889234), (1024, 0.00025629621814005077), (2048, 0.00025332404766231775), (4096, 0.0002727633691392839), (8192, 0.00032649608328938484), (16384, 0.00042658497113734484), (32768, 0.00044248157064430416), (65536, 0.0005176409322302788), (131072, 0.0006386159802787006), (262144, 0.0009244787215720862), (524288, 0.0014817705377936365), (1048576, 0.00257017754483968), (2097152, 0.005014546914026141), (4194304, 0.009379981085658073), (8388608, 0.018759962171316147), (16777216, 0.037519924342632294), (33554432, 0.07503984868526459), (67108864, 0.15007969737052917), (134217728, 0.30015939474105835), (268435456, 0.5992124557495118), (536870912, 1.1984249114990235), (1073741824, 2.3915106296539306), (2147483648, 4.783021259307861)]
(((0, 4, 8, 12), (1, 5, 9, 13), (2, 6, 10, 14), (3, 7, 11, 15)), 'f32'): [(0, 1.7618003766983747e-05), (1, 1.762786996550858e-05), (2, 1.7614976968616247e-05), (4, 3.5229953937232494e-05), (8, 7.045990787446499e-05), (16, 0.00014091981574892998), (32, 0.0002523945295251906), (64, 0.00026112195337191224), (128, 0.0002595173427835107), (256, 0.0002579786232672632), (512, 0.00026331355911679566), (1024, 0.00025125755928456783), (2048, 0.00026495064957998693), (4096, 0.0003111276892013848), (8192, 0.0004083517414983362), (16384, 0.0004433020076248794), (32768, 0.0005077806708868593), (65536, 0.0006262723763938993), (131072, 0.0008855793566908687), (262144, 0.0014563678414560854), (524288, 0.0026018156204372644), (1048576, 0.0050340641755610704), (2097152, 0.009367973078042269), (4194304, 0.018735946156084538), (8388608, 0.037471892312169075), (16777216, 0.07494378462433815), (33554432, 0.1498875692486763), (67108864, 0.2997751384973526), (134217728, 0.5995502769947052), (268435456, 1.1991005539894104), (536870912, 2.398201107978821), (1073741824, 4.796402215957642)]
(((0, 8), (1, 9), (2, 10), (3, 11), (4, 12), (5, 13), (6, 14), (7, 15)), 'f16'): [(0, 1.691031502559781e-05), (1, 1.7064041458070278e-05), (2, 3.4128082916140556e-05), (4, 6.825616583228111e-05), (8, 0.00013651233166456223), (16, 0.00015688137500546873), (32, 0.00015601463383063674), (64, 0.0001537568459752947), (128, 0.00015876590623520315), (256, 0.00015722171519882977), (512, 0.00014424658729694784), (1024, 0.00014325976371765137), (2048, 0.00015916425036266446), (4096, 0.00016026347293518484), (8192, 0.00019497828907333314), (16384, 0.0002782683295663446), (32768, 0.0003130510449409485), (65536, 0.0003957727458328008), (131072, 0.0005652921099681407), (262144, 0.0009562116756569594), (524288, 0.0017568267649039628), (1048576, 0.0035136535298079257), (2097152, 0.007027307059615851), (4194304, 0.014054614119231703), (8388608, 0.028109228238463405), (16777216, 0.05621845647692681), (33554432, 0.11243691295385362), (67108864, 0.22487382590770724), (134217728, 0.4497476518154145), (268435456, 0.899495303630829), (536870912, 1.798990607261658), (1073741824, 3.597981214523316), (2147483648, 7.195962429046632)]
(((0, 8), (1, 9), (2, 10), (3, 11), (4, 12), (5, 13), (6, 14), (7, 15)), 'f32'): [(0, 2.0600593416020274e-05), (1, 1.7234240658581257e-05), (2, 3.4468481317162514e-05), (4, 6.893696263432503e-05), (8, 0.00013787392526865005), (16, 0.00014095046208240092), (32, 0.000145213445648551), (64, 0.00015013196389190853), (128, 0.00016330965445376933), (256, 0.00014237314462661743), (512, 0.00014968833420425653), (1024, 0.00017264869529753923), (2048, 0.0001804196508601308), (4096, 0.0002315877354703844), (8192, 0.0002813484170474112), (16384, 0.00031855361885391176), (32768, 0.000391752168070525), (65536, 0.0005578759883064777), (131072, 0.0009684634569566697), (262144, 0.0017728887614794075), (524288, 0.003545777522958815), (1048576, 0.00709155504591763), (2097152, 0.01418311009183526), (4194304, 0.02836622018367052), (8388608, 0.05673244036734104), (16777216, 0.11346488073468208), (33554432, 0.22692976146936417), (67108864, 0.45385952293872833), (134217728, 0.9077190458774567), (268435456, 1.8154380917549133), (536870912, 3.6308761835098267), (1073741824, 7.261752367019653)]
(((0, 1, 2, 3, 4, 5, 6, 7), (8, 9, 10, 11, 12, 13, 14, 15)), 'f16'): [(0, 1.7350103007629514e-05), (1, 1.7337501049041748e-05), (2, 1.7358630429953337e-05), (4, 1.7740967450663447e-05), (8, 3.5481934901326895e-05), (16, 7.096386980265379e-05), (32, 0.00014192773960530758), (64, 0.00020851937006227675), (128, 0.0002070888294838369), (256, 0.00020445068366825578), (512, 0.0002054045908153057), (1024, 0.00020494722411967814), (2048, 0.00020770003902725875), (4096, 0.0002076017262879759), (8192, 0.00020694779232144356), (16384, 0.00020749965915456414), (32768, 0.00021627053502015767), (65536, 0.00021456420654430985), (131072, 0.00022415415151044724), (262144, 0.00023420469369739294), (524288, 0.00023173162480816245), (1048576, 0.0002675049181561917), (2097152, 0.0002941639395430684), (4194304, 0.00032416137401014566), (8388608, 0.0003778391983360052), (16777216, 0.0004911879077553749), (33554432, 0.0007416950538754463), (67108864, 0.0012096837162971497), (134217728, 0.002245388925075531), (268435456, 0.0041288286447525024), (536870912, 0.008072033524513245), (1073741824, 0.015749120712280275), (2147483648, 0.03117074966430664)]
(((0, 1, 2, 3, 4, 5, 6, 7), (8, 9, 10, 11, 12, 13, 14, 15)), 'f32'): [(0, 1.817505108192563e-05), (1, 1.7811486031860113e-05), (2, 1.7962127458304167e-05), (4, 1.942188828252256e-05), (8, 3.884377656504512e-05), (16, 7.768755313009024e-05), (32, 0.00015537510626018047), (64, 0.0002097742399200797), (128, 0.00020615415996871889), (256, 0.00021085143089294434), (512, 0.00021097433636896312), (1024, 0.00020856724586337802), (2048, 0.0002069943875540048), (4096, 0.00020396488253027203), (8192, 0.00021144282072782516), (16384, 0.0002181872259825468), (32768, 0.0002177223504986614), (65536, 0.00021775954519398513), (131072, 0.00022489772527478635), (262144, 0.0002348964917473495), (524288, 0.0002646379289217293), (1048576, 0.00029133225325495005), (2097152, 0.0003220436628907919), (4194304, 0.0003786867018789053), (8388608, 0.00048045441508293157), (16777216, 0.0006996355950832367), (33554432, 0.0011661946773529053), (67108864, 0.0021284259855747223), (134217728, 0.004010148346424103), (268435456, 0.007803678512573242), (536870912, 0.015362858772277832), (1073741824, 0.03041884899139404)]
(((0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11), (12, 13, 14, 15)), 'f16'): [(0, 1.7472077161073685e-05), (1, 1.741803134791553e-05), (2, 1.7459591617807746e-05), (4, 3.491918323561549e-05), (8, 6.983836647123098e-05), (16, 0.0001020750787574798), (32, 0.0001029049453791231), (64, 0.00010286556789651513), (128, 0.00010272121289744973), (256, 0.00010452303104102612), (512, 0.00010254394146613777), (1024, 9.65955841820687e-05), (2048, 9.584779036231339e-05), (4096, 0.00010479043703526258), (8192, 0.00010813336120918392), (16384, 0.00011233679833821954), (32768, 0.0001137519720941782), (65536, 0.0001220629783347249), (131072, 0.00012146829976700246), (262144, 0.00012651504948735237), (524288, 0.0001444698718842119), (1048576, 0.00015507350326515734), (2097152, 0.00017690076492726803), (4194304, 0.00021818559616804126), (8388608, 0.0003180340863764286), (16777216, 0.0004946491681039333), (33554432, 0.0008386494591832161), (67108864, 0.0014877021312713623), (134217728, 0.002914275974035263), (268435456, 0.005676396191120148), (536870912, 0.011352792382240295), (1073741824, 0.02270558476448059), (2147483648, 0.04541116952896118)]
(((0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11), (12, 13, 14, 15)), 'f32'): [(0, 1.7746788216754794e-05), (1, 1.7717713490128517e-05), (2, 1.797013101167977e-05), (4, 3.594026202335954e-05), (8, 7.188052404671907e-05), (16, 0.00010957609629258513), (32, 0.00010871049016714096), (64, 0.00010935895261354746), (128, 0.00010656091035343705), (256, 0.00010825935169123113), (512, 0.0001088259741663933), (1024, 0.00010921325883828104), (2048, 0.00010979987564496697), (4096, 0.00010877777822315693), (8192, 0.00011078364332206547), (16384, 0.00011414790060371159), (32768, 0.00011766649549826978), (65536, 0.00011978394468314944), (131072, 0.00011874956544488668), (262144, 0.0001358960580546409), (524288, 0.00015369217726401985), (1048576, 0.00017363991355523467), (2097152, 0.00021632155403494835), (4194304, 0.00029962649568915367), (8388608, 0.0004665269516408443), (16777216, 0.0007962090894579887), (33554432, 0.0013906210660934448), (67108864, 0.0027482137084007263), (134217728, 0.0053545162081718445), (268435456, 0.010709032416343689), (536870912, 0.021418064832687378), (1073741824, 0.042836129665374756)]
(((0, 1), (2, 3), (4, 5), (6, 7), (8, 9), (10, 11), (12, 13), (14, 15)), 'f16'): [(0, 1.7728889361023903e-05), (1, 1.7893471522256732e-05), (2, 3.5786943044513464e-05), (4, 6.606214446946979e-05), (8, 6.470849621109664e-05), (16, 6.550224497914314e-05), (32, 6.498410948552191e-05), (64, 6.448186468333006e-05), (128, 6.462432793341577e-05), (256, 6.60959049127996e-05), (512, 6.676180055364966e-05), (1024, 6.560975452885032e-05), (2048, 6.482904427684844e-05), (4096, 6.438768468797207e-05), (8192, 6.942494655959308e-05), (16384, 7.320783333852887e-05), (32768, 7.660975097678602e-05), (65536, 7.615803042426705e-05), (131072, 8.224041084758937e-05), (262144, 8.970717317424715e-05), (524288, 9.89328254945576e-05), (1048576, 0.0001402371854055673), (2097152, 0.00022051553241908548), (4194304, 0.0003643847303465009), (8388608, 0.0006530683021992445), (16777216, 0.001215009018778801), (33554432, 0.0023464104160666466), (67108864, 0.0045421067625284195), (134217728, 0.008686147630214691), (268435456, 0.01643192768096924), (536870912, 0.03158856928348541), (1073741824, 0.061679887771606455), (2147483648, 0.12335977554321291)]
(((0, 1), (2, 3), (4, 5), (6, 7), (8, 9), (10, 11), (12, 13), (14, 15)), 'f32'): [(0, 1.7615151591598988e-05), (1, 1.7618585843592882e-05), (2, 3.5237171687185764e-05), (4, 6.787540041841567e-05), (8, 7.199906394816935e-05), (16, 6.791012128815055e-05), (32, 6.704652332700789e-05), (64, 6.735685747116804e-05), (128, 6.813023355789483e-05), (256, 6.727754953317344e-05), (512, 6.760965334251523e-05), (1024, 6.839414709247649e-05), (2048, 6.812287028878927e-05), (4096, 7.093892781995237e-05), (8192, 7.628349703736603e-05), (16384, 7.725899922661483e-05), (32768, 7.894611917436123e-05), (65536, 8.743361104279757e-05), (131072, 9.309948654845357e-05), (262144, 0.00011044647544622423), (524288, 0.00014198926510289311), (1048576, 0.00021336774807423353), (2097152, 0.00035004562232643366), (4194304, 0.0006233737803995609), (8388608, 0.0011618221178650856), (16777216, 0.002219780348241329), (33554432, 0.0042945463210344315), (67108864, 0.008328191936016083), (134217728, 0.016450971364974976), (268435456, 0.03270751237869263), (536870912, 0.06508071422576904), (1073741824, 0.12952983379364014)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,), (8,), (9,), (10,), (11,), (12,), (13,), (14,), (15,)), 'f16'): [(0, 1.7491576727479696e-05), (1, 2.0441511878743768e-05), (2, 2.0582752767950296e-05), (4, 2.0573788788169622e-05), (8, 2.0667066564783454e-05), (16, 2.0663777831941843e-05), (32, 2.0564242731779814e-05), (64, 2.063394640572369e-05), (128, 2.0520557882264256e-05), (256, 2.0578212570399046e-05), (512, 2.064224099740386e-05), (1024, 2.0368839614093304e-05), (2048, 2.0584906451404095e-05), (4096, 2.0742823835462332e-05), (8192, 2.0568491891026497e-05), (16384, 2.037733793258667e-05), (32768, 2.022783155553043e-05), (65536, 2.03122035600245e-05), (131072, 2.0310224499553442e-05), (262144, 2.0455452613532543e-05), (524288, 2.0017585484310985e-05), (1048576, 2.012492041103542e-05), (2097152, 3.2159558031708e-05), (4194304, 4.400080069899559e-05), (8388608, 6.522634066641331e-05), (16777216, 0.00010808883234858513), (33554432, 0.00019280984997749329), (67108864, 0.00036198273301124573), (134217728, 0.00070219486951828), (268435456, 0.00140438973903656), (536870912, 0.0027228742837905884), (1073741824, 0.005413508415222168), (2147483648, 0.010750555992126464)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,), (8,), (9,), (10,), (11,), (12,), (13,), (14,), (15,)), 'f32'): [(0, 1.731011434458196e-05), (1, 2.014788333326578e-05), (2, 2.0440667867660522e-05), (4, 2.0259234588593245e-05), (8, 2.03854579012841e-05), (16, 2.1378305973485112e-05), (32, 1.981464447453618e-05), (64, 1.9990402506664395e-05), (128, 1.9817467546090484e-05), (256, 1.9893312128260732e-05), (512, 1.9856495782732964e-05), (1024, 2.0260544260963798e-05), (2048, 1.9771949155256152e-05), (4096, 1.9719445845112205e-05), (8192, 2.002634573727846e-05), (16384, 2.0121224224567413e-05), (32768, 2.0089675672352314e-05), (65536, 2.0312232663854957e-05), (131072, 2.0994688384234905e-05), (262144, 2.0721839973703027e-05), (524288, 2.078642137348652e-05), (1048576, 3.265996929258108e-05), (2097152, 4.456867463886738e-05), (4194304, 6.58784992992878e-05), (8388608, 0.00010842643678188325), (16777216, 0.00019234884530305862), (33554432, 0.0003672763705253601), (67108864, 0.0007038526237010956), (134217728, 0.0013815388083457947), (268435456, 0.00273054838180542), (536870912, 0.005430674552917481), (1073741824, 0.010732126235961915)]
available_memory_per_device:
14957700710
dot_cost_dict:
((), 'f16'): [(0, 1.662003342062235e-05), (4194304, 2.9675080440938473e-05), (33554432, 2.9678398277610537e-05), (113246208, 3.9767706766724586e-05), (268435456, 2.968066837638617e-05), (524288000, 3.7421355955302715e-05), (905969664, 3.7816702388226986e-05), (1438646272, 5.177216371521354e-05), (2147483648, 5.5833079386502504e-05), (3057647616, 7.949670873586001e-05), (4194304000, 8.927180897444487e-05), (5582618624, 9.796867379918694e-05), (7247757312, 0.00010636699153110385), (9214885888, 0.00013523627337606203), (11509170176, 0.00016890684303318807), (14155776000, 0.0002077480303341872), (17179869184, 0.0002307849936187267), (20606615552, 0.00024195347214117646), (24461180928, 0.0002872120190367069), (28768731136, 0.00033778930702550974), (33554432000, 0.0003939808217238778), (38843449344, 0.0004220617401016342), (44660948992, 0.000485273016801879), (51032096768, 0.000554500074702147), (57982058496, 0.0006300163584024394), (65536000000, 0.0006726105486760374), (73719087104, 0.0007565953922259221), (82556485632, 0.0008472955794937884), (92073361408, 0.0009154096712284546), (102294880256, 0.0010170338225032243), (113246208000, 0.0011259138631180882), (124952510464, 0.0012299516797065735), (137438953472, 0.0013328195438096415), (150730702848, 0.0013959321501028485), (164852924416, 0.0015267194598225326), (179830784000, 0.0016654309189876625), (195689447424, 0.0018122995908172218), (212454080512, 0.0019675585385301943), (230149849088, 0.0021314408253455632), (248801918976, 0.0023041795144823126), (268435456000, 0.002394599299277029), (289075625984, 0.0025787215360230016), (310747594752, 0.0027720480138255705), (333476528128, 0.0029748032263690427), (357287591936, 0.0031872116673377256), (382205952000, 0.0034094978304159257), (408256774144, 0.003641886209287951), (435465224192, 0.0038497791540154204), (463856467968, 0.004100775128833432), (493455671296, 0.004362450205549447), (524288000000, 0.004635026865453007), (556378619904, 0.004918727589833655), (589752696832, 0.005189138051320815), (624435396608, 0.005494305485224802), (660451885056, 0.005811208708702071), (697827328000, 0.006124526470691174), (736586891264, 0.006464702273039226), (776755740672, 0.006817246765960695), (818359042048, 0.0071823808189301055), (861421961216, 0.007560325301421984), (905969664000, 0.007951301082910856), (952027316224, 0.008355529032871244), (999620083712, 0.008773230020777678), (1048773132288, 0.009177684783935547)]
((), 'f32'): [(0, 1.65026867762208e-05), (4194304, 2.9112619813531634e-05), (33554432, 2.9380840715020895e-05), (113246208, 4.260562127456069e-05), (268435456, 4.5800465159118176e-05), (524288000, 7.021648343652487e-05), (905969664, 9.645731188356875e-05), (1438646272, 0.00013177230721339583), (2147483648, 0.00019034423166885972), (3057647616, 0.0002501418930478394), (4194304000, 0.0003141497145406902), (5582618624, 0.00041813327005365864), (7247757312, 0.0005278292810544372), (9214885888, 0.0006554973078891635), (11509170176, 0.0008187003244642079), (14155776000, 0.0009950751761482904), (17179869184, 0.0011966387430826823), (20606615552, 0.0014353237658118207), (24461180928, 0.0016767866690798048), (28768731136, 0.001972064431278872), (33554432000, 0.0022670427424795678), (38843449344, 0.0026243853547629095), (44660948992, 0.0030174338902403046), (51032096768, 0.0034478886309686125), (57982058496, 0.003917449859004693), (65536000000, 0.004365294311303166), (73719087104, 0.004910362420189725), (82556485632, 0.005457679102243471), (92073361408, 0.006086824754989009), (102294880256, 0.006762553250247219), (113246208000, 0.007437059938772465), (124952510464, 0.008205831579110019), (137438953472, 0.009025836299025782), (150730702848, 0.00989872677850615), (164852924416, 0.010826155697537516), (179830784000, 0.011809775736106273), (195689447424, 0.012851239574198817), (212454080512, 0.01395219989180154), (230149849088, 0.015109097353324363), (248801918976, 0.01621807098388672), (268435456000, 0.01731490960685156), (289075625984, 0.018646263828340878), (310747594752, 0.020044172233631533), (333476528128, 0.02151025809549917), (357287591936, 0.023046144686719422), (382205952000, 0.02465345528006794), (408256774144, 0.026333813148320364), (435465224192, 0.028088841564252332), (463856467968, 0.029920163800639492), (493455671296, 0.031829403130257485), (524288000000, 0.03381818282588195), (556378619904, 0.03588812616028853), (589752696832, 0.03804085640625287), (624435396608, 0.04027799683655062), (660451885056, 0.0426011707239574), (697827328000, 0.04501200134124887), (736586891264, 0.04751211196120067), (776755740672, 0.05010312585658844), (818359042048, 0.05278666630018782), (861421961216, 0.05556435656477446), (905969664000, 0.058437819923123996), (952027316224, 0.061408679648012075), (999620083712, 0.06447855901221433), (1048773132288, 0.06764908128850641)]
conv_cost_dict:
[]
op_cost_dict:
[]
--------------------------------------------------
Key: ('default', (1, 8))
all_gather_cost_dict:
(((0, 1, 2, 3, 4, 5, 6, 7),), 'f16'): [(0, 1.7325131921097636e-05), (1, 1.7006153939291835e-05), (2, 1.7107377061620355e-05), (4, 1.8333055777475238e-05), (8, 3.6666111554950476e-05), (16, 7.333222310990095e-05), (32, 0.0001466644462198019), (64, 0.0002023277920670807), (128, 0.0002009982708841562), (256, 0.000202561728656292), (512, 0.00020683312322944403), (1024, 0.00020383443916216493), (2048, 0.00019965233514085412), (4096, 0.00020524143474176523), (8192, 0.00021099986042827368), (16384, 0.00021124311024323106), (32768, 0.00021722738165408373), (65536, 0.00021424356964416802), (131072, 0.00021907591144554314), (262144, 0.00022044286015443504), (524288, 0.00023251477978192273), (1048576, 0.0002500249247532338), (2097152, 0.000261944776866585), (4194304, 0.00030463445000350475), (8388608, 0.0003798315301537514), (16777216, 0.00047455588355660444), (33554432, 0.000685068778693676), (67108864, 0.0011362824589014053), (134217728, 0.0020334087312221527), (268435456, 0.0038242116570472717), (536870912, 0.007411390542984009), (1073741824, 0.014822781085968018), (2147483648, 0.028885555267333985)]
(((0, 1, 2, 3, 4, 5, 6, 7),), 'f32'): [(0, 1.7301266780123115e-05), (1, 1.7223122995346785e-05), (2, 1.722155138850212e-05), (4, 1.7285696230828762e-05), (8, 3.4571392461657524e-05), (16, 6.914278492331505e-05), (32, 0.0001382855698466301), (64, 0.00020883532124571502), (128, 0.0002094202791340649), (256, 0.00020237194257788363), (512, 0.0002034951758105308), (1024, 0.00020046342979185283), (2048, 0.0002038183156400919), (4096, 0.00020257307915017011), (8192, 0.0002049050817731768), (16384, 0.00021457826369442048), (32768, 0.00021160717005841437), (65536, 0.000216582790017128), (131072, 0.0002293921425007284), (262144, 0.00023417314514517787), (524288, 0.0002563595480751246), (1048576, 0.0002605167683213949), (2097152, 0.00026681553572416306), (4194304, 0.000322605948895216), (8388608, 0.0004264218732714653), (16777216, 0.000659717246890068), (33554432, 0.0011212937533855438), (67108864, 0.00201304629445076), (134217728, 0.0037718191742897034), (268435456, 0.007400736212730408), (536870912, 0.014562201499938966), (1073741824, 0.02896029949188233)]
(((0, 2, 4, 6), (1, 3, 5, 7)), 'f16'): [(0, 1.754047116264701e-05), (1, 1.7561862478032708e-05), (2, 1.7645623302087188e-05), (4, 3.5291246604174376e-05), (8, 7.058249320834875e-05), (16, 0.00010278401896357538), (32, 0.00010321941226720811), (64, 0.00010379942250438035), (128, 0.00010305686737410724), (256, 0.00010327165364287794), (512, 0.00010309010394848883), (1024, 0.00010441357153467834), (2048, 0.0001069027348421514), (4096, 0.00010841537732630967), (8192, 0.00011166275362484158), (16384, 0.00011424333206377923), (32768, 0.000117060961201787), (65536, 0.00011998036643490195), (131072, 0.00012582910130731761), (262144, 0.00013630575267598033), (524288, 0.00014486355939880013), (1048576, 0.00016327781486324966), (2097152, 0.00019722827710211277), (4194304, 0.00027310545556247234), (8388608, 0.0004123742692172527), (16777216, 0.0006883959285914898), (33554432, 0.0012654270976781845), (67108864, 0.0024219900369644165), (134217728, 0.004700757563114166), (268435456, 0.00928834080696106), (536870912, 0.01846197247505188), (1073741824, 0.03692394495010376), (2147483648, 0.07353839874267579)]
(((0, 2, 4, 6), (1, 3, 5, 7)), 'f32'): [(0, 1.7542217392474413e-05), (1, 1.764274202287197e-05), (2, 1.7600279534235597e-05), (4, 3.520055906847119e-05), (8, 7.040111813694239e-05), (16, 0.00010622991248965265), (32, 0.00010596797801554204), (64, 0.00010660645784810187), (128, 0.00010612278128974138), (256, 0.00010517047485336663), (512, 0.00010619586100801827), (1024, 0.00010746778571046889), (2048, 0.00010756775736808778), (4096, 0.0001119965745601803), (8192, 0.00011499293032102287), (16384, 0.00011552695650607346), (32768, 0.00011960032861679792), (65536, 0.00012186283129267396), (131072, 0.0001349504163954407), (262144, 0.00014509039465337992), (524288, 0.00016267437604255974), (1048576, 0.00019217858789488673), (2097152, 0.0002657516160979867), (4194304, 0.0004076776094734669), (8388608, 0.0006849993951618671), (16777216, 0.0012636184692382812), (33554432, 0.0024044234305620193), (67108864, 0.004704535007476807), (134217728, 0.0092826709151268), (268435456, 0.018457144498825073), (536870912, 0.036772942543029784), (1073741824, 0.07344613075256348)]
(((0, 4), (1, 5), (2, 6), (3, 7)), 'f16'): [(0, 1.8073274986818433e-05), (1, 1.819172757677734e-05), (2, 3.638345515355468e-05), (4, 7.049614214338362e-05), (8, 6.971124093979597e-05), (16, 7.0524140028283e-05), (32, 7.017888128757477e-05), (64, 7.011598790995777e-05), (128, 7.111503509804606e-05), (256, 6.969037349335849e-05), (512, 7.055472815409303e-05), (1024, 6.907864008098841e-05), (2048, 6.943100015632808e-05), (4096, 6.909415242262185e-05), (8192, 7.310183718800545e-05), (16384, 7.549789734184742e-05), (32768, 7.82989664003253e-05), (65536, 8.093897486105561e-05), (131072, 8.87662754394114e-05), (262144, 9.019806748256087e-05), (524288, 9.739422239363194e-05), (1048576, 0.00012172272545285524), (2097152, 0.00017607869813218713), (4194304, 0.00027218100149184465), (8388608, 0.00046325917355716223), (16777216, 0.0008446318097412586), (33554432, 0.0016153808683156967), (67108864, 0.003154648467898369), (134217728, 0.006217516958713531), (268435456, 0.012345165014266968), (536870912, 0.024690330028533936), (1073741824, 0.04910581111907959), (2147483648, 0.0980856418609619)]
(((0, 4), (1, 5), (2, 6), (3, 7)), 'f32'): [(0, 1.7556565580889583e-05), (1, 1.762609463185072e-05), (2, 3.525218926370144e-05), (4, 6.825468153692782e-05), (8, 5.903621786274015e-05), (16, 6.90379529260099e-05), (32, 6.761000258848071e-05), (64, 5.660267197526991e-05), (128, 5.574524402618408e-05), (256, 5.596168921329081e-05), (512, 5.70706615690142e-05), (1024, 5.5779441026970744e-05), (2048, 6.313365884125233e-05), (4096, 7.384966011159122e-05), (8192, 7.693681982345879e-05), (16384, 7.726356852799654e-05), (32768, 8.069680188782513e-05), (65536, 8.762048673816025e-05), (131072, 8.97824647836387e-05), (262144, 0.00010074971942231059), (524288, 0.00012489495566114783), (1048576, 0.00017411034787073731), (2097152, 0.00026701658498495817), (4194304, 0.0004594631027430296), (8388608, 0.0008426397107541562), (16777216, 0.0016099605709314348), (33554432, 0.003143331035971642), (67108864, 0.006208471953868866), (134217728, 0.012351199984550474), (268435456, 0.02461715042591095), (536870912, 0.04910862445831299), (1073741824, 0.09805183410644533)]
(((0, 1, 2, 3), (4, 5, 6, 7)), 'f16'): [(0, 1.713994424790144e-05), (1, 1.72698637470603e-05), (2, 1.7329672118648887e-05), (4, 3.465934423729777e-05), (8, 6.931868847459555e-05), (16, 0.00010785038466565311), (32, 0.00010497766197659075), (64, 0.00010543887037783861), (128, 0.00010818804730661213), (256, 0.0001070139987859875), (512, 0.00010680168634280562), (1024, 0.00010741950245574118), (2048, 0.00010740236029960215), (4096, 0.00010761927114799619), (8192, 0.00011000270023941994), (16384, 0.00011443177936598657), (32768, 0.00011445203563198447), (65536, 0.00012196210445836185), (131072, 0.00012590549886226654), (262144, 0.00012865752796642482), (524288, 0.00013997076894156635), (1048576, 0.00015083723701536655), (2097152, 0.00017472112085670233), (4194304, 0.00020768621470779183), (8388608, 0.00029447139240801334), (16777216, 0.00043768063187599177), (33554432, 0.0007156059145927429), (67108864, 0.0013442058116197586), (134217728, 0.0025404728949069977), (268435456, 0.005058273673057556), (536870912, 0.009901255369186401), (1073741824, 0.019679737091064454), (2147483648, 0.03935947418212891)]
(((0, 1, 2, 3), (4, 5, 6, 7)), 'f32'): [(0, 1.707489718683064e-05), (1, 1.6981939552351832e-05), (2, 1.702739973552525e-05), (4, 3.40547994710505e-05), (8, 6.8109598942101e-05), (16, 0.00010418007150292397), (32, 0.00010667002061381936), (64, 0.00010732028749771416), (128, 0.00010635345824994147), (256, 0.00010634961654432117), (512, 0.0001057333138305694), (1024, 0.00010387322981841862), (2048, 0.00010445126099511981), (4096, 0.00011089764302596448), (8192, 0.00011531487689353526), (16384, 0.00011613205424509943), (32768, 0.00012148162932135165), (65536, 0.00012572170817293227), (131072, 0.0001282331650145352), (262144, 0.00013661218690685928), (524288, 0.00014777801698073745), (1048576, 0.00017074844799935818), (2097152, 0.0002077125245705247), (4194304, 0.00028439750894904137), (8388608, 0.00042684003710746765), (16777216, 0.0007256520912051201), (33554432, 0.0013350248336791992), (67108864, 0.00254875048995018), (134217728, 0.004984244704246521), (268435456, 0.009881511330604553), (536870912, 0.019763022661209106), (1073741824, 0.03952604532241821)]
(((0, 1), (2, 3), (4, 5), (6, 7)), 'f16'): [(0, 1.8108461517840624e-05), (1, 1.8149556126445532e-05), (2, 3.6299112252891064e-05), (4, 6.87228748574853e-05), (8, 6.923096952959895e-05), (16, 6.877040141262114e-05), (32, 6.874508108012378e-05), (64, 6.89048320055008e-05), (128, 6.721419049426913e-05), (256, 6.79045042488724e-05), (512, 6.792484782636166e-05), (1024, 6.815939559601247e-05), (2048, 6.887846393510699e-05), (4096, 6.862953887321055e-05), (8192, 7.147734868340194e-05), (16384, 7.249394548125565e-05), (32768, 7.676862878724933e-05), (65536, 8.181703742593527e-05), (131072, 8.988819899968803e-05), (262144, 9.273848263546824e-05), (524288, 9.897848940454423e-05), (1048576, 0.00012530229287222028), (2097152, 0.00017652794485911727), (4194304, 0.0002658519661054015), (8388608, 0.0004660349804908037), (16777216, 0.0008440068922936917), (33554432, 0.00161567609757185), (67108864, 0.0031443722546100616), (134217728, 0.006234865635633468), (268435456, 0.012343183159828186), (536870912, 0.024594932794570923), (1073741824, 0.049189865589141846), (2147483648, 0.09812676906585693)]
(((0, 1), (2, 3), (4, 5), (6, 7)), 'f32'): [(0, 1.919813803397119e-05), (1, 1.9186671124771237e-05), (2, 3.8373342249542475e-05), (4, 6.588862743228674e-05), (8, 6.645606481470168e-05), (16, 6.577558815479279e-05), (32, 6.790220504626632e-05), (64, 6.886484334245324e-05), (128, 6.80985685903579e-05), (256, 6.669631693512201e-05), (512, 6.909327930770814e-05), (1024, 6.992000271566212e-05), (2048, 6.886647315695882e-05), (4096, 7.017503958195448e-05), (8192, 7.755329716019332e-05), (16384, 7.628824096173048e-05), (32768, 8.325761882588267e-05), (65536, 8.629672811366618e-05), (131072, 9.100488387048244e-05), (262144, 0.00010255517554469408), (524288, 0.00012710594455711544), (1048576, 0.00017516303341835737), (2097152, 0.00026900111697614193), (4194304, 0.0004652703646570444), (8388608, 0.0008473340421915054), (16777216, 0.0016149738803505898), (33554432, 0.0031457487493753433), (67108864, 0.006217360496520996), (134217728, 0.012357220053672789), (268435456, 0.024624869227409363), (536870912, 0.049134254455566406), (1073741824, 0.09813923835754394)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,)), 'f16'): [(0, 1.7425132682546973e-05), (1, 2.0407838746905327e-05), (2, 2.051482442766428e-05), (4, 2.0520732505246997e-05), (8, 2.062396379187703e-05), (16, 2.0715000573545694e-05), (32, 2.0824227249249816e-05), (64, 2.0945852156728506e-05), (128, 2.341310027986765e-05), (256, 2.106005558744073e-05), (512, 2.094064257107675e-05), (1024, 2.06739641726017e-05), (2048, 2.0657869754359126e-05), (4096, 2.0740902982652187e-05), (8192, 2.066069282591343e-05), (16384, 2.080734702758491e-05), (32768, 2.0640785805881023e-05), (65536, 2.0633131498470902e-05), (131072, 2.0372128346934915e-05), (262144, 2.0549341570585966e-05), (524288, 2.0326144294813275e-05), (1048576, 2.085216692648828e-05), (2097152, 3.8330384995788336e-05), (4194304, 5.251495167613029e-05), (8388608, 7.206248119473457e-05), (16777216, 0.00011388771235942839), (33554432, 0.000199328176677227), (67108864, 0.00036855041980743403), (134217728, 0.0007220543920993805), (268435456, 0.0014298558235168457), (536870912, 0.002733185887336731), (1073741824, 0.005430889129638672), (2147483648, 0.010767197608947754)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,)), 'f32'): [(0, 1.676971442066133e-05), (1, 1.9518309272825718e-05), (2, 1.9688770407810807e-05), (4, 1.9658473320305347e-05), (8, 1.963999238796532e-05), (16, 1.979852095246315e-05), (32, 1.9704923033714294e-05), (64, 1.9723287550732493e-05), (128, 1.967049320228398e-05), (256, 1.9666535081341863e-05), (512, 1.9646977307274938e-05), (1024, 1.9594008335843682e-05), (2048, 1.9690924091264606e-05), (4096, 1.9621686078608036e-05), (8192, 2.0222942112013698e-05), (16384, 2.0364997908473015e-05), (32768, 2.0879902876913548e-05), (65536, 2.0453677279874682e-05), (131072, 2.103840233758092e-05), (262144, 2.093191142193973e-05), (524288, 2.0471663447096944e-05), (1048576, 3.237125929445028e-05), (2097152, 4.42346790805459e-05), (4194304, 6.570504046976566e-05), (8388608, 0.00010818382725119591), (16777216, 0.00019390415400266645), (33554432, 0.0003686770796775818), (67108864, 0.0007060356438159943), (134217728, 0.001392759382724762), (268435456, 0.002742394804954529), (536870912, 0.005433440208435059), (1073741824, 0.01075572967529297)]
all_reduce_cost_dict:
(((0, 1, 2, 3, 4, 5, 6, 7),), 'f16'): [(0, 0.00020855889306403697), (1, 0.00021091659436933696), (2, 0.00020925624994561076), (4, 0.00021351693430915475), (8, 0.00021367202862165868), (16, 0.0002060204860754311), (32, 0.00021109837689436972), (64, 0.00020899478113278747), (128, 0.00020627398043870926), (256, 0.0002138516865670681), (512, 0.0002080299018416554), (1024, 0.00021254239254631105), (2048, 0.0002143807359971106), (4096, 0.00021433940855786202), (8192, 0.00021681623184122148), (16384, 0.0002262699999846518), (32768, 0.0002239481545984745), (65536, 0.00022920375340618196), (131072, 0.00023692756076343358), (262144, 0.00023956297081895173), (524288, 0.00025427722721360624), (1048576, 0.00027985888300463557), (2097152, 0.000329027883708477), (4194304, 0.00041683390736579895), (8388608, 0.0005050944164395332), (16777216, 0.0007126247510313988), (33554432, 0.001163555309176445), (67108864, 0.0020355023443698883), (134217728, 0.00383661687374115), (268435456, 0.0074012428522109985), (536870912, 0.014561867713928225), (1073741824, 0.028852629661560058), (2147483648, 0.05723099708557129)]
(((0, 1, 2, 3, 4, 5, 6, 7),), 'f32'): [(0, 0.00021047194604761899), (1, 0.0002083250437863171), (2, 0.00020404622773639858), (4, 0.0002127127663698047), (8, 0.00021234893938526514), (16, 0.00021399528486654162), (32, 0.0002131866931449622), (64, 0.0002121705620083958), (128, 0.00021937707788310945), (256, 0.00020932545885443687), (512, 0.00021571043180301785), (1024, 0.00022348426864482462), (2048, 0.00022053319844417274), (4096, 0.00022220236132852733), (8192, 0.0002285604423377663), (16384, 0.00022795895347371697), (32768, 0.00023155059898272157), (65536, 0.00023616899852640927), (131072, 0.00024448809563182294), (262144, 0.00026369583792984486), (524288, 0.0002852866309694946), (1048576, 0.00032915431074798107), (2097152, 0.0004134026821702719), (4194304, 0.0005117026157677174), (8388608, 0.0007166806608438492), (16777216, 0.001145634800195694), (33554432, 0.002023167908191681), (67108864, 0.00382290780544281), (134217728, 0.007385432720184325), (268435456, 0.014517211914062498), (536870912, 0.028688502311706544), (1073741824, 0.05702638626098633)]
(((0, 2, 4, 6), (1, 3, 5, 7)), 'f16'): [(0, 0.00010339589789509773), (1, 9.706191485747696e-05), (2, 9.603373473510146e-05), (4, 9.463276364840567e-05), (8, 9.397012763656677e-05), (16, 9.399975533597171e-05), (32, 9.43031336646527e-05), (64, 9.420551941730083e-05), (128, 9.514167322777213e-05), (256, 9.551836410537361e-05), (512, 9.575590956956148e-05), (1024, 9.903043974190949e-05), (2048, 0.00010769083746708931), (4096, 0.00011049665044993162), (8192, 0.00010892824502661824), (16384, 0.00011253904085606337), (32768, 0.00011812627781182529), (65536, 0.0001215457741636783), (131072, 0.00013835958088748157), (262144, 0.00014765153173357248), (524288, 0.0001778041187208146), (1048576, 0.00020599388517439365), (2097152, 0.0002697656163945794), (4194304, 0.0004138732329010964), (8388608, 0.0007067411206662655), (16777216, 0.0012535946443676949), (33554432, 0.002394607290625572), (67108864, 0.00469386950135231), (134217728, 0.009246297180652618), (268435456, 0.01840531826019287), (536870912, 0.036641907691955564), (1073741824, 0.07302846908569335), (2147483648, 0.14600963592529298)]
(((0, 2, 4, 6), (1, 3, 5, 7)), 'f32'): [(0, 0.00010959035716950893), (1, 0.0001051256840582937), (2, 0.00010924259549938141), (4, 0.00010468676919117569), (8, 0.00010752130765467882), (16, 0.00010879230103455484), (32, 0.00010865335934795439), (64, 0.00011239040759392081), (128, 0.000110317807411775), (256, 0.00010940691572614014), (512, 0.00010970915900543332), (1024, 0.00010805699275806546), (2048, 0.00011259649181738496), (4096, 0.00011813864693976939), (8192, 0.00011865826672874391), (16384, 0.00012087731738574803), (32768, 0.00012743493425659835), (65536, 0.00014069731696508825), (131072, 0.00015339418314397335), (262144, 0.00017773962463252246), (524288, 0.00019240553956478834), (1048576, 0.0002573904348537326), (2097152, 0.0003979685716331005), (4194304, 0.0006822678260505199), (8388608, 0.001241249032318592), (16777216, 0.002382393926382065), (33554432, 0.00465928390622139), (67108864, 0.009221047163009644), (134217728, 0.018364638090133667), (268435456, 0.036555910110473634), (536870912, 0.07290983200073242), (1073741824, 0.1456575870513916)]
(((0, 4), (1, 5), (2, 6), (3, 7)), 'f16'): [(0, 5.47646195627749e-05), (1, 5.441068788059056e-05), (2, 6.758540985174477e-05), (4, 6.75628543831408e-05), (8, 6.633653538301587e-05), (16, 6.692876922897995e-05), (32, 6.671922164969146e-05), (64, 6.831332575529814e-05), (128, 6.98321673553437e-05), (256, 6.855203537270427e-05), (512, 6.187133840285242e-05), (1024, 5.85682864766568e-05), (2048, 5.768676055595279e-05), (4096, 6.547218072228134e-05), (8192, 7.436872692778707e-05), (16384, 7.54237116780132e-05), (32768, 8.098548278212547e-05), (65536, 8.793018059805037e-05), (131072, 9.119062451645733e-05), (262144, 0.00010244519216939806), (524288, 0.00012305699056014419), (1048576, 0.00017109198961406946), (2097152, 0.00026353728026151657), (4194304, 0.0004549941513687372), (8388608, 0.0008260915055871011), (16777216, 0.0015838677063584326), (33554432, 0.003076396882534027), (67108864, 0.006076309829950333), (134217728, 0.012069076299667358), (268435456, 0.024064883589744568), (536870912, 0.04807114601135254), (1073741824, 0.09585230350494386), (2147483648, 0.19149904251098632)]
(((0, 4), (1, 5), (2, 6), (3, 7)), 'f32'): [(0, 6.820244016125798e-05), (1, 6.766634760424495e-05), (2, 6.705941632390022e-05), (4, 5.824098479934037e-05), (8, 5.452011828310788e-05), (16, 6.767193553969264e-05), (32, 6.727245636284351e-05), (64, 6.916632992215455e-05), (128, 6.856003892607987e-05), (256, 6.76462659612298e-05), (512, 6.880657747387886e-05), (1024, 6.970393587835133e-05), (2048, 7.165840361267328e-05), (4096, 7.361194002442062e-05), (8192, 7.76431115809828e-05), (16384, 8.212547982111573e-05), (32768, 8.752039866521955e-05), (65536, 9.543675696477294e-05), (131072, 0.00010250002378597856), (262144, 0.00011665376950986682), (524288, 0.00017317791935056448), (1048576, 0.0002666451036930084), (2097152, 0.0004537447821348906), (4194304, 0.0008276556618511678), (8388608, 0.001581041142344475), (16777216, 0.0030725542455911636), (33554432, 0.006077650934457779), (67108864, 0.012050651013851166), (134217728, 0.024021118879318237), (268435456, 0.048001432418823244), (536870912, 0.09581637382507324), (1073741824, 0.1913677453994751)]
(((0, 1, 2, 3), (4, 5, 6, 7)), 'f16'): [(0, 0.00010234932415187359), (1, 0.00010291041689924894), (2, 0.00010308125638402998), (4, 0.0001006779493764043), (8, 0.00010201591067016125), (16, 9.950241656042635e-05), (32, 0.0001028967963065952), (64, 0.00010500909411348403), (128, 0.0001069233112502843), (256, 0.00010327232303097843), (512, 0.00010741283767856658), (1024, 0.00011041603283956646), (2048, 0.000113765214337036), (4096, 0.000116279989015311), (8192, 0.00011764123337343337), (16384, 0.00012004579184576869), (32768, 0.00011951211490668355), (65536, 0.00012397280079312623), (131072, 0.00013043617946095765), (262144, 0.00014606144395656884), (524288, 0.00015498753054998815), (1048576, 0.00018095428822562099), (2097152, 0.00022277352400124073), (4194304, 0.00029819877818226814), (8388608, 0.0004546497948467732), (16777216, 0.0007572034373879433), (33554432, 0.0013435091823339462), (67108864, 0.002572830766439438), (134217728, 0.005064867436885834), (268435456, 0.009969651699066162), (536870912, 0.019934487342834473), (1073741824, 0.03956634998321533), (2147483648, 0.0791234016418457)]
(((0, 1, 2, 3), (4, 5, 6, 7)), 'f32'): [(0, 0.00010364819900132716), (1, 0.00010253587970510125), (2, 0.0001032980508171022), (4, 0.00010346437920816243), (8, 0.00010576506610959768), (16, 9.797804523259403e-05), (32, 0.0001033905427902937), (64, 0.0001038476184476167), (128, 0.00010843022027984262), (256, 0.00010846558143384753), (512, 0.00011066455044783652), (1024, 0.00010993209434673191), (2048, 0.00011617448762990534), (4096, 0.00011523650027811527), (8192, 0.00011474129860289395), (16384, 0.00012028063065372409), (32768, 0.00012666953261941671), (65536, 0.00013198945089243352), (131072, 0.000143644108902663), (262144, 0.00015747841098345816), (524288, 0.000183035503141582), (1048576, 0.00022311462089419365), (2097152, 0.00029032654128968716), (4194304, 0.0004461328499019146), (8388608, 0.0007482878863811493), (16777216, 0.001343410462141037), (33554432, 0.0025437474250793457), (67108864, 0.004985824227333069), (134217728, 0.009892880916595459), (268435456, 0.019623970985412596), (536870912, 0.03924794197082519), (1073741824, 0.07849588394165039)]
(((0, 1), (2, 3), (4, 5), (6, 7)), 'f16'): [(0, 6.844662129878998e-05), (1, 6.793628563173115e-05), (2, 6.811422645114362e-05), (4, 6.871437653899193e-05), (8, 6.827185279689729e-05), (16, 6.904217298142612e-05), (32, 6.877703708596528e-05), (64, 6.89688022248447e-05), (128, 6.857540574856102e-05), (256, 6.910806405358016e-05), (512, 6.799193215556443e-05), (1024, 6.846213364042342e-05), (2048, 6.908626528456807e-05), (4096, 7.052635191939771e-05), (8192, 7.739517604932189e-05), (16384, 7.818735321052372e-05), (32768, 8.135740063153207e-05), (65536, 9.478311403654516e-05), (131072, 9.509007213637233e-05), (262144, 0.00010444939834997059), (524288, 0.0001251243520528078), (1048576, 0.00017509295139461756), (2097152, 0.00026792671997100115), (4194304, 0.00046042958274483686), (8388608, 0.0008289390243589879), (16777216, 0.0015857676044106483), (33554432, 0.0030934959650039673), (67108864, 0.006072979420423508), (134217728, 0.012145958840847015), (268435456, 0.024092063307762146), (536870912, 0.04796578884124756), (1073741824, 0.09573917388916016), (2147483648, 0.19130206108093262)]
(((0, 1), (2, 3), (4, 5), (6, 7)), 'f32'): [(0, 6.96303031872958e-05), (1, 6.788034806959331e-05), (2, 6.829472840763628e-05), (4, 6.890931399539113e-05), (8, 7.001115591265261e-05), (16, 5.565871833823621e-05), (32, 5.56999584659934e-05), (64, 5.7814206229522824e-05), (128, 6.793363718315959e-05), (256, 6.925952038727701e-05), (512, 7.057678885757923e-05), (1024, 7.009532419033349e-05), (2048, 6.936516729183495e-05), (4096, 7.226556772366166e-05), (8192, 7.776511483825743e-05), (16384, 8.08358599897474e-05), (32768, 9.037667769007385e-05), (65536, 9.678950300440192e-05), (131072, 0.00010461322381161153), (262144, 0.00012576559674926102), (524288, 0.00017413817113265395), (1048576, 0.0002707020612433553), (2097152, 0.0004533436149358749), (4194304, 0.000828250776976347), (8388608, 0.0015801247209310532), (16777216, 0.003086870536208153), (33554432, 0.006082676351070404), (67108864, 0.012044563889503479), (134217728, 0.02402450144290924), (268435456, 0.04795317649841309), (536870912, 0.09572908878326418), (1073741824, 0.19145817756652836)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,)), 'f16'): [(0, 1.7035577911883593e-05), (1, 2.004514681175351e-05), (2, 2.0071136532351375e-05), (4, 2.0717765437439084e-05), (8, 2.0158884581178427e-05), (16, 2.010565367527306e-05), (32, 2.0386709365993738e-05), (64, 1.994954072870314e-05), (128, 1.9997329218313098e-05), (256, 1.9958417396992445e-05), (512, 2.00418580789119e-05), (1024, 1.995917409658432e-05), (2048, 2.0110892364755273e-05), (4096, 1.9930273992940784e-05), (8192, 1.965853152796626e-05), (16384, 1.9883096683770418e-05), (32768, 1.9784201867878437e-05), (65536, 2.005638089030981e-05), (131072, 2.0062230760231614e-05), (262144, 1.965291448868811e-05), (524288, 1.9618863007053733e-05), (1048576, 1.9989791326224804e-05), (2097152, 3.2148673199117184e-05), (4194304, 4.9718655645847314e-05), (8388608, 7.057236507534981e-05), (16777216, 0.00011414755135774612), (33554432, 0.0002002120018005371), (67108864, 0.00036538019776344294), (134217728, 0.0007085129618644714), (268435456, 0.0014013797044754028), (536870912, 0.0028027594089508057), (1073741824, 0.0054299354553222655), (2147483648, 0.01075451374053955)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,)), 'f32'): [(0, 1.72128202393651e-05), (1, 2.0148057956248522e-05), (2, 2.060498809441924e-05), (4, 2.0244100596755743e-05), (8, 2.0416773622855544e-05), (16, 2.076494274660945e-05), (32, 2.014380879700184e-05), (64, 2.017721999436617e-05), (128, 2.0213657990098e-05), (256, 1.9967759726569057e-05), (512, 2.0260835299268365e-05), (1024, 2.0323204807937145e-05), (2048, 2.058548852801323e-05), (4096, 2.01780057977885e-05), (8192, 2.033903729170561e-05), (16384, 2.0429433789104223e-05), (32768, 2.0371429855003953e-05), (65536, 2.2343272576108578e-05), (131072, 2.0354753360152245e-05), (262144, 2.0209845388308167e-05), (524288, 2.1717336494475603e-05), (1048576, 3.9434642530977726e-05), (2097152, 5.06893265992403e-05), (4194304, 7.191114127635956e-05), (8388608, 0.00011478923261165618), (16777216, 0.00020365417003631592), (33554432, 0.00036624446511268616), (67108864, 0.0007170736789703369), (134217728, 0.0014035999774932861), (268435456, 0.0027559280395507814), (536870912, 0.005511856079101563), (1073741824, 0.010742473602294921)]
all_to_all_cost_dict:
(((0, 1, 2, 3, 4, 5, 6, 7),), 'f16'): [(0, 1.7858896171674132e-05), (1, 1.7830636352300644e-05), (2, 1.7773621948435903e-05), (4, 1.7870712326839566e-05), (8, 1.7752288840711117e-05), (16, 1.7740181647241116e-05), (32, 1.775252167135477e-05), (64, 3.550504334270954e-05), (128, 7.101008668541908e-05), (256, 0.00014202017337083817), (512, 0.00024053818196989599), (1024, 0.00024153897538781166), (2048, 0.00023976876400411126), (4096, 0.00024291191948577764), (8192, 0.00024233313160948455), (16384, 0.00024160143220797178), (32768, 0.00023688035435043275), (65536, 0.00022045540390536186), (131072, 0.00021185734658502042), (262144, 0.00022473183344118297), (524288, 0.0002493863576091826), (1048576, 0.00026706227799877524), (2097152, 0.00030547776259481907), (4194304, 0.0003786305896937847), (8388608, 0.0005036387592554092), (16777216, 0.0007328423671424389), (33554432, 0.0011523645371198654), (67108864, 0.0020058024674654007), (134217728, 0.004011604934930801), (268435456, 0.007863909006118774), (536870912, 0.015182822942733763), (1073741824, 0.02986915111541748), (2147483648, 0.05921704769134522)]
(((0, 1, 2, 3, 4, 5, 6, 7),), 'f32'): [(0, 1.7091631889343262e-05), (1, 1.7430633306503296e-05), (2, 1.7406593542546034e-05), (4, 1.7808983102440834e-05), (8, 1.7482059774920344e-05), (16, 1.743514440022409e-05), (32, 1.7800048226490617e-05), (64, 3.5600096452981234e-05), (128, 7.120019290596247e-05), (256, 0.00014240038581192493), (512, 0.00024603039491921663), (1024, 0.0002475679211784154), (2048, 0.00024455160018987954), (4096, 0.00024536874843761325), (8192, 0.00024611817207187414), (16384, 0.0002444257843308151), (32768, 0.000245533708948642), (65536, 0.00024714856408536434), (131072, 0.0002496697416063398), (262144, 0.00025640614330768585), (524288, 0.0002666162035893649), (1048576, 0.00030271068681031466), (2097152, 0.0003807629691436886), (4194304, 0.0005024205893278122), (8388608, 0.0007274327799677849), (16777216, 0.0011602649465203285), (33554432, 0.002024298533797264), (67108864, 0.004048597067594528), (134217728, 0.007969409227371216), (268435456, 0.015501618385314941), (536870912, 0.03033246994018555), (1073741824, 0.060025596618652345)]
(((0, 2, 4, 6), (1, 3, 5, 7)), 'f16'): [(0, 1.780092134140432e-05), (1, 1.781134051270783e-05), (2, 1.7651822417974472e-05), (4, 1.7733051208779216e-05), (8, 1.7761281924322248e-05), (16, 3.5522563848644495e-05), (32, 7.104512769728899e-05), (64, 0.00013116028276272118), (128, 0.00013236311497166753), (256, 0.00013289955677464604), (512, 0.00013225898146629333), (1024, 0.00013257405953481793), (2048, 0.00013224061694927514), (4096, 0.00013258718536235392), (8192, 0.00013114511966705322), (16384, 0.00013001522165723145), (32768, 0.00012880514259450138), (65536, 0.00012597604654729366), (131072, 0.00013492454309016466), (262144, 0.0001373297709506005), (524288, 0.0001424771617166698), (1048576, 0.00016052016871981323), (2097152, 0.000193272833712399), (4194304, 0.0002810613950714469), (8388608, 0.00045386026613414293), (16777216, 0.0007567727006971836), (33554432, 0.0014081764966249466), (67108864, 0.0026777051389217377), (134217728, 0.005245652049779892), (268435456, 0.010491304099559784), (536870912, 0.020982608199119568), (1073741824, 0.041965216398239136), (2147483648, 0.08393043279647827)]
(((0, 2, 4, 6), (1, 3, 5, 7)), 'f32'): [(0, 1.767839421518147e-05), (1, 1.7640908481553197e-05), (2, 1.770968083292246e-05), (4, 1.7669430235400796e-05), (8, 1.781116588972509e-05), (16, 3.562233177945018e-05), (32, 7.124466355890036e-05), (64, 0.00013079895870760083), (128, 0.00013028830289840698), (256, 0.0001312217500526458), (512, 0.00013222487177699804), (1024, 0.0001293757522944361), (2048, 0.0001307621132582426), (4096, 0.00013047608081251383), (8192, 0.0001307605125475675), (16384, 0.00012926396448165178), (32768, 0.0001292091910727322), (65536, 0.00013092695735394955), (131072, 0.00013417578884400427), (262144, 0.0001434705627616495), (524288, 0.00015507516218349338), (1048576, 0.00018837861716747284), (2097152, 0.00026891997549682856), (4194304, 0.000445404089987278), (8388608, 0.0007669595070183276), (16777216, 0.0014002975076436996), (33554432, 0.0028005950152873993), (67108864, 0.00548451766371727), (134217728, 0.01096903532743454), (268435456, 0.02193807065486908), (536870912, 0.04387614130973816), (1073741824, 0.08775228261947632)]
(((0, 4), (1, 5), (2, 6), (3, 7)), 'f16'): [(0, 1.7517042579129338e-05), (1, 1.7274636775255203e-05), (2, 1.7183716408908367e-05), (4, 3.4367432817816734e-05), (8, 6.873486563563347e-05), (16, 8.227524813264608e-05), (32, 8.283465285785496e-05), (64, 8.159811841323972e-05), (128, 8.355267345905304e-05), (256, 8.206430356949568e-05), (512, 8.183135651051998e-05), (1024, 8.275292930193245e-05), (2048, 8.280231850221753e-05), (4096, 8.252583211287856e-05), (8192, 8.364656241610646e-05), (16384, 8.383448584936559e-05), (32768, 8.464243728667498e-05), (65536, 8.54847312439233e-05), (131072, 8.609570795670152e-05), (262144, 8.669440285302699e-05), (524288, 9.820281411521137e-05), (1048576, 0.00010974498582072556), (2097152, 0.00013851444236934185), (4194304, 0.00019938475452363488), (8388608, 0.00036904658190906053), (16777216, 0.0006463550962507725), (33554432, 0.0012380992993712425), (67108864, 0.0024090204387903214), (134217728, 0.0047750212252140045), (268435456, 0.009472228586673737), (536870912, 0.018909022212028503), (1073741824, 0.037717580795288086), (2147483648, 0.07543516159057617)]
(((0, 4), (1, 5), (2, 6), (3, 7)), 'f32'): [(0, 1.9621627870947123e-05), (1, 1.78734480869025e-05), (2, 1.7730897525325418e-05), (4, 3.5461795050650835e-05), (8, 7.092359010130167e-05), (16, 8.295968291349709e-05), (32, 8.303584763780236e-05), (64, 8.23412265162915e-05), (128, 8.284513023681939e-05), (256, 8.282388444058597e-05), (512, 8.219224400818348e-05), (1024, 8.373125456273556e-05), (2048, 8.637426071800292e-05), (4096, 8.517122478224337e-05), (8192, 8.50581272970885e-05), (16384, 8.206162601709366e-05), (32768, 8.451342000626028e-05), (65536, 8.522675489075482e-05), (131072, 8.69166397023946e-05), (262144, 9.499656152911484e-05), (524288, 0.00010834023123607038), (1048576, 0.00013673340436071157), (2097152, 0.00019725318998098373), (4194304, 0.0003668048884719611), (8388608, 0.0006517781876027584), (16777216, 0.0012402059510350227), (33554432, 0.0024242643266916275), (67108864, 0.004781316965818405), (134217728, 0.009511657059192657), (268435456, 0.018963217735290527), (536870912, 0.03787791728973389), (1073741824, 0.07561740875244141)]
(((0, 1, 2, 3), (4, 5, 6, 7)), 'f16'): [(0, 1.7681100871413946e-05), (1, 1.7830985598266125e-05), (2, 1.7777696484699845e-05), (4, 1.7898622900247574e-05), (8, 1.7727434169501066e-05), (16, 3.545486833900213e-05), (32, 7.090973667800426e-05), (64, 0.00012957380386069417), (128, 0.00012977243750356138), (256, 0.00012863986194133759), (512, 0.00012622331269085407), (1024, 0.00012289831647649407), (2048, 0.0001234633382409811), (4096, 0.0001322160824202001), (8192, 0.00013229195610620081), (16384, 0.00013112672604620457), (32768, 0.00012859251000918448), (65536, 0.00011721759801730514), (131072, 0.0001298784336540848), (262144, 0.0001280121796298772), (524288, 0.0001313120883423835), (1048576, 0.00013747482444159687), (2097152, 0.00014604348689317703), (4194304, 0.00016085745301097631), (8388608, 0.00019454630091786385), (16777216, 0.0002642432227730751), (33554432, 0.0004360424354672432), (67108864, 0.0007331632077693939), (134217728, 0.0013958476483821869), (268435456, 0.00272572785615921), (536870912, 0.005394876003265381), (1073741824, 0.010705018043518066), (2147483648, 0.021331381797790528)]
(((0, 1, 2, 3), (4, 5, 6, 7)), 'f32'): [(0, 1.7809623386710882e-05), (1, 1.7713988199830055e-05), (2, 1.7777609173208475e-05), (4, 1.7566897440701723e-05), (8, 1.750740921124816e-05), (16, 3.501481842249632e-05), (32, 7.002963684499264e-05), (64, 0.00012735379277728498), (128, 0.00012882112059742212), (256, 0.00012749360757879913), (512, 0.00012746642460115254), (1024, 0.00012733493349514902), (2048, 0.0001274494279641658), (4096, 0.00013247065362520516), (8192, 0.00012711909948848188), (16384, 0.0001247632026206702), (32768, 0.00013044511433690786), (65536, 0.00013002418563701212), (131072, 0.00012932191020809114), (262144, 0.00013530559954233468), (524288, 0.0001371862308587879), (1048576, 0.00014482933329418302), (2097152, 0.00016268598847091198), (4194304, 0.00020748167298734188), (8388608, 0.0002722530625760555), (16777216, 0.00043595489114522934), (33554432, 0.0007419995963573456), (67108864, 0.0014105625450611115), (134217728, 0.0027661994099617004), (268435456, 0.005478337407112122), (536870912, 0.010938906669616699), (1073741824, 0.021757888793945312)]
(((0, 1), (2, 3), (4, 5), (6, 7)), 'f16'): [(0, 1.7442594980821013e-05), (1, 1.7583282897248864e-05), (2, 1.7205893527716398e-05), (4, 3.4411787055432796e-05), (8, 6.882357411086559e-05), (16, 8.235947461798787e-05), (32, 8.173301466740668e-05), (64, 8.171063382178545e-05), (128, 8.136784890666604e-05), (256, 8.161421283148229e-05), (512, 8.199017611332238e-05), (1024, 7.654138607904315e-05), (2048, 7.241324055939913e-05), (4096, 8.597300620749593e-05), (8192, 8.025500574149191e-05), (16384, 8.089042967185378e-05), (32768, 8.309294935315847e-05), (65536, 8.603956666775048e-05), (131072, 8.534465450793505e-05), (262144, 8.829127182252705e-05), (524288, 9.53754351940006e-05), (1048576, 0.00011046318104490638), (2097152, 0.00013685121666640043), (4194304, 0.00019633513875305653), (8388608, 0.00037365639582276344), (16777216, 0.0006424183957278728), (33554432, 0.0012303274124860764), (67108864, 0.002406710758805275), (134217728, 0.00476473942399025), (268435456, 0.009479299187660217), (536870912, 0.018921643495559692), (1073741824, 0.03771717548370361), (2147483648, 0.07543435096740722)]
(((0, 1), (2, 3), (4, 5), (6, 7)), 'f32'): [(0, 1.7131795175373554e-05), (1, 1.7105223378166556e-05), (2, 1.6984675312414765e-05), (4, 3.396935062482953e-05), (8, 6.793870124965906e-05), (16, 8.54056270327419e-05), (32, 8.543577860109508e-05), (64, 8.539913687855005e-05), (128, 8.581459405831993e-05), (256, 8.559162961319089e-05), (512, 8.509831968694925e-05), (1024, 8.598991553299129e-05), (2048, 8.455658098682761e-05), (4096, 8.363733650185168e-05), (8192, 8.41846049297601e-05), (16384, 8.155248360708356e-05), (32768, 8.37000843603164e-05), (65536, 8.331993012689054e-05), (131072, 8.740241173654795e-05), (262144, 8.701282786205411e-05), (524288, 0.00010331577504985034), (1048576, 0.0001369180390611291), (2097152, 0.00019535166211426258), (4194304, 0.0003681809175759554), (8388608, 0.0006473525427281857), (16777216, 0.0012344783172011375), (33554432, 0.002415470778942108), (67108864, 0.004779178649187088), (134217728, 0.009510897099971771), (268435456, 0.018953919410705566), (536870912, 0.03783040046691895), (1073741824, 0.07557251453399658)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,)), 'f16'): [(0, 1.6920908819884062e-05), (1, 1.9964121747761965e-05), (2, 2.0168954506516457e-05), (4, 2.0187406335026026e-05), (8, 2.010236494243145e-05), (16, 1.9899045582860708e-05), (32, 2.0460603991523385e-05), (64, 2.081625279970467e-05), (128, 2.0627485355362296e-05), (256, 2.0292471162974834e-05), (512, 2.0648149074986577e-05), (1024, 2.0570005290210247e-05), (2048, 2.0787527319043875e-05), (4096, 2.1486252080649137e-05), (8192, 2.041459083557129e-05), (16384, 2.0112143829464912e-05), (32768, 2.0370673155412078e-05), (65536, 2.0670151570811868e-05), (131072, 2.08771089091897e-05), (262144, 2.0213890820741653e-05), (524288, 2.2787484340369698e-05), (1048576, 2.112268703058362e-05), (2097152, 3.215367905795574e-05), (4194304, 4.977895878255367e-05), (8388608, 7.038959302008152e-05), (16777216, 0.00011107372120022772), (33554432, 0.0002000965178012848), (67108864, 0.0003691110759973526), (134217728, 0.0007052868604660034), (268435456, 0.0013819336891174316), (536870912, 0.002733558416366577), (1073741824, 0.0054179906845092775), (2147483648, 0.01074812412261963)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,)), 'f32'): [(0, 1.7307669622823596e-05), (1, 2.065979060716927e-05), (2, 2.1213461877778172e-05), (4, 2.053871867246926e-05), (8, 2.074337680824101e-05), (16, 2.0744046196341515e-05), (32, 2.059314283542335e-05), (64, 2.0761333871632814e-05), (128, 2.047803718596697e-05), (256, 2.042847336269915e-05), (512, 2.0494568161666393e-05), (1024, 2.0648352801799774e-05), (2048, 2.046528970822692e-05), (4096, 2.051884075626731e-05), (8192, 2.0062143448740244e-05), (16384, 2.0389416022226214e-05), (32768, 2.037858939729631e-05), (65536, 2.0320381736382842e-05), (131072, 1.9970786524936557e-05), (262144, 2.0216655684635043e-05), (524288, 2.017151564359665e-05), (1048576, 3.563490463420749e-05), (2097152, 5.03521878272295e-05), (4194304, 7.110787555575371e-05), (8388608, 0.00010862899944186209), (16777216, 0.00019347667694091797), (33554432, 0.00036131590604782104), (67108864, 0.000698331743478775), (134217728, 0.00139666348695755), (268435456, 0.0027267783880233765), (536870912, 0.005405902862548828), (1073741824, 0.01075601577758789)]
reduce_scatter_cost_dict:
(((0, 1, 2, 3, 4, 5, 6, 7),), 'f16'): [(0, 1.830604742281139e-05), (1, 1.837554736994207e-05), (2, 1.8243794329464436e-05), (4, 1.8079415895044804e-05), (8, 3.615883179008961e-05), (16, 7.231766358017921e-05), (32, 0.00014463532716035843), (64, 0.00021338937222026289), (128, 0.00020951437181793153), (256, 0.00020305500947870311), (512, 0.0001823121856432408), (1024, 0.00017472507897764444), (2048, 0.00019893742864951491), (4096, 0.00017339910846203566), (8192, 0.00018659152556210756), (16384, 0.00020534000941552222), (32768, 0.00021517896675504744), (65536, 0.00021901819854974747), (131072, 0.0002242370101157576), (262144, 0.0002263911301270127), (524288, 0.0002236807777080685), (1048576, 0.0002637410070747137), (2097152, 0.00030082318698987365), (4194304, 0.00032441632356494665), (8388608, 0.00037484127096831804), (16777216, 0.0004870332777500153), (33554432, 0.000730183906853199), (67108864, 0.0012041181325912476), (134217728, 0.0021951012313365936), (268435456, 0.004139572381973267), (536870912, 0.008044302463531494), (1073741824, 0.015786170959472656), (2147483648, 0.031194138526916503)]
(((0, 1, 2, 3, 4, 5, 6, 7),), 'f32'): [(0, 1.833133865147829e-05), (1, 1.7876445781439543e-05), (2, 1.8026214092969894e-05), (4, 1.8058402929455042e-05), (8, 3.6116805858910084e-05), (16, 7.223361171782017e-05), (32, 0.00014446722343564034), (64, 0.0002099030825775117), (128, 0.0002077421231660992), (256, 0.0002093831135425717), (512, 0.00020282328478060663), (1024, 0.00020253288676030934), (2048, 0.00020536361262202263), (4096, 0.00019952861475758255), (8192, 0.00019766355399042368), (16384, 0.00022203990374691782), (32768, 0.0002097667893394828), (65536, 0.00021730983280576763), (131072, 0.00021490402286872268), (262144, 0.00019737976253964004), (524288, 0.00023028010036796331), (1048576, 0.0002578162821009755), (2097152, 0.000312750693410635), (4194304, 0.0003416445106267929), (8388608, 0.00044430699199438095), (16777216, 0.0006721001118421555), (33554432, 0.0011475365608930588), (67108864, 0.0021248124539852142), (134217728, 0.0040314942598342896), (268435456, 0.007829979062080383), (536870912, 0.015367746353149416), (1073741824, 0.03048975467681885)]
(((0, 2, 4, 6), (1, 3, 5, 7)), 'f16'): [(0, 1.8241582438349724e-05), (1, 1.7974467482417822e-05), (2, 1.7983518773689866e-05), (4, 3.596703754737973e-05), (8, 7.193407509475946e-05), (16, 9.697335190139711e-05), (32, 9.99341136775911e-05), (64, 0.00010714199743233621), (128, 9.836972458288074e-05), (256, 9.997101733461021e-05), (512, 0.00010056135943159461), (1024, 0.00010197309893555939), (2048, 0.00010188485612161456), (4096, 0.0001086140691768378), (8192, 0.00010818365262821317), (16384, 0.00011627579806372522), (32768, 0.0001159830717369914), (65536, 0.0001139631203841418), (131072, 0.0001195597869809717), (262144, 0.00013230208423919976), (524288, 0.0001433469878975302), (1048576, 0.00016937946202233434), (2097152, 0.00020836986368522048), (4194304, 0.0002869745949283242), (8388608, 0.00045244069769978523), (16777216, 0.0008009392768144608), (33554432, 0.0014630146324634552), (67108864, 0.0028165411204099655), (134217728, 0.005529046058654785), (268435456, 0.010868951678276062), (536870912, 0.02160455286502838), (1073741824, 0.04320910573005676), (2147483648, 0.08589334487915039)]
(((0, 2, 4, 6), (1, 3, 5, 7)), 'f32'): [(0, 1.7701531760394573e-05), (1, 1.7750455299392343e-05), (2, 1.799044548533857e-05), (4, 3.598089097067714e-05), (8, 7.196178194135427e-05), (16, 9.851931827142835e-05), (32, 9.932162356562912e-05), (64, 0.00010557047789916396), (128, 0.00010654755169525744), (256, 0.00010037134052254261), (512, 0.00010724662570282818), (1024, 0.00010756906704045831), (2048, 0.0001079245121218264), (4096, 0.00010908712283708157), (8192, 0.00011718555469997227), (16384, 0.00011814921163022518), (32768, 0.0001183298008982092), (65536, 0.00012312500621192157), (131072, 0.00014014964108355343), (262144, 0.00014986985479481518), (524288, 0.00016551362932659686), (1048576, 0.00020279776072129607), (2097152, 0.0002843154361471534), (4194304, 0.0004446334205567836), (8388608, 0.0007650721818208696), (16777216, 0.0014356430619955063), (33554432, 0.0027315188199281693), (67108864, 0.00534169003367424), (134217728, 0.010495074093341827), (268435456, 0.02085644006729126), (536870912, 0.04154603481292725), (1073741824, 0.0830920696258545)]
(((0, 4), (1, 5), (2, 6), (3, 7)), 'f16'): [(0, 1.7443206161260605e-05), (1, 1.7473503248766065e-05), (2, 3.494700649753213e-05), (4, 6.74965267535299e-05), (8, 6.649733404628932e-05), (16, 6.599983316846192e-05), (32, 6.772048072889447e-05), (64, 6.931088864803314e-05), (128, 6.813774234615266e-05), (256, 6.64808030705899e-05), (512, 6.53099559713155e-05), (1024, 6.749900057911873e-05), (2048, 6.686095730401576e-05), (4096, 6.684998515993357e-05), (8192, 6.95080088917166e-05), (16384, 7.116829510778189e-05), (32768, 7.57037487346679e-05), (65536, 7.833173731341958e-05), (131072, 8.666428038850427e-05), (262144, 9.583530481904745e-05), (524288, 0.00010764587204903363), (1048576, 0.00014230908709578216), (2097152, 0.0002201905590482056), (4194304, 0.0003646350232884288), (8388608, 0.000660550082102418), (16777216, 0.001227628905326128), (33554432, 0.0023689353838562965), (67108864, 0.0045807622373104095), (134217728, 0.00868910551071167), (268435456, 0.016712822020053864), (536870912, 0.03200061619281769), (1073741824, 0.059775233268737786), (2147483648, 0.11655592918395995)]
(((0, 4), (1, 5), (2, 6), (3, 7)), 'f32'): [(0, 1.7575861420482397e-05), (1, 1.761459861882031e-05), (2, 3.522919723764062e-05), (4, 6.485090125352144e-05), (8, 6.145791849121451e-05), (16, 5.4148549679666764e-05), (32, 6.573714199475944e-05), (64, 6.709888111799955e-05), (128, 6.615708116441965e-05), (256, 6.683386163786054e-05), (512, 6.670624134130776e-05), (1024, 6.585550727322698e-05), (2048, 6.853742524981499e-05), (4096, 7.042408105917275e-05), (8192, 7.180724060162902e-05), (16384, 7.674164953641593e-05), (32768, 7.897248724475503e-05), (65536, 8.773332228884101e-05), (131072, 9.400246199220419e-05), (262144, 0.00011116091627627611), (524288, 0.00014033340266905725), (1048576, 0.00021302950335666535), (2097152, 0.0003517451696097851), (4194304, 0.0006222573574632406), (8388608, 0.0011590709909796715), (16777216, 0.0022149421274662018), (33554432, 0.004286434501409531), (67108864, 0.008332278579473495), (134217728, 0.016441889107227325), (268435456, 0.032694920897483826), (536870912, 0.06466288566589355), (1073741824, 0.1293257713317871)]
(((0, 1, 2, 3), (4, 5, 6, 7)), 'f16'): [(0, 1.7258396837860346e-05), (1, 1.6987643903121352e-05), (2, 1.702419831417501e-05), (4, 3.404839662835002e-05), (8, 6.809679325670004e-05), (16, 0.00010754528921097518), (32, 0.00010615491191856563), (64, 0.00010777884745039047), (128, 0.00010636335355229676), (256, 0.00010678585385903717), (512, 0.00010768187348730862), (1024, 0.00011039225501008332), (2048, 0.00010776161798276007), (4096, 0.00010644021676853299), (8192, 0.00010947213741019367), (16384, 0.00011371282744221389), (32768, 0.00011791210272349417), (65536, 0.00011642512981779873), (131072, 0.00012088727089576422), (262144, 0.0001259025593753904), (524288, 0.00014445040142163634), (1048576, 0.00015352308400906622), (2097152, 0.00017562939319759607), (4194304, 0.0002215367276221514), (8388608, 0.00031212647445499897), (16777216, 0.0005108071491122246), (33554432, 0.0008596479892730714), (67108864, 0.0015307627618312838), (134217728, 0.002917703241109848), (268435456, 0.005668222904205322), (536870912, 0.011336445808410645), (1073741824, 0.02267289161682129), (2147483648, 0.04534578323364258)]
(((0, 1, 2, 3), (4, 5, 6, 7)), 'f32'): [(0, 1.7356855096295476e-05), (1, 1.7212005332112312e-05), (2, 1.7099693650379777e-05), (4, 3.4199387300759554e-05), (8, 6.839877460151911e-05), (16, 0.00010104896500706673), (32, 0.0001003043435048312), (64, 0.00010660901898518205), (128, 0.00010784523328766225), (256, 0.0001076695916708559), (512, 0.0001065906835719943), (1024, 0.0001083108363673091), (2048, 0.00010717075201682746), (4096, 0.00010334132821299134), (8192, 0.00011124409502372146), (16384, 0.00011552183423191308), (32768, 0.00011168853961862624), (65536, 0.0001189670292660594), (131072, 0.00011858317884616555), (262144, 0.00014279130846261978), (524288, 0.00015320233069360256), (1048576, 0.00017814745660871267), (2097152, 0.00022041075862944126), (4194304, 0.00029995315708220005), (8388608, 0.0004664934240281582), (16777216, 0.0007965248078107834), (33554432, 0.001400873064994812), (67108864, 0.002755057066679001), (134217728, 0.005381137132644653), (268435456, 0.010762274265289307), (536870912, 0.021524548530578613), (1073741824, 0.04304909706115723)]
(((0, 1), (2, 3), (4, 5), (6, 7)), 'f16'): [(0, 1.775796408765018e-05), (1, 1.81731884367764e-05), (2, 3.63463768735528e-05), (4, 6.591822602786124e-05), (8, 6.795968511141837e-05), (16, 6.822854629717767e-05), (32, 6.79529330227524e-05), (64, 6.815840606577694e-05), (128, 6.790182669647038e-05), (256, 6.808966281823814e-05), (512, 6.205422687344253e-05), (1024, 6.64989638607949e-05), (2048, 6.256243796087801e-05), (4096, 6.472383392974734e-05), (8192, 7.206742884591222e-05), (16384, 7.462350185960531e-05), (32768, 7.272619404830039e-05), (65536, 7.43301643524319e-05), (131072, 8.348407573066652e-05), (262144, 8.90145602170378e-05), (524288, 0.00010084972018375993), (1048576, 0.00013996480265632272), (2097152, 0.00022079620976001024), (4194304, 0.0003636527108028531), (8388608, 0.000652700662612915), (16777216, 0.0012130714021623135), (33554432, 0.0023523326963186264), (67108864, 0.004522467032074928), (134217728, 0.008714158087968826), (268435456, 0.01630590856075287), (536870912, 0.030906528234481815), (1073741824, 0.06001377105712891), (2147483648, 0.11676633358001709)]
(((0, 1), (2, 3), (4, 5), (6, 7)), 'f32'): [(0, 1.773150870576501e-05), (1, 1.7644139006733894e-05), (2, 3.528827801346779e-05), (4, 6.661281804554164e-05), (8, 6.781020783819258e-05), (16, 6.688441499136388e-05), (32, 6.799714174121618e-05), (64, 6.859979475848377e-05), (128, 6.843334995210171e-05), (256, 6.924563786014915e-05), (512, 6.646948168054223e-05), (1024, 6.636590114794672e-05), (2048, 6.600574124604464e-05), (4096, 7.29092862457037e-05), (8192, 6.801437120884657e-05), (16384, 7.562030805274844e-05), (32768, 7.461284985765815e-05), (65536, 8.635129779577255e-05), (131072, 8.8690867414698e-05), (262144, 9.865820175036787e-05), (524288, 0.00013714272063225508), (1048576, 0.00021147291408851743), (2097152, 0.00034632172901183367), (4194304, 0.0006235740147531033), (8388608, 0.0011557834222912788), (16777216, 0.0022090673446655273), (33554432, 0.004267748445272446), (67108864, 0.00830325111746788), (134217728, 0.01639954000711441), (268435456, 0.03257283568382263), (536870912, 0.06436097621917725), (1073741824, 0.1287219524383545)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,)), 'f16'): [(0, 1.7307582311332226e-05), (1, 2.0637089619413018e-05), (2, 2.0879029761999846e-05), (4, 2.040344406850636e-05), (8, 2.040807157754898e-05), (16, 2.0457751816138625e-05), (32, 2.0496401702985168e-05), (64, 2.045798464678228e-05), (128, 2.058182144537568e-05), (256, 2.045868313871324e-05), (512, 2.1287647541612387e-05), (1024, 2.0746869267895818e-05), (2048, 2.0879029761999846e-05), (4096, 2.1071522496640682e-05), (8192, 2.140123979188502e-05), (16384, 2.074684016406536e-05), (32768, 2.071549533866346e-05), (65536, 2.296071033924818e-05), (131072, 2.049619797617197e-05), (262144, 2.051197225227952e-05), (524288, 2.0793784642592072e-05), (1048576, 2.042046980932355e-05), (2097152, 3.245065454393625e-05), (4194304, 4.464678931981325e-05), (8388608, 6.609759293496609e-05), (16777216, 0.00010875612497329712), (33554432, 0.0001964382827281952), (67108864, 0.00037703849375247955), (134217728, 0.000704687088727951), (268435456, 0.0013797804713249207), (536870912, 0.002736106514930725), (1073741824, 0.005419445037841797), (2147483648, 0.010752058029174805)]
(((0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,)), 'f32'): [(0, 1.7280748579651117e-05), (1, 2.037759986706078e-05), (2, 2.028545713983476e-05), (4, 2.0671141101047397e-05), (8, 2.058199606835842e-05), (16, 2.103153383359313e-05), (32, 2.0512932678684592e-05), (64, 2.0432722521945834e-05), (128, 2.0435894839465618e-05), (256, 2.0461913663893938e-05), (512, 2.086616586893797e-05), (1024, 2.0839914213865995e-05), (2048, 2.055463846772909e-05), (4096, 2.0528619643300772e-05), (8192, 2.0543113350868225e-05), (16384, 2.0482984837144613e-05), (32768, 2.0583916921168566e-05), (65536, 2.0531908376142383e-05), (131072, 2.035309444181621e-05), (262144, 2.0382023649290204e-05), (524288, 2.0366423996165395e-05), (1048576, 3.2363401260226965e-05), (2097152, 4.4756801798939705e-05), (4194304, 6.570504046976566e-05), (8388608, 0.00011119479313492774), (16777216, 0.00019679125398397446), (33554432, 0.0003748144954442978), (67108864, 0.0007024966180324554), (134217728, 0.0013835206627845764), (268435456, 0.002735629677772522), (536870912, 0.0054320812225341795), (1073741824, 0.010754060745239259)]
available_memory_per_device:
14957700710
dot_cost_dict:
((), 'f16'): [(0, 1.662003342062235e-05), (4194304, 2.9675080440938473e-05), (33554432, 2.9678398277610537e-05), (113246208, 3.9767706766724586e-05), (268435456, 2.968066837638617e-05), (524288000, 3.7421355955302715e-05), (905969664, 3.7816702388226986e-05), (1438646272, 5.177216371521354e-05), (2147483648, 5.5833079386502504e-05), (3057647616, 7.949670873586001e-05), (4194304000, 8.927180897444487e-05), (5582618624, 9.796867379918694e-05), (7247757312, 0.00010636699153110385), (9214885888, 0.00013523627337606203), (11509170176, 0.00016890684303318807), (14155776000, 0.0002077480303341872), (17179869184, 0.0002307849936187267), (20606615552, 0.00024195347214117646), (24461180928, 0.0002872120190367069), (28768731136, 0.00033778930702550974), (33554432000, 0.0003939808217238778), (38843449344, 0.0004220617401016342), (44660948992, 0.000485273016801879), (51032096768, 0.000554500074702147), (57982058496, 0.0006300163584024394), (65536000000, 0.0006726105486760374), (73719087104, 0.0007565953922259221), (82556485632, 0.0008472955794937884), (92073361408, 0.0009154096712284546), (102294880256, 0.0010170338225032243), (113246208000, 0.0011259138631180882), (124952510464, 0.0012299516797065735), (137438953472, 0.0013328195438096415), (150730702848, 0.0013959321501028485), (164852924416, 0.0015267194598225326), (179830784000, 0.0016654309189876625), (195689447424, 0.0018122995908172218), (212454080512, 0.0019675585385301943), (230149849088, 0.0021314408253455632), (248801918976, 0.0023041795144823126), (268435456000, 0.002394599299277029), (289075625984, 0.0025787215360230016), (310747594752, 0.0027720480138255705), (333476528128, 0.0029748032263690427), (357287591936, 0.0031872116673377256), (382205952000, 0.0034094978304159257), (408256774144, 0.003641886209287951), (435465224192, 0.0038497791540154204), (463856467968, 0.004100775128833432), (493455671296, 0.004362450205549447), (524288000000, 0.004635026865453007), (556378619904, 0.004918727589833655), (589752696832, 0.005189138051320815), (624435396608, 0.005494305485224802), (660451885056, 0.005811208708702071), (697827328000, 0.006124526470691174), (736586891264, 0.006464702273039226), (776755740672, 0.006817246765960695), (818359042048, 0.0071823808189301055), (861421961216, 0.007560325301421984), (905969664000, 0.007951301082910856), (952027316224, 0.008355529032871244), (999620083712, 0.008773230020777678), (1048773132288, 0.009177684783935547)]
((), 'f32'): [(0, 1.65026867762208e-05), (4194304, 2.9112619813531634e-05), (33554432, 2.9380840715020895e-05), (113246208, 4.260562127456069e-05), (268435456, 4.5800465159118176e-05), (524288000, 7.021648343652487e-05), (905969664, 9.645731188356875e-05), (1438646272, 0.00013177230721339583), (2147483648, 0.00019034423166885972), (3057647616, 0.0002501418930478394), (4194304000, 0.0003141497145406902), (5582618624, 0.00041813327005365864), (7247757312, 0.0005278292810544372), (9214885888, 0.0006554973078891635), (11509170176, 0.0008187003244642079), (14155776000, 0.0009950751761482904), (17179869184, 0.0011966387430826823), (20606615552, 0.0014353237658118207), (24461180928, 0.0016767866690798048), (28768731136, 0.001972064431278872), (33554432000, 0.0022670427424795678), (38843449344, 0.0026243853547629095), (44660948992, 0.0030174338902403046), (51032096768, 0.0034478886309686125), (57982058496, 0.003917449859004693), (65536000000, 0.004365294311303166), (73719087104, 0.004910362420189725), (82556485632, 0.005457679102243471), (92073361408, 0.006086824754989009), (102294880256, 0.006762553250247219), (113246208000, 0.007437059938772465), (124952510464, 0.008205831579110019), (137438953472, 0.009025836299025782), (150730702848, 0.00989872677850615), (164852924416, 0.010826155697537516), (179830784000, 0.011809775736106273), (195689447424, 0.012851239574198817), (212454080512, 0.01395219989180154), (230149849088, 0.015109097353324363), (248801918976, 0.01621807098388672), (268435456000, 0.01731490960685156), (289075625984, 0.018646263828340878), (310747594752, 0.020044172233631533), (333476528128, 0.02151025809549917), (357287591936, 0.023046144686719422), (382205952000, 0.02465345528006794), (408256774144, 0.026333813148320364), (435465224192, 0.028088841564252332), (463856467968, 0.029920163800639492), (493455671296, 0.031829403130257485), (524288000000, 0.03381818282588195), (556378619904, 0.03588812616028853), (589752696832, 0.03804085640625287), (624435396608, 0.04027799683655062), (660451885056, 0.0426011707239574), (697827328000, 0.04501200134124887), (736586891264, 0.04751211196120067), (776755740672, 0.05010312585658844), (818359042048, 0.05278666630018782), (861421961216, 0.05556435656477446), (905969664000, 0.058437819923123996), (952027316224, 0.061408679648012075), (999620083712, 0.06447855901221433), (1048773132288, 0.06764908128850641)]
conv_cost_dict:
[]
op_cost_dict:
[]
--------------------------------------------------
Key: ('default', (1, 4))
all_gather_cost_dict:
(((0, 1, 2, 3),), 'f16'): [(0, 1.70939601957798e-05), (1, 1.7453887267038226e-05), (2, 1.7247628420591354e-05), (4, 3.449525684118271e-05), (8, 6.899051368236542e-05), (16, 0.00010592021862976253), (32, 0.00010700398706831038), (64, 0.00010602961992844938), (128, 0.00010540318908169866), (256, 0.00010474416194483639), (512, 0.00010605694842524827), (1024, 0.00010446473606862128), (2048, 0.00010116296471096575), (4096, 0.00010366912465542554), (8192, 0.0001101514499168843), (16384, 0.00011390287545509638), (32768, 0.0001148005248978734), (65536, 0.00011914229253306986), (131072, 0.000117743038572371), (262144, 0.00012893829261884093), (524288, 0.00014126882888376713), (1048576, 0.00014304794603958726), (2097152, 0.0001709875068627298), (4194304, 0.00019109121058136222), (8388608, 0.0002665654756128788), (16777216, 0.0004154024645686149), (33554432, 0.0007011592388153076), (67108864, 0.0013114716857671738), (134217728, 0.0025184266269207), (268435456, 0.005016602575778961), (536870912, 0.00981605052947998), (1073741824, 0.01963210105895996), (2147483648, 0.03926420211791992)]
(((0, 1, 2, 3),), 'f32'): [(0, 1.7255457350984216e-05), (1, 1.7211015801876783e-05), (2, 1.743549364618957e-05), (4, 3.487098729237914e-05), (8, 6.974197458475828e-05), (16, 0.00011137031833641231), (32, 0.00010452853166498242), (64, 0.00011095058289356528), (128, 0.00011251203250139953), (256, 0.000112298468593508), (512, 0.00010798426228575408), (1024, 0.00010783137986436488), (2048, 0.00011080052354373039), (4096, 0.00011131761129945517), (8192, 0.00011882957187481225), (16384, 0.00011623930186033249), (32768, 0.00011913015623576939), (65536, 0.00012311071623116732), (131072, 0.00012860560673289), (262144, 0.00013244431465864182), (524288, 0.0001475213502999395), (1048576, 0.00017200398724526167), (2097152, 0.0002099429257214069), (4194304, 0.0002809474244713783), (8388608, 0.000425062607973814), (16777216, 0.0007132655009627342), (33554432, 0.0013249162584543228), (67108864, 0.0025407448410987854), (134217728, 0.004967018961906433), (268435456, 0.009841054677963257), (536870912, 0.019682109355926514), (1073741824, 0.03936421871185303)]
(((0, 2), (1, 3)), 'f16'): [(0, 1.7684942577034235e-05), (1, 1.76884641405195e-05), (2, 3.5376928281039e-05), (4, 6.73655595164746e-05), (8, 6.729562301188707e-05), (16, 6.701415986754e-05), (32, 6.728197331540287e-05), (64, 6.898189894855022e-05), (128, 6.810398190282285e-05), (256, 6.80220837239176e-05), (512, 6.73359609209001e-05), (1024, 6.774807116016746e-05), (2048, 6.886626943014562e-05), (4096, 6.861175643280149e-05), (8192, 7.04638659954071e-05), (16384, 7.539885700680315e-05), (32768, 7.539871148765087e-05), (65536, 7.258830009959638e-05), (131072, 8.46889743115753e-05), (262144, 9.276971104554832e-05), (524288, 0.00010510499123483896), (1048576, 0.0001287042978219688), (2097152, 0.00017884152475744486), (4194304, 0.00027003593277186155), (8388608, 0.00046513671986758704), (16777216, 0.0008494053035974503), (33554432, 0.001622355543076992), (67108864, 0.003144560381770134), (134217728, 0.006235022097826004), (268435456, 0.012347601354122164), (536870912, 0.024695202708244327), (1073741824, 0.049098539352416995), (2147483648, 0.09819707870483399)]
(((0, 2), (1, 3)), 'f32'): [(0, 1.6880163457244635e-05), (1, 1.6891804989427328e-05), (2, 3.3783609978854656e-05), (4, 6.168422987684608e-05), (8, 6.612457218579948e-05), (16, 6.573172868229449e-05), (32, 6.485066842287779e-05), (64, 6.401297287084162e-05), (128, 6.424152525141835e-05), (256, 6.30159629508853e-05), (512, 6.118984310887754e-05), (1024, 6.867334013804793e-05), (2048, 6.596674211323261e-05), (4096, 7.130031008273363e-05), (8192, 7.417629240080714e-05), (16384, 7.57130328565836e-05), (32768, 8.451734902337193e-05), (65536, 8.724178769625723e-05), (131072, 9.241071529686451e-05), (262144, 0.00010216652299277483), (524288, 0.00012565404176712036), (1048576, 0.00017468159785494208), (2097152, 0.000267957104369998), (4194304, 0.0004611399490386248), (8388608, 0.0008461740799248219), (16777216, 0.0016230661422014236), (33554432, 0.0031692590564489365), (67108864, 0.006236206740140915), (134217728, 0.012335486710071564), (268435456, 0.02457784116268158), (536870912, 0.049081611633300784), (1073741824, 0.09802756309509278)]
(((0, 1), (2, 3)), 'f16'): [(0, 1.75797613337636e-05), (1, 1.7586105968803167e-05), (2, 3.5172211937606335e-05), (4, 6.524543277919292e-05), (8, 6.501286407001317e-05), (16, 6.137479795143008e-05), (32, 6.736157229170203e-05), (64, 6.809557089582086e-05), (128, 6.633691373281181e-05), (256, 6.656840560026467e-05), (512, 6.574249709956348e-05), (1024, 6.771099288016558e-05), (2048, 6.826157914474607e-05), (4096, 6.5494270529598e-05), (8192, 7.023254875093699e-05), (16384, 7.412608829326928e-05), (32768, 7.690730853937566e-05), (65536, 8.522314601577818e-05), (131072, 8.753512520343065e-05), (262144, 9.420295828022063e-05), (524288, 0.00010170278255827725), (1048576, 0.00012530735693871975), (2097152, 0.00017581332940608263), (4194304, 0.0002703442005440593), (8388608, 0.00045997207053005695), (16777216, 0.0008545066229999066), (33554432, 0.0016274424269795418), (67108864, 0.0031454842537641525), (134217728, 0.006226658821105957), (268435456, 0.012343332171440125), (536870912, 0.02459174394607544), (1073741824, 0.04907839298248291), (2147483648, 0.09811558723449706)]
(((0, 1), (2, 3)), 'f32'): [(0, 1.7662940081208944e-05), (1, 1.8019054550677538e-05), (2, 3.6038109101355076e-05), (4, 6.575355655513704e-05), (8, 6.590050179511309e-05), (16, 6.591831333935261e-05), (32, 6.497735739685595e-05), (64, 6.491993553936481e-05), (128, 6.598423351533711e-05), (256, 6.619622581638396e-05), (512, 5.284187500365079e-05), (1024, 5.122018046677113e-05), (2048, 5.2149436669424176e-05), (4096, 6.244506221264601e-05), (8192, 7.042541983537376e-05), (16384, 7.138372166082263e-05), (32768, 7.713559898547828e-05), (65536, 8.655560668557882e-05), (131072, 8.684990461915731e-05), (262144, 0.00010126779670827091), (524288, 0.0001275784452445805), (1048576, 0.00017593213124200702), (2097152, 0.0002694829599931836), (4194304, 0.00046353298239409924), (8388608, 0.0008436217904090881), (16777216, 0.00161033496260643), (33554432, 0.0031523313373327255), (67108864, 0.006213482469320297), (134217728, 0.01240132749080658), (268435456, 0.024596959352493286), (536870912, 0.049146723747253415), (1073741824, 0.09731228351593017)]
(((0,), (1,), (2,), (3,)), 'f16'): [(0, 1.7260346794500947e-05), (1, 2.0049454178661108e-05), (2, 2.0432722521945834e-05), (4, 2.034418866969645e-05), (8, 2.0297069568187e-05), (16, 2.045219298452139e-05), (32, 2.039640094153583e-05), (64, 2.0458566723391414e-05), (128, 2.015172503888607e-05), (256, 2.0331150153651834e-05), (512, 2.0748848328366876e-05), (1024, 2.0397186744958162e-05), (2048, 2.068965113721788e-05), (4096, 2.0837003830820322e-05), (8192, 2.0892650354653597e-05), (16384, 2.1143205231055617e-05), (32768, 2.1187006495893e-05), (65536, 2.0610576029866934e-05), (131072, 2.0809005945920944e-05), (262144, 2.0866747945547104e-05), (524288, 2.0795443560928106e-05), (1048576, 2.0929350284859538e-05), (2097152, 3.221730003133416e-05), (4194304, 4.4264248572289944e-05), (8388608, 6.670085713267326e-05), (16777216, 0.00010816985741257668), (33554432, 0.00019230879843235016), (67108864, 0.0003657527267932892), (134217728, 0.0006975904107093811), (268435456, 0.0013951808214187622), (536870912, 0.00271819531917572), (1073741824, 0.005396652221679688), (2147483648, 0.010731387138366699)]
(((0,), (1,), (2,), (3,)), 'f32'): [(0, 1.7400860087946057e-05), (1, 2.0320876501500607e-05), (2, 2.095571835525334e-05), (4, 2.0683568436652422e-05), (8, 2.0613078959286213e-05), (16, 2.0642008166760206e-05), (32, 2.0620267605409026e-05), (64, 2.074567601084709e-05), (128, 2.0737119484692812e-05), (256, 2.1012878278270364e-05), (512, 2.098074764944613e-05), (1024, 2.0907347789034247e-05), (2048, 2.1073559764772654e-05), (4096, 2.096578828059137e-05), (8192, 2.1081097656860948e-05), (16384, 2.0744570065289736e-05), (32768, 2.075923839583993e-05), (65536, 2.057230449281633e-05), (131072, 2.0399835193529725e-05), (262144, 2.023371052928269e-05), (524288, 2.0522915292531252e-05), (1048576, 3.2252923119813204e-05), (2097152, 4.4133746996521957e-05), (4194304, 6.573018617928028e-05), (8388608, 0.00011007627472281456), (16777216, 0.00019230879843235016), (33554432, 0.00036618486046791077), (67108864, 0.0006964802742004395), (134217728, 0.0013779550790786743), (268435456, 0.002718716859817505), (536870912, 0.005406665802001953), (1073741824, 0.010724997520446778)]
all_reduce_cost_dict:
(((0, 1, 2, 3),), 'f16'): [(0, 9.980122558772564e-05), (1, 9.587925160303712e-05), (2, 0.00010110760922543705), (4, 0.00010221204138360918), (8, 0.00010074247256852685), (16, 0.00010120504884980619), (32, 0.00010097370250150561), (64, 0.00010165318963117898), (128, 0.00010182717232964933), (256, 0.00010216876398772001), (512, 0.00010828921222127975), (1024, 0.00010755922994576395), (2048, 0.00010906753595918418), (4096, 0.00011479418026283382), (8192, 0.0001156956423074007), (16384, 0.00011182768503203987), (32768, 0.00011957515380345286), (65536, 0.00012477653217501938), (131072, 0.00013352936366572976), (262144, 0.00014578798436559737), (524288, 0.00015321632963605225), (1048576, 0.0001865104422904551), (2097152, 0.0002174333203583956), (4194304, 0.0002892331685870886), (8388608, 0.0004545249976217747), (16777216, 0.0007523270323872566), (33554432, 0.0013355705887079239), (67108864, 0.0026067905128002167), (134217728, 0.005042359232902527), (268435456, 0.009952276945114136), (536870912, 0.019862866401672362), (1073741824, 0.03948831558227539), (2147483648, 0.07886075973510742)]
(((0, 1, 2, 3),), 'f32'): [(0, 0.00010513103916309774), (1, 0.00010203401325270534), (2, 0.00010109235881827773), (4, 0.00010164169361814857), (8, 0.00010295520769432187), (16, 0.00010500822099857032), (32, 0.00010207839659415185), (64, 9.984939242713153e-05), (128, 9.952153777703643e-05), (256, 9.703502291813493e-05), (512, 9.971606777980924e-05), (1024, 0.00010406170622445644), (2048, 0.00010630107135511936), (4096, 0.00010660602129064499), (8192, 0.00011168018681928515), (16384, 0.00011576790711842476), (32768, 0.00011461297981441022), (65536, 0.00012335588689893484), (131072, 0.00014224648475646973), (262144, 0.00015365093713626266), (524288, 0.00018289953004568815), (1048576, 0.00021786836441606283), (2097152, 0.00028759916312992573), (4194304, 0.00044103851541876793), (8388608, 0.0007451726123690604), (16777216, 0.0013380814343690872), (33554432, 0.002533853054046631), (67108864, 0.004971444606781006), (134217728, 0.009875476360321045), (268435456, 0.019606852531433107), (536870912, 0.039213705062866214), (1073741824, 0.07842741012573243)]
(((0, 2), (1, 3)), 'f16'): [(0, 6.947742076590657e-05), (1, 6.810799823142588e-05), (2, 6.704905536025763e-05), (4, 6.69339788146317e-05), (8, 6.730662425979972e-05), (16, 6.670085713267326e-05), (32, 6.637643673457205e-05), (64, 6.588676478713751e-05), (128, 6.77123898640275e-05), (256, 6.676840712316334e-05), (512, 6.673822645097971e-05), (1024, 6.635402678512037e-05), (2048, 6.736445357091725e-05), (4096, 6.728857988491654e-05), (8192, 7.497347542084754e-05), (16384, 7.588273729197681e-05), (32768, 7.938180351629853e-05), (65536, 8.957812679000199e-05), (131072, 9.548652451485395e-05), (262144, 0.00010651577031239867), (524288, 0.0001258089323528111), (1048576, 0.0001743332832120359), (2097152, 0.0002676446456462145), (4194304, 0.00045255757868289953), (8388608, 0.0008365423418581486), (16777216, 0.001574961468577385), (33554432, 0.003074437379837036), (67108864, 0.0060828328132629395), (134217728, 0.012103527784347534), (268435456, 0.024071335792541504), (536870912, 0.0479475736618042), (1073741824, 0.09576416015625), (2147483648, 0.19148764610290528)]
(((0, 2), (1, 3)), 'f32'): [(0, 6.499787559732795e-05), (1, 6.463861791417003e-05), (2, 6.468628998845816e-05), (4, 6.594034493900836e-05), (8, 6.581019260920584e-05), (16, 6.44249957986176e-05), (32, 6.625612149946392e-05), (64, 6.527328514494002e-05), (128, 6.65838597342372e-05), (256, 6.643787492066622e-05), (512, 6.506699719466269e-05), (1024, 6.605157977901399e-05), (2048, 6.588699761778116e-05), (4096, 7.628221646882594e-05), (8192, 7.617572555318475e-05), (16384, 7.754514808766544e-05), (32768, 8.783693192526698e-05), (65536, 9.308478911407292e-05), (131072, 0.00010515085887163877), (262144, 0.0001261454599443823), (524288, 0.0001728871138766408), (1048576, 0.00026619306299835443), (2097152, 0.00045573781244456773), (4194304, 0.0008255895227193832), (8388608, 0.0015856781974434853), (16777216, 0.003087917342782021), (33554432, 0.006067413836717606), (67108864, 0.01205626130104065), (134217728, 0.02401033043861389), (268435456, 0.04791247844696045), (536870912, 0.09572179317474365), (1073741824, 0.19134087562561036)]
(((0, 1), (2, 3)), 'f16'): [(0, 6.443075835704803e-05), (1, 6.594776641577482e-05), (2, 6.559499888680875e-05), (4, 6.594788283109665e-05), (8, 6.577774183824658e-05), (16, 6.707425927743316e-05), (32, 6.604357622563839e-05), (64, 6.453238893300295e-05), (128, 6.45069230813533e-05), (256, 6.492863758467138e-05), (512, 6.353261414915323e-05), (1024, 6.337545346468687e-05), (2048, 6.257131462916732e-05), (4096, 6.685586413368583e-05), (8192, 7.444250513799489e-05), (16384, 7.401956827379763e-05), (32768, 7.890607230365276e-05), (65536, 8.626768249087036e-05), (131072, 9.193431469611825e-05), (262144, 0.00010221923002973199), (524288, 0.00012300614616833627), (1048576, 0.00017134618246927857), (2097152, 0.0002648357767611742), (4194304, 0.0004595078062266111), (8388608, 0.0008376589976251125), (16777216, 0.0015804516151547432), (33554432, 0.0030927471816539764), (67108864, 0.0060901157557964325), (134217728, 0.012070246040821077), (268435456, 0.024050667881965637), (536870912, 0.04795651435852051), (1073741824, 0.09576926231384278), (2147483648, 0.19123666286468505)]
(((0, 1), (2, 3)), 'f32'): [(0, 6.632308941334486e-05), (1, 6.730580935254693e-05), (2, 6.53555616736412e-05), (4, 6.648839917033911e-05), (8, 6.217439658939838e-05), (16, 6.356844096444547e-05), (32, 6.404775194823742e-05), (64, 6.455439142882824e-05), (128, 6.337990635074675e-05), (256, 6.274876068346202e-05), (512, 6.428666529245675e-05), (1024, 6.452019442804158e-05), (2048, 6.580987246707082e-05), (4096, 7.398295565508306e-05), (8192, 7.338912109844387e-05), (16384, 7.765792543068528e-05), (32768, 8.401920786127448e-05), (65536, 9.070395026355982e-05), (131072, 0.00010339022264815867), (262144, 0.00012195488670840858), (524288, 0.0001684824819676578), (1048576, 0.00026423484086990356), (2097152, 0.00044910795986652374), (4194304, 0.0008237184956669807), (8388608, 0.0015825917944312096), (16777216, 0.0030995458364486694), (33554432, 0.006098896265029907), (67108864, 0.012049421668052675), (134217728, 0.023996785283088684), (268435456, 0.04792289733886719), (536870912, 0.09570529460906983), (1073741824, 0.19141058921813967)]
(((0,), (1,), (2,), (3,)), 'f16'): [(0, 1.7514510545879602e-05), (1, 2.0423176465556026e-05), (2, 2.056919038295746e-05), (4, 2.0544073777273297e-05), (8, 2.042893902398646e-05), (16, 2.051412593573332e-05), (32, 2.042617416009307e-05), (64, 2.035417128354311e-05), (128, 2.0184030290693045e-05), (256, 2.0244362531229854e-05), (512, 2.018790110014379e-05), (1024, 2.0091829355806112e-05), (2048, 2.0139559637755156e-05), (4096, 1.9979604985564947e-05), (8192, 1.9987113773822784e-05), (16384, 2.0237872377038002e-05), (32768, 2.0434963516891003e-05), (65536, 2.0610459614545107e-05), (131072, 2.0468170987442136e-05), (262144, 2.012838376685977e-05), (524288, 1.997343497350812e-05), (1048576, 2.1528161596506834e-05), (2097152, 3.9619975723326206e-05), (4194304, 5.0199683755636215e-05), (8388608, 6.631296128034592e-05), (16777216, 0.00010877661406993867), (33554432, 0.0001934841275215149), (67108864, 0.0003650002181529999), (134217728, 0.0007031932473182678), (268435456, 0.0013846606016159058), (536870912, 0.00273284912109375), (1073741824, 0.005395030975341797), (2147483648, 0.010712814331054688)]
(((0,), (1,), (2,), (3,)), 'f32'): [(0, 1.7113139620050788e-05), (1, 1.9905826775357127e-05), (2, 2.005300484597683e-05), (4, 2.0041188690811396e-05), (8, 2.033411874435842e-05), (16, 2.0119186956435442e-05), (32, 2.0007399143651128e-05), (64, 2.0093226339668036e-05), (128, 2.011476317420602e-05), (256, 2.0287989173084497e-05), (512, 2.0055565983057022e-05), (1024, 2.003347617574036e-05), (2048, 2.019974635913968e-05), (4096, 1.993885962292552e-05), (8192, 2.0008475985378027e-05), (16384, 2.007716102525592e-05), (32768, 1.9983970560133457e-05), (65536, 1.9944418454542756e-05), (131072, 2.0340405171737075e-05), (262144, 2.0625244360417128e-05), (524288, 2.1347368601709604e-05), (1048576, 3.42406565323472e-05), (2097152, 4.45640180259943e-05), (4194304, 6.824731826782227e-05), (8388608, 0.00011265184730291368), (16777216, 0.0001946520060300827), (33554432, 0.00036466121673583984), (67108864, 0.0007043331861495972), (134217728, 0.0013884156942367554), (268435456, 0.002734804153442383), (536870912, 0.005393886566162109), (1073741824, 0.010737133026123048)]
all_to_all_cost_dict:
(((0, 1, 2, 3),), 'f16'): [(0, 1.7005077097564936e-05), (1, 1.7184909665957093e-05), (2, 1.6625941498205066e-05), (4, 1.6926846001297235e-05), (8, 1.69435515999794e-05), (16, 3.38871031999588e-05), (32, 6.77742063999176e-05), (64, 0.00013231183402240276), (128, 0.00013004225911572576), (256, 0.00011948123574256896), (512, 0.00011640909360721709), (1024, 0.00011605507461354135), (2048, 0.00012193244765512647), (4096, 0.0001326177443843335), (8192, 0.00012354535283520818), (16384, 0.00011780089698731899), (32768, 0.00012036357657052578), (65536, 0.0001223065482918173), (131072, 0.00013341705198399723), (262144, 0.0001298657152801752), (524288, 0.00013374764239415526), (1048576, 0.0001407044765073806), (2097152, 0.00014820409705862403), (4194304, 0.00015232455916702747), (8388608, 0.00018219579942524433), (16777216, 0.0002481718547642231), (33554432, 0.00040781404823064804), (67108864, 0.0007224585860967636), (134217728, 0.0014043450355529785), (268435456, 0.002758920192718506), (536870912, 0.005377739667892456), (1073741824, 0.010755479335784912), (2147483648, 0.021254873275756835)]
(((0, 1, 2, 3),), 'f32'): [(0, 1.670458004809916e-05), (1, 1.6775360563769937e-05), (2, 1.6805133782327175e-05), (4, 1.677079126238823e-05), (8, 1.676596002653241e-05), (16, 3.353192005306482e-05), (32, 6.706384010612965e-05), (64, 0.00012561865150928497), (128, 0.00012643300578929484), (256, 0.00013091095024719834), (512, 0.00012972686090506613), (1024, 0.00013054534792900085), (2048, 0.00013000742183066905), (4096, 0.00013093571760691702), (8192, 0.0001297792769037187), (16384, 0.00013105032849125564), (32768, 0.0001300223229918629), (65536, 0.00013314513489603996), (131072, 0.00013309731730259955), (262144, 0.00013341638259589672), (524288, 0.00014172671944834292), (1048576, 0.00014797330368310213), (2097152, 0.000162479467689991), (4194304, 0.00019547133706510067), (8388608, 0.00027280859649181366), (16777216, 0.0004294943064451218), (33554432, 0.0007551033049821854), (67108864, 0.0014059394598007202), (134217728, 0.002757139503955841), (268435456, 0.005440235137939453), (536870912, 0.010824894905090332), (1073741824, 0.021520590782165526)]
(((0, 2), (1, 3)), 'f16'): [(0, 1.690667704679072e-05), (1, 1.6801990568637848e-05), (2, 1.686345785856247e-05), (4, 3.372691571712494e-05), (8, 6.745383143424988e-05), (16, 8.166642510332167e-05), (32, 8.33595113363117e-05), (64, 7.848258246667683e-05), (128, 8.427977445535362e-05), (256, 7.303673191927373e-05), (512, 8.292309939861298e-05), (1024, 8.387741399928927e-05), (2048, 8.103184518404305e-05), (4096, 8.470891043543816e-05), (8192, 8.364827954210341e-05), (16384, 7.065679528750479e-05), (32768, 7.014680886641145e-05), (65536, 6.894097896292806e-05), (131072, 6.970419781282544e-05), (262144, 8.53360106702894e-05), (524288, 9.151693666353822e-05), (1048576, 9.698621579445899e-05), (2097152, 0.00013647961895912886), (4194304, 0.00019460683688521385), (8388608, 0.00036272010765969753), (16777216, 0.0006408980116248131), (33554432, 0.0012372694909572601), (67108864, 0.0024119168519973755), (134217728, 0.004823833703994751), (268435456, 0.00947299599647522), (536870912, 0.01894599199295044), (1073741824, 0.03774507045745849), (2147483648, 0.0754704475402832)]
(((0, 2), (1, 3)), 'f32'): [(0, 1.8280174117535353e-05), (1, 1.8024962628260255e-05), (2, 1.7689104424789548e-05), (4, 3.5378208849579096e-05), (8, 7.075641769915819e-05), (16, 8.332228753715754e-05), (32, 8.130123023875058e-05), (64, 7.964076939970255e-05), (128, 7.992048631422222e-05), (256, 8.017502841539681e-05), (512, 7.333621033467352e-05), (1024, 7.270422065630555e-05), (2048, 7.147568976506591e-05), (4096, 7.199335959739983e-05), (8192, 7.073767483234406e-05), (16384, 7.265762542374432e-05), (32768, 7.060999632813036e-05), (65536, 7.317663403227925e-05), (131072, 8.513202192261815e-05), (262144, 9.120628237724304e-05), (524288, 9.866547770798206e-05), (1048576, 0.00013421580661088228), (2097152, 0.0001932312734425068), (4194304, 0.0003636730834841728), (8388608, 0.0006423117592930794), (16777216, 0.0012333160266280174), (33554432, 0.002431081607937813), (67108864, 0.004794184118509293), (134217728, 0.009498052299022675), (268435456, 0.018950045108795166), (536870912, 0.03781259059906006), (1073741824, 0.07557864189147949)]
(((0, 1), (2, 3)), 'f16'): [(0, 1.7107842722907662e-05), (1, 1.705993781797588e-05), (2, 1.693089143373072e-05), (4, 3.386178286746144e-05), (8, 6.772356573492289e-05), (16, 7.789445226080716e-05), (32, 7.118983194231987e-05), (64, 7.070120773278177e-05), (128, 7.114690379239619e-05), (256, 7.091506267897785e-05), (512, 7.173744961619377e-05), (1024, 7.154268678277731e-05), (2048, 7.128121796995401e-05), (4096, 7.099309004843235e-05), (8192, 7.059570634737611e-05), (16384, 7.750626537017524e-05), (32768, 7.800193270668387e-05), (65536, 8.332912693731487e-05), (131072, 8.309289114549756e-05), (262144, 8.652746328152716e-05), (524288, 9.170046541839838e-05), (1048576, 0.00010748382192105055), (2097152, 0.00013652537018060684), (4194304, 0.0001992520410567522), (8388608, 0.00036924262531101704), (16777216, 0.0006510401144623756), (33554432, 0.0012400588020682335), (67108864, 0.0024070553481578827), (134217728, 0.004791393876075745), (268435456, 0.009496711194515228), (536870912, 0.01890304684638977), (1073741824, 0.03769595623016357), (2147483648, 0.07527847290039062)]
(((0, 1), (2, 3)), 'f32'): [(0, 1.7657788703218102e-05), (1, 1.774434349499643e-05), (2, 1.750685623846948e-05), (4, 3.501371247693896e-05), (8, 7.002742495387793e-05), (16, 7.884632213972509e-05), (32, 7.820824976079166e-05), (64, 7.49194878153503e-05), (128, 8.409438305534422e-05), (256, 8.563726441934705e-05), (512, 8.351611904799938e-05), (1024, 8.408827125094831e-05), (2048, 8.550900383852422e-05), (4096, 8.627769420854747e-05), (8192, 8.306954987347126e-05), (16384, 8.596506086178124e-05), (32768, 8.529413025826216e-05), (65536, 8.573391824029386e-05), (131072, 8.697278099134564e-05), (262144, 9.204560774378479e-05), (524288, 0.00010519433999434115), (1048576, 0.00013270555064082146), (2097152, 0.00019335781689733264), (4194304, 0.00036160298623144627), (8388608, 0.0006426107138395309), (16777216, 0.001245332881808281), (33554432, 0.002412937581539154), (67108864, 0.004774987697601318), (134217728, 0.009496994316577911), (268435456, 0.018943950533866882), (536870912, 0.03784489631652832), (1073741824, 0.07558145523071289)]
(((0,), (1,), (2,), (3,)), 'f16'): [(0, 1.717387931421399e-05), (1, 2.0733714336529374e-05), (2, 2.018327359110117e-05), (4, 2.032495103776455e-05), (8, 2.0131177734583616e-05), (16, 2.0321807824075222e-05), (32, 2.0467094145715237e-05), (64, 2.0452134776860476e-05), (128, 2.038883394561708e-05), (256, 2.045181463472545e-05), (512, 2.046633744612336e-05), (1024, 2.041662810370326e-05), (2048, 2.073249197565019e-05), (4096, 2.0444567780941725e-05), (8192, 2.057355595752597e-05), (16384, 2.0251231035217643e-05), (32768, 2.0581792341545224e-05), (65536, 2.0650913938879967e-05), (131072, 2.0422565285116434e-05), (262144, 2.0697305444628e-05), (524288, 2.0716659491881728e-05), (1048576, 2.0647974452003837e-05), (2097152, 3.3166666980832815e-05), (4194304, 4.4937594793736935e-05), (8388608, 6.596860475838184e-05), (16777216, 0.00010811397805809975), (33554432, 0.00019620638340711594), (67108864, 0.0003663897514343261), (134217728, 0.0007002390921115875), (268435456, 0.001400478184223175), (536870912, 0.002721443772315979), (1073741824, 0.005394792556762696), (2147483648, 0.010725641250610351)]
(((0,), (1,), (2,), (3,)), 'f32'): [(0, 1.720854197628796e-05), (1, 2.0133476937189698e-05), (2, 2.01739021576941e-05), (4, 2.0243460312485695e-05), (8, 2.0215753465890884e-05), (16, 2.0134320948272943e-05), (32, 2.045129076577723e-05), (64, 2.0066945580765605e-05), (128, 2.0124251022934914e-05), (256, 2.0088482415303588e-05), (512, 2.007686998695135e-05), (1024, 2.0069011952728033e-05), (2048, 2.007954753935337e-05), (4096, 2.01231159735471e-05), (8192, 2.013694029301405e-05), (16384, 2.0122184650972486e-05), (32768, 2.0186707843095064e-05), (65536, 2.000507083721459e-05), (131072, 2.022329135797918e-05), (262144, 2.0724284695461392e-05), (524288, 2.076779492199421e-05), (1048576, 3.265670966356993e-05), (2097152, 4.458264447748661e-05), (4194304, 6.56286720186472e-05), (8388608, 0.0001079309731721878), (16777216, 0.0001922696828842163), (33554432, 0.00036201439797878265), (67108864, 0.0006978921592235565), (134217728, 0.0013715177774429321), (268435456, 0.002719387412071228), (536870912, 0.0054044008255004885), (1073741824, 0.010731911659240723)]
reduce_scatter_cost_dict:
(((0, 1, 2, 3),), 'f16'): [(0, 1.7271871911361814e-05), (1, 1.744343899190426e-05), (2, 1.749600050970912e-05), (4, 3.499200101941824e-05), (8, 6.998400203883648e-05), (16, 0.00011013561743311585), (32, 0.00011103803990408778), (64, 0.00011218764120712875), (128, 0.0001056346227414906), (256, 9.378354297950864e-05), (512, 9.964150376617908e-05), (1024, 0.00010847896919585764), (2048, 0.00011178565910086034), (4096, 0.00010944635141640902), (8192, 0.00011016998905688526), (16384, 0.00011449921294115484), (32768, 0.00011638933210633695), (65536, 0.00012057070853188633), (131072, 0.00012468869681470096), (262144, 0.00012639310443773866), (524288, 0.00014654622646048665), (1048576, 0.00015515738050453365), (2097152, 0.00017683272017166018), (4194304, 0.00022633990738540888), (8388608, 0.0003142673522233963), (16777216, 0.0004946459084749222), (33554432, 0.0008472967892885208), (67108864, 0.0014954768121242523), (134217728, 0.0029295869171619415), (268435456, 0.005654618144035339), (536870912, 0.011309236288070679), (1073741824, 0.022618472576141357), (2147483648, 0.045236945152282715)]
(((0, 1, 2, 3),), 'f32'): [(0, 1.68186379596591e-05), (1, 1.700306893326342e-05), (2, 1.6887177480384707e-05), (4, 3.3774354960769415e-05), (8, 6.754870992153883e-05), (16, 0.00010556794586591423), (32, 0.00010563514661043884), (64, 0.00010623576235957444), (128, 0.00010554611799307168), (256, 0.00010485513485036789), (512, 0.00010518010822124778), (1024, 0.00010410961112938823), (2048, 0.00010577274952083826), (4096, 0.00010582516551949084), (8192, 0.0001135289785452187), (16384, 0.00011478475062176587), (32768, 0.00011799926869571209), (65536, 0.00012322980910539627), (131072, 0.0001227374596055597), (262144, 0.00014777638716623187), (524288, 0.00014972087228670716), (1048576, 0.0001788257504813373), (2097152, 0.00022273068316280842), (4194304, 0.00030253035947680473), (8388608, 0.0004677590914070607), (16777216, 0.0007918020710349083), (33554432, 0.0013889670372009277), (67108864, 0.002749856561422348), (134217728, 0.005372405052185059), (268435456, 0.010744810104370117), (536870912, 0.021489620208740234), (1073741824, 0.04297924041748047)]
(((0, 2), (1, 3)), 'f16'): [(0, 1.7153273802250624e-05), (1, 1.7238373402506113e-05), (2, 3.4476746805012226e-05), (4, 5.350899300538003e-05), (8, 5.392907769419253e-05), (16, 6.0417427448555834e-05), (32, 5.5213749874383204e-05), (64, 5.602929741144181e-05), (128, 5.960397538729012e-05), (256, 6.741919787600636e-05), (512, 6.753945490345359e-05), (1024, 6.446984480135143e-05), (2048, 6.549435784108937e-05), (4096, 6.494743865914643e-05), (8192, 6.939005106687546e-05), (16384, 7.460892084054649e-05), (32768, 7.576346979476511e-05), (65536, 7.796558202244341e-05), (131072, 8.651890675537288e-05), (262144, 9.384169243276119e-05), (524288, 0.0001085686672013253), (1048576, 0.00014407761045731604), (2097152, 0.00022305012680590153), (4194304, 0.00036341382656246424), (8388608, 0.0006506652571260929), (16777216, 0.0012243213132023811), (33554432, 0.0023406967520713806), (67108864, 0.004513729363679886), (134217728, 0.00850645825266838), (268435456, 0.015751004219055176), (536870912, 0.031100451946258545), (1073741824, 0.05871906280517578), (2147483648, 0.11577873229980469)]
(((0, 2), (1, 3)), 'f32'): [(0, 1.6785546904429793e-05), (1, 1.681435969658196e-05), (2, 3.362871939316392e-05), (4, 4.981318488717079e-05), (8, 5.178913124836981e-05), (16, 4.9891386879608035e-05), (32, 5.044287536293268e-05), (64, 4.9555732402950525e-05), (128, 5.0411035772413015e-05), (256, 5.1037612138316035e-05), (512, 4.91415266878903e-05), (1024, 4.883727524429559e-05), (2048, 4.901798092760146e-05), (4096, 6.056317943148314e-05), (8192, 5.794959724880755e-05), (16384, 5.833452451042831e-05), (32768, 7.860962068662047e-05), (65536, 8.469034219160676e-05), (131072, 8.932116907089949e-05), (262144, 9.771177428774536e-05), (524288, 0.00013922527432441711), (1048576, 0.00021199422189965844), (2097152, 0.00034656317438930273), (4194304, 0.0006201204378157854), (8388608, 0.001155739650130272), (16777216, 0.0022214436903595924), (33554432, 0.00428306870162487), (67108864, 0.008395016193389893), (134217728, 0.016489580273628235), (268435456, 0.03268122673034668), (536870912, 0.06535093784332276), (1073741824, 0.1298403263092041)]
(((0, 1), (2, 3)), 'f16'): [(0, 1.7081067198887467e-05), (1, 1.7143640434369445e-05), (2, 3.428728086873889e-05), (4, 5.290756234899163e-05), (8, 5.0496484618633986e-05), (16, 5.1790208090096705e-05), (32, 5.3460418712347746e-05), (64, 5.523857544176281e-05), (128, 5.260563921183348e-05), (256, 5.2888965001329786e-05), (512, 5.596538539975881e-05), (1024, 6.698566721752286e-05), (2048, 5.314729060046375e-05), (4096, 5.166759365238249e-05), (8192, 6.688115536235273e-05), (16384, 6.073879194445908e-05), (32768, 6.123533239588141e-05), (65536, 7.701735012233257e-05), (131072, 8.733727736398578e-05), (262144, 9.26546344999224e-05), (524288, 0.00010846037184819578), (1048576, 0.00014296939480118454), (2097152, 0.00022020476171746847), (4194304, 0.0003647505072876811), (8388608, 0.0006616236642003059), (16777216, 0.001219616737216711), (33554432, 0.0023636305704712868), (67108864, 0.004477227106690407), (134217728, 0.008435580879449844), (268435456, 0.015236616134643553), (536870912, 0.029457628726959225), (1073741824, 0.05623476505279541), (2147483648, 0.11040878295898438)]
(((0, 1), (2, 3)), 'f32'): [(0, 1.698607229627669e-05), (1, 1.7023063264787197e-05), (2, 3.4046126529574394e-05), (4, 6.560227484442294e-05), (8, 6.741206743754447e-05), (16, 6.559689063578844e-05), (32, 6.689911242574453e-05), (64, 6.834350642748177e-05), (128, 6.670632865279913e-05), (256, 6.54681061860174e-05), (512, 6.568001117557287e-05), (1024, 6.644154200330377e-05), (2048, 6.586770177818835e-05), (4096, 7.122592069208622e-05), (8192, 7.611812907271087e-05), (16384, 7.65258155297488e-05), (32768, 7.810568786226213e-05), (65536, 8.80522420629859e-05), (131072, 9.281100938096642e-05), (262144, 0.00010773693793453276), (524288, 0.00014060104149393737), (1048576, 0.00021198001923039556), (2097152, 0.00034696992952376604), (4194304, 0.0006225889082998037), (8388608, 0.0011617662385106087), (16777216, 0.0022145677357912064), (33554432, 0.004282524809241295), (67108864, 0.00835343822836876), (134217728, 0.016413524746894836), (268435456, 0.032542139291763306), (536870912, 0.06495256423950195), (1073741824, 0.12938883304595947)]
(((0,), (1,), (2,), (3,)), 'f16'): [(0, 1.7750775441527367e-05), (1, 2.1043349988758564e-05), (2, 2.107064938172698e-05), (4, 2.0480307284742594e-05), (8, 2.0512554328888655e-05), (16, 2.0447623683139682e-05), (32, 2.051892806775868e-05), (64, 2.0284816855564713e-05), (128, 2.0359992049634457e-05), (256, 2.055562799796462e-05), (512, 2.025821595452726e-05), (1024, 2.0450097508728504e-05), (2048, 2.032698830589652e-05), (4096, 2.039247192442417e-05), (8192, 2.0797160686925054e-05), (16384, 2.0823354134336114e-05), (32768, 2.1069194190204144e-05), (65536, 2.0949315512552857e-05), (131072, 2.084995503537357e-05), (262144, 2.0642997696995735e-05), (524288, 2.0693696569651365e-05), (1048576, 2.0525738364085555e-05), (2097152, 3.25917499139905e-05), (4194304, 4.47278143838048e-05), (8388608, 6.53369352221489e-05), (16777216, 0.00011126091703772545), (33554432, 0.00019212346524000168), (67108864, 0.00036024488508701324), (134217728, 0.0006956197321414948), (268435456, 0.0013691708445549011), (536870912, 0.0027128010988235474), (1073741824, 0.005389523506164551), (2147483648, 0.010712552070617675)]
(((0,), (1,), (2,), (3,)), 'f32'): [(0, 1.7463258700445294e-05), (1, 2.039872924797237e-05), (2, 2.0523410057649016e-05), (4, 2.0563311409205198e-05), (8, 2.0523526472970843e-05), (16, 2.0577776012942195e-05), (32, 2.090536872856319e-05), (64, 2.0652543753385544e-05), (128, 2.0685663912445307e-05), (256, 2.0435196347534657e-05), (512, 2.0527339074760675e-05), (1024, 2.0417297491803765e-05), (2048, 2.034063800238073e-05), (4096, 2.0294683054089546e-05), (8192, 2.0401057554408908e-05), (16384, 2.0337756723165512e-05), (32768, 2.0290230168029666e-05), (65536, 2.0300823962315917e-05), (131072, 2.0313600543886423e-05), (262144, 2.0376784959807992e-05), (524288, 2.0375766325742006e-05), (1048576, 3.2409385312348604e-05), (2097152, 4.410417750477791e-05), (4194304, 6.527639925479889e-05), (8388608, 0.00010846974328160286), (16777216, 0.00019289273768663406), (33554432, 0.00036153756082057953), (67108864, 0.000699903815984726), (134217728, 0.0013720467686653137), (268435456, 0.0027210116386413574), (536870912, 0.005442023277282715), (1073741824, 0.010737872123718262)]
available_memory_per_device:
14957700710
dot_cost_dict:
((), 'f16'): [(0, 1.662003342062235e-05), (4194304, 2.9675080440938473e-05), (33554432, 2.9678398277610537e-05), (113246208, 3.9767706766724586e-05), (268435456, 2.968066837638617e-05), (524288000, 3.7421355955302715e-05), (905969664, 3.7816702388226986e-05), (1438646272, 5.177216371521354e-05), (2147483648, 5.5833079386502504e-05), (3057647616, 7.949670873586001e-05), (4194304000, 8.927180897444487e-05), (5582618624, 9.796867379918694e-05), (7247757312, 0.00010636699153110385), (9214885888, 0.00013523627337606203), (11509170176, 0.00016890684303318807), (14155776000, 0.0002077480303341872), (17179869184, 0.0002307849936187267), (20606615552, 0.00024195347214117646), (24461180928, 0.0002872120190367069), (28768731136, 0.00033778930702550974), (33554432000, 0.0003939808217238778), (38843449344, 0.0004220617401016342), (44660948992, 0.000485273016801879), (51032096768, 0.000554500074702147), (57982058496, 0.0006300163584024394), (65536000000, 0.0006726105486760374), (73719087104, 0.0007565953922259221), (82556485632, 0.0008472955794937884), (92073361408, 0.0009154096712284546), (102294880256, 0.0010170338225032243), (113246208000, 0.0011259138631180882), (124952510464, 0.0012299516797065735), (137438953472, 0.0013328195438096415), (150730702848, 0.0013959321501028485), (164852924416, 0.0015267194598225326), (179830784000, 0.0016654309189876625), (195689447424, 0.0018122995908172218), (212454080512, 0.0019675585385301943), (230149849088, 0.0021314408253455632), (248801918976, 0.0023041795144823126), (268435456000, 0.002394599299277029), (289075625984, 0.0025787215360230016), (310747594752, 0.0027720480138255705), (333476528128, 0.0029748032263690427), (357287591936, 0.0031872116673377256), (382205952000, 0.0034094978304159257), (408256774144, 0.003641886209287951), (435465224192, 0.0038497791540154204), (463856467968, 0.004100775128833432), (493455671296, 0.004362450205549447), (524288000000, 0.004635026865453007), (556378619904, 0.004918727589833655), (589752696832, 0.005189138051320815), (624435396608, 0.005494305485224802), (660451885056, 0.005811208708702071), (697827328000, 0.006124526470691174), (736586891264, 0.006464702273039226), (776755740672, 0.006817246765960695), (818359042048, 0.0071823808189301055), (861421961216, 0.007560325301421984), (905969664000, 0.007951301082910856), (952027316224, 0.008355529032871244), (999620083712, 0.008773230020777678), (1048773132288, 0.009177684783935547)]
((), 'f32'): [(0, 1.65026867762208e-05), (4194304, 2.9112619813531634e-05), (33554432, 2.9380840715020895e-05), (113246208, 4.260562127456069e-05), (268435456, 4.5800465159118176e-05), (524288000, 7.021648343652487e-05), (905969664, 9.645731188356875e-05), (1438646272, 0.00013177230721339583), (2147483648, 0.00019034423166885972), (3057647616, 0.0002501418930478394), (4194304000, 0.0003141497145406902), (5582618624, 0.00041813327005365864), (7247757312, 0.0005278292810544372), (9214885888, 0.0006554973078891635), (11509170176, 0.0008187003244642079), (14155776000, 0.0009950751761482904), (17179869184, 0.0011966387430826823), (20606615552, 0.0014353237658118207), (24461180928, 0.0016767866690798048), (28768731136, 0.001972064431278872), (33554432000, 0.0022670427424795678), (38843449344, 0.0026243853547629095), (44660948992, 0.0030174338902403046), (51032096768, 0.0034478886309686125), (57982058496, 0.003917449859004693), (65536000000, 0.004365294311303166), (73719087104, 0.004910362420189725), (82556485632, 0.005457679102243471), (92073361408, 0.006086824754989009), (102294880256, 0.006762553250247219), (113246208000, 0.007437059938772465), (124952510464, 0.008205831579110019), (137438953472, 0.009025836299025782), (150730702848, 0.00989872677850615), (164852924416, 0.010826155697537516), (179830784000, 0.011809775736106273), (195689447424, 0.012851239574198817), (212454080512, 0.01395219989180154), (230149849088, 0.015109097353324363), (248801918976, 0.01621807098388672), (268435456000, 0.01731490960685156), (289075625984, 0.018646263828340878), (310747594752, 0.020044172233631533), (333476528128, 0.02151025809549917), (357287591936, 0.023046144686719422), (382205952000, 0.02465345528006794), (408256774144, 0.026333813148320364), (435465224192, 0.028088841564252332), (463856467968, 0.029920163800639492), (493455671296, 0.031829403130257485), (524288000000, 0.03381818282588195), (556378619904, 0.03588812616028853), (589752696832, 0.03804085640625287), (624435396608, 0.04027799683655062), (660451885056, 0.0426011707239574), (697827328000, 0.04501200134124887), (736586891264, 0.04751211196120067), (776755740672, 0.05010312585658844), (818359042048, 0.05278666630018782), (861421961216, 0.05556435656477446), (905969664000, 0.058437819923123996), (952027316224, 0.061408679648012075), (999620083712, 0.06447855901221433), (1048773132288, 0.06764908128850641)]
conv_cost_dict:
[]
op_cost_dict:
[]
--------------------------------------------------
Key: ('default', (1, 2))
all_gather_cost_dict:
(((0, 1),), 'f16'): [(0, 1.7996266251429915e-05), (1, 1.8008897313848138e-05), (2, 3.6017794627696276e-05), (4, 5.299592157825828e-05), (8, 6.569002289324999e-05), (16, 6.179077900014818e-05), (32, 6.87073334120214e-05), (64, 6.906507769599557e-05), (128, 6.757545634172857e-05), (256, 5.063344724476337e-05), (512, 5.136962863616645e-05), (1024, 5.141779547557235e-05), (2048, 5.1616516429930925e-05), (4096, 6.367493188008666e-05), (8192, 7.313478272408247e-05), (16384, 7.335672853514552e-05), (32768, 7.415525033138692e-05), (65536, 8.192347013391554e-05), (131072, 8.7030726717785e-05), (262144, 9.372280328534544e-05), (524288, 9.82773199211806e-05), (1048576, 0.00011075919610448182), (2097152, 0.0001771107199601829), (4194304, 0.0002706334926187992), (8388608, 0.0004640957340598106), (16777216, 0.0008462988771498204), (33554432, 0.001636403612792492), (67108864, 0.0031436067074537277), (134217728, 0.0062316544353961945), (268435456, 0.012397728860378267), (536870912, 0.024703457951545715), (1073741824, 0.049082303047180184), (2147483648, 0.09816460609436037)]
(((0, 1),), 'f32'): [(0, 1.7837330233305693e-05), (1, 1.7844460671767592e-05), (2, 3.5688921343535185e-05), (4, 6.77080824971199e-05), (8, 6.582264904864132e-05), (16, 6.618813495151699e-05), (32, 6.751026376150548e-05), (64, 6.57233758829534e-05), (128, 6.766477599740028e-05), (256, 6.669506547041237e-05), (512, 6.769903120584786e-05), (1024, 6.699436926282942e-05), (2048, 6.763474084436893e-05), (4096, 7.10341555532068e-05), (8192, 7.586868014186621e-05), (16384, 7.770679076202214e-05), (32768, 8.24114540591836e-05), (65536, 9.018331184051931e-05), (131072, 8.218252332881093e-05), (262144, 9.961368050426245e-05), (524288, 0.00012486675404943526), (1048576, 0.0001751451636664569), (2097152, 0.0002693325513973832), (4194304, 0.00046475883573293686), (8388608, 0.0008472846820950509), (16777216, 0.0016102520748972895), (33554432, 0.0031660608947277073), (67108864, 0.006217125803232193), (134217728, 0.012329846620559692), (268435456, 0.024659693241119385), (536870912, 0.04914016723632812), (1073741824, 0.09813740253448486)]
(((0,), (1,)), 'f16'): [(0, 1.758555299602449e-05), (1, 2.0371749997138977e-05), (2, 2.0658160792663693e-05), (4, 2.0557752577587962e-05), (8, 2.054858487099409e-05), (16, 2.043027780018747e-05), (32, 2.064142609015107e-05), (64, 2.0504900021478534e-05), (128, 2.053024945780635e-05), (256, 2.0572682842612267e-05), (512, 2.047410816885531e-05), (1024, 2.0405714167281985e-05), (2048, 2.041511470451951e-05), (4096, 2.0428415155038238e-05), (8192, 2.036959631368518e-05), (16384, 2.0521983969956636e-05), (32768, 2.0524021238088608e-05), (65536, 2.0741892512887716e-05), (131072, 2.0634965039789677e-05), (262144, 2.0430423319339752e-05), (524288, 2.0670238882303238e-05), (1048576, 2.0781444618478417e-05), (2097152, 3.3145188353955746e-05), (4194304, 4.499626811593771e-05), (8388608, 6.675487384200096e-05), (16777216, 0.00010893028229475021), (33554432, 0.00019286107271909714), (67108864, 0.0003623124212026596), (134217728, 0.0006978660821914673), (268435456, 0.0013957321643829346), (536870912, 0.002720743417739868), (1073741824, 0.005387067794799805), (2147483648, 0.010727310180664062)]
(((0,), (1,)), 'f32'): [(0, 1.746774069033563e-05), (1, 2.0910956664010882e-05), (2, 2.0466744899749756e-05), (4, 2.044666325673461e-05), (8, 2.044977736659348e-05), (16, 2.0738079911097884e-05), (32, 2.0415434846654534e-05), (64, 2.0478531951084733e-05), (128, 2.0449195289984345e-05), (256, 2.043452695943415e-05), (512, 2.0647363271564245e-05), (1024, 2.0694395061582327e-05), (2048, 2.0715000573545694e-05), (4096, 2.0659936126321554e-05), (8192, 2.068420872092247e-05), (16384, 2.0765321096405387e-05), (32768, 2.1084066247567534e-05), (65536, 2.0563311409205198e-05), (131072, 1.9687286112457514e-05), (262144, 1.9671395421028137e-05), (524288, 2.0149163901805878e-05), (1048576, 3.247725544497371e-05), (2097152, 4.360196180641651e-05), (4194304, 6.532506085932255e-05), (8388608, 0.00010748812928795815), (16777216, 0.0001975083723664284), (33554432, 0.0003595948219299316), (67108864, 0.0006954409182071686), (134217728, 0.001370951533317566), (268435456, 0.002715185284614563), (536870912, 0.005390310287475586), (1073741824, 0.010711193084716797)]
all_reduce_cost_dict:
(((0, 1),), 'f16'): [(0, 4.9959984607994556e-05), (1, 5.306029925122857e-05), (2, 6.481868331320584e-05), (4, 6.601776112802327e-05), (8, 5.095827509649098e-05), (16, 4.9512978876009583e-05), (32, 5.284370854496956e-05), (64, 6.614634185098112e-05), (128, 6.834935629740357e-05), (256, 6.489668157882988e-05), (512, 5.575412069447338e-05), (1024, 5.4277974413707846e-05), (2048, 6.890983786433935e-05), (4096, 6.981269689276814e-05), (8192, 7.587677100673318e-05), (16384, 6.522462354041636e-05), (32768, 7.709141937084496e-05), (65536, 9.224863606505096e-05), (131072, 9.289095760323107e-05), (262144, 0.00010667761671356857), (524288, 0.00011916717630811036), (1048576, 0.0001739170984365046), (2097152, 0.0002724860096350312), (4194304, 0.0004572519101202488), (8388608, 0.0008260076865553856), (16777216, 0.0015772236511111257), (33554432, 0.0030838493257761006), (67108864, 0.006070982664823532), (134217728, 0.012057438492774963), (268435456, 0.024024978280067444), (536870912, 0.04804995656013489), (1073741824, 0.09566421508789062), (2147483648, 0.19132843017578124)]
(((0, 1),), 'f32'): [(0, 6.457837298512459e-05), (1, 6.226939149200916e-05), (2, 6.389903137460351e-05), (4, 6.494985427707434e-05), (8, 6.281939567998052e-05), (16, 5.389982834458351e-05), (32, 5.480964318849147e-05), (64, 5.7316618040204055e-05), (128, 6.494400440715253e-05), (256, 6.539165042340755e-05), (512, 6.531545659527183e-05), (1024, 6.667946581728756e-05), (2048, 7.089943392202258e-05), (4096, 7.364552584476769e-05), (8192, 7.427093805745244e-05), (16384, 8.242516196332872e-05), (32768, 9.099062299355865e-05), (65536, 9.45526408031583e-05), (131072, 0.00010604228009469807), (262144, 0.0001250236527994275), (524288, 0.0001720512518659234), (1048576, 0.0002665348583832383), (2097152, 0.0004504772368818521), (4194304, 0.0008314605802297592), (8388608, 0.001574937254190445), (16777216, 0.003078768029808998), (33554432, 0.006063397973775864), (67108864, 0.012056536972522736), (134217728, 0.023996129631996155), (268435456, 0.04799225926399231), (536870912, 0.09572079181671143), (1073741824, 0.19143099784851075)]
(((0,), (1,)), 'f16'): [(0, 1.8243008526042104e-05), (1, 2.1196698071435094e-05), (2, 2.1560699678957462e-05), (4, 2.1152751287445426e-05), (8, 2.0973413484171033e-05), (16, 2.034651697613299e-05), (32, 1.9994040485471487e-05), (64, 1.999188680201769e-05), (128, 1.993181649595499e-05), (256, 2.063502324745059e-05), (512, 2.1310901502147317e-05), (1024, 2.0959210814908147e-05), (2048, 2.0960142137482762e-05), (4096, 2.0843668607994914e-05), (8192, 2.097006654366851e-05), (16384, 2.1293963072821498e-05), (32768, 2.109329216182232e-05), (65536, 2.1073821699246764e-05), (131072, 2.039695391431451e-05), (262144, 2.0034436602145433e-05), (524288, 1.998367952182889e-05), (1048576, 2.022675471380353e-05), (2097152, 3.223400563001633e-05), (4194304, 4.4645508751273155e-05), (8388608, 6.539467722177505e-05), (16777216, 0.00010836776345968245), (33554432, 0.00019440054893493652), (67108864, 0.00036339834332466125), (134217728, 0.0006999894976615906), (268435456, 0.001380428671836853), (536870912, 0.002729058265686035), (1073741824, 0.005395197868347168), (2147483648, 0.01071183681488037)]
(((0,), (1,)), 'f32'): [(0, 1.7870392184704542e-05), (1, 2.0954030333086848e-05), (2, 2.1044048480689526e-05), (4, 2.1148996893316507e-05), (8, 2.127612242475152e-05), (16, 2.120877616107464e-05), (32, 2.1127198124304414e-05), (64, 2.1244457457214594e-05), (128, 2.1344429114833474e-05), (256, 2.1244166418910027e-05), (512, 2.1654792362824082e-05), (1024, 2.1591607946902514e-05), (2048, 2.005897113122046e-05), (4096, 2.009639865718782e-05), (8192, 2.0025967387482524e-05), (16384, 2.0053412299603224e-05), (32768, 2.0238978322595358e-05), (65536, 2.0238454453647137e-05), (131072, 1.9909231923520565e-05), (262144, 2.0041043171659112e-05), (524288, 2.001877874135971e-05), (1048576, 3.233796451240778e-05), (2097152, 4.392722621560097e-05), (4194304, 7.18352384865284e-05), (8388608, 0.000108182430267334), (16777216, 0.00021540001034736633), (33554432, 0.00036319345235824585), (67108864, 0.0006997883319854736), (134217728, 0.0013995766639709473), (268435456, 0.0027278900146484376), (536870912, 0.0053881645202636715), (1073741824, 0.010723400115966796)]
all_to_all_cost_dict:
(((0, 1),), 'f16'): [(0, 1.7263024346902966e-05), (1, 1.752175739966333e-05), (2, 1.7157552065327764e-05), (4, 3.431510413065553e-05), (8, 6.863020826131105e-05), (16, 8.080975385382771e-05), (32, 7.891596760600805e-05), (64, 7.650506449863315e-05), (128, 8.28425690997392e-05), (256, 8.370436262339354e-05), (512, 8.466053986921906e-05), (1024, 8.157466072589159e-05), (2048, 8.136837277561426e-05), (4096, 8.40871362015605e-05), (8192, 7.804669439792633e-05), (16384, 8.702368359081447e-05), (32768, 8.16179090179503e-05), (65536, 7.81926210038364e-05), (131072, 7.638809620402753e-05), (262144, 8.880888344720006e-05), (524288, 8.454101043753326e-05), (1048576, 0.00010898226173594594), (2097152, 0.00013951450819149613), (4194304, 0.00020161049906164408), (8388608, 0.0003734712954610586), (16777216, 0.0006427271291613579), (33554432, 0.001230829395353794), (67108864, 0.002408258616924286), (134217728, 0.004771873354911804), (268435456, 0.009472765028476715), (536870912, 0.018912747502326965), (1073741824, 0.03774092197418213), (2147483648, 0.07528243064880372)]
(((0, 1),), 'f32'): [(0, 1.773782423697412e-05), (1, 1.772810355760157e-05), (2, 1.7849059076979756e-05), (4, 3.569811815395951e-05), (8, 7.139623630791903e-05), (16, 8.478673407807946e-05), (32, 8.62350279930979e-05), (64, 8.755899034440517e-05), (128, 8.747985702939332e-05), (256, 8.651905227452517e-05), (512, 8.676864672452211e-05), (1024, 8.690322283655405e-05), (2048, 8.651672396808863e-05), (4096, 8.585030445829034e-05), (8192, 8.717895252630115e-05), (16384, 8.57449194882065e-05), (32768, 8.67737689986825e-05), (65536, 8.695322321727872e-05), (131072, 9.147575474344195e-05), (262144, 9.751474135555327e-05), (524288, 0.00011168676428496838), (1048576, 0.00013907084940001369), (2097152, 0.00019795203115791082), (4194304, 0.00036689708940684795), (8388608, 0.0006528361700475216), (16777216, 0.0012494921684265137), (33554432, 0.0024242401123046875), (67108864, 0.004780307412147522), (134217728, 0.009504355490207672), (268435456, 0.018953517079353333), (536870912, 0.03782908916473389), (1073741824, 0.07552168369293213)]
(((0,), (1,)), 'f16'): [(0, 1.7497048247605562e-05), (1, 2.0507752196863294e-05), (2, 2.0816107280552387e-05), (4, 2.0455277990549803e-05), (8, 2.062183921225369e-05), (16, 2.048219903372228e-05), (32, 2.0898296497762203e-05), (64, 2.079203841276467e-05), (128, 2.0368141122162342e-05), (256, 2.0826206309720874e-05), (512, 2.0468520233407617e-05), (1024, 2.0522973500192165e-05), (2048, 2.104943268932402e-05), (4096, 2.0373729057610035e-05), (8192, 2.048176247626543e-05), (16384, 2.1256913896650076e-05), (32768, 2.0517298253253102e-05), (65536, 2.0542211132124066e-05), (131072, 2.112812944687903e-05), (262144, 2.073563518933952e-05), (524288, 2.083167782984674e-05), (1048576, 2.0105420844629407e-05), (2097152, 3.1563686206936836e-05), (4194304, 4.308705683797598e-05), (8388608, 6.898236460983753e-05), (16777216, 0.00011264905333518982), (33554432, 0.00019166897982358932), (67108864, 0.0003681238740682602), (134217728, 0.0006934776902198792), (268435456, 0.0013682171702384949), (536870912, 0.0027116388082504272), (1073741824, 0.005400347709655762), (2147483648, 0.01071760654449463)]
(((0,), (1,)), 'f32'): [(0, 1.756273559294641e-05), (1, 2.072608913294971e-05), (2, 2.1393265342339873e-05), (4, 2.099687117151916e-05), (8, 2.1015875972807407e-05), (16, 2.0621460862457752e-05), (32, 2.0732841221615672e-05), (64, 2.068671165034175e-05), (128, 2.062780549749732e-05), (256, 2.0638923160731792e-05), (512, 2.0812556613236666e-05), (1024, 2.0785635570064187e-05), (2048, 2.0496780052781105e-05), (4096, 2.0715349819511175e-05), (8192, 2.065481385216117e-05), (16384, 2.0737876184284687e-05), (32768, 2.043464337475598e-05), (65536, 2.0101055270060897e-05), (131072, 2.1292828023433685e-05), (262144, 2.1117768483236432e-05), (524288, 2.0813196897506714e-05), (1048576, 3.210717113688588e-05), (2097152, 4.3778098188340664e-05), (4194304, 6.542610935866833e-05), (8388608, 0.00010785646736621857), (16777216, 0.00019280333071947098), (33554432, 0.0003610942512750625), (67108864, 0.000694621354341507), (134217728, 0.0013692975044250488), (268435456, 0.0027115345001220703), (536870912, 0.005389571189880371), (1073741824, 0.010711741447448731)]
reduce_scatter_cost_dict:
(((0, 1),), 'f16'): [(0, 1.6685982700437307e-05), (1, 1.681694993749261e-05), (2, 3.363389987498522e-05), (4, 5.1018811063840985e-05), (8, 4.8837391659617424e-05), (16, 5.672185216099023e-05), (32, 6.712047616019845e-05), (64, 6.626505637541413e-05), (128, 6.435444811359048e-05), (256, 5.043737473897636e-05), (512, 4.928946145810187e-05), (1024, 5.097914254292846e-05), (2048, 5.424045957624912e-05), (4096, 5.1788752898573875e-05), (8192, 6.428314372897148e-05), (16384, 6.221051444299519e-05), (32768, 6.775878136977553e-05), (65536, 7.31878390070051e-05), (131072, 8.587574120610952e-05), (262144, 9.19473241083324e-05), (524288, 0.00010885985102504492), (1048576, 0.00014009204460307956), (2097152, 0.00022164190886542204), (4194304, 0.00036025315057486296), (8388608, 0.0006537975277751684), (16777216, 0.001222042366862297), (33554432, 0.002355906181037426), (67108864, 0.004524068906903267), (134217728, 0.008347507566213608), (268435456, 0.015454329550266264), (536870912, 0.02856515347957611), (1073741824, 0.05503942966461182), (2147483648, 0.10764012336730958)]
(((0, 1),), 'f32'): [(0, 1.747746136970818e-05), (1, 1.7970072804018855e-05), (2, 3.594014560803771e-05), (4, 6.405691965483129e-05), (8, 6.453329115174711e-05), (16, 6.475247209891677e-05), (32, 6.604782538488507e-05), (64, 6.227297126315534e-05), (128, 6.305283750407398e-05), (256, 6.236557965166867e-05), (512, 6.226444384083152e-05), (1024, 6.19499187450856e-05), (2048, 6.28091802354902e-05), (4096, 6.987593951635063e-05), (8192, 7.11043830960989e-05), (16384, 7.318466668948531e-05), (32768, 7.342835306189954e-05), (65536, 8.859657100401819e-05), (131072, 9.27108631003648e-05), (262144, 0.00010321062291041017), (524288, 0.00013755104737356305), (1048576, 0.00021174934227019548), (2097152, 0.0003451491938903928), (4194304, 0.0006323247216641903), (8388608, 0.0011589527130126953), (16777216, 0.0022168494760990143), (33554432, 0.004275310784578323), (67108864, 0.00836198776960373), (134217728, 0.01642690598964691), (268435456, 0.032572388648986816), (536870912, 0.06475470066070557), (1073741824, 0.12947664260864258)]
(((0,), (1,)), 'f16'): [(0, 1.7533660866320133e-05), (1, 2.0241772290319204e-05), (2, 2.052623312920332e-05), (4, 2.0437175408005714e-05), (8, 2.0665815100073814e-05), (16, 2.061229315586388e-05), (32, 2.0587642211467028e-05), (64, 2.0543549908325076e-05), (128, 2.0898034563288093e-05), (256, 2.0431092707440257e-05), (512, 2.0440405933186412e-05), (1024, 2.0822539227083325e-05), (2048, 2.051828778348863e-05), (4096, 2.0416395273059607e-05), (8192, 2.071628114208579e-05), (16384, 2.0378269255161285e-05), (32768, 2.0849925931543112e-05), (65536, 2.0250794477760792e-05), (131072, 2.065283479169011e-05), (262144, 2.0283274352550507e-05), (524288, 2.0668376237154007e-05), (1048576, 2.047125599347055e-05), (2097152, 3.332551568746567e-05), (4194304, 4.405446816235781e-05), (8388608, 6.78221695125103e-05), (16777216, 0.00011271331459283829), (33554432, 0.00019310880452394483), (67108864, 0.0003620460629463196), (134217728, 0.0006961934268474579), (268435456, 0.0013737007975578308), (536870912, 0.002712249755859375), (1073741824, 0.00542449951171875), (2147483648, 0.010716700553894043)]
(((0,), (1,)), 'f32'): [(0, 1.773613621480763e-05), (1, 2.075798693113029e-05), (2, 2.0745617803186178e-05), (4, 2.099340781569481e-05), (8, 2.096273237839341e-05), (16, 2.120938734151423e-05), (32, 2.0448380382731557e-05), (64, 2.0105799194425344e-05), (128, 2.0911305909976363e-05), (256, 2.0972336642444134e-05), (512, 2.1189713152125478e-05), (1024, 2.099366975016892e-05), (2048, 2.1139421733096242e-05), (4096, 2.136951661668718e-05), (8192, 2.1553627448156476e-05), (16384, 2.1044339518994093e-05), (32768, 2.081715501844883e-05), (65536, 2.104623126797378e-05), (131072, 2.1539104636758566e-05), (262144, 2.1510582882910967e-05), (524288, 2.090359339490533e-05), (1048576, 3.343418939039111e-05), (2097152, 4.479184281080961e-05), (4194304, 6.713694892823696e-05), (8388608, 0.00010895496234297752), (16777216, 0.0001906631514430046), (33554432, 0.00035928934812545776), (67108864, 0.0007046051323413849), (134217728, 0.0013687089085578918), (268435456, 0.0027114897966384888), (536870912, 0.005390667915344238), (1073741824, 0.010781335830688476)]
available_memory_per_device:
14957700710
dot_cost_dict:
((), 'f16'): [(0, 1.662003342062235e-05), (4194304, 2.9675080440938473e-05), (33554432, 2.9678398277610537e-05), (113246208, 3.9767706766724586e-05), (268435456, 2.968066837638617e-05), (524288000, 3.7421355955302715e-05), (905969664, 3.7816702388226986e-05), (1438646272, 5.177216371521354e-05), (2147483648, 5.5833079386502504e-05), (3057647616, 7.949670873586001e-05), (4194304000, 8.927180897444487e-05), (5582618624, 9.796867379918694e-05), (7247757312, 0.00010636699153110385), (9214885888, 0.00013523627337606203), (11509170176, 0.00016890684303318807), (14155776000, 0.0002077480303341872), (17179869184, 0.0002307849936187267), (20606615552, 0.00024195347214117646), (24461180928, 0.0002872120190367069), (28768731136, 0.00033778930702550974), (33554432000, 0.0003939808217238778), (38843449344, 0.0004220617401016342), (44660948992, 0.000485273016801879), (51032096768, 0.000554500074702147), (57982058496, 0.0006300163584024394), (65536000000, 0.0006726105486760374), (73719087104, 0.0007565953922259221), (82556485632, 0.0008472955794937884), (92073361408, 0.0009154096712284546), (102294880256, 0.0010170338225032243), (113246208000, 0.0011259138631180882), (124952510464, 0.0012299516797065735), (137438953472, 0.0013328195438096415), (150730702848, 0.0013959321501028485), (164852924416, 0.0015267194598225326), (179830784000, 0.0016654309189876625), (195689447424, 0.0018122995908172218), (212454080512, 0.0019675585385301943), (230149849088, 0.0021314408253455632), (248801918976, 0.0023041795144823126), (268435456000, 0.002394599299277029), (289075625984, 0.0025787215360230016), (310747594752, 0.0027720480138255705), (333476528128, 0.0029748032263690427), (357287591936, 0.0031872116673377256), (382205952000, 0.0034094978304159257), (408256774144, 0.003641886209287951), (435465224192, 0.0038497791540154204), (463856467968, 0.004100775128833432), (493455671296, 0.004362450205549447), (524288000000, 0.004635026865453007), (556378619904, 0.004918727589833655), (589752696832, 0.005189138051320815), (624435396608, 0.005494305485224802), (660451885056, 0.005811208708702071), (697827328000, 0.006124526470691174), (736586891264, 0.006464702273039226), (776755740672, 0.006817246765960695), (818359042048, 0.0071823808189301055), (861421961216, 0.007560325301421984), (905969664000, 0.007951301082910856), (952027316224, 0.008355529032871244), (999620083712, 0.008773230020777678), (1048773132288, 0.009177684783935547)]
((), 'f32'): [(0, 1.65026867762208e-05), (4194304, 2.9112619813531634e-05), (33554432, 2.9380840715020895e-05), (113246208, 4.260562127456069e-05), (268435456, 4.5800465159118176e-05), (524288000, 7.021648343652487e-05), (905969664, 9.645731188356875e-05), (1438646272, 0.00013177230721339583), (2147483648, 0.00019034423166885972), (3057647616, 0.0002501418930478394), (4194304000, 0.0003141497145406902), (5582618624, 0.00041813327005365864), (7247757312, 0.0005278292810544372), (9214885888, 0.0006554973078891635), (11509170176, 0.0008187003244642079), (14155776000, 0.0009950751761482904), (17179869184, 0.0011966387430826823), (20606615552, 0.0014353237658118207), (24461180928, 0.0016767866690798048), (28768731136, 0.001972064431278872), (33554432000, 0.0022670427424795678), (38843449344, 0.0026243853547629095), (44660948992, 0.0030174338902403046), (51032096768, 0.0034478886309686125), (57982058496, 0.003917449859004693), (65536000000, 0.004365294311303166), (73719087104, 0.004910362420189725), (82556485632, 0.005457679102243471), (92073361408, 0.006086824754989009), (102294880256, 0.006762553250247219), (113246208000, 0.007437059938772465), (124952510464, 0.008205831579110019), (137438953472, 0.009025836299025782), (150730702848, 0.00989872677850615), (164852924416, 0.010826155697537516), (179830784000, 0.011809775736106273), (195689447424, 0.012851239574198817), (212454080512, 0.01395219989180154), (230149849088, 0.015109097353324363), (248801918976, 0.01621807098388672), (268435456000, 0.01731490960685156), (289075625984, 0.018646263828340878), (310747594752, 0.020044172233631533), (333476528128, 0.02151025809549917), (357287591936, 0.023046144686719422), (382205952000, 0.02465345528006794), (408256774144, 0.026333813148320364), (435465224192, 0.028088841564252332), (463856467968, 0.029920163800639492), (493455671296, 0.031829403130257485), (524288000000, 0.03381818282588195), (556378619904, 0.03588812616028853), (589752696832, 0.03804085640625287), (624435396608, 0.04027799683655062), (660451885056, 0.0426011707239574), (697827328000, 0.04501200134124887), (736586891264, 0.04751211196120067), (776755740672, 0.05010312585658844), (818359042048, 0.05278666630018782), (861421961216, 0.05556435656477446), (905969664000, 0.058437819923123996), (952027316224, 0.061408679648012075), (999620083712, 0.06447855901221433), (1048773132288, 0.06764908128850641)]
conv_cost_dict:
[]
op_cost_dict:
[]
--------------------------------------------------
Key: ('default', (1, 1))
all_gather_cost_dict:
(((0,),), 'f16'): [(0, 1.6640842659398913e-05), (1, 1.955989864654839e-05), (2, 1.966842683032155e-05), (4, 1.996487844735384e-05), (8, 1.992567558772862e-05), (16, 1.9677623640745878e-05), (32, 2.0005449187010527e-05), (64, 1.9938452169299126e-05), (128, 1.989654265344143e-05), (256, 1.952142338268459e-05), (512, 1.9702973077073693e-05), (1024, 1.958513166755438e-05), (2048, 1.9751809304580092e-05), (4096, 1.9794271793216467e-05), (8192, 1.954991603270173e-05), (16384, 1.9762141164392233e-05), (32768, 1.9741739379242063e-05), (65536, 1.978926593437791e-05), (131072, 1.9796134438365698e-05), (262144, 1.96364417206496e-05), (524288, 1.9782630261033773e-05), (1048576, 1.988656003959477e-05), (2097152, 3.23272543027997e-05), (4194304, 4.423316568136215e-05), (8388608, 6.580352783203125e-05), (16777216, 0.00010732561349868773), (33554432, 0.00019161123782396317), (67108864, 0.0003599170595407486), (134217728, 0.0006927214562892914), (268435456, 0.0013661235570907593), (536870912, 0.0027042627334594727), (1073741824, 0.0053749561309814455), (2147483648, 0.010706138610839844)]
(((0,),), 'f32'): [(0, 1.701197470538318e-05), (1, 1.9627332221716642e-05), (2, 1.9967672415077686e-05), (4, 1.9844475900754333e-05), (8, 2.003356348723173e-05), (16, 1.991249155253172e-05), (32, 1.988414442166686e-05), (64, 1.982381218113005e-05), (128, 1.989875454455614e-05), (256, 1.9866653019562364e-05), (512, 1.987881842069328e-05), (1024, 1.999374944716692e-05), (2048, 1.9803119357675314e-05), (4096, 1.981348032131791e-05), (8192, 1.991115277633071e-05), (16384, 2.010620664805174e-05), (32768, 2.0023289835080504e-05), (65536, 2.006796421483159e-05), (131072, 1.9971863366663456e-05), (262144, 2.0187057089060545e-05), (524288, 2.014503115788102e-05), (1048576, 3.2794661819934845e-05), (2097152, 4.3960753828287125e-05), (4194304, 6.541842594742775e-05), (8388608, 0.00010736659169197083), (16777216, 0.00019161682575941089), (33554432, 0.00035958364605903625), (67108864, 0.000692218542098999), (134217728, 0.0013675689697265625), (268435456, 0.0027043819427490234), (536870912, 0.005374097824096679), (1073741824, 0.010700368881225586)]
all_reduce_cost_dict:
(((0,),), 'f16'): [(0, 1.7084763385355473e-05), (1, 1.9517116015776992e-05), (2, 2.003420377150178e-05), (4, 1.9741244614124298e-05), (8, 1.9997649360448122e-05), (16, 1.9962259102612734e-05), (32, 1.9985222024843097e-05), (64, 2.003874396905303e-05), (128, 2.000032691285014e-05), (256, 1.990864984691143e-05), (512, 2.0081090042367578e-05), (1024, 1.9905332010239363e-05), (2048, 1.9955652533099055e-05), (4096, 2.001112443394959e-05), (8192, 1.998181687667966e-05), (16384, 1.9870116375386715e-05), (32768, 2.0082894479855895e-05), (65536, 2.013193443417549e-05), (131072, 2.0014471374452114e-05), (262144, 2.023379784077406e-05), (524288, 2.0025589037686586e-05), (1048576, 2.0353589206933975e-05), (2097152, 3.2200710847973824e-05), (4194304, 4.438706673681736e-05), (8388608, 6.639398634433746e-05), (16777216, 0.0001087551936507225), (33554432, 0.00019289925694465637), (67108864, 0.00036265701055526733), (134217728, 0.000699259340763092), (268435456, 0.001382291316986084), (536870912, 0.0027161121368408205), (1073741824, 0.005376195907592774), (2147483648, 0.010697531700134277)]
(((0,),), 'f32'): [(0, 1.701776636764407e-05), (1, 2.0014529582113028e-05), (2, 2.0159146515652537e-05), (4, 2.0384788513183594e-05), (8, 2.047704765573144e-05), (16, 2.0002422388643026e-05), (32, 2.0011066226288676e-05), (64, 1.9983737729489803e-05), (128, 2.001968096010387e-05), (256, 1.985562266781926e-05), (512, 2.0147592294961214e-05), (1024, 2.0052859326824546e-05), (2048, 1.9958883058279753e-05), (4096, 2.002698602154851e-05), (8192, 1.9761151634156704e-05), (16384, 1.9906728994101286e-05), (32768, 1.9799743313342333e-05), (65536, 1.9884202629327774e-05), (131072, 1.9947794498875737e-05), (262144, 1.986071583814919e-05), (524288, 1.9834027625620365e-05), (1048576, 3.211002331227064e-05), (2097152, 4.419521428644657e-05), (4194304, 6.635859608650208e-05), (8388608, 0.00010827556252479553), (16777216, 0.00019273534417152405), (33554432, 0.00036291033029556274), (67108864, 0.0006963163614273071), (134217728, 0.0013780295848846436), (268435456, 0.002714729309082031), (536870912, 0.005376482009887695), (1073741824, 0.010699176788330078)]
all_to_all_cost_dict:
(((0,),), 'f16'): [(0, 1.6965524991974235e-05), (1, 2.032573684118688e-05), (2, 1.9912462448701262e-05), (4, 1.9833329133689404e-05), (8, 2.0013278117403388e-05), (16, 2.009497256949544e-05), (32, 1.99474161490798e-05), (64, 1.9924133084714413e-05), (128, 1.988461008295417e-05), (256, 1.990757300518453e-05), (512, 1.9942002836614847e-05), (1024, 1.975495251826942e-05), (2048, 1.9833649275824428e-05), (4096, 1.9993167370557785e-05), (8192, 1.9902712665498257e-05), (16384, 1.988504664041102e-05), (32768, 2.002625842578709e-05), (65536, 1.996595528908074e-05), (131072, 2.017969381995499e-05), (262144, 1.9812898244708776e-05), (524288, 2.0093488274142146e-05), (1048576, 2.029037568718195e-05), (2097152, 3.229064168408513e-05), (4194304, 4.466273821890354e-05), (8388608, 6.596441380679607e-05), (16777216, 0.00010805996134877205), (33554432, 0.00019219424575567245), (67108864, 0.00036099925637245184), (134217728, 0.0006941035389900208), (268435456, 0.0013684704899787903), (536870912, 0.0027048587799072266), (1073741824, 0.005376863479614258), (2147483648, 0.010697603225708008)]
(((0,),), 'f32'): [(0, 1.7055135685950518e-05), (1, 1.9771308870986104e-05), (2, 1.9854371203109622e-05), (4, 1.9860308384522796e-05), (8, 2.0044768461957574e-05), (16, 1.988513395190239e-05), (32, 2.0012492313981056e-05), (64, 1.9800820155069232e-05), (128, 1.9729137420654297e-05), (256, 1.9846833311021328e-05), (512, 1.991185126826167e-05), (1024, 1.9726052414625883e-05), (2048, 1.9884726498275995e-05), (4096, 1.9696366507560015e-05), (8192, 1.9809725927188993e-05), (16384, 2.0041392417624593e-05), (32768, 1.979299122467637e-05), (65536, 1.983309630304575e-05), (131072, 1.992683974094689e-05), (262144, 2.002983819693327e-05), (524288, 2.0086619770154357e-05), (1048576, 3.234989708289504e-05), (2097152, 4.4806511141359806e-05), (4194304, 6.633787415921688e-05), (8388608, 0.00010787742212414742), (16777216, 0.0001922864466905594), (33554432, 0.0003598686307668686), (67108864, 0.0006917305290699005), (134217728, 0.0013662055134773254), (268435456, 0.0027046799659729004), (536870912, 0.0053725004196166996), (1073741824, 0.01069943904876709)]
reduce_scatter_cost_dict:
(((0,),), 'f16'): [(0, 1.785962376743555e-05), (1, 2.0570092601701617e-05), (2, 2.0610488718375564e-05), (4, 2.0423729438334703e-05), (8, 2.0672247046604753e-05), (16, 2.0279461750760674e-05), (32, 2.0417734049260616e-05), (64, 2.0566541934385896e-05), (128, 2.0281324395909905e-05), (256, 2.0337174646556377e-05), (512, 2.0616513211280107e-05), (1024, 2.0457955542951822e-05), (2048, 2.0541949197649956e-05), (4096, 2.0570907508954406e-05), (8192, 2.0754989236593246e-05), (16384, 2.0501611288636923e-05), (32768, 2.0846084225922823e-05), (65536, 2.0892592146992683e-05), (131072, 2.05693650059402e-05), (262144, 2.02867086045444e-05), (524288, 2.0586972823366523e-05), (1048576, 2.0444218534976244e-05), (2097152, 3.253103932365775e-05), (4194304, 4.409626126289368e-05), (8388608, 6.653531454503536e-05), (16777216, 0.00010843481868505478), (33554432, 0.00019160006195306778), (67108864, 0.00036102160811424255), (134217728, 0.0006933510303497314), (268435456, 0.001367807388305664), (536870912, 0.002703070640563965), (1073741824, 0.005372786521911621), (2147483648, 0.01069934368133545)]
(((0,),), 'f32'): [(0, 1.7696787836030126e-05), (1, 2.0379346096888185e-05), (2, 2.0674837287515402e-05), (4, 2.0821316866204143e-05), (8, 2.0749197574332356e-05), (16, 2.0762119675055146e-05), (32, 2.0852399757131934e-05), (64, 2.0553619833663106e-05), (128, 2.079468686133623e-05), (256, 2.060338738374412e-05), (512, 2.066459273919463e-05), (1024, 2.0595063688233495e-05), (2048, 2.0540785044431686e-05), (4096, 2.0520586986094713e-05), (8192, 2.064785803668201e-05), (16384, 2.0588457118719816e-05), (32768, 2.04991374630481e-05), (65536, 2.056383527815342e-05), (131072, 2.0603270968422294e-05), (262144, 2.0580977434292436e-05), (524288, 2.045932342298329e-05), (1048576, 3.24331340380013e-05), (2097152, 4.395400173962116e-05), (4194304, 6.568292155861855e-05), (8388608, 0.00010808417573571205), (16777216, 0.00019206944853067398), (33554432, 0.000361097976565361), (67108864, 0.0006938129663467407), (134217728, 0.0013689994812011719), (268435456, 0.0027085542678833008), (536870912, 0.005379390716552734), (1073741824, 0.01069798469543457)]
available_memory_per_device:
14957700710
dot_cost_dict:
((), 'f16'): [(0, 1.662003342062235e-05), (4194304, 2.9675080440938473e-05), (33554432, 2.9678398277610537e-05), (113246208, 3.9767706766724586e-05), (268435456, 2.968066837638617e-05), (524288000, 3.7421355955302715e-05), (905969664, 3.7816702388226986e-05), (1438646272, 5.177216371521354e-05), (2147483648, 5.5833079386502504e-05), (3057647616, 7.949670873586001e-05), (4194304000, 8.927180897444487e-05), (5582618624, 9.796867379918694e-05), (7247757312, 0.00010636699153110385), (9214885888, 0.00013523627337606203), (11509170176, 0.00016890684303318807), (14155776000, 0.0002077480303341872), (17179869184, 0.0002307849936187267), (20606615552, 0.00024195347214117646), (24461180928, 0.0002872120190367069), (28768731136, 0.00033778930702550974), (33554432000, 0.0003939808217238778), (38843449344, 0.0004220617401016342), (44660948992, 0.000485273016801879), (51032096768, 0.000554500074702147), (57982058496, 0.0006300163584024394), (65536000000, 0.0006726105486760374), (73719087104, 0.0007565953922259221), (82556485632, 0.0008472955794937884), (92073361408, 0.0009154096712284546), (102294880256, 0.0010170338225032243), (113246208000, 0.0011259138631180882), (124952510464, 0.0012299516797065735), (137438953472, 0.0013328195438096415), (150730702848, 0.0013959321501028485), (164852924416, 0.0015267194598225326), (179830784000, 0.0016654309189876625), (195689447424, 0.0018122995908172218), (212454080512, 0.0019675585385301943), (230149849088, 0.0021314408253455632), (248801918976, 0.0023041795144823126), (268435456000, 0.002394599299277029), (289075625984, 0.0025787215360230016), (310747594752, 0.0027720480138255705), (333476528128, 0.0029748032263690427), (357287591936, 0.0031872116673377256), (382205952000, 0.0034094978304159257), (408256774144, 0.003641886209287951), (435465224192, 0.0038497791540154204), (463856467968, 0.004100775128833432), (493455671296, 0.004362450205549447), (524288000000, 0.004635026865453007), (556378619904, 0.004918727589833655), (589752696832, 0.005189138051320815), (624435396608, 0.005494305485224802), (660451885056, 0.005811208708702071), (697827328000, 0.006124526470691174), (736586891264, 0.006464702273039226), (776755740672, 0.006817246765960695), (818359042048, 0.0071823808189301055), (861421961216, 0.007560325301421984), (905969664000, 0.007951301082910856), (952027316224, 0.008355529032871244), (999620083712, 0.008773230020777678), (1048773132288, 0.009177684783935547)]
((), 'f32'): [(0, 1.65026867762208e-05), (4194304, 2.9112619813531634e-05), (33554432, 2.9380840715020895e-05), (113246208, 4.260562127456069e-05), (268435456, 4.5800465159118176e-05), (524288000, 7.021648343652487e-05), (905969664, 9.645731188356875e-05), (1438646272, 0.00013177230721339583), (2147483648, 0.00019034423166885972), (3057647616, 0.0002501418930478394), (4194304000, 0.0003141497145406902), (5582618624, 0.00041813327005365864), (7247757312, 0.0005278292810544372), (9214885888, 0.0006554973078891635), (11509170176, 0.0008187003244642079), (14155776000, 0.0009950751761482904), (17179869184, 0.0011966387430826823), (20606615552, 0.0014353237658118207), (24461180928, 0.0016767866690798048), (28768731136, 0.001972064431278872), (33554432000, 0.0022670427424795678), (38843449344, 0.0026243853547629095), (44660948992, 0.0030174338902403046), (51032096768, 0.0034478886309686125), (57982058496, 0.003917449859004693), (65536000000, 0.004365294311303166), (73719087104, 0.004910362420189725), (82556485632, 0.005457679102243471), (92073361408, 0.006086824754989009), (102294880256, 0.006762553250247219), (113246208000, 0.007437059938772465), (124952510464, 0.008205831579110019), (137438953472, 0.009025836299025782), (150730702848, 0.00989872677850615), (164852924416, 0.010826155697537516), (179830784000, 0.011809775736106273), (195689447424, 0.012851239574198817), (212454080512, 0.01395219989180154), (230149849088, 0.015109097353324363), (248801918976, 0.01621807098388672), (268435456000, 0.01731490960685156), (289075625984, 0.018646263828340878), (310747594752, 0.020044172233631533), (333476528128, 0.02151025809549917), (357287591936, 0.023046144686719422), (382205952000, 0.02465345528006794), (408256774144, 0.026333813148320364), (435465224192, 0.028088841564252332), (463856467968, 0.029920163800639492), (493455671296, 0.031829403130257485), (524288000000, 0.03381818282588195), (556378619904, 0.03588812616028853), (589752696832, 0.03804085640625287), (624435396608, 0.04027799683655062), (660451885056, 0.0426011707239574), (697827328000, 0.04501200134124887), (736586891264, 0.04751211196120067), (776755740672, 0.05010312585658844), (818359042048, 0.05278666630018782), (861421961216, 0.05556435656477446), (905969664000, 0.058437819923123996), (952027316224, 0.061408679648012075), (999620083712, 0.06447855901221433), (1048773132288, 0.06764908128850641)]
conv_cost_dict:
[]
op_cost_dict:
[]
--------------------------------------------------