QDP++
sse_spin_recon_inlines.h
Go to the documentation of this file.
1#ifndef SSE_SPIN_RECON_INLINES_H
2#define SSE_SPIN_RECON_INLINES_H
3
4#include "qdp_sse_intrin.h"
5
6/* File: generic_spin_recon_inlines.h
7 Purpose: Supply inline functions to do spin reconstruction
8 Author: $Id: sse_spin_recon_inlines.h,v 1.6 2007-08-31 14:41:18 bjoo Exp $
9*/
10namespace QDP {
11
24inline
25void inlineSpinReconDir0Plus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
26{
27
28#ifdef DEBUG_GENERIC_SPIN_RECON_INLINES
29 QDPIO::cout << "inlineSpinProjDir0Plus" << endl;
30#endif
31
32 /* 1 + \gamma_0 = 1 0 0 i
33 0 1 i 0
34 0 -i 1 0
35 -i 0 0 1
36
37 * ( b2r + i b2i ) = ( {a2r + a1i} + i{a2i - a1r} ) = ( b1i - i b1r )
38 * ( b3r + i b3i ) ( {a3r + a0i} + i{a3i - a0r} ) ( b0i - i b0r )
39 */
40
41 // 12 components in source, 24 in dest
42 // 3 vectors for source
43
44 const REAL32* src_shadow = src;
45 REAL32* dst_shadow = dst;
46
47
48 SSEVec v0, v1, v2, v3, v4, v5, v6;
49 v6.floats[0] = +1;
50 v6.floats[1] = -1;
51 v6.floats[2] = +1;
52 v6.floats[3] = -1;
53
54 v0.vector = _mm_load_ps(src_shadow);
55 v1.vector = _mm_load_ps(src_shadow+4);
56 v2.vector = _mm_load_ps(src_shadow+8);
57
58 for(unsigned int site=0; site < n_vec-1; site++) {
59
60 src_shadow += 12;
61 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
62
63 // Now we have
64 // v0.floats[0] = tmp[0][col=0][re]
65 // v0.floats[1] = tmp[0][col=0][im]
66 // v0.floats[2] = tmp[0][col=1][re]
67 // v0.floats[3] = tmp[0][col=1][im]
68 //
69 // v1.floats[0] = tmp[0][col=2][re]
70 // v1.floats[1] = tmp[0][col=2][im]
71 // v1.floats[2] = tmp[1][col=0][re]
72 // v1.floats[3] = tmp[1][col=0][im]
73 //
74 // v2.floats[0] = tmp[1][col=1][re]
75 // v2.floats[1] = tmp[1][col=1][im]
76 // v2.floats[2] = tmp[1][col=2][re]
77 // v2.floats[3] = tmp[1][col=2][im]
78
79 // These are the top 2 components of the result
80 // so I can store them out already.
81 _mm_store_ps(dst_shadow, v0.vector);
82 _mm_store_ps(dst_shadow+4, v1.vector);
83 _mm_store_ps(dst_shadow+8, v2.vector);
84
85 // I want to set up
86 //
87 // v3[0] = v1[3] = tmp[1][col=0][im]
88 // v3[1] = v1[2] = tmp[1][col=0][re]
89 // v3[2] = v2[1] = tmp[1][col=1][im]
90 // v3[3] = v2[0] = tmp[1][col=1][re]
91 //
92 // We can do this with v3 = v1, v3 = shuf( v3, v1 ). Shuf code is 00 01 10 11 = x1B
93 v3.vector = v1.vector;
94 v3.vector = _mm_shuffle_ps(v3.vector, v2.vector, 0x1B);
95
96 // Now I need to do multiply the mask (+1, -1, +1, -1)
97 v3.vector = _mm_mul_ps(v3.vector, v6.vector);
98
99 // I want to set up
100 //
101 // v4[0] = v2[3] = tmp[1][col=2][im]
102 // v4[1] = v2[2] = tmp[1][col=2][re]
103 // v4[2] = v0[1] = tmp[0][col=0][im]
104 // v4[3] = v0[0] = tmp[0][col=0][re]
105 //
106 // We can do this with v4 = v2, v4 = shuf( v4, v2 ). Shuf code is 00 01 10 11 = x1B
107 v4.vector = v2.vector;
108 v4.vector = _mm_shuffle_ps(v4.vector, v0.vector, 0x1B);
109
110 // Now I need to do multiply the mask (+1, -1, +1, -1)
111 v4.vector = _mm_mul_ps(v4.vector, v6.vector);
112
113 // I want to set up
114 //
115 // v5[0] = v0[3] = tmp[0][col=1][im]
116 // v5[1] = v0[2] = tmp[0][col=1][re]
117 // v5[2] = v1[1] = tmp[0][col=2][im]
118 // v5[3] = v1[0] = tmp[0][col=2][re]
119 //
120 // We can do this with v5 = v0, v5 = shuf( v5, v1 ). Shuf code is 00 01 10 11 = x1B
121 v5.vector = v0.vector;
122 v5.vector = _mm_shuffle_ps(v5.vector, v1.vector, 0x1B);
123
124 // Now I need to do multiply the mask (+1, -1, +1, -1)
125 v5.vector = _mm_mul_ps(v5.vector, v6.vector);
126
127 // so I can them out already - bypass cache.
128 _mm_store_ps(dst_shadow+12, v3.vector);
129 _mm_store_ps(dst_shadow+16, v4.vector);
130 _mm_store_ps(dst_shadow+20, v5.vector);
131
132 dst_shadow+=24;
133
134 v0.vector = _mm_load_ps(src_shadow);
135 v1.vector = _mm_load_ps(src_shadow+4);
136 v2.vector = _mm_load_ps(src_shadow+8);
137
138 }
139
140 // Now we have
141 // v0.floats[0] = tmp[0][col=0][re]
142 // v0.floats[1] = tmp[0][col=0][im]
143 // v0.floats[2] = tmp[0][col=1][re]
144 // v0.floats[3] = tmp[0][col=1][im]
145 //
146 // v1.floats[0] = tmp[0][col=2][re]
147 // v1.floats[1] = tmp[0][col=2][im]
148 // v1.floats[2] = tmp[1][col=0][re]
149 // v1.floats[3] = tmp[1][col=0][im]
150 //
151 // v2.floats[0] = tmp[1][col=1][re]
152 // v2.floats[1] = tmp[1][col=1][im]
153 // v2.floats[2] = tmp[1][col=2][re]
154 // v2.floats[3] = tmp[1][col=2][im]
155
156 // These are the top 2 components of the result
157 // so I can store them out already.
158 _mm_store_ps(dst_shadow, v0.vector);
159 _mm_store_ps(dst_shadow+4, v1.vector);
160 _mm_store_ps(dst_shadow+8, v2.vector);
161
162 // I want to set up
163 //
164 // v3[0] = v1[3] = tmp[1][col=0][im]
165 // v3[1] = v1[2] = tmp[1][col=0][re]
166 // v3[2] = v2[1] = tmp[1][col=1][im]
167 // v3[3] = v2[0] = tmp[1][col=1][re]
168 //
169 // We can do this with v3 = v1, v3 = shuf( v3, v1 ). Shuf code is 00 01 10 11 = x1B
170 v3.vector = v1.vector;
171 v3.vector = _mm_shuffle_ps(v3.vector, v2.vector, 0x1B);
172
173 // Now I need to do multiply the mask (+1, -1, +1, -1)
174 v3.vector = _mm_mul_ps(v3.vector, v6.vector);
175
176 // I want to set up
177 //
178 // v4[0] = v2[3] = tmp[1][col=2][im]
179 // v4[1] = v2[2] = tmp[1][col=2][re]
180 // v4[2] = v0[1] = tmp[0][col=0][im]
181 // v4[3] = v0[0] = tmp[0][col=0][re]
182 //
183 // We can do this with v4 = v2, v4 = shuf( v4, v2 ). Shuf code is 00 01 10 11 = x1B
184 v4.vector = v2.vector;
185 v4.vector = _mm_shuffle_ps(v4.vector, v0.vector, 0x1B);
186
187 // Now I need to do multiply the mask (+1, -1, +1, -1)
188 v4.vector = _mm_mul_ps(v4.vector, v6.vector);
189
190 // I want to set up
191 //
192 // v5[0] = v0[3] = tmp[0][col=1][im]
193 // v5[1] = v0[2] = tmp[0][col=1][re]
194 // v5[2] = v1[1] = tmp[0][col=2][im]
195 // v5[3] = v1[0] = tmp[0][col=2][re]
196 //
197 // We can do this with v5 = v0, v5 = shuf( v5, v1 ). Shuf code is 00 01 10 11 = x1B
198 v5.vector = v0.vector;
199 v5.vector = _mm_shuffle_ps(v5.vector, v1.vector, 0x1B);
200
201 // Now I need to do multiply the mask (+1, -1, +1, -1)
202 v5.vector = _mm_mul_ps(v5.vector, v6.vector);
203
204 // so I can them out already - bypass cache.
205 _mm_store_ps(dst_shadow+12, v3.vector);
206 _mm_store_ps(dst_shadow+16, v4.vector);
207 _mm_store_ps(dst_shadow+20, v5.vector);
208
209}
210
223inline
224void inlineSpinReconDir0Minus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
225{
226
227#ifdef DEBUG_GENERIC_SPIN_RECON_INLINES
228 QDPIO::cout << "inlineSpinReconDir0Minus" << endl;
229#endif
230
231 /* ( 1 0 0 -i) ( a0 ) ( a0 - i a3 )
232 * B := ( 1 - Gamma ) A = ( 0 1 -i 0) ( a1 ) = ( a1 - i a2 )
233 * 0 ( 0 i 1 0) ( a2 ) ( a2 + i a1 )
234 * ( i 0 0 1) ( a3 ) ( a3 + i a0 )
235
236 * The bottom components of be may be reconstructed using the formula
237 * ( b2r + i b2i ) = ( {a2r - a1i} + i{a2i + a1r} ) = ( - b1i + i b1r )
238 * ( b3r + i b3i ) ( {a3r - a0i} + i{a3i + a0r} ) ( - b0i + i b0r )
239 */
240
241
242 const REAL32* src_shadow = src;
243 REAL32* dst_shadow = dst;
244
245
246 SSEVec v0, v1, v2, v3, v4, v5, v6;
247 v6.floats[0] = -1;
248 v6.floats[1] = +1;
249 v6.floats[2] = -1;
250 v6.floats[3] = +1;
251
252 v0.vector = _mm_load_ps(src_shadow);
253 v1.vector = _mm_load_ps(src_shadow+4);
254 v2.vector = _mm_load_ps(src_shadow+8);
255
256 for(unsigned int site=0; site < n_vec-1; site++) {
257 src_shadow += 12;
258 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
259
260 // Now we have
261 // v0.floats[0] = tmp[0][col=0][re]
262 // v0.floats[1] = tmp[0][col=0][im]
263 // v0.floats[2] = tmp[0][col=1][re]
264 // v0.floats[3] = tmp[0][col=1][im]
265 //
266 // v1.floats[0] = tmp[0][col=2][re]
267 // v1.floats[1] = tmp[0][col=2][im]
268 // v1.floats[2] = tmp[1][col=0][re]
269 // v1.floats[3] = tmp[1][col=0][im]
270 //
271 // v2.floats[0] = tmp[1][col=1][re]
272 // v2.floats[1] = tmp[1][col=1][im]
273 // v2.floats[2] = tmp[1][col=2][re]
274 // v2.floats[3] = tmp[1][col=2][im]
275
276 // These are the top 2 components of the result
277 // so I can store them out already.
278 _mm_store_ps(dst_shadow, v0.vector);
279 _mm_store_ps(dst_shadow+4, v1.vector);
280 _mm_store_ps(dst_shadow+8, v2.vector);
281
282 // I want to set up
283 //
284 // v3[0] = v1[3] = tmp[1][col=0][im]
285 // v3[1] = v1[2] = tmp[1][col=0][re]
286 // v3[2] = v2[1] = tmp[1][col=1][im]
287 // v3[3] = v2[0] = tmp[1][col=1][re]
288 //
289 // We can do this with v3 = v1, v3 = shuf( v3, v1 ). Shuf code is 00 01 10 11 = x1B
290 v3.vector = v1.vector;
291 v3.vector = _mm_shuffle_ps(v3.vector, v2.vector, 0x1B);
292
293 // Now I need to do multiply the mask (-1,+1,-1,+1)
294 v3.vector = _mm_mul_ps(v3.vector, v6.vector);
295
296 // I want to set up
297 //
298 // v4[0] = v2[3] = tmp[1][col=2][im]
299 // v4[1] = v2[2] = tmp[1][col=2][re]
300 // v4[2] = v0[1] = tmp[0][col=0][im]
301 // v4[3] = v0[0] = tmp[0][col=0][re]
302 //
303 // We can do this with v4 = v2, v4 = shuf( v4, v2 ). Shuf code is 00 01 10 11 = x1B
304 v4.vector = v2.vector;
305 v4.vector = _mm_shuffle_ps(v4.vector, v0.vector, 0x1B);
306
307 // Now I need to do multiply the mask (-1,+1,-1,+1)
308 v4.vector = _mm_mul_ps(v4.vector, v6.vector);
309
310 // I want to set up
311 //
312 // v5[0] = v0[3] = tmp[0][col=1][im]
313 // v5[1] = v0[2] = tmp[0][col=1][re]
314 // v5[2] = v1[1] = tmp[0][col=2][im]
315 // v5[3] = v1[0] = tmp[0][col=2][re]
316 //
317 // We can do this with v5 = v0, v5 = shuf( v5, v1 ). Shuf code is 00 01 10 11 = x1B
318 v5.vector = v0.vector;
319 v5.vector = _mm_shuffle_ps(v5.vector, v1.vector, 0x1B);
320
321 // Now I need to do multiply the mask (-1,+1,-1,+1)
322 v5.vector = _mm_mul_ps(v5.vector, v6.vector);
323
324 // so I can them out already - bypass cache.
325 _mm_store_ps(dst_shadow+12, v3.vector);
326 _mm_store_ps(dst_shadow+16, v4.vector);
327 _mm_store_ps(dst_shadow+20, v5.vector);
328
329 dst_shadow+=24;
330
331 v0.vector = _mm_load_ps(src_shadow);
332 v1.vector = _mm_load_ps(src_shadow+4);
333 v2.vector = _mm_load_ps(src_shadow+8);
334
335 }
336
337 // Now we have
338 // v0.floats[0] = tmp[0][col=0][re]
339 // v0.floats[1] = tmp[0][col=0][im]
340 // v0.floats[2] = tmp[0][col=1][re]
341 // v0.floats[3] = tmp[0][col=1][im]
342 //
343 // v1.floats[0] = tmp[0][col=2][re]
344 // v1.floats[1] = tmp[0][col=2][im]
345 // v1.floats[2] = tmp[1][col=0][re]
346 // v1.floats[3] = tmp[1][col=0][im]
347 //
348 // v2.floats[0] = tmp[1][col=1][re]
349 // v2.floats[1] = tmp[1][col=1][im]
350 // v2.floats[2] = tmp[1][col=2][re]
351 // v2.floats[3] = tmp[1][col=2][im]
352
353 // These are the top 2 components of the result
354 // so I can store them out already.
355 _mm_store_ps(dst_shadow, v0.vector);
356 _mm_store_ps(dst_shadow+4, v1.vector);
357 _mm_store_ps(dst_shadow+8, v2.vector);
358
359 // I want to set up
360 //
361 // v3[0] = v1[3] = tmp[1][col=0][im]
362 // v3[1] = v1[2] = tmp[1][col=0][re]
363 // v3[2] = v2[1] = tmp[1][col=1][im]
364 // v3[3] = v2[0] = tmp[1][col=1][re]
365 //
366 // We can do this with v3 = v1, v3 = shuf( v3, v1 ). Shuf code is 00 01 10 11 = x1B
367 v3.vector = v1.vector;
368 v3.vector = _mm_shuffle_ps(v3.vector, v2.vector, 0x1B);
369
370 // Now I need to do multiply the mask (-1,+1,-1,+1)
371 v3.vector = _mm_mul_ps(v3.vector, v6.vector);
372
373 // I want to set up
374 //
375 // v4[0] = v2[3] = tmp[1][col=2][im]
376 // v4[1] = v2[2] = tmp[1][col=2][re]
377 // v4[2] = v0[1] = tmp[0][col=0][im]
378 // v4[3] = v0[0] = tmp[0][col=0][re]
379 //
380 // We can do this with v4 = v2, v4 = shuf( v4, v2 ). Shuf code is 00 01 10 11 = x1B
381 v4.vector = v2.vector;
382 v4.vector = _mm_shuffle_ps(v4.vector, v0.vector, 0x1B);
383
384 // Now I need to do multiply the mask (-1,+1,-1,+1)
385 v4.vector = _mm_mul_ps(v4.vector, v6.vector);
386
387 // I want to set up
388 //
389 // v5[0] = v0[3] = tmp[0][col=1][im]
390 // v5[1] = v0[2] = tmp[0][col=1][re]
391 // v5[2] = v1[1] = tmp[0][col=2][im]
392 // v5[3] = v1[0] = tmp[0][col=2][re]
393 //
394 // We can do this with v5 = v0, v5 = shuf( v5, v1 ). Shuf code is 00 01 10 11 = x1B
395 v5.vector = v0.vector;
396 v5.vector = _mm_shuffle_ps(v5.vector, v1.vector, 0x1B);
397
398 // Now I need to do multiply the mask (-1,+1,-1,+1)
399 v5.vector = _mm_mul_ps(v5.vector, v6.vector);
400
401 // so I can them out already - bypass cache.
402 _mm_store_ps(dst_shadow+12, v3.vector);
403 _mm_store_ps(dst_shadow+16, v4.vector);
404 _mm_store_ps(dst_shadow+20, v5.vector);
405
406}
407
420inline
421void inlineSpinReconDir1Plus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
422{
423
424#ifdef DEBUG_GENERIC_SPIN_RECON_INLINES
425 QDPIO::cout << "inlineSpinReconDir0Plus" << endl;
426#endif
427
428 /* 1 + \gamma_1 = 1 0 0 -1
429 * 0 1 1 0
430 * 0 1 1 0
431 * -1 0 0 1
432
433 * ( b2r + i b2i ) = ( {a2r + a1r} + i{a2i + a1i} ) = ( b1r + i b1i )
434 * ( b3r + i b3i ) ( {a3r - a0r} + i{a3i - a0i} ) ( - b0r - i b0i )
435
436 */
437
438 const REAL32* src_shadow = src;
439 REAL32* dst_shadow = dst;
440
441
442 SSEVec v0, v1, v2, v3, v4, v5, v6, v7;
443 v6.floats[0] = +1;
444 v6.floats[1] = +1;
445 v6.floats[2] = -1;
446 v6.floats[3] = -1;
447
448
449 v7.floats[0] = -1;
450 v7.floats[1] = -1;
451 v7.floats[2] = -1;
452 v7.floats[3] = -1;
453
454
455 v0.vector = _mm_load_ps(src_shadow);
456 v1.vector = _mm_load_ps(src_shadow+4);
457 v2.vector = _mm_load_ps(src_shadow+8);
458
459 for(unsigned int site=0; site < n_vec-1; site++) {
460
461 src_shadow += 12;
462 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
463
464 // Now we have
465 // v0.floats[0] = tmp[0][col=0][re]
466 // v0.floats[1] = tmp[0][col=0][im]
467 // v0.floats[2] = tmp[0][col=1][re]
468 // v0.floats[3] = tmp[0][col=1][im]
469 //
470 // v1.floats[0] = tmp[0][col=2][re]
471 // v1.floats[1] = tmp[0][col=2][im]
472 // v1.floats[2] = tmp[1][col=0][re]
473 // v1.floats[3] = tmp[1][col=0][im]
474 //
475 // v2.floats[0] = tmp[1][col=1][re]
476 // v2.floats[1] = tmp[1][col=1][im]
477 // v2.floats[2] = tmp[1][col=2][re]
478 // v2.floats[3] = tmp[1][col=2][im]
479
480 // These are the top 2 components of the result
481 // so I can store them out already.
482 _mm_store_ps(dst_shadow, v0.vector);
483 _mm_store_ps(dst_shadow+4, v1.vector);
484 _mm_store_ps(dst_shadow+8, v2.vector);
485
486 // I want to set up
487 //
488 // v3[0] = v1[2] = tmp[1][col=0][re]
489 // v3[1] = v1[3] = tmp[1][col=0][im]
490 // v3[2] = v2[0] = tmp[1][col=1][re]
491 // v3[3] = v2[1] = tmp[1][col=1][im]
492 //
493 // We can do this with v3 = v1, v3 = shuf( v3, v2 ). Shuf code is 01 00 11 10 = x4E
494 v3.vector = v1.vector;
495 v3.vector = _mm_shuffle_ps(v3.vector, v2.vector, 0x4E);
496
497 // I want to set up
498 //
499 // v4[0] = v2[2] = tmp[1][col=2][re]
500 // v4[1] = v2[3] = tmp[1][col=2][im]
501 // v4[2] = v0[0] = tmp[0][col=0][re]
502 // v4[3] = v0[1] = tmp[0][col=0][im]
503 //
504 // We can do this with v4 = v2, v4 = shuf( v4, v0 ). Shuf code is 01 00 11 10 = x4E
505 v4.vector = v2.vector;
506 v4.vector = _mm_shuffle_ps(v4.vector, v0.vector, 0x4E);
507
508 // Need to multiply in mask (+1,+1,-1,-1)
509 v4.vector = _mm_mul_ps(v4.vector, v6.vector);
510
511 // I want to set up
512 //
513 // v5[0] = v0[2] = tmp[0][col=1][re]
514 // v5[1] = v0[3] = tmp[0][col=1][im]
515 // v5[2] = v1[0] = tmp[0][col=2][re]
516 // v5[3] = v1[1] = tmp[0][col=2][im]
517 //
518 // We can do this with v5 = v0, v5 = shuf( v5, v1 ). Shuf code is 01 00 11 10 = x4E
519 v5.vector = v0.vector;
520 v5.vector = _mm_shuffle_ps(v5.vector, v1.vector, 0x4E);
521
522 // Now I need to do multiply the mask (-1,-1,-1,-11)
523 v5.vector = _mm_mul_ps(v5.vector, v7.vector);
524
525 // so I can them out already - bypass cache.
526 _mm_store_ps(dst_shadow+12, v3.vector);
527 _mm_store_ps(dst_shadow+16, v4.vector);
528 _mm_store_ps(dst_shadow+20, v5.vector);
529
530 dst_shadow+=24;
531
532 v0.vector = _mm_load_ps(src_shadow);
533 v1.vector = _mm_load_ps(src_shadow+4);
534 v2.vector = _mm_load_ps(src_shadow+8);
535
536 }
537
538 // Now we have
539 // v0.floats[0] = tmp[0][col=0][re]
540 // v0.floats[1] = tmp[0][col=0][im]
541 // v0.floats[2] = tmp[0][col=1][re]
542 // v0.floats[3] = tmp[0][col=1][im]
543 //
544 // v1.floats[0] = tmp[0][col=2][re]
545 // v1.floats[1] = tmp[0][col=2][im]
546 // v1.floats[2] = tmp[1][col=0][re]
547 // v1.floats[3] = tmp[1][col=0][im]
548 //
549 // v2.floats[0] = tmp[1][col=1][re]
550 // v2.floats[1] = tmp[1][col=1][im]
551 // v2.floats[2] = tmp[1][col=2][re]
552 // v2.floats[3] = tmp[1][col=2][im]
553
554 // These are the top 2 components of the result
555 // so I can store them out already.
556 _mm_store_ps(dst_shadow, v0.vector);
557 _mm_store_ps(dst_shadow+4, v1.vector);
558 _mm_store_ps(dst_shadow+8, v2.vector);
559
560 // I want to set up
561 //
562 // v3[0] = v1[2] = tmp[1][col=0][re]
563 // v3[1] = v1[3] = tmp[1][col=0][im]
564 // v3[2] = v2[0] = tmp[1][col=1][re]
565 // v3[3] = v2[1] = tmp[1][col=1][im]
566 //
567 // We can do this with v3 = v1, v3 = shuf( v3, v2 ). Shuf code is 01 00 11 10 = x4E
568 v3.vector = v1.vector;
569 v3.vector = _mm_shuffle_ps(v3.vector, v2.vector, 0x4E);
570
571 // I want to set up
572 //
573 // v4[0] = v2[2] = tmp[1][col=2][re]
574 // v4[1] = v2[3] = tmp[1][col=2][im]
575 // v4[2] = v0[0] = tmp[0][col=0][re]
576 // v4[3] = v0[1] = tmp[0][col=0][im]
577 //
578 // We can do this with v4 = v2, v4 = shuf( v4, v0 ). Shuf code is 01 00 11 10 = x4E
579 v4.vector = v2.vector;
580 v4.vector = _mm_shuffle_ps(v4.vector, v0.vector, 0x4E);
581
582 // Need to multiply in mask (+1,+1,-1,-1)
583 v4.vector = _mm_mul_ps(v4.vector, v6.vector);
584
585 // I want to set up
586 //
587 // v5[0] = v0[2] = tmp[0][col=1][re]
588 // v5[1] = v0[3] = tmp[0][col=1][im]
589 // v5[2] = v1[0] = tmp[0][col=2][re]
590 // v5[3] = v1[1] = tmp[0][col=2][im]
591 //
592 // We can do this with v5 = v0, v5 = shuf( v5, v1 ). Shuf code is 01 00 11 10 = x4E
593 v5.vector = v0.vector;
594 v5.vector = _mm_shuffle_ps(v5.vector, v1.vector, 0x4E);
595
596 // Now I need to do multiply the mask (-1,-1,-1,-11)
597 v5.vector = _mm_mul_ps(v5.vector, v7.vector);
598
599 // so I can them out already - bypass cache.
600 _mm_store_ps(dst_shadow+12, v3.vector);
601 _mm_store_ps(dst_shadow+16, v4.vector);
602 _mm_store_ps(dst_shadow+20, v5.vector);
603
604}
605
618inline
619void inlineSpinReconDir1Minus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
620{
621
622#ifdef DEBUG_GENERIC_SPIN_RECON_INLINES
623 QDPIO::cout << "inlineSpinReconDir0Minus" << endl;
624#endif
625
626 const REAL32* src_shadow = src;
627 REAL32* dst_shadow = dst;
628
629
630 SSEVec v0, v1, v2, v3, v4, v5, v6, v7;
631 v6.floats[0] = -1;
632 v6.floats[1] = -1;
633 v6.floats[2] = +1;
634 v6.floats[3] = +1;
635
636
637 v7.floats[0] = -1;
638 v7.floats[1] = -1;
639 v7.floats[2] = -1;
640 v7.floats[3] = -1;
641
642 v0.vector = _mm_load_ps(src_shadow);
643 v1.vector = _mm_load_ps(src_shadow+4);
644 v2.vector = _mm_load_ps(src_shadow+8);
645
646 for(unsigned int site=0; site < n_vec-1; site++) {
647
648 src_shadow += 12;
649 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
650
651 // Now we have
652 // v0.floats[0] = tmp[0][col=0][re]
653 // v0.floats[1] = tmp[0][col=0][im]
654 // v0.floats[2] = tmp[0][col=1][re]
655 // v0.floats[3] = tmp[0][col=1][im]
656 //
657 // v1.floats[0] = tmp[0][col=2][re]
658 // v1.floats[1] = tmp[0][col=2][im]
659 // v1.floats[2] = tmp[1][col=0][re]
660 // v1.floats[3] = tmp[1][col=0][im]
661 //
662 // v2.floats[0] = tmp[1][col=1][re]
663 // v2.floats[1] = tmp[1][col=1][im]
664 // v2.floats[2] = tmp[1][col=2][re]
665 // v2.floats[3] = tmp[1][col=2][im]
666
667 // These are the top 2 components of the result
668 // so I can store them out already.
669 _mm_store_ps(dst_shadow, v0.vector);
670 _mm_store_ps(dst_shadow+4, v1.vector);
671 _mm_store_ps(dst_shadow+8, v2.vector);
672
673 // I want to set up
674 //
675 // v3[0] = v1[2] = tmp[1][col=0][re]
676 // v3[1] = v1[3] = tmp[1][col=0][im]
677 // v3[2] = v2[0] = tmp[1][col=1][re]
678 // v3[3] = v2[1] = tmp[1][col=1][im]
679 //
680 // We can do this with v3 = v1, v3 = shuf( v3, v2 ). Shuf code is 01 00 11 10 = x4E
681 v3.vector = v1.vector;
682 v3.vector = _mm_shuffle_ps(v3.vector, v2.vector, 0x4E);
683
684 // Mask (-1,-1,-1,-1)
685 v3.vector = _mm_mul_ps(v3.vector, v7.vector);
686
687 // I want to set up
688 //
689 // v4[0] = v2[2] = tmp[1][col=2][re]
690 // v4[1] = v2[3] = tmp[1][col=2][im]
691 // v4[2] = v0[0] = tmp[0][col=0][re]
692 // v4[3] = v0[1] = tmp[0][col=0][im]
693 //
694 // We can do this with v4 = v2, v4 = shuf( v4, v0 ). Shuf code is 01 00 11 10 = x4E
695 v4.vector = v2.vector;
696 v4.vector = _mm_shuffle_ps(v4.vector, v0.vector, 0x4E);
697
698 // Need to multiply in mask (-1,-1,+1,+1)
699 v4.vector = _mm_mul_ps(v4.vector, v6.vector);
700
701 // I want to set up
702 //
703 // v5[0] = v0[2] = tmp[0][col=1][re]
704 // v5[1] = v0[3] = tmp[0][col=1][im]
705 // v5[2] = v1[0] = tmp[0][col=2][re]
706 // v5[3] = v1[1] = tmp[0][col=2][im]
707 //
708 // We can do this with v5 = v0, v5 = shuf( v5, v1 ). Shuf code is 01 00 11 10 = x4E
709 v5.vector = v0.vector;
710 v5.vector = _mm_shuffle_ps(v5.vector, v1.vector, 0x4E);
711
712 // so I can them out already - bypass cache.
713 _mm_store_ps(dst_shadow+12, v3.vector);
714 _mm_store_ps(dst_shadow+16, v4.vector);
715 _mm_store_ps(dst_shadow+20, v5.vector);
716
717 dst_shadow+=24;
718
719 v0.vector = _mm_load_ps(src_shadow);
720 v1.vector = _mm_load_ps(src_shadow+4);
721 v2.vector = _mm_load_ps(src_shadow+8);
722
723
724 }
725
726 // Now we have
727 // v0.floats[0] = tmp[0][col=0][re]
728 // v0.floats[1] = tmp[0][col=0][im]
729 // v0.floats[2] = tmp[0][col=1][re]
730 // v0.floats[3] = tmp[0][col=1][im]
731 //
732 // v1.floats[0] = tmp[0][col=2][re]
733 // v1.floats[1] = tmp[0][col=2][im]
734 // v1.floats[2] = tmp[1][col=0][re]
735 // v1.floats[3] = tmp[1][col=0][im]
736 //
737 // v2.floats[0] = tmp[1][col=1][re]
738 // v2.floats[1] = tmp[1][col=1][im]
739 // v2.floats[2] = tmp[1][col=2][re]
740 // v2.floats[3] = tmp[1][col=2][im]
741
742 // These are the top 2 components of the result
743 // so I can store them out already.
744 _mm_store_ps(dst_shadow, v0.vector);
745 _mm_store_ps(dst_shadow+4, v1.vector);
746 _mm_store_ps(dst_shadow+8, v2.vector);
747
748 // I want to set up
749 //
750 // v3[0] = v1[2] = tmp[1][col=0][re]
751 // v3[1] = v1[3] = tmp[1][col=0][im]
752 // v3[2] = v2[0] = tmp[1][col=1][re]
753 // v3[3] = v2[1] = tmp[1][col=1][im]
754 //
755 // We can do this with v3 = v1, v3 = shuf( v3, v2 ). Shuf code is 01 00 11 10 = x4E
756 v3.vector = v1.vector;
757 v3.vector = _mm_shuffle_ps(v3.vector, v2.vector, 0x4E);
758
759 // Mask (-1,-1,-1,-1)
760 v3.vector = _mm_mul_ps(v3.vector, v7.vector);
761
762 // I want to set up
763 //
764 // v4[0] = v2[2] = tmp[1][col=2][re]
765 // v4[1] = v2[3] = tmp[1][col=2][im]
766 // v4[2] = v0[0] = tmp[0][col=0][re]
767 // v4[3] = v0[1] = tmp[0][col=0][im]
768 //
769 // We can do this with v4 = v2, v4 = shuf( v4, v0 ). Shuf code is 01 00 11 10 = x4E
770 v4.vector = v2.vector;
771 v4.vector = _mm_shuffle_ps(v4.vector, v0.vector, 0x4E);
772
773 // Need to multiply in mask (-1,-1,+1,+1)
774 v4.vector = _mm_mul_ps(v4.vector, v6.vector);
775
776 // I want to set up
777 //
778 // v5[0] = v0[2] = tmp[0][col=1][re]
779 // v5[1] = v0[3] = tmp[0][col=1][im]
780 // v5[2] = v1[0] = tmp[0][col=2][re]
781 // v5[3] = v1[1] = tmp[0][col=2][im]
782 //
783 // We can do this with v5 = v0, v5 = shuf( v5, v1 ). Shuf code is 01 00 11 10 = x4E
784 v5.vector = v0.vector;
785 v5.vector = _mm_shuffle_ps(v5.vector, v1.vector, 0x4E);
786
787 // so I can them out already - bypass cache.
788 _mm_store_ps(dst_shadow+12, v3.vector);
789 _mm_store_ps(dst_shadow+16, v4.vector);
790 _mm_store_ps(dst_shadow+20, v5.vector);
791
792}
793
794
807inline
808void inlineSpinReconDir2Plus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
809{
810
811#ifdef DEBUG_GENERIC_SPIN_RECON_INLINES
812 QDPIO::cout << "inlineSpinReconDir0Plus" << endl;
813#endif
814
815
816 /* 1 + \gamma_2 = 1 0 i 0
817 * 0 1 0 -i
818 * -i 0 1 0
819 * 0 i 0 1
820 *
821 * ( b2r + i b2i ) = ( {a2r + a0i} + i{a2i - a0r} ) = ( b0i - i b0r )
822 * ( b3r + i b3i ) ( {a3r - a1i} + i{a3i + a1r} ) ( - b1i + i b1r )
823 */
824
825 SSEVec v0, v1, v2, v5, v6, v7;
826 const REAL32* src_shadow = src;
827 REAL32* dst_shadow = dst;
828
829 v5.floats[0] = +1;
830 v5.floats[1] = -1;
831 v5.floats[2] = +1;
832 v5.floats[3] = -1;
833
834 v6.floats[0] = +1;
835 v6.floats[1] = -1;
836 v6.floats[2] = -1;
837 v6.floats[3] = +1;
838
839 v7.floats[0] = -1;
840 v7.floats[1] = +1;
841 v7.floats[2] = -1;
842 v7.floats[3] = +1;
843
844 v0.vector = _mm_load_ps(src_shadow);
845 v1.vector = _mm_load_ps(src_shadow+4);
846 v2.vector = _mm_load_ps(src_shadow+8);
847
848 for(unsigned int site=0; site < n_vec-1; site++) {
849
850 src_shadow += 12;
851 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
852
853 // Now we have
854 // v0.floats[0] = tmp[0][col=0][re]
855 // v0.floats[1] = tmp[0][col=0][im]
856 // v0.floats[2] = tmp[0][col=1][re]
857 // v0.floats[3] = tmp[0][col=1][im]
858 //
859 // v1.floats[0] = tmp[0][col=2][re]
860 // v1.floats[1] = tmp[0][col=2][im]
861 // v1.floats[2] = tmp[1][col=0][re]
862 // v1.floats[3] = tmp[1][col=0][im]
863 //
864 // v2.floats[0] = tmp[1][col=1][re]
865 // v2.floats[1] = tmp[1][col=1][im]
866 // v2.floats[2] = tmp[1][col=2][re]
867 // v2.floats[3] = tmp[1][col=2][im]
868
869 // These are the top 2 components of the result
870 // so I can store them out already.
871 _mm_store_ps(dst_shadow, v0.vector);
872 _mm_store_ps(dst_shadow+4, v1.vector);
873 _mm_store_ps(dst_shadow+8, v2.vector);
874
875 // I want to set up
876 //
877 // v0[0] = v0[1] = tmp[0][col=0][im]
878 // v0[1] = v0[0] = tmp[0][col=0][re]
879 // v0[2] = v0[3] = tmp[0][col=1][im]
880 // v0[3] = v0[2] = tmp[0][col=1][re]
881 //
882 // We can do this with v0 = shuf( v0, v0 ). Shuf code is 10 11 00 01 = xB1
883 v0.vector = _mm_shuffle_ps(v0.vector, v0.vector, 0xB1);
884
885 // Mask (+1,-1,+1,-1)
886 v0.vector = _mm_mul_ps(v0.vector, v5.vector);
887
888 // I want to set up
889 //
890 // v1[0] = v1[1] = tmp[0][col=2][im]
891 // v1[1] = v1[0] = tmp[0][col=2][re]
892 // v1[2] = v1[3] = tmp[1][col=0][im]
893 // v1[3] = v1[2] = tmp[1][col=0][re]
894 //
895 // We can do this with v1 = shuf( v1, v1 ). Shuf code is 10 11 00 01 = xB1
896 v1.vector = _mm_shuffle_ps(v1.vector, v1.vector, 0xB1);
897
898 // Need to multiply in mask (+1,-1,-1,+1)
899 v1.vector = _mm_mul_ps(v1.vector, v6.vector);
900
901 // I want to set up
902 //
903 // v2[0] = v2[1] = tmp[1][col=1][im]
904 // v2[1] = v2[0] = tmp[1][col=1][re]
905 // v2[2] = v2[2] = tmp[1][col=2][im]
906 // v2[3] = v2[3] = tmp[1][col=2][re]
907 //
908 // We can do this with v2 = shuf( v2, v2 ). Shuf code is 10 11 00 01 = xB1
909 v2.vector = _mm_shuffle_ps(v2.vector, v2.vector, 0xB1);
910
911 // Mask in -1, +1, -1, +1 from v7
912 v2.vector = _mm_mul_ps(v2.vector, v7.vector);
913
914 // so I can them out already - bypass cache.
915 _mm_store_ps(dst_shadow+12, v0.vector);
916 _mm_store_ps(dst_shadow+16, v1.vector);
917 _mm_store_ps(dst_shadow+20, v2.vector);
918
919 dst_shadow+=24;
920
921 v0.vector = _mm_load_ps(src_shadow);
922 v1.vector = _mm_load_ps(src_shadow+4);
923 v2.vector = _mm_load_ps(src_shadow+8);
924
925
926 }
927
928 // Now we have
929 // v0.floats[0] = tmp[0][col=0][re]
930 // v0.floats[1] = tmp[0][col=0][im]
931 // v0.floats[2] = tmp[0][col=1][re]
932 // v0.floats[3] = tmp[0][col=1][im]
933 //
934 // v1.floats[0] = tmp[0][col=2][re]
935 // v1.floats[1] = tmp[0][col=2][im]
936 // v1.floats[2] = tmp[1][col=0][re]
937 // v1.floats[3] = tmp[1][col=0][im]
938 //
939 // v2.floats[0] = tmp[1][col=1][re]
940 // v2.floats[1] = tmp[1][col=1][im]
941 // v2.floats[2] = tmp[1][col=2][re]
942 // v2.floats[3] = tmp[1][col=2][im]
943
944 // These are the top 2 components of the result
945 // so I can store them out already.
946 _mm_store_ps(dst_shadow, v0.vector);
947 _mm_store_ps(dst_shadow+4, v1.vector);
948 _mm_store_ps(dst_shadow+8, v2.vector);
949
950 // I want to set up
951 //
952 // v0[0] = v0[1] = tmp[0][col=0][im]
953 // v0[1] = v0[0] = tmp[0][col=0][re]
954 // v0[2] = v0[3] = tmp[0][col=1][im]
955 // v0[3] = v0[2] = tmp[0][col=1][re]
956 //
957 // We can do this with v0 = shuf( v0, v0 ). Shuf code is 10 11 00 01 = xB1
958 v0.vector = _mm_shuffle_ps(v0.vector, v0.vector, 0xB1);
959
960 // Mask (+1,-1,+1,-1)
961 v0.vector = _mm_mul_ps(v0.vector, v5.vector);
962
963 // I want to set up
964 //
965 // v1[0] = v1[1] = tmp[0][col=2][im]
966 // v1[1] = v1[0] = tmp[0][col=2][re]
967 // v1[2] = v1[3] = tmp[1][col=0][im]
968 // v1[3] = v1[2] = tmp[1][col=0][re]
969 //
970 // We can do this with v1 = shuf( v1, v1 ). Shuf code is 10 11 00 01 = xB1
971 v1.vector = _mm_shuffle_ps(v1.vector, v1.vector, 0xB1);
972
973 // Need to multiply in mask (+1,-1,-1,+1)
974 v1.vector = _mm_mul_ps(v1.vector, v6.vector);
975
976 // I want to set up
977 //
978 // v2[0] = v2[1] = tmp[1][col=1][im]
979 // v2[1] = v2[0] = tmp[1][col=1][re]
980 // v2[2] = v2[2] = tmp[1][col=2][im]
981 // v2[3] = v2[3] = tmp[1][col=2][re]
982 //
983 // We can do this with v2 = shuf( v2, v2 ). Shuf code is 10 11 00 01 = xB1
984 v2.vector = _mm_shuffle_ps(v2.vector, v2.vector, 0xB1);
985
986 // Mask in -1, +1, -1, +1 from v7
987 v2.vector = _mm_mul_ps(v2.vector, v7.vector);
988
989 // so I can them out already - bypass cache.
990 _mm_store_ps(dst_shadow+12, v0.vector);
991 _mm_store_ps(dst_shadow+16, v1.vector);
992 _mm_store_ps(dst_shadow+20, v2.vector);
993
994
995}
996
1009inline
1010void inlineSpinReconDir2Minus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
1011{
1012
1013#ifdef DEBUG_GENERIC_SPIN_RECON_INLINES
1014 QDPIO::cout << "inlineSpinReconDir2Minus" << endl;
1015#endif
1016
1017 SSEVec v0, v1, v2, v5, v6, v7;
1018 const REAL32* src_shadow = src;
1019 REAL32* dst_shadow = dst;
1020
1021 v5.floats[0] = -1;
1022 v5.floats[1] = +1;
1023 v5.floats[2] = -1;
1024 v5.floats[3] = +1;
1025
1026 v6.floats[0] = -1;
1027 v6.floats[1] = +1;
1028 v6.floats[2] = +1;
1029 v6.floats[3] = -1;
1030
1031 v7.floats[0] = +1;
1032 v7.floats[1] = -1;
1033 v7.floats[2] = +1;
1034 v7.floats[3] = -1;
1035
1036 v0.vector = _mm_load_ps(src_shadow);
1037 v1.vector = _mm_load_ps(src_shadow+4);
1038 v2.vector = _mm_load_ps(src_shadow+8);
1039
1040 for(unsigned int site=0; site < n_vec-1; site++) {
1041
1042 src_shadow += 12;
1043 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
1044
1045 // Now we have
1046 // v0.floats[0] = tmp[0][col=0][re]
1047 // v0.floats[1] = tmp[0][col=0][im]
1048 // v0.floats[2] = tmp[0][col=1][re]
1049 // v0.floats[3] = tmp[0][col=1][im]
1050 //
1051 // v1.floats[0] = tmp[0][col=2][re]
1052 // v1.floats[1] = tmp[0][col=2][im]
1053 // v1.floats[2] = tmp[1][col=0][re]
1054 // v1.floats[3] = tmp[1][col=0][im]
1055 //
1056 // v2.floats[0] = tmp[1][col=1][re]
1057 // v2.floats[1] = tmp[1][col=1][im]
1058 // v2.floats[2] = tmp[1][col=2][re]
1059 // v2.floats[3] = tmp[1][col=2][im]
1060
1061 // These are the top 2 components of the result
1062 // so I can store them out already.
1063 _mm_store_ps(dst_shadow, v0.vector);
1064 _mm_store_ps(dst_shadow+4, v1.vector);
1065 _mm_store_ps(dst_shadow+8, v2.vector);
1066
1067 // I want to set up
1068 //
1069 // v0[0] = v0[1] = tmp[0][col=0][im]
1070 // v0[1] = v0[0] = tmp[0][col=0][re]
1071 // v0[2] = v0[3] = tmp[0][col=1][im]
1072 // v0[3] = v0[2] = tmp[0][col=1][re]
1073 //
1074 // We can do this with v0 = shuf( v0, v0 ). Shuf code is 10 11 00 01 = xB1
1075 v0.vector = _mm_shuffle_ps(v0.vector, v0.vector, 0xB1);
1076
1077 // Mask (+1,-1,+1,-1)
1078 v0.vector = _mm_mul_ps(v0.vector, v5.vector);
1079
1080 // I want to set up
1081 //
1082 // v1[0] = v1[1] = tmp[0][col=2][im]
1083 // v1[1] = v1[0] = tmp[0][col=2][re]
1084 // v1[2] = v1[3] = tmp[1][col=0][im]
1085 // v1[3] = v1[2] = tmp[1][col=0][re]
1086 //
1087 // We can do this with v1 = shuf( v1, v1 ). Shuf code is 10 11 00 01 = xB1
1088 v1.vector = _mm_shuffle_ps(v1.vector, v1.vector, 0xB1);
1089
1090 // Need to multiply in mask (+1,-1,-1,+1)
1091 v1.vector = _mm_mul_ps(v1.vector, v6.vector);
1092
1093 // I want to set up
1094 //
1095 // v2[0] = v2[1] = tmp[1][col=1][im]
1096 // v2[1] = v2[0] = tmp[1][col=1][re]
1097 // v2[2] = v2[2] = tmp[1][col=2][im]
1098 // v2[3] = v2[3] = tmp[1][col=2][re]
1099 //
1100 // We can do this with v2 = shuf( v2, v2 ). Shuf code is 10 11 00 01 = xB1
1101 v2.vector = _mm_shuffle_ps(v2.vector, v2.vector, 0xB1);
1102
1103 // Mask in -1, +1, -1, +1 from v7
1104 v2.vector = _mm_mul_ps(v2.vector, v7.vector);
1105
1106 // so I can them out already - bypass cache.
1107 _mm_store_ps(dst_shadow+12, v0.vector);
1108 _mm_store_ps(dst_shadow+16, v1.vector);
1109 _mm_store_ps(dst_shadow+20, v2.vector);
1110
1111 dst_shadow+=24;
1112
1113 v0.vector = _mm_load_ps(src_shadow);
1114 v1.vector = _mm_load_ps(src_shadow+4);
1115 v2.vector = _mm_load_ps(src_shadow+8);
1116
1117
1118 }
1119
1120 // Now we have
1121 // v0.floats[0] = tmp[0][col=0][re]
1122 // v0.floats[1] = tmp[0][col=0][im]
1123 // v0.floats[2] = tmp[0][col=1][re]
1124 // v0.floats[3] = tmp[0][col=1][im]
1125 //
1126 // v1.floats[0] = tmp[0][col=2][re]
1127 // v1.floats[1] = tmp[0][col=2][im]
1128 // v1.floats[2] = tmp[1][col=0][re]
1129 // v1.floats[3] = tmp[1][col=0][im]
1130 //
1131 // v2.floats[0] = tmp[1][col=1][re]
1132 // v2.floats[1] = tmp[1][col=1][im]
1133 // v2.floats[2] = tmp[1][col=2][re]
1134 // v2.floats[3] = tmp[1][col=2][im]
1135
1136 // These are the top 2 components of the result
1137 // so I can store them out already.
1138 _mm_store_ps(dst_shadow, v0.vector);
1139 _mm_store_ps(dst_shadow+4, v1.vector);
1140 _mm_store_ps(dst_shadow+8, v2.vector);
1141
1142 // I want to set up
1143 //
1144 // v0[0] = v0[1] = tmp[0][col=0][im]
1145 // v0[1] = v0[0] = tmp[0][col=0][re]
1146 // v0[2] = v0[3] = tmp[0][col=1][im]
1147 // v0[3] = v0[2] = tmp[0][col=1][re]
1148 //
1149 // We can do this with v0 = shuf( v0, v0 ). Shuf code is 10 11 00 01 = xB1
1150 v0.vector = _mm_shuffle_ps(v0.vector, v0.vector, 0xB1);
1151
1152 // Mask (+1,-1,+1,-1)
1153 v0.vector = _mm_mul_ps(v0.vector, v5.vector);
1154
1155 // I want to set up
1156 //
1157 // v1[0] = v1[1] = tmp[0][col=2][im]
1158 // v1[1] = v1[0] = tmp[0][col=2][re]
1159 // v1[2] = v1[3] = tmp[1][col=0][im]
1160 // v1[3] = v1[2] = tmp[1][col=0][re]
1161 //
1162 // We can do this with v1 = shuf( v1, v1 ). Shuf code is 10 11 00 01 = xB1
1163 v1.vector = _mm_shuffle_ps(v1.vector, v1.vector, 0xB1);
1164
1165 // Need to multiply in mask (+1,-1,-1,+1)
1166 v1.vector = _mm_mul_ps(v1.vector, v6.vector);
1167
1168 // I want to set up
1169 //
1170 // v2[0] = v2[1] = tmp[1][col=1][im]
1171 // v2[1] = v2[0] = tmp[1][col=1][re]
1172 // v2[2] = v2[2] = tmp[1][col=2][im]
1173 // v2[3] = v2[3] = tmp[1][col=2][re]
1174 //
1175 // We can do this with v2 = shuf( v2, v2 ). Shuf code is 10 11 00 01 = xB1
1176 v2.vector = _mm_shuffle_ps(v2.vector, v2.vector, 0xB1);
1177
1178 // Mask in -1, +1, -1, +1 from v7
1179 v2.vector = _mm_mul_ps(v2.vector, v7.vector);
1180
1181 // so I can them out already - bypass cache.
1182 _mm_store_ps(dst_shadow+12, v0.vector);
1183 _mm_store_ps(dst_shadow+16, v1.vector);
1184 _mm_store_ps(dst_shadow+20, v2.vector);
1185
1186}
1187
1188
1201inline
1202void inlineSpinReconDir3Plus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
1203{
1204
1205#ifdef DEBUG_GENERIC_SPIN_RECON_INLINES
1206 QDPIO::cout << "inlineSpinReconDir0Plus" << endl;
1207#endif
1208
1209 SSEVec v0, v1, v2;
1210 const REAL32* src_shadow = src;
1211 REAL32* dst_shadow = dst;
1212
1213 /* ( 1 0 1 0) ( a0 ) ( a0 + a2 )
1214 * B := ( 1 + Gamma ) A = ( 0 1 0 1) ( a1 ) = ( a1 + a3 )
1215 * 3 ( 1 0 1 0) ( a2 ) ( a2 + a0 )
1216 * ( 0 1 0 1) ( a3 ) ( a3 + a1 )
1217
1218 * The bottom components of be may be reconstructed using the formula
1219
1220 * ( b2r + i b2i ) = ( {a2r + a0r} + i{a2i + a0i} ) = ( b0r + i b0i )
1221 * ( b3r + i b3i ) ( {a3r + a1r} + i{a3i + a1i} ) ( b1r + i b1i )
1222 */
1223
1224 v0.vector = _mm_load_ps(src_shadow);
1225 v1.vector = _mm_load_ps(src_shadow+4);
1226 v2.vector = _mm_load_ps(src_shadow+8);
1227
1228 for(unsigned int site=0; site < n_vec-1; site++) {
1229
1230 src_shadow += 12;
1231 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
1232
1233 // Now we have
1234 // v0.floats[0] = tmp[0][col=0][re]
1235 // v0.floats[1] = tmp[0][col=0][im]
1236 // v0.floats[2] = tmp[0][col=1][re]
1237 // v0.floats[3] = tmp[0][col=1][im]
1238 //
1239 // v1.floats[0] = tmp[0][col=2][re]
1240 // v1.floats[1] = tmp[0][col=2][im]
1241 // v1.floats[2] = tmp[1][col=0][re]
1242 // v1.floats[3] = tmp[1][col=0][im]
1243 //
1244 // v2.floats[0] = tmp[1][col=1][re]
1245 // v2.floats[1] = tmp[1][col=1][im]
1246 // v2.floats[2] = tmp[1][col=2][re]
1247 // v2.floats[3] = tmp[1][col=2][im]
1248
1249 // This one is easy no shufs needed
1250 _mm_store_ps(dst_shadow, v0.vector);
1251 _mm_store_ps(dst_shadow+4, v1.vector);
1252 _mm_store_ps(dst_shadow+8, v2.vector);
1253
1254 _mm_store_ps(dst_shadow+12, v0.vector);
1255 _mm_store_ps(dst_shadow+16, v1.vector);
1256 _mm_store_ps(dst_shadow+20, v2.vector);
1257
1258 dst_shadow+=24;
1259
1260 v0.vector = _mm_load_ps(src_shadow);
1261 v1.vector = _mm_load_ps(src_shadow+4);
1262 v2.vector = _mm_load_ps(src_shadow+8);
1263
1264
1265 }
1266
1267 // Now we have
1268 // v0.floats[0] = tmp[0][col=0][re]
1269 // v0.floats[1] = tmp[0][col=0][im]
1270 // v0.floats[2] = tmp[0][col=1][re]
1271 // v0.floats[3] = tmp[0][col=1][im]
1272 //
1273 // v1.floats[0] = tmp[0][col=2][re]
1274 // v1.floats[1] = tmp[0][col=2][im]
1275 // v1.floats[2] = tmp[1][col=0][re]
1276 // v1.floats[3] = tmp[1][col=0][im]
1277 //
1278 // v2.floats[0] = tmp[1][col=1][re]
1279 // v2.floats[1] = tmp[1][col=1][im]
1280 // v2.floats[2] = tmp[1][col=2][re]
1281 // v2.floats[3] = tmp[1][col=2][im]
1282
1283 // These are the top 2 components of the result
1284 // so I can store them out already.
1285 // Again no shufs needed
1286 _mm_store_ps(dst_shadow, v0.vector);
1287 _mm_store_ps(dst_shadow+4, v1.vector);
1288 _mm_store_ps(dst_shadow+8, v2.vector);
1289
1290 // so I can them out already - bypass cache.
1291 _mm_store_ps(dst_shadow+12, v0.vector);
1292 _mm_store_ps(dst_shadow+16, v1.vector);
1293 _mm_store_ps(dst_shadow+20, v2.vector);
1294}
1295
1308inline
1309void inlineSpinReconDir3Minus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
1310{
1311
1312#ifdef DEBUG_GENERIC_SPIN_RECON_INLINES
1313 QDPIO::cout << "inlineSpinReconDir0Minus" << endl;
1314#endif
1315
1316 SSEVec v0, v1, v2, v7;
1317 const REAL32* src_shadow = src;
1318 REAL32* dst_shadow = dst;
1319
1320
1321 v7.floats[0] = -1;
1322 v7.floats[1] = -1;
1323 v7.floats[2] = -1;
1324 v7.floats[3] = -1;
1325 /* ( 1 0 -1 0) ( a0 ) ( a0 - a2 )
1326 * B := ( 1 - Gamma ) A = ( 0 1 0 -1) ( a1 ) = ( a1 - a3 )
1327 * 3 (-1 0 1 0) ( a2 ) ( a2 - a0 )
1328 * ( 0 -1 0 1) ( a3 ) ( a3 - a1 )
1329
1330 * The bottom components of be may be reconstructed using the formula
1331 * ( b2r + i b2i ) = ( {a2r - a0r} + i{a2i - a0i} ) = ( - b0r - i b0i )
1332 * ( b3r + i b3i ) ( {a3r - a1r} + i{a3i - a1i} ) ( - b1r - i b1i )
1333 */
1334
1335 v0.vector = _mm_load_ps(src_shadow);
1336 v1.vector = _mm_load_ps(src_shadow+4);
1337 v2.vector = _mm_load_ps(src_shadow+8);
1338
1339 for(unsigned int site=0; site < n_vec-1; site++) {
1340
1341 src_shadow += 12;
1342 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
1343
1344 // Now we have
1345 // v0.floats[0] = tmp[0][col=0][re]
1346 // v0.floats[1] = tmp[0][col=0][im]
1347 // v0.floats[2] = tmp[0][col=1][re]
1348 // v0.floats[3] = tmp[0][col=1][im]
1349 //
1350 // v1.floats[0] = tmp[0][col=2][re]
1351 // v1.floats[1] = tmp[0][col=2][im]
1352 // v1.floats[2] = tmp[1][col=0][re]
1353 // v1.floats[3] = tmp[1][col=0][im]
1354 //
1355 // v2.floats[0] = tmp[1][col=1][re]
1356 // v2.floats[1] = tmp[1][col=1][im]
1357 // v2.floats[2] = tmp[1][col=2][re]
1358 // v2.floats[3] = tmp[1][col=2][im]
1359 _mm_store_ps(dst_shadow, v0.vector);
1360 _mm_store_ps(dst_shadow+4, v1.vector);
1361 _mm_store_ps(dst_shadow+8, v2.vector);
1362
1363 // This one is easy no shufs needed
1364 // multiply by -1,-1,-1,-1
1365 v0.vector = _mm_mul_ps(v0.vector, v7.vector);
1366 v1.vector = _mm_mul_ps(v1.vector, v7.vector);
1367 v2.vector = _mm_mul_ps(v2.vector, v7.vector);
1368
1369 _mm_store_ps(dst_shadow+12, v0.vector);
1370 _mm_store_ps(dst_shadow+16, v1.vector);
1371 _mm_store_ps(dst_shadow+20, v2.vector);
1372
1373 dst_shadow+=24;
1374
1375 v0.vector = _mm_load_ps(src_shadow);
1376 v1.vector = _mm_load_ps(src_shadow+4);
1377 v2.vector = _mm_load_ps(src_shadow+8);
1378
1379
1380 }
1381 // Now we have
1382 // v0.floats[0] = tmp[0][col=0][re]
1383 // v0.floats[1] = tmp[0][col=0][im]
1384 // v0.floats[2] = tmp[0][col=1][re]
1385 // v0.floats[3] = tmp[0][col=1][im]
1386 //
1387 // v1.floats[0] = tmp[0][col=2][re]
1388 // v1.floats[1] = tmp[0][col=2][im]
1389 // v1.floats[2] = tmp[1][col=0][re]
1390 // v1.floats[3] = tmp[1][col=0][im]
1391 //
1392 // v2.floats[0] = tmp[1][col=1][re]
1393 // v2.floats[1] = tmp[1][col=1][im]
1394 // v2.floats[2] = tmp[1][col=2][re]
1395 // v2.floats[3] = tmp[1][col=2][im]
1396 _mm_store_ps(dst_shadow, v0.vector);
1397 _mm_store_ps(dst_shadow+4, v1.vector);
1398 _mm_store_ps(dst_shadow+8, v2.vector);
1399
1400 // This one is easy no shufs needed
1401 // multiply by -1,-1,-1,-1
1402 v0.vector = _mm_mul_ps(v0.vector, v7.vector);
1403 v1.vector = _mm_mul_ps(v1.vector, v7.vector);
1404 v2.vector = _mm_mul_ps(v2.vector, v7.vector);
1405
1406 _mm_store_ps(dst_shadow+12, v0.vector);
1407 _mm_store_ps(dst_shadow+16, v1.vector);
1408 _mm_store_ps(dst_shadow+20, v2.vector);
1409
1410}
1411
1412
1413
1426inline
1427void inlineAddSpinReconDir0Plus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
1428{
1429
1430#ifdef DEBUG_GENERIC_SPIN_RECON_INLINES
1431 QDPIO::cout << "inlinaAddSpinReconDir0Plus" << endl;
1432#endif
1433
1434 /* 1 + \gamma_0 = 1 0 0 i
1435 0 1 i 0
1436 0 -i 1 0
1437 -i 0 0 1
1438
1439 * ( b2r + i b2i ) = ( {a2r + a1i} + i{a2i - a1r} ) = ( b1i - i b1r )
1440 * ( b3r + i b3i ) ( {a3r + a0i} + i{a3i - a0r} ) ( b0i - i b0r )
1441 */
1442
1443 // 12 components in source, 24 in dest
1444 // 3 vectors for source
1445
1446 const REAL32* src_shadow = src;
1447 REAL32* dst_shadow = dst;
1448
1449
1450 SSEVec v0, v1, v2, v3, v4, v5, v6;
1451 v6.floats[0] = +1;
1452 v6.floats[1] = -1;
1453 v6.floats[2] = +1;
1454 v6.floats[3] = -1;
1455
1456
1457 // Load source
1458 v0.vector = _mm_load_ps(src_shadow);
1459 v1.vector = _mm_load_ps(src_shadow+4);
1460 v2.vector = _mm_load_ps(src_shadow+8);
1461
1462 // Load top half of result
1463 v3.vector = _mm_load_ps(dst_shadow);
1464 v4.vector = _mm_load_ps(dst_shadow+4);
1465 v5.vector = _mm_load_ps(dst_shadow+8);
1466
1467 // Add source to result
1468 v3.vector = _mm_add_ps(v3.vector, v0.vector);
1469 v4.vector = _mm_add_ps(v4.vector, v1.vector);
1470 v5.vector = _mm_add_ps(v5.vector, v2.vector);
1471
1472 // Store out result
1473 _mm_store_ps(dst_shadow, v3.vector);
1474 _mm_store_ps(dst_shadow+4, v4.vector);
1475 _mm_store_ps(dst_shadow+8, v5.vector);
1476
1477 for(unsigned int site=0; site < n_vec-1; site++) {
1478
1479
1480 src_shadow += 12;
1481 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
1482
1483 // Now the bottom half. -- preload v4 and v5
1484 v4.vector = _mm_load_ps(dst_shadow+12);
1485 v5.vector = _mm_load_ps(dst_shadow+16);
1486
1487 // I want to set up
1488 //
1489 // v3[0] = v1[3] = tmp[1][col=0][im]
1490 // v3[1] = v1[2] = tmp[1][col=0][re]
1491 // v3[2] = v2[1] = tmp[1][col=1][im]
1492 // v3[3] = v2[0] = tmp[1][col=1][re]
1493 //
1494 // We can do this with v3 = v1, v3 = shuf( v3, v1 ). Shuf code is 00 01 10 11 = x1B
1495 v3.vector = v1.vector;
1496 v3.vector = _mm_shuffle_ps(v3.vector, v2.vector, 0x1B);
1497
1498 // Now I need to do multiply the mask (+1, -1, +1, -1)
1499 v3.vector = _mm_mul_ps(v3.vector, v6.vector);
1500
1501 // Add to the destination
1502 v4.vector = _mm_add_ps(v3.vector, v4.vector);
1503
1504 // v3 now free reuse it with the last element of the destination
1505 v3.vector = _mm_load_ps(dst_shadow+20);
1506 _mm_prefetch((const char *) dst_shadow+20, _MM_HINT_T0);
1507 // Store result
1508 _mm_store_ps(dst_shadow+12, v4.vector);
1509
1510
1511 // V4 is now free?
1512 // I want to set up
1513 //
1514 // v4[0] = v2[3] = tmp[1][col=2][im]
1515 // v4[1] = v2[2] = tmp[1][col=2][re]
1516 // v4[2] = v0[1] = tmp[0][col=0][im]
1517 // v4[3] = v0[0] = tmp[0][col=0][re]
1518 //
1519 // We can do this with v4 = v2, v4 = shuf( v4, v2 ). Shuf code is 00 01 10 11 = x1B
1520 v4.vector = v2.vector;
1521 v4.vector = _mm_shuffle_ps(v4.vector, v0.vector, 0x1B);
1522
1523 // Now I need to do multiply the mask (+1, -1, +1, -1)
1524 v4.vector = _mm_mul_ps(v4.vector, v6.vector);
1525 v5.vector = _mm_add_ps(v5.vector, v4.vector);
1526 _mm_store_ps(dst_shadow+16, v5.vector);
1527
1528 // I want to set up
1529 //
1530 // v5[0] = v0[3] = tmp[0][col=1][im]
1531 // v5[1] = v0[2] = tmp[0][col=1][re]
1532 // v5[2] = v1[1] = tmp[0][col=2][im]
1533 // v5[3] = v1[0] = tmp[0][col=2][re]
1534 //
1535 // We can do this with v5 = v0, v5 = shuf( v5, v1 ). Shuf code is 00 01 10 11 = x1B
1536 v5.vector = v0.vector;
1537 v5.vector = _mm_shuffle_ps(v5.vector, v1.vector, 0x1B);
1538
1539 // Now I need to do multiply the mask (+1, -1, +1, -1)
1540 v5.vector = _mm_mul_ps(v5.vector, v6.vector);
1541 v3.vector = _mm_add_ps(v3.vector, v5.vector);
1542 _mm_store_ps(dst_shadow+20, v3.vector);
1543
1544 dst_shadow+=24;
1545
1546 // Load source
1547 v0.vector = _mm_load_ps(src_shadow);
1548 v1.vector = _mm_load_ps(src_shadow+4);
1549 v2.vector = _mm_load_ps(src_shadow+8);
1550
1551 // Load top half of result
1552 v3.vector = _mm_load_ps(dst_shadow);
1553 v4.vector = _mm_load_ps(dst_shadow+4);
1554 v5.vector = _mm_load_ps(dst_shadow+8);
1555
1556 // Add source to result
1557 v3.vector = _mm_add_ps(v3.vector, v0.vector);
1558 v4.vector = _mm_add_ps(v4.vector, v1.vector);
1559 v5.vector = _mm_add_ps(v5.vector, v2.vector);
1560
1561 // Store out result
1562 _mm_store_ps(dst_shadow, v3.vector);
1563 _mm_store_ps(dst_shadow+4, v4.vector);
1564 _mm_store_ps(dst_shadow+8, v5.vector);
1565
1566 }
1567
1568 // Now the bottom half. -- preload v4 and v5
1569 v4.vector = _mm_load_ps(dst_shadow+12);
1570 v5.vector = _mm_load_ps(dst_shadow+16);
1571
1572 // I want to set up
1573 //
1574 // v3[0] = v1[3] = tmp[1][col=0][im]
1575 // v3[1] = v1[2] = tmp[1][col=0][re]
1576 // v3[2] = v2[1] = tmp[1][col=1][im]
1577 // v3[3] = v2[0] = tmp[1][col=1][re]
1578 //
1579 // We can do this with v3 = v1, v3 = shuf( v3, v1 ). Shuf code is 00 01 10 11 = x1B
1580 v3.vector = v1.vector;
1581 v3.vector = _mm_shuffle_ps(v3.vector, v2.vector, 0x1B);
1582
1583 // Now I need to do multiply the mask (+1, -1, +1, -1)
1584 v3.vector = _mm_mul_ps(v3.vector, v6.vector);
1585
1586 // Add to the destination
1587 v4.vector = _mm_add_ps(v3.vector, v4.vector);
1588
1589 // v3 now free reuse it with the last element of the destination
1590 v3.vector = _mm_load_ps(dst_shadow+20);
1591 _mm_prefetch((const char *) dst_shadow+20, _MM_HINT_T0);
1592
1593 // Store result
1594 _mm_store_ps(dst_shadow+12, v4.vector);
1595
1596
1597 // V4 is now free?
1598 // I want to set up
1599 //
1600 // v4[0] = v2[3] = tmp[1][col=2][im]
1601 // v4[1] = v2[2] = tmp[1][col=2][re]
1602 // v4[2] = v0[1] = tmp[0][col=0][im]
1603 // v4[3] = v0[0] = tmp[0][col=0][re]
1604 //
1605 // We can do this with v4 = v2, v4 = shuf( v4, v2 ). Shuf code is 00 01 10 11 = x1B
1606 v4.vector = v2.vector;
1607 v4.vector = _mm_shuffle_ps(v4.vector, v0.vector, 0x1B);
1608
1609 // Now I need to do multiply the mask (+1, -1, +1, -1)
1610 v4.vector = _mm_mul_ps(v4.vector, v6.vector);
1611 v5.vector = _mm_add_ps(v5.vector, v4.vector);
1612 _mm_store_ps(dst_shadow+16, v5.vector);
1613
1614 // I want to set up
1615 //
1616 // v5[0] = v0[3] = tmp[0][col=1][im]
1617 // v5[1] = v0[2] = tmp[0][col=1][re]
1618 // v5[2] = v1[1] = tmp[0][col=2][im]
1619 // v5[3] = v1[0] = tmp[0][col=2][re]
1620 //
1621 // We can do this with v5 = v0, v5 = shuf( v5, v1 ). Shuf code is 00 01 10 11 = x1B
1622 v5.vector = v0.vector;
1623 v5.vector = _mm_shuffle_ps(v5.vector, v1.vector, 0x1B);
1624
1625 // Now I need to do multiply the mask (+1, -1, +1, -1)
1626 v5.vector = _mm_mul_ps(v5.vector, v6.vector);
1627 v3.vector = _mm_add_ps(v3.vector, v5.vector);
1628 _mm_store_ps(dst_shadow+20, v3.vector);
1629
1630}
1631
1644inline
1645void inlineAddSpinReconDir0Minus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
1646{
1647
1648#ifdef DEBUG_GENERIC_SPIN_RECON_INLINES
1649 QDPIO::cout << "inlinaAddSpinReconDir0Minus" << endl;
1650#endif
1651
1652
1653 /* ( 1 0 0 -i) ( a0 ) ( a0 - i a3 )
1654 * B := ( 1 - Gamma ) A = ( 0 1 -i 0) ( a1 ) = ( a1 - i a2 )
1655 * 0 ( 0 i 1 0) ( a2 ) ( a2 + i a1 )
1656 * ( i 0 0 1) ( a3 ) ( a3 + i a0 )
1657
1658 * The bottom components of be may be reconstructed using the formula
1659 * ( b2r + i b2i ) = ( {a2r - a1i} + i{a2i + a1r} ) = ( - b1i + i b1r )
1660 * ( b3r + i b3i ) ( {a3r - a0i} + i{a3i + a0r} ) ( - b0i + i b0r )
1661 */
1662
1663 const REAL32* src_shadow = src;
1664 REAL32* dst_shadow = dst;
1665
1666
1667 SSEVec v0, v1, v2, v3, v4, v5, v6;
1668 v6.floats[0] = -1;
1669 v6.floats[1] = +1;
1670 v6.floats[2] = -1;
1671 v6.floats[3] = +1;
1672
1673
1674 // Load source
1675 v0.vector = _mm_load_ps(src_shadow);
1676 v1.vector = _mm_load_ps(src_shadow+4);
1677 v2.vector = _mm_load_ps(src_shadow+8);
1678
1679 // Load top half of result
1680 v3.vector = _mm_load_ps(dst_shadow);
1681 v4.vector = _mm_load_ps(dst_shadow+4);
1682 v5.vector = _mm_load_ps(dst_shadow+8);
1683
1684 // Add source to result
1685 v3.vector = _mm_add_ps(v3.vector, v0.vector);
1686 v4.vector = _mm_add_ps(v4.vector, v1.vector);
1687 v5.vector = _mm_add_ps(v5.vector, v2.vector);
1688
1689 // Store out result
1690 _mm_store_ps(dst_shadow, v3.vector);
1691 _mm_store_ps(dst_shadow+4, v4.vector);
1692 _mm_store_ps(dst_shadow+8, v5.vector);
1693
1694 for(unsigned int site=0; site < n_vec-1; site++) {
1695
1696
1697 src_shadow += 12;
1698 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
1699
1700 // Now the bottom half. -- preload v4 and v5
1701 v4.vector = _mm_load_ps(dst_shadow+12);
1702 v5.vector = _mm_load_ps(dst_shadow+16);
1703
1704 // I want to set up
1705 //
1706 // v3[0] = v1[3] = tmp[1][col=0][im]
1707 // v3[1] = v1[2] = tmp[1][col=0][re]
1708 // v3[2] = v2[1] = tmp[1][col=1][im]
1709 // v3[3] = v2[0] = tmp[1][col=1][re]
1710 //
1711 // We can do this with v3 = v1, v3 = shuf( v3, v1 ). Shuf code is 00 01 10 11 = x1B
1712 v3.vector = v1.vector;
1713 v3.vector = _mm_shuffle_ps(v3.vector, v2.vector, 0x1B);
1714
1715 // Now I need to do multiply the mask (+1, -1, +1, -1)
1716 v3.vector = _mm_mul_ps(v3.vector, v6.vector);
1717
1718 // Add to the destination
1719 v4.vector = _mm_add_ps(v3.vector, v4.vector);
1720
1721 // v3 now free reuse it with the last element of the destination
1722 v3.vector = _mm_load_ps(dst_shadow+20);
1723 _mm_prefetch((const char *) dst_shadow+20, _MM_HINT_T0);
1724
1725 // Store result
1726 _mm_store_ps(dst_shadow+12, v4.vector);
1727
1728
1729 // V4 is now free?
1730 // I want to set up
1731 //
1732 // v4[0] = v2[3] = tmp[1][col=2][im]
1733 // v4[1] = v2[2] = tmp[1][col=2][re]
1734 // v4[2] = v0[1] = tmp[0][col=0][im]
1735 // v4[3] = v0[0] = tmp[0][col=0][re]
1736 //
1737 // We can do this with v4 = v2, v4 = shuf( v4, v2 ). Shuf code is 00 01 10 11 = x1B
1738 v4.vector = v2.vector;
1739 v4.vector = _mm_shuffle_ps(v4.vector, v0.vector, 0x1B);
1740
1741 // Now I need to do multiply the mask (+1, -1, +1, -1)
1742 v4.vector = _mm_mul_ps(v4.vector, v6.vector);
1743 v5.vector = _mm_add_ps(v5.vector, v4.vector);
1744 _mm_store_ps(dst_shadow+16, v5.vector);
1745
1746 // I want to set up
1747 //
1748 // v5[0] = v0[3] = tmp[0][col=1][im]
1749 // v5[1] = v0[2] = tmp[0][col=1][re]
1750 // v5[2] = v1[1] = tmp[0][col=2][im]
1751 // v5[3] = v1[0] = tmp[0][col=2][re]
1752 //
1753 // We can do this with v5 = v0, v5 = shuf( v5, v1 ). Shuf code is 00 01 10 11 = x1B
1754 v5.vector = v0.vector;
1755 v5.vector = _mm_shuffle_ps(v5.vector, v1.vector, 0x1B);
1756
1757 // Now I need to do multiply the mask (+1, -1, +1, -1)
1758 v5.vector = _mm_mul_ps(v5.vector, v6.vector);
1759 v3.vector = _mm_add_ps(v3.vector, v5.vector);
1760 _mm_store_ps(dst_shadow+20, v3.vector);
1761
1762 dst_shadow+=24;
1763
1764 // Load source
1765 v0.vector = _mm_load_ps(src_shadow);
1766 v1.vector = _mm_load_ps(src_shadow+4);
1767 v2.vector = _mm_load_ps(src_shadow+8);
1768
1769 // Load top half of result
1770 v3.vector = _mm_load_ps(dst_shadow);
1771 v4.vector = _mm_load_ps(dst_shadow+4);
1772 v5.vector = _mm_load_ps(dst_shadow+8);
1773
1774 // Add source to result
1775 v3.vector = _mm_add_ps(v3.vector, v0.vector);
1776 v4.vector = _mm_add_ps(v4.vector, v1.vector);
1777 v5.vector = _mm_add_ps(v5.vector, v2.vector);
1778
1779 // Store out result
1780 _mm_store_ps(dst_shadow, v3.vector);
1781 _mm_store_ps(dst_shadow+4, v4.vector);
1782 _mm_store_ps(dst_shadow+8, v5.vector);
1783
1784 }
1785
1786 // Now the bottom half. -- preload v4 and v5
1787 v4.vector = _mm_load_ps(dst_shadow+12);
1788 v5.vector = _mm_load_ps(dst_shadow+16);
1789
1790 // I want to set up
1791 //
1792 // v3[0] = v1[3] = tmp[1][col=0][im]
1793 // v3[1] = v1[2] = tmp[1][col=0][re]
1794 // v3[2] = v2[1] = tmp[1][col=1][im]
1795 // v3[3] = v2[0] = tmp[1][col=1][re]
1796 //
1797 // We can do this with v3 = v1, v3 = shuf( v3, v1 ). Shuf code is 00 01 10 11 = x1B
1798 v3.vector = v1.vector;
1799 v3.vector = _mm_shuffle_ps(v3.vector, v2.vector, 0x1B);
1800
1801 // Now I need to do multiply the mask (+1, -1, +1, -1)
1802 v3.vector = _mm_mul_ps(v3.vector, v6.vector);
1803
1804 // Add to the destination
1805 v4.vector = _mm_add_ps(v3.vector, v4.vector);
1806
1807 // v3 now free reuse it with the last element of the destination
1808 v3.vector = _mm_load_ps(dst_shadow+20);
1809 _mm_prefetch((const char *) dst_shadow+20, _MM_HINT_T0);
1810
1811 // Store result
1812 _mm_store_ps(dst_shadow+12, v4.vector);
1813
1814
1815 // V4 is now free?
1816 // I want to set up
1817 //
1818 // v4[0] = v2[3] = tmp[1][col=2][im]
1819 // v4[1] = v2[2] = tmp[1][col=2][re]
1820 // v4[2] = v0[1] = tmp[0][col=0][im]
1821 // v4[3] = v0[0] = tmp[0][col=0][re]
1822 //
1823 // We can do this with v4 = v2, v4 = shuf( v4, v2 ). Shuf code is 00 01 10 11 = x1B
1824 v4.vector = v2.vector;
1825 v4.vector = _mm_shuffle_ps(v4.vector, v0.vector, 0x1B);
1826
1827 // Now I need to do multiply the mask (+1, -1, +1, -1)
1828 v4.vector = _mm_mul_ps(v4.vector, v6.vector);
1829 v5.vector = _mm_add_ps(v5.vector, v4.vector);
1830 _mm_store_ps(dst_shadow+16, v5.vector);
1831
1832 // I want to set up
1833 //
1834 // v5[0] = v0[3] = tmp[0][col=1][im]
1835 // v5[1] = v0[2] = tmp[0][col=1][re]
1836 // v5[2] = v1[1] = tmp[0][col=2][im]
1837 // v5[3] = v1[0] = tmp[0][col=2][re]
1838 //
1839 // We can do this with v5 = v0, v5 = shuf( v5, v1 ). Shuf code is 00 01 10 11 = x1B
1840 v5.vector = v0.vector;
1841 v5.vector = _mm_shuffle_ps(v5.vector, v1.vector, 0x1B);
1842
1843 // Now I need to do multiply the mask (+1, -1, +1, -1)
1844 v5.vector = _mm_mul_ps(v5.vector, v6.vector);
1845 v3.vector = _mm_add_ps(v3.vector, v5.vector);
1846 _mm_store_ps(dst_shadow+20, v3.vector);
1847
1848
1849
1850}
1851
1864inline
1865void inlineAddSpinReconDir1Plus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
1866{
1867
1868#ifdef DEBUG_GENERIC_SPIN_RECON_INLINES
1869 QDPIO::cout << "inlinaAddSpinReconDir0Plus" << endl;
1870#endif
1871
1872 const REAL32* src_shadow = src;
1873 REAL32* dst_shadow = dst;
1874
1875 /* 1 + \gamma_1 = 1 0 0 -1
1876 * 0 1 1 0
1877 * 0 1 1 0
1878 * -1 0 0 1
1879
1880 * ( b2r + i b2i ) = ( {a2r + a1r} + i{a2i + a1i} ) = ( b1r + i b1i )
1881 * ( b3r + i b3i ) ( {a3r - a0r} + i{a3i - a0i} ) ( - b0r - i b0i )
1882
1883 */
1884
1885
1886 SSEVec v0, v1, v2, v3, v4, v5, v6;
1887 v6.floats[0] = +1;
1888 v6.floats[1] = +1;
1889 v6.floats[2] = -1;
1890 v6.floats[3] = -1;
1891
1892
1893 // Load source
1894 v0.vector = _mm_load_ps(src_shadow);
1895 v1.vector = _mm_load_ps(src_shadow+4);
1896 v2.vector = _mm_load_ps(src_shadow+8);
1897
1898 // Load top half of result
1899 v3.vector = _mm_load_ps(dst_shadow);
1900 v4.vector = _mm_load_ps(dst_shadow+4);
1901 v5.vector = _mm_load_ps(dst_shadow+8);
1902
1903 // Add source to result
1904 v3.vector = _mm_add_ps(v3.vector, v0.vector);
1905 v4.vector = _mm_add_ps(v4.vector, v1.vector);
1906 v5.vector = _mm_add_ps(v5.vector, v2.vector);
1907
1908 // Store out result
1909 _mm_store_ps(dst_shadow, v3.vector);
1910 _mm_store_ps(dst_shadow+4, v4.vector);
1911 _mm_store_ps(dst_shadow+8, v5.vector);
1912
1913 for(unsigned int site=0; site < n_vec-1; site++) {
1914
1915
1916 src_shadow += 12;
1917 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
1918
1919 // Now the bottom half. -- preload v4 and v5
1920 v4.vector = _mm_load_ps(dst_shadow+12);
1921 v5.vector = _mm_load_ps(dst_shadow+16);
1922
1923 // I want to set up
1924 //
1925 // v3[0] = v1[2] = tmp[1][col=0][re]
1926 // v3[1] = v1[3] = tmp[1][col=0][im]
1927 // v3[2] = v2[0] = tmp[1][col=1][re]
1928 // v3[3] = v2[1] = tmp[1][col=1][im]
1929 //
1930 // We can do this with v3 = v1, v3 = shuf( v3, v2 ). Shuf code is 01 00 11 10 = x4E
1931 v3.vector = v1.vector;
1932 v3.vector = _mm_shuffle_ps(v3.vector, v2.vector, 0x4E);
1933
1934 // Add to result
1935 v4.vector = _mm_add_ps(v4.vector, v3.vector);
1936 _mm_store_ps( dst_shadow+12, v4.vector);
1937
1938 v3.vector = _mm_load_ps(dst_shadow+20);
1939 _mm_prefetch((const char *) dst_shadow+20, _MM_HINT_T0);
1940
1941 // I want to set up
1942 //
1943 // v4[0] = v2[2] = tmp[1][col=2][re]
1944 // v4[1] = v2[3] = tmp[1][col=2][im]
1945 // v4[2] = v0[0] = tmp[0][col=0][re]
1946 // v4[3] = v0[1] = tmp[0][col=0][im]
1947 //
1948 // We can do this with v4 = v2, v4 = shuf( v4, v0 ). Shuf code is 01 00 11 10 = x4E
1949 v4.vector = v2.vector;
1950 v4.vector = _mm_shuffle_ps(v4.vector, v0.vector, 0x4E);
1951
1952 // Need to multiply in mask (+1,+1,-1,-1)
1953 v4.vector = _mm_mul_ps(v4.vector, v6.vector);
1954
1955 // Add to result
1956 v5.vector = _mm_add_ps(v5.vector, v4.vector);
1957 _mm_store_ps( dst_shadow+16, v5.vector);
1958
1959 // I want to set up
1960 //
1961 // v5[0] = v0[2] = tmp[0][col=1][re]
1962 // v5[1] = v0[3] = tmp[0][col=1][im]
1963 // v5[2] = v1[0] = tmp[0][col=2][re]
1964 // v5[3] = v1[1] = tmp[0][col=2][im]
1965 //
1966 // We can do this with v5 = v0, v5 = shuf( v5, v1 ). Shuf code is 01 00 11 10 = x4E
1967 v5.vector = v0.vector;
1968 v5.vector = _mm_shuffle_ps(v5.vector, v1.vector, 0x4E);
1969
1970 // Now I need to do multiply the mask (-1,-1,-1,-11)
1971 v3.vector = _mm_sub_ps(v3.vector, v5.vector);
1972
1973 // so I can them out already - bypass cache.
1974 _mm_store_ps(dst_shadow+20, v3.vector);
1975
1976 dst_shadow+=24;
1977
1978 // Load source
1979 v0.vector = _mm_load_ps(src_shadow);
1980 v1.vector = _mm_load_ps(src_shadow+4);
1981 v2.vector = _mm_load_ps(src_shadow+8);
1982
1983 // Load top half of result
1984 v3.vector = _mm_load_ps(dst_shadow);
1985 v4.vector = _mm_load_ps(dst_shadow+4);
1986 v5.vector = _mm_load_ps(dst_shadow+8);
1987
1988 // Add source to result
1989 v3.vector = _mm_add_ps(v3.vector, v0.vector);
1990 v4.vector = _mm_add_ps(v4.vector, v1.vector);
1991 v5.vector = _mm_add_ps(v5.vector, v2.vector);
1992
1993 // Store out result
1994 _mm_store_ps(dst_shadow, v3.vector);
1995 _mm_store_ps(dst_shadow+4, v4.vector);
1996 _mm_store_ps(dst_shadow+8, v5.vector);
1997
1998 }
1999
2000 // Now the bottom half. -- preload v4 and v5
2001 v4.vector = _mm_load_ps(dst_shadow+12);
2002 v5.vector = _mm_load_ps(dst_shadow+16);
2003
2004 // I want to set up
2005 //
2006 // v3[0] = v1[2] = tmp[1][col=0][re]
2007 // v3[1] = v1[3] = tmp[1][col=0][im]
2008 // v3[2] = v2[0] = tmp[1][col=1][re]
2009 // v3[3] = v2[1] = tmp[1][col=1][im]
2010 //
2011 // We can do this with v3 = v1, v3 = shuf( v3, v2 ). Shuf code is 01 00 11 10 = x4E
2012 v3.vector = v1.vector;
2013 v3.vector = _mm_shuffle_ps(v3.vector, v2.vector, 0x4E);
2014
2015 // Add to result
2016 v4.vector = _mm_add_ps(v4.vector, v3.vector);
2017 _mm_store_ps( dst_shadow+12, v4.vector);
2018
2019 v3.vector = _mm_load_ps(dst_shadow+20);
2020 _mm_prefetch((const char *) dst_shadow+20, _MM_HINT_T0);
2021 // I want to set up
2022 //
2023 // v4[0] = v2[2] = tmp[1][col=2][re]
2024 // v4[1] = v2[3] = tmp[1][col=2][im]
2025 // v4[2] = v0[0] = tmp[0][col=0][re]
2026 // v4[3] = v0[1] = tmp[0][col=0][im]
2027 //
2028 // We can do this with v4 = v2, v4 = shuf( v4, v0 ). Shuf code is 01 00 11 10 = x4E
2029 v4.vector = v2.vector;
2030 v4.vector = _mm_shuffle_ps(v4.vector, v0.vector, 0x4E);
2031
2032 // Need to multiply in mask (+1,+1,-1,-1)
2033 v4.vector = _mm_mul_ps(v4.vector, v6.vector);
2034
2035 // Add to result
2036 v5.vector = _mm_add_ps(v5.vector, v4.vector);
2037 _mm_store_ps( dst_shadow+16, v5.vector);
2038
2039 // I want to set up
2040 //
2041 // v5[0] = v0[2] = tmp[0][col=1][re]
2042 // v5[1] = v0[3] = tmp[0][col=1][im]
2043 // v5[2] = v1[0] = tmp[0][col=2][re]
2044 // v5[3] = v1[1] = tmp[0][col=2][im]
2045 //
2046 // We can do this with v5 = v0, v5 = shuf( v5, v1 ). Shuf code is 01 00 11 10 = x4E
2047 v5.vector = v0.vector;
2048 v5.vector = _mm_shuffle_ps(v5.vector, v1.vector, 0x4E);
2049
2050 // Now I need to do multiply the mask (-1,-1,-1,-11)
2051 v3.vector = _mm_sub_ps(v3.vector, v5.vector);
2052
2053 // so I can them out already - bypass cache.
2054 _mm_store_ps(dst_shadow+20, v3.vector);
2055
2056#if 0
2057
2058 REAL32 tmp_hspinor[4][3][2];
2059
2060 const REAL32* src_shadow = src;
2061 REAL32* dst_shadow = dst;
2062
2063 const int re = 0;
2064 const int im = 1;
2065 const int Ncmpx = 2;
2066 const int Nsby2 = 2;
2067
2068 for(int site=0; site < n_vec; site++) {
2069 REAL32* tmp_shadow = &(tmp_hspinor[0][0][0]);
2070
2071 // Store in the half spinor - write out the first two components
2072 for(int store=0; store < Nsby2*Nc*Ncmpx; store++) {
2073 REAL32 tmp = *(src_shadow++);
2074 *(tmp_shadow++) = tmp;
2075 *(dst_shadow++) += tmp;
2076 }
2077
2078 // Reconstruct Spin 2
2079 for(int col=0; col < Nc; col++) {
2080 *(dst_shadow++) += tmp_hspinor[1][col][re];
2081 *(dst_shadow++) += tmp_hspinor[1][col][im];
2082 }
2083
2084 // Reconstruct Spin 3;
2085 for(int col=0; col < Nc; col++) {
2086 *(dst_shadow++) -= tmp_hspinor[0][col][re];
2087 *(dst_shadow++) -= tmp_hspinor[0][col][im];
2088 }
2089
2090
2091 }
2092#endif
2093}
2094
2107inline
2108void inlineAddSpinReconDir1Minus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
2109{
2110
2111#ifdef DEBUG_GENERIC_SPIN_RECON_INLINES
2112 QDPIO::cout << "inlinaAddSpinReconDir0Minus" << endl;
2113#endif
2114
2115 const REAL32* src_shadow = src;
2116 REAL32* dst_shadow = dst;
2117
2118 /* 1 + \gamma_1 = 1 0 0 -1
2119 * 0 1 1 0
2120 * 0 1 1 0
2121 * -1 0 0 1
2122
2123 * ( b2r + i b2i ) = ( {a2r + a1r} + i{a2i + a1i} ) = ( b1r + i b1i )
2124 * ( b3r + i b3i ) ( {a3r - a0r} + i{a3i - a0i} ) ( - b0r - i b0i )
2125
2126 */
2127
2128
2129 SSEVec v0, v1, v2, v3, v4, v5, v6;
2130 v6.floats[0] = -1;
2131 v6.floats[1] = -1;
2132 v6.floats[2] = +1;
2133 v6.floats[3] = +1;
2134
2135
2136 // Load source
2137 v0.vector = _mm_load_ps(src_shadow);
2138 v1.vector = _mm_load_ps(src_shadow+4);
2139 v2.vector = _mm_load_ps(src_shadow+8);
2140
2141 // Load top half of result
2142 v3.vector = _mm_load_ps(dst_shadow);
2143 v4.vector = _mm_load_ps(dst_shadow+4);
2144 v5.vector = _mm_load_ps(dst_shadow+8);
2145
2146 // Add source to result
2147 v3.vector = _mm_add_ps(v3.vector, v0.vector);
2148 v4.vector = _mm_add_ps(v4.vector, v1.vector);
2149 v5.vector = _mm_add_ps(v5.vector, v2.vector);
2150
2151 // Store out result
2152 _mm_store_ps(dst_shadow, v3.vector);
2153 _mm_store_ps(dst_shadow+4, v4.vector);
2154 _mm_store_ps(dst_shadow+8, v5.vector);
2155
2156 for(unsigned int site=0; site < n_vec-1; site++) {
2157
2158 src_shadow += 12;
2159 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
2160
2161 // Now the bottom half. -- preload v4 and v5
2162 v4.vector = _mm_load_ps(dst_shadow+12);
2163 v5.vector = _mm_load_ps(dst_shadow+16);
2164
2165 // I want to set up
2166 //
2167 // v3[0] = v1[2] = tmp[1][col=0][re]
2168 // v3[1] = v1[3] = tmp[1][col=0][im]
2169 // v3[2] = v2[0] = tmp[1][col=1][re]
2170 // v3[3] = v2[1] = tmp[1][col=1][im]
2171 //
2172 // We can do this with v3 = v1, v3 = shuf( v3, v2 ). Shuf code is 01 00 11 10 = x4E
2173 v3.vector = v1.vector;
2174 v3.vector = _mm_shuffle_ps(v3.vector, v2.vector, 0x4E);
2175
2176 // Add to result
2177 v4.vector = _mm_sub_ps(v4.vector, v3.vector);
2178 _mm_store_ps( dst_shadow+12, v4.vector);
2179
2180 v3.vector = _mm_load_ps(dst_shadow+20);
2181 _mm_prefetch((const char *) dst_shadow+20, _MM_HINT_T0);
2182 // I want to set up
2183 //
2184 // v4[0] = v2[2] = tmp[1][col=2][re]
2185 // v4[1] = v2[3] = tmp[1][col=2][im]
2186 // v4[2] = v0[0] = tmp[0][col=0][re]
2187 // v4[3] = v0[1] = tmp[0][col=0][im]
2188 //
2189 // We can do this with v4 = v2, v4 = shuf( v4, v0 ). Shuf code is 01 00 11 10 = x4E
2190 v4.vector = v2.vector;
2191 v4.vector = _mm_shuffle_ps(v4.vector, v0.vector, 0x4E);
2192
2193 // Need to multiply in mask (+1,+1,-1,-1)
2194 v4.vector = _mm_mul_ps(v4.vector, v6.vector);
2195
2196 // Add to result
2197 v5.vector = _mm_add_ps(v5.vector, v4.vector);
2198 _mm_store_ps( dst_shadow+16, v5.vector);
2199
2200 // I want to set up
2201 //
2202 // v5[0] = v0[2] = tmp[0][col=1][re]
2203 // v5[1] = v0[3] = tmp[0][col=1][im]
2204 // v5[2] = v1[0] = tmp[0][col=2][re]
2205 // v5[3] = v1[1] = tmp[0][col=2][im]
2206 //
2207 // We can do this with v5 = v0, v5 = shuf( v5, v1 ). Shuf code is 01 00 11 10 = x4E
2208 v5.vector = v0.vector;
2209 v5.vector = _mm_shuffle_ps(v5.vector, v1.vector, 0x4E);
2210
2211 // Now I need to do multiply the mask (-1,-1,-1,-11)
2212 v3.vector = _mm_add_ps(v3.vector, v5.vector);
2213
2214 // so I can them out already - bypass cache.
2215 _mm_store_ps(dst_shadow+20, v3.vector);
2216
2217 dst_shadow+=24;
2218
2219 // Load source
2220 v0.vector = _mm_load_ps(src_shadow);
2221 v1.vector = _mm_load_ps(src_shadow+4);
2222 v2.vector = _mm_load_ps(src_shadow+8);
2223
2224 // Load top half of result
2225 v3.vector = _mm_load_ps(dst_shadow);
2226 v4.vector = _mm_load_ps(dst_shadow+4);
2227 v5.vector = _mm_load_ps(dst_shadow+8);
2228
2229 // Add source to result
2230 v3.vector = _mm_add_ps(v3.vector, v0.vector);
2231 v4.vector = _mm_add_ps(v4.vector, v1.vector);
2232 v5.vector = _mm_add_ps(v5.vector, v2.vector);
2233
2234 // Store out result
2235 _mm_store_ps(dst_shadow, v3.vector);
2236 _mm_store_ps(dst_shadow+4, v4.vector);
2237 _mm_store_ps(dst_shadow+8, v5.vector);
2238
2239 }
2240
2241 // Now the bottom half. -- preload v4 and v5
2242 v4.vector = _mm_load_ps(dst_shadow+12);
2243 v5.vector = _mm_load_ps(dst_shadow+16);
2244
2245 // I want to set up
2246 //
2247 // v3[0] = v1[2] = tmp[1][col=0][re]
2248 // v3[1] = v1[3] = tmp[1][col=0][im]
2249 // v3[2] = v2[0] = tmp[1][col=1][re]
2250 // v3[3] = v2[1] = tmp[1][col=1][im]
2251 //
2252 // We can do this with v3 = v1, v3 = shuf( v3, v2 ). Shuf code is 01 00 11 10 = x4E
2253 v3.vector = v1.vector;
2254 v3.vector = _mm_shuffle_ps(v3.vector, v2.vector, 0x4E);
2255
2256 // Add to result
2257 v4.vector = _mm_sub_ps(v4.vector, v3.vector);
2258 _mm_store_ps( dst_shadow+12, v4.vector);
2259
2260 v3.vector = _mm_load_ps(dst_shadow+20);
2261 _mm_prefetch((const char *) dst_shadow+20, _MM_HINT_T0);
2262 // I want to set up
2263 //
2264 // v4[0] = v2[2] = tmp[1][col=2][re]
2265 // v4[1] = v2[3] = tmp[1][col=2][im]
2266 // v4[2] = v0[0] = tmp[0][col=0][re]
2267 // v4[3] = v0[1] = tmp[0][col=0][im]
2268 //
2269 // We can do this with v4 = v2, v4 = shuf( v4, v0 ). Shuf code is 01 00 11 10 = x4E
2270 v4.vector = v2.vector;
2271 v4.vector = _mm_shuffle_ps(v4.vector, v0.vector, 0x4E);
2272
2273 // Need to multiply in mask (+1,+1,-1,-1)
2274 v4.vector = _mm_mul_ps(v4.vector, v6.vector);
2275
2276 // Add to result
2277 v5.vector = _mm_add_ps(v5.vector, v4.vector);
2278 _mm_store_ps( dst_shadow+16, v5.vector);
2279
2280 // I want to set up
2281 //
2282 // v5[0] = v0[2] = tmp[0][col=1][re]
2283 // v5[1] = v0[3] = tmp[0][col=1][im]
2284 // v5[2] = v1[0] = tmp[0][col=2][re]
2285 // v5[3] = v1[1] = tmp[0][col=2][im]
2286 //
2287 // We can do this with v5 = v0, v5 = shuf( v5, v1 ). Shuf code is 01 00 11 10 = x4E
2288 v5.vector = v0.vector;
2289 v5.vector = _mm_shuffle_ps(v5.vector, v1.vector, 0x4E);
2290
2291 // Now I need to do multiply the mask (-1,-1,-1,-11)
2292 v3.vector = _mm_add_ps(v3.vector, v5.vector);
2293
2294 // so I can them out already - bypass cache.
2295 _mm_store_ps(dst_shadow+20, v3.vector);
2296
2297}
2298
2299
2312inline
2313void inlineAddSpinReconDir2Plus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
2314{
2315
2316#ifdef DEBUG_GENERIC_SPIN_RECON_INLINES
2317 QDPIO::cout << "inlinaAddSpinReconDir0Plus" << endl;
2318#endif
2319
2320
2321 const REAL32* src_shadow = src;
2322 REAL32* dst_shadow = dst;
2323
2324
2325 /* 1 + \gamma_2 = 1 0 i 0
2326 * 0 1 0 -i
2327 * -i 0 1 0
2328 * 0 i 0 1
2329 *
2330 * ( b2r + i b2i ) = ( {a2r + a0i} + i{a2i - a0r} ) = ( b0i - i b0r )
2331 * ( b3r + i b3i ) ( {a3r - a1i} + i{a3i + a1r} ) ( - b1i + i b1r )
2332 */
2333 SSEVec v0, v1, v2, v3, v4, v5, v6, v7;
2334 v6.floats[0] = +1;
2335 v6.floats[1] = -1;
2336 v6.floats[2] = +1;
2337 v6.floats[3] = -1;
2338
2339 v7.floats[0] = +1;
2340 v7.floats[1] = -1;
2341 v7.floats[2] = -1;
2342 v7.floats[3] = +1;
2343
2344 // Load source
2345 v0.vector = _mm_load_ps(src_shadow);
2346 v1.vector = _mm_load_ps(src_shadow+4);
2347 v2.vector = _mm_load_ps(src_shadow+8);
2348
2349 // Load top half of result
2350 v3.vector = _mm_load_ps(dst_shadow);
2351 v4.vector = _mm_load_ps(dst_shadow+4);
2352 v5.vector = _mm_load_ps(dst_shadow+8);
2353
2354 // Add source to result
2355 v3.vector = _mm_add_ps(v3.vector, v0.vector);
2356 v4.vector = _mm_add_ps(v4.vector, v1.vector);
2357 v5.vector = _mm_add_ps(v5.vector, v2.vector);
2358
2359 // Store out result
2360 _mm_store_ps(dst_shadow, v3.vector);
2361 _mm_store_ps(dst_shadow+4, v4.vector);
2362 _mm_store_ps(dst_shadow+8, v5.vector);
2363
2364 for(unsigned int site=0; site < n_vec-1; site++) {
2365
2366 src_shadow += 12;
2367 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
2368
2369 // Now the bottom half. -- preload v4 and v5
2370 v3.vector = _mm_load_ps(dst_shadow+12);
2371 v4.vector = _mm_load_ps(dst_shadow+16);
2372 v5.vector = _mm_load_ps(dst_shadow+20);
2373 _mm_prefetch((const char *) dst_shadow+20, _MM_HINT_T0);
2374
2375 // I want to set up
2376 //
2377 // v0[0] = v0[1] = tmp[0][col=0][im]
2378 // v0[1] = v0[0] = tmp[0][col=0][re]
2379 // v0[2] = v0[3] = tmp[0][col=1][im]
2380 // v0[3] = v0[2] = tmp[0][col=1][re]
2381 //
2382 // We can do this with v0 = shuf( v0, v0 ). Shuf code is 10 11 00 01 = xB1
2383 v0.vector = _mm_shuffle_ps(v0.vector, v0.vector, 0xB1);
2384
2385 // Mask (+1,-1,+1,-1)
2386 v0.vector = _mm_mul_ps(v0.vector, v6.vector);
2387 v3.vector = _mm_add_ps(v3.vector, v0.vector);
2388 _mm_store_ps(dst_shadow+12, v3.vector);
2389
2390
2391 // I want to set up
2392 //
2393 // v1[0] = v1[1] = tmp[0][col=2][im]
2394 // v1[1] = v1[0] = tmp[0][col=2][re]
2395 // v1[2] = v1[3] = tmp[1][col=0][im]
2396 // v1[3] = v1[2] = tmp[1][col=0][re]
2397 //
2398 // We can do this with v1 = shuf( v1, v1 ). Shuf code is 10 11 00 01 = xB1
2399 v1.vector = _mm_shuffle_ps(v1.vector, v1.vector, 0xB1);
2400
2401 // Need to multiply in mask (+1,-1,-1,+1)
2402 v1.vector = _mm_mul_ps(v1.vector, v7.vector);
2403 v4.vector = _mm_add_ps(v4.vector, v1.vector);
2404 _mm_store_ps(dst_shadow+16, v4.vector);
2405
2406 // I want to set up
2407 //
2408 // v2[0] = v2[1] = tmp[1][col=1][im]
2409 // v2[1] = v2[0] = tmp[1][col=1][re]
2410 // v2[2] = v2[2] = tmp[1][col=2][im]
2411 // v2[3] = v2[3] = tmp[1][col=2][re]
2412 //
2413 // We can do this with v2 = shuf( v2, v2 ). Shuf code is 10 11 00 01 = xB1
2414 v2.vector = _mm_shuffle_ps(v2.vector, v2.vector, 0xB1);
2415
2416 // Mask in -1, +1, -1, +1 from v7
2417 v2.vector = _mm_mul_ps(v2.vector, v6.vector);
2418 v5.vector = _mm_sub_ps(v5.vector, v2.vector);
2419 _mm_store_ps(dst_shadow+20, v5.vector);
2420
2421 dst_shadow+=24;
2422
2423 // Load source
2424 v0.vector = _mm_load_ps(src_shadow);
2425 v1.vector = _mm_load_ps(src_shadow+4);
2426 v2.vector = _mm_load_ps(src_shadow+8);
2427
2428 // Load top half of result
2429 v3.vector = _mm_load_ps(dst_shadow);
2430 v4.vector = _mm_load_ps(dst_shadow+4);
2431 v5.vector = _mm_load_ps(dst_shadow+8);
2432
2433 // Add source to result
2434 v3.vector = _mm_add_ps(v3.vector, v0.vector);
2435 v4.vector = _mm_add_ps(v4.vector, v1.vector);
2436 v5.vector = _mm_add_ps(v5.vector, v2.vector);
2437
2438 // Store out result
2439 _mm_store_ps(dst_shadow, v3.vector);
2440 _mm_store_ps(dst_shadow+4, v4.vector);
2441 _mm_store_ps(dst_shadow+8, v5.vector);
2442
2443 }
2444
2445 // Now the bottom half. -- preload v4 and v5
2446 // Now the bottom half. -- preload v4 and v5
2447 v3.vector = _mm_load_ps(dst_shadow+12);
2448 v4.vector = _mm_load_ps(dst_shadow+16);
2449 v5.vector = _mm_load_ps(dst_shadow+20);
2450 _mm_prefetch((const char *) dst_shadow+20, _MM_HINT_T0);
2451
2452 // I want to set up
2453 //
2454 // v0[0] = v0[1] = tmp[0][col=0][im]
2455 // v0[1] = v0[0] = tmp[0][col=0][re]
2456 // v0[2] = v0[3] = tmp[0][col=1][im]
2457 // v0[3] = v0[2] = tmp[0][col=1][re]
2458 //
2459 // We can do this with v0 = shuf( v0, v0 ). Shuf code is 10 11 00 01 = xB1
2460 v0.vector = _mm_shuffle_ps(v0.vector, v0.vector, 0xB1);
2461
2462 // Mask (+1,-1,+1,-1)
2463 v0.vector = _mm_mul_ps(v0.vector, v6.vector);
2464 v3.vector = _mm_add_ps(v3.vector, v0.vector);
2465 _mm_store_ps(dst_shadow+12,v3.vector);
2466
2467
2468 // I want to set up
2469 //
2470 // v1[0] = v1[1] = tmp[0][col=2][im]
2471 // v1[1] = v1[0] = tmp[0][col=2][re]
2472 // v1[2] = v1[3] = tmp[1][col=0][im]
2473 // v1[3] = v1[2] = tmp[1][col=0][re]
2474 //
2475 // We can do this with v1 = shuf( v1, v1 ). Shuf code is 10 11 00 01 = xB1
2476 v1.vector = _mm_shuffle_ps(v1.vector, v1.vector, 0xB1);
2477
2478 // Need to multiply in mask (+1,-1,-1,+1)
2479 v1.vector = _mm_mul_ps(v1.vector, v7.vector);
2480 v4.vector = _mm_add_ps(v4.vector, v1.vector);
2481 _mm_store_ps(dst_shadow+16, v4.vector);
2482
2483 // I want to set up
2484 //
2485 // v2[0] = v2[1] = tmp[1][col=1][im]
2486 // v2[1] = v2[0] = tmp[1][col=1][re]
2487 // v2[2] = v2[2] = tmp[1][col=2][im]
2488 // v2[3] = v2[3] = tmp[1][col=2][re]
2489 //
2490 // We can do this with v2 = shuf( v2, v2 ). Shuf code is 10 11 00 01 = xB1
2491 v2.vector = _mm_shuffle_ps(v2.vector, v2.vector, 0xB1);
2492
2493 // Mask in -1, +1, -1, +1 from v7
2494 v2.vector = _mm_mul_ps(v2.vector, v6.vector);
2495 v5.vector = _mm_sub_ps(v5.vector, v2.vector);
2496 _mm_store_ps(dst_shadow+20, v5.vector);
2497
2498}
2499
2512inline
2513void inlineAddSpinReconDir2Minus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
2514{
2515
2516#ifdef DEBUG_GENERIC_SPIN_RECON_INLINES
2517 QDPIO::cout << "inlinaAddSpinReconDir0Minus" << endl;
2518#endif
2519
2520 /* ( 1 0 -i 0) ( a0 ) ( a0 - i a2 )
2521 * B := ( 1 - Gamma ) A = ( 0 1 0 i) ( a1 ) = ( a1 + i a3 )
2522 * 2 ( i 0 1 0) ( a2 ) ( a2 + i a0 )
2523 * ( 0 -i 0 1) ( a3 ) ( a3 - i a1 )
2524
2525 * The bottom components of be may be reconstructed using the formula
2526 * ( b2r + i b2i ) = ( {a2r - a0i} + i{a2i + a0r} ) = ( - b0i + i b0r )
2527 * ( b3r + i b3i ) ( {a3r + a1i} + i{a3i - a1r} ) ( b1i - i b1r )
2528 */
2529
2530 const REAL32* src_shadow = src;
2531 REAL32* dst_shadow = dst;
2532
2533 SSEVec v0, v1, v2, v3, v4, v5, v6, v7;
2534 v6.floats[0] = -1;
2535 v6.floats[1] = +1;
2536 v6.floats[2] = -1;
2537 v6.floats[3] = +1;
2538
2539 v7.floats[0] = -1;
2540 v7.floats[1] = +1;
2541 v7.floats[2] = +1;
2542 v7.floats[3] = -1;
2543
2544 // Load source
2545 v0.vector = _mm_load_ps(src_shadow);
2546 v1.vector = _mm_load_ps(src_shadow+4);
2547 v2.vector = _mm_load_ps(src_shadow+8);
2548
2549 // Load top half of result
2550 v3.vector = _mm_load_ps(dst_shadow);
2551 v4.vector = _mm_load_ps(dst_shadow+4);
2552 v5.vector = _mm_load_ps(dst_shadow+8);
2553
2554 // Add source to result
2555 v3.vector = _mm_add_ps(v3.vector, v0.vector);
2556 v4.vector = _mm_add_ps(v4.vector, v1.vector);
2557 v5.vector = _mm_add_ps(v5.vector, v2.vector);
2558
2559 // Store out result
2560 _mm_store_ps(dst_shadow, v3.vector);
2561 _mm_store_ps(dst_shadow+4, v4.vector);
2562 _mm_store_ps(dst_shadow+8, v5.vector);
2563
2564 for(unsigned int site=0; site < n_vec-1; site++) {
2565
2566 src_shadow += 12;
2567 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
2568
2569 // Now the bottom half. -- preload v4 and v5
2570 v3.vector = _mm_load_ps(dst_shadow+12);
2571 v4.vector = _mm_load_ps(dst_shadow+16);
2572 v5.vector = _mm_load_ps(dst_shadow+20);
2573 _mm_prefetch((const char *) dst_shadow+20, _MM_HINT_T0);
2574
2575 // I want to set up
2576 //
2577 // v0[0] = v0[1] = tmp[0][col=0][im]
2578 // v0[1] = v0[0] = tmp[0][col=0][re]
2579 // v0[2] = v0[3] = tmp[0][col=1][im]
2580 // v0[3] = v0[2] = tmp[0][col=1][re]
2581 //
2582 // We can do this with v0 = shuf( v0, v0 ). Shuf code is 10 11 00 01 = xB1
2583 v0.vector = _mm_shuffle_ps(v0.vector, v0.vector, 0xB1);
2584
2585 // Mask (+1,-1,+1,-1)
2586 v0.vector = _mm_mul_ps(v0.vector, v6.vector);
2587 v3.vector = _mm_add_ps(v3.vector, v0.vector);
2588 _mm_store_ps(dst_shadow+12, v3.vector);
2589
2590
2591 // I want to set up
2592 //
2593 // v1[0] = v1[1] = tmp[0][col=2][im]
2594 // v1[1] = v1[0] = tmp[0][col=2][re]
2595 // v1[2] = v1[3] = tmp[1][col=0][im]
2596 // v1[3] = v1[2] = tmp[1][col=0][re]
2597 //
2598 // We can do this with v1 = shuf( v1, v1 ). Shuf code is 10 11 00 01 = xB1
2599 v1.vector = _mm_shuffle_ps(v1.vector, v1.vector, 0xB1);
2600
2601 // Need to multiply in mask (+1,-1,-1,+1)
2602 v1.vector = _mm_mul_ps(v1.vector, v7.vector);
2603 v4.vector = _mm_add_ps(v4.vector, v1.vector);
2604 _mm_store_ps(dst_shadow+16, v4.vector);
2605
2606 // I want to set up
2607 //
2608 // v2[0] = v2[1] = tmp[1][col=1][im]
2609 // v2[1] = v2[0] = tmp[1][col=1][re]
2610 // v2[2] = v2[2] = tmp[1][col=2][im]
2611 // v2[3] = v2[3] = tmp[1][col=2][re]
2612 //
2613 // We can do this with v2 = shuf( v2, v2 ). Shuf code is 10 11 00 01 = xB1
2614 v2.vector = _mm_shuffle_ps(v2.vector, v2.vector, 0xB1);
2615
2616 // Mask in -1, +1, -1, +1 from v7
2617 v2.vector = _mm_mul_ps(v2.vector, v6.vector);
2618 v5.vector = _mm_sub_ps(v5.vector, v2.vector);
2619 _mm_store_ps(dst_shadow+20, v5.vector);
2620
2621 dst_shadow+=24;
2622
2623 // Load source
2624 v0.vector = _mm_load_ps(src_shadow);
2625 v1.vector = _mm_load_ps(src_shadow+4);
2626 v2.vector = _mm_load_ps(src_shadow+8);
2627
2628 // Load top half of result
2629 v3.vector = _mm_load_ps(dst_shadow);
2630 v4.vector = _mm_load_ps(dst_shadow+4);
2631 v5.vector = _mm_load_ps(dst_shadow+8);
2632
2633 // Add source to result
2634 v3.vector = _mm_add_ps(v3.vector, v0.vector);
2635 v4.vector = _mm_add_ps(v4.vector, v1.vector);
2636 v5.vector = _mm_add_ps(v5.vector, v2.vector);
2637
2638 // Store out result
2639 _mm_store_ps(dst_shadow, v3.vector);
2640 _mm_store_ps(dst_shadow+4, v4.vector);
2641 _mm_store_ps(dst_shadow+8, v5.vector);
2642
2643 }
2644
2645 // Now the bottom half. -- preload v4 and v5
2646 // Now the bottom half. -- preload v4 and v5
2647 v3.vector = _mm_load_ps(dst_shadow+12);
2648 v4.vector = _mm_load_ps(dst_shadow+16);
2649 v5.vector = _mm_load_ps(dst_shadow+20);
2650 _mm_prefetch((const char *) dst_shadow+20, _MM_HINT_T0);
2651
2652 // I want to set up
2653 //
2654 // v0[0] = v0[1] = tmp[0][col=0][im]
2655 // v0[1] = v0[0] = tmp[0][col=0][re]
2656 // v0[2] = v0[3] = tmp[0][col=1][im]
2657 // v0[3] = v0[2] = tmp[0][col=1][re]
2658 //
2659 // We can do this with v0 = shuf( v0, v0 ). Shuf code is 10 11 00 01 = xB1
2660 v0.vector = _mm_shuffle_ps(v0.vector, v0.vector, 0xB1);
2661
2662 // Mask (+1,-1,+1,-1)
2663 v0.vector = _mm_mul_ps(v0.vector, v6.vector);
2664 v3.vector = _mm_add_ps(v3.vector, v0.vector);
2665 _mm_store_ps(dst_shadow+12, v3.vector);
2666
2667
2668 // I want to set up
2669 //
2670 // v1[0] = v1[1] = tmp[0][col=2][im]
2671 // v1[1] = v1[0] = tmp[0][col=2][re]
2672 // v1[2] = v1[3] = tmp[1][col=0][im]
2673 // v1[3] = v1[2] = tmp[1][col=0][re]
2674 //
2675 // We can do this with v1 = shuf( v1, v1 ). Shuf code is 10 11 00 01 = xB1
2676 v1.vector = _mm_shuffle_ps(v1.vector, v1.vector, 0xB1);
2677
2678 // Need to multiply in mask (+1,-1,-1,+1)
2679 v1.vector = _mm_mul_ps(v1.vector, v7.vector);
2680 v4.vector = _mm_add_ps(v4.vector, v1.vector);
2681 _mm_store_ps(dst_shadow+16, v4.vector);
2682
2683 // I want to set up
2684 //
2685 // v2[0] = v2[1] = tmp[1][col=1][im]
2686 // v2[1] = v2[0] = tmp[1][col=1][re]
2687 // v2[2] = v2[2] = tmp[1][col=2][im]
2688 // v2[3] = v2[3] = tmp[1][col=2][re]
2689 //
2690 // We can do this with v2 = shuf( v2, v2 ). Shuf code is 10 11 00 01 = xB1
2691 v2.vector = _mm_shuffle_ps(v2.vector, v2.vector, 0xB1);
2692
2693 // Mask in -1, +1, -1, +1 from v7
2694 v2.vector = _mm_mul_ps(v2.vector, v6.vector);
2695 v5.vector = _mm_sub_ps(v5.vector, v2.vector);
2696 _mm_store_ps(dst_shadow+20, v5.vector);
2697
2698
2699
2700}
2701
2702
2715inline
2716void inlineAddSpinReconDir3Plus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
2717{
2718
2719#ifdef DEBUG_GENERIC_SPIN_RECON_INLINES
2720 QDPIO::cout << "inlinaAddSpinReconDir0Plus" << endl;
2721#endif
2722
2723 /* ( 1 0 1 0) ( a0 ) ( a0 + a2 )
2724 * B := ( 1 + Gamma ) A = ( 0 1 0 1) ( a1 ) = ( a1 + a3 )
2725 * 3 ( 1 0 1 0) ( a2 ) ( a2 + a0 )
2726 * ( 0 1 0 1) ( a3 ) ( a3 + a1 )
2727
2728 * The bottom components of be may be reconstructed using the formula
2729
2730 * ( b2r + i b2i ) = ( {a2r + a0r} + i{a2i + a0i} ) = ( b0r + i b0i )
2731 * ( b3r + i b3i ) ( {a3r + a1r} + i{a3i + a1i} ) ( b1r + i b1i )
2732 */
2733
2734
2735 const REAL32* src_shadow = src;
2736 REAL32* dst_shadow = dst;
2737
2738 SSEVec v0, v1, v2, v3, v4, v5;
2739
2740 // Load source
2741 v0.vector = _mm_load_ps(src_shadow);
2742 v1.vector = _mm_load_ps(src_shadow+4);
2743 v2.vector = _mm_load_ps(src_shadow+8);
2744
2745 // Load top half of result
2746 v3.vector = _mm_load_ps(dst_shadow);
2747 v4.vector = _mm_load_ps(dst_shadow+4);
2748 v5.vector = _mm_load_ps(dst_shadow+8);
2749
2750 // Add source to result
2751 v3.vector = _mm_add_ps(v3.vector, v0.vector);
2752 v4.vector = _mm_add_ps(v4.vector, v1.vector);
2753 v5.vector = _mm_add_ps(v5.vector, v2.vector);
2754
2755 // Store out result
2756 _mm_store_ps(dst_shadow, v3.vector);
2757 _mm_store_ps(dst_shadow+4, v4.vector);
2758 _mm_store_ps(dst_shadow+8, v5.vector);
2759
2760 for(unsigned int site=0; site < n_vec-1; site++) {
2761
2762 src_shadow += 12;
2763 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
2764
2765
2766 // Now the bottom half. -- preload v4 and v5
2767 v3.vector = _mm_load_ps(dst_shadow+12);
2768 v4.vector = _mm_load_ps(dst_shadow+16);
2769 v5.vector = _mm_load_ps(dst_shadow+20);
2770
2771 v3.vector = _mm_add_ps(v3.vector, v0.vector);
2772 v4.vector = _mm_add_ps(v4.vector, v1.vector);
2773 v5.vector = _mm_add_ps(v5.vector, v2.vector);
2774
2775 _mm_store_ps(dst_shadow+12, v3.vector);
2776 _mm_store_ps(dst_shadow+16, v4.vector);
2777 _mm_store_ps(dst_shadow+20, v5.vector);
2778
2779 dst_shadow+=24;
2780
2781 // Load source
2782 v0.vector = _mm_load_ps(src_shadow);
2783 v1.vector = _mm_load_ps(src_shadow+4);
2784 v2.vector = _mm_load_ps(src_shadow+8);
2785
2786 // Load top half of result
2787 v3.vector = _mm_load_ps(dst_shadow);
2788 v4.vector = _mm_load_ps(dst_shadow+4);
2789 v5.vector = _mm_load_ps(dst_shadow+8);
2790
2791 // Add source to result (no shufs needed, no mask needed)
2792 v3.vector = _mm_add_ps(v3.vector, v0.vector);
2793 v4.vector = _mm_add_ps(v4.vector, v1.vector);
2794 v5.vector = _mm_add_ps(v5.vector, v2.vector);
2795
2796 // Store out result
2797 _mm_store_ps(dst_shadow, v3.vector);
2798 _mm_store_ps(dst_shadow+4, v4.vector);
2799 _mm_store_ps(dst_shadow+8, v5.vector);
2800
2801 }
2802
2803 // Now the bottom half. -- preload v4 and v5
2804 // Now the bottom half. -- preload v4 and v5
2805 v3.vector = _mm_load_ps(dst_shadow+12);
2806 v4.vector = _mm_load_ps(dst_shadow+16);
2807 v5.vector = _mm_load_ps(dst_shadow+20);
2808
2809 v3.vector = _mm_add_ps(v3.vector, v0.vector);
2810 v4.vector = _mm_add_ps(v4.vector, v1.vector);
2811 v5.vector = _mm_add_ps(v5.vector, v2.vector);
2812
2813 _mm_store_ps(dst_shadow+12, v3.vector);
2814 _mm_store_ps(dst_shadow+16, v4.vector);
2815 _mm_store_ps(dst_shadow+20, v5.vector);
2816
2817}
2818
2831inline
2832void inlineAddSpinReconDir3Minus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
2833{
2834
2835#ifdef DEBUG_GENERIC_SPIN_RECON_INLINES
2836 QDPIO::cout << "inlinaAddSpinReconDir0Minus" << endl;
2837#endif
2838
2839 /* ( 1 0 -1 0) ( a0 ) ( a0 - a2 )
2840 * B := ( 1 - Gamma ) A = ( 0 1 0 -1) ( a1 ) = ( a1 - a3 )
2841 * 3 (-1 0 1 0) ( a2 ) ( a2 - a0 )
2842 * ( 0 -1 0 1) ( a3 ) ( a3 - a1 )
2843
2844 * The bottom components of be may be reconstructed using the formula
2845 * ( b2r + i b2i ) = ( {a2r - a0r} + i{a2i - a0i} ) = ( - b0r - i b0i )
2846 * ( b3r + i b3i ) ( {a3r - a1r} + i{a3i - a1i} ) ( - b1r - i b1i )
2847 */
2848
2849 const REAL32* src_shadow = src;
2850 REAL32* dst_shadow = dst;
2851
2852 SSEVec v0, v1, v2, v3, v4, v5;
2853
2854 // Load source
2855 v0.vector = _mm_load_ps(src_shadow);
2856 v1.vector = _mm_load_ps(src_shadow+4);
2857 v2.vector = _mm_load_ps(src_shadow+8);
2858
2859 // Load top half of result
2860 v3.vector = _mm_load_ps(dst_shadow);
2861 v4.vector = _mm_load_ps(dst_shadow+4);
2862 v5.vector = _mm_load_ps(dst_shadow+8);
2863
2864 // Add source to result
2865 v3.vector = _mm_add_ps(v3.vector, v0.vector);
2866 v4.vector = _mm_add_ps(v4.vector, v1.vector);
2867 v5.vector = _mm_add_ps(v5.vector, v2.vector);
2868
2869 // Store out result
2870 _mm_store_ps(dst_shadow, v3.vector);
2871 _mm_store_ps(dst_shadow+4, v4.vector);
2872 _mm_store_ps(dst_shadow+8, v5.vector);
2873
2874 for(unsigned int site=0; site < n_vec-1; site++) {
2875
2876 src_shadow += 12;
2877 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
2878
2879 // Now the bottom half. -- preload v4 and v5
2880 v3.vector = _mm_load_ps(dst_shadow+12);
2881 v4.vector = _mm_load_ps(dst_shadow+16);
2882 v5.vector = _mm_load_ps(dst_shadow+20);
2883
2884 v3.vector = _mm_sub_ps(v3.vector, v0.vector);
2885 v4.vector = _mm_sub_ps(v4.vector, v1.vector);
2886 v5.vector = _mm_sub_ps(v5.vector, v2.vector);
2887
2888 _mm_store_ps(dst_shadow+12, v3.vector);
2889 _mm_store_ps(dst_shadow+16, v4.vector);
2890 _mm_store_ps(dst_shadow+20, v5.vector);
2891
2892 dst_shadow+=24;
2893
2894 // Load source
2895 v0.vector = _mm_load_ps(src_shadow);
2896 v1.vector = _mm_load_ps(src_shadow+4);
2897 v2.vector = _mm_load_ps(src_shadow+8);
2898
2899 // Load top half of result
2900 v3.vector = _mm_load_ps(dst_shadow);
2901 v4.vector = _mm_load_ps(dst_shadow+4);
2902 v5.vector = _mm_load_ps(dst_shadow+8);
2903
2904 // Add source to result (no shufs needed, no mask needed)
2905 v3.vector = _mm_add_ps(v3.vector, v0.vector);
2906 v4.vector = _mm_add_ps(v4.vector, v1.vector);
2907 v5.vector = _mm_add_ps(v5.vector, v2.vector);
2908
2909 // Store out result
2910 _mm_store_ps(dst_shadow, v3.vector);
2911 _mm_store_ps(dst_shadow+4, v4.vector);
2912 _mm_store_ps(dst_shadow+8, v5.vector);
2913
2914 }
2915
2916 // Now the bottom half. -- preload v4 and v5
2917 // Now the bottom half. -- preload v4 and v5
2918 v3.vector = _mm_load_ps(dst_shadow+12);
2919 v4.vector = _mm_load_ps(dst_shadow+16);
2920 v5.vector = _mm_load_ps(dst_shadow+20);
2921
2922 v3.vector = _mm_sub_ps(v3.vector, v0.vector);
2923 v4.vector = _mm_sub_ps(v4.vector, v1.vector);
2924 v5.vector = _mm_sub_ps(v5.vector, v2.vector);
2925
2926 _mm_store_ps(dst_shadow+12, v3.vector);
2927 _mm_store_ps(dst_shadow+16, v4.vector);
2928 _mm_store_ps(dst_shadow+20, v5.vector);
2929
2930}
2931
2932
2933} // namespace QDP
2934
2935#endif
QDP::REAL32
float REAL32
Definition qdp_scalarsite_defs.h:21
QDP::Nc
const int Nc
Definition qdp_params.h:25
QDP::QDPIO::cout
StandardOutputStream cout
Definition qdp_stdio.cc:21
QDP
Yet another random number generator.
Definition qdp_bgq_threadbind.cc:16
QDP::inlineSpinReconDir0Minus
void inlineSpinReconDir0Minus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin recon (1/2)(1-\gamma_0).
Definition generic_spin_recon_inlines.h:88
QDP::inlineAddSpinReconDir1Plus
void inlineAddSpinReconDir1Plus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin recon (1/2)(1+\gamma_1).
Definition generic_spin_recon_inlines.h:694
QDP::inlineAddSpinReconDir2Minus
void inlineAddSpinReconDir2Minus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin recon (1/2)(1-\gamma_2).
Definition generic_spin_recon_inlines.h:899
QDP::inlineAddSpinReconDir1Minus
void inlineAddSpinReconDir1Minus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin recon (1/2)(1-\gamma_1).
Definition generic_spin_recon_inlines.h:761
QDP::inlineSpinReconDir2Plus
void inlineSpinReconDir2Plus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin recon (1/2)(1+\gamma_2).
Definition generic_spin_recon_inlines.h:292
QDP::inlineSpinReconDir1Minus
void inlineSpinReconDir1Minus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin recon (1/2)(1-\gamma_1).
Definition generic_spin_recon_inlines.h:221
QDP::inlineAddSpinReconDir3Plus
void inlineAddSpinReconDir3Plus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin recon (1/2)(1+\gamma3).
Definition generic_spin_recon_inlines.h:967
QDP::inlineSpinReconDir2Minus
void inlineSpinReconDir2Minus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin recon (1/2)(1-\gamma_2).
Definition generic_spin_recon_inlines.h:359
QDP::inlineAddSpinReconDir3Minus
void inlineAddSpinReconDir3Minus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin recon (1/2)(1-\gamma3).
Definition generic_spin_recon_inlines.h:1035
QDP::inlineSpinReconDir3Minus
void inlineSpinReconDir3Minus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin recon (1/2)(1-\gamma3).
Definition generic_spin_recon_inlines.h:495
QDP::inlineAddSpinReconDir0Minus
void inlineAddSpinReconDir0Minus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin recon (1/2)(1-\gamma_0).
Definition generic_spin_recon_inlines.h:628
QDP::inlineAddSpinReconDir2Plus
void inlineAddSpinReconDir2Plus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin recon (1/2)(1+\gamma_2).
Definition generic_spin_recon_inlines.h:832
QDP::inlineSpinReconDir3Plus
void inlineSpinReconDir3Plus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin recon (1/2)(1+\gamma3).
Definition generic_spin_recon_inlines.h:427
QDP::inlineAddSpinReconDir0Plus
void inlineAddSpinReconDir0Plus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin recon (1/2)(1+\gamma_0).
Definition generic_spin_recon_inlines.h:563
QDP::inlineSpinReconDir1Plus
void inlineSpinReconDir1Plus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin recon (1/2)(1+\gamma_1).
Definition generic_spin_recon_inlines.h:154
QDP::inlineSpinReconDir0Plus
void inlineSpinReconDir0Plus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin recon (1/2)(1+\gamma_0).
Definition generic_spin_recon_inlines.h:23
qdp_sse_intrin.h
QDP::SSEVec::floats
float floats[4]
Definition qdp_sse_intrin.h:10
Generated by doxygen 1.16.1