QDP++
sse_spin_proj_inlines.h
Go to the documentation of this file.
1#ifndef SSE_SPIN_PROJ_INLINES_H
2#define SSE_SPIN_PROJ_INLINES_H
3
4#include "qdp_sse_intrin.h"
5
6/* File: generic_spin_proj_inlines.h
7 Purpose: Supply inline functions to do spin projection
8 Author: $Id: sse_spin_proj_inlines.h,v 1.6 2009-02-11 20:50:45 bjoo Exp $
9*/
10namespace QDP {
11#include <stdio.h>
12
25inline
26void inlineSpinProjDir0Plus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
27{
28
29#ifdef DEBUG_GENERIC_SPIN_PROJ_INLINES
30 QDPIO::cout << "inlineSpinProjDir0Plus" << endl;
31#endif
32
33
34 SSEVec v0, v1, v2, v3, v4, v5, v6, v7;
35
36 /* 1 + \gamma_0 = 1 0 0 i
37 0 1 i 0
38 0 -i 1 0
39 -i 0 0 1
40
41 * ( d0r + i d0i ) = ( {x0r - x3i} + i{x0i + x3r} )
42 * ( d1r + i d1i ) ( {x1r - x2i} + i{x1i + x2r} )
43 */
44 const REAL32* src_shadow = src;
45 REAL32* dst_shadow = dst;
46
47
48 // Store in the spinor - top half
49 v0.vector = _mm_load_ps(src_shadow);
50 v1.vector = _mm_load_ps(src_shadow+4);
51 v2.vector = _mm_load_ps(src_shadow+8);
52 v3.vector = _mm_load_ps(src_shadow+12);
53 v4.vector = _mm_load_ps(src_shadow+16);
54 v5.vector = _mm_load_ps(src_shadow+20);
55
56 v7.floats[0] = -1;
57 v7.floats[1] = +1;
58 v7.floats[2] = -1;
59 v7.floats[3] = +1;
60
61 for(unsigned int site=0; site < n_vec-1; site++) {
62 src_shadow += 24;
63
64 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
65
66 v6.vector = v4.vector; // V6 is dest so we can move its
67 // I want to shuffle so that:
68 // v6[0] <- v6[3]=v4[3] [1:0]=3 = x11 <- tmp[3][col=0][im]
69 // v6[1] <- v6[2]=v4[2] [3:2]=2 = x10 <- tmp[3][col=0][re]
70 // v6[2] <- v5[1] [5:4]=1 = x01 <- tmp[3][col=1][im]
71 // v6[3] <- v5[0] [7:6]=0 = x00 <- tmp[3][col=1][re]
72 // So Immediate for shufps is: 0x0001 1011 = x1B = 27
73 v6.vector = _mm_shuffle_ps(v6.vector, v5.vector, 0x1B);
74
75 // Does a v0 + v6*v7
76 // v0[0] <- v0[0] + v6[0]*v7[0] = tmp[0][col=0][re] - tmp[3][col=0][im]
77 // v0[1] <- v0[1] + v6[1]*v7[1] = tmp[0][col=0][im] + tmp[3][col=0][re]
78 // v0[2] <- v0[2] + v6[2]*v7[2] = tmp[0][col=1][re] - tmp[3][col=1][im]
79 // v0[3] <- v0[3] + v6[3]*v7[3] = tmp[0][col=1][im] + tmp[3][col=1][re]
80 v6.vector = _mm_mul_ps(v6.vector, v7.vector);
81 v0.vector = _mm_add_ps(v0.vector, v6.vector);
82 _mm_store_ps(dst_shadow, v0.vector);
83
84 v6.vector = v5.vector;
85 // Now setup v7 so that
86 // v6[0] <- v5[3] [1:0]=3 = x11 <- tmp[3][col=2][im]
87 // v6[1] <- v5[2] [3:2]=2 = x10 <- tmp[3][col=2][re]
88 // v6[2] <- v3[1] [5:4]=1 = x01 <- tmp[2][col=0][im]
89 // v6[3] <- v3[0] [7:6]=0 = x00 <- tmp[2][col=0][re]
90 // So Immediate for shufps is: 0x0001 1011 = x1B = 27
91 v6.vector = _mm_shuffle_ps(v6.vector, v3.vector, 0x1B);
92
93 // Does a v1 + v6*v7
94 // v1[0] <- v1[0] + v6[0]*v7[0] = tmp[0][col=2][re] - tmp[3][col=2][im]
95 // v1[1] <- v1[1] + v6[1]*v7[1] = tmp[0][col=2][im] + tmp[3][col=2][re]
96 // v1[2] <- v1[2] + v6[2]*v7[2] = tmp[1][col=0][re] - tmp[2][col=0][im]
97 // v1[3] <- v1[3] + v6[3]*v7[3] = tmp[1][col=0][im] + tmp[2][col=0][re]
98 v6.vector = _mm_mul_ps(v6.vector, v7.vector);
99 v1.vector = _mm_add_ps(v1.vector, v6.vector);
100 _mm_store_ps(dst_shadow+4, v1.vector);
101
102 v6.vector = v3.vector;
103 // v6[0] <- v6[3]=v3[3] [1:0]=3 = x11 <- tmp[2][col=1][im]
104 // v6[1] <- v6[3]=v3[2] [3:2]=2 = x10 <- tmp[2][col=1][re]
105 // v6[2] <- v4[1] [5:4]=1 = x01 <- tmp[2][col=2][im]
106 // v6[3] <- v4[0] [7:6]=0 = x00 <- tmp[2][col=2][re]
107 // So Immediate for shufps is: 0x0001 1011 = x1B = 27
108 v6.vector = _mm_shuffle_ps(v6.vector, v4.vector, 0x1B);
109
110 // Does a v2 + v6*v7
111 // v2[0] <- v2[0] + v6[0]*v7[0] = tmp[1][col=1][re] - tmp[2][col=1][im]
112 // v2[1] <- v2[1] + v6[1]*v7[1] = tmp[1][col=1][im] + tmp[2][col=1][re]
113 // v2[2] <- v2[2] + v6[2]*v7[2] = tmp[1][col=2][re] - tmp[2][col=2][im]
114 // v2[3] <- v2[3] + v6[3]*v7[3] = tmp[1][col=2][im] + tmp[2][col=2][re]
115 v6.vector = _mm_mul_ps(v6.vector, v7.vector);
116 v2.vector = _mm_add_ps(v2.vector, v6.vector);
117 _mm_store_ps(dst_shadow+8, v2.vector);
118
119 // Push the next one
120 dst_shadow+=12;
121
122 // Store in the next spinor -- should be prefetched if all has gone well
123 v0.vector = _mm_load_ps(src_shadow);
124 v1.vector = _mm_load_ps(src_shadow+4);
125 v2.vector = _mm_load_ps(src_shadow+8);
126 v3.vector = _mm_load_ps(src_shadow+12);
127 v4.vector = _mm_load_ps(src_shadow+16);
128 v5.vector = _mm_load_ps(src_shadow+20);
129
130 }
131
132 // Last bit
133
134 v6.vector = v4.vector; // V6 is dest so we can move its
135 // I want to shuffle so that:
136 // v6[0] <- v6[3]=v4[3] [1:0]=3 = x11 <- tmp[3][col=0][im]
137 // v6[1] <- v6[2]=v4[2] [3:2]=2 = x10 <- tmp[3][col=0][re]
138 // v6[2] <- v5[1] [5:4]=1 = x01 <- tmp[3][col=1][im]
139 // v6[3] <- v5[0] [7:6]=0 = x00 <- tmp[3][col=1][re]
140 // So Immediate for shufps is: 0x0001 1011 = x1B = 27
141 v6.vector = _mm_shuffle_ps(v6.vector, v5.vector, 0x1B);
142
143 // Does a v0 + v6*v7
144 // v0[0] <- v0[0] + v6[0]*v7[0] = tmp[0][col=0][re] - tmp[3][col=0][im]
145 // v0[1] <- v0[1] + v6[1]*v7[1] = tmp[0][col=0][im] + tmp[3][col=0][re]
146 // v0[2] <- v0[2] + v6[2]*v7[2] = tmp[0][col=1][re] - tmp[3][col=1][im]
147 // v0[3] <- v0[3] + v6[3]*v7[3] = tmp[0][col=1][im] + tmp[3][col=1][re]
148 v6.vector = _mm_mul_ps(v6.vector, v7.vector);
149 v0.vector = _mm_add_ps(v0.vector, v6.vector);
150 _mm_store_ps(dst_shadow, v0.vector);
151
152 v6.vector = v5.vector;
153 // Now setup v7 so that
154 // v6[0] <- v5[3] [1:0]=3 = x11 <- tmp[3][col=2][im]
155 // v6[1] <- v5[2] [3:2]=2 = x10 <- tmp[3][col=2][re]
156 // v6[2] <- v3[1] [5:4]=1 = x01 <- tmp[2][col=0][im]
157 // v6[3] <- v3[0] [7:6]=0 = x00 <- tmp[2][col=0][re]
158 // So Immediate for shufps is: 0x0001 1011 = x1B = 27
159 v6.vector = _mm_shuffle_ps(v6.vector, v3.vector, 0x1B);
160
161 // Does a v1 + v6*v7
162 // v1[0] <- v1[0] + v6[0]*v7[0] = tmp[0][col=2][re] - tmp[3][col=2][im]
163 // v1[1] <- v1[1] + v6[1]*v7[1] = tmp[0][col=2][im] + tmp[3][col=2][re]
164 // v1[2] <- v1[2] + v6[2]*v7[2] = tmp[1][col=0][re] - tmp[2][col=0][im]
165 // v1[3] <- v1[3] + v6[3]*v7[3] = tmp[1][col=0][im] + tmp[2][col=0][re]
166 v6.vector = _mm_mul_ps(v6.vector, v7.vector);
167 v1.vector = _mm_add_ps(v1.vector, v6.vector);
168 _mm_store_ps(dst_shadow+4, v1.vector);
169
170 v6.vector = v3.vector;
171 // v6[0] <- v6[3]=v3[3] [1:0]=3 = x11 <- tmp[2][col=1][im]
172 // v6[1] <- v6[3]=v3[2] [3:2]=2 = x10 <- tmp[2][col=1][re]
173 // v6[2] <- v4[1] [5:4]=1 = x01 <- tmp[2][col=2][im]
174 // v6[3] <- v4[0] [7:6]=0 = x00 <- tmp[2][col=2][re]
175 // So Immediate for shufps is: 0x0001 1011 = x1B = 27
176 v6.vector = _mm_shuffle_ps(v6.vector, v4.vector, 0x1B);
177
178 // Does a v2 + v6*v7
179 // v2[0] <- v2[0] + v6[0]*v7[0] = tmp[1][col=1][re] - tmp[2][col=1][im]
180 // v2[1] <- v2[1] + v6[1]*v7[1] = tmp[1][col=1][im] + tmp[2][col=1][re]
181 // v2[2] <- v2[2] + v6[2]*v7[2] = tmp[1][col=2][re] - tmp[2][col=2][im]
182 // v2[3] <- v2[3] + v6[3]*v7[3] = tmp[1][col=2][im] + tmp[2][col=2][re]
183 v6.vector = _mm_mul_ps(v6.vector, v7.vector);
184 v2.vector = _mm_add_ps(v2.vector, v6.vector);
185 _mm_store_ps(dst_shadow+8, v2.vector);
186
187}
188
200inline
201void inlineSpinProjDir0Minus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
202{
203
204#ifdef DEBUG_GENERIC_SPIN_PROJ_INLINES
205 QDPIO::cout << "inlineSpinProjDir0Minus" << endl;
206#endif
207
208
209 /* ( 1 0 0 -i) ( a0 ) ( a0 - i a3 )
210 * B := ( 1 - Gamma ) A = ( 0 1 -i 0) ( a1 ) = ( a1 - i a2 )
211 * 0 ( 0 i 1 0) ( a2 ) ( a2 + i a1 )
212 * ( i 0 0 1) ( a3 ) ( a3 + i a0 )
213
214 * Therefore the top components are
215
216 * ( b0r + i b0i ) = ( {a0r + a3i} + i{a0i - a3r} )
217 * ( b1r + i b1i ) ( {a1r + a2i} + i{a1i - a2r} )
218 */
219
220
221 const REAL32* src_shadow = src;
222 REAL32* dst_shadow = dst;
223 SSEVec v0, v1, v2, v3, v4, v5, v6, v7;
224
225 v7.floats[0] = -1;
226 v7.floats[1] = +1;
227 v7.floats[2] = -1;
228 v7.floats[3] = +1;
229
230 // Store in the spinor
231 v0.vector = _mm_load_ps(src_shadow);
232 v1.vector = _mm_load_ps(src_shadow+4);
233 v2.vector = _mm_load_ps(src_shadow+8);
234 v3.vector = _mm_load_ps(src_shadow+12);
235 v4.vector = _mm_load_ps(src_shadow+16);
236 v5.vector = _mm_load_ps(src_shadow+20);
237
238 for(unsigned int site=0; site < n_vec-1; site++) {
239 src_shadow += 24;
240
241 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
242
243 v6.vector = v4.vector; // V6 is dest so we can move its
244 // I want to shuffle so that:
245 // v6[0] <- v6[3]=v4[3] [1:0]=3 = x11 <- tmp[3][col=0][im]
246 // v6[1] <- v6[2]=v4[2] [3:2]=2 = x10 <- tmp[3][col=0][re]
247 // v6[2] <- v5[1] [5:4]=1 = x01 <- tmp[3][col=1][im]
248 // v6[3] <- v5[0] [7:6]=0 = x00 <- tmp[3][col=1][re]
249 // So Immediate for shufps is: 0x0001 1011 = x1B = 27
250 v6.vector = _mm_shuffle_ps(v6.vector, v5.vector, 0x1B);
251
252 // Does a v0 - v6*v7
253 // v0[0] <- v0[0] - v6[0]*v7[0] = tmp[0][col=0][re] + tmp[3][col=0][im]
254 // v0[1] <- v0[1] - v6[1]*v7[1] = tmp[0][col=0][im] - tmp[3][col=0][re]
255 // v0[2] <- v0[2] - v6[2]*v7[2] = tmp[0][col=1][re] + tmp[3][col=1][im]
256 // v0[3] <- v0[3] - v6[3]*v7[3] = tmp[0][col=1][im] - tmp[3][col=1][re]
257 v6.vector = _mm_mul_ps(v6.vector, v7.vector);
258 v0.vector = _mm_sub_ps(v0.vector, v6.vector);
259 _mm_store_ps(dst_shadow, v0.vector);
260
261 v6.vector = v5.vector;
262 // I want to shuffle so that:
263 // v6[0] <- v6[3]=v5[3] [1:0]=3 = x11 <- tmp[3][col=0][im]
264 // v6[1] <- v6[2]=v5[2] [3:2]=2 = x10 <- tmp[3][col=0][re]
265 // v6[2] <- v3[1] [5:4]=1 = x01 <- tmp[3][col=1][im]
266 // v6[3] <- v3[0] [7:6]=0 = x00 <- tmp[3][col=1][re]
267 // So Immediate for shufps is: 0x0001 1011 = x1B = 27
268 v6.vector = _mm_shuffle_ps(v6.vector, v3.vector, 0x1B);
269
270 // Does a v1 - v6*v7
271 // v1[0] <- v1[0] - v6[0]*v7[0] = tmp[0][col=2][re] + tmp[3][col=2][im]
272 // v1[1] <- v1[1] - v6[1]*v7[1] = tmp[0][col=2][im] - tmp[3][col=2][re]
273 // v1[2] <- v1[2] - v6[2]*v7[2] = tmp[1][col=0][re] + tmp[2][col=0][im]
274 // v1[3] <- v1[3] - v6[3]*v7[3] = tmp[1][col=0][im] - tmp[2][col=0][re]
275 v6.vector = _mm_mul_ps(v6.vector, v7.vector);
276 v1.vector = _mm_sub_ps(v1.vector, v6.vector);
277 _mm_store_ps(dst_shadow+4, v1.vector);
278
279 v6.vector = v3.vector;
280 // v6[0] <- v6[3]=v3[3] [1:0]=3 = x11 <- tmp[2][col=1][im]
281 // v6[1] <- v6[3]=v3[2] [3:2]=2 = x10 <- tmp[2][col=1][re]
282 // v6[2] <- v4[1] [5:4]=1 = x01 <- tmp[2][col=2][im]
283 // v6[3] <- v4[0] [7:6]=0 = x00 <- tmp[2][col=2][re]
284 // So Immediate for shufps is: 0x0001 1011 = x1B = 27
285 v6.vector = _mm_shuffle_ps(v6.vector, v4.vector, 0x1B);
286
287 // Does a v2 - v6*v7
288 // v2[0] <- v2[0] - v6[0]*v7[0] = tmp[1][col=1][re] + tmp[2][col=1][im]
289 // v2[1] <- v2[1] - v6[1]*v7[1] = tmp[1][col=1][im] - tmp[2][col=1][re]
290 // v2[2] <- v2[2] - v6[2]*v7[2] = tmp[1][col=2][re] + tmp[2][col=2][im]
291 // v2[3] <- v2[3] - v6[3]*v7[3] = tmp[1][col=2][im] - tmp[2][col=2][re]
292 v6.vector = _mm_mul_ps(v6.vector, v7.vector);
293 v2.vector = _mm_sub_ps(v2.vector, v6.vector);
294 _mm_store_ps(dst_shadow+8, v2.vector);
295
296 dst_shadow+=12;
297
298 // Store in the next spinor -- should be prefetched if all has gone well
299 v0.vector = _mm_load_ps(src_shadow);
300 v1.vector = _mm_load_ps(src_shadow+4);
301 v2.vector = _mm_load_ps(src_shadow+8);
302 v3.vector = _mm_load_ps(src_shadow+12);
303 v4.vector = _mm_load_ps(src_shadow+16);
304 v5.vector = _mm_load_ps(src_shadow+20);
305
306 }
307
308
309 v6.vector = v4.vector; // V6 is dest so we can move its
310 // I want to shuffle so that:
311 // v6[0] <- v6[3]=v4[3] [1:0]=3 = x11 <- tmp[3][col=0][im]
312 // v6[1] <- v6[2]=v4[2] [3:2]=2 = x10 <- tmp[3][col=0][re]
313 // v6[2] <- v5[1] [5:4]=1 = x01 <- tmp[3][col=1][im]
314 // v6[3] <- v5[0] [7:6]=0 = x00 <- tmp[3][col=1][re]
315 // So Immediate for shufps is: 0x0001 1011 = x1B = 27
316 v6.vector = _mm_shuffle_ps(v6.vector, v5.vector, 0x1B);
317
318 // Does a v0 - v6*v7
319 // v0[0] <- v0[0] - v6[0]*v7[0] = tmp[0][col=0][re] + tmp[3][col=0][im]
320 // v0[1] <- v0[1] - v6[1]*v7[1] = tmp[0][col=0][im] - tmp[3][col=0][re]
321 // v0[2] <- v0[2] - v6[2]*v7[2] = tmp[0][col=1][re] + tmp[3][col=1][im]
322 // v0[3] <- v0[3] - v6[3]*v7[3] = tmp[0][col=1][im] - tmp[3][col=1][re]
323 v6.vector = _mm_mul_ps(v6.vector, v7.vector);
324 v0.vector = _mm_sub_ps(v0.vector, v6.vector);
325 _mm_store_ps(dst_shadow, v0.vector);
326
327 v6.vector = v5.vector;
328 // I want to shuffle so that:
329 // v6[0] <- v6[3]=v5[3] [1:0]=3 = x11 <- tmp[3][col=0][im]
330 // v6[1] <- v6[2]=v5[2] [3:2]=2 = x10 <- tmp[3][col=0][re]
331 // v6[2] <- v3[1] [5:4]=1 = x01 <- tmp[3][col=1][im]
332 // v6[3] <- v3[0] [7:6]=0 = x00 <- tmp[3][col=1][re]
333 // So Immediate for shufps is: 0x0001 1011 = x1B = 27
334 v6.vector = _mm_shuffle_ps(v6.vector, v3.vector, 0x1B);
335
336 // Does a v1 - v6*v7
337 // v1[0] <- v1[0] - v6[0]*v7[0] = tmp[0][col=2][re] + tmp[3][col=2][im]
338 // v1[1] <- v1[1] - v6[1]*v7[1] = tmp[0][col=2][im] - tmp[3][col=2][re]
339 // v1[2] <- v1[2] - v6[2]*v7[2] = tmp[1][col=0][re] + tmp[2][col=0][im]
340 // v1[3] <- v1[3] - v6[3]*v7[3] = tmp[1][col=0][im] - tmp[2][col=0][re]
341 v6.vector = _mm_mul_ps(v6.vector, v7.vector);
342 v1.vector = _mm_sub_ps(v1.vector, v6.vector);
343 _mm_store_ps(dst_shadow+4, v1.vector);
344
345 v6.vector = v3.vector;
346 // v6[0] <- v6[3]=v3[3] [1:0]=3 = x11 <- tmp[2][col=1][im]
347 // v6[1] <- v6[3]=v3[2] [3:2]=2 = x10 <- tmp[2][col=1][re]
348 // v6[2] <- v4[1] [5:4]=1 = x01 <- tmp[2][col=2][im]
349 // v6[3] <- v4[0] [7:6]=0 = x00 <- tmp[2][col=2][re]
350 // So Immediate for shufps is: 0x0001 1011 = x1B = 27
351 v6.vector = _mm_shuffle_ps(v6.vector, v4.vector, 0x1B);
352
353 // Does a v2 - v6*v7
354 // v2[0] <- v2[0] - v6[0]*v7[0] = tmp[1][col=1][re] + tmp[2][col=1][im]
355 // v2[1] <- v2[1] - v6[1]*v7[1] = tmp[1][col=1][im] - tmp[2][col=1][re]
356 // v2[2] <- v2[2] - v6[2]*v7[2] = tmp[1][col=2][re] + tmp[2][col=2][im]
357 // v2[3] <- v2[3] - v6[3]*v7[3] = tmp[1][col=2][im] - tmp[2][col=2][re]
358 v6.vector = _mm_mul_ps(v6.vector, v7.vector);
359 v2.vector = _mm_sub_ps(v2.vector, v6.vector);
360 _mm_store_ps(dst_shadow+8, v2.vector);
361
362}
363
364
365
378inline
379void inlineSpinProjDir1Plus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
380{
381
382#ifdef DEBUG_GENERIC_SPIN_PROJ_INLINES
383 QDPIO::cout << "inlineSpinProjDir1Plus" << endl;
384#endif
385
386 /* 1 + \gamma_1 = 1 0 0 -1
387 0 1 1 0
388 0 1 1 0
389 -1 0 0 1
390
391 * ( b0r + i b0i ) = ( {a0r - a3r} + i{a0i - a3i} )
392 * ( b1r + i b1i ) ( {a1r + a2r} + i{a1i + a2i} )
393 */
394
395 const REAL32* src_shadow = src;
396 REAL32* dst_shadow = dst;
397 SSEVec v0, v1, v2, v3, v4, v5, v6, v7;
398
399 v7.floats[0] = -1;
400 v7.floats[1] = -1;
401 v7.floats[2] = +1;
402 v7.floats[3] = +1;
403
404 // Store in the spinor - top half
405 v0.vector = _mm_load_ps(src_shadow);
406 v1.vector = _mm_load_ps(src_shadow+4);
407 v2.vector = _mm_load_ps(src_shadow+8);
408 v3.vector = _mm_load_ps(src_shadow+12);
409 v4.vector = _mm_load_ps(src_shadow+16);
410 v5.vector = _mm_load_ps(src_shadow+20);
411
412 for(unsigned int site=0; site < n_vec-1; site++) {
413 src_shadow += 24;
414
415 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
416
417 v6.vector = v4.vector; // V6 is dest so we can move its
418 // I want to shuffle so that:
419 // v6[0] <- v6[2]=v4[2] [1:0]=2 = x10 <- tmp[3][col=0][re]
420 // v6[1] <- v6[3]=v4[3] [3:2]=3 = x11 <- tmp[3][col=0][im]
421 // v6[2] <- v5[0] [5:4]=0 = x00 <- tmp[3][col=1][re]
422 // v6[3] <- v5[1] [7:6]=1 = x01 <- tmp[3][col=1][im]
423 // So Immediate for shufps is: 0x0100 1110 = x4E
424 v6.vector = _mm_shuffle_ps(v6.vector, v5.vector, 0x4E);
425
426 // Does a v0 - v6
427 // v0[0] <- v0[0] - v6[0] = tmp[0][col=0][re] - tmp[3][col=0][re]
428 // v0[1] <- v0[1] - v6[1] = tmp[0][col=0][im] - tmp[3][col=0][im]
429 // v0[2] <- v0[2] - v6[2] = tmp[0][col=1][re] - tmp[3][col=1][re]
430 // v0[3] <- v0[3] - v6[3] = tmp[0][col=1][im] - tmp[3][col=1][im]
431 v0.vector = _mm_sub_ps(v0.vector, v6.vector);
432 _mm_store_ps(dst_shadow, v0.vector);
433
434 v6.vector = v5.vector;
435 // Now setup v6 so that
436 // v6[0] <- v6[2]=v5[2] [1:0]=2 = x11 <- tmp[3][col=2][re]
437 // v6[1] <- v6[3]=v5[3] [3:2]=3 = x10 <- tmp[3][col=2][im]
438 // v6[2] <- v3[0] [5:4]=0 = x01 <- tmp[2][col=0][re]
439 // v6[3] <- v3[1] [7:6]=1 = x00 <- tmp[2][col=0][im]
440 // So Immediate for shufps is: 0x0100 1110 = x4E
441 v6.vector = _mm_shuffle_ps(v6.vector, v3.vector, 0x4E);
442
443 // Does a v1 + v7*v6
444 // v1[0] <- v1[0] + v7[0]*v6[0] = tmp[0][col=2][re] - tmp[3][col=2][re]
445 // v1[1] <- v1[1] + v7[1]*v6[1] = tmp[0][col=2][im] - tmp[3][col=2][im]
446 // v1[2] <- v1[2] + v7[2]*v6[2] = tmp[1][col=0][re] + tmp[2][col=0][re]
447 // v1[3] <- v1[3] + v7[3]*v6[3] = tmp[1][col=0][im] + tmp[2][col=0][im]
448 v6.vector = _mm_mul_ps(v6.vector, v7.vector);
449 v1.vector = _mm_add_ps(v1.vector, v6.vector);
450 _mm_store_ps(dst_shadow+4, v1.vector);
451
452 v6.vector = v3.vector;
453 // v6[0] <- v6[2]=v3[2] [1:0]=2 = x10 <- tmp[2][col=1][re]
454 // v6[1] <- v6[3]=v3[3] [3:2]=3 = x11 <- tmp[2][col=1][im]
455 // v6[2] <- v4[0] [5:4]=0 = x00 <- tmp[2][col=2][re]
456 // v6[3] <- v4[1] [7:6]=1 = x01 <- tmp[2][col=2][im]
457 // So Immediate for shufps is: 0x0100 1110 = x4E
458
459 v6.vector = _mm_shuffle_ps(v6.vector, v4.vector, 0x4E);
460
461 // Does a v2 + v6
462 // v2[0] <- v2[0] + v6[0] = tmp[1][col=1][re] + tmp[2][col=1][re]
463 // v2[1] <- v2[1] + v6[1] = tmp[1][col=1][im] + tmp[2][col=1][im]
464 // v2[2] <- v2[2] + v6[2] = tmp[1][col=2][re] + tmp[2][col=2][re]
465 // v2[3] <- v2[3] + v6[3] = tmp[1][col=2][im] + tmp[2][col=2][im]
466 v2.vector = _mm_add_ps(v2.vector, v6.vector);
467 _mm_store_ps(dst_shadow+8, v2.vector);
468
469 dst_shadow+=12;
470
471 // Store in the next spinor -- should be prefetched if all has gone well
472 v0.vector = _mm_load_ps(src_shadow);
473 v1.vector = _mm_load_ps(src_shadow+4);
474 v2.vector = _mm_load_ps(src_shadow+8);
475 v3.vector = _mm_load_ps(src_shadow+12);
476 v4.vector = _mm_load_ps(src_shadow+16);
477 v5.vector = _mm_load_ps(src_shadow+20);
478
479 }
480
481 v6.vector = v4.vector; // V6 is dest so we can move its
482 // I want to shuffle so that:
483 // v6[0] <- v6[2]=v4[2] [1:0]=2 = x10 <- tmp[3][col=0][re]
484 // v6[1] <- v6[3]=v4[3] [3:2]=3 = x11 <- tmp[3][col=0][im]
485 // v6[2] <- v5[0] [5:4]=0 = x00 <- tmp[3][col=1][re]
486 // v6[3] <- v5[1] [7:6]=1 = x01 <- tmp[3][col=1][im]
487 // So Immediate for shufps is: 0x0100 1110 = x4E
488 v6.vector = _mm_shuffle_ps(v6.vector, v5.vector, 0x4E);
489
490 // Does a v0 - v6
491 // v0[0] <- v0[0] - v6[0] = tmp[0][col=0][re] - tmp[3][col=0][re]
492 // v0[1] <- v0[1] - v6[1] = tmp[0][col=0][im] - tmp[3][col=0][im]
493 // v0[2] <- v0[2] - v6[2] = tmp[0][col=1][re] - tmp[3][col=1][re]
494 // v0[3] <- v0[3] - v6[3] = tmp[0][col=1][im] - tmp[3][col=1][im]
495 v0.vector = _mm_sub_ps(v0.vector, v6.vector);
496 _mm_store_ps(dst_shadow, v0.vector);
497
498 v6.vector = v5.vector;
499 // Now setup v6 so that
500 // v6[0] <- v6[2]=v5[2] [1:0]=2 = x11 <- tmp[3][col=2][re]
501 // v6[1] <- v6[3]=v5[3] [3:2]=3 = x10 <- tmp[3][col=2][im]
502 // v6[2] <- v3[0] [5:4]=0 = x01 <- tmp[2][col=0][re]
503 // v6[3] <- v3[1] [7:6]=1 = x00 <- tmp[2][col=0][im]
504 // So Immediate for shufps is: 0x0100 1110 = x4E
505 v6.vector = _mm_shuffle_ps(v6.vector, v3.vector, 0x4E);
506
507 // Does a v1 + v7*v6
508 // v1[0] <- v1[0] + v7[0]*v6[0] = tmp[0][col=2][re] - tmp[3][col=2][re]
509 // v1[1] <- v1[1] + v7[1]*v6[1] = tmp[0][col=2][im] - tmp[3][col=2][im]
510 // v1[2] <- v1[2] + v7[2]*v6[2] = tmp[1][col=0][re] + tmp[2][col=0][re]
511 // v1[3] <- v1[3] + v7[3]*v6[3] = tmp[1][col=0][im] + tmp[2][col=0][im]
512 v6.vector = _mm_mul_ps(v6.vector, v7.vector);
513 v1.vector = _mm_add_ps(v1.vector, v6.vector);
514 _mm_store_ps(dst_shadow+4, v1.vector);
515
516 v6.vector = v3.vector;
517 // v6[0] <- v6[2]=v3[2] [1:0]=2 = x10 <- tmp[2][col=1][re]
518 // v6[1] <- v6[3]=v3[3] [3:2]=3 = x11 <- tmp[2][col=1][im]
519 // v6[2] <- v4[0] [5:4]=0 = x00 <- tmp[2][col=2][re]
520 // v6[3] <- v4[1] [7:6]=1 = x01 <- tmp[2][col=2][im]
521 // So Immediate for shufps is: 0x0100 1110 = x4E
522
523 v6.vector = _mm_shuffle_ps(v6.vector, v4.vector, 0x4E);
524
525 // Does a v2 + v6
526 // v2[0] <- v2[0] + v6[0] = tmp[1][col=1][re] + tmp[2][col=1][re]
527 // v2[1] <- v2[1] + v6[1] = tmp[1][col=1][im] + tmp[2][col=1][im]
528 // v2[2] <- v2[2] + v6[2] = tmp[1][col=2][re] + tmp[2][col=2][re]
529 // v2[3] <- v2[3] + v6[3] = tmp[1][col=2][im] + tmp[2][col=2][im]
530 v2.vector = _mm_add_ps(v2.vector, v6.vector);
531 _mm_store_ps(dst_shadow+8, v2.vector);
532
533}
534
546inline
547void inlineSpinProjDir1Minus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
548{
549
550#ifdef DEBUG_GENERIC_SPIN_PROJ_INLINES
551 QDPIO::cout << "inlineSpinProjDir1Minus" << endl;
552#endif
553
554 /* 1 - \gamma_1 = 1 0 0 +1
555 0 1 -1 0
556 0 -1 1 0
557 +1 0 0 1
558
559 * ( b0r + i b0i ) = ( {a0r + a3r} + i{a0i + a3i} )
560 * ( b1r + i b1i ) ( {a1r - a2r} + i{a1i - a2i} )
561 */
562 const REAL32* src_shadow = src;
563 REAL32* dst_shadow = dst;
564 SSEVec v0, v1, v2, v3, v4, v5, v6, v7;
565
566 v7.floats[0] = -1;
567 v7.floats[1] = -1;
568 v7.floats[2] = +1;
569 v7.floats[3] = +1;
570
571 // Store in the spinor - top half
572 v0.vector = _mm_load_ps(src_shadow);
573 v1.vector = _mm_load_ps(src_shadow+4);
574 v2.vector = _mm_load_ps(src_shadow+8);
575 v3.vector = _mm_load_ps(src_shadow+12);
576 v4.vector = _mm_load_ps(src_shadow+16);
577 v5.vector = _mm_load_ps(src_shadow+20);
578
579 for(unsigned int site=0; site < n_vec-1; site++) {
580 src_shadow += 24;
581 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
582
583 v6.vector = v4.vector; // V6 is dest so we can move its
584 // I want to shuffle so that:
585 // v6[0] <- v6[2]=v4[2] [1:0]=2 = x10 <- tmp[3][col=0][re]
586 // v6[1] <- v6[3]=v4[3] [3:2]=3 = x11 <- tmp[3][col=0][im]
587 // v6[2] <- v5[0] [5:4]=0 = x00 <- tmp[3][col=1][re]
588 // v6[3] <- v5[1] [7:6]=1 = x01 <- tmp[3][col=1][im]
589 // So Immediate for shufps is: 0x0100 1110 = x4E
590 v6.vector = _mm_shuffle_ps(v6.vector, v5.vector, 0x4E);
591
592 // Does a v0 + v6
593 // v0[0] <- v0[0] + v6[0] = tmp[0][col=0][re] + tmp[3][col=0][re]
594 // v0[1] <- v0[1] + v6[1] = tmp[0][col=0][im] + tmp[3][col=0][im]
595 // v0[2] <- v0[2] + v6[2] = tmp[0][col=1][re] + tmp[3][col=1][re]
596 // v0[3] <- v0[3] + v6[3] = tmp[0][col=1][im] + tmp[3][col=1][im]
597 v0.vector = _mm_add_ps(v0.vector, v6.vector);
598 _mm_store_ps(dst_shadow, v0.vector);
599
600 v6.vector = v5.vector;
601 // Now setup v6 so that
602 // v6[0] <- v6[2]=v5[2] [1:0]=2 = x11 <- tmp[3][col=2][re]
603 // v6[1] <- v6[3]=v5[3] [3:2]=3 = x10 <- tmp[3][col=2][im]
604 // v6[2] <- v3[0] [5:4]=0 = x01 <- tmp[2][col=0][re]
605 // v6[3] <- v3[1] [7:6]=1 = x00 <- tmp[2][col=0][im]
606 // So Immediate for shufps is: 0x0100 1110 = x4E
607 v6.vector = _mm_shuffle_ps(v6.vector, v3.vector, 0x4E);
608
609
610 // Does a v1 - v7*v6
611 // v1[0] <- v1[0] - v7[0]*v6[0] = tmp[0][col=2][re] + tmp[3][col=2][re]
612 // v1[1] <- v1[1] - v7[1]*v6[1] = tmp[0][col=2][im] + tmp[3][col=2][im]
613 // v1[2] <- v1[2] - v7[2]*v6[2] = tmp[1][col=0][re] - tmp[2][col=0][re]
614 // v1[3] <- v1[3] - v7[3]*v6[3] = tmp[1][col=0][im] - tmp[2][col=0][im]
615 v6.vector = _mm_mul_ps(v6.vector, v7.vector);
616 v1.vector = _mm_sub_ps(v1.vector, v6.vector);
617 _mm_store_ps(dst_shadow+4, v1.vector);
618
619 v6.vector = v3.vector;
620 // v6[0] <- v6[2]=v3[2] [1:0]=2 = x10 <- tmp[2][col=1][re]
621 // v6[1] <- v6[3]=v3[3] [3:2]=3 = x11 <- tmp[2][col=1][im]
622 // v6[2] <- v4[0] [5:4]=0 = x00 <- tmp[2][col=2][re]
623 // v6[3] <- v4[1] [7:6]=1 = x01 <- tmp[2][col=2][im]
624 // So Immediate for shufps is: 0x0100 1110 = x4E
625
626 v6.vector = _mm_shuffle_ps(v6.vector, v4.vector, 0x4E);
627
628 // Does a v2 - v6
629 // v2[0] <- v2[0] - v6[0] = tmp[1][col=1][re] - tmp[2][col=1][re]
630 // v2[1] <- v2[1] - v6[1] = tmp[1][col=1][im] - tmp[2][col=1][im]
631 // v2[2] <- v2[2] - v6[2] = tmp[1][col=2][re] - tmp[2][col=2][re]
632 // v2[3] <- v2[3] - v6[3] = tmp[1][col=2][im] - tmp[2][col=2][im]
633 v2.vector = _mm_sub_ps(v2.vector, v6.vector);
634 _mm_store_ps(dst_shadow+8, v2.vector);
635
636 dst_shadow+=12;
637
638 v0.vector = _mm_load_ps(src_shadow);
639 v1.vector = _mm_load_ps(src_shadow+4);
640 v2.vector = _mm_load_ps(src_shadow+8);
641 v3.vector = _mm_load_ps(src_shadow+12);
642 v4.vector = _mm_load_ps(src_shadow+16);
643 v5.vector = _mm_load_ps(src_shadow+20);
644
645 }
646
647
648 v6.vector = v4.vector; // V6 is dest so we can move its
649 // I want to shuffle so that:
650 // v6[0] <- v6[2]=v4[2] [1:0]=2 = x10 <- tmp[3][col=0][re]
651 // v6[1] <- v6[3]=v4[3] [3:2]=3 = x11 <- tmp[3][col=0][im]
652 // v6[2] <- v5[0] [5:4]=0 = x00 <- tmp[3][col=1][re]
653 // v6[3] <- v5[1] [7:6]=1 = x01 <- tmp[3][col=1][im]
654 // So Immediate for shufps is: 0x0100 1110 = x4E
655 v6.vector = _mm_shuffle_ps(v6.vector, v5.vector, 0x4E);
656
657 // Does a v0 + v6
658 // v0[0] <- v0[0] + v6[0] = tmp[0][col=0][re] + tmp[3][col=0][re]
659 // v0[1] <- v0[1] + v6[1] = tmp[0][col=0][im] + tmp[3][col=0][im]
660 // v0[2] <- v0[2] + v6[2] = tmp[0][col=1][re] + tmp[3][col=1][re]
661 // v0[3] <- v0[3] + v6[3] = tmp[0][col=1][im] + tmp[3][col=1][im]
662 v0.vector = _mm_add_ps(v0.vector, v6.vector);
663 _mm_store_ps(dst_shadow, v0.vector);
664
665 v6.vector = v5.vector;
666 // Now setup v6 so that
667 // v6[0] <- v6[2]=v5[2] [1:0]=2 = x11 <- tmp[3][col=2][re]
668 // v6[1] <- v6[3]=v5[3] [3:2]=3 = x10 <- tmp[3][col=2][im]
669 // v6[2] <- v3[0] [5:4]=0 = x01 <- tmp[2][col=0][re]
670 // v6[3] <- v3[1] [7:6]=1 = x00 <- tmp[2][col=0][im]
671 // So Immediate for shufps is: 0x0100 1110 = x4E
672 v6.vector = _mm_shuffle_ps(v6.vector, v3.vector, 0x4E);
673
674
675 // Does a v1 - v7*v6
676 // v1[0] <- v1[0] - v7[0]*v6[0] = tmp[0][col=2][re] + tmp[3][col=2][re]
677 // v1[1] <- v1[1] - v7[1]*v6[1] = tmp[0][col=2][im] + tmp[3][col=2][im]
678 // v1[2] <- v1[2] - v7[2]*v6[2] = tmp[1][col=0][re] - tmp[2][col=0][re]
679 // v1[3] <- v1[3] - v7[3]*v6[3] = tmp[1][col=0][im] - tmp[2][col=0][im]
680 v6.vector = _mm_mul_ps(v6.vector, v7.vector);
681 v1.vector = _mm_sub_ps(v1.vector, v6.vector);
682 _mm_store_ps(dst_shadow+4, v1.vector);
683
684 v6.vector = v3.vector;
685 // v6[0] <- v6[2]=v3[2] [1:0]=2 = x10 <- tmp[2][col=1][re]
686 // v6[1] <- v6[3]=v3[3] [3:2]=3 = x11 <- tmp[2][col=1][im]
687 // v6[2] <- v4[0] [5:4]=0 = x00 <- tmp[2][col=2][re]
688 // v6[3] <- v4[1] [7:6]=1 = x01 <- tmp[2][col=2][im]
689 // So Immediate for shufps is: 0x0100 1110 = x4E
690
691 v6.vector = _mm_shuffle_ps(v6.vector, v4.vector, 0x4E);
692
693 // Does a v2 - v6
694 // v2[0] <- v2[0] - v6[0] = tmp[1][col=1][re] - tmp[2][col=1][re]
695 // v2[1] <- v2[1] - v6[1] = tmp[1][col=1][im] - tmp[2][col=1][im]
696 // v2[2] <- v2[2] - v6[2] = tmp[1][col=2][re] - tmp[2][col=2][re]
697 // v2[3] <- v2[3] - v6[3] = tmp[1][col=2][im] - tmp[2][col=2][im]
698 v2.vector = _mm_sub_ps(v2.vector, v6.vector);
699 _mm_store_ps(dst_shadow+8, v2.vector);
700
701}
702
703
716inline
717void inlineSpinProjDir2Plus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
718{
719
720#ifdef DEBUG_GENERIC_SPIN_PROJ_INLINES
721 QDPIO::cout << "inlineSpinProjDir2Plus" << endl;
722#endif
723 /* 1 + \gamma_2 = 1 0 i 0
724 0 1 0 -i
725 -i 0 1 0
726 0 i 0 1
727
728
729 * ( b0r + i b0i ) = ( {a0r - a2i} + i{a0i + a2r} )
730 * ( b1r + i b1i ) ( {a1r + a3i} + i{a1i - a3r} )
731 */
732
733 const REAL32* src_shadow = src;
734 REAL32* dst_shadow = dst;
735 SSEVec v0, v1, v2, v3, v4, v5, v6, v7;
736
737 v7.floats[0] = -1;
738 v7.floats[1] = +1;
739 v7.floats[2] = -1;
740 v7.floats[3] = +1;
741
742 // The shuffling will not need extra registers
743 // so I can reuse v6 for a set of signs.
744 v6.floats[0] = -1;
745 v6.floats[1] = +1;
746 v6.floats[2] = +1;
747 v6.floats[3] = -1;
748
749 // Store in the spinor - top half
750 v0.vector = _mm_load_ps(src_shadow);
751 v1.vector = _mm_load_ps(src_shadow+4);
752 v2.vector = _mm_load_ps(src_shadow+8);
753 v3.vector = _mm_load_ps(src_shadow+12);
754 v4.vector = _mm_load_ps(src_shadow+16);
755 v5.vector = _mm_load_ps(src_shadow+20);
756
757 for(unsigned int site=0; site < n_vec-1; site++) {
758 src_shadow += 24;
759 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
760
761 // I want to shuffle so that
762 // v3[0] <- v3[1] [1:0]=1 = x01 <- tmp[2][col=0][im]
763 // v3[1] <- v3[0] [3:2]=0 = x00 <- tmp[2][col=0][re]
764 // v3[2] <- v3[3] [5:4]=3 = x11 <- tmp[2][col=1][im]
765 // v3[3] <- v3[2] [7:6]=2 = x10 <- tmp[2][col=1][re]
766 //
767 // Note I don't need an extra here. Shuf from v3 to
768 // v3 directly.
769 //
770 // So Immediate for shufps is: 0x1011 0001 = xB1
771 v3.vector = _mm_shuffle_ps(v3.vector, v3.vector, 0xB1);
772
773 // Does a v0 + v7*v3
774 // v0[0] <- v0[0] + v7[0]*v3[0] = tmp[0][col=0][re] - tmp[2][col=0][im]
775 // v0[1] <- v0[1] + v7[1]*v3[1] = tmp[0][col=0][im] + tmp[2][col=0][re]
776 // v0[2] <- v0[2] + v7[2]*v3[2] = tmp[0][col=1][re] - tmp[2][col=1][im]
777 // v0[3] <- v0[3] + v7[3]*v3[3] = tmp[0][col=1][im] + tmp[2][col=1][re]
778 v3.vector = _mm_mul_ps(v3.vector, v7.vector);
779 v0.vector = _mm_add_ps(v0.vector, v3.vector);
780 _mm_store_ps(dst_shadow, v0.vector);
781
782 // Now setup v5 so that
783 // v4[0] <- v4[1] [1:0]=1 = x01 <- tmp[2][col=2][im]
784 // v4[1] <- v4[0] [3:2]=0 = x00 <- tmp[2][col=2][re]
785 // v4[2] <- v4[3] [5:4]=3 = x11 <- tmp[3][col=0][im]
786 // v4[3] <- v4[2] [7:6]=2 = x10 <- tmp[3][col=0][re]
787 // So Immediate for shufps is: 0x1011 0001 = xB1
788 v4.vector = _mm_shuffle_ps(v4.vector, v4.vector, 0xB1);
789
790 // Does a v1 + v6*v4
791 // v1[0] <- v1[0] + v6[0]*v4[0] = tmp[0][col=2][re] - tmp[2][col=2][im]
792 // v1[1] <- v1[1] + v6[1]*v4[1] = tmp[0][col=2][im] + tmp[2][col=2][re]
793 // v1[2] <- v1[2] + v6[2]*v4[2] = tmp[1][col=0][re] + tmp[3][col=0][im]
794 // v1[3] <- v1[3] + v6[3]*v4[3] = tmp[1][col=0][im] - tmp[3][col=0][re]
795 v4.vector = _mm_mul_ps(v4.vector, v6.vector);
796 v1.vector = _mm_add_ps(v1.vector, v4.vector);
797 _mm_store_ps(dst_shadow+4, v1.vector);
798
799 // v5[0] <- v5[1] [1:0]=1 = x01 <- tmp[3][col=1][im]
800 // v5[1] <- v5[0] [3:2]=0 = x00 <- tmp[3][col=1][re]
801 // v5[2] <- v5[3] [5:4]=3 = x11 <- tmp[3][col=2][im]
802 // v5[3] <- v5[2] [7:6]=2 = x10 <- tmp[3][col=2][re]
803 // So Immediate for shufps is: 0x1011 0001 = xB1
804 v5.vector = _mm_shuffle_ps(v5.vector, v5.vector, 0xB1);
805
806 // Does a v0 - v7*v5
807 // v2[0] <- v2[0] - v7[0]*v5[0] = tmp[1][col=1][re] + tmp[2][col=1][im]
808 // v2[1] <- v2[1] - v7[1]*v5[1] = tmp[1][col=1][im] - tmp[2][col=1][re]
809 // v2[2] <- v2[2] - v7[2]*v5[2] = tmp[1][col=2][re] + tmp[2][col=2][im]
810 // v2[3] <- v2[3] - v7[3]*v5[3] = tmp[1][col=2][im] - tmp[2][col=2][re]
811 v5.vector = _mm_mul_ps(v5.vector, v7.vector );
812 v2.vector = _mm_sub_ps(v2.vector, v5.vector);
813 _mm_store_ps(dst_shadow+8, v2.vector);
814
815 dst_shadow+=12;
816
817 // Store in the next spinor - top half
818 v0.vector = _mm_load_ps(src_shadow);
819 v1.vector = _mm_load_ps(src_shadow+4);
820 v2.vector = _mm_load_ps(src_shadow+8);
821 v3.vector = _mm_load_ps(src_shadow+12);
822 v4.vector = _mm_load_ps(src_shadow+16);
823 v5.vector = _mm_load_ps(src_shadow+20);
824
825 }
826
827 // I want to shuffle so that
828 // v3[0] <- v3[1] [1:0]=1 = x01 <- tmp[2][col=0][im]
829 // v3[1] <- v3[0] [3:2]=0 = x00 <- tmp[2][col=0][re]
830 // v3[2] <- v3[3] [5:4]=3 = x11 <- tmp[2][col=1][im]
831 // v3[3] <- v3[2] [7:6]=2 = x10 <- tmp[2][col=1][re]
832 //
833 // Note I don't need an extra here. Shuf from v3 to
834 // v3 directly.
835 //
836 // So Immediate for shufps is: 0x1011 0001 = xB1
837 v3.vector = _mm_shuffle_ps(v3.vector, v3.vector, 0xB1);
838
839 // Does a v0 + v7*v3
840 // v0[0] <- v0[0] + v7[0]*v3[0] = tmp[0][col=0][re] - tmp[2][col=0][im]
841 // v0[1] <- v0[1] + v7[1]*v3[1] = tmp[0][col=0][im] + tmp[2][col=0][re]
842 // v0[2] <- v0[2] + v7[2]*v3[2] = tmp[0][col=1][re] - tmp[2][col=1][im]
843 // v0[3] <- v0[3] + v7[3]*v3[3] = tmp[0][col=1][im] + tmp[2][col=1][re]
844 v3.vector = _mm_mul_ps(v3.vector, v7.vector);
845 v0.vector = _mm_add_ps(v0.vector, v3.vector);
846 _mm_store_ps(dst_shadow, v0.vector);
847
848 // Now setup v5 so that
849 // v4[0] <- v4[1] [1:0]=1 = x01 <- tmp[2][col=2][im]
850 // v4[1] <- v4[0] [3:2]=0 = x00 <- tmp[2][col=2][re]
851 // v4[2] <- v4[3] [5:4]=3 = x11 <- tmp[3][col=0][im]
852 // v4[3] <- v4[2] [7:6]=2 = x10 <- tmp[3][col=0][re]
853 // So Immediate for shufps is: 0x1011 0001 = xB1
854 v4.vector = _mm_shuffle_ps(v4.vector, v4.vector, 0xB1);
855
856 // Does a v1 + v6*v4
857 // v1[0] <- v1[0] + v6[0]*v4[0] = tmp[0][col=2][re] - tmp[2][col=2][im]
858 // v1[1] <- v1[1] + v6[1]*v4[1] = tmp[0][col=2][im] + tmp[2][col=2][re]
859 // v1[2] <- v1[2] + v6[2]*v4[2] = tmp[1][col=0][re] + tmp[3][col=0][im]
860 // v1[3] <- v1[3] + v6[3]*v4[3] = tmp[1][col=0][im] - tmp[3][col=0][re]
861 v4.vector = _mm_mul_ps(v4.vector, v6.vector);
862 v1.vector = _mm_add_ps(v1.vector, v4.vector);
863 _mm_store_ps(dst_shadow+4, v1.vector);
864
865 // v5[0] <- v5[1] [1:0]=1 = x01 <- tmp[3][col=1][im]
866 // v5[1] <- v5[0] [3:2]=0 = x00 <- tmp[3][col=1][re]
867 // v5[2] <- v5[3] [5:4]=3 = x11 <- tmp[3][col=2][im]
868 // v5[3] <- v5[2] [7:6]=2 = x10 <- tmp[3][col=2][re]
869 // So Immediate for shufps is: 0x1011 0001 = xB1
870 v5.vector = _mm_shuffle_ps(v5.vector, v5.vector, 0xB1);
871
872 // Does a v0 - v7*v5
873 // v2[0] <- v2[0] - v7[0]*v5[0] = tmp[1][col=1][re] + tmp[2][col=1][im]
874 // v2[1] <- v2[1] - v7[1]*v5[1] = tmp[1][col=1][im] - tmp[2][col=1][re]
875 // v2[2] <- v2[2] - v7[2]*v5[2] = tmp[1][col=2][re] + tmp[2][col=2][im]
876 // v2[3] <- v2[3] - v7[3]*v5[3] = tmp[1][col=2][im] - tmp[2][col=2][re]
877 v5.vector = _mm_mul_ps(v5.vector, v7.vector );
878 v2.vector = _mm_sub_ps(v2.vector, v5.vector);
879 _mm_store_ps(dst_shadow+8, v2.vector);
880
881}
882
894inline
895void inlineSpinProjDir2Minus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
896{
897
898#ifdef DEBUG_GENERIC_SPIN_PROJ_INLINES
899 QDPIO::cout << "inlineSpinProjDir2Minus" << endl;
900#endif
901
902 /* 1 - \gamma_2 = 1 0 -i 0
903 0 1 0 +i
904 +i 0 1 0
905 0 -i 0 1
906
907
908 * ( b0r + i b0i ) = ( {a0r + a2i} + i{a0i - a2r} )
909 * ( b1r + i b1i ) ( {a1r - a3i} + i{a1i + a3r} )
910 */
911
912 const REAL32* src_shadow = src;
913 REAL32* dst_shadow = dst;
914 SSEVec v0, v1, v2, v3, v4, v5, v6, v7;
915
916 v7.floats[0] = -1;
917 v7.floats[1] = +1;
918 v7.floats[2] = -1;
919 v7.floats[3] = +1;
920
921 // The shuffling will not need extra registers
922 // so I can reuse v6 for a set of signs.
923 v6.floats[0] = -1;
924 v6.floats[1] = +1;
925 v6.floats[2] = +1;
926 v6.floats[3] = -1;
927
928 // Store in the spinor - top half
929 v0.vector = _mm_load_ps(src_shadow);
930 v1.vector = _mm_load_ps(src_shadow+4);
931 v2.vector = _mm_load_ps(src_shadow+8);
932 v3.vector = _mm_load_ps(src_shadow+12);
933 v4.vector = _mm_load_ps(src_shadow+16);
934 v5.vector = _mm_load_ps(src_shadow+20);
935
936 for(unsigned int site=0; site < n_vec-1; site++) {
937 src_shadow += 24;
938 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
939
940 // I want to shuffle so that
941 // v3[0] <- v3[1] [1:0]=1 = x01 <- tmp[2][col=0][im]
942 // v3[1] <- v3[0] [3:2]=0 = x00 <- tmp[2][col=0][re]
943 // v3[2] <- v3[3] [5:4]=3 = x11 <- tmp[2][col=1][im]
944 // v3[3] <- v3[2] [7:6]=2 = x10 <- tmp[2][col=1][re]
945 //
946 // Note I don't need an extra here. Shuf from v3 to
947 // v3 directly.
948 //
949 // So Immediate for shufps is: 0x1011 0001 = xB1
950 v3.vector = _mm_shuffle_ps(v3.vector, v3.vector, 0xB1);
951
952 // Does a v0 - v7*v3
953 // v0[0] <- v0[0] - v7[0]*v3[0] = tmp[0][col=0][re] + tmp[2][col=0][im]
954 // v0[1] <- v0[1] - v7[1]*v3[1] = tmp[0][col=0][im] - tmp[2][col=0][re]
955 // v0[2] <- v0[2] - v7[2]*v3[2] = tmp[0][col=1][re] + tmp[2][col=1][im]
956 // v0[3] <- v0[3] - v7[3]*v3[3] = tmp[0][col=1][im] - tmp[2][col=1][re]
957 v3.vector = _mm_mul_ps(v3.vector, v7.vector);
958 v0.vector = _mm_sub_ps(v0.vector, v3.vector);
959 _mm_store_ps(dst_shadow, v0.vector);
960
961 // Now setup v4 so that
962 // v4[0] <- v4[1] [1:0]=1 = x01 <- tmp[2][col=2][im]
963 // v4[1] <- v4[0] [3:2]=0 = x00 <- tmp[2][col=2][re]
964 // v4[2] <- v4[3] [5:4]=3 = x11 <- tmp[3][col=0][im]
965 // v4[3] <- v4[2] [7:6]=2 = x10 <- tmp[3][col=0][re]
966 // So Immediate for shufps is: 0x1011 0001 = xB1
967 v4.vector = _mm_shuffle_ps(v4.vector, v4.vector, 0xB1);
968
969 // Does a v1 - v6*v4
970 // v1[0] <- v1[0] + v6[0]*v4[0] = tmp[0][col=2][re] + tmp[2][col=2][im]
971 // v1[1] <- v1[1] + v6[1]*v4[1] = tmp[0][col=2][im] - tmp[2][col=2][re]
972 // v1[2] <- v1[2] + v6[2]*v4[2] = tmp[1][col=0][re] - tmp[3][col=0][im]
973 // v1[3] <- v1[3] + v6[3]*v4[3] = tmp[1][col=0][im] + tmp[3][col=0][re]
974 v4.vector = _mm_mul_ps(v4.vector, v6.vector);
975 v1.vector = _mm_sub_ps(v1.vector, v4.vector);
976 _mm_store_ps(dst_shadow+4, v1.vector);
977
978 // v5[0] <- v5[1] [1:0]=1 = x01 <- tmp[3][col=1][im]
979 // v5[1] <- v5[0] [3:2]=0 = x00 <- tmp[3][col=1][re]
980 // v5[2] <- v5[3] [5:4]=3 = x11 <- tmp[3][col=2][im]
981 // v5[3] <- v5[2] [7:6]=2 = x10 <- tmp[3][col=2][re]
982 // So Immediate for shufps is: 0x1011 0001 = xB1
983 v5.vector = _mm_shuffle_ps(v5.vector, v5.vector, 0xB1);
984
985 // Does a v0 + v7*v5
986 // v2[0] <- v2[0] + v7[0]*v5[0] = tmp[1][col=1][re] - tmp[2][col=1][im]
987 // v2[1] <- v2[1] + v7[1]*v5[1] = tmp[1][col=1][im] + tmp[2][col=1][re]
988 // v2[2] <- v2[2] + v7[2]*v5[2] = tmp[1][col=2][re] - tmp[2][col=2][im]
989 // v2[3] <- v2[3] + v7[3]*v5[3] = tmp[1][col=2][im] + tmp[2][col=2][re]
990 v5.vector = _mm_mul_ps(v5.vector, v7.vector );
991 v2.vector = _mm_add_ps(v2.vector, v5.vector);
992 _mm_store_ps(dst_shadow+8, v2.vector);
993
994 dst_shadow+=12;
995
996 // Store in the spinor - top half
997 v0.vector = _mm_load_ps(src_shadow);
998 v1.vector = _mm_load_ps(src_shadow+4);
999 v2.vector = _mm_load_ps(src_shadow+8);
1000 v3.vector = _mm_load_ps(src_shadow+12);
1001 v4.vector = _mm_load_ps(src_shadow+16);
1002 v5.vector = _mm_load_ps(src_shadow+20);
1003
1004 }
1005
1006 // I want to shuffle so that
1007 // v3[0] <- v3[1] [1:0]=1 = x01 <- tmp[2][col=0][im]
1008 // v3[1] <- v3[0] [3:2]=0 = x00 <- tmp[2][col=0][re]
1009 // v3[2] <- v3[3] [5:4]=3 = x11 <- tmp[2][col=1][im]
1010 // v3[3] <- v3[2] [7:6]=2 = x10 <- tmp[2][col=1][re]
1011 //
1012 // Note I don't need an extra here. Shuf from v3 to
1013 // v3 directly.
1014 //
1015 // So Immediate for shufps is: 0x1011 0001 = xB1
1016 v3.vector = _mm_shuffle_ps(v3.vector, v3.vector, 0xB1);
1017
1018 // Does a v0 - v7*v3
1019 // v0[0] <- v0[0] - v7[0]*v3[0] = tmp[0][col=0][re] + tmp[2][col=0][im]
1020 // v0[1] <- v0[1] - v7[1]*v3[1] = tmp[0][col=0][im] - tmp[2][col=0][re]
1021 // v0[2] <- v0[2] - v7[2]*v3[2] = tmp[0][col=1][re] + tmp[2][col=1][im]
1022 // v0[3] <- v0[3] - v7[3]*v3[3] = tmp[0][col=1][im] - tmp[2][col=1][re]
1023 v3.vector = _mm_mul_ps(v3.vector, v7.vector);
1024 v0.vector = _mm_sub_ps(v0.vector, v3.vector);
1025 _mm_store_ps(dst_shadow, v0.vector);
1026
1027 // Now setup v4 so that
1028 // v4[0] <- v4[1] [1:0]=1 = x01 <- tmp[2][col=2][im]
1029 // v4[1] <- v4[0] [3:2]=0 = x00 <- tmp[2][col=2][re]
1030 // v4[2] <- v4[3] [5:4]=3 = x11 <- tmp[3][col=0][im]
1031 // v4[3] <- v4[2] [7:6]=2 = x10 <- tmp[3][col=0][re]
1032 // So Immediate for shufps is: 0x1011 0001 = xB1
1033 v4.vector = _mm_shuffle_ps(v4.vector, v4.vector, 0xB1);
1034
1035 // Does a v1 - v6*v4
1036 // v1[0] <- v1[0] + v6[0]*v4[0] = tmp[0][col=2][re] + tmp[2][col=2][im]
1037 // v1[1] <- v1[1] + v6[1]*v4[1] = tmp[0][col=2][im] - tmp[2][col=2][re]
1038 // v1[2] <- v1[2] + v6[2]*v4[2] = tmp[1][col=0][re] - tmp[3][col=0][im]
1039 // v1[3] <- v1[3] + v6[3]*v4[3] = tmp[1][col=0][im] + tmp[3][col=0][re]
1040 v4.vector = _mm_mul_ps(v4.vector, v6.vector);
1041 v1.vector = _mm_sub_ps(v1.vector, v4.vector);
1042 _mm_store_ps(dst_shadow+4, v1.vector);
1043
1044 // v5[0] <- v5[1] [1:0]=1 = x01 <- tmp[3][col=1][im]
1045 // v5[1] <- v5[0] [3:2]=0 = x00 <- tmp[3][col=1][re]
1046 // v5[2] <- v5[3] [5:4]=3 = x11 <- tmp[3][col=2][im]
1047 // v5[3] <- v5[2] [7:6]=2 = x10 <- tmp[3][col=2][re]
1048 // So Immediate for shufps is: 0x1011 0001 = xB1
1049 v5.vector = _mm_shuffle_ps(v5.vector, v5.vector, 0xB1);
1050
1051 // Does a v0 + v7*v5
1052 // v2[0] <- v2[0] + v7[0]*v5[0] = tmp[1][col=1][re] - tmp[2][col=1][im]
1053 // v2[1] <- v2[1] + v7[1]*v5[1] = tmp[1][col=1][im] + tmp[2][col=1][re]
1054 // v2[2] <- v2[2] + v7[2]*v5[2] = tmp[1][col=2][re] - tmp[2][col=2][im]
1055 // v2[3] <- v2[3] + v7[3]*v5[3] = tmp[1][col=2][im] + tmp[2][col=2][re]
1056 v5.vector = _mm_mul_ps(v5.vector, v7.vector );
1057 v2.vector = _mm_add_ps(v2.vector, v5.vector);
1058 _mm_store_ps(dst_shadow+8, v2.vector);
1059
1060}
1061
1074inline
1075void inlineSpinProjDir3Plus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
1076{
1077
1078#ifdef DEBUG_GENERIC_SPIN_PROJ_INLINES
1079 QDPIO::cout << "inlineSpinProjDir3Plus" << endl;
1080#endif
1081 /* 1 + \gamma_3 = 1 0 1 0
1082 0 1 0 1
1083 1 0 1 0
1084 0 1 0 1
1085
1086 * ( b0r + i b0i ) = ( {a0r + a2r} + i{a0i + a2i} )
1087 * ( b1r + i b1i ) ( {a1r + a3r} + i{a1i + a3i} )
1088 */
1089 const REAL32* src_shadow = src;
1090 REAL32* dst_shadow = dst;
1091 SSEVec v0, v1, v2, v3, v4, v5;
1092
1093 // Store in the spinor - top half
1094 v0.vector = _mm_load_ps(src_shadow);
1095 v1.vector = _mm_load_ps(src_shadow+4);
1096 v2.vector = _mm_load_ps(src_shadow+8);
1097 v3.vector = _mm_load_ps(src_shadow+12);
1098 v4.vector = _mm_load_ps(src_shadow+16);
1099 v5.vector = _mm_load_ps(src_shadow+20);
1100
1101 for(unsigned int site=0; site < n_vec-1; site++) {
1102 src_shadow += 24;
1103 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
1104
1105 // This is easy - I don't need any shuffling
1106 v0.vector = _mm_add_ps(v0.vector, v3.vector);
1107 v1.vector = _mm_add_ps(v1.vector, v4.vector);
1108 v2.vector = _mm_add_ps(v2.vector, v5.vector);
1109
1110 _mm_store_ps(dst_shadow, v0.vector);
1111 _mm_store_ps(dst_shadow+4, v1.vector);
1112 _mm_store_ps(dst_shadow+8, v2.vector);
1113
1114 dst_shadow+=12;
1115
1116 v0.vector = _mm_load_ps(src_shadow);
1117 v1.vector = _mm_load_ps(src_shadow+4);
1118 v2.vector = _mm_load_ps(src_shadow+8);
1119 v3.vector = _mm_load_ps(src_shadow+12);
1120 v4.vector = _mm_load_ps(src_shadow+16);
1121 v5.vector = _mm_load_ps(src_shadow+20);
1122
1123 }
1124
1125
1126 // This is easy - I don't need any shuffling
1127 v0.vector = _mm_add_ps(v0.vector, v3.vector);
1128 v1.vector = _mm_add_ps(v1.vector, v4.vector);
1129 v2.vector = _mm_add_ps(v2.vector, v5.vector);
1130
1131 _mm_store_ps(dst_shadow, v0.vector);
1132 _mm_store_ps(dst_shadow+4, v1.vector);
1133 _mm_store_ps(dst_shadow+8, v2.vector);
1134
1135}
1136
1148inline
1149void inlineSpinProjDir3Minus(const REAL32* src, REAL32 *dst, unsigned int n_vec)
1150{
1151
1152#ifdef DEBUG_GENERIC_SPIN_PROJ_INLINES
1153 QDPIO::cout << "inlineSpinProjDir3Minus" << endl;
1154#endif
1155
1156 /* 1 - \gamma_3 = 1 0 -1 0
1157 0 1 0 -1
1158 -1 0 1 0
1159 0 -1 0 1
1160
1161 * ( b0r + i b0i ) = ( {a0r - a2r} + i{a0i - a2i} )
1162 * ( b1r + i b1i ) ( {a1r - a3r} + i{a1i - a3i} )
1163 */
1164
1165 const REAL32* src_shadow = src;
1166 REAL32* dst_shadow = dst;
1167 SSEVec v0, v1, v2, v3, v4, v5;
1168
1169 // Store in the spinor - top half
1170 v0.vector = _mm_load_ps(src_shadow);
1171 v1.vector = _mm_load_ps(src_shadow+4);
1172 v2.vector = _mm_load_ps(src_shadow+8);
1173 v3.vector = _mm_load_ps(src_shadow+12);
1174 v4.vector = _mm_load_ps(src_shadow+16);
1175 v5.vector = _mm_load_ps(src_shadow+20);
1176
1177 for(unsigned int site=0; site < n_vec-1; site++) {
1178 src_shadow += 24;
1179 _mm_prefetch((const char *)src_shadow, _MM_HINT_T0);
1180
1181 // This is easy - I don't need any shuffling
1182 v0.vector = _mm_sub_ps(v0.vector, v3.vector);
1183 v1.vector = _mm_sub_ps(v1.vector, v4.vector);
1184 v2.vector = _mm_sub_ps(v2.vector, v5.vector);
1185
1186 _mm_store_ps(dst_shadow, v0.vector);
1187 _mm_store_ps(dst_shadow+4, v1.vector);
1188 _mm_store_ps(dst_shadow+8, v2.vector);
1189
1190 dst_shadow+=12;
1191
1192 v0.vector = _mm_load_ps(src_shadow);
1193 v1.vector = _mm_load_ps(src_shadow+4);
1194 v2.vector = _mm_load_ps(src_shadow+8);
1195 v3.vector = _mm_load_ps(src_shadow+12);
1196 v4.vector = _mm_load_ps(src_shadow+16);
1197 v5.vector = _mm_load_ps(src_shadow+20);
1198 }
1199
1200
1201 // This is easy - I don't need any shuffling
1202 v0.vector = _mm_sub_ps(v0.vector, v3.vector);
1203 v1.vector = _mm_sub_ps(v1.vector, v4.vector);
1204 v2.vector = _mm_sub_ps(v2.vector, v5.vector);
1205
1206 _mm_store_ps(dst_shadow, v0.vector);
1207 _mm_store_ps(dst_shadow+4, v1.vector);
1208 _mm_store_ps(dst_shadow+8, v2.vector);
1209
1210}
1211
1212} // namespace QDP;
1213
1214#endif
QDP::REAL32
float REAL32
Definition qdp_scalarsite_defs.h:21
QDP::QDPIO::cout
StandardOutputStream cout
Definition qdp_stdio.cc:21
QDP
Yet another random number generator.
Definition qdp_bgq_threadbind.cc:16
QDP::inlineSpinProjDir0Minus
void inlineSpinProjDir0Minus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin Project (1/2)(1-\gamma_0).
Definition generic_spin_proj_inlines.h:85
QDP::inlineSpinProjDir0Plus
void inlineSpinProjDir0Plus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin Project (1/2)(1+\gamma_0).
Definition generic_spin_proj_inlines.h:24
QDP::inlineSpinProjDir1Minus
void inlineSpinProjDir1Minus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin Project (1/2)(1-\gamma_1).
Definition generic_spin_proj_inlines.h:208
QDP::inlineSpinProjDir1Plus
void inlineSpinProjDir1Plus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin Project (1/2)(1+\gamma_1).
Definition generic_spin_proj_inlines.h:149
QDP::inlineSpinProjDir3Plus
void inlineSpinProjDir3Plus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin Project (1/2)(1+\gamma_3).
Definition generic_spin_proj_inlines.h:392
QDP::inlineSpinProjDir3Minus
void inlineSpinProjDir3Minus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin Project (1/2)(1-\gamma_3).
Definition generic_spin_proj_inlines.h:451
QDP::inlineSpinProjDir2Minus
void inlineSpinProjDir2Minus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin Project (1/2)(1-\gamma_2).
Definition generic_spin_proj_inlines.h:330
QDP::inlineSpinProjDir2Plus
void inlineSpinProjDir2Plus(const REAL *src, REAL *dst, unsigned int n_vec)
Spin Project (1/2)(1+\gamma_2).
Definition generic_spin_proj_inlines.h:270
qdp_sse_intrin.h
QDP::SSEVec::floats
float floats[4]
Definition qdp_sse_intrin.h:10
Generated by doxygen 1.16.1