QDP++
sse_blas_vaxpy4_double.cc
Go to the documentation of this file.
1
6
7#include "scalarsite_sse/sse_blas_vaxpy4_double.h"
8#include <xmmintrin.h>
9#include "scalarsite_sse/sse_prefetch.h"
10
11namespace QDP {
12
13
14
15#ifndef L2BY2
16#define L2BY2 1365 /* L2 / 2 in SPINORS */
17#endif
18
19
20void vaxpy4(REAL64 *Out,REAL64 *scalep,REAL64 *InScale, int n_4spin)
21{
22 __m128d scalar;
23 __m128d tmp1;
24 __m128d tmp2;
25 __m128d tmp3;
26 __m128d in1;
27 __m128d add1;
28 __m128d in2;
29 __m128d add2;
30 __m128d in3;
31 __m128d add3;
32 __m128d in4;
33 __m128d add4;
34 __m128d out1;
35 __m128d out2;
36 __m128d out3;
37
38 // Load the scalar into low bytes of scalar
39 scalar = _mm_load_sd(scalep);
40
41 // cross components into tmp
42 // Zero tmp
43 tmp1 = _mm_setzero_pd();
44 tmp1 = _mm_shuffle_pd(scalar, scalar, 0x1);
45 scalar = _mm_add_pd(scalar, tmp1);
46
47 double *in_p=InScale;
48 double *out_p=Out;
49
50 if( n_4spin < L2BY2 ) {
51
52 // Less than L2 size.
53 // Out p is sequential read/write: USE PREFETCHW
54 // in_p is sequential read only: USE PREFETCH + HW PREFETCHER
55 PREFETCH(((const char *)in_p)+16);
56
57 for(int i=0; i < 3*n_4spin; i++) {
58 PREFETCHW(((const char *)out_p)+16);
59
60 add1 = _mm_load_pd(out_p);
61 in1 = _mm_load_pd(in_p);
62 tmp1 = _mm_mul_pd(scalar, in1);
63 out1 = _mm_add_pd(tmp1,add1);
64 _mm_store_pd(out_p, out1);
65
66 add2 = _mm_load_pd(out_p+2);
67 in2 = _mm_load_pd(in_p+2);
68 tmp2 = _mm_mul_pd(scalar, in2);
69 out2 = _mm_add_pd(tmp2,add2);
70 _mm_store_pd(out_p+2, out2);
71
72 add3 = _mm_load_pd(out_p+4);
73 in3 = _mm_load_pd(in_p+4);
74 tmp3 = _mm_mul_pd(scalar, in3);
75 out3 = _mm_add_pd(tmp3,add3);
76 _mm_store_pd(out_p+4, out3);
77
78 add1 = _mm_load_pd(out_p+6);
79 in1 = _mm_load_pd(in_p+6);
80 tmp1 = _mm_mul_pd(scalar, in1);
81 out1 = _mm_add_pd(tmp1,add1);
82 _mm_store_pd(out_p+6, out1);
83
84 out_p+=8; in_p+=8;
85
86 }
87 }
88 else {
89 // > L2BY2 Size
90 // Out_p is sequential read/write: PREFETCHNTA
91 // In_p is sequential read: PREFETCHNTA
92 for(int i=0; i < 3*n_4spin; i++) {
93
94 PREFETCHNTA(((const char *)out_p)+56);
95 PREFETCHNTA(((const char *)in_p)+56);
96 add1 = _mm_load_pd(out_p);
97 in1 = _mm_load_pd(in_p);
98 tmp1 = _mm_mul_pd(scalar, in1);
99 out1 = _mm_add_pd(tmp1,add1);
100 _mm_store_pd(out_p, out1);
101
102 add2 = _mm_load_pd(out_p+2);
103 in2 = _mm_load_pd(in_p+2);
104 tmp2 = _mm_mul_pd(scalar, in2);
105 out2 = _mm_add_pd(tmp2,add2);
106 _mm_store_pd(out_p+2, out2);
107
108 add3 = _mm_load_pd(out_p+4);
109 in3 = _mm_load_pd(in_p+4);
110 tmp3 = _mm_mul_pd(scalar, in3);
111 out3 = _mm_add_pd(tmp3,add3);
112 _mm_store_pd(out_p+4, out3);
113
114 add1 = _mm_load_pd(out_p+6);
115 in1 = _mm_load_pd(in_p+6);
116 tmp1 = _mm_mul_pd(scalar, in1);
117 out1 = _mm_add_pd(tmp1,add1);
118 _mm_store_pd(out_p+6, out1);
119
120 out_p+=8; in_p+=8;
121
122 }
123
124 }
125}
126
127
128
129void vaxpyz4(REAL64 *Out,REAL64 *scalep,REAL64 *InScale, REAL64 *Add,int n_4vec)
130{
131 __m128d scalar;
132 __m128d tmp1;
133 __m128d tmp2;
134 __m128d tmp3;
135 __m128d in1;
136 __m128d add1;
137 __m128d in2;
138 __m128d add2;
139 __m128d in3;
140 __m128d add3;
141 __m128d in4;
142 __m128d add4;
143 __m128d out1;
144 __m128d out2;
145 __m128d out3;
146
147 // Load the scalar into low bytes of scalar
148 scalar = _mm_load_sd(scalep);
149
150 // cross components into tmp
151 // Zero tmp
152 tmp1 = _mm_setzero_pd();
153
154 tmp1 = _mm_shuffle_pd(scalar, scalar, 0x1);
155 scalar = _mm_add_pd(scalar, tmp1);
156
157 // Do n_3vec 3vectors.
158 double *in_p=InScale;
159 double *add_p=Add;
160 double *out_p=Out;
161
162
163 if( n_4vec < L2BY2 ) {
164
165 // Less than L2
166 // in_p sequential read only :
167 // add_p sequential read only :
168 // out_p sequential read/write :
169 PREFETCH(((const char *)add_p)+16);
170 PREFETCH(((const char *)in_p)+16);
171
172 for(int i=0; i < 3*n_4vec; i++) {
173 PREFETCHW(((const char *)out_p)+16);
174 add1 = _mm_load_pd(add_p);
175 in1 = _mm_load_pd(in_p);
176 tmp1 = _mm_mul_pd(scalar, in1);
177 out1 = _mm_add_pd(tmp1,add1);
178 _mm_store_pd(out_p, out1);
179
180 add2 = _mm_load_pd(add_p+2);
181 in2 = _mm_load_pd(in_p+2);
182 tmp2 = _mm_mul_pd(scalar, in2);
183 out2 = _mm_add_pd(tmp2,add2);
184 _mm_store_pd(out_p+2, out2);
185
186 add3 = _mm_load_pd(add_p+4);
187 in3 = _mm_load_pd(in_p+4);
188 tmp3 = _mm_mul_pd(scalar, in3);
189 out3 = _mm_add_pd(tmp3,add3);
190 _mm_store_pd(out_p+4, out3);
191
192 add1 = _mm_load_pd(add_p+6);
193 in1 = _mm_load_pd(in_p+6);
194 tmp1 = _mm_mul_pd(scalar, in1);
195 out1 = _mm_add_pd(tmp1,add1);
196 _mm_store_pd(out_p+6, out1);
197
198 out_p+=8; in_p+=8; add_p+=8;
199
200 }
201 }
202 else {
203
204 for(int i=0; i < 3*n_4vec; i++) {
205
206 PREFETCHNTA(((const char *)add_p)+56);
207 PREFETCHNTA(((const char *)in_p)+56);
208
209 add1 = _mm_load_pd(add_p);
210 in1 = _mm_load_pd(in_p);
211 tmp1 = _mm_mul_pd(scalar, in1);
212 out1 = _mm_add_pd(tmp1,add1);
213 _mm_stream_pd(out_p, out1);
214
215
216 add2 = _mm_load_pd(add_p+2);
217 in2 = _mm_load_pd(in_p+2);
218 tmp2 = _mm_mul_pd(scalar, in2);
219 out2 = _mm_add_pd(tmp2,add2);
220 _mm_stream_pd(out_p+2, out2);
221
222 add3 = _mm_load_pd(add_p+4);
223 in3 = _mm_load_pd(in_p+4);
224 tmp3 = _mm_mul_pd(scalar, in3);
225 out3 = _mm_add_pd(tmp3,add3);
226 _mm_stream_pd(out_p+4, out3);
227
228
229 add1 = _mm_load_pd(add_p+6);
230 in1 = _mm_load_pd(in_p+6);
231 tmp1 = _mm_mul_pd(scalar, in1);
232 out1 = _mm_add_pd(tmp1,add1);
233 _mm_stream_pd(out_p+6, out1);
234
235 out_p+=8; in_p+=8; add_p+=8;
236
237 }
238 }
239}
240
241
242} // namespace QDP;
243
QDP::REAL64
double REAL64
Definition qdp_scalarsite_defs.h:22
QDP
Yet another random number generator.
Definition qdp_bgq_threadbind.cc:16
QDP::vaxpy4
void vaxpy4(REAL64 *Out, REAL64 *scalep, REAL64 *InScale, int n_4spin)
Definition sse_blas_vaxpy4_double.cc:20
QDP::vaxpyz4
void vaxpyz4(REAL64 *Out, REAL64 *scalep, REAL64 *InScale, REAL64 *Add, int n_4vec)
Definition sse_blas_vaxpy4_double.cc:129
L2BY2
#define L2BY2
Definition sse_blas_vaxmbyz4_double.cc:15
sse_blas_vaxpy4_double.h
Generic Scalar VAXPY routine.
sse_prefetch.h
PREFETCH
#define PREFETCH(a)
Definition sse_prefetch.h:11
PREFETCHW
#define PREFETCHW(a)
Definition sse_prefetch.h:21
PREFETCHNTA
#define PREFETCHNTA(a)
Definition sse_prefetch.h:9
Generated by doxygen 1.16.1