QDP++
qdp_parscalar_init.cc
Go to the documentation of this file.
1
7
8#include <stdlib.h>
9#include <unistd.h>
10
11#include "qdp.h"
12#include "qmp.h"
13
14#if defined(QDP_USE_QMT_THREADS)
15#include <qmt.h>
16#endif
17
18#if defined(QDP_USE_OMP_THREADS)
19#include "qdp_threadbind.h"
20#endif
21
22namespace QDP {
23
28
30 static bool isInit = false;
31
32
33 static bool threadbind = false;
34 static int n_cores = -1;
35 static int n_threads_per_core = -1;
36
37 extern float pool_size_in_gb;
38
40 void QDP_initialize(int *argc, char ***argv)
41 {
42 if (isInit)
43 {
44 QDPIO::cerr << "QDP already inited" << std::endl;
45 QDP_abort(1);
46 }
47
48 //
49 // Process command line
50 //
51 // Default pool size
52 pool_size_in_gb = 8;
53
54 // Look for help
55 bool help_flag = false;
56 for (int i=0; i<*argc; i++)
57 {
58 if (strcmp((*argv)[i], "-h")==0)
59 help_flag = true;
60 }
61
62 bool setGeomP = false;
63 multi1d<int> logical_geom(Nd); // apriori logical geometry of the machine
64
65 logical_geom = 0;
66 bool setIOGeomP = false;
67 multi1d<int> logical_iogeom(Nd); // apriori logical
68 logical_iogeom = 0;
69
70#ifdef USE_REMOTE_QIO
71 int rtiP = 0;
72#endif
73 int QMP_verboseP = 0;
74 const int maxlen = 256;
75 char rtinode[maxlen];
76 strncpy(rtinode, "your_local_food_store", maxlen);
77
78 // Usage
79 if (Layout::primaryNode()) {
80 if (help_flag)
81 {
82 fprintf(stderr,"Usage: %s options\n",(*argv)[0]);
83 fprintf(stderr,"options:\n");
84 fprintf(stderr," -h help\n");
85 fprintf(stderr," -V %%d [%d] verbose mode for QMP\n",
86 QMP_verboseP);
87#if defined(QDP_USE_PROFILING)
88 fprintf(stderr," -p %%d [%d] profile level\n",
89 getProfileLevel());
90#endif
91
92 // logical geometry info
93 fprintf(stderr," -geom %%d");
94 for(int i=1; i < Nd; i++)
95 fprintf(stderr," %%d");
96
97 fprintf(stderr," [-1");
98 for(int i=1; i < Nd; i++)
99 fprintf(stderr,",-1");
100 fprintf(stderr,"] logical machine geometry\n");
101
102#ifdef USE_REMOTE_QIO
103 fprintf(stderr," -cd %%s [.] set working dir for QIO interface\n");
104 fprintf(stderr," -rti %%d [%d] use run-time interface\n",
105 rtiP);
106 fprintf(stderr," -rtinode %%s [%s] run-time interface fileserver node\n",
107 rtinode);
108#endif
109
110 fprintf(stderr, " -poolsize <X> Create a fixed pool of X GB for Pool Alloc\n");
111 fprintf(stderr, " -bind c:s Bind Threads -- BlueGene Q only, c cores per node and s SMT threads to run per core \n");
112
113
114 QDP_abort(1);
115 }
116 }
117
118 for (int i=1; i<*argc; i++)
119 {
120 if (strcmp((*argv)[i], "-V")==0)
121 {
122 QMP_verboseP = 1;
123 }
124#if defined(QDP_USE_PROFILING)
125 else if (strcmp((*argv)[i], "-p")==0)
126 {
127 int lev;
128 sscanf((*argv)[++i], "%d", &lev);
129 setProgramProfileLevel(lev);
130 }
131#endif
132 else if ( strcmp((*argv)[i],"-poolsize")==0)
133 {
134 sscanf((*argv)[++i], "%f", &pool_size_in_gb);
135
136 }
137 else if (strcmp((*argv)[i], "-geom")==0)
138 {
139 setGeomP = true;
140 for(int j=0; j < Nd; j++)
141 {
142 int uu;
143 sscanf((*argv)[++i], "%d", &uu);
144 logical_geom[j] = uu;
145 }
146 }
147 else if (strcmp((*argv)[i], "-iogeom")==0)
148 {
149 setIOGeomP = true;
150 for(int j=0; j < Nd; j++)
151 {
152 int uu;
153 sscanf((*argv)[++i], "%d", &uu);
154 logical_iogeom[j] = uu;
155 }
156 }
157 else if (strcmp((*argv)[i], "-bind")==0)
158 {
159 threadbind = true;
160 sscanf((*argv)[++i], "%d:%d", &n_cores, &n_threads_per_core);
161 }
162#ifdef USE_REMOTE_QIO
163 else if (strcmp((*argv)[i], "-cd")==0)
164 {
165 /* push the dir into the environment vars so qio.c can pick it up */
166 setenv("QHOSTDIR", (*argv)[++i], 0);
167 }
168 else if (strcmp((*argv)[i], "-rti")==0)
169 {
170 sscanf((*argv)[++i], "%d", &rtiP);
171 }
172 else if (strcmp((*argv)[i], "-rtinode")==0)
173 {
174 int n = strlen((*argv)[++i]);
175 if (n >= maxlen)
176 {
177 QDPIO::cerr << __func__ << ": rtinode name too long" << std::endl;
178 QDP_abort(1);
179 }
180 sscanf((*argv)[i], "%s", rtinode);
181 }
182#endif
183#if 0
184 else
185 {
186 QDPIO::cerr << __func__ << ": Unknown argument = " << (*argv)[i] << std::endl;
187 QDP_abort(1);
188 }
189#endif
190
191 if (i >= *argc)
192 {
193 QDPIO::cerr << __func__ << ": missing argument at the end" << std::endl;
194 QDP_abort(1);
195 }
196 }
197
198
199 QMP_verbose (QMP_verboseP);
200
201#if QDP_DEBUG >= 1
202 // Print command line args
203 for (int i=0; i<*argc; i++)
204 QDP_info("QDP_init: arg[%d] = XX%sXX",i,(*argv)[i]);
205#endif
206
207#if QDP_DEBUG >= 1
208 QDP_info("Now initialize QMP");
209#endif
210
211
212
213 if (QMP_is_initialized() == QMP_FALSE)
214 {
215 QMP_thread_level_t prv;
216 if (QMP_init_msg_passing(argc, argv, QMP_THREAD_MULTIPLE, &prv) != QMP_SUCCESS)
217 {
218 QDPIO::cerr << __func__ << ": QMP_init_msg_passing failed" << std::endl;
219 QDP_abort(1);
220 }
221 }
222
223#if QDP_DEBUG >= 1
224 QDP_info("QMP inititalized");
225#endif
226
227 if (setGeomP)
228 if (QMP_declare_logical_topology(logical_geom.slice(), Nd) != QMP_SUCCESS)
229 {
230 QDPIO::cerr << __func__ << ": QMP_declare_logical_topology failed" << std::endl;
231 QDP_abort(1);
232 }
233
234#if QDP_DEBUG >= 1
235 QDP_info("Some layout init");
236#endif
237
238
239 Layout::init(); // setup extremely basic functionality in Layout
240
241 isInit = true;
242
243#if QDP_DEBUG >= 1
244 QDP_info("Init qio");
245#endif
246 // OK, I need to set up the IO geometry here...
247 // I should make it part of layout...
248 if( setIOGeomP ) {
249#if QDP_DEBUG >=1
250 std::ostringstream outbuf;
251 for(int mu=0; mu < Nd; mu++) {
252 outbuf << " " << logical_iogeom[mu];
253 }
254
255 QDP_info("Setting IO Geometry: %s\n", outbuf.str().c_str());
256#endif
257
258 Layout::setIONodeGrid(logical_iogeom);
259
260 }
261 //
262 // add qmt inilisisation
263 //
264#ifdef QDP_USE_QMT_THREADS
265
266 // Initialize threads
267 if( Layout::primaryNode() ) {
268 std::cout << "QDP use qmt threading: Initializing threads..." ;
269 }
270 int thread_status = qmt_init();
271
272 if( thread_status == 0 ) {
273 if ( Layout::primaryNode() ) {
274 std::cout << "Success. We have " << qdpNumThreads() << " threads \n";
275 }
276 }
277 else {
278 std::cout << "Failure... qmt_init() returned " << thread_status << std::endl;
279 QDP_abort(1);
280 }
281
282#else
283#ifdef QDP_USE_OMP_THREADS
284
285 if( Layout::primaryNode()) {
286 std::cout << "QDP use OpenMP threading. We have " << qdpNumThreads() << " threads\n";
287 }
288
289
290#endif
291#endif
292
293 // Alloc space for reductions
294 ThreadReductions::norm2_results = new REAL64 [ qdpNumThreads() ];
295 if( ThreadReductions::norm2_results == 0x0 ) {
296 std::cout << "Failure... space for norm2 results failed " << std::endl;
297 QDP_abort(1);
298 }
299
300 ThreadReductions::innerProd_results = new REAL64 [ 2*qdpNumThreads() ];
301 if( ThreadReductions::innerProd_results == 0x0 ) {
302 std::cout << "Failure... space for innerProd results failed " << std::endl;
303 QDP_abort(1);
304 }
305
306
307 // initialize the global streams
308 QDPIO::cin.init(&std::cin);
309 QDPIO::cout.init(&std::cout);
310 QDPIO::cerr.init(&std::cerr);
311
312#ifdef QDP_USE_OMP_THREADS
313 if ( threadbind ) {
314 QDPIO::cout << "Attempting to bind threads: " << n_cores << " cores with " << n_threads_per_core << " threads per core" << std::endl;
315 setThreadAffinity(n_cores, n_threads_per_core);
316 }
317 reportAffinity();
318#endif
319 initProfile(__FILE__, __func__, __LINE__);
320
321 QDPIO::cout << "Initialize done" << std::endl;
322 }
323
325 bool QDP_isInitialized() {return isInit;}
326
328 void QDP_finalize()
329 {
330 if ( ! QDP_isInitialized() )
331 {
332 QDPIO::cerr << "QDP is not inited" << std::endl;
333 QDP_abort(1);
334 }
335
336#if defined(QDP_USE_HDF5)
337 H5close();
338#endif
339
340 //
341 // finalise qmt
342 //
343 delete [] ThreadReductions::norm2_results;
344 delete [] ThreadReductions::innerProd_results;
345#if defined(QMT_USE_QMT_THREADS)
346 // Finalize threads
347 std::cout << "QDP use qmt threading: Finalizing threads" << std::endl;
348 qmt_finalize();
349#endif
350
351 printProfile();
352
353 QMP_finalize_msg_passing();
354
355 isInit = false;
356 }
357
359 void QDP_abort(int status)
360 {
361 QMP_abort(status);
362 }
363
365 void QDP_resume() {}
366
368 void QDP_suspend() {}
369
370
371} // namespace QDP;
QDP::multi1d
Container for a multi-dimensional 1D array.
Definition qdp_multi.h:25
QDP::multi1d::slice
const T * slice() const
Return ref to a column slice.
Definition qdp_multi.h:225
QDP::REAL64
double REAL64
Definition qdp_scalarsite_defs.h:22
QDP::Nd
const int Nd
Definition qdp_params.h:24
QDP::Layout::setIONodeGrid
void setIONodeGrid(const multi1d< int > &io_grid)
Set the I/O Node grid – satisfy interface.
Definition qdp_parscalar_layout.cc:197
QDP::Layout::init
void init()
Initializer for layout.
Definition qdp_parscalar_layout.cc:157
QDP::Layout::primaryNode
bool primaryNode()
Returns whether this is the primary node.
Definition qdp_parscalar_layout.cc:112
QDP::QDPIO::cout
StandardOutputStream cout
Definition qdp_stdio.cc:21
QDP::QDPIO::cerr
StandardOutputStream cerr
Definition qdp_stdio.cc:22
QDP::QDPIO::cin
StandardInputStream cin
Definition qdp_stdio.cc:20
QDP
Yet another random number generator.
Definition qdp_bgq_threadbind.cc:16
QDP::setThreadAffinity
void setThreadAffinity(int nCores, int threadsPerCore)
Definition qdp_bgq_threadbind.cc:72
QDP::printProfile
void printProfile()
Definition qdp_profile.cc:53
QDP::threadbind
static bool threadbind
Definition qdp_parscalar_init.cc:33
QDP::getProfileLevel
int getProfileLevel()
Definition qdp_profile.cc:25
QDP::QDP_isInitialized
bool QDP_isInitialized()
Is the machine initialized?
Definition qdp_parscalar_init.cc:325
QDP::n_cores
static int n_cores
Definition qdp_parscalar_init.cc:34
QDP::QDP_initialize
void QDP_initialize(int *argc, char ***argv)
Turn on the machine.
Definition qdp_parscalar_init.cc:40
QDP::isInit
static bool isInit
Private flag for status.
Definition qdp_parscalar_init.cc:30
QDP::QDP_suspend
void QDP_suspend()
Suspends QDP communications.
Definition qdp_parscalar_init.cc:368
QDP::reportAffinity
void reportAffinity()
Definition qdp_bgq_threadbind.cc:77
QDP::setProgramProfileLevel
int setProgramProfileLevel(int n)
Definition qdp_profile.cc:55
QDP::n_threads_per_core
static int n_threads_per_core
Definition qdp_parscalar_init.cc:35
QDP::QDP_abort
void QDP_abort(int status)
Panic button.
Definition qdp_parscalar_init.cc:359
QDP::QDP_resume
void QDP_resume()
Resumes QDP communications.
Definition qdp_parscalar_init.cc:365
QDP::initProfile
void initProfile(const std::string &file, const std::string &caller, int line)
Definition qdp_profile.cc:51
QDP::pool_size_in_gb
float pool_size_in_gb
Definition qdp_parscalar_layout.cc:38
QDP::qdpNumThreads
int qdpNumThreads()
Definition qdp_dispatch.h:93
QDP::QDP_finalize
void QDP_finalize()
Turn off the machine.
Definition qdp_parscalar_init.cc:328
QDP::QDP_info
int QDP_info(const char *format,...)
Simple information display routine.
Definition qdp_util.cc:43
qdp.h
Primary include file for QDP.
QMP_verbose
void QMP_verbose(QMP_bool_t verbose)
Definition qdp_qmp_extensions.cc:9
qdp_threadbind.h
Generated by doxygen 1.16.1