Line | % of fetches | Source |
---|---|---|
1 | /* | |
2 | * node_server.cpp | |
3 | * | |
4 | * Created on: Jun 11, 2015 | |
5 | * Author: dmarce1 | |
6 | */ | |
7 | ||
8 | #include "node_server.hpp" | |
9 | #include "problem.hpp" | |
10 | #include "future.hpp" | |
11 | #include "options.hpp" | |
12 | #include "taylor.hpp" | |
13 | ||
14 | #include <streambuf> | |
15 | #include <fstream> | |
16 | #include <iostream> | |
17 | #include <sys/stat.h> | |
18 | #if !defined(_MSC_VER) | |
19 | #include <unistd.h> | |
20 | #endif | |
21 | extern options opts; | |
22 | ||
23 | #include <hpx/include/lcos.hpp> | |
24 | ||
25 | HPX_REGISTER_MINIMAL_COMPONENT_FACTORY(hpx::components::managed_component<node_server>, node_server); | |
26 | ||
27 | bool node_server::static_initialized(false); | |
28 | std::atomic<integer> node_server::static_initializing(0); | |
29 | ||
30 | bool node_server::hydro_on = true; | |
31 | bool node_server::gravity_on = true; | |
32 | ||
33 | void node_server::set_gravity(bool b) { | |
34 | gravity_on = b; | |
35 | } | |
36 | ||
37 | void node_server::set_hydro(bool b) { | |
38 | hydro_on = b; | |
39 | } | |
40 | ||
41 | real node_server::get_time() const { | |
42 | return current_time; | |
43 | } | |
44 | ||
45 | real node_server::get_rotation_count() const { | |
46 | if (opts.problem == DWD) { | |
47 | return rotational_time / (2.0 * M_PI); | |
48 | } else { | |
49 | return current_time; | |
50 | } | |
51 | } | |
52 | ||
53 | hpx::future<void> node_server::exchange_flux_corrections() { | |
54 | const geo::octant ci = my_location.get_child_index(); | |
55 | std::vector<hpx::future<void>> futs; | |
56 | constexpr auto full_set = geo::face::full_set(); | |
57 | futs.reserve(full_set.size()); | |
58 | for (auto& f : full_set) { | |
59 | const auto face_dim = f.get_dimension(); | |
60 | auto& this_aunt = aunts[f]; | |
61 | if (!this_aunt.empty()) { | |
62 | std::array<integer, NDIM> lb, ub; | |
63 | lb[XDIM] = lb[YDIM] = lb[ZDIM] = 0; | |
64 | ub[XDIM] = ub[YDIM] = ub[ZDIM] = INX; | |
65 | if (f.get_side() == geo::MINUS) { | |
66 | lb[face_dim] = 0; | |
67 | } else { | |
68 | lb[face_dim] = INX; | |
69 | } | |
70 | ub[face_dim] = lb[face_dim] + 1; | |
71 | auto data = grid_ptr->get_flux_restrict(lb, ub, face_dim); | |
72 | futs.push_back(this_aunt.send_hydro_flux_correct(std::move(data), f.flip(), ci)); | |
73 | } | |
74 | } | |
75 | ||
76 | return hpx::async([this](hpx::future<void> fut) { | |
77 | for (auto& f : geo::face::full_set()) { | |
78 | if (this->nieces[f].size()) { | |
79 | const auto face_dim = f.get_dimension(); | |
80 | for (auto& quadrant : geo::quadrant::full_set()) { | |
81 | std::array<integer, NDIM> lb, ub; | |
82 | switch (face_dim) { | |
83 | case XDIM: | |
84 | lb[XDIM] = f.get_side() == geo::MINUS ? 0 : INX; | |
85 | lb[YDIM] = quadrant.get_side(0) * (INX / 2); | |
86 | lb[ZDIM] = quadrant.get_side(1) * (INX / 2); | |
87 | ub[XDIM] = lb[XDIM] + 1; | |
88 | ub[YDIM] = lb[YDIM] + (INX / 2); | |
89 | ub[ZDIM] = lb[ZDIM] + (INX / 2); | |
90 | break; | |
91 | case YDIM: | |
92 | lb[XDIM] = quadrant.get_side(0) * (INX / 2); | |
93 | lb[YDIM] = f.get_side() == geo::MINUS ? 0 : INX; | |
94 | lb[ZDIM] = quadrant.get_side(1) * (INX / 2); | |
95 | ub[XDIM] = lb[XDIM] + (INX / 2); | |
96 | ub[YDIM] = lb[YDIM] + 1; | |
97 | ub[ZDIM] = lb[ZDIM] + (INX / 2); | |
98 | break; | |
99 | case ZDIM: | |
100 | lb[XDIM] = quadrant.get_side(0) * (INX / 2); | |
101 | lb[YDIM] = quadrant.get_side(1) * (INX / 2); | |
102 | lb[ZDIM] = f.get_side() == geo::MINUS ? 0 : INX; | |
103 | ub[XDIM] = lb[XDIM] + (INX / 2); | |
104 | ub[YDIM] = lb[YDIM] + (INX / 2); | |
105 | ub[ZDIM] = lb[ZDIM] + 1; | |
106 | break; | |
107 | } | |
108 | std::vector<real> data = niece_hydro_channels[f][quadrant].get_future().get(); | |
109 | grid_ptr->set_flux_restrict(data, lb, ub, face_dim); | |
110 | } | |
111 | } | |
112 | } | |
113 | fut.get(); | |
114 | }, hpx::when_all(futs)); | |
115 | } | |
116 | ||
117 | hpx::future<void> node_server::all_hydro_bounds(bool tau_only) { | |
118 | std::vector<hpx::future<void>> fut; | |
119 | fut.push_back(exchange_interlevel_hydro_data()); | |
120 | fut.push_back(collect_hydro_boundaries(tau_only)); | |
121 | fut.push_back(send_hydro_amr_boundaries(tau_only)); | |
122 | return hpx::when_all(fut); | |
123 | } | |
124 | ||
125 | hpx::future<void> node_server::exchange_interlevel_hydro_data() { | |
126 | ||
127 | hpx::future<void> fut; | |
128 | std::vector < real > outflow(NF, ZERO); | |
129 | if (is_refined) { | |
130 | for (auto& ci : geo::octant::full_set()) { | |
131 | std::vector < real > data = child_hydro_channels[ci].get_future().get(); | |
132 | grid_ptr->set_restrict(data, ci); | |
133 | integer fi = 0; | |
134 | for (auto i = data.end() - NF; i != data.end(); ++i) { | |
135 | outflow[fi] += *i; | |
136 | ++fi; | |
137 | } | |
138 | } | |
139 | grid_ptr->set_outflows(std::move(outflow)); | |
140 | } | |
141 | if (my_location.level() > 0) { | |
142 | std::vector < real > data = grid_ptr->get_restrict(); | |
143 | integer ci = my_location.get_child_index(); | |
144 | fut = parent.send_hydro_children(std::move(data), ci); | |
145 | } else { | |
146 | fut = hpx::make_ready_future(); | |
147 | } | |
148 | return fut; | |
149 | } | |
150 | ||
151 | hpx::future<void> node_server::collect_hydro_boundaries(bool tau_only) { | |
152 | std::vector<hpx::future<void>> futs; | |
153 | constexpr auto full_set = geo::direction::full_set(); | |
154 | futs.reserve(full_set.size()); | |
155 | for (auto& dir : full_set) { | |
156 | // if (!dir.is_vertex()) { | |
157 | if (!neighbors[dir].empty()) { | |
158 | // const integer width = dir.is_face() ? H_BW : 1; | |
159 | const integer width = H_BW; | |
160 | auto bdata = grid_ptr->get_hydro_boundary(dir, width, tau_only); | |
161 | futs.push_back(neighbors[dir].send_hydro_boundary(std::move(bdata), dir.flip())); | |
162 | } | |
163 | // } | |
164 | } | |
165 | ||
166 | for (auto& dir : geo::direction::full_set()) { | |
167 | // if (!dir.is_vertex()) { | |
168 | if (!(neighbors[dir].empty() && my_location.level() == 0)) { | |
169 | auto tmp = sibling_hydro_channels[dir].get_future().get(); | |
170 | // const integer width = dir.is_face() ? H_BW : 1; | |
171 | const integer width = H_BW; | |
172 | grid_ptr->set_hydro_boundary(tmp.data, tmp.direction, width, tau_only); | |
173 | } | |
174 | // } | |
175 | } | |
176 | ||
177 | for (auto& face : geo::face::full_set()) { | |
178 | if (my_location.is_physical_boundary(face)) { | |
179 | grid_ptr->set_physical_boundaries(face, current_time); | |
180 | } | |
181 | } | |
182 | ||
183 | return hpx::when_all(futs); | |
184 | } | |
185 | ||
186 | hpx::future<void> node_server::send_hydro_amr_boundaries(bool tau_only) { | |
187 | hpx::future<void> fut; | |
188 | if (is_refined) { | |
189 | std::vector<hpx::future<void>> futs; | |
190 | constexpr auto full_set = geo::octant::full_set(); | |
191 | futs.reserve(full_set.size()); | |
192 | for (auto& ci : full_set) { | |
193 | const auto& flags = amr_flags[ci]; | |
194 | for (auto& dir : geo::direction::full_set()) { | |
195 | // if (!dir.is_vertex()) { | |
196 | if (flags[dir]) { | |
197 | std::array<integer, NDIM> lb, ub; | |
198 | std::vector < real > data; | |
199 | // const integer width = dir.is_face() ? H_BW : 1; | |
200 | const integer width = H_BW; | |
201 | if (!tau_only) { | |
202 | get_boundary_size(lb, ub, dir, OUTER, INX, width); | |
203 | } else { | |
204 | get_boundary_size(lb, ub, dir, OUTER, INX, width); | |
205 | } | |
206 | for (integer dim = 0; dim != NDIM; ++dim) { | |
207 | lb[dim] = ((lb[dim] - H_BW)) + 2 * H_BW + ci.get_side(dim) * (INX); | |
208 | ub[dim] = ((ub[dim] - H_BW)) + 2 * H_BW + ci.get_side(dim) * (INX); | |
209 | } | |
210 | data = grid_ptr->get_prolong(lb, ub, tau_only); | |
211 | futs.push_back(children[ci].send_hydro_boundary(std::move(data), dir)); | |
212 | } | |
213 | } | |
214 | // } | |
215 | } | |
216 | fut = hpx::when_all(futs); | |
217 | } else { | |
218 | fut = hpx::make_ready_future(); | |
219 | ||
220 | } | |
221 | return fut; | |
222 | ||
223 | } | |
224 | ||
225 | inline bool file_exists(const std::string& name) { | |
226 | struct stat buffer; | |
227 | return (stat(name.c_str(), &buffer) == 0); | |
228 | } | |
229 | ||
230 | //HPX_PLAIN_ACTION(grid::set_omega, set_omega_action2); | |
231 | //HPX_PLAIN_ACTION(grid::set_pivot, set_pivot_action2); | |
232 | ||
233 | std::size_t node_server::load_me(FILE* fp) { | |
234 | std::size_t cnt = 0; | |
235 | auto foo = std::fread; | |
236 | refinement_flag = false; | |
237 | cnt += foo(&step_num, sizeof(integer), 1, fp) * sizeof(integer); | |
238 | cnt += foo(¤t_time, sizeof(real), 1, fp) * sizeof(real); | |
239 | cnt += foo(&rotational_time, sizeof(real), 1, fp) * sizeof(real); | |
240 | cnt += grid_ptr->load(fp); | |
241 | return cnt; | |
242 | } | |
243 | ||
244 | std::size_t node_server::save_me(FILE* fp) const { | |
245 | auto foo = std::fwrite; | |
246 | std::size_t cnt = 0; | |
247 | ||
248 | cnt += foo(&step_num, sizeof(integer), 1, fp) * sizeof(integer); | |
249 | cnt += foo(¤t_time, sizeof(real), 1, fp) * sizeof(real); | |
250 | cnt += foo(&rotational_time, sizeof(real), 1, fp) * sizeof(real); | |
251 | assert(grid_ptr != nullptr); | |
252 | cnt += grid_ptr->save(fp); | |
253 | return cnt; | |
254 | } | |
255 | ||
256 | #include "util.hpp" | |
257 | ||
258 | void node_server::save_to_file(const std::string& fname) const { | |
259 | save(0, fname); | |
260 | file_copy(fname.c_str(), "restart.chk"); | |
261 | // std::string command = std::string("cp ") + fname + std::string(" restart.chk\n"); | |
262 | // SYSTEM(command); | |
263 | } | |
264 | ||
265 | void node_server::load_from_file(const std::string& fname) { | |
266 | load(0, hpx::id_type(), false, fname); | |
267 | } | |
268 | ||
269 | void node_server::load_from_file_and_output(const std::string& fname, const std::string& outname) { | |
270 | load(0, hpx::id_type(), true, fname); | |
271 | file_copy("data.silo", outname.c_str()); | |
272 | // std::string command = std::string("mv data.silo ") + outname + std::string("\n"); | |
273 | // SYSTEM(command); | |
274 | } | |
275 | void node_server::clear_family() { | |
276 | parent = hpx::invalid_id; | |
277 | me = hpx::invalid_id; | |
278 | std::fill(aunts.begin(), aunts.end(), hpx::invalid_id); | |
279 | std::fill(neighbors.begin(), neighbors.end(), hpx::invalid_id); | |
280 | std::fill(nieces.begin(), nieces.end(), std::vector<node_client>()); | |
281 | } | |
282 | ||
283 | integer child_index_to_quadrant_index(integer ci, integer dim) { | |
284 | integer index; | |
285 | if (dim == XDIM) { | |
286 | index = ci >> 1; | |
287 | } else if (dim == ZDIM) { | |
288 | index = ci & 0x3; | |
289 | } else { | |
290 | index = (ci & 1) | ((ci >> 1) & 0x2); | |
291 | } | |
292 | return index; | |
293 | } | |
294 | ||
295 | node_server::node_server(const node_location& _my_location, integer _step_num, bool _is_refined, real _current_time, real _rotational_time, | |
296 | const std::array<integer, NCHILD>& _child_d, grid _grid, const std::vector<hpx::id_type>& _c) { | |
297 | my_location = _my_location; | |
298 | initialize(_current_time, _rotational_time); | |
299 | is_refined = _is_refined; | |
300 | step_num = _step_num; | |
301 | current_time = _current_time; | |
302 | rotational_time = _rotational_time; | |
303 | // grid test; | |
304 | grid_ptr = std::make_shared < grid > (std::move(_grid)); | |
305 | if (is_refined) { | |
306 | children.resize(NCHILD); | |
307 | std::copy(_c.begin(), _c.end(), children.begin()); | |
308 | } | |
309 | child_descendant_count = _child_d; | |
310 | } | |
311 | ||
312 | bool node_server::child_is_on_face(integer ci, integer face) { | |
313 | return (((ci >> (face / 2)) & 1) == (face & 1)); | |
314 | } | |
315 | ||
316 | void node_server::static_initialize() { | |
317 | if (!static_initialized) { | |
318 | bool test = (static_initializing++ != 0) ? true : false; | |
319 | if (!test) { | |
320 | static_initialized = true; | |
321 | } | |
322 | while (!static_initialized) { | |
323 | hpx::this_thread::yield(); | |
324 | } | |
325 | } | |
326 | } | |
327 | ||
328 | void node_server::initialize(real t, real rt) { | |
329 | step_num = 0; | |
330 | refinement_flag = 0; | |
331 | static_initialize(); | |
332 | is_refined = false; | |
333 | neighbors.resize(geo::direction::count()); | |
334 | nieces.resize(NFACE); | |
335 | aunts.resize(NFACE); | |
336 | current_time = t; | |
337 | rotational_time = rt; | |
338 | dx = TWO * grid::get_scaling_factor() / real(INX << my_location.level()); | |
339 | for (auto& d : geo::dimension::full_set()) { | |
340 | xmin[d] = grid::get_scaling_factor() * my_location.x_location(d); | |
341 | } | |
342 | if (current_time == ZERO) { | |
343 | const auto p = get_problem(); | |
344 | grid_ptr = std::make_shared < grid > (p, dx, xmin); | |
345 | } else { | |
346 | grid_ptr = std::make_shared < grid > (dx, xmin); | |
347 | } | |
348 | if (my_location.level() == 0) { | |
349 | grid_ptr->set_root(); | |
350 | } | |
351 | } | |
352 | node_server::~node_server() { | |
353 | } | |
354 | ||
355 | node_server::node_server() { | |
356 | initialize(ZERO, ZERO); | |
357 | } | |
358 | ||
359 | node_server::node_server(const node_location& loc, const node_client& parent_id, real t, real rt) : | |
360 | my_location(loc), parent(parent_id) { | |
361 | initialize(t, rt); | |
362 | } | |
363 | ||
364 | void node_server::compute_fmm(gsolve_type type, bool energy_account) { | |
365 | if (!gravity_on) { | |
366 | return; | |
367 | } | |
368 | ||
369 | hpx::future<void> parent_fut; | |
370 | if (energy_account) { | |
371 | // printf( "!\n"); | |
372 | grid_ptr->egas_to_etot(); | |
373 | } | |
374 | multipole_pass_type m_out; | |
375 | m_out.first.resize(INX * INX * INX); | |
376 | m_out.second.resize(INX * INX * INX); | |
377 | if (is_refined) { | |
378 | std::vector<hpx::future<void>> futs; | |
379 | constexpr auto full_set = geo::octant::full_set(); | |
380 | futs.reserve(full_set.size()); | |
381 | for (auto& ci : full_set) { | |
382 | hpx::future<multipole_pass_type> m_in_future = child_gravity_channels[ci].get_future(); | |
383 | ||
384 | futs.push_back( | |
385 | m_in_future.then( | |
386 | [&m_out, ci](hpx::future<multipole_pass_type>&& fut) | |
387 | { | |
388 | const integer x0 = ci.get_side(XDIM) * INX / 2; | |
389 | const integer y0 = ci.get_side(YDIM) * INX / 2; | |
390 | const integer z0 = ci.get_side(ZDIM) * INX / 2; | |
391 | auto m_in = fut.get(); | |
392 | for (integer i = 0; i != INX / 2; ++i) { | |
393 | for (integer j = 0; j != INX / 2; ++j) { | |
394 | for (integer k = 0; k != INX / 2; ++k) { | |
395 | const integer ii = i * INX * INX / 4 + j * INX / 2 + k; | |
396 | const integer io = (i + x0) * INX * INX + (j + y0) * INX + k + z0; | |
397 | m_out.first[io] = m_in.first[ii]; | |
398 | m_out.second[io] = m_in.second[ii]; | |
399 | } | |
400 | } | |
401 | } | |
402 | } | |
403 | ) | |
404 | ); | |
405 | } | |
406 | wait_all_and_propagate_exceptions(futs); | |
407 | m_out = grid_ptr->compute_multipoles(type, &m_out); | |
408 | } else { | |
409 | m_out = grid_ptr->compute_multipoles(type); | |
410 | } | |
411 | ||
412 | if (my_location.level() != 0) { | |
413 | parent_fut = parent.send_gravity_multipoles(std::move(m_out), my_location.get_child_index()); | |
414 | } else { | |
415 | parent_fut = hpx::make_ready_future(); | |
416 | } | |
417 | ||
418 | std::vector<hpx::future<void>> neighbor_futs; | |
419 | constexpr auto full_set = geo::direction::full_set(); | |
420 | neighbor_futs.reserve(full_set.size()); | |
421 | for (auto& dir : full_set) { | |
422 | if (!neighbors[dir].empty()) { | |
423 | auto ndir = dir.flip(); | |
424 | const bool is_monopole = !is_refined; | |
425 | // const auto gid = neighbors[dir].get_gid(); | |
426 | const bool is_local = neighbors[dir].is_local(); | |
427 | neighbor_futs.push_back( | |
428 | neighbors[dir].send_gravity_boundary( | |
429 | grid_ptr->get_gravity_boundary(dir, is_local), ndir, is_monopole)); | |
430 | } | |
431 | } | |
432 | ||
433 | grid_ptr->compute_interactions(type); | |
434 | #ifdef USE_GRAV_PAR | |
435 | std::vector<hpx::future<void>> boundary_futs; | |
436 | boundary_futs.reserve(full_set.size()); | |
437 | for (auto& dir : full_set) { | |
438 | if (!neighbors[dir].empty()) { | |
439 | auto f = neighbor_gravity_channels[dir].get_future(); | |
440 | boundary_futs.push_back(f.then( | |
441 | [this, type](hpx::future<neighbor_gravity_type> fut) | |
442 | { | |
443 | auto && tmp = fut.get(); | |
444 | grid_ptr->compute_boundary_interactions(type, tmp.direction, tmp.is_monopole, tmp.data); | |
445 | }) | |
446 | ); | |
447 | } | |
448 | } | |
449 | wait_all_and_propagate_exceptions(parent_fut, boundary_futs); | |
450 | #else | |
451 | for (auto& dir : geo::direction::full_set()) { | |
452 | if (!neighbors[dir].empty()) { | |
453 | auto tmp = neighbor_gravity_channels[dir].get_future().get(); | |
454 | grid_ptr->compute_boundary_interactions(type, tmp.direction, tmp.is_monopole, tmp.data); | |
455 | } | |
456 | } | |
457 | parent_fut.get(); | |
458 | #endif | |
459 | /************************************************************************************************/ | |
460 | ||
461 | expansion_pass_type l_in; | |
462 | if (my_location.level() != 0) { | |
463 | l_in = parent_gravity_channel.get_future().get(); | |
464 | } | |
465 | const expansion_pass_type ltmp = grid_ptr->compute_expansions(type, my_location.level() == 0 ? nullptr : &l_in); | |
466 | ||
467 | auto f = [this, type, energy_account](expansion_pass_type ltmp) | |
468 | { | |
469 | std::vector<hpx::future<void>> child_futs; | |
470 | if (is_refined) { | |
471 | constexpr auto full_set = geo::octant::full_set(); | |
472 | child_futs.reserve(full_set.size()); | |
473 | for (auto& ci : full_set) { | |
474 | expansion_pass_type l_out; | |
475 | l_out.first.resize(INX * INX * INX / NCHILD); | |
476 | if (type == RHO) { | |
477 | l_out.second.resize(INX * INX * INX / NCHILD); | |
478 | } | |
479 | const integer x0 = ci.get_side(XDIM) * INX / 2; | |
480 | const integer y0 = ci.get_side(YDIM) * INX / 2; | |
481 | const integer z0 = ci.get_side(ZDIM) * INX / 2; | |
482 | for (integer i = 0; i != INX / 2; ++i) { | |
483 | for (integer j = 0; j != INX / 2; ++j) { | |
484 | for (integer k = 0; k != INX / 2; ++k) { | |
485 | const integer io = i * INX * INX / 4 + j * INX / 2 + k; | |
486 | const integer ii = (i + x0) * INX * INX + (j + y0) * INX + k + z0; | |
487 | l_out.first[io] = ltmp.first[ii]; | |
488 | if (type == RHO) { | |
489 | l_out.second[io] = ltmp.second[ii]; | |
490 | } | |
491 | } | |
492 | } | |
493 | } | |
494 | child_futs.push_back(children[ci].send_gravity_expansions(std::move(l_out))); | |
495 | } | |
496 | } | |
497 | ||
498 | if (energy_account) { | |
499 | grid_ptr->etot_to_egas(); | |
500 | } | |
501 | ||
502 | return hpx::when_all(child_futs); | |
503 | }; | |
504 | ||
505 | wait_all_and_propagate_exceptions(f(ltmp), neighbor_futs); | |
506 | } | |
507 | ||
508 | void node_server::report_timing() | |
509 | { | |
510 | timings_.report("..."); | |
511 | } | |
512 |
Copyright (c) 2006-2012 Rogue Wave Software, Inc. All Rights Reserved.
Patents pending.