/home/users/khuck/src/octotiger/src/node_server.cpp

Line% of fetchesSource
1  
/*
2  
 * node_server.cpp
3  
 *
4  
 *  Created on: Jun 11, 2015
5  
 *      Author: dmarce1
6  
 */
7  
8  
#include "node_server.hpp"
9  
#include "problem.hpp"
10  
#include "future.hpp"
11  
#include "options.hpp"
12  
#include "taylor.hpp"
13  
14  
#include <streambuf>
15  
#include <fstream>
16  
#include <iostream>
17  
#include <sys/stat.h>
18  
#if !defined(_MSC_VER)
19  
#include <unistd.h>
20  
#endif
21  
extern options opts;
22  
23  
#include <hpx/include/lcos.hpp>
24  
25  
HPX_REGISTER_MINIMAL_COMPONENT_FACTORY(hpx::components::managed_component<node_server>, node_server);
26  
27  
bool node_server::static_initialized(false);
28  
std::atomic<integer> node_server::static_initializing(0);
29  
30  
bool node_server::hydro_on = true;
31  
bool node_server::gravity_on = true;
32  
33  
void node_server::set_gravity(bool b) {
34  
    gravity_on = b;
35  
}
36  
37  
void node_server::set_hydro(bool b) {
38  
    hydro_on = b;
39  
}
40  
41  
real node_server::get_time() const {
42  
    return current_time;
43  
}
44  
45  
real node_server::get_rotation_count() const {
46  
    if (opts.problem == DWD) {
47  
        return rotational_time / (2.0 * M_PI);
48  
    } else {
49  
        return current_time;
50  
    }
51  
}
52  
53  
hpx::future<void> node_server::exchange_flux_corrections() {
54  
    const geo::octant ci = my_location.get_child_index();
55  
    std::vector<hpx::future<void>> futs;
56  
    constexpr auto full_set = geo::face::full_set();
57  
    futs.reserve(full_set.size());
58  
    for (auto& f : full_set) {
59  
        const auto face_dim = f.get_dimension();
60  
        auto& this_aunt = aunts[f];
61  
        if (!this_aunt.empty()) {
62  
            std::array<integer, NDIM> lb, ub;
63  
            lb[XDIM] = lb[YDIM] = lb[ZDIM] = 0;
64  
            ub[XDIM] = ub[YDIM] = ub[ZDIM] = INX;
65  
            if (f.get_side() == geo::MINUS) {
66  
                lb[face_dim] = 0;
67  
            } else {
68  
                lb[face_dim] = INX;
69  
            }
70  
            ub[face_dim] = lb[face_dim] + 1;
71  
            auto data = grid_ptr->get_flux_restrict(lb, ub, face_dim);
72  
            futs.push_back(this_aunt.send_hydro_flux_correct(std::move(data), f.flip(), ci));
73  
        }
74  
    }
75  
76  
    return hpx::async([this](hpx::future<void> fut) {
77  
        for (auto& f : geo::face::full_set()) {
78  
            if (this->nieces[f].size()) {
79  
                const auto face_dim = f.get_dimension();
80  
                for (auto& quadrant : geo::quadrant::full_set()) {
81  
                    std::array<integer, NDIM> lb, ub;
82  
                    switch (face_dim) {
83  
                        case XDIM:
84  
                        lb[XDIM] = f.get_side() == geo::MINUS ? 0 : INX;
85  
                        lb[YDIM] = quadrant.get_side(0) * (INX / 2);
86  
                        lb[ZDIM] = quadrant.get_side(1) * (INX / 2);
87  
                        ub[XDIM] = lb[XDIM] + 1;
88  
                        ub[YDIM] = lb[YDIM] + (INX / 2);
89  
                        ub[ZDIM] = lb[ZDIM] + (INX / 2);
90  
                        break;
91  
                        case YDIM:
92  
                        lb[XDIM] = quadrant.get_side(0) * (INX / 2);
93  
                        lb[YDIM] = f.get_side() == geo::MINUS ? 0 : INX;
94  
                        lb[ZDIM] = quadrant.get_side(1) * (INX / 2);
95  
                        ub[XDIM] = lb[XDIM] + (INX / 2);
96  
                        ub[YDIM] = lb[YDIM] + 1;
97  
                        ub[ZDIM] = lb[ZDIM] + (INX / 2);
98  
                        break;
99  
                        case ZDIM:
100  
                        lb[XDIM] = quadrant.get_side(0) * (INX / 2);
101  
                        lb[YDIM] = quadrant.get_side(1) * (INX / 2);
102  
                        lb[ZDIM] = f.get_side() == geo::MINUS ? 0 : INX;
103  
                        ub[XDIM] = lb[XDIM] + (INX / 2);
104  
                        ub[YDIM] = lb[YDIM] + (INX / 2);
105  
                        ub[ZDIM] = lb[ZDIM] + 1;
106  
                        break;
107  
                    }
108  
                    std::vector<real> data = niece_hydro_channels[f][quadrant].get_future().get();
109  
                    grid_ptr->set_flux_restrict(data, lb, ub, face_dim);
110  
                }
111  
            }
112  
        }
113  
        fut.get();
114  
    }, hpx::when_all(futs));
115  
}
116  
117  
hpx::future<void> node_server::all_hydro_bounds(bool tau_only) {
118  
    std::vector<hpx::future<void>> fut;
119  
    fut.push_back(exchange_interlevel_hydro_data());
120  
    fut.push_back(collect_hydro_boundaries(tau_only));
121  
    fut.push_back(send_hydro_amr_boundaries(tau_only));
122  
    return hpx::when_all(fut);
123  
}
124  
125  
hpx::future<void> node_server::exchange_interlevel_hydro_data() {
126  
127  
    hpx::future<void> fut;
128  
    std::vector < real > outflow(NF, ZERO);
129  
    if (is_refined) {
130  
        for (auto& ci : geo::octant::full_set()) {
131  
            std::vector < real > data = child_hydro_channels[ci].get_future().get();
132  
            grid_ptr->set_restrict(data, ci);
133  
            integer fi = 0;
134  
            for (auto i = data.end() - NF; i != data.end(); ++i) {
135  
                outflow[fi] += *i;
136  
                ++fi;
137  
            }
138  
        }
139  
        grid_ptr->set_outflows(std::move(outflow));
140  
    }
141  
    if (my_location.level() > 0) {
142  
        std::vector < real > data = grid_ptr->get_restrict();
143  
        integer ci = my_location.get_child_index();
144  
        fut = parent.send_hydro_children(std::move(data), ci);
145  
    } else {
146  
        fut = hpx::make_ready_future();
147  
    }
148  
    return fut;
149  
}
150  
151  
hpx::future<void> node_server::collect_hydro_boundaries(bool tau_only) {
152  
    std::vector<hpx::future<void>> futs;
153  
    constexpr auto full_set = geo::direction::full_set();
154  
    futs.reserve(full_set.size());
155  
    for (auto& dir : full_set) {
156  
//		if (!dir.is_vertex()) {
157  
        if (!neighbors[dir].empty()) {
158  
//				const integer width = dir.is_face() ? H_BW : 1;
159  
            const integer width = H_BW;
160  
            auto bdata = grid_ptr->get_hydro_boundary(dir, width, tau_only);
161  
            futs.push_back(neighbors[dir].send_hydro_boundary(std::move(bdata), dir.flip()));
162  
        }
163  
//		}
164  
    }
165  
166  
    for (auto& dir : geo::direction::full_set()) {
167  
//		if (!dir.is_vertex()) {
168  
        if (!(neighbors[dir].empty() && my_location.level() == 0)) {
169  
            auto tmp = sibling_hydro_channels[dir].get_future().get();
170  
            //				const integer width = dir.is_face() ? H_BW : 1;
171  
            const integer width = H_BW;
172  
            grid_ptr->set_hydro_boundary(tmp.data, tmp.direction, width, tau_only);
173  
        }
174  
//		}
175  
    }
176  
177  
    for (auto& face : geo::face::full_set()) {
178  
        if (my_location.is_physical_boundary(face)) {
179  
            grid_ptr->set_physical_boundaries(face, current_time);
180  
        }
181  
    }
182  
183  
    return hpx::when_all(futs);
184  
}
185  
186  
hpx::future<void> node_server::send_hydro_amr_boundaries(bool tau_only) {
187  
    hpx::future<void> fut;
188  
    if (is_refined) {
189  
        std::vector<hpx::future<void>> futs;
190  
        constexpr auto full_set = geo::octant::full_set();
191  
        futs.reserve(full_set.size());
192  
        for (auto& ci : full_set) {
193  
            const auto& flags = amr_flags[ci];
194  
            for (auto& dir : geo::direction::full_set()) {
195  
                //			if (!dir.is_vertex()) {
196  
                if (flags[dir]) {
197  
                    std::array<integer, NDIM> lb, ub;
198  
                    std::vector < real > data;
199  
//						const integer width = dir.is_face() ? H_BW : 1;
200  
                    const integer width = H_BW;
201  
                    if (!tau_only) {
202  
                        get_boundary_size(lb, ub, dir, OUTER, INX, width);
203  
                    } else {
204  
                        get_boundary_size(lb, ub, dir, OUTER, INX, width);
205  
                    }
206  
                    for (integer dim = 0; dim != NDIM; ++dim) {
207  
                        lb[dim] = ((lb[dim] - H_BW)) + 2 * H_BW + ci.get_side(dim) * (INX);
208  
                        ub[dim] = ((ub[dim] - H_BW)) + 2 * H_BW + ci.get_side(dim) * (INX);
209  
                    }
210  
                    data = grid_ptr->get_prolong(lb, ub, tau_only);
211  
                    futs.push_back(children[ci].send_hydro_boundary(std::move(data), dir));
212  
                }
213  
            }
214  
//			}
215  
        }
216  
        fut = hpx::when_all(futs);
217  
    } else {
218  
        fut = hpx::make_ready_future();
219  
220  
    }
221  
    return fut;
222  
223  
}
224  
225  
inline bool file_exists(const std::string& name) {
226  
    struct stat buffer;
227  
    return (stat(name.c_str(), &buffer) == 0);
228  
}
229  
230  
//HPX_PLAIN_ACTION(grid::set_omega, set_omega_action2);
231  
//HPX_PLAIN_ACTION(grid::set_pivot, set_pivot_action2);
232  
233  
std::size_t node_server::load_me(FILE* fp) {
234  
    std::size_t cnt = 0;
235  
    auto foo = std::fread;
236  
    refinement_flag = false;
237  
    cnt += foo(&step_num, sizeof(integer), 1, fp) * sizeof(integer);
238  
    cnt += foo(&current_time, sizeof(real), 1, fp) * sizeof(real);
239  
    cnt += foo(&rotational_time, sizeof(real), 1, fp) * sizeof(real);
240  
    cnt += grid_ptr->load(fp);
241  
    return cnt;
242  
}
243  
244  
std::size_t node_server::save_me(FILE* fp) const {
245  
    auto foo = std::fwrite;
246  
    std::size_t cnt = 0;
247  
248  
    cnt += foo(&step_num, sizeof(integer), 1, fp) * sizeof(integer);
249  
    cnt += foo(&current_time, sizeof(real), 1, fp) * sizeof(real);
250  
    cnt += foo(&rotational_time, sizeof(real), 1, fp) * sizeof(real);
251  
    assert(grid_ptr != nullptr);
252  
    cnt += grid_ptr->save(fp);
253  
    return cnt;
254  
}
255  
256  
#include "util.hpp"
257  
258  
void node_server::save_to_file(const std::string& fname) const {
259  
    save(0, fname);
260  
    file_copy(fname.c_str(), "restart.chk");
261  
//	std::string command = std::string("cp ") + fname + std::string(" restart.chk\n");
262  
//	SYSTEM(command);
263  
}
264  
265  
void node_server::load_from_file(const std::string& fname) {
266  
    load(0, hpx::id_type(), false, fname);
267  
}
268  
269  
void node_server::load_from_file_and_output(const std::string& fname, const std::string& outname) {
270  
    load(0, hpx::id_type(), true, fname);
271  
    file_copy("data.silo", outname.c_str());
272  
//	std::string command = std::string("mv data.silo ") + outname + std::string("\n");
273  
//	SYSTEM(command);
274  
}
275  
void node_server::clear_family() {
276  
    parent = hpx::invalid_id;
277  
    me = hpx::invalid_id;
278  
    std::fill(aunts.begin(), aunts.end(), hpx::invalid_id);
279  
    std::fill(neighbors.begin(), neighbors.end(), hpx::invalid_id);
280  
    std::fill(nieces.begin(), nieces.end(), std::vector<node_client>());
281  
}
282  
283  
integer child_index_to_quadrant_index(integer ci, integer dim) {
284  
    integer index;
285  
    if (dim == XDIM) {
286  
        index = ci >> 1;
287  
    } else if (dim == ZDIM) {
288  
        index = ci & 0x3;
289  
    } else {
290  
        index = (ci & 1) | ((ci >> 1) & 0x2);
291  
    }
292  
    return index;
293  
}
294  
295  
node_server::node_server(const node_location& _my_location, integer _step_num, bool _is_refined, real _current_time, real _rotational_time,
296  
    const std::array<integer, NCHILD>& _child_d, grid _grid, const std::vector<hpx::id_type>& _c) {
297  
    my_location = _my_location;
298  
    initialize(_current_time, _rotational_time);
299  
    is_refined = _is_refined;
300  
    step_num = _step_num;
301  
    current_time = _current_time;
302  
    rotational_time = _rotational_time;
303  
//     grid test;
304  
    grid_ptr = std::make_shared < grid > (std::move(_grid));
305  
    if (is_refined) {
306  
        children.resize(NCHILD);
307  
        std::copy(_c.begin(), _c.end(), children.begin());
308  
    }
309  
    child_descendant_count = _child_d;
310  
}
311  
312  
bool node_server::child_is_on_face(integer ci, integer face) {
313  
    return (((ci >> (face / 2)) & 1) == (face & 1));
314  
}
315  
316  
void node_server::static_initialize() {
317  
    if (!static_initialized) {
318  
        bool test = (static_initializing++ != 0) ? true : false;
319  
        if (!test) {
320  
            static_initialized = true;
321  
        }
322  
        while (!static_initialized) {
323  
            hpx::this_thread::yield();
324  
        }
325  
    }
326  
}
327  
328  
void node_server::initialize(real t, real rt) {
329  
    step_num = 0;
330  
    refinement_flag = 0;
331  
    static_initialize();
332  
    is_refined = false;
333  
    neighbors.resize(geo::direction::count());
334  
    nieces.resize(NFACE);
335  
    aunts.resize(NFACE);
336  
    current_time = t;
337  
    rotational_time = rt;
338  
    dx = TWO * grid::get_scaling_factor() / real(INX << my_location.level());
339  
    for (auto& d : geo::dimension::full_set()) {
340  
        xmin[d] = grid::get_scaling_factor() * my_location.x_location(d);
341  
    }
342  
    if (current_time == ZERO) {
343  
        const auto p = get_problem();
344  
        grid_ptr = std::make_shared < grid > (p, dx, xmin);
345  
    } else {
346  
        grid_ptr = std::make_shared < grid > (dx, xmin);
347  
    }
348  
    if (my_location.level() == 0) {
349  
        grid_ptr->set_root();
350  
    }
351  
}
352  
node_server::~node_server() {
353  
}
354  
355  
node_server::node_server() {
356  
    initialize(ZERO, ZERO);
357  
}
358  
359  
node_server::node_server(const node_location& loc, const node_client& parent_id, real t, real rt) :
360  
    my_location(loc), parent(parent_id) {
361  
    initialize(t, rt);
362  
}
363  
364  
void node_server::compute_fmm(gsolve_type type, bool energy_account) {
365  
    if (!gravity_on) {
366  
        return;
367  
    }
368  
369  
    hpx::future<void> parent_fut;
370  
    if (energy_account) {
371  
    //	printf( "!\n");
372  
        grid_ptr->egas_to_etot();
373  
    }
374  
    multipole_pass_type m_out;
375  
    m_out.first.resize(INX * INX * INX);
376  
    m_out.second.resize(INX * INX * INX);
377  
    if (is_refined) {
378  
        std::vector<hpx::future<void>> futs;
379  
        constexpr auto full_set = geo::octant::full_set();
380  
        futs.reserve(full_set.size());
381  
        for (auto& ci : full_set) {
382  
            hpx::future<multipole_pass_type> m_in_future = child_gravity_channels[ci].get_future();
383  
384  
            futs.push_back(
385  
                m_in_future.then(
386  
                    [&m_out, ci](hpx::future<multipole_pass_type>&& fut)
387  
                    {
388  
                        const integer x0 = ci.get_side(XDIM) * INX / 2;
389  
                        const integer y0 = ci.get_side(YDIM) * INX / 2;
390  
                        const integer z0 = ci.get_side(ZDIM) * INX / 2;
391  
                        auto m_in = fut.get();
392  
                        for (integer i = 0; i != INX / 2; ++i) {
393  
                            for (integer j = 0; j != INX / 2; ++j) {
394  
                                for (integer k = 0; k != INX / 2; ++k) {
395  
                                    const integer ii = i * INX * INX / 4 + j * INX / 2 + k;
396  
                                    const integer io = (i + x0) * INX * INX + (j + y0) * INX + k + z0;
397  
                                    m_out.first[io] = m_in.first[ii];
398  
                                    m_out.second[io] = m_in.second[ii];
399  
                                }
400  
                            }
401  
                        }
402  
                    }
403  
                )
404  
            );
405  
        }
406  
        wait_all_and_propagate_exceptions(futs);
407  
        m_out = grid_ptr->compute_multipoles(type, &m_out);
408  
    } else {
409  
        m_out = grid_ptr->compute_multipoles(type);
410  
    }
411  
412  
    if (my_location.level() != 0) {
413  
        parent_fut = parent.send_gravity_multipoles(std::move(m_out), my_location.get_child_index());
414  
    } else {
415  
        parent_fut = hpx::make_ready_future();
416  
    }
417  
418  
    std::vector<hpx::future<void>> neighbor_futs;
419  
    constexpr auto full_set = geo::direction::full_set();
420  
    neighbor_futs.reserve(full_set.size());
421  
    for (auto& dir : full_set) {
422  
        if (!neighbors[dir].empty()) {
423  
            auto ndir = dir.flip();
424  
            const bool is_monopole = !is_refined;
425  
//             const auto gid = neighbors[dir].get_gid();
426  
            const bool is_local = neighbors[dir].is_local();
427  
            neighbor_futs.push_back(
428  
                neighbors[dir].send_gravity_boundary(
429  
                    grid_ptr->get_gravity_boundary(dir, is_local), ndir, is_monopole));
430  
        }
431  
    }
432  
433  
    grid_ptr->compute_interactions(type);
434  
#ifdef USE_GRAV_PAR
435  
    std::vector<hpx::future<void>> boundary_futs;
436  
    boundary_futs.reserve(full_set.size());
437  
    for (auto& dir : full_set) {
438  
        if (!neighbors[dir].empty()) {
439  
            auto f = neighbor_gravity_channels[dir].get_future();
440  
            boundary_futs.push_back(f.then(
441  
                [this, type](hpx::future<neighbor_gravity_type> fut)
442  
                {
443  
                    auto && tmp = fut.get();
444  
                    grid_ptr->compute_boundary_interactions(type, tmp.direction, tmp.is_monopole, tmp.data);
445  
                })
446  
            );
447  
        }
448  
    }
449  
    wait_all_and_propagate_exceptions(parent_fut, boundary_futs);
450  
#else
451  
     for (auto& dir : geo::direction::full_set()) {
452  
 		if (!neighbors[dir].empty()) {
453  
 			auto tmp = neighbor_gravity_channels[dir].get_future().get();
454  
 			grid_ptr->compute_boundary_interactions(type, tmp.direction, tmp.is_monopole, tmp.data);
455  
 		}
456  
 	}
457  
 	parent_fut.get();
458  
#endif
459  
	/************************************************************************************************/
460  
461  
    expansion_pass_type l_in;
462  
    if (my_location.level() != 0) {
463  
        l_in = parent_gravity_channel.get_future().get();
464  
    }
465  
    const expansion_pass_type ltmp = grid_ptr->compute_expansions(type, my_location.level() == 0 ? nullptr : &l_in);
466  
467  
    auto f = [this, type, energy_account](expansion_pass_type ltmp)
468  
    {
469  
        std::vector<hpx::future<void>> child_futs;
470  
        if (is_refined) {
471  
            constexpr auto full_set = geo::octant::full_set();
472  
            child_futs.reserve(full_set.size());
473  
            for (auto& ci : full_set) {
474  
                expansion_pass_type l_out;
475  
                l_out.first.resize(INX * INX * INX / NCHILD);
476  
                if (type == RHO) {
477  
                    l_out.second.resize(INX * INX * INX / NCHILD);
478  
                }
479  
                const integer x0 = ci.get_side(XDIM) * INX / 2;
480  
                const integer y0 = ci.get_side(YDIM) * INX / 2;
481  
                const integer z0 = ci.get_side(ZDIM) * INX / 2;
482  
                for (integer i = 0; i != INX / 2; ++i) {
483  
                    for (integer j = 0; j != INX / 2; ++j) {
484  
                        for (integer k = 0; k != INX / 2; ++k) {
485  
                            const integer io = i * INX * INX / 4 + j * INX / 2 + k;
486  
                            const integer ii = (i + x0) * INX * INX + (j + y0) * INX + k + z0;
487  
                            l_out.first[io] = ltmp.first[ii];
488  
                            if (type == RHO) {
489  
                                l_out.second[io] = ltmp.second[ii];
490  
                            }
491  
                        }
492  
                    }
493  
                }
494  
                child_futs.push_back(children[ci].send_gravity_expansions(std::move(l_out)));
495  
            }
496  
        }
497  
498  
        if (energy_account) {
499  
            grid_ptr->etot_to_egas();
500  
        }
501  
502  
        return hpx::when_all(child_futs);
503  
    };
504  
505  
    wait_all_and_propagate_exceptions(f(ltmp), neighbor_futs);
506  
}
507  
508  
void node_server::report_timing()
509  
{
510  
    timings_.report("...");
511  
}
512  

Copyright (c) 2006-2012 Rogue Wave Software, Inc. All Rights Reserved.
Patents pending.