yohai/phigrape
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Now working as the hybrid code

Browse source 
Removed the need for the annoying ETICS_CEP flag; now dt_scf is read from the config file.
This commit is contained in:
Yohai Meiron 2020-04-13 23:08:42 -04:00
parent 75ad0f0e89
commit b51613695f
5 changed files with 140 additions and 95 deletions
Download patch file Download diff file Expand all files Collapse all files

103
phigrape.cpp
View file

@ -62,11 +62,6 @@ Config *config;
#ifdef ETICS
#include "grapite.h"
// why do we need CEP as a compilaion flag... just have it always on when ETICS is on. IF there is no CEP, there should be a graceful skipping of those operations.
//#define ETICS_CEP
#ifndef ETICS_DTSCF
#error "ETICS_DTSCF must be defined"
#endif
#endif
#define TIMING
@ -140,8 +135,6 @@ int new_tunit=51, new_xunit=51;
double ti=0.0;
double eps_BH=0.0;
/* external potential... */
double3 x_bh1, x_bh2, v_bh1, v_bh2;
@ -156,11 +149,8 @@ double3 a_pn1[7], adot_pn1[7], a_pn2[7], adot_pn2[7];
#include "pn_bh_spin.c"
#ifdef ETICS
double t_exp, dt_exp=ETICS_DTSCF; // t_exp is just the initial value
#ifdef ETICS_CEP
int grapite_cep_index;
#endif
#endif
void get_CPU_time(double *time_real, double *time_user, double *time_syst)
{
@ -989,10 +979,9 @@ int main(int argc, char *argv[])
g6_set_xunit(new_xunit);
#ifdef ETICS
grapite_read_particle_tags(N, "grapite.mask", myRank, n_loc);
grapite_set_dt_exp(dt_exp);
dt_exp = time_cur; // if we don't have a binary restart then we don't remember the coefficients, and we need to calculate them now.
grapite_set_t_exp(t_exp);
grapite_read_particle_tags(N, config->grapite_mask_file_name.c_str(), myRank, n_loc);
grapite_set_dt_exp(config->dt_scf);
grapite_set_t_exp(time_cur);
#endif
/* load the nj particles to the G6 */
@ -1004,24 +993,17 @@ int main(int argc, char *argv[])
#ifdef ETICS
double etics_length_scale;
if (myRank == rootRank) etics_length_scale = grapite_get_length_scale(N, m, x, v); // We don't want all ranks to do it, because they need to write a file and might confuse each other
if (myRank == rootRank) etics_length_scale = grapite_get_length_scale(N, m, (double(*)[3])x, (double(*)[3])v); // We don't want all ranks to do it, because they need to write a file and might confuse each other
MPI_Bcast(&etics_length_scale, 1, MPI_DOUBLE, rootRank, MPI_COMM_WORLD);
grapite_set_length_scale(etics_length_scale);
#endif
#ifdef ETICS_CEP
// First time only: get the CEP index
grapite_cep_index = grapite_get_cep_index();
// First calculate the DC
grapite_calc_center(N, m, x, v, xcm, vcm, xdc, vdc);
// Now copy it to the global particle list
memcpy(x[grapite_cep_index], xdc, 3*sizeof(double));
memcpy(v[grapite_cep_index], vdc, 3*sizeof(double));
double zeros[3] = {0., 0., 0.}; // We haven't calculated the force yet!
grapite_update_cep(time_cur, xdc, vdc, zeros, zeros);
if (grapite_cep_index >= 0) {
grapite_calc_center(N, m, (double(*)[3])x, (double(*)[3])v, xcm, vcm, xdc, vdc);
memcpy(x[grapite_cep_index], xdc, 3*sizeof(double));
memcpy(v[grapite_cep_index], vdc, 3*sizeof(double));
grapite_update_cep(time_cur, xdc, vdc, zeros, zeros);
}
#endif
/* define the all particles as a active on all the processors for the first time grav calc. */
@ -1150,25 +1132,19 @@ int main(int argc, char *argv[])
energy_contr(time_cur, timesteps, n_act_sum, g6_calls, rcm_sum, vcm_sum, E_tot_0, E_tot_corr_0, E_tot_corr_sd_0, skip_con, N, m, x, v, pot, pot_ext);
} /* if (myRank == rootRank) */
#ifdef ETICS_DUMP
if (diskstep==0) {
#ifdef ETICS
if (config->etics_dump_coeffs && (diskstep==0)) {
char out_fname[256];
sprintf(out_fname, "coeffs.%06d.%02d.dat", 0, myRank);
grapite_dump(out_fname, 2);
}
#endif
/* Wait to all processors to finish his works... */
MPI_Barrier(MPI_COMM_WORLD);
#ifdef ETICS_CEP
// First calculate the DC
grapite_calc_center(N, m, x, v, xcm, vcm, xdc, vdc);
// Now copy it to the global particle list
memcpy(x[grapite_cep_index], xdc, 3*sizeof(double));
memcpy(v[grapite_cep_index], vdc, 3*sizeof(double));
grapite_update_cep(time_cur, xdc, vdc, a[grapite_cep_index], adot[grapite_cep_index]);
if (grapite_cep_index >= 0) {
grapite_calc_center(N, m, (double(*)[3])x, (double(*)[3])v, xcm, vcm, xdc, vdc);
memcpy(x[grapite_cep_index], xdc, 3*sizeof(double));
memcpy(v[grapite_cep_index], vdc, 3*sizeof(double));
grapite_update_cep(time_cur, xdc, vdc, a[grapite_cep_index], adot[grapite_cep_index]);
}
#endif
/* Wait to all processors to finish his works... */
@ -1506,9 +1482,9 @@ int main(int argc, char *argv[])
get_CPU_time(&CPU_tmp_real0, &CPU_tmp_user0, &CPU_tmp_syst0);
#endif
for (int i=0; i<n_act; i++) { // TODO would be nicer to use i instead of j here
#ifdef ETICS_CEP
if (ind_act[i] == grapite_cep_index) grapite_update_cep(t_act[i], x_act[i], v_act[i], a_act[i], adot_act[i]); // All ranks should do it.
for (int i=0; i<n_act; i++) {
#ifdef ETICS
if (ind_act[i] == grapite_cep_index) grapite_update_cep(t[grapite_cep_index], x[grapite_cep_index], v[grapite_cep_index], a[grapite_cep_index], adot[grapite_cep_index]); // All ranks should do it.
#endif
cur_rank = ind_act[i]/n_loc;
@ -1586,48 +1562,39 @@ int main(int argc, char *argv[])
} /* if (myRank == rootRank) */
#ifdef ETICS_CEP
#ifdef ETICS
// We are /inside/ a control step, so all particles must be synchronized; we can safely calculate their density centre. The acceleration and jerk currently in the memory are for the predicted position of the CEP, by calling grapite_calc_center we "correct" the position and velocity, but not the gravity at that point.
// First calculate the DC
grapite_calc_center(N, m, x, v, xcm, vcm, xdc, vdc);
// Now copy it to the global particle list
memcpy(x[grapite_cep_index], xdc, 3*sizeof(double));
memcpy(v[grapite_cep_index], vdc, 3*sizeof(double));
// Now copy it to the local particle list for tha appropriate rank
if (myRank == grapite_cep_index/n_loc) {
memcpy(x_loc[grapite_cep_index - myRank*n_loc], xdc, 3*sizeof(double));
memcpy(v_loc[grapite_cep_index - myRank*n_loc], vdc, 3*sizeof(double));
if (grapite_cep_index >= 0) {
grapite_calc_center(N, m, (double(*)[3])x, (double(*)[3])v, xcm, vcm, xdc, vdc);
memcpy(x[grapite_cep_index], xdc, 3*sizeof(double));
memcpy(v[grapite_cep_index], vdc, 3*sizeof(double));
grapite_update_cep(time_cur, xdc, vdc, a[grapite_cep_index], adot[grapite_cep_index]);
}
grapite_update_cep(time_cur, xdc, vdc, a[grapite_cep_index], adot[grapite_cep_index]);
#endif
t_contr += dt_contr;
} /* if (time_cur >= t_contr) */
if (time_cur >= t_disk) {
char out_fname[256];
diskstep++;
if (myRank == rootRank) {
diskstep++;
char out_fname[256];
sprintf(out_fname, "%06d", diskstep);
if (config->output_hdf5) h5_write(std::string(out_fname) + ".h5", diskstep, N, time_cur, m, x, v, pot, a, adot, config->output_extra_mode, config->output_hdf5_double_precision);
else ascii_write(std::string(out_fname) + ".dat", diskstep, N, time_cur, m, x, v, config->output_ascii_precision);
} /* if (myRank == rootRank) */
#ifdef ETICS_DUMP
sprintf(out_fname, "coeffs.%06d.%02d.dat", diskstep, myRank);
grapite_dump(out_fname, 2);
#ifdef ETICS
if (config->etics_dump_coeffs) {
sprintf(out_fname, "coeffs.%06d.%02d.dat", diskstep, myRank);
grapite_dump(out_fname, 2);
}
#endif
t_disk += dt_disk;
} /* if (time_cur >= t_disk) */
} /* while (time_cur < t_end) */
/* close the local GRAPEs */
g6_close(clusterid);
/* Wait to all processors to finish his works... */