Number of devices can be overriden in config file

config.cpp

@@ -158,16 +158,15 @@ Config::Config(std::string file_name)
     dt_contr = get_parameter<double>(dictionary, "dt_contr");
     dt_bh = get_parameter<double>(dictionary, "dt_bh", dt_contr);
     eta = get_parameter<double>(dictionary, "eta");
-
     input_file_name = get_parameter<std::string>(dictionary, "input_file_name", "data.con");
+    devices_per_node = get_parameter<int>(dictionary, "devices_per_node", 0);
+    dt_min_warning = get_parameter<bool>(dictionary, "dt_min_warning", false);
 
     output_hdf5 = get_parameter<bool>(dictionary, "output_hdf5", false);
     output_hdf5_double_precision = get_parameter<bool>(dictionary, "output_hdf5_double_precision", true);
     output_ascii_precision = get_parameter<int>(dictionary, "output_ascii_precision", 10);
     output_extra_mode = get_parameter<int>(dictionary, "output_extra_mode", 10);
 
-    dt_min_warning = get_parameter<bool>(dictionary, "dt_min_warning", false);
-
     live_smbh_count = get_parameter<int>(dictionary, "live_smbh_count", 0);
     live_smbh_custom_eps = get_parameter<double>(dictionary, "live_smbh_custom_eps", -1);
     live_smbh_output = get_parameter<bool>(dictionary, "live_smbh_output", false);

config.h

@@ -14,21 +14,26 @@ public:
     double           dt_bh;
     double           eta;
     std::string      input_file_name;
+    int              devices_per_node;
+    bool             dt_min_warning;
+
     bool             output_hdf5;
     bool             output_hdf5_double_precision;
     int              output_ascii_precision;
     int              output_extra_mode;
-    bool             dt_min_warning;
+
     int              live_smbh_count;
     double           live_smbh_custom_eps;
     bool             live_smbh_output;
     bool             live_smbh_neighbor_output;
     int              live_smbh_neighbor_number;
+
     bool             binary_smbh_pn;
     bool             binary_smbh_influence_sphere_output;
     double           binary_smbh_influence_radius_factor;
     std::vector<int> pn_usage;
     double           pn_c;
+
     bool             ext_units_physical;
     double           unit_mass;
     double           unit_length;

phigrape.conf

@@ -21,7 +21,12 @@ dt_bh =                                 0.125
 eta =                                   0.01
 
 # Name of the input file; use "data.con" in most cases [default: data.con]
-input_file_name =                       data.con
+#input_file_name =                       data.con
+
+
+# Number of devices (GRAPEs or GPUs per node) [default: 0]
+# By default, each MPI process will attempt to bind to one device. To change this behaviour, set the value to the number of available devices in the system. For example, if from some reason you want to run multiple MPI processes on a machine with a single device, set the value to 1 and use the mpirun utility (or whatever is used in your job scheduler) to launch as many processes as you like.
+#devices_per_node =                      1
 
 
 ##########
@@ -123,7 +128,7 @@ input_file_name =                       data.con
 #binary_smbh_influence_sphere_output =   true
 
 # The influence sphere is centred at the binary SMBH's centre of mass, and its radius is the semi-major axis of the binary times the factor below. [default: 10]
-#binary_smbh_influence_radius_factor =   10
+#binary_smbh_influence_radius_factor =   20
 
 # Add post Newtonian terms to SMBH-SMBH gravity. [default: false]
 #binary_smbh_pn =                        true

phigrape.cpp

@@ -942,15 +942,15 @@ int main(int argc, char *argv[])
 
     /* init the local GRAPE's */
 
-#ifdef MPI_OVERRIDE
+    if (config->devices_per_node==0) {
-    numGPU = 1; // TODO get this from config file
+        MPI_Comm shmcomm;
-    clusterid = myRank % numGPU;
+        MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &shmcomm);
-#else
+        MPI_Comm_size(shmcomm, &numGPU);
-    MPI_Comm shmcomm;
+        MPI_Comm_rank(shmcomm, &clusterid);
-    MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &shmcomm);
+    } else {
-    MPI_Comm_size(shmcomm, &numGPU);
+        numGPU = config->devices_per_node;
-    MPI_Comm_rank(shmcomm, &clusterid);
+        clusterid = myRank % numGPU;
-#endif
+    }
     printf("Rank of the processor %03d : Number of GPUs %01d : Cluster ID %01d \n", myRank, numGPU, clusterid);
     fflush(stdout);