<html>
<head>
<meta http-equiv="content-type" content="text/html; charset=utf-8">
</head>
<body bgcolor="#FFFFFF" text="#000000">
<b>Hi all,</b><b><br>
</b><b><br>
</b><b>I currently try to run enzo with embedded parallelized
python/yt with the following user_script.py :</b><br>
<br>
<i>import yt</i><i><br>
</i><i><br>
</i><i>from yt.frontends.enzo.api import EnzoDatasetInMemory</i><i><br>
</i><i><br>
</i><i>from yt.fields.field_plugin_registry import \</i><i><br>
</i><i> register_field_plugin</i><i><br>
</i><i>from yt.fields.fluid_fields import \</i><i><br>
</i><i> setup_gradient_fields</i><i><br>
</i><i><br>
</i><i>@register_field_plugin</i><i><br>
</i><i>def setup_my_fields(registry, ftype="enzo", slice_info=None):</i><i><br>
</i><i> setup_gradient_fields(registry, ('enzo','AxImaginary'),
'', slice_info)</i><i><br>
</i><i><br>
</i><i>yt.enable_parallelism()</i><i><br>
</i><i><br>
</i><i>def main():</i><i><br>
</i><i><br>
</i><i> ds = EnzoDatasetInMemory()</i><i><br>
</i><i> ds.index</i><i><br>
</i><i> dd = ds.all_data()</i><i><br>
</i><i> print
dd.quantities.total_quantity('AxImaginary_gradient_z')</i><br>
<br>
<br>
<b>It works fine on a single processor (mpirun -n 1 ...) but gives
my the following error message for multiple processors:</b><br>
<br>
<i>Global Dir set to .</i><i><br>
</i><i>ENZO_layout 1 x 1 x 2</i><i><br>
</i><i>Successfully read in parameter file Bosonstar.enzo.</i><i><br>
</i><i>INITIALIZATION TIME = 3.75330448e-02</i><i><br>
</i><i>yt : [INFO ] 2016-08-30 15:02:40,868 Global parallel
computation enabled: 0 / 2</i><i><br>
</i><i>yt : [INFO ] 2016-08-30 15:02:40,868 Global parallel
computation enabled: 1 / 2</i><i><br>
</i><i>Continuation Flag = 1</i><i><br>
</i><i>TopGrid dt = 2.000000e-05 time = 0 cycle = 0</i><i><br>
</i><i>P000 yt : [INFO ] 2016-08-30 15:02:41,001 Parameters:
current_time = 2e-05</i><i><br>
</i><i>P000 yt : [INFO ] 2016-08-30 15:02:41,001 Parameters:
domain_dimensions = [32 32 32]</i><i><br>
</i><i>P000 yt : [INFO ] 2016-08-30 15:02:41,002 Parameters:
domain_left_edge = [-0.256 -0.256 -0.256]</i><i><br>
</i><i>P000 yt : [INFO ] 2016-08-30 15:02:41,002 Parameters:
domain_right_edge = [ 0.256 0.256 0.256]</i><i><br>
</i><i>P000 yt : [INFO ] 2016-08-30 15:02:41,003 Parameters:
cosmological_simulation = 0.0</i><i><br>
</i><i>P000 yt : [INFO ] 2016-08-30 15:02:41,005 Gathering a
field list (this may take a moment.)</i><i><br>
</i><i> File "<string>", line 1, in <module></i><i><br>
</i><i> File "./user_script.py", line 21, in main</i><i><br>
</i><i> print
dd.quantities.total_quantity('AxImaginary_gradient_z')</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/derived_quantities.py",
line 176, in __call__</i><i><br>
</i><i> rv = super(TotalQuantity, self).__call__(fields)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/derived_quantities.py",
line 67, in __call__</i><i><br>
</i><i> sto.result = self.process_chunk(ds, *args, **kwargs)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/derived_quantities.py",
line 182, in process_chunk</i><i><br>
</i><i> for field in fields]</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/data_containers.py",
line 279, in __getitem__</i><i><br>
</i><i> File "<string>", line 1, in <module></i><i><br>
</i><i> File "./user_script.py", line 21, in main</i><i><br>
</i><i> print
dd.quantities.total_quantity('AxImaginary_gradient_z')</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/derived_quantities.py",
line 176, in __call__</i><i><br>
</i><i> self.get_data(f)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/data_containers.py",
line 1291, in get_data</i><i><br>
</i><i> self._generate_fields(fields_to_generate)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/data_containers.py",
line 1311, in _generate_fields</i><i><br>
</i><i> fd = self._generate_field(field)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/data_containers.py",
line 316, in _generate_field</i><i><br>
</i><i> tr = self._generate_fluid_field(field)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/data_containers.py",
line 334, in _generate_fluid_field</i><i><br>
</i><i> rv = self._generate_spatial_fluid(field,
ngt_exception.ghost_zones)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/data_containers.py",
line 364, in _generate_spatial_fluid</i><i><br>
</i><i> gz[field][ngz:-ngz, ngz:-ngz, ngz:-ngz],</i><i><br>
</i><i> rv = super(TotalQuantity, self).__call__(fields)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/derived_quantities.py",
line 67, in __call__</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/data_containers.py",
line 279, in __getitem__</i><i><br>
</i><i> sto.result = self.process_chunk(ds, *args, **kwargs)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/derived_quantities.py",
line 182, in process_chunk</i><i><br>
</i><i> for field in fields]</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/data_containers.py",
line 279, in __getitem__</i><i><br>
</i><i> self.get_data(f)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/construction_data_containers.py",
line 628, in get_data</i><i><br>
</i><i> self.get_data(f)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/data_containers.py",
line 1291, in get_data</i><i><br>
</i><i> self._generate_fields(fields_to_generate)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/data_containers.py",
line 1311, in _generate_fields</i><i><br>
</i><i> fd = self._generate_field(field)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/data_containers.py",
line 316, in _generate_field</i><i><br>
</i><i> tr = self._generate_fluid_field(field)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/data_containers.py",
line 334, in _generate_fluid_field</i><i><br>
</i><i> rv = self._generate_spatial_fluid(field,
ngt_exception.ghost_zones)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/data_containers.py",
line 364, in _generate_spatial_fluid</i><i><br>
</i><i> if len(fill) > 0: self._fill_fields(fill)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/construction_data_containers.py",
line 947, in _fill_fields</i><i><br>
</i><i> gz[field][ngz:-ngz, ngz:-ngz, ngz:-ngz],</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/data_containers.py",
line 279, in __getitem__</i><i><br>
</i><i> for chunk in ls.data_source.chunks(fields, "io"):</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/data_containers.py",
line 1190, in chunks</i><i><br>
</i><i> self.get_data(f)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/construction_data_containers.py",
line 628, in get_data</i><i><br>
</i><i> self.get_data(fields)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/data_containers.py",
line 1279, in get_data</i><i><br>
</i><i> fluids, self, self._current_chunk)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/geometry/geometry_handler.py",
line 245, in _read_fluid_fields</i><i><br>
</i><i> chunk_size)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/frontends/enzo/io.py",
line 380, in _read_fluid_selection</i><i><br>
</i><i> if len(fill) > 0: self._fill_fields(fill)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/construction_data_containers.py",
line 947, in _fill_fields</i><i><br>
</i><i> for chunk in ls.data_source.chunks(fields, "io"):</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/data_containers.py",
line 1190, in chunks</i><i><br>
</i><i> self.get_data(fields)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/data_objects/data_containers.py",
line 1279, in get_data</i><i><br>
</i><i> fluids, self, self._current_chunk)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/geometry/geometry_handler.py",
line 245, in _read_fluid_fields</i><i><br>
</i><i> data_view =
self.grids_in_memory[g.id][fname][self.my_slice].swapaxes(0,2)</i><i><br>
</i><i>P001 yt : [ERROR ] 2016-08-30 15:02:41,783 KeyError: 1</i><i><br>
</i><i> chunk_size)</i><i><br>
</i><i> File
"/gfs2/work/nipbschw/yt-shared-smp/src/yt-hg/yt/frontends/enzo/io.py",
line 380, in _read_fluid_selection</i><i><br>
</i><i> data_view =
self.grids_in_memory[g.id][fname][self.my_slice].swapaxes(0,2)</i><i><br>
</i><i>P000 yt : [ERROR ] 2016-08-30 15:02:41,784 KeyError: 2</i><i><br>
</i><i>[hsmp16:59147] 1 more process has sent help message
help-mpi-btl-openib.txt / default subnet prefix</i><i><br>
</i><i>[hsmp16:59147] Set MCA parameter "orte_base_help_aggregate"
to 0 to see all help / error messages</i><i><br>
</i><i>[hsmp16:59147] 1 more process has sent help message
help-mpi-runtime.txt / mpi_init:warn-fork</i><i><br>
</i><i>[hsmp16:59147] 1 more process has sent help message
help-mpi-api.txt / mpi-abort<br>
<br>
</i><b>Thank you very much for any helpful suggestions,</b><b><br>
</b><b>Bodo</b><i><br>
</i>
</body>
</html>