[yt-svn] commit/yt: ngoldbaum: Merged in hyschive/yt-hyschive (pull request #2366)
commits-noreply at bitbucket.org
commits-noreply at bitbucket.org
Wed Sep 7 11:15:05 PDT 2016
1 new commit in yt:
https://bitbucket.org/yt_analysis/yt/commits/c144003a5fd2/
Changeset: c144003a5fd2
Branch: yt
User: ngoldbaum
Date: 2016-09-07 18:14:39+00:00
Summary: Merged in hyschive/yt-hyschive (pull request #2366)
Support loading particles in the GAMER frontend
Affected #: 8 files
diff -r eaabc53f4933c4f46b032631563f6b1af087bd95 -r c144003a5fd281a25c7d283899141cca10cb340b doc/source/developing/testing.rst
--- a/doc/source/developing/testing.rst
+++ b/doc/source/developing/testing.rst
@@ -250,6 +250,7 @@
* ``InteractingJets/jet_000002``
* ``WaveDarkMatter/psiDM_000020``
+* ``Plummer/plummer_000000``
Halo Catalog
~~~~~~~~~~~~
diff -r eaabc53f4933c4f46b032631563f6b1af087bd95 -r c144003a5fd281a25c7d283899141cca10cb340b doc/source/examining/loading_data.rst
--- a/doc/source/examining/loading_data.rst
+++ b/doc/source/examining/loading_data.rst
@@ -1069,6 +1069,8 @@
This means that the yt fields, e.g., ``("gas","density")``, will be in cgs units, but the GAMER fields,
e.g., ``("gamer","Dens")``, will be in code units.
+Particle data are supported and are always stored in the same file as the grid data.
+
.. rubric:: Caveats
* GAMER data in raw binary format (i.e., OPT__OUTPUT_TOTAL = C-binary) is not supported.
diff -r eaabc53f4933c4f46b032631563f6b1af087bd95 -r c144003a5fd281a25c7d283899141cca10cb340b doc/source/reference/code_support.rst
--- a/doc/source/reference/code_support.rst
+++ b/doc/source/reference/code_support.rst
@@ -34,7 +34,7 @@
+-----------------------+------------+-----------+------------+-------+----------+----------+------------+----------+
| Gadget | Y | Y | Y | Y | Y [#f2]_ | Y | Y | Full |
+-----------------------+------------+-----------+------------+-------+----------+----------+------------+----------+
-| GAMER | Y | N | Y | Y | Y | Y | Y | Full |
+| GAMER | Y | Y | Y | Y | Y | Y | Y | Full |
+-----------------------+------------+-----------+------------+-------+----------+----------+------------+----------+
| Gasoline | Y | Y | Y | Y | Y [#f2]_ | Y | Y | Full |
+-----------------------+------------+-----------+------------+-------+----------+----------+------------+----------+
diff -r eaabc53f4933c4f46b032631563f6b1af087bd95 -r c144003a5fd281a25c7d283899141cca10cb340b tests/tests.yaml
--- a/tests/tests.yaml
+++ b/tests/tests.yaml
@@ -20,7 +20,7 @@
local_gadget_000:
- yt/frontends/gadget/tests/test_outputs.py
- local_gamer_000:
+ local_gamer_001:
- yt/frontends/gamer/tests/test_outputs.py
local_gdf_000:
diff -r eaabc53f4933c4f46b032631563f6b1af087bd95 -r c144003a5fd281a25c7d283899141cca10cb340b yt/frontends/gamer/data_structures.py
--- a/yt/frontends/gamer/data_structures.py
+++ b/yt/frontends/gamer/data_structures.py
@@ -52,33 +52,39 @@
class GAMERHierarchy(GridIndex):
grid = GAMERGrid
_preload_implemented = True # since gamer defines "_read_chunk_data" in io.py
-
+
def __init__(self, ds, dataset_type = 'gamer'):
self.dataset_type = dataset_type
self.dataset = weakref.proxy(ds)
self.index_filename = self.dataset.parameter_filename
self.directory = os.path.dirname(self.index_filename)
self._handle = ds._handle
+ self._group_grid = ds._group_grid
+ self._group_particle = ds._group_particle
self.float_type = 'float64' # fixed even when FLOAT8 is off
self._particle_handle = ds._particle_handle
GridIndex.__init__(self, ds, dataset_type)
def _detect_output_fields(self):
# find all field names in the current dataset
- self.field_list = [ ('gamer', v) for v in self._handle['Data'].keys() ]
-
+ # grid fields
+ self.field_list = [ ('gamer', v) for v in self._group_grid.keys() ]
+
+ # particle fields
+ if self._group_particle is not None:
+ self.field_list += [ ('io', v) for v in self._group_particle.keys() ]
+
def _count_grids(self):
- # count the total number of patches at all levels
+ # count the total number of patches at all levels
self.num_grids = self.dataset.parameters['NPatch'].sum()
-
+
def _parse_index(self):
parameters = self.dataset.parameters
gid0 = 0
grid_corner = self._handle['Tree/Corner'].value
convert2physical = self._handle['Tree/Corner'].attrs['Cvt2Phy']
- self.grid_dimensions [:] = parameters['PatchSize']
- self.grid_particle_count[:] = 0
+ self.grid_dimensions[:] = parameters['PatchSize']
for lv in range(0, parameters['NLevel']):
num_grids_level = parameters['NPatch'][lv]
@@ -103,7 +109,19 @@
# maximum level with patches (which can be lower than MAX_LEVEL)
self.max_level = self.grid_levels.max()
-
+
+ # number of particles in each grid
+ try:
+ self.grid_particle_count[:] = self._handle['Tree/NPar'].value[:,None]
+ except KeyError:
+ self.grid_particle_count[:] = 0.0
+
+ # calculate the starting particle indices for each grid (starting from 0)
+ # --> note that the last element must store the total number of particles
+ # (see _read_particle_coords and _read_particle_fields in io.py)
+ self._particle_indices = np.zeros(self.num_grids + 1, dtype='int64')
+ np.add.accumulate(self.grid_particle_count.squeeze(), out=self._particle_indices[1:])
+
def _populate_grid_objects(self):
son_list = self._handle["Tree/Son"].value
@@ -144,7 +162,7 @@
'Grid %d, Children %d, Children->Parent %d' % \
(grid.id, c.id, c.Parent.id)
- # all refinement grids should have parent
+ # all refinement grids should have parent
if grid.Level > 0:
assert grid.Parent is not None and grid.Parent.id >= 0, \
'Grid %d, Level %d, Parent %d' % \
@@ -163,18 +181,20 @@
assert_equal(grid.LeftEdge, grid.Children[0].LeftEdge )
assert_equal(grid.RightEdge, grid.Children[7].RightEdge)
mylog.info('Check passed')
-
+
class GAMERDataset(Dataset):
_index_class = GAMERHierarchy
_field_info_class = GAMERFieldInfo
_handle = None
+ _group_grid = None
+ _group_particle = None
_debug = False # debug mode for the GAMER frontend
-
+
def __init__(self, filename,
dataset_type = 'gamer',
storage_filename = None,
- particle_filename = None,
+ particle_filename = None,
units_override = None,
unit_system = "cgs"):
@@ -184,6 +204,15 @@
self._handle = HDF5FileHandler(filename)
self.particle_filename = particle_filename
+ # to catch both the new and old data formats for the grid data
+ try:
+ self._group_grid = self._handle['GridData']
+ except KeyError:
+ self._group_grid = self._handle['Data']
+
+ if 'Particle' in self._handle:
+ self._group_particle = self._handle['Particle']
+
if self.particle_filename is None:
self._particle_handle = self._handle
else:
@@ -199,7 +228,7 @@
units_override = units_override,
unit_system = unit_system)
self.storage_filename = storage_filename
-
+
def _set_code_unit_attributes(self):
# GAMER does not assume any unit yet ...
if len(self.units_override) == 0:
@@ -211,7 +240,7 @@
if len(self.units_override) == 0:
mylog.warning("Assuming 1.0 = 1.0 %s", cgs)
-
+
def _parse_parameter_file(self):
self.unique_identifier = \
int(os.stat(self.parameter_filename)[stat.ST_CTIME])
@@ -237,7 +266,7 @@
if Makefile['Comoving']:
self.cosmological_simulation = 1
self.current_redshift = 1.0/self.current_time - 1.0
- self.omega_matter = InputPara['OmegaM0']
+ self.omega_matter = InputPara['OmegaM0']
self.omega_lambda = 1.0 - self.omega_matter
self.hubble_constant = 0.6955 # H0 is not set in GAMER
else:
diff -r eaabc53f4933c4f46b032631563f6b1af087bd95 -r c144003a5fd281a25c7d283899141cca10cb340b yt/frontends/gamer/fields.py
--- a/yt/frontends/gamer/fields.py
+++ b/yt/frontends/gamer/fields.py
@@ -30,19 +30,30 @@
class GAMERFieldInfo(FieldInfoContainer):
known_other_fields = (
# hydro fields on disk (GAMER outputs conservative variables)
- ( "Dens", (rho_units, ["density"], r"\rho") ),
- ( "MomX", (mom_units, ["momentum_x"], None ) ),
- ( "MomY", (mom_units, ["momentum_y"], None ) ),
- ( "MomZ", (mom_units, ["momentum_z"], None ) ),
- ( "Engy", (erg_units, ["total_energy_per_volume"], None ) ),
- ( "Pote", (pot_units, ["gravitational_potential"], None ) ),
+ ( "Dens", (rho_units, ["density"], r"\rho") ),
+ ( "MomX", (mom_units, ["momentum_x"], None ) ),
+ ( "MomY", (mom_units, ["momentum_y"], None ) ),
+ ( "MomZ", (mom_units, ["momentum_z"], None ) ),
+ ( "Engy", (erg_units, ["total_energy_per_volume"], None ) ),
+ ( "Pote", (pot_units, ["gravitational_potential"], None ) ),
# psiDM fields on disk
- ( "Real", (psi_units, ["psidm_real_part"], None ) ),
- ( "Imag", (psi_units, ["psidm_imaginary_part"], None ) ),
+ ( "Real", (psi_units, ["psidm_real_part"], None ) ),
+ ( "Imag", (psi_units, ["psidm_imaginary_part"], None ) ),
+
+ # particle fields on disk (deposited onto grids)
+ ("ParDens", (rho_units, ["particle_density_on_grid"], None ) ),
+ ("TotalDens", (rho_units, ["total_density_on_grid"], None ) ),
)
known_particle_fields = (
+ ( "ParMass", ("code_mass", ["particle_mass"], None) ),
+ ( "ParPosX", ("code_length", ["particle_position_x"], None) ),
+ ( "ParPosY", ("code_length", ["particle_position_y"], None) ),
+ ( "ParPosZ", ("code_length", ["particle_position_z"], None) ),
+ ( "ParVelX", ("code_velocity", ["particle_velocity_x"], None) ),
+ ( "ParVelY", ("code_velocity", ["particle_velocity_y"], None) ),
+ ( "ParVelZ", ("code_velocity", ["particle_velocity_z"], None) ),
)
def __init__(self, ds, field_list):
@@ -106,5 +117,4 @@
units = unit_system["temperature"] )
def setup_particle_fields(self, ptype):
- # This will get called for every particle type.
- pass
+ super(GAMERFieldInfo, self).setup_particle_fields(ptype)
diff -r eaabc53f4933c4f46b032631563f6b1af087bd95 -r c144003a5fd281a25c7d283899141cca10cb340b yt/frontends/gamer/io.py
--- a/yt/frontends/gamer/io.py
+++ b/yt/frontends/gamer/io.py
@@ -19,6 +19,7 @@
from yt.utilities.io_handler import \
BaseIOHandler
from yt.utilities.logger import ytLogger as mylog
+from yt.geometry.selection_routines import AlwaysSelector
#-----------------------------------------------------------------------------
@@ -27,25 +28,79 @@
# group grids with consecutive indices together to improve the I/O performance
+# --> grids are assumed to be sorted into ascending numerical order already
def grid_sequences(grids):
- for k, g in groupby( enumerate(grids), lambda i_x1:i_x1[0]-i_x1[1].id ):
+ for k, g in groupby( enumerate(grids), lambda i_x:i_x[0]-i_x[1].id ):
seq = list(v[1] for v in g)
yield seq
+def particle_sequences(grids):
+ for k, g in groupby( enumerate(grids), lambda i_x:i_x[0]-i_x[1].id ):
+ seq = list(v[1] for v in g)
+ yield seq[0], seq[-1]
+
class IOHandlerGAMER(BaseIOHandler):
_particle_reader = False
_dataset_type = "gamer"
def __init__(self, ds):
super(IOHandlerGAMER, self).__init__(ds)
- self._handle = ds._handle
- self._field_dtype = "float64" # fixed even when FLOAT8 is off
+ self._handle = ds._handle
+ self._group_grid = ds._group_grid
+ self._group_particle = ds._group_particle
+ self._field_dtype = "float64" # fixed even when FLOAT8 is off
+ self._particle_handle = ds._particle_handle
def _read_particle_coords(self, chunks, ptf):
- pass
+ chunks = list(chunks) # generator --> list
+ p_idx = self.ds.index._particle_indices
+
+ # shortcuts
+ par_posx = self._group_particle["ParPosX"]
+ par_posy = self._group_particle["ParPosY"]
+ par_posz = self._group_particle["ParPosZ"]
+
+ # currently GAMER does not support multiple particle types
+ assert( len(ptf) == 1 )
+ ptype = list( ptf.keys() )[0]
+
+ for chunk in chunks:
+ for g1, g2 in particle_sequences(chunk.objs):
+ start = p_idx[g1.id ]
+ end = p_idx[g2.id + 1]
+ x = np.asarray( par_posx[start:end], dtype=self._field_dtype )
+ y = np.asarray( par_posy[start:end], dtype=self._field_dtype )
+ z = np.asarray( par_posz[start:end], dtype=self._field_dtype )
+ yield ptype, (x, y, z)
def _read_particle_fields(self, chunks, ptf, selector):
- pass
+ chunks = list(chunks) # generator --> list
+ p_idx = self.ds.index._particle_indices
+
+ # shortcuts
+ par_posx = self._group_particle["ParPosX"]
+ par_posy = self._group_particle["ParPosY"]
+ par_posz = self._group_particle["ParPosZ"]
+
+ # currently GAMER does not support multiple particle types
+ assert( len(ptf) == 1 )
+ ptype = list( ptf.keys() )[0]
+ pfields = ptf[ptype]
+
+ for chunk in chunks:
+ for g1, g2 in particle_sequences(chunk.objs):
+ start = p_idx[g1.id ]
+ end = p_idx[g2.id + 1]
+ x = np.asarray( par_posx[start:end], dtype=self._field_dtype )
+ y = np.asarray( par_posy[start:end], dtype=self._field_dtype )
+ z = np.asarray( par_posz[start:end], dtype=self._field_dtype )
+
+ mask = selector.select_points(x, y, z, 0.0)
+ if mask is None: continue
+
+ for field in pfields:
+ data = self._group_particle[field][start:end]
+ yield (ptype, field), data[mask]
def _read_fluid_selection(self, chunks, selector, fields, size):
chunks = list(chunks) # generator --> list
@@ -61,7 +116,7 @@
size, [f2 for f1, f2 in fields], ng )
for field in fields:
- ds = self._handle[ "/Data/%s" % field[1] ]
+ ds = self._group_grid[ field[1] ]
offset = 0
for chunk in chunks:
for gs in grid_sequences(chunk.objs):
@@ -74,12 +129,29 @@
def _read_chunk_data(self, chunk, fields):
rv = {}
- if len(chunk.objs) == 0: return rv
+ if len(chunk.objs) == 0: return rv
for g in chunk.objs: rv[g.id] = {}
- for field in fields:
- ds = self._handle[ "/Data/%s" % field[1] ]
+ # Split into particles and non-particles
+ fluid_fields, particle_fields = [], []
+ for ftype, fname in fields:
+ if ftype in self.ds.particle_types:
+ particle_fields.append( (ftype, fname) )
+ else:
+ fluid_fields.append( (ftype, fname) )
+
+ # particles
+ if len(particle_fields) > 0:
+ selector = AlwaysSelector(self.ds)
+ rv.update( self._read_particle_selection(
+ [chunk], selector, particle_fields) )
+
+ # fluid
+ if len(fluid_fields) == 0: return rv
+
+ for field in fluid_fields:
+ ds = self._group_grid[ field[1] ]
for gs in grid_sequences(chunk.objs):
start = gs[ 0].id
diff -r eaabc53f4933c4f46b032631563f6b1af087bd95 -r c144003a5fd281a25c7d283899141cca10cb340b yt/frontends/gamer/tests/test_outputs.py
--- a/yt/frontends/gamer/tests/test_outputs.py
+++ b/yt/frontends/gamer/tests/test_outputs.py
@@ -52,6 +52,18 @@
yield test
+plummer = "Plummer/plummer_000000"
+_fields_plummer = ( ("gamer","ParDens"), ("deposit","io_cic") )
+
+ at requires_ds(plummer, big_data=True)
+def test_plummer():
+ ds = data_dir_load(plummer)
+ yield assert_equal, str(ds), "plummer_000000"
+ for test in small_patch_amr(ds, _fields_plummer):
+ test_plummer.__name__ = test.description
+ yield test
+
+
@requires_file(psiDM)
def test_GAMERDataset():
assert isinstance(data_dir_load(psiDM), GAMERDataset)
Repository URL: https://bitbucket.org/yt_analysis/yt/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the yt-svn
mailing list