[yt-svn] commit/yt-3.0: 2 new changesets
commits-noreply at bitbucket.org
commits-noreply at bitbucket.org
Fri Oct 18 20:08:50 PDT 2013
2 new commits in yt-3.0:
https://bitbucket.org/yt_analysis/yt-3.0/commits/b94c1e8965b4/
Changeset: b94c1e8965b4
Branch: yt-3.0
User: xarthisius
Date: 2013-10-12 12:15:13
Summary: [gdf] update wrt recent changes, projections and slices are working now
Affected #: 3 files
diff -r 5df0d5742ff2e8abcb6dd032f82247af9c6f0409 -r b94c1e8965b4174aadbd35d2d7bfd93d6d77ea52 yt/frontends/gdf/data_structures.py
--- a/yt/frontends/gdf/data_structures.py
+++ b/yt/frontends/gdf/data_structures.py
@@ -16,35 +16,37 @@
import h5py
import numpy as np
import weakref
-from yt.funcs import *
+import os
+from yt.funcs import \
+ just_one, ensure_tuple
from yt.data_objects.grid_patch import \
- AMRGridPatch
+ AMRGridPatch
from yt.geometry.grid_geometry_handler import \
- GridGeometryHandler
+ GridGeometryHandler
from yt.data_objects.static_output import \
- StaticOutput
+ StaticOutput
from yt.utilities.lib import \
get_box_grids_level
-from yt.utilities.io_handler import \
- io_registry
from yt.utilities.definitions import \
mpc_conversion, sec_conversion
from .fields import GDFFieldInfo, KnownGDFFields
from yt.data_objects.field_info_container import \
- FieldInfoContainer, NullFunc
-import pdb
+ NullFunc
+
def _get_convert(fname):
def _conv(data):
- return data.convert(fname)
+ return 1.0 # data.convert(fname) FIXME
return _conv
+
class GDFGrid(AMRGridPatch):
_id_offset = 0
+
def __init__(self, id, hierarchy, level, start, dimensions):
- AMRGridPatch.__init__(self, id, filename = hierarchy.hierarchy_filename,
- hierarchy = hierarchy)
+ AMRGridPatch.__init__(self, id, filename=hierarchy.hierarchy_filename,
+ hierarchy=hierarchy)
self.Parent = []
self.Children = []
self.Level = level
@@ -59,39 +61,41 @@
if len(self.Parent) > 0:
self.dds = self.Parent[0].dds / self.pf.refine_by
else:
- LE, RE = self.hierarchy.grid_left_edge[id,:], \
- self.hierarchy.grid_right_edge[id,:]
- self.dds = np.array((RE-LE)/self.ActiveDimensions)
+ LE, RE = self.hierarchy.grid_left_edge[id, :], \
+ self.hierarchy.grid_right_edge[id, :]
+ self.dds = np.array((RE - LE) / self.ActiveDimensions)
if self.pf.data_software != "piernik":
- if self.pf.dimensionality < 2: self.dds[1] = 1.0
- if self.pf.dimensionality < 3: self.dds[2] = 1.0
- self.field_data['dx'], self.field_data['dy'], self.field_data['dz'] = self.dds
+ if self.pf.dimensionality < 2:
+ self.dds[1] = 1.0
+ if self.pf.dimensionality < 3:
+ self.dds[2] = 1.0
+ self.field_data['dx'], self.field_data['dy'], self.field_data['dz'] = \
+ self.dds
- @property
- def filename(self):
- return None
class GDFHierarchy(GridGeometryHandler):
grid = GDFGrid
+ filtered_particle_types = []
def __init__(self, pf, data_style='grid_data_format'):
self.parameter_file = weakref.proxy(pf)
+ self.hierarchy_filename = self.parameter_file.parameter_filename
+ h5f = h5py.File(self.hierarchy_filename, 'r')
self.data_style = data_style
+ GridGeometryHandler.__init__(self, pf, data_style)
self.max_level = 10 # FIXME
# for now, the hierarchy file is the parameter file!
- self.hierarchy_filename = self.parameter_file.parameter_filename
self.directory = os.path.dirname(self.hierarchy_filename)
- self._fhandle = h5py.File(self.hierarchy_filename,'r')
- GridGeometryHandler.__init__(self,pf,data_style)
-
- self._fhandle.close()
+ h5f.close()
def _initialize_data_storage(self):
pass
def _detect_fields(self):
- self.field_list = self._fhandle['field_types'].keys()
+ h5f = h5py.File(self.hierarchy_filename, 'r')
+ self.field_list = h5f['field_types'].keys()
+ h5f.close()
def _setup_classes(self):
dd = self._get_data_reader_dict()
@@ -99,15 +103,17 @@
self.object_types.sort()
def _count_grids(self):
- self.num_grids = self._fhandle['/grid_parent_id'].shape[0]
+ h5f = h5py.File(self.hierarchy_filename, 'r')
+ self.num_grids = h5f['/grid_parent_id'].shape[0]
+ h5f.close()
def _parse_hierarchy(self):
- f = self._fhandle
+ h5f = h5py.File(self.hierarchy_filename, 'r')
dxs = []
self.grids = np.empty(self.num_grids, dtype='object')
- levels = (f['grid_level'][:]).copy()
- glis = (f['grid_left_index'][:]).copy()
- gdims = (f['grid_dimensions'][:]).copy()
+ levels = (h5f['grid_level'][:]).copy()
+ glis = (h5f['grid_left_index'][:]).copy()
+ gdims = (h5f['grid_dimensions'][:]).copy()
active_dims = ~((np.max(gdims, axis=0) == 1) &
(self.parameter_file.domain_dimensions == 1))
@@ -117,16 +123,18 @@
gdims[i])
self.grids[i]._level_id = levels[i]
- dx = (self.parameter_file.domain_right_edge-
- self.parameter_file.domain_left_edge)/self.parameter_file.domain_dimensions
- dx[active_dims] = dx[active_dims]/self.parameter_file.refine_by**(levels[i])
+ dx = (self.parameter_file.domain_right_edge -
+ self.parameter_file.domain_left_edge) / \
+ self.parameter_file.domain_dimensions
+ dx[active_dims] /= self.parameter_file.refine_by ** levels[i]
dxs.append(dx)
dx = np.array(dxs)
- self.grid_left_edge = self.parameter_file.domain_left_edge + dx*glis
+ self.grid_left_edge = self.parameter_file.domain_left_edge + dx * glis
self.grid_dimensions = gdims.astype("int32")
- self.grid_right_edge = self.grid_left_edge + dx*self.grid_dimensions
- self.grid_particle_count = f['grid_particle_count'][:]
+ self.grid_right_edge = self.grid_left_edge + dx * self.grid_dimensions
+ self.grid_particle_count = h5f['grid_particle_count'][:]
del levels, glis, gdims
+ h5f.close()
def _populate_grid_objects(self):
mask = np.empty(self.grids.size, dtype='int32')
@@ -138,8 +146,8 @@
g.Children = self._get_grid_children(g)
for g1 in g.Children:
g1.Parent.append(g)
- get_box_grids_level(self.grid_left_edge[gi,:],
- self.grid_right_edge[gi,:],
+ get_box_grids_level(self.grid_left_edge[gi, :],
+ self.grid_right_edge[gi, :],
self.grid_levels[gi],
self.grid_left_edge, self.grid_right_edge,
self.grid_levels, mask)
@@ -158,32 +166,33 @@
Gets back all the grids between a left edge and right edge
"""
eps = np.finfo(np.float64).eps
- grid_i = np.where((np.all((self.grid_right_edge - left_edge) > eps, axis=1) \
- & np.all((right_edge - self.grid_left_edge) > eps, axis=1)) == True)
+ grid_i = np.where(np.all((self.grid_right_edge - left_edge) > eps, axis=1) &
+ np.all((right_edge - self.grid_left_edge) > eps, axis=1))
return self.grids[grid_i], grid_i
-
def _get_grid_children(self, grid):
mask = np.zeros(self.num_grids, dtype='bool')
grids, grid_ind = self._get_box_grids(grid.LeftEdge, grid.RightEdge)
mask[grid_ind] = True
return [g for g in self.grids[mask] if g.Level == grid.Level + 1]
+
class GDFStaticOutput(StaticOutput):
_hierarchy_class = GDFHierarchy
_fieldinfo_fallback = GDFFieldInfo
_fieldinfo_known = KnownGDFFields
def __init__(self, filename, data_style='grid_data_format',
- storage_filename = None):
+ storage_filename=None):
StaticOutput.__init__(self, filename, data_style)
self.storage_filename = storage_filename
self.filename = filename
def _set_units(self):
"""
- Generates the conversion to various physical _units based on the parameter file
+ Generates the conversion to various physical _units
+ based on the parameter file
"""
self.units = {}
self.time_units = {}
@@ -192,16 +201,17 @@
self.time_units['1'] = 1
self.units['1'] = 1.0
self.units['cm'] = 1.0
- self.units['unitary'] = 1.0 / (self.domain_right_edge - self.domain_left_edge).max()
+ self.units['unitary'] = 1.0 / (self.domain_right_edge -
+ self.domain_left_edge).max()
for unit in mpc_conversion.keys():
- self.units[unit] = 1.0 * mpc_conversion[unit] / mpc_conversion["cm"]
+ self.units[unit] = mpc_conversion[unit] / mpc_conversion["cm"]
for unit in sec_conversion.keys():
self.time_units[unit] = 1.0 / sec_conversion[unit]
# This should be improved.
- self._handle = h5py.File(self.parameter_filename, "r")
- for field_name in self._handle["/field_types"]:
- current_field = self._handle["/field_types/%s" % field_name]
+ h5f = h5py.File(self.parameter_filename, "r")
+ for field_name in h5f["/field_types"]:
+ current_field = h5f["/field_types/%s" % field_name]
if 'field_to_cgs' in current_field.attrs:
self.units[field_name] = current_field.attrs['field_to_cgs']
else:
@@ -210,15 +220,16 @@
if type(current_field.attrs['field_units']) == str:
current_fields_unit = current_field.attrs['field_units']
else:
- current_fields_unit = just_one(current_field.attrs['field_units'])
+ current_fields_unit = \
+ just_one(current_field.attrs['field_units'])
else:
current_fields_unit = ""
- self._fieldinfo_known.add_field(field_name, function=NullFunc, take_log=False,
- units=current_fields_unit, projected_units="",
- convert_function=_get_convert(field_name))
+ self._fieldinfo_known.add_field(
+ field_name, function=NullFunc, take_log=False,
+ units=current_fields_unit, projected_units="",
+ convert_function=_get_convert(field_name))
- self._handle.close()
- del self._handle
+ h5f.close()
def _parse_parameter_file(self):
self._handle = h5py.File(self.parameter_filename, "r")
@@ -232,13 +243,15 @@
self.domain_right_edge = sp["domain_right_edge"][:]
self.domain_dimensions = sp["domain_dimensions"][:]
refine_by = sp["refine_by"]
- if refine_by is None: refine_by = 2
+ if refine_by is None:
+ refine_by = 2
self.refine_by = refine_by
self.dimensionality = sp["dimensionality"]
self.current_time = sp["current_time"]
self.unique_identifier = sp["unique_identifier"]
self.cosmological_simulation = sp["cosmological_simulation"]
- if sp["num_ghost_zones"] != 0: raise RuntimeError
+ if sp["num_ghost_zones"] != 0:
+ raise RuntimeError
self.num_ghost_zones = sp["num_ghost_zones"]
self.field_ordering = sp["field_ordering"]
self.boundary_conditions = sp["boundary_conditions"][:]
@@ -252,15 +265,16 @@
else:
self.current_redshift = self.omega_lambda = self.omega_matter = \
self.hubble_constant = self.cosmological_simulation = 0.0
- self.parameters['Time'] = 1.0 # Hardcode time conversion for now.
- self.parameters["HydroMethod"] = 0 # Hardcode for now until field staggering is supported.
+ self.parameters['Time'] = 1.0 # Hardcode time conversion for now.
+ # Hardcode for now until field staggering is supported.
+ self.parameters["HydroMethod"] = 0
self._handle.close()
del self._handle
@classmethod
def _is_valid(self, *args, **kwargs):
try:
- fileh = h5py.File(args[0],'r')
+ fileh = h5py.File(args[0], 'r')
if "gridded_data_format" in fileh:
fileh.close()
return True
@@ -271,4 +285,3 @@
def __repr__(self):
return self.basename.rsplit(".", 1)[0]
-
diff -r 5df0d5742ff2e8abcb6dd032f82247af9c6f0409 -r b94c1e8965b4174aadbd35d2d7bfd93d6d77ea52 yt/frontends/gdf/fields.py
--- a/yt/frontends/gdf/fields.py
+++ b/yt/frontends/gdf/fields.py
@@ -16,14 +16,8 @@
from yt.data_objects.field_info_container import \
FieldInfoContainer, \
FieldInfo, \
- ValidateParameter, \
- ValidateDataField, \
- ValidateProperty, \
- ValidateSpatial, \
- ValidateGridType, \
NullFunc, \
TranslationFunc
-import yt.fields.universal_fields
log_translation_dict = {"Density": "density",
"Pressure": "pressure"}
@@ -31,7 +25,7 @@
translation_dict = {"x-velocity": "velocity_x",
"y-velocity": "velocity_y",
"z-velocity": "velocity_z"}
-
+
# translation_dict = {"mag_field_x": "cell_centered_B_x ",
# "mag_field_y": "cell_centered_B_y ",
# "mag_field_z": "cell_centered_B_z "}
@@ -43,39 +37,39 @@
add_gdf_field = KnownGDFFields.add_field
add_gdf_field("density", function=NullFunc, take_log=True,
- units=r"\rm{g}/\rm{cm}^3",
- projected_units =r"\rm{g}/\rm{cm}^2")
+ units=r"\rm{g}/\rm{cm}^3",
+ projected_units=r"\rm{g}/\rm{cm}^2")
add_gdf_field("specific_energy", function=NullFunc, take_log=True,
- units=r"\rm{erg}/\rm{g}")
+ units=r"\rm{erg}/\rm{g}")
add_gdf_field("pressure", function=NullFunc, take_log=True,
- units=r"\rm{erg}/\rm{g}")
+ units=r"\rm{erg}/\rm{g}")
add_gdf_field("velocity_x", function=NullFunc, take_log=False,
- units=r"\rm{cm}/\rm{s}")
+ units=r"\rm{cm}/\rm{s}")
add_gdf_field("velocity_y", function=NullFunc, take_log=False,
- units=r"\rm{cm}/\rm{s}")
+ units=r"\rm{cm}/\rm{s}")
add_gdf_field("velocity_z", function=NullFunc, take_log=False,
- units=r"\rm{cm}/\rm{s}")
+ units=r"\rm{cm}/\rm{s}")
add_gdf_field("mag_field_x", function=NullFunc, take_log=False,
- units=r"\rm{cm}/\rm{s}")
+ units=r"\rm{cm}/\rm{s}")
add_gdf_field("mag_field_y", function=NullFunc, take_log=False,
- units=r"\rm{cm}/\rm{s}")
+ units=r"\rm{cm}/\rm{s}")
add_gdf_field("mag_field_z", function=NullFunc, take_log=False,
- units=r"\rm{cm}/\rm{s}")
+ units=r"\rm{cm}/\rm{s}")
-for f,v in log_translation_dict.items():
+for f, v in log_translation_dict.items():
add_field(f, TranslationFunc(v), take_log=True,
units=KnownGDFFields[v].get_units(),
projected_units=KnownGDFFields[v].get_projected_units())
-for f,v in translation_dict.items():
+for f, v in translation_dict.items():
add_field(f, TranslationFunc(v), take_log=False,
units=KnownGDFFields[v].get_units(),
projected_units=KnownGDFFields[v].get_projected_units())
diff -r 5df0d5742ff2e8abcb6dd032f82247af9c6f0409 -r b94c1e8965b4174aadbd35d2d7bfd93d6d77ea52 yt/frontends/gdf/io.py
--- a/yt/frontends/gdf/io.py
+++ b/yt/frontends/gdf/io.py
@@ -14,14 +14,20 @@
#-----------------------------------------------------------------------------
import numpy as np
+import h5py
+import exceptions
from yt.funcs import \
mylog
from yt.utilities.io_handler import \
BaseIOHandler
-def field_dname(grid_id, field_name):
- return "/data/grid_%010i/%s" % (grid_id, field_name)
+def _grid_dname(grid_id):
+ return "/data/grid_%010i" % grid_id
+
+
+def _field_dname(grid_id, field_name):
+ return "%s/%s" % (_grid_dname(grid_id), field_name)
# TODO all particle bits were removed
@@ -30,50 +36,84 @@
_offset_string = 'data:offsets=0'
_data_string = 'data:datatype=0'
- def __init__(self, pf, *args, **kwargs):
- # TODO check if _num_per_stride is needed
- self._num_per_stride = kwargs.pop("num_per_stride", 1000000)
- BaseIOHandler.__init__(self, *args, **kwargs)
- self.pf = pf
- self._handle = pf._handle
+ def _read_field_names(self, grid):
+ if grid.filename is None:
+ return []
+ print 'grid.filename = %', grid.filename
+ h5f = h5py.File(grid.filename, mode="r")
+ group = h5f[_grid_dname(grid.id)]
+ fields = []
+ for name, v in group.iteritems():
+ # NOTE: This won't work with 1D datasets.
+ if not hasattr(v, "shape"):
+ continue
+ elif len(v.dims) == 1:
+ fields.append(("io", str(name)))
+ else:
+ fields.append(("gas", str(name)))
+ h5f.close()
+ return fields
-
- def _read_data_set(self, grid, field):
- if self.pf.field_ordering == 1:
- data = self._handle[field_dname(grid.id, field)][:].swapaxes(0, 2)
- else:
- data = self._handle[field_dname(grid.id, field)][:, :, :]
- return data.astype("float64")
-
- def _read_data_slice(self, grid, field, axis, coord):
- slc = [slice(None), slice(None), slice(None)]
- slc[axis] = slice(coord, coord + 1)
- if self.pf.field_ordering == 1:
- data = self._handle[field_dname(grid.id, field)][:].swapaxes(0, 2)[slc]
- else:
- data = self._handle[field_dname(grid.id, field)][slc]
- return data.astype("float64")
+ @property
+ def _read_exception(self):
+ return (exceptions.KeyError, )
def _read_fluid_selection(self, chunks, selector, fields, size):
+ rv = {}
chunks = list(chunks)
+
+ if selector.__class__.__name__ == "GridSelector":
+ if not (len(chunks) == len(chunks[0].objs) == 1):
+ raise RuntimeError
+ grid = chunks[0].objs[0]
+ h5f = h5py.File(grid.filename, 'r')
+ gds = h5f.get(_grid_dname(grid.id))
+ for ftype, fname in fields:
+ if self.pf.field_ordering == 1:
+ rv[(ftype, fname)] = gds.get(fname).value.swapaxes(0, 2)
+ else:
+ rv[(ftype, fname)] = gds.get(fname).value
+ h5f.close()
+ return rv
+ if size is None:
+ size = sum((grid.count(selector) for chunk in chunks
+ for grid in chunk.objs))
+
if any((ftype != "gas" for ftype, fname in fields)):
raise NotImplementedError
- fhandle = self._handle
- rv = {}
+
for field in fields:
ftype, fname = field
- rv[field] = np.empty(
- size, dtype=fhandle[field_dname(0, fname)].dtype)
+ fsize = size
+ # check the dtype instead
+ rv[field] = np.empty(fsize, dtype="float64")
ngrids = sum(len(chunk.objs) for chunk in chunks)
mylog.debug("Reading %s cells of %s fields in %s blocks",
size, [fname for ftype, fname in fields], ngrids)
- for field in fields:
- ftype, fname = field
- ind = 0
- for chunk in chunks:
- for grid in chunk.objs:
- data = fhandle[field_dname(grid.id, fname)][:]
- if self.pf.field_ordering == 1:
- data = data.swapaxes(0, 2)
- ind += g.select(selector, data, rv[field], ind) # caches
+ ind = 0
+ for chunk in chunks:
+ fid = None
+ for grid in chunk.objs:
+ if grid.filename is None:
+ continue
+ if fid is None:
+ fid = h5py.h5f.open(grid.filename, h5py.h5f.ACC_RDONLY)
+ if self.pf.field_ordering == 1:
+ # check the dtype instead
+ data = np.empty(grid.ActiveDimensions[::-1],
+ dtype="float64")
+ data_view = data.swapaxes(0, 2)
+ else:
+ # check the dtype instead
+ data_view = data = np.empty(grid.ActiveDimensions,
+ dtype="float64")
+ for field in fields:
+ ftype, fname = field
+ dg = h5py.h5d.open(fid, _field_dname(grid.id, fname))
+ dg.read(h5py.h5s.ALL, h5py.h5s.ALL, data)
+ # caches
+ nd = grid.select(selector, data_view, rv[field], ind)
+ ind += nd # I don't get that part, only last nd is added
+ if fid is not None:
+ fid.close()
return rv
https://bitbucket.org/yt_analysis/yt-3.0/commits/529acf3ab809/
Changeset: 529acf3ab809
Branch: yt-3.0
User: MatthewTurk
Date: 2013-10-19 05:08:46
Summary: Merged in xarthisius/yt-3.0 (pull request #116)
Update for GDF frontend
Affected #: 3 files
diff -r 09fcbdcc5ae8b38bc37a5a95901d936303497086 -r 529acf3ab809bbc3031a75197d3c8fffb443f6b2 yt/frontends/gdf/data_structures.py
--- a/yt/frontends/gdf/data_structures.py
+++ b/yt/frontends/gdf/data_structures.py
@@ -16,35 +16,37 @@
import h5py
import numpy as np
import weakref
-from yt.funcs import *
+import os
+from yt.funcs import \
+ just_one, ensure_tuple
from yt.data_objects.grid_patch import \
- AMRGridPatch
+ AMRGridPatch
from yt.geometry.grid_geometry_handler import \
- GridGeometryHandler
+ GridGeometryHandler
from yt.data_objects.static_output import \
- StaticOutput
+ StaticOutput
from yt.utilities.lib import \
get_box_grids_level
-from yt.utilities.io_handler import \
- io_registry
from yt.utilities.definitions import \
mpc_conversion, sec_conversion
from .fields import GDFFieldInfo, KnownGDFFields
from yt.data_objects.field_info_container import \
- FieldInfoContainer, NullFunc
-import pdb
+ NullFunc
+
def _get_convert(fname):
def _conv(data):
- return data.convert(fname)
+ return 1.0 # data.convert(fname) FIXME
return _conv
+
class GDFGrid(AMRGridPatch):
_id_offset = 0
+
def __init__(self, id, hierarchy, level, start, dimensions):
- AMRGridPatch.__init__(self, id, filename = hierarchy.hierarchy_filename,
- hierarchy = hierarchy)
+ AMRGridPatch.__init__(self, id, filename=hierarchy.hierarchy_filename,
+ hierarchy=hierarchy)
self.Parent = []
self.Children = []
self.Level = level
@@ -59,39 +61,41 @@
if len(self.Parent) > 0:
self.dds = self.Parent[0].dds / self.pf.refine_by
else:
- LE, RE = self.hierarchy.grid_left_edge[id,:], \
- self.hierarchy.grid_right_edge[id,:]
- self.dds = np.array((RE-LE)/self.ActiveDimensions)
+ LE, RE = self.hierarchy.grid_left_edge[id, :], \
+ self.hierarchy.grid_right_edge[id, :]
+ self.dds = np.array((RE - LE) / self.ActiveDimensions)
if self.pf.data_software != "piernik":
- if self.pf.dimensionality < 2: self.dds[1] = 1.0
- if self.pf.dimensionality < 3: self.dds[2] = 1.0
- self.field_data['dx'], self.field_data['dy'], self.field_data['dz'] = self.dds
+ if self.pf.dimensionality < 2:
+ self.dds[1] = 1.0
+ if self.pf.dimensionality < 3:
+ self.dds[2] = 1.0
+ self.field_data['dx'], self.field_data['dy'], self.field_data['dz'] = \
+ self.dds
- @property
- def filename(self):
- return None
class GDFHierarchy(GridGeometryHandler):
grid = GDFGrid
+ filtered_particle_types = []
def __init__(self, pf, data_style='grid_data_format'):
self.parameter_file = weakref.proxy(pf)
+ self.hierarchy_filename = self.parameter_file.parameter_filename
+ h5f = h5py.File(self.hierarchy_filename, 'r')
self.data_style = data_style
+ GridGeometryHandler.__init__(self, pf, data_style)
self.max_level = 10 # FIXME
# for now, the hierarchy file is the parameter file!
- self.hierarchy_filename = self.parameter_file.parameter_filename
self.directory = os.path.dirname(self.hierarchy_filename)
- self._fhandle = h5py.File(self.hierarchy_filename,'r')
- GridGeometryHandler.__init__(self,pf,data_style)
-
- self._fhandle.close()
+ h5f.close()
def _initialize_data_storage(self):
pass
def _detect_fields(self):
- self.field_list = self._fhandle['field_types'].keys()
+ h5f = h5py.File(self.hierarchy_filename, 'r')
+ self.field_list = h5f['field_types'].keys()
+ h5f.close()
def _setup_classes(self):
dd = self._get_data_reader_dict()
@@ -99,15 +103,17 @@
self.object_types.sort()
def _count_grids(self):
- self.num_grids = self._fhandle['/grid_parent_id'].shape[0]
+ h5f = h5py.File(self.hierarchy_filename, 'r')
+ self.num_grids = h5f['/grid_parent_id'].shape[0]
+ h5f.close()
def _parse_hierarchy(self):
- f = self._fhandle
+ h5f = h5py.File(self.hierarchy_filename, 'r')
dxs = []
self.grids = np.empty(self.num_grids, dtype='object')
- levels = (f['grid_level'][:]).copy()
- glis = (f['grid_left_index'][:]).copy()
- gdims = (f['grid_dimensions'][:]).copy()
+ levels = (h5f['grid_level'][:]).copy()
+ glis = (h5f['grid_left_index'][:]).copy()
+ gdims = (h5f['grid_dimensions'][:]).copy()
active_dims = ~((np.max(gdims, axis=0) == 1) &
(self.parameter_file.domain_dimensions == 1))
@@ -117,16 +123,18 @@
gdims[i])
self.grids[i]._level_id = levels[i]
- dx = (self.parameter_file.domain_right_edge-
- self.parameter_file.domain_left_edge)/self.parameter_file.domain_dimensions
- dx[active_dims] = dx[active_dims]/self.parameter_file.refine_by**(levels[i])
+ dx = (self.parameter_file.domain_right_edge -
+ self.parameter_file.domain_left_edge) / \
+ self.parameter_file.domain_dimensions
+ dx[active_dims] /= self.parameter_file.refine_by ** levels[i]
dxs.append(dx)
dx = np.array(dxs)
- self.grid_left_edge = self.parameter_file.domain_left_edge + dx*glis
+ self.grid_left_edge = self.parameter_file.domain_left_edge + dx * glis
self.grid_dimensions = gdims.astype("int32")
- self.grid_right_edge = self.grid_left_edge + dx*self.grid_dimensions
- self.grid_particle_count = f['grid_particle_count'][:]
+ self.grid_right_edge = self.grid_left_edge + dx * self.grid_dimensions
+ self.grid_particle_count = h5f['grid_particle_count'][:]
del levels, glis, gdims
+ h5f.close()
def _populate_grid_objects(self):
mask = np.empty(self.grids.size, dtype='int32')
@@ -138,8 +146,8 @@
g.Children = self._get_grid_children(g)
for g1 in g.Children:
g1.Parent.append(g)
- get_box_grids_level(self.grid_left_edge[gi,:],
- self.grid_right_edge[gi,:],
+ get_box_grids_level(self.grid_left_edge[gi, :],
+ self.grid_right_edge[gi, :],
self.grid_levels[gi],
self.grid_left_edge, self.grid_right_edge,
self.grid_levels, mask)
@@ -158,32 +166,33 @@
Gets back all the grids between a left edge and right edge
"""
eps = np.finfo(np.float64).eps
- grid_i = np.where((np.all((self.grid_right_edge - left_edge) > eps, axis=1) \
- & np.all((right_edge - self.grid_left_edge) > eps, axis=1)) == True)
+ grid_i = np.where(np.all((self.grid_right_edge - left_edge) > eps, axis=1) &
+ np.all((right_edge - self.grid_left_edge) > eps, axis=1))
return self.grids[grid_i], grid_i
-
def _get_grid_children(self, grid):
mask = np.zeros(self.num_grids, dtype='bool')
grids, grid_ind = self._get_box_grids(grid.LeftEdge, grid.RightEdge)
mask[grid_ind] = True
return [g for g in self.grids[mask] if g.Level == grid.Level + 1]
+
class GDFStaticOutput(StaticOutput):
_hierarchy_class = GDFHierarchy
_fieldinfo_fallback = GDFFieldInfo
_fieldinfo_known = KnownGDFFields
def __init__(self, filename, data_style='grid_data_format',
- storage_filename = None):
+ storage_filename=None):
StaticOutput.__init__(self, filename, data_style)
self.storage_filename = storage_filename
self.filename = filename
def _set_units(self):
"""
- Generates the conversion to various physical _units based on the parameter file
+ Generates the conversion to various physical _units
+ based on the parameter file
"""
self.units = {}
self.time_units = {}
@@ -192,16 +201,17 @@
self.time_units['1'] = 1
self.units['1'] = 1.0
self.units['cm'] = 1.0
- self.units['unitary'] = 1.0 / (self.domain_right_edge - self.domain_left_edge).max()
+ self.units['unitary'] = 1.0 / (self.domain_right_edge -
+ self.domain_left_edge).max()
for unit in mpc_conversion.keys():
- self.units[unit] = 1.0 * mpc_conversion[unit] / mpc_conversion["cm"]
+ self.units[unit] = mpc_conversion[unit] / mpc_conversion["cm"]
for unit in sec_conversion.keys():
self.time_units[unit] = 1.0 / sec_conversion[unit]
# This should be improved.
- self._handle = h5py.File(self.parameter_filename, "r")
- for field_name in self._handle["/field_types"]:
- current_field = self._handle["/field_types/%s" % field_name]
+ h5f = h5py.File(self.parameter_filename, "r")
+ for field_name in h5f["/field_types"]:
+ current_field = h5f["/field_types/%s" % field_name]
if 'field_to_cgs' in current_field.attrs:
self.units[field_name] = current_field.attrs['field_to_cgs']
else:
@@ -210,15 +220,16 @@
if type(current_field.attrs['field_units']) == str:
current_fields_unit = current_field.attrs['field_units']
else:
- current_fields_unit = just_one(current_field.attrs['field_units'])
+ current_fields_unit = \
+ just_one(current_field.attrs['field_units'])
else:
current_fields_unit = ""
- self._fieldinfo_known.add_field(field_name, function=NullFunc, take_log=False,
- units=current_fields_unit, projected_units="",
- convert_function=_get_convert(field_name))
+ self._fieldinfo_known.add_field(
+ field_name, function=NullFunc, take_log=False,
+ units=current_fields_unit, projected_units="",
+ convert_function=_get_convert(field_name))
- self._handle.close()
- del self._handle
+ h5f.close()
def _parse_parameter_file(self):
self._handle = h5py.File(self.parameter_filename, "r")
@@ -232,13 +243,15 @@
self.domain_right_edge = sp["domain_right_edge"][:]
self.domain_dimensions = sp["domain_dimensions"][:]
refine_by = sp["refine_by"]
- if refine_by is None: refine_by = 2
+ if refine_by is None:
+ refine_by = 2
self.refine_by = refine_by
self.dimensionality = sp["dimensionality"]
self.current_time = sp["current_time"]
self.unique_identifier = sp["unique_identifier"]
self.cosmological_simulation = sp["cosmological_simulation"]
- if sp["num_ghost_zones"] != 0: raise RuntimeError
+ if sp["num_ghost_zones"] != 0:
+ raise RuntimeError
self.num_ghost_zones = sp["num_ghost_zones"]
self.field_ordering = sp["field_ordering"]
self.boundary_conditions = sp["boundary_conditions"][:]
@@ -252,15 +265,16 @@
else:
self.current_redshift = self.omega_lambda = self.omega_matter = \
self.hubble_constant = self.cosmological_simulation = 0.0
- self.parameters['Time'] = 1.0 # Hardcode time conversion for now.
- self.parameters["HydroMethod"] = 0 # Hardcode for now until field staggering is supported.
+ self.parameters['Time'] = 1.0 # Hardcode time conversion for now.
+ # Hardcode for now until field staggering is supported.
+ self.parameters["HydroMethod"] = 0
self._handle.close()
del self._handle
@classmethod
def _is_valid(self, *args, **kwargs):
try:
- fileh = h5py.File(args[0],'r')
+ fileh = h5py.File(args[0], 'r')
if "gridded_data_format" in fileh:
fileh.close()
return True
@@ -271,4 +285,3 @@
def __repr__(self):
return self.basename.rsplit(".", 1)[0]
-
diff -r 09fcbdcc5ae8b38bc37a5a95901d936303497086 -r 529acf3ab809bbc3031a75197d3c8fffb443f6b2 yt/frontends/gdf/fields.py
--- a/yt/frontends/gdf/fields.py
+++ b/yt/frontends/gdf/fields.py
@@ -16,14 +16,8 @@
from yt.data_objects.field_info_container import \
FieldInfoContainer, \
FieldInfo, \
- ValidateParameter, \
- ValidateDataField, \
- ValidateProperty, \
- ValidateSpatial, \
- ValidateGridType, \
NullFunc, \
TranslationFunc
-import yt.fields.universal_fields
log_translation_dict = {"Density": "density",
"Pressure": "pressure"}
@@ -31,7 +25,7 @@
translation_dict = {"x-velocity": "velocity_x",
"y-velocity": "velocity_y",
"z-velocity": "velocity_z"}
-
+
# translation_dict = {"mag_field_x": "cell_centered_B_x ",
# "mag_field_y": "cell_centered_B_y ",
# "mag_field_z": "cell_centered_B_z "}
@@ -43,39 +37,39 @@
add_gdf_field = KnownGDFFields.add_field
add_gdf_field("density", function=NullFunc, take_log=True,
- units=r"\rm{g}/\rm{cm}^3",
- projected_units =r"\rm{g}/\rm{cm}^2")
+ units=r"\rm{g}/\rm{cm}^3",
+ projected_units=r"\rm{g}/\rm{cm}^2")
add_gdf_field("specific_energy", function=NullFunc, take_log=True,
- units=r"\rm{erg}/\rm{g}")
+ units=r"\rm{erg}/\rm{g}")
add_gdf_field("pressure", function=NullFunc, take_log=True,
- units=r"\rm{erg}/\rm{g}")
+ units=r"\rm{erg}/\rm{g}")
add_gdf_field("velocity_x", function=NullFunc, take_log=False,
- units=r"\rm{cm}/\rm{s}")
+ units=r"\rm{cm}/\rm{s}")
add_gdf_field("velocity_y", function=NullFunc, take_log=False,
- units=r"\rm{cm}/\rm{s}")
+ units=r"\rm{cm}/\rm{s}")
add_gdf_field("velocity_z", function=NullFunc, take_log=False,
- units=r"\rm{cm}/\rm{s}")
+ units=r"\rm{cm}/\rm{s}")
add_gdf_field("mag_field_x", function=NullFunc, take_log=False,
- units=r"\rm{cm}/\rm{s}")
+ units=r"\rm{cm}/\rm{s}")
add_gdf_field("mag_field_y", function=NullFunc, take_log=False,
- units=r"\rm{cm}/\rm{s}")
+ units=r"\rm{cm}/\rm{s}")
add_gdf_field("mag_field_z", function=NullFunc, take_log=False,
- units=r"\rm{cm}/\rm{s}")
+ units=r"\rm{cm}/\rm{s}")
-for f,v in log_translation_dict.items():
+for f, v in log_translation_dict.items():
add_field(f, TranslationFunc(v), take_log=True,
units=KnownGDFFields[v].get_units(),
projected_units=KnownGDFFields[v].get_projected_units())
-for f,v in translation_dict.items():
+for f, v in translation_dict.items():
add_field(f, TranslationFunc(v), take_log=False,
units=KnownGDFFields[v].get_units(),
projected_units=KnownGDFFields[v].get_projected_units())
diff -r 09fcbdcc5ae8b38bc37a5a95901d936303497086 -r 529acf3ab809bbc3031a75197d3c8fffb443f6b2 yt/frontends/gdf/io.py
--- a/yt/frontends/gdf/io.py
+++ b/yt/frontends/gdf/io.py
@@ -14,14 +14,20 @@
#-----------------------------------------------------------------------------
import numpy as np
+import h5py
+import exceptions
from yt.funcs import \
mylog
from yt.utilities.io_handler import \
BaseIOHandler
-def field_dname(grid_id, field_name):
- return "/data/grid_%010i/%s" % (grid_id, field_name)
+def _grid_dname(grid_id):
+ return "/data/grid_%010i" % grid_id
+
+
+def _field_dname(grid_id, field_name):
+ return "%s/%s" % (_grid_dname(grid_id), field_name)
# TODO all particle bits were removed
@@ -30,50 +36,84 @@
_offset_string = 'data:offsets=0'
_data_string = 'data:datatype=0'
- def __init__(self, pf, *args, **kwargs):
- # TODO check if _num_per_stride is needed
- self._num_per_stride = kwargs.pop("num_per_stride", 1000000)
- BaseIOHandler.__init__(self, *args, **kwargs)
- self.pf = pf
- self._handle = pf._handle
+ def _read_field_names(self, grid):
+ if grid.filename is None:
+ return []
+ print 'grid.filename = %', grid.filename
+ h5f = h5py.File(grid.filename, mode="r")
+ group = h5f[_grid_dname(grid.id)]
+ fields = []
+ for name, v in group.iteritems():
+ # NOTE: This won't work with 1D datasets.
+ if not hasattr(v, "shape"):
+ continue
+ elif len(v.dims) == 1:
+ fields.append(("io", str(name)))
+ else:
+ fields.append(("gas", str(name)))
+ h5f.close()
+ return fields
-
- def _read_data_set(self, grid, field):
- if self.pf.field_ordering == 1:
- data = self._handle[field_dname(grid.id, field)][:].swapaxes(0, 2)
- else:
- data = self._handle[field_dname(grid.id, field)][:, :, :]
- return data.astype("float64")
-
- def _read_data_slice(self, grid, field, axis, coord):
- slc = [slice(None), slice(None), slice(None)]
- slc[axis] = slice(coord, coord + 1)
- if self.pf.field_ordering == 1:
- data = self._handle[field_dname(grid.id, field)][:].swapaxes(0, 2)[slc]
- else:
- data = self._handle[field_dname(grid.id, field)][slc]
- return data.astype("float64")
+ @property
+ def _read_exception(self):
+ return (exceptions.KeyError, )
def _read_fluid_selection(self, chunks, selector, fields, size):
+ rv = {}
chunks = list(chunks)
+
+ if selector.__class__.__name__ == "GridSelector":
+ if not (len(chunks) == len(chunks[0].objs) == 1):
+ raise RuntimeError
+ grid = chunks[0].objs[0]
+ h5f = h5py.File(grid.filename, 'r')
+ gds = h5f.get(_grid_dname(grid.id))
+ for ftype, fname in fields:
+ if self.pf.field_ordering == 1:
+ rv[(ftype, fname)] = gds.get(fname).value.swapaxes(0, 2)
+ else:
+ rv[(ftype, fname)] = gds.get(fname).value
+ h5f.close()
+ return rv
+ if size is None:
+ size = sum((grid.count(selector) for chunk in chunks
+ for grid in chunk.objs))
+
if any((ftype != "gas" for ftype, fname in fields)):
raise NotImplementedError
- fhandle = self._handle
- rv = {}
+
for field in fields:
ftype, fname = field
- rv[field] = np.empty(
- size, dtype=fhandle[field_dname(0, fname)].dtype)
+ fsize = size
+ # check the dtype instead
+ rv[field] = np.empty(fsize, dtype="float64")
ngrids = sum(len(chunk.objs) for chunk in chunks)
mylog.debug("Reading %s cells of %s fields in %s blocks",
size, [fname for ftype, fname in fields], ngrids)
- for field in fields:
- ftype, fname = field
- ind = 0
- for chunk in chunks:
- for grid in chunk.objs:
- data = fhandle[field_dname(grid.id, fname)][:]
- if self.pf.field_ordering == 1:
- data = data.swapaxes(0, 2)
- ind += g.select(selector, data, rv[field], ind) # caches
+ ind = 0
+ for chunk in chunks:
+ fid = None
+ for grid in chunk.objs:
+ if grid.filename is None:
+ continue
+ if fid is None:
+ fid = h5py.h5f.open(grid.filename, h5py.h5f.ACC_RDONLY)
+ if self.pf.field_ordering == 1:
+ # check the dtype instead
+ data = np.empty(grid.ActiveDimensions[::-1],
+ dtype="float64")
+ data_view = data.swapaxes(0, 2)
+ else:
+ # check the dtype instead
+ data_view = data = np.empty(grid.ActiveDimensions,
+ dtype="float64")
+ for field in fields:
+ ftype, fname = field
+ dg = h5py.h5d.open(fid, _field_dname(grid.id, fname))
+ dg.read(h5py.h5s.ALL, h5py.h5s.ALL, data)
+ # caches
+ nd = grid.select(selector, data_view, rv[field], ind)
+ ind += nd # I don't get that part, only last nd is added
+ if fid is not None:
+ fid.close()
return rv
Repository URL: https://bitbucket.org/yt_analysis/yt-3.0/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the yt-svn
mailing list