[yt-svn] commit/yt-3.0: 2 new changesets

commits-noreply at bitbucket.org commits-noreply at bitbucket.org
Fri Oct 18 20:08:50 PDT 2013


2 new commits in yt-3.0:

https://bitbucket.org/yt_analysis/yt-3.0/commits/b94c1e8965b4/
Changeset:   b94c1e8965b4
Branch:      yt-3.0
User:        xarthisius
Date:        2013-10-12 12:15:13
Summary:     [gdf] update wrt recent changes, projections and slices are working now
Affected #:  3 files

diff -r 5df0d5742ff2e8abcb6dd032f82247af9c6f0409 -r b94c1e8965b4174aadbd35d2d7bfd93d6d77ea52 yt/frontends/gdf/data_structures.py
--- a/yt/frontends/gdf/data_structures.py
+++ b/yt/frontends/gdf/data_structures.py
@@ -16,35 +16,37 @@
 import h5py
 import numpy as np
 import weakref
-from yt.funcs import *
+import os
+from yt.funcs import \
+    just_one, ensure_tuple
 from yt.data_objects.grid_patch import \
-           AMRGridPatch
+    AMRGridPatch
 from yt.geometry.grid_geometry_handler import \
-           GridGeometryHandler
+    GridGeometryHandler
 from yt.data_objects.static_output import \
-           StaticOutput
+    StaticOutput
 from yt.utilities.lib import \
     get_box_grids_level
-from yt.utilities.io_handler import \
-    io_registry
 from yt.utilities.definitions import \
     mpc_conversion, sec_conversion
 
 from .fields import GDFFieldInfo, KnownGDFFields
 from yt.data_objects.field_info_container import \
-    FieldInfoContainer, NullFunc
-import pdb
+    NullFunc
+
 
 def _get_convert(fname):
     def _conv(data):
-        return data.convert(fname)
+        return 1.0  # data.convert(fname) FIXME
     return _conv
 
+
 class GDFGrid(AMRGridPatch):
     _id_offset = 0
+
     def __init__(self, id, hierarchy, level, start, dimensions):
-        AMRGridPatch.__init__(self, id, filename = hierarchy.hierarchy_filename,
-                              hierarchy = hierarchy)
+        AMRGridPatch.__init__(self, id, filename=hierarchy.hierarchy_filename,
+                              hierarchy=hierarchy)
         self.Parent = []
         self.Children = []
         self.Level = level
@@ -59,39 +61,41 @@
         if len(self.Parent) > 0:
             self.dds = self.Parent[0].dds / self.pf.refine_by
         else:
-            LE, RE = self.hierarchy.grid_left_edge[id,:], \
-                     self.hierarchy.grid_right_edge[id,:]
-            self.dds = np.array((RE-LE)/self.ActiveDimensions)
+            LE, RE = self.hierarchy.grid_left_edge[id, :], \
+                self.hierarchy.grid_right_edge[id, :]
+            self.dds = np.array((RE - LE) / self.ActiveDimensions)
         if self.pf.data_software != "piernik":
-            if self.pf.dimensionality < 2: self.dds[1] = 1.0
-            if self.pf.dimensionality < 3: self.dds[2] = 1.0
-        self.field_data['dx'], self.field_data['dy'], self.field_data['dz'] = self.dds
+            if self.pf.dimensionality < 2:
+                self.dds[1] = 1.0
+            if self.pf.dimensionality < 3:
+                self.dds[2] = 1.0
+        self.field_data['dx'], self.field_data['dy'], self.field_data['dz'] = \
+            self.dds
 
-    @property
-    def filename(self):
-        return None
 
 class GDFHierarchy(GridGeometryHandler):
 
     grid = GDFGrid
+    filtered_particle_types = []
 
     def __init__(self, pf, data_style='grid_data_format'):
         self.parameter_file = weakref.proxy(pf)
+        self.hierarchy_filename = self.parameter_file.parameter_filename
+        h5f = h5py.File(self.hierarchy_filename, 'r')
         self.data_style = data_style
+        GridGeometryHandler.__init__(self, pf, data_style)
         self.max_level = 10  # FIXME
         # for now, the hierarchy file is the parameter file!
-        self.hierarchy_filename = self.parameter_file.parameter_filename
         self.directory = os.path.dirname(self.hierarchy_filename)
-        self._fhandle = h5py.File(self.hierarchy_filename,'r')
-        GridGeometryHandler.__init__(self,pf,data_style)
-
-        self._fhandle.close()
+        h5f.close()
 
     def _initialize_data_storage(self):
         pass
 
     def _detect_fields(self):
-        self.field_list = self._fhandle['field_types'].keys()
+        h5f = h5py.File(self.hierarchy_filename, 'r')
+        self.field_list = h5f['field_types'].keys()
+        h5f.close()
 
     def _setup_classes(self):
         dd = self._get_data_reader_dict()
@@ -99,15 +103,17 @@
         self.object_types.sort()
 
     def _count_grids(self):
-        self.num_grids = self._fhandle['/grid_parent_id'].shape[0]
+        h5f = h5py.File(self.hierarchy_filename, 'r')
+        self.num_grids = h5f['/grid_parent_id'].shape[0]
+        h5f.close()
 
     def _parse_hierarchy(self):
-        f = self._fhandle
+        h5f = h5py.File(self.hierarchy_filename, 'r')
         dxs = []
         self.grids = np.empty(self.num_grids, dtype='object')
-        levels = (f['grid_level'][:]).copy()
-        glis = (f['grid_left_index'][:]).copy()
-        gdims = (f['grid_dimensions'][:]).copy()
+        levels = (h5f['grid_level'][:]).copy()
+        glis = (h5f['grid_left_index'][:]).copy()
+        gdims = (h5f['grid_dimensions'][:]).copy()
         active_dims = ~((np.max(gdims, axis=0) == 1) &
                         (self.parameter_file.domain_dimensions == 1))
 
@@ -117,16 +123,18 @@
                                       gdims[i])
             self.grids[i]._level_id = levels[i]
 
-            dx = (self.parameter_file.domain_right_edge-
-                  self.parameter_file.domain_left_edge)/self.parameter_file.domain_dimensions
-            dx[active_dims] = dx[active_dims]/self.parameter_file.refine_by**(levels[i])
+            dx = (self.parameter_file.domain_right_edge -
+                  self.parameter_file.domain_left_edge) / \
+                self.parameter_file.domain_dimensions
+            dx[active_dims] /= self.parameter_file.refine_by ** levels[i]
             dxs.append(dx)
         dx = np.array(dxs)
-        self.grid_left_edge = self.parameter_file.domain_left_edge + dx*glis
+        self.grid_left_edge = self.parameter_file.domain_left_edge + dx * glis
         self.grid_dimensions = gdims.astype("int32")
-        self.grid_right_edge = self.grid_left_edge + dx*self.grid_dimensions
-        self.grid_particle_count = f['grid_particle_count'][:]
+        self.grid_right_edge = self.grid_left_edge + dx * self.grid_dimensions
+        self.grid_particle_count = h5f['grid_particle_count'][:]
         del levels, glis, gdims
+        h5f.close()
 
     def _populate_grid_objects(self):
         mask = np.empty(self.grids.size, dtype='int32')
@@ -138,8 +146,8 @@
             g.Children = self._get_grid_children(g)
             for g1 in g.Children:
                 g1.Parent.append(g)
-            get_box_grids_level(self.grid_left_edge[gi,:],
-                                self.grid_right_edge[gi,:],
+            get_box_grids_level(self.grid_left_edge[gi, :],
+                                self.grid_right_edge[gi, :],
                                 self.grid_levels[gi],
                                 self.grid_left_edge, self.grid_right_edge,
                                 self.grid_levels, mask)
@@ -158,32 +166,33 @@
         Gets back all the grids between a left edge and right edge
         """
         eps = np.finfo(np.float64).eps
-        grid_i = np.where((np.all((self.grid_right_edge - left_edge) > eps, axis=1) \
-                        &  np.all((right_edge - self.grid_left_edge) > eps, axis=1)) == True)
+        grid_i = np.where(np.all((self.grid_right_edge - left_edge) > eps, axis=1) &
+                          np.all((right_edge - self.grid_left_edge) > eps, axis=1))
 
         return self.grids[grid_i], grid_i
 
-
     def _get_grid_children(self, grid):
         mask = np.zeros(self.num_grids, dtype='bool')
         grids, grid_ind = self._get_box_grids(grid.LeftEdge, grid.RightEdge)
         mask[grid_ind] = True
         return [g for g in self.grids[mask] if g.Level == grid.Level + 1]
 
+
 class GDFStaticOutput(StaticOutput):
     _hierarchy_class = GDFHierarchy
     _fieldinfo_fallback = GDFFieldInfo
     _fieldinfo_known = KnownGDFFields
 
     def __init__(self, filename, data_style='grid_data_format',
-                 storage_filename = None):
+                 storage_filename=None):
         StaticOutput.__init__(self, filename, data_style)
         self.storage_filename = storage_filename
         self.filename = filename
 
     def _set_units(self):
         """
-        Generates the conversion to various physical _units based on the parameter file
+        Generates the conversion to various physical _units
+        based on the parameter file
         """
         self.units = {}
         self.time_units = {}
@@ -192,16 +201,17 @@
         self.time_units['1'] = 1
         self.units['1'] = 1.0
         self.units['cm'] = 1.0
-        self.units['unitary'] = 1.0 / (self.domain_right_edge - self.domain_left_edge).max()
+        self.units['unitary'] = 1.0 / (self.domain_right_edge -
+                                       self.domain_left_edge).max()
         for unit in mpc_conversion.keys():
-            self.units[unit] = 1.0 * mpc_conversion[unit] / mpc_conversion["cm"]
+            self.units[unit] = mpc_conversion[unit] / mpc_conversion["cm"]
         for unit in sec_conversion.keys():
             self.time_units[unit] = 1.0 / sec_conversion[unit]
 
         # This should be improved.
-        self._handle = h5py.File(self.parameter_filename, "r")
-        for field_name in self._handle["/field_types"]:
-            current_field = self._handle["/field_types/%s" % field_name]
+        h5f = h5py.File(self.parameter_filename, "r")
+        for field_name in h5f["/field_types"]:
+            current_field = h5f["/field_types/%s" % field_name]
             if 'field_to_cgs' in current_field.attrs:
                 self.units[field_name] = current_field.attrs['field_to_cgs']
             else:
@@ -210,15 +220,16 @@
                 if type(current_field.attrs['field_units']) == str:
                     current_fields_unit = current_field.attrs['field_units']
                 else:
-                    current_fields_unit = just_one(current_field.attrs['field_units'])
+                    current_fields_unit = \
+                        just_one(current_field.attrs['field_units'])
             else:
                 current_fields_unit = ""
-            self._fieldinfo_known.add_field(field_name, function=NullFunc, take_log=False,
-                   units=current_fields_unit, projected_units="",
-                   convert_function=_get_convert(field_name))
+            self._fieldinfo_known.add_field(
+                field_name, function=NullFunc, take_log=False,
+                units=current_fields_unit, projected_units="",
+                convert_function=_get_convert(field_name))
 
-        self._handle.close()
-        del self._handle
+        h5f.close()
 
     def _parse_parameter_file(self):
         self._handle = h5py.File(self.parameter_filename, "r")
@@ -232,13 +243,15 @@
         self.domain_right_edge = sp["domain_right_edge"][:]
         self.domain_dimensions = sp["domain_dimensions"][:]
         refine_by = sp["refine_by"]
-        if refine_by is None: refine_by = 2
+        if refine_by is None:
+            refine_by = 2
         self.refine_by = refine_by
         self.dimensionality = sp["dimensionality"]
         self.current_time = sp["current_time"]
         self.unique_identifier = sp["unique_identifier"]
         self.cosmological_simulation = sp["cosmological_simulation"]
-        if sp["num_ghost_zones"] != 0: raise RuntimeError
+        if sp["num_ghost_zones"] != 0:
+            raise RuntimeError
         self.num_ghost_zones = sp["num_ghost_zones"]
         self.field_ordering = sp["field_ordering"]
         self.boundary_conditions = sp["boundary_conditions"][:]
@@ -252,15 +265,16 @@
         else:
             self.current_redshift = self.omega_lambda = self.omega_matter = \
                 self.hubble_constant = self.cosmological_simulation = 0.0
-        self.parameters['Time'] = 1.0 # Hardcode time conversion for now.
-        self.parameters["HydroMethod"] = 0 # Hardcode for now until field staggering is supported.
+        self.parameters['Time'] = 1.0  # Hardcode time conversion for now.
+        # Hardcode for now until field staggering is supported.
+        self.parameters["HydroMethod"] = 0
         self._handle.close()
         del self._handle
 
     @classmethod
     def _is_valid(self, *args, **kwargs):
         try:
-            fileh = h5py.File(args[0],'r')
+            fileh = h5py.File(args[0], 'r')
             if "gridded_data_format" in fileh:
                 fileh.close()
                 return True
@@ -271,4 +285,3 @@
 
     def __repr__(self):
         return self.basename.rsplit(".", 1)[0]
-

diff -r 5df0d5742ff2e8abcb6dd032f82247af9c6f0409 -r b94c1e8965b4174aadbd35d2d7bfd93d6d77ea52 yt/frontends/gdf/fields.py
--- a/yt/frontends/gdf/fields.py
+++ b/yt/frontends/gdf/fields.py
@@ -16,14 +16,8 @@
 from yt.data_objects.field_info_container import \
     FieldInfoContainer, \
     FieldInfo, \
-    ValidateParameter, \
-    ValidateDataField, \
-    ValidateProperty, \
-    ValidateSpatial, \
-    ValidateGridType, \
     NullFunc, \
     TranslationFunc
-import yt.fields.universal_fields
 
 log_translation_dict = {"Density": "density",
                         "Pressure": "pressure"}
@@ -31,7 +25,7 @@
 translation_dict = {"x-velocity": "velocity_x",
                     "y-velocity": "velocity_y",
                     "z-velocity": "velocity_z"}
-                    
+
 # translation_dict = {"mag_field_x": "cell_centered_B_x ",
 #                     "mag_field_y": "cell_centered_B_y ",
 #                     "mag_field_z": "cell_centered_B_z "}
@@ -43,39 +37,39 @@
 add_gdf_field = KnownGDFFields.add_field
 
 add_gdf_field("density", function=NullFunc, take_log=True,
-          units=r"\rm{g}/\rm{cm}^3",
-          projected_units =r"\rm{g}/\rm{cm}^2")
+              units=r"\rm{g}/\rm{cm}^3",
+              projected_units=r"\rm{g}/\rm{cm}^2")
 
 add_gdf_field("specific_energy", function=NullFunc, take_log=True,
-          units=r"\rm{erg}/\rm{g}")
+              units=r"\rm{erg}/\rm{g}")
 
 add_gdf_field("pressure", function=NullFunc, take_log=True,
-          units=r"\rm{erg}/\rm{g}")
+              units=r"\rm{erg}/\rm{g}")
 
 add_gdf_field("velocity_x", function=NullFunc, take_log=False,
-          units=r"\rm{cm}/\rm{s}")
+              units=r"\rm{cm}/\rm{s}")
 
 add_gdf_field("velocity_y", function=NullFunc, take_log=False,
-          units=r"\rm{cm}/\rm{s}")
+              units=r"\rm{cm}/\rm{s}")
 
 add_gdf_field("velocity_z", function=NullFunc, take_log=False,
-          units=r"\rm{cm}/\rm{s}")
+              units=r"\rm{cm}/\rm{s}")
 
 add_gdf_field("mag_field_x", function=NullFunc, take_log=False,
-          units=r"\rm{cm}/\rm{s}")
+              units=r"\rm{cm}/\rm{s}")
 
 add_gdf_field("mag_field_y", function=NullFunc, take_log=False,
-          units=r"\rm{cm}/\rm{s}")
+              units=r"\rm{cm}/\rm{s}")
 
 add_gdf_field("mag_field_z", function=NullFunc, take_log=False,
-          units=r"\rm{cm}/\rm{s}")
+              units=r"\rm{cm}/\rm{s}")
 
-for f,v in log_translation_dict.items():
+for f, v in log_translation_dict.items():
     add_field(f, TranslationFunc(v), take_log=True,
               units=KnownGDFFields[v].get_units(),
               projected_units=KnownGDFFields[v].get_projected_units())
 
-for f,v in translation_dict.items():
+for f, v in translation_dict.items():
     add_field(f, TranslationFunc(v), take_log=False,
               units=KnownGDFFields[v].get_units(),
               projected_units=KnownGDFFields[v].get_projected_units())

diff -r 5df0d5742ff2e8abcb6dd032f82247af9c6f0409 -r b94c1e8965b4174aadbd35d2d7bfd93d6d77ea52 yt/frontends/gdf/io.py
--- a/yt/frontends/gdf/io.py
+++ b/yt/frontends/gdf/io.py
@@ -14,14 +14,20 @@
 #-----------------------------------------------------------------------------
 
 import numpy as np
+import h5py
+import exceptions
 from yt.funcs import \
     mylog
 from yt.utilities.io_handler import \
     BaseIOHandler
 
 
-def field_dname(grid_id, field_name):
-    return "/data/grid_%010i/%s" % (grid_id, field_name)
+def _grid_dname(grid_id):
+    return "/data/grid_%010i" % grid_id
+
+
+def _field_dname(grid_id, field_name):
+    return "%s/%s" % (_grid_dname(grid_id), field_name)
 
 
 # TODO all particle bits were removed
@@ -30,50 +36,84 @@
     _offset_string = 'data:offsets=0'
     _data_string = 'data:datatype=0'
 
-    def __init__(self, pf, *args, **kwargs):
-        # TODO check if _num_per_stride is needed
-        self._num_per_stride = kwargs.pop("num_per_stride", 1000000)
-        BaseIOHandler.__init__(self, *args, **kwargs)
-        self.pf = pf
-        self._handle = pf._handle
+    def _read_field_names(self, grid):
+        if grid.filename is None:
+            return []
+        print 'grid.filename = %', grid.filename
+        h5f = h5py.File(grid.filename, mode="r")
+        group = h5f[_grid_dname(grid.id)]
+        fields = []
+        for name, v in group.iteritems():
+            # NOTE: This won't work with 1D datasets.
+            if not hasattr(v, "shape"):
+                continue
+            elif len(v.dims) == 1:
+                fields.append(("io", str(name)))
+            else:
+                fields.append(("gas", str(name)))
+        h5f.close()
+        return fields
 
-
-    def _read_data_set(self, grid, field):
-        if self.pf.field_ordering == 1:
-            data = self._handle[field_dname(grid.id, field)][:].swapaxes(0, 2)
-        else:
-            data = self._handle[field_dname(grid.id, field)][:, :, :]
-        return data.astype("float64")
-
-    def _read_data_slice(self, grid, field, axis, coord):
-        slc = [slice(None), slice(None), slice(None)]
-        slc[axis] = slice(coord, coord + 1)
-        if self.pf.field_ordering == 1:
-            data = self._handle[field_dname(grid.id, field)][:].swapaxes(0, 2)[slc]
-        else:
-            data = self._handle[field_dname(grid.id, field)][slc]
-        return data.astype("float64")
+    @property
+    def _read_exception(self):
+        return (exceptions.KeyError, )
 
     def _read_fluid_selection(self, chunks, selector, fields, size):
+        rv = {}
         chunks = list(chunks)
+
+        if selector.__class__.__name__ == "GridSelector":
+            if not (len(chunks) == len(chunks[0].objs) == 1):
+                raise RuntimeError
+            grid = chunks[0].objs[0]
+            h5f = h5py.File(grid.filename, 'r')
+            gds = h5f.get(_grid_dname(grid.id))
+            for ftype, fname in fields:
+                if self.pf.field_ordering == 1:
+                    rv[(ftype, fname)] = gds.get(fname).value.swapaxes(0, 2)
+                else:
+                    rv[(ftype, fname)] = gds.get(fname).value
+            h5f.close()
+            return rv
+        if size is None:
+            size = sum((grid.count(selector) for chunk in chunks
+                        for grid in chunk.objs))
+
         if any((ftype != "gas" for ftype, fname in fields)):
             raise NotImplementedError
-        fhandle = self._handle
-        rv = {}
+
         for field in fields:
             ftype, fname = field
-            rv[field] = np.empty(
-                size, dtype=fhandle[field_dname(0, fname)].dtype)
+            fsize = size
+            # check the dtype instead
+            rv[field] = np.empty(fsize, dtype="float64")
         ngrids = sum(len(chunk.objs) for chunk in chunks)
         mylog.debug("Reading %s cells of %s fields in %s blocks",
                     size, [fname for ftype, fname in fields], ngrids)
-        for field in fields:
-            ftype, fname = field
-            ind = 0
-            for chunk in chunks:
-                for grid in chunk.objs:
-                    data = fhandle[field_dname(grid.id, fname)][:]
-                    if self.pf.field_ordering == 1:
-                        data = data.swapaxes(0, 2)
-                    ind += g.select(selector, data, rv[field], ind) # caches
+        ind = 0
+        for chunk in chunks:
+            fid = None
+            for grid in chunk.objs:
+                if grid.filename is None:
+                    continue
+                if fid is None:
+                    fid = h5py.h5f.open(grid.filename, h5py.h5f.ACC_RDONLY)
+                if self.pf.field_ordering == 1:
+                    # check the dtype instead
+                    data = np.empty(grid.ActiveDimensions[::-1],
+                                    dtype="float64")
+                    data_view = data.swapaxes(0, 2)
+                else:
+                    # check the dtype instead
+                    data_view = data = np.empty(grid.ActiveDimensions,
+                                                dtype="float64")
+                for field in fields:
+                    ftype, fname = field
+                    dg = h5py.h5d.open(fid, _field_dname(grid.id, fname))
+                    dg.read(h5py.h5s.ALL, h5py.h5s.ALL, data)
+                    # caches
+                    nd = grid.select(selector, data_view, rv[field], ind)
+                ind += nd    # I don't get that part, only last nd is added
+            if fid is not None:
+                fid.close()
         return rv


https://bitbucket.org/yt_analysis/yt-3.0/commits/529acf3ab809/
Changeset:   529acf3ab809
Branch:      yt-3.0
User:        MatthewTurk
Date:        2013-10-19 05:08:46
Summary:     Merged in xarthisius/yt-3.0 (pull request #116)

Update for GDF frontend
Affected #:  3 files

diff -r 09fcbdcc5ae8b38bc37a5a95901d936303497086 -r 529acf3ab809bbc3031a75197d3c8fffb443f6b2 yt/frontends/gdf/data_structures.py
--- a/yt/frontends/gdf/data_structures.py
+++ b/yt/frontends/gdf/data_structures.py
@@ -16,35 +16,37 @@
 import h5py
 import numpy as np
 import weakref
-from yt.funcs import *
+import os
+from yt.funcs import \
+    just_one, ensure_tuple
 from yt.data_objects.grid_patch import \
-           AMRGridPatch
+    AMRGridPatch
 from yt.geometry.grid_geometry_handler import \
-           GridGeometryHandler
+    GridGeometryHandler
 from yt.data_objects.static_output import \
-           StaticOutput
+    StaticOutput
 from yt.utilities.lib import \
     get_box_grids_level
-from yt.utilities.io_handler import \
-    io_registry
 from yt.utilities.definitions import \
     mpc_conversion, sec_conversion
 
 from .fields import GDFFieldInfo, KnownGDFFields
 from yt.data_objects.field_info_container import \
-    FieldInfoContainer, NullFunc
-import pdb
+    NullFunc
+
 
 def _get_convert(fname):
     def _conv(data):
-        return data.convert(fname)
+        return 1.0  # data.convert(fname) FIXME
     return _conv
 
+
 class GDFGrid(AMRGridPatch):
     _id_offset = 0
+
     def __init__(self, id, hierarchy, level, start, dimensions):
-        AMRGridPatch.__init__(self, id, filename = hierarchy.hierarchy_filename,
-                              hierarchy = hierarchy)
+        AMRGridPatch.__init__(self, id, filename=hierarchy.hierarchy_filename,
+                              hierarchy=hierarchy)
         self.Parent = []
         self.Children = []
         self.Level = level
@@ -59,39 +61,41 @@
         if len(self.Parent) > 0:
             self.dds = self.Parent[0].dds / self.pf.refine_by
         else:
-            LE, RE = self.hierarchy.grid_left_edge[id,:], \
-                     self.hierarchy.grid_right_edge[id,:]
-            self.dds = np.array((RE-LE)/self.ActiveDimensions)
+            LE, RE = self.hierarchy.grid_left_edge[id, :], \
+                self.hierarchy.grid_right_edge[id, :]
+            self.dds = np.array((RE - LE) / self.ActiveDimensions)
         if self.pf.data_software != "piernik":
-            if self.pf.dimensionality < 2: self.dds[1] = 1.0
-            if self.pf.dimensionality < 3: self.dds[2] = 1.0
-        self.field_data['dx'], self.field_data['dy'], self.field_data['dz'] = self.dds
+            if self.pf.dimensionality < 2:
+                self.dds[1] = 1.0
+            if self.pf.dimensionality < 3:
+                self.dds[2] = 1.0
+        self.field_data['dx'], self.field_data['dy'], self.field_data['dz'] = \
+            self.dds
 
-    @property
-    def filename(self):
-        return None
 
 class GDFHierarchy(GridGeometryHandler):
 
     grid = GDFGrid
+    filtered_particle_types = []
 
     def __init__(self, pf, data_style='grid_data_format'):
         self.parameter_file = weakref.proxy(pf)
+        self.hierarchy_filename = self.parameter_file.parameter_filename
+        h5f = h5py.File(self.hierarchy_filename, 'r')
         self.data_style = data_style
+        GridGeometryHandler.__init__(self, pf, data_style)
         self.max_level = 10  # FIXME
         # for now, the hierarchy file is the parameter file!
-        self.hierarchy_filename = self.parameter_file.parameter_filename
         self.directory = os.path.dirname(self.hierarchy_filename)
-        self._fhandle = h5py.File(self.hierarchy_filename,'r')
-        GridGeometryHandler.__init__(self,pf,data_style)
-
-        self._fhandle.close()
+        h5f.close()
 
     def _initialize_data_storage(self):
         pass
 
     def _detect_fields(self):
-        self.field_list = self._fhandle['field_types'].keys()
+        h5f = h5py.File(self.hierarchy_filename, 'r')
+        self.field_list = h5f['field_types'].keys()
+        h5f.close()
 
     def _setup_classes(self):
         dd = self._get_data_reader_dict()
@@ -99,15 +103,17 @@
         self.object_types.sort()
 
     def _count_grids(self):
-        self.num_grids = self._fhandle['/grid_parent_id'].shape[0]
+        h5f = h5py.File(self.hierarchy_filename, 'r')
+        self.num_grids = h5f['/grid_parent_id'].shape[0]
+        h5f.close()
 
     def _parse_hierarchy(self):
-        f = self._fhandle
+        h5f = h5py.File(self.hierarchy_filename, 'r')
         dxs = []
         self.grids = np.empty(self.num_grids, dtype='object')
-        levels = (f['grid_level'][:]).copy()
-        glis = (f['grid_left_index'][:]).copy()
-        gdims = (f['grid_dimensions'][:]).copy()
+        levels = (h5f['grid_level'][:]).copy()
+        glis = (h5f['grid_left_index'][:]).copy()
+        gdims = (h5f['grid_dimensions'][:]).copy()
         active_dims = ~((np.max(gdims, axis=0) == 1) &
                         (self.parameter_file.domain_dimensions == 1))
 
@@ -117,16 +123,18 @@
                                       gdims[i])
             self.grids[i]._level_id = levels[i]
 
-            dx = (self.parameter_file.domain_right_edge-
-                  self.parameter_file.domain_left_edge)/self.parameter_file.domain_dimensions
-            dx[active_dims] = dx[active_dims]/self.parameter_file.refine_by**(levels[i])
+            dx = (self.parameter_file.domain_right_edge -
+                  self.parameter_file.domain_left_edge) / \
+                self.parameter_file.domain_dimensions
+            dx[active_dims] /= self.parameter_file.refine_by ** levels[i]
             dxs.append(dx)
         dx = np.array(dxs)
-        self.grid_left_edge = self.parameter_file.domain_left_edge + dx*glis
+        self.grid_left_edge = self.parameter_file.domain_left_edge + dx * glis
         self.grid_dimensions = gdims.astype("int32")
-        self.grid_right_edge = self.grid_left_edge + dx*self.grid_dimensions
-        self.grid_particle_count = f['grid_particle_count'][:]
+        self.grid_right_edge = self.grid_left_edge + dx * self.grid_dimensions
+        self.grid_particle_count = h5f['grid_particle_count'][:]
         del levels, glis, gdims
+        h5f.close()
 
     def _populate_grid_objects(self):
         mask = np.empty(self.grids.size, dtype='int32')
@@ -138,8 +146,8 @@
             g.Children = self._get_grid_children(g)
             for g1 in g.Children:
                 g1.Parent.append(g)
-            get_box_grids_level(self.grid_left_edge[gi,:],
-                                self.grid_right_edge[gi,:],
+            get_box_grids_level(self.grid_left_edge[gi, :],
+                                self.grid_right_edge[gi, :],
                                 self.grid_levels[gi],
                                 self.grid_left_edge, self.grid_right_edge,
                                 self.grid_levels, mask)
@@ -158,32 +166,33 @@
         Gets back all the grids between a left edge and right edge
         """
         eps = np.finfo(np.float64).eps
-        grid_i = np.where((np.all((self.grid_right_edge - left_edge) > eps, axis=1) \
-                        &  np.all((right_edge - self.grid_left_edge) > eps, axis=1)) == True)
+        grid_i = np.where(np.all((self.grid_right_edge - left_edge) > eps, axis=1) &
+                          np.all((right_edge - self.grid_left_edge) > eps, axis=1))
 
         return self.grids[grid_i], grid_i
 
-
     def _get_grid_children(self, grid):
         mask = np.zeros(self.num_grids, dtype='bool')
         grids, grid_ind = self._get_box_grids(grid.LeftEdge, grid.RightEdge)
         mask[grid_ind] = True
         return [g for g in self.grids[mask] if g.Level == grid.Level + 1]
 
+
 class GDFStaticOutput(StaticOutput):
     _hierarchy_class = GDFHierarchy
     _fieldinfo_fallback = GDFFieldInfo
     _fieldinfo_known = KnownGDFFields
 
     def __init__(self, filename, data_style='grid_data_format',
-                 storage_filename = None):
+                 storage_filename=None):
         StaticOutput.__init__(self, filename, data_style)
         self.storage_filename = storage_filename
         self.filename = filename
 
     def _set_units(self):
         """
-        Generates the conversion to various physical _units based on the parameter file
+        Generates the conversion to various physical _units
+        based on the parameter file
         """
         self.units = {}
         self.time_units = {}
@@ -192,16 +201,17 @@
         self.time_units['1'] = 1
         self.units['1'] = 1.0
         self.units['cm'] = 1.0
-        self.units['unitary'] = 1.0 / (self.domain_right_edge - self.domain_left_edge).max()
+        self.units['unitary'] = 1.0 / (self.domain_right_edge -
+                                       self.domain_left_edge).max()
         for unit in mpc_conversion.keys():
-            self.units[unit] = 1.0 * mpc_conversion[unit] / mpc_conversion["cm"]
+            self.units[unit] = mpc_conversion[unit] / mpc_conversion["cm"]
         for unit in sec_conversion.keys():
             self.time_units[unit] = 1.0 / sec_conversion[unit]
 
         # This should be improved.
-        self._handle = h5py.File(self.parameter_filename, "r")
-        for field_name in self._handle["/field_types"]:
-            current_field = self._handle["/field_types/%s" % field_name]
+        h5f = h5py.File(self.parameter_filename, "r")
+        for field_name in h5f["/field_types"]:
+            current_field = h5f["/field_types/%s" % field_name]
             if 'field_to_cgs' in current_field.attrs:
                 self.units[field_name] = current_field.attrs['field_to_cgs']
             else:
@@ -210,15 +220,16 @@
                 if type(current_field.attrs['field_units']) == str:
                     current_fields_unit = current_field.attrs['field_units']
                 else:
-                    current_fields_unit = just_one(current_field.attrs['field_units'])
+                    current_fields_unit = \
+                        just_one(current_field.attrs['field_units'])
             else:
                 current_fields_unit = ""
-            self._fieldinfo_known.add_field(field_name, function=NullFunc, take_log=False,
-                   units=current_fields_unit, projected_units="",
-                   convert_function=_get_convert(field_name))
+            self._fieldinfo_known.add_field(
+                field_name, function=NullFunc, take_log=False,
+                units=current_fields_unit, projected_units="",
+                convert_function=_get_convert(field_name))
 
-        self._handle.close()
-        del self._handle
+        h5f.close()
 
     def _parse_parameter_file(self):
         self._handle = h5py.File(self.parameter_filename, "r")
@@ -232,13 +243,15 @@
         self.domain_right_edge = sp["domain_right_edge"][:]
         self.domain_dimensions = sp["domain_dimensions"][:]
         refine_by = sp["refine_by"]
-        if refine_by is None: refine_by = 2
+        if refine_by is None:
+            refine_by = 2
         self.refine_by = refine_by
         self.dimensionality = sp["dimensionality"]
         self.current_time = sp["current_time"]
         self.unique_identifier = sp["unique_identifier"]
         self.cosmological_simulation = sp["cosmological_simulation"]
-        if sp["num_ghost_zones"] != 0: raise RuntimeError
+        if sp["num_ghost_zones"] != 0:
+            raise RuntimeError
         self.num_ghost_zones = sp["num_ghost_zones"]
         self.field_ordering = sp["field_ordering"]
         self.boundary_conditions = sp["boundary_conditions"][:]
@@ -252,15 +265,16 @@
         else:
             self.current_redshift = self.omega_lambda = self.omega_matter = \
                 self.hubble_constant = self.cosmological_simulation = 0.0
-        self.parameters['Time'] = 1.0 # Hardcode time conversion for now.
-        self.parameters["HydroMethod"] = 0 # Hardcode for now until field staggering is supported.
+        self.parameters['Time'] = 1.0  # Hardcode time conversion for now.
+        # Hardcode for now until field staggering is supported.
+        self.parameters["HydroMethod"] = 0
         self._handle.close()
         del self._handle
 
     @classmethod
     def _is_valid(self, *args, **kwargs):
         try:
-            fileh = h5py.File(args[0],'r')
+            fileh = h5py.File(args[0], 'r')
             if "gridded_data_format" in fileh:
                 fileh.close()
                 return True
@@ -271,4 +285,3 @@
 
     def __repr__(self):
         return self.basename.rsplit(".", 1)[0]
-

diff -r 09fcbdcc5ae8b38bc37a5a95901d936303497086 -r 529acf3ab809bbc3031a75197d3c8fffb443f6b2 yt/frontends/gdf/fields.py
--- a/yt/frontends/gdf/fields.py
+++ b/yt/frontends/gdf/fields.py
@@ -16,14 +16,8 @@
 from yt.data_objects.field_info_container import \
     FieldInfoContainer, \
     FieldInfo, \
-    ValidateParameter, \
-    ValidateDataField, \
-    ValidateProperty, \
-    ValidateSpatial, \
-    ValidateGridType, \
     NullFunc, \
     TranslationFunc
-import yt.fields.universal_fields
 
 log_translation_dict = {"Density": "density",
                         "Pressure": "pressure"}
@@ -31,7 +25,7 @@
 translation_dict = {"x-velocity": "velocity_x",
                     "y-velocity": "velocity_y",
                     "z-velocity": "velocity_z"}
-                    
+
 # translation_dict = {"mag_field_x": "cell_centered_B_x ",
 #                     "mag_field_y": "cell_centered_B_y ",
 #                     "mag_field_z": "cell_centered_B_z "}
@@ -43,39 +37,39 @@
 add_gdf_field = KnownGDFFields.add_field
 
 add_gdf_field("density", function=NullFunc, take_log=True,
-          units=r"\rm{g}/\rm{cm}^3",
-          projected_units =r"\rm{g}/\rm{cm}^2")
+              units=r"\rm{g}/\rm{cm}^3",
+              projected_units=r"\rm{g}/\rm{cm}^2")
 
 add_gdf_field("specific_energy", function=NullFunc, take_log=True,
-          units=r"\rm{erg}/\rm{g}")
+              units=r"\rm{erg}/\rm{g}")
 
 add_gdf_field("pressure", function=NullFunc, take_log=True,
-          units=r"\rm{erg}/\rm{g}")
+              units=r"\rm{erg}/\rm{g}")
 
 add_gdf_field("velocity_x", function=NullFunc, take_log=False,
-          units=r"\rm{cm}/\rm{s}")
+              units=r"\rm{cm}/\rm{s}")
 
 add_gdf_field("velocity_y", function=NullFunc, take_log=False,
-          units=r"\rm{cm}/\rm{s}")
+              units=r"\rm{cm}/\rm{s}")
 
 add_gdf_field("velocity_z", function=NullFunc, take_log=False,
-          units=r"\rm{cm}/\rm{s}")
+              units=r"\rm{cm}/\rm{s}")
 
 add_gdf_field("mag_field_x", function=NullFunc, take_log=False,
-          units=r"\rm{cm}/\rm{s}")
+              units=r"\rm{cm}/\rm{s}")
 
 add_gdf_field("mag_field_y", function=NullFunc, take_log=False,
-          units=r"\rm{cm}/\rm{s}")
+              units=r"\rm{cm}/\rm{s}")
 
 add_gdf_field("mag_field_z", function=NullFunc, take_log=False,
-          units=r"\rm{cm}/\rm{s}")
+              units=r"\rm{cm}/\rm{s}")
 
-for f,v in log_translation_dict.items():
+for f, v in log_translation_dict.items():
     add_field(f, TranslationFunc(v), take_log=True,
               units=KnownGDFFields[v].get_units(),
               projected_units=KnownGDFFields[v].get_projected_units())
 
-for f,v in translation_dict.items():
+for f, v in translation_dict.items():
     add_field(f, TranslationFunc(v), take_log=False,
               units=KnownGDFFields[v].get_units(),
               projected_units=KnownGDFFields[v].get_projected_units())

diff -r 09fcbdcc5ae8b38bc37a5a95901d936303497086 -r 529acf3ab809bbc3031a75197d3c8fffb443f6b2 yt/frontends/gdf/io.py
--- a/yt/frontends/gdf/io.py
+++ b/yt/frontends/gdf/io.py
@@ -14,14 +14,20 @@
 #-----------------------------------------------------------------------------
 
 import numpy as np
+import h5py
+import exceptions
 from yt.funcs import \
     mylog
 from yt.utilities.io_handler import \
     BaseIOHandler
 
 
-def field_dname(grid_id, field_name):
-    return "/data/grid_%010i/%s" % (grid_id, field_name)
+def _grid_dname(grid_id):
+    return "/data/grid_%010i" % grid_id
+
+
+def _field_dname(grid_id, field_name):
+    return "%s/%s" % (_grid_dname(grid_id), field_name)
 
 
 # TODO all particle bits were removed
@@ -30,50 +36,84 @@
     _offset_string = 'data:offsets=0'
     _data_string = 'data:datatype=0'
 
-    def __init__(self, pf, *args, **kwargs):
-        # TODO check if _num_per_stride is needed
-        self._num_per_stride = kwargs.pop("num_per_stride", 1000000)
-        BaseIOHandler.__init__(self, *args, **kwargs)
-        self.pf = pf
-        self._handle = pf._handle
+    def _read_field_names(self, grid):
+        if grid.filename is None:
+            return []
+        print 'grid.filename = %', grid.filename
+        h5f = h5py.File(grid.filename, mode="r")
+        group = h5f[_grid_dname(grid.id)]
+        fields = []
+        for name, v in group.iteritems():
+            # NOTE: This won't work with 1D datasets.
+            if not hasattr(v, "shape"):
+                continue
+            elif len(v.dims) == 1:
+                fields.append(("io", str(name)))
+            else:
+                fields.append(("gas", str(name)))
+        h5f.close()
+        return fields
 
-
-    def _read_data_set(self, grid, field):
-        if self.pf.field_ordering == 1:
-            data = self._handle[field_dname(grid.id, field)][:].swapaxes(0, 2)
-        else:
-            data = self._handle[field_dname(grid.id, field)][:, :, :]
-        return data.astype("float64")
-
-    def _read_data_slice(self, grid, field, axis, coord):
-        slc = [slice(None), slice(None), slice(None)]
-        slc[axis] = slice(coord, coord + 1)
-        if self.pf.field_ordering == 1:
-            data = self._handle[field_dname(grid.id, field)][:].swapaxes(0, 2)[slc]
-        else:
-            data = self._handle[field_dname(grid.id, field)][slc]
-        return data.astype("float64")
+    @property
+    def _read_exception(self):
+        return (exceptions.KeyError, )
 
     def _read_fluid_selection(self, chunks, selector, fields, size):
+        rv = {}
         chunks = list(chunks)
+
+        if selector.__class__.__name__ == "GridSelector":
+            if not (len(chunks) == len(chunks[0].objs) == 1):
+                raise RuntimeError
+            grid = chunks[0].objs[0]
+            h5f = h5py.File(grid.filename, 'r')
+            gds = h5f.get(_grid_dname(grid.id))
+            for ftype, fname in fields:
+                if self.pf.field_ordering == 1:
+                    rv[(ftype, fname)] = gds.get(fname).value.swapaxes(0, 2)
+                else:
+                    rv[(ftype, fname)] = gds.get(fname).value
+            h5f.close()
+            return rv
+        if size is None:
+            size = sum((grid.count(selector) for chunk in chunks
+                        for grid in chunk.objs))
+
         if any((ftype != "gas" for ftype, fname in fields)):
             raise NotImplementedError
-        fhandle = self._handle
-        rv = {}
+
         for field in fields:
             ftype, fname = field
-            rv[field] = np.empty(
-                size, dtype=fhandle[field_dname(0, fname)].dtype)
+            fsize = size
+            # check the dtype instead
+            rv[field] = np.empty(fsize, dtype="float64")
         ngrids = sum(len(chunk.objs) for chunk in chunks)
         mylog.debug("Reading %s cells of %s fields in %s blocks",
                     size, [fname for ftype, fname in fields], ngrids)
-        for field in fields:
-            ftype, fname = field
-            ind = 0
-            for chunk in chunks:
-                for grid in chunk.objs:
-                    data = fhandle[field_dname(grid.id, fname)][:]
-                    if self.pf.field_ordering == 1:
-                        data = data.swapaxes(0, 2)
-                    ind += g.select(selector, data, rv[field], ind) # caches
+        ind = 0
+        for chunk in chunks:
+            fid = None
+            for grid in chunk.objs:
+                if grid.filename is None:
+                    continue
+                if fid is None:
+                    fid = h5py.h5f.open(grid.filename, h5py.h5f.ACC_RDONLY)
+                if self.pf.field_ordering == 1:
+                    # check the dtype instead
+                    data = np.empty(grid.ActiveDimensions[::-1],
+                                    dtype="float64")
+                    data_view = data.swapaxes(0, 2)
+                else:
+                    # check the dtype instead
+                    data_view = data = np.empty(grid.ActiveDimensions,
+                                                dtype="float64")
+                for field in fields:
+                    ftype, fname = field
+                    dg = h5py.h5d.open(fid, _field_dname(grid.id, fname))
+                    dg.read(h5py.h5s.ALL, h5py.h5s.ALL, data)
+                    # caches
+                    nd = grid.select(selector, data_view, rv[field], ind)
+                ind += nd    # I don't get that part, only last nd is added
+            if fid is not None:
+                fid.close()
         return rv

Repository URL: https://bitbucket.org/yt_analysis/yt-3.0/

--

This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.



More information about the yt-svn mailing list