[yt-svn] commit/yt-3.0: MatthewTurk: Converting back to hdf5_light_reader for fluid and particle IO in Enzo. Caught

Bitbucket commits-noreply at bitbucket.org
Thu Aug 9 13:05:57 PDT 2012


1 new commit in yt-3.0:


https://bitbucket.org/yt_analysis/yt-3.0/changeset/724a9fbc17aa/
changeset:   724a9fbc17aa
branch:      yt-3.0
user:        MatthewTurk
date:        2012-08-09 22:05:47
summary:     Converting back to hdf5_light_reader for fluid and particle IO in Enzo.  Caught
a missed handle.close().
affected #:  3 files

diff -r 665f55c8d8b270a9b2cc2e9d4e8671a3d3a8b944 -r 724a9fbc17aacebb698578044e17e07faee2c884 yt/data_objects/data_containers.py
--- a/yt/data_objects/data_containers.py
+++ b/yt/data_objects/data_containers.py
@@ -452,7 +452,7 @@
             inspected += 1
             if field not in self.pf.field_dependencies: continue
             fd = self.pf.field_dependencies[field]
-            requested = self._determine_fields(fd.requested)
+            requested = self._determine_fields(list(set(fd.requested)))
             deps = [d for d in requested if d not in fields_to_get]
             fields_to_get += deps
         # We now split up into readers for the types of fields


diff -r 665f55c8d8b270a9b2cc2e9d4e8671a3d3a8b944 -r 724a9fbc17aacebb698578044e17e07faee2c884 yt/frontends/enzo/io.py
--- a/yt/frontends/enzo/io.py
+++ b/yt/frontends/enzo/io.py
@@ -43,6 +43,7 @@
     def _read_data_set(self, grid, field):
         handle = h5py.File(grid.filename)
         tr = handle["/Grid%08i/%s" % (grid.id, field)][:]
+        handle.close()
         return tr.swapaxes(0, 2)
 
     def _read_field_names(self, grid):
@@ -64,86 +65,82 @@
             raise NotImplementedError("Sorry, but you'll have to manually " +
                     "discriminate without a particle_type field")
         mylog.debug("First pass: counting particles.")
-        last = chunks[0].objs[0].filename
-        handle = h5py.File(last)
         xn, yn, zn = ("particle_position_%s" % ax for ax in 'xyz')
         size = 0
+        pfields = [("all", "particle_position_%s" % ax) for ax in 'xyz']
         for chunk in chunks:
+            data = self._read_chunk_data(chunk, pfields, True)
             for g in chunk.objs:
                 if g.NumberOfParticles == 0: continue
-                if last != g.filename:
-                    last = g.filename
-                    handle = h5py.File(last)
-                base = handle["/Grid%08i" % g.id]
-                x, y, z = (base["particle_position_%s" %
-                            ax][:].astype("float64") for ax in 'xyz')
+                x, y, z = (data[g.id].pop("particle_position_%s" % ax)
+                           for ax in 'xyz')
                 size += g.count_particles(selector, x, y, z)
-            handle.close()
-        last = chunks[0].objs[0].filename
-        handle = h5py.File(last)
+        read_fields = fields[:]
         for field in fields:
             # TODO: figure out dataset types
             rv[field] = na.empty(size, dtype='float64')
+        for pfield in pfields:
+            if pfield not in fields: read_fields.append(pfield)
         ng = sum(len(c.objs) for c in chunks)
         mylog.debug("Reading %s cells of %s fields in %s grids",
                    size, [f2 for f1, f2 in fields], ng)
         ind = 0
         for chunk in chunks:
+            data = self._read_chunk_data(chunk, read_fields, True)
             for g in chunk.objs:
                 if g.NumberOfParticles == 0: continue
-                if last != g.filename:
-                    last = g.filename
-                    handle = h5py.File(last)
-                base = handle["/Grid%08i" % g.id]
-                x, y, z = (base["particle_position_%s" %
-                            ax][:].astype("float64") for ax in 'xyz')
+                x, y, z = (data[g.id]["particle_position_%s" % ax]
+                           for ax in 'xyz')
                 mask = g.select_particles(selector, x, y, z)
                 if mask is None: continue
                 for field in fields:
                     ftype, fname = field
-                    ds = handle["/Grid%08i/%s" % (g.id, fname)]
-                    data = ds[self._base][mask]
-                    data 
-                    rv[field][ind:ind+data.size] = data
-                ind += data.size
-            handle.close()
+                    gdata = data[g.id].pop(fname)[mask]
+                    rv[field][ind:ind+gdata.size] = gdata
+                ind += gdata.size
         return rv
         
     def _read_fluid_selection(self, chunks, selector, fields, size):
-        last = None
         rv = {}
         if any((ftype != "gas" for ftype, fname in fields)):
             raise NotImplementedError
-
         # Now we have to do something unpleasant
         chunks = list(chunks)
-        last = chunks[0].objs[0].filename
-        handle = h5py.File(last)
         for field in fields:
             ftype, fname = field
-            ds = handle["/Grid%08i/%s" % (chunks[0].objs[0].id, fname)]
             fsize = size
-            rv[field] = na.empty(fsize, dtype=ds.dtype)
+            rv[field] = na.empty(fsize, dtype="float64")
         ind = 0
         ng = sum(len(c.objs) for c in chunks)
         mylog.debug("Reading %s cells of %s fields in %s grids",
                    size, [f2 for f1, f2 in fields], ng)
         for chunk in chunks:
+            data = self._read_chunk_data(chunk, fields)
             for g in chunk.objs:
-                if last != g.filename:
-                    last = g.filename
-                    handle = h5py.File(last)
                 mask = g.select(selector)
                 if mask is None: continue
                 for field in fields:
                     ftype, fname = field
-                    ds = handle["/Grid%08i/%s" % (g.id, fname)]
-                    data = ds[self._base].swapaxes(0,2)[mask]
-                    rv[field][ind:ind+data.size] = data
-                ind += data.size
-            handle.close()
+                    gdata = data[g.id].pop(fname).swapaxes(0,2)[mask]
+                    rv[field][ind:ind+gdata.size] = gdata
+                ind += gdata.size
+                data.pop(g.id)
         return rv
 
+    def _read_chunk_data(self, chunk, fields, filter_particles = False):
+        data = {}
+        grids_by_file = defaultdict(list)
+        for g in chunk.objs:
+            if filter_particles and g.NumberOfParticles == 0: continue
+            grids_by_file[g.filename].append(g.id)
+        sets = [fname for ftype, fname in fields]
+        for filename in grids_by_file:
+            nodes = grids_by_file[filename]
+            nodes.sort()
+            data.update(hdf5_light_reader.ReadMultipleGrids(
+                filename, nodes, sets))
+        return data
+
 class IOHandlerPackedHDF5GhostZones(IOHandlerPackedHDF5):
     _data_style = "enzo_packed_3d_gz"
     _base = (slice(3, -3), slice(3, -3), slice(3, -3))


diff -r 665f55c8d8b270a9b2cc2e9d4e8671a3d3a8b944 -r 724a9fbc17aacebb698578044e17e07faee2c884 yt/utilities/hdf5_light_reader.c
--- a/yt/utilities/hdf5_light_reader.c
+++ b/yt/utilities/hdf5_light_reader.c
@@ -614,6 +614,7 @@
     if (file_id < 0) {
         PyErr_Format(_hdf5ReadError,
                  "ReadMultipleGrids: Unable to open %s", filename);
+        H5Eprint(stderr);
         goto _fail;
     }

Repository URL: https://bitbucket.org/yt_analysis/yt-3.0/

--

This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.



More information about the yt-svn mailing list