[yt-svn] commit/yt: MatthewTurk: Merged in MatthewTurk/yt/yt-3.0 (pull request #962)

[commits-noreply at bitbucket.org]

1 new commit in yt:

https://bitbucket.org/yt_analysis/yt/commits/3e627557e0bb/
Changeset:   3e627557e0bb
Branch:      yt-3.0
User:        MatthewTurk
Date:        2014-06-20 20:08:22
Summary:     Merged in MatthewTurk/yt/yt-3.0 (pull request #962)

Reduce memory usage for grid datasets
Affected #:  3 files

diff -r 93f97b13eb7ff1509cc45457a17574c19d91f5fb -r 3e627557e0bb1f730575c4990e6fc0f940e7a147 yt/data_objects/grid_patch.py
--- a/yt/data_objects/grid_patch.py
+++ b/yt/data_objects/grid_patch.py
@@ -38,6 +38,7 @@
     _num_ghost_zones = 0
     _grids = None
     _id_offset = 1
+    _cache_mask = True
 
     _type_name = 'grid'
     _skip_add = True
@@ -351,10 +352,12 @@
         yield self, mask
 
     def _get_selector_mask(self, selector):
-        if hash(selector) == self._last_selector_id:
+        if self._cache_mask and hash(selector) == self._last_selector_id:
             mask = self._last_mask
         else:
-            self._last_mask = mask = selector.fill_mask(self)
+            mask = selector.fill_mask(self)
+            if self._cache_mask:
+                self._last_mask = mask
             self._last_selector_id = hash(selector)
             if mask is None:
                 self._last_count = 0

diff -r 93f97b13eb7ff1509cc45457a17574c19d91f5fb -r 3e627557e0bb1f730575c4990e6fc0f940e7a147 yt/geometry/grid_geometry_handler.py
--- a/yt/geometry/grid_geometry_handler.py
+++ b/yt/geometry/grid_geometry_handler.py
@@ -317,6 +317,7 @@
             # individual grids.
             yield YTDataChunk(dobj, "spatial", [g], size, cache = False)
 
+    _grid_chunksize = 1000
     def _chunk_io(self, dobj, cache = True, local_only = False):
         # local_only is only useful for inline datasets and requires
         # implementation by subclasses.
@@ -325,7 +326,14 @@
         for g in gobjs:
             gfiles[g.filename].append(g)
         for fn in sorted(gfiles):
+            # We can apply a heuristic here to make sure we aren't loading too
+            # many grids all at once.
             gs = gfiles[fn]
-            yield YTDataChunk(dobj, "io", gs, self._count_selection(dobj, gs),
-                              cache = cache)
+            size = self._grid_chunksize
+            
+            for grids in (gs[pos:pos + size] for pos
+                          in xrange(0, len(gs), size)):
+                yield YTDataChunk(dobj, "io", grids,
+                        self._count_selection(dobj, grids),
+                        cache = cache)

Repository URL: https://bitbucket.org/yt_analysis/yt/

--

This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.