[yt-svn] commit/yt: xarthisius: Merged in MatthewTurk/yt/yt-3.0 (pull request #959)

[commits-noreply at bitbucket.org]

1 new commit in yt:

https://bitbucket.org/yt_analysis/yt/commits/b5463f9e432a/
Changeset:   b5463f9e432a
Branch:      yt-3.0
User:        xarthisius
Date:        2014-06-19 17:33:58
Summary:     Merged in MatthewTurk/yt/yt-3.0 (pull request #959)

Considerably reduce peak memory for Tipsy datasets.
Affected #:  1 file

diff -r 191ae91b5d908beb9156652b4e81fd68ea26f588 -r b5463f9e432ad9b1b6d712691d0d0f7b873bbd8b yt/frontends/sph/io.py
--- a/yt/frontends/sph/io.py
+++ b/yt/frontends/sph/io.py
@@ -388,6 +388,7 @@
     _ptypes = ( "Gas",
                 "DarkMatter",
                 "Stars" )
+    _chunksize = 64*64*64
 
     _aux_fields = None
     _fields = ( ("Gas", "Mass"),
@@ -502,12 +503,15 @@
             tp = data_file.total_particles
             f = open(data_file.filename, "rb")
             for ptype, field_list in sorted(ptf.items(), key=lambda a: poff[a[0]]):
-                if tp[ptype] == 0: continue
                 f.seek(poff[ptype], os.SEEK_SET)
-                p = np.fromfile(f, self._pdtypes[ptype], count=tp[ptype])
-                d = [p["Coordinates"][ax].astype("float64") for ax in 'xyz']
-                del p
-                yield ptype, d
+                total = 0
+                while total < tp[ptype]:
+                    p = np.fromfile(f, self._pdtypes[ptype],
+                            count=min(self._chunksize, tp[ptype] - total))
+                    total += p.size
+                    d = [p["Coordinates"][ax].astype("float64") for ax in 'xyz']
+                    del p
+                    yield ptype, d
 
     def _read_particle_fields(self, chunks, ptf, selector):
         chunks = list(chunks)
@@ -521,15 +525,19 @@
             f = open(data_file.filename, "rb")
             for ptype, field_list in sorted(ptf.items(), key=lambda a: poff[a[0]]):
                 f.seek(poff[ptype], os.SEEK_SET)
-                p = np.fromfile(f, self._pdtypes[ptype], count=tp[ptype])
-                mask = selector.select_points(
-                    p["Coordinates"]['x'].astype("float64"),
-                    p["Coordinates"]['y'].astype("float64"),
-                    p["Coordinates"]['z'].astype("float64"), 0.0)
-                if mask is None: continue
-                tf = self._fill_fields(field_list, p, mask, data_file)
-                for field in field_list:
-                    yield (ptype, field), tf.pop(field)
+                total = 0
+                while total < tp[ptype]:
+                    p = np.fromfile(f, self._pdtypes[ptype],
+                        count=min(self._chunksize, tp[ptype] - total))
+                    mask = selector.select_points(
+                        p["Coordinates"]['x'].astype("float64"),
+                        p["Coordinates"]['y'].astype("float64"),
+                        p["Coordinates"]['z'].astype("float64"), 0.0)
+                    if mask is None: continue
+                    tf = self._fill_fields(field_list, p, mask, data_file)
+                    for field in field_list:
+                        yield (ptype, field), tf.pop(field)
+                    total += p.size
             f.close()
 
     def _update_domain(self, data_file):

Repository URL: https://bitbucket.org/yt_analysis/yt/

--

This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.