[yt-svn] commit/yt-3.0: 5 new changesets
commits-noreply at bitbucket.org
commits-noreply at bitbucket.org
Tue Jul 2 08:06:06 PDT 2013
5 new commits in yt-3.0:
https://bitbucket.org/yt_analysis/yt-3.0/commits/b02f9ede5df5/
Changeset: b02f9ede5df5
Branch: yt-3.0
User: MatthewTurk
Date: 2013-07-01 21:33:22
Summary: Simplifying particle_mass for Enzo by pushing it all right into the IO routine.
This does change how Enzo's particles are regarded -- but at the benefit of
having much better IO and reduced complexity for calculating particle masses.
I believe this is worth it; we special case for other frontends, and I do not
think it is a problem to do it here.
Affected #: 2 files
diff -r 92a91071ac6b84e1008646de7f2210e6841db9f0 -r b02f9ede5df50cbc312b42e6c5565f523029ddd1 yt/frontends/enzo/fields.py
--- a/yt/frontends/enzo/fields.py
+++ b/yt/frontends/enzo/fields.py
@@ -388,14 +388,16 @@
"""
particle_field = field.name[4:]
pos = data[('all', 'Coordinates')]
+ # Get back into density
+ pden = data['all', 'particle_mass'] / data["CellVolume"]
top = data.deposit(
pos,
- [data[('all', particle_field)]*data[('all', 'particle_mass')]],
+ [data[('all', particle_field)]*pden],
method = 'cic'
)
bottom = data.deposit(
pos,
- [data[('all', 'particle_mass')]],
+ [pden],
method = 'cic'
)
top[bottom == 0] = 0.0
@@ -513,7 +515,22 @@
add_enzo_field(pf, function=NullFunc,
validators = [ValidateDataField(pf)],
particle_type=True)
-add_field(('all', "particle_mass"), function=NullFunc, particle_type=True)
+
+def _convertParticleMass(data):
+ return data.convert("Density")*(data.convert("cm")**3.0)
+def _convertParticleMassMsun(data):
+ return data.convert("Density")*((data.convert("cm")**3.0)/mass_sun_cgs)
+# We have now multiplied by grid.dds.prod() inside the IO function.
+# So here we multiply just by the conversion to density.
+add_field(('all', "particle_mass"), function=NullFunc,
+ particle_type=True, convert_function = _convertParticleMass)
+
+add_field("ParticleMass",
+ function=TranslationFunc("particle_mass"),
+ particle_type=True, convert_function=_convertParticleMass)
+add_field("ParticleMassMsun",
+ function=TranslationFunc("particle_mass"),
+ particle_type=True, convert_function=_convertParticleMassMsun)
def _ParticleAge(field, data):
current_time = data.pf.current_time
@@ -524,32 +541,6 @@
validators=[ValidateDataField("creation_time")],
particle_type=True, convert_function=_convertParticleAge)
-def _ParticleMass(field, data):
- particles = data['all', "particle_mass"].astype('float64') * \
- just_one(data["CellVolumeCode"].ravel())
- # Note that we mandate grid-type here, so this is okay
- return particles
-
-def _convertParticleMass(data):
- return data.convert("Density")*(data.convert("cm")**3.0)
-def _IOLevelParticleMass(grid):
- dd = dict(particle_mass = np.ones(1), CellVolumeCode=grid["CellVolumeCode"])
- cf = (_ParticleMass(None, dd) * _convertParticleMass(grid))[0]
- return cf
-def _convertParticleMassMsun(data):
- return data.convert("Density")*((data.convert("cm")**3.0)/1.989e33)
-def _IOLevelParticleMassMsun(grid):
- dd = dict(particle_mass = np.ones(1), CellVolumeCode=grid["CellVolumeCode"])
- cf = (_ParticleMass(None, dd) * _convertParticleMassMsun(grid))[0]
- return cf
-add_field("ParticleMass",
- function=_ParticleMass, validators=[ValidateSpatial(0)],
- particle_type=True, convert_function=_convertParticleMass,
- particle_convert_function=_IOLevelParticleMass)
-add_field("ParticleMassMsun",
- function=_ParticleMass, validators=[ValidateSpatial(0)],
- particle_type=True, convert_function=_convertParticleMassMsun,
- particle_convert_function=_IOLevelParticleMassMsun)
#
# Now we do overrides for 2D fields
diff -r 92a91071ac6b84e1008646de7f2210e6841db9f0 -r b02f9ede5df50cbc312b42e6c5565f523029ddd1 yt/frontends/enzo/io.py
--- a/yt/frontends/enzo/io.py
+++ b/yt/frontends/enzo/io.py
@@ -36,6 +36,8 @@
import numpy as np
from yt.funcs import *
+_convert_mass = ("particle_mass",)
+
class IOHandlerPackedHDF5(BaseIOHandler):
_data_style = "enzo_packed_3d"
@@ -81,6 +83,8 @@
for field in set(fields):
ftype, fname = field
gdata = data[g.id].pop(fname)[mask]
+ if fname == "particle_mass":
+ gdata *= g.dds.prod()
rv[field][ind:ind+gdata.size] = gdata
ind += gdata.size
data.pop(g.id)
@@ -130,6 +134,8 @@
for field in set(fields):
ftype, fname = field
gdata = data[g.id].pop(fname)[mask]
+ if fname == "particle_mass":
+ gdata *= g.dds.prod()
rv[field][ind:ind+gdata.size] = gdata
ind += gdata.size
return rv
https://bitbucket.org/yt_analysis/yt-3.0/commits/4452846ad4b6/
Changeset: 4452846ad4b6
Branch: yt-3.0
User: MatthewTurk
Date: 2013-07-01 23:11:41
Summary: Remove usage of alloca in QuadTree in favor of explicit malloc/free.
Affected #: 1 file
diff -r b02f9ede5df50cbc312b42e6c5565f523029ddd1 -r 4452846ad4b6c93c9f304b82afe85b970e593c5e yt/utilities/lib/QuadTree.pyx
--- a/yt/utilities/lib/QuadTree.pyx
+++ b/yt/utilities/lib/QuadTree.pyx
@@ -131,7 +131,7 @@
cdef int i, j
cdef QuadTreeNode *node
cdef np.int64_t pos[2]
- cdef np.float64_t *vals = <np.float64_t *> alloca(
+ cdef np.float64_t *vals = <np.float64_t *> malloc(
sizeof(np.float64_t)*nvals)
cdef np.float64_t weight_val = 0.0
self.nvals = nvals
@@ -160,6 +160,7 @@
self.root_nodes[i][j] = QTN_initialize(
pos, nvals, vals, weight_val)
self.num_cells = self.top_grid_dims[0] * self.top_grid_dims[1]
+ free(vals)
cdef int count_total_cells(self, QuadTreeNode *root):
cdef int total = 0
@@ -373,7 +374,7 @@
cdef np.float64_t *vdata = <np.float64_t *> nvals.data
cdef np.float64_t *wdata = <np.float64_t *> nwvals.data
cdef np.float64_t wtoadd
- cdef np.float64_t *vtoadd = <np.float64_t *> alloca(
+ cdef np.float64_t *vtoadd = <np.float64_t *> malloc(
sizeof(np.float64_t)*self.nvals)
for i in range(self.top_grid_dims[0]):
for j in range(self.top_grid_dims[1]):
@@ -381,6 +382,7 @@
wtoadd = 0.0
curpos += self.fill(self.root_nodes[i][j],
curpos, px, py, pdx, pdy, vdata, wdata, vtoadd, wtoadd, 0)
+ free(vtoadd)
return opx, opy, opdx, opdy, nvals, nwvals
cdef int count(self, QuadTreeNode *node):
@@ -406,7 +408,7 @@
np.int64_t level):
cdef int i, j, n
cdef np.float64_t *vorig
- vorig = <np.float64_t *> alloca(sizeof(np.float64_t) * self.nvals)
+ vorig = <np.float64_t *> malloc(sizeof(np.float64_t) * self.nvals)
if node.children[0][0] == NULL:
if self.merged == -1:
for i in range(self.nvals):
@@ -444,6 +446,7 @@
for i in range(self.nvals):
vtoadd[i] = vorig[i]
wtoadd -= node.weight_val
+ free(vorig)
return added
@cython.boundscheck(False)
https://bitbucket.org/yt_analysis/yt-3.0/commits/0f91890a2fad/
Changeset: 0f91890a2fad
Branch: yt-3.0
User: MatthewTurk
Date: 2013-07-01 23:15:40
Summary: Adding cached_property decorator for chunks.
This allows YTDataChunk objects to only optionally track results of operations
like fcoords, fwidth, etc.
Affected #: 1 file
diff -r 4452846ad4b6c93c9f304b82afe85b970e593c5e -r 0f91890a2fad5813b6085198a1316d00459c8be6 yt/geometry/geometry_handler.py
--- a/yt/geometry/geometry_handler.py
+++ b/yt/geometry/geometry_handler.py
@@ -453,20 +453,30 @@
else:
raise NotImplementedError
+def cached_property(func):
+ n = '_%s' % func.func_name
+ def cached_func(self):
+ if self._cache and getattr(self, n, None) is not None:
+ return getattr(self, n)
+ if self.data_size is None:
+ tr = self._accumulate_values(n[1:])
+ else:
+ tr = func(self)
+ if self._cache:
+ setattr(self, n, tr)
+ return tr
+ return property(cached_func)
+
class YTDataChunk(object):
- def __init__(self, dobj, chunk_type, objs, data_size = None, field_type = None):
+ def __init__(self, dobj, chunk_type, objs, data_size = None,
+ field_type = None, cache = True):
self.dobj = dobj
self.chunk_type = chunk_type
self.objs = objs
- self._data_size = data_size
+ self.data_size = data_size
self._field_type = field_type
-
- @property
- def data_size(self):
- if callable(self._data_size):
- self._data_size = self._data_size(self.dobj, self.objs)
- return self._data_size
+ self._cache = cache
def _accumulate_values(self, method):
# We call this generically. It's somewhat slower, since we're doing
@@ -477,35 +487,25 @@
f = getattr(obj, mname)
arrs.append(f(self.dobj))
arrs = np.concatenate(arrs)
- self._data_size = arrs.shape[0]
+ self.data_size = arrs.shape[0]
return arrs
- _fcoords = None
- @property
+ @cached_property
def fcoords(self):
- if self.data_size is None:
- self._fcoords = self._accumulate_values("fcoords")
- if self._fcoords is not None: return self._fcoords
ci = np.empty((self.data_size, 3), dtype='float64')
- self._fcoords = ci
- if self.data_size == 0: return self._fcoords
+ if self.data_size == 0: return ci
ind = 0
for obj in self.objs:
c = obj.select_fcoords(self.dobj)
if c.shape[0] == 0: continue
ci[ind:ind+c.shape[0], :] = c
ind += c.shape[0]
- return self._fcoords
+ return ci
- _icoords = None
- @property
+ @cached_property
def icoords(self):
- if self.data_size is None:
- self._icoords = self._accumulate_values("icoords")
- if self._icoords is not None: return self._icoords
ci = np.empty((self.data_size, 3), dtype='int64')
- self._icoords = ci
- if self.data_size == 0: return self._icoords
+ if self.data_size == 0: return ci
ind = 0
for obj in self.objs:
c = obj.select_icoords(self.dobj)
@@ -514,15 +514,10 @@
ind += c.shape[0]
return ci
- _fwidth = None
- @property
+ @cached_property
def fwidth(self):
- if self.data_size is None:
- self._fwidth = self._accumulate_values("fwidth")
- if self._fwidth is not None: return self._fwidth
ci = np.empty((self.data_size, 3), dtype='float64')
- self._fwidth = ci
- if self.data_size == 0: return self._fwidth
+ if self.data_size == 0: return ci
ind = 0
for obj in self.objs:
c = obj.select_fwidth(self.dobj)
@@ -531,15 +526,10 @@
ind += c.shape[0]
return ci
- _ires = None
- @property
+ @cached_property
def ires(self):
- if self.data_size is None:
- self._ires = self._accumulate_values("ires")
- if self._ires is not None: return self._ires
ci = np.empty(self.data_size, dtype='int64')
- self._ires = ci
- if self.data_size == 0: return self._ires
+ if self.data_size == 0: return ci
ind = 0
for obj in self.objs:
c = obj.select_ires(self.dobj)
@@ -548,22 +538,17 @@
ind += c.size
return ci
- _tcoords = None
- @property
+ @cached_property
def tcoords(self):
- if self._tcoords is None:
- self.dtcoords
+ self.dtcoords
return self._tcoords
- _dtcoords = None
- @property
+ @cached_property
def dtcoords(self):
- if self._dtcoords is not None: return self._dtcoords
ct = np.empty(self.data_size, dtype='float64')
cdt = np.empty(self.data_size, dtype='float64')
- self._tcoords = ct
- self._dtcoords = cdt
- if self.data_size == 0: return self._dtcoords
+ self._tcoords = ct # Se this for tcoords
+ if self.data_size == 0: return cdt
ind = 0
for obj in self.objs:
gdt, gt = obj.tcoords(self.dobj)
https://bitbucket.org/yt_analysis/yt-3.0/commits/2b865392b01b/
Changeset: 2b865392b01b
Branch: yt-3.0
User: MatthewTurk
Date: 2013-07-01 23:39:10
Summary: Thread cache arguments through the various frontends.
Note that by default, caching is off, but we turn it on when explicitly asking
for an 'all'. I think this will replicate the behavior users expect. However,
one thing to keep a look out for is the relative performance of various
operations, and I think we will also want to consider adding either a heuristic
or a contextmanager that will preserve and traverse simultaneously for the
various coordinates. This would be helpful, for instance, in simulations like
ARTIO where the traversal could potentially be expensive.
This largely completes the enzodiet, as memory usage is now down considerably
-- although not completely -- in my tests.
Affected #: 9 files
diff -r 0f91890a2fad5813b6085198a1316d00459c8be6 -r 2b865392b01b1f5458947d629dee2707c8140765 yt/data_objects/construction_data_containers.py
--- a/yt/data_objects/construction_data_containers.py
+++ b/yt/data_objects/construction_data_containers.py
@@ -286,7 +286,8 @@
# This needs to be parallel_objects-ified
for chunk in parallel_objects(self.data_source.chunks(
chunk_fields, "io")):
- mylog.debug("Adding chunk (%s) to tree", chunk.ires.size)
+ mylog.debug("Adding chunk (%s) to tree (%0.3e GB RAM)", chunk.ires.size,
+ get_memory_usage()/1024.)
self._handle_chunk(chunk, fields, tree)
# Note that this will briefly double RAM usage
if self.proj_style == "mip":
diff -r 0f91890a2fad5813b6085198a1316d00459c8be6 -r 2b865392b01b1f5458947d629dee2707c8140765 yt/data_objects/data_containers.py
--- a/yt/data_objects/data_containers.py
+++ b/yt/data_objects/data_containers.py
@@ -246,7 +246,7 @@
rv = np.empty(self.ires.size, dtype="float64")
ind = 0
if ngz == 0:
- for io_chunk in self.chunks([], "io"):
+ for io_chunk in self.chunks([], "io", cache = False):
for i,chunk in enumerate(self.chunks(field, "spatial", ngz = 0)):
ind += self._current_chunk.objs[0].select(
self.selector, self[field], rv, ind)
@@ -279,8 +279,8 @@
size = self._count_particles(ftype)
rv = np.empty(size, dtype="float64")
ind = 0
- for io_chunk in self.chunks([], "io"):
- for i,chunk in enumerate(self.chunks(field, "spatial")):
+ for io_chunk in self.chunks([], "io", cache = False):
+ for i, chunk in enumerate(self.chunks(field, "spatial")):
x, y, z = (self[ftype, 'particle_position_%s' % ax]
for ax in 'xyz')
if x.size == 0: continue
@@ -301,7 +301,7 @@
if f1 == ftype:
return val.size
size = 0
- for io_chunk in self.chunks([], "io"):
+ for io_chunk in self.chunks([], "io", cache = False):
for i,chunk in enumerate(self.chunks([], "spatial")):
x, y, z = (self[ftype, 'particle_position_%s' % ax]
for ax in 'xyz')
diff -r 0f91890a2fad5813b6085198a1316d00459c8be6 -r 2b865392b01b1f5458947d629dee2707c8140765 yt/frontends/art/data_structures.py
--- a/yt/frontends/art/data_structures.py
+++ b/yt/frontends/art/data_structures.py
@@ -180,7 +180,7 @@
g = og
yield YTDataChunk(dobj, "spatial", [g], None)
- def _chunk_io(self, dobj):
+ def _chunk_io(self, dobj, cache = True):
"""
Since subsets are calculated per domain,
i.e. per file, yield each domain at a time to
@@ -189,7 +189,8 @@
"""
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
for subset in oobjs:
- yield YTDataChunk(dobj, "io", [subset], None)
+ yield YTDataChunk(dobj, "io", [subset], None,
+ cache = cache)
class ARTStaticOutput(StaticOutput):
diff -r 0f91890a2fad5813b6085198a1316d00459c8be6 -r 2b865392b01b1f5458947d629dee2707c8140765 yt/frontends/artio/data_structures.py
--- a/yt/frontends/artio/data_structures.py
+++ b/yt/frontends/artio/data_structures.py
@@ -232,11 +232,12 @@
def _chunk_spatial(self, dobj, ngz):
raise NotImplementedError
- def _chunk_io(self, dobj):
+ def _chunk_io(self, dobj, cache = True):
# _current_chunk is made from identify_base_chunk
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
for chunk in oobjs:
- yield YTDataChunk(dobj, "io", [chunk], self._data_size)
+ yield YTDataChunk(dobj, "io", [chunk], self._data_size,
+ cache = cache)
def _read_fluid_fields(self, fields, dobj, chunk=None):
if len(fields) == 0:
diff -r 0f91890a2fad5813b6085198a1316d00459c8be6 -r 2b865392b01b1f5458947d629dee2707c8140765 yt/frontends/flash/data_structures.py
--- a/yt/frontends/flash/data_structures.py
+++ b/yt/frontends/flash/data_structures.py
@@ -212,12 +212,13 @@
def _setup_data_io(self):
self.io = io_registry[self.data_style](self.parameter_file)
- def _chunk_io(self, dobj):
+ def _chunk_io(self, dobj, cache = True):
gobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
# We'll take the max of 128 and the number of processors
nl = max(16, ytcfg.getint("yt", "__topcomm_parallel_size"))
for gs in list_chunks(gobjs, nl):
- yield YTDataChunk(dobj, "io", gs, self._count_selection)
+ yield YTDataChunk(dobj, "io", gs, self._count_selection,
+ cache = cache)
class FLASHStaticOutput(StaticOutput):
_hierarchy_class = FLASHHierarchy
diff -r 0f91890a2fad5813b6085198a1316d00459c8be6 -r 2b865392b01b1f5458947d629dee2707c8140765 yt/frontends/ramses/data_structures.py
--- a/yt/frontends/ramses/data_structures.py
+++ b/yt/frontends/ramses/data_structures.py
@@ -348,10 +348,10 @@
g = og
yield YTDataChunk(dobj, "spatial", [g], None)
- def _chunk_io(self, dobj):
+ def _chunk_io(self, dobj, cache = True):
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
for subset in oobjs:
- yield YTDataChunk(dobj, "io", [subset], None)
+ yield YTDataChunk(dobj, "io", [subset], None, cache = cache)
class RAMSESStaticOutput(StaticOutput):
_hierarchy_class = RAMSESGeometryHandler
diff -r 0f91890a2fad5813b6085198a1316d00459c8be6 -r 2b865392b01b1f5458947d629dee2707c8140765 yt/geometry/geometry_handler.py
--- a/yt/geometry/geometry_handler.py
+++ b/yt/geometry/geometry_handler.py
@@ -403,8 +403,9 @@
if len(fields_to_read) == 0:
return {}, fields_to_generate
fields_to_return = self.io._read_particle_selection(
- self._chunk_io(dobj), selector,
- fields_to_read)
+ self._chunk_io(dobj, cache = False),
+ selector,
+ fields_to_read)
for field in fields_to_read:
ftype, fname = field
finfo = self.pf._get_field_info(*field)
@@ -425,10 +426,11 @@
fields_to_read, fields_to_generate = self._split_fields(fields)
if len(fields_to_read) == 0:
return {}, fields_to_generate
- fields_to_return = self.io._read_fluid_selection(self._chunk_io(dobj),
- selector,
- fields_to_read,
- chunk_size)
+ fields_to_return = self.io._read_fluid_selection(
+ self._chunk_io(dobj, cache = False),
+ selector,
+ fields_to_read,
+ chunk_size)
for field in fields_to_read:
ftype, fname = field
conv_factor = self.pf.field_info[fname]._convert_function(self)
@@ -470,7 +472,7 @@
class YTDataChunk(object):
def __init__(self, dobj, chunk_type, objs, data_size = None,
- field_type = None, cache = True):
+ field_type = None, cache = False):
self.dobj = dobj
self.chunk_type = chunk_type
self.objs = objs
diff -r 0f91890a2fad5813b6085198a1316d00459c8be6 -r 2b865392b01b1f5458947d629dee2707c8140765 yt/geometry/grid_geometry_handler.py
--- a/yt/geometry/grid_geometry_handler.py
+++ b/yt/geometry/grid_geometry_handler.py
@@ -245,16 +245,16 @@
dobj.size = self._count_selection(dobj)
if getattr(dobj, "shape", None) is None:
dobj.shape = (dobj.size,)
- dobj._current_chunk = list(self._chunk_all(dobj))[0]
+ dobj._current_chunk = list(self._chunk_all(dobj, cache = False))[0]
def _count_selection(self, dobj, grids = None):
if grids is None: grids = dobj._chunk_info
count = sum((g.count(dobj.selector) for g in grids))
return count
- def _chunk_all(self, dobj):
+ def _chunk_all(self, dobj, cache = True):
gobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
- yield YTDataChunk(dobj, "all", gobjs, dobj.size)
+ yield YTDataChunk(dobj, "all", gobjs, dobj.size, cache)
def _chunk_spatial(self, dobj, ngz, sort = None):
gobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
@@ -271,13 +271,16 @@
g = og
size = self._count_selection(dobj, [og])
if size == 0: continue
- yield YTDataChunk(dobj, "spatial", [g], size)
+ # We don't want to cache any of the masks or icoords or fcoords for
+ # individual grids.
+ yield YTDataChunk(dobj, "spatial", [g], size, cache = False)
- def _chunk_io(self, dobj):
+ def _chunk_io(self, dobj, cache = True):
gfiles = defaultdict(list)
gobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
for g in gobjs:
gfiles[g.filename].append(g)
for fn in sorted(gfiles):
gs = gfiles[fn]
- yield YTDataChunk(dobj, "io", gs, self._count_selection(dobj, gs))
+ yield YTDataChunk(dobj, "io", gs, self._count_selection(dobj, gs),
+ cache = cache)
diff -r 0f91890a2fad5813b6085198a1316d00459c8be6 -r 2b865392b01b1f5458947d629dee2707c8140765 yt/geometry/particle_geometry_handler.py
--- a/yt/geometry/particle_geometry_handler.py
+++ b/yt/geometry/particle_geometry_handler.py
@@ -170,10 +170,10 @@
g = og
yield YTDataChunk(dobj, "spatial", [g])
- def _chunk_io(self, dobj):
+ def _chunk_io(self, dobj, cache = True):
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
for subset in oobjs:
- yield YTDataChunk(dobj, "io", [subset], None)
+ yield YTDataChunk(dobj, "io", [subset], None, cache = cache)
class ParticleDataChunk(YTDataChunk):
def __init__(self, oct_handler, regions, *args, **kwargs):
https://bitbucket.org/yt_analysis/yt-3.0/commits/67d98f2ef85c/
Changeset: 67d98f2ef85c
Branch: yt-3.0
User: ngoldbaum
Date: 2013-07-02 17:06:01
Summary: Merged in MatthewTurk/yt-3.0 (pull request #55)
Reduce overall memory usage and set Enzo particle_mass to be mass always
Affected #: 12 files
diff -r 294e62e00fa88b8aa3a9e35875178d70aecc964c -r 67d98f2ef85c7831a1f5bb0c640b06ed89ea151a yt/data_objects/construction_data_containers.py
--- a/yt/data_objects/construction_data_containers.py
+++ b/yt/data_objects/construction_data_containers.py
@@ -286,7 +286,8 @@
# This needs to be parallel_objects-ified
for chunk in parallel_objects(self.data_source.chunks(
chunk_fields, "io")):
- mylog.debug("Adding chunk (%s) to tree", chunk.ires.size)
+ mylog.debug("Adding chunk (%s) to tree (%0.3e GB RAM)", chunk.ires.size,
+ get_memory_usage()/1024.)
self._handle_chunk(chunk, fields, tree)
# Note that this will briefly double RAM usage
if self.proj_style == "mip":
diff -r 294e62e00fa88b8aa3a9e35875178d70aecc964c -r 67d98f2ef85c7831a1f5bb0c640b06ed89ea151a yt/data_objects/data_containers.py
--- a/yt/data_objects/data_containers.py
+++ b/yt/data_objects/data_containers.py
@@ -246,7 +246,7 @@
rv = np.empty(self.ires.size, dtype="float64")
ind = 0
if ngz == 0:
- for io_chunk in self.chunks([], "io"):
+ for io_chunk in self.chunks([], "io", cache = False):
for i,chunk in enumerate(self.chunks(field, "spatial", ngz = 0)):
ind += self._current_chunk.objs[0].select(
self.selector, self[field], rv, ind)
@@ -279,8 +279,8 @@
size = self._count_particles(ftype)
rv = np.empty(size, dtype="float64")
ind = 0
- for io_chunk in self.chunks([], "io"):
- for i,chunk in enumerate(self.chunks(field, "spatial")):
+ for io_chunk in self.chunks([], "io", cache = False):
+ for i, chunk in enumerate(self.chunks(field, "spatial")):
x, y, z = (self[ftype, 'particle_position_%s' % ax]
for ax in 'xyz')
if x.size == 0: continue
@@ -301,7 +301,7 @@
if f1 == ftype:
return val.size
size = 0
- for io_chunk in self.chunks([], "io"):
+ for io_chunk in self.chunks([], "io", cache = False):
for i,chunk in enumerate(self.chunks([], "spatial")):
x, y, z = (self[ftype, 'particle_position_%s' % ax]
for ax in 'xyz')
diff -r 294e62e00fa88b8aa3a9e35875178d70aecc964c -r 67d98f2ef85c7831a1f5bb0c640b06ed89ea151a yt/frontends/art/data_structures.py
--- a/yt/frontends/art/data_structures.py
+++ b/yt/frontends/art/data_structures.py
@@ -180,7 +180,7 @@
g = og
yield YTDataChunk(dobj, "spatial", [g], None)
- def _chunk_io(self, dobj):
+ def _chunk_io(self, dobj, cache = True):
"""
Since subsets are calculated per domain,
i.e. per file, yield each domain at a time to
@@ -189,7 +189,8 @@
"""
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
for subset in oobjs:
- yield YTDataChunk(dobj, "io", [subset], None)
+ yield YTDataChunk(dobj, "io", [subset], None,
+ cache = cache)
class ARTStaticOutput(StaticOutput):
diff -r 294e62e00fa88b8aa3a9e35875178d70aecc964c -r 67d98f2ef85c7831a1f5bb0c640b06ed89ea151a yt/frontends/artio/data_structures.py
--- a/yt/frontends/artio/data_structures.py
+++ b/yt/frontends/artio/data_structures.py
@@ -232,11 +232,12 @@
def _chunk_spatial(self, dobj, ngz):
raise NotImplementedError
- def _chunk_io(self, dobj):
+ def _chunk_io(self, dobj, cache = True):
# _current_chunk is made from identify_base_chunk
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
for chunk in oobjs:
- yield YTDataChunk(dobj, "io", [chunk], self._data_size)
+ yield YTDataChunk(dobj, "io", [chunk], self._data_size,
+ cache = cache)
def _read_fluid_fields(self, fields, dobj, chunk=None):
if len(fields) == 0:
diff -r 294e62e00fa88b8aa3a9e35875178d70aecc964c -r 67d98f2ef85c7831a1f5bb0c640b06ed89ea151a yt/frontends/enzo/fields.py
--- a/yt/frontends/enzo/fields.py
+++ b/yt/frontends/enzo/fields.py
@@ -388,14 +388,16 @@
"""
particle_field = field.name[4:]
pos = data[('all', 'Coordinates')]
+ # Get back into density
+ pden = data['all', 'particle_mass'] / data["CellVolume"]
top = data.deposit(
pos,
- [data[('all', particle_field)]*data[('all', 'particle_mass')]],
+ [data[('all', particle_field)]*pden],
method = 'cic'
)
bottom = data.deposit(
pos,
- [data[('all', 'particle_mass')]],
+ [pden],
method = 'cic'
)
top[bottom == 0] = 0.0
@@ -513,7 +515,22 @@
add_enzo_field(pf, function=NullFunc,
validators = [ValidateDataField(pf)],
particle_type=True)
-add_field(('all', "particle_mass"), function=NullFunc, particle_type=True)
+
+def _convertParticleMass(data):
+ return data.convert("Density")*(data.convert("cm")**3.0)
+def _convertParticleMassMsun(data):
+ return data.convert("Density")*((data.convert("cm")**3.0)/mass_sun_cgs)
+# We have now multiplied by grid.dds.prod() inside the IO function.
+# So here we multiply just by the conversion to density.
+add_field(('all', "particle_mass"), function=NullFunc,
+ particle_type=True, convert_function = _convertParticleMass)
+
+add_field("ParticleMass",
+ function=TranslationFunc("particle_mass"),
+ particle_type=True, convert_function=_convertParticleMass)
+add_field("ParticleMassMsun",
+ function=TranslationFunc("particle_mass"),
+ particle_type=True, convert_function=_convertParticleMassMsun)
def _ParticleAge(field, data):
current_time = data.pf.current_time
@@ -524,32 +541,6 @@
validators=[ValidateDataField("creation_time")],
particle_type=True, convert_function=_convertParticleAge)
-def _ParticleMass(field, data):
- particles = data['all', "particle_mass"].astype('float64') * \
- just_one(data["CellVolumeCode"].ravel())
- # Note that we mandate grid-type here, so this is okay
- return particles
-
-def _convertParticleMass(data):
- return data.convert("Density")*(data.convert("cm")**3.0)
-def _IOLevelParticleMass(grid):
- dd = dict(particle_mass = np.ones(1), CellVolumeCode=grid["CellVolumeCode"])
- cf = (_ParticleMass(None, dd) * _convertParticleMass(grid))[0]
- return cf
-def _convertParticleMassMsun(data):
- return data.convert("Density")*((data.convert("cm")**3.0)/1.989e33)
-def _IOLevelParticleMassMsun(grid):
- dd = dict(particle_mass = np.ones(1), CellVolumeCode=grid["CellVolumeCode"])
- cf = (_ParticleMass(None, dd) * _convertParticleMassMsun(grid))[0]
- return cf
-add_field("ParticleMass",
- function=_ParticleMass, validators=[ValidateSpatial(0)],
- particle_type=True, convert_function=_convertParticleMass,
- particle_convert_function=_IOLevelParticleMass)
-add_field("ParticleMassMsun",
- function=_ParticleMass, validators=[ValidateSpatial(0)],
- particle_type=True, convert_function=_convertParticleMassMsun,
- particle_convert_function=_IOLevelParticleMassMsun)
#
# Now we do overrides for 2D fields
diff -r 294e62e00fa88b8aa3a9e35875178d70aecc964c -r 67d98f2ef85c7831a1f5bb0c640b06ed89ea151a yt/frontends/enzo/io.py
--- a/yt/frontends/enzo/io.py
+++ b/yt/frontends/enzo/io.py
@@ -36,6 +36,8 @@
import numpy as np
from yt.funcs import *
+_convert_mass = ("particle_mass",)
+
class IOHandlerPackedHDF5(BaseIOHandler):
_data_style = "enzo_packed_3d"
@@ -81,6 +83,8 @@
for field in set(fields):
ftype, fname = field
gdata = data[g.id].pop(fname)[mask]
+ if fname == "particle_mass":
+ gdata *= g.dds.prod()
rv[field][ind:ind+gdata.size] = gdata
ind += gdata.size
data.pop(g.id)
@@ -130,6 +134,8 @@
for field in set(fields):
ftype, fname = field
gdata = data[g.id].pop(fname)[mask]
+ if fname == "particle_mass":
+ gdata *= g.dds.prod()
rv[field][ind:ind+gdata.size] = gdata
ind += gdata.size
return rv
diff -r 294e62e00fa88b8aa3a9e35875178d70aecc964c -r 67d98f2ef85c7831a1f5bb0c640b06ed89ea151a yt/frontends/flash/data_structures.py
--- a/yt/frontends/flash/data_structures.py
+++ b/yt/frontends/flash/data_structures.py
@@ -212,12 +212,13 @@
def _setup_data_io(self):
self.io = io_registry[self.data_style](self.parameter_file)
- def _chunk_io(self, dobj):
+ def _chunk_io(self, dobj, cache = True):
gobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
# We'll take the max of 128 and the number of processors
nl = max(16, ytcfg.getint("yt", "__topcomm_parallel_size"))
for gs in list_chunks(gobjs, nl):
- yield YTDataChunk(dobj, "io", gs, self._count_selection)
+ yield YTDataChunk(dobj, "io", gs, self._count_selection,
+ cache = cache)
class FLASHStaticOutput(StaticOutput):
_hierarchy_class = FLASHHierarchy
diff -r 294e62e00fa88b8aa3a9e35875178d70aecc964c -r 67d98f2ef85c7831a1f5bb0c640b06ed89ea151a yt/frontends/ramses/data_structures.py
--- a/yt/frontends/ramses/data_structures.py
+++ b/yt/frontends/ramses/data_structures.py
@@ -348,10 +348,10 @@
g = og
yield YTDataChunk(dobj, "spatial", [g], None)
- def _chunk_io(self, dobj):
+ def _chunk_io(self, dobj, cache = True):
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
for subset in oobjs:
- yield YTDataChunk(dobj, "io", [subset], None)
+ yield YTDataChunk(dobj, "io", [subset], None, cache = cache)
class RAMSESStaticOutput(StaticOutput):
_hierarchy_class = RAMSESGeometryHandler
diff -r 294e62e00fa88b8aa3a9e35875178d70aecc964c -r 67d98f2ef85c7831a1f5bb0c640b06ed89ea151a yt/geometry/geometry_handler.py
--- a/yt/geometry/geometry_handler.py
+++ b/yt/geometry/geometry_handler.py
@@ -403,8 +403,9 @@
if len(fields_to_read) == 0:
return {}, fields_to_generate
fields_to_return = self.io._read_particle_selection(
- self._chunk_io(dobj), selector,
- fields_to_read)
+ self._chunk_io(dobj, cache = False),
+ selector,
+ fields_to_read)
for field in fields_to_read:
ftype, fname = field
finfo = self.pf._get_field_info(*field)
@@ -425,10 +426,11 @@
fields_to_read, fields_to_generate = self._split_fields(fields)
if len(fields_to_read) == 0:
return {}, fields_to_generate
- fields_to_return = self.io._read_fluid_selection(self._chunk_io(dobj),
- selector,
- fields_to_read,
- chunk_size)
+ fields_to_return = self.io._read_fluid_selection(
+ self._chunk_io(dobj, cache = False),
+ selector,
+ fields_to_read,
+ chunk_size)
for field in fields_to_read:
ftype, fname = field
conv_factor = self.pf.field_info[fname]._convert_function(self)
@@ -453,20 +455,30 @@
else:
raise NotImplementedError
+def cached_property(func):
+ n = '_%s' % func.func_name
+ def cached_func(self):
+ if self._cache and getattr(self, n, None) is not None:
+ return getattr(self, n)
+ if self.data_size is None:
+ tr = self._accumulate_values(n[1:])
+ else:
+ tr = func(self)
+ if self._cache:
+ setattr(self, n, tr)
+ return tr
+ return property(cached_func)
+
class YTDataChunk(object):
- def __init__(self, dobj, chunk_type, objs, data_size = None, field_type = None):
+ def __init__(self, dobj, chunk_type, objs, data_size = None,
+ field_type = None, cache = False):
self.dobj = dobj
self.chunk_type = chunk_type
self.objs = objs
- self._data_size = data_size
+ self.data_size = data_size
self._field_type = field_type
-
- @property
- def data_size(self):
- if callable(self._data_size):
- self._data_size = self._data_size(self.dobj, self.objs)
- return self._data_size
+ self._cache = cache
def _accumulate_values(self, method):
# We call this generically. It's somewhat slower, since we're doing
@@ -477,35 +489,25 @@
f = getattr(obj, mname)
arrs.append(f(self.dobj))
arrs = np.concatenate(arrs)
- self._data_size = arrs.shape[0]
+ self.data_size = arrs.shape[0]
return arrs
- _fcoords = None
- @property
+ @cached_property
def fcoords(self):
- if self.data_size is None:
- self._fcoords = self._accumulate_values("fcoords")
- if self._fcoords is not None: return self._fcoords
ci = np.empty((self.data_size, 3), dtype='float64')
- self._fcoords = ci
- if self.data_size == 0: return self._fcoords
+ if self.data_size == 0: return ci
ind = 0
for obj in self.objs:
c = obj.select_fcoords(self.dobj)
if c.shape[0] == 0: continue
ci[ind:ind+c.shape[0], :] = c
ind += c.shape[0]
- return self._fcoords
+ return ci
- _icoords = None
- @property
+ @cached_property
def icoords(self):
- if self.data_size is None:
- self._icoords = self._accumulate_values("icoords")
- if self._icoords is not None: return self._icoords
ci = np.empty((self.data_size, 3), dtype='int64')
- self._icoords = ci
- if self.data_size == 0: return self._icoords
+ if self.data_size == 0: return ci
ind = 0
for obj in self.objs:
c = obj.select_icoords(self.dobj)
@@ -514,15 +516,10 @@
ind += c.shape[0]
return ci
- _fwidth = None
- @property
+ @cached_property
def fwidth(self):
- if self.data_size is None:
- self._fwidth = self._accumulate_values("fwidth")
- if self._fwidth is not None: return self._fwidth
ci = np.empty((self.data_size, 3), dtype='float64')
- self._fwidth = ci
- if self.data_size == 0: return self._fwidth
+ if self.data_size == 0: return ci
ind = 0
for obj in self.objs:
c = obj.select_fwidth(self.dobj)
@@ -531,15 +528,10 @@
ind += c.shape[0]
return ci
- _ires = None
- @property
+ @cached_property
def ires(self):
- if self.data_size is None:
- self._ires = self._accumulate_values("ires")
- if self._ires is not None: return self._ires
ci = np.empty(self.data_size, dtype='int64')
- self._ires = ci
- if self.data_size == 0: return self._ires
+ if self.data_size == 0: return ci
ind = 0
for obj in self.objs:
c = obj.select_ires(self.dobj)
@@ -548,22 +540,17 @@
ind += c.size
return ci
- _tcoords = None
- @property
+ @cached_property
def tcoords(self):
- if self._tcoords is None:
- self.dtcoords
+ self.dtcoords
return self._tcoords
- _dtcoords = None
- @property
+ @cached_property
def dtcoords(self):
- if self._dtcoords is not None: return self._dtcoords
ct = np.empty(self.data_size, dtype='float64')
cdt = np.empty(self.data_size, dtype='float64')
- self._tcoords = ct
- self._dtcoords = cdt
- if self.data_size == 0: return self._dtcoords
+ self._tcoords = ct # Se this for tcoords
+ if self.data_size == 0: return cdt
ind = 0
for obj in self.objs:
gdt, gt = obj.tcoords(self.dobj)
diff -r 294e62e00fa88b8aa3a9e35875178d70aecc964c -r 67d98f2ef85c7831a1f5bb0c640b06ed89ea151a yt/geometry/grid_geometry_handler.py
--- a/yt/geometry/grid_geometry_handler.py
+++ b/yt/geometry/grid_geometry_handler.py
@@ -245,16 +245,16 @@
dobj.size = self._count_selection(dobj)
if getattr(dobj, "shape", None) is None:
dobj.shape = (dobj.size,)
- dobj._current_chunk = list(self._chunk_all(dobj))[0]
+ dobj._current_chunk = list(self._chunk_all(dobj, cache = False))[0]
def _count_selection(self, dobj, grids = None):
if grids is None: grids = dobj._chunk_info
count = sum((g.count(dobj.selector) for g in grids))
return count
- def _chunk_all(self, dobj):
+ def _chunk_all(self, dobj, cache = True):
gobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
- yield YTDataChunk(dobj, "all", gobjs, dobj.size)
+ yield YTDataChunk(dobj, "all", gobjs, dobj.size, cache)
def _chunk_spatial(self, dobj, ngz, sort = None):
gobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
@@ -271,13 +271,16 @@
g = og
size = self._count_selection(dobj, [og])
if size == 0: continue
- yield YTDataChunk(dobj, "spatial", [g], size)
+ # We don't want to cache any of the masks or icoords or fcoords for
+ # individual grids.
+ yield YTDataChunk(dobj, "spatial", [g], size, cache = False)
- def _chunk_io(self, dobj):
+ def _chunk_io(self, dobj, cache = True):
gfiles = defaultdict(list)
gobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
for g in gobjs:
gfiles[g.filename].append(g)
for fn in sorted(gfiles):
gs = gfiles[fn]
- yield YTDataChunk(dobj, "io", gs, self._count_selection(dobj, gs))
+ yield YTDataChunk(dobj, "io", gs, self._count_selection(dobj, gs),
+ cache = cache)
diff -r 294e62e00fa88b8aa3a9e35875178d70aecc964c -r 67d98f2ef85c7831a1f5bb0c640b06ed89ea151a yt/geometry/particle_geometry_handler.py
--- a/yt/geometry/particle_geometry_handler.py
+++ b/yt/geometry/particle_geometry_handler.py
@@ -170,10 +170,10 @@
g = og
yield YTDataChunk(dobj, "spatial", [g])
- def _chunk_io(self, dobj):
+ def _chunk_io(self, dobj, cache = True):
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
for subset in oobjs:
- yield YTDataChunk(dobj, "io", [subset], None)
+ yield YTDataChunk(dobj, "io", [subset], None, cache = cache)
class ParticleDataChunk(YTDataChunk):
def __init__(self, oct_handler, regions, *args, **kwargs):
diff -r 294e62e00fa88b8aa3a9e35875178d70aecc964c -r 67d98f2ef85c7831a1f5bb0c640b06ed89ea151a yt/utilities/lib/QuadTree.pyx
--- a/yt/utilities/lib/QuadTree.pyx
+++ b/yt/utilities/lib/QuadTree.pyx
@@ -131,7 +131,7 @@
cdef int i, j
cdef QuadTreeNode *node
cdef np.int64_t pos[2]
- cdef np.float64_t *vals = <np.float64_t *> alloca(
+ cdef np.float64_t *vals = <np.float64_t *> malloc(
sizeof(np.float64_t)*nvals)
cdef np.float64_t weight_val = 0.0
self.nvals = nvals
@@ -160,6 +160,7 @@
self.root_nodes[i][j] = QTN_initialize(
pos, nvals, vals, weight_val)
self.num_cells = self.top_grid_dims[0] * self.top_grid_dims[1]
+ free(vals)
cdef int count_total_cells(self, QuadTreeNode *root):
cdef int total = 0
@@ -373,7 +374,7 @@
cdef np.float64_t *vdata = <np.float64_t *> nvals.data
cdef np.float64_t *wdata = <np.float64_t *> nwvals.data
cdef np.float64_t wtoadd
- cdef np.float64_t *vtoadd = <np.float64_t *> alloca(
+ cdef np.float64_t *vtoadd = <np.float64_t *> malloc(
sizeof(np.float64_t)*self.nvals)
for i in range(self.top_grid_dims[0]):
for j in range(self.top_grid_dims[1]):
@@ -381,6 +382,7 @@
wtoadd = 0.0
curpos += self.fill(self.root_nodes[i][j],
curpos, px, py, pdx, pdy, vdata, wdata, vtoadd, wtoadd, 0)
+ free(vtoadd)
return opx, opy, opdx, opdy, nvals, nwvals
cdef int count(self, QuadTreeNode *node):
@@ -406,7 +408,7 @@
np.int64_t level):
cdef int i, j, n
cdef np.float64_t *vorig
- vorig = <np.float64_t *> alloca(sizeof(np.float64_t) * self.nvals)
+ vorig = <np.float64_t *> malloc(sizeof(np.float64_t) * self.nvals)
if node.children[0][0] == NULL:
if self.merged == -1:
for i in range(self.nvals):
@@ -444,6 +446,7 @@
for i in range(self.nvals):
vtoadd[i] = vorig[i]
wtoadd -= node.weight_val
+ free(vorig)
return added
@cython.boundscheck(False)
Repository URL: https://bitbucket.org/yt_analysis/yt-3.0/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the yt-svn
mailing list