[Yt-svn] yt-commit r1603 - branches/yt-1.6/yt/lagos
mturk at wrangler.dreamhost.com
mturk at wrangler.dreamhost.com
Thu Jan 28 18:01:33 PST 2010
Author: mturk
Date: Thu Jan 28 18:01:28 2010
New Revision: 1603
URL: http://yt.enzotools.org/changeset/1603
Log:
Backporting a number of fixes from trunk, including:
* a fix for the hierarchy for problematic storage files
* Fixes for halo finding if there are no halos found
* Fix for calculating the mass in a given region in halo finding
* Adding 'storage_filename' parameter to the EnzoStaticOutput
* Removing clear_data() calls
* MAJOR Fix for the periodic region ParticleIO
Modified:
branches/yt-1.6/yt/lagos/BaseDataTypes.py
branches/yt-1.6/yt/lagos/ContourFinder.py
branches/yt-1.6/yt/lagos/HDF5LightReader.c
branches/yt-1.6/yt/lagos/HaloFinding.py
branches/yt-1.6/yt/lagos/HierarchyType.py
branches/yt-1.6/yt/lagos/OutputTypes.py
branches/yt-1.6/yt/lagos/ParallelTools.py
Modified: branches/yt-1.6/yt/lagos/BaseDataTypes.py
==============================================================================
--- branches/yt-1.6/yt/lagos/BaseDataTypes.py (original)
+++ branches/yt-1.6/yt/lagos/BaseDataTypes.py Thu Jan 28 18:01:28 2010
@@ -1762,7 +1762,6 @@
if cache: cached_fields = defaultdict(lambda: dict())
else: cached_fields = None
for level in range(num_levels):
- self.clear_data()
contours[level] = {}
if cumulative:
mv = max_val
Modified: branches/yt-1.6/yt/lagos/ContourFinder.py
==============================================================================
--- branches/yt-1.6/yt/lagos/ContourFinder.py (original)
+++ branches/yt-1.6/yt/lagos/ContourFinder.py Thu Jan 28 18:01:28 2010
@@ -262,7 +262,6 @@
total_contours = 0
tree = []
for gi,grid in enumerate(grids):
- grid.clear_data()
pbar.update(gi+1)
cm = data_source._get_cut_mask(grid)
if cm is True: cm = na.ones(grid.ActiveDimensions, dtype='bool')
Modified: branches/yt-1.6/yt/lagos/HDF5LightReader.c
==============================================================================
--- branches/yt-1.6/yt/lagos/HDF5LightReader.c (original)
+++ branches/yt-1.6/yt/lagos/HDF5LightReader.c Thu Jan 28 18:01:28 2010
@@ -1280,14 +1280,14 @@
} else if ( (tempx > vdata->left_edge[0]) && (tempx > vdata->right_edge[0]) ) {
tempx -= vdata->period[0];
}
- if ( (tempy < vdata->left_edge[1]) && (tempx < vdata->right_edge[1]) ) {
+ if ( (tempy < vdata->left_edge[1]) && (tempy < vdata->right_edge[1]) ) {
tempy += vdata->period[1];
- } else if ( (tempy > vdata->left_edge[1]) && (tempx > vdata->right_edge[1]) ) {
+ } else if ( (tempy > vdata->left_edge[1]) && (tempy > vdata->right_edge[1]) ) {
tempy -= vdata->period[1];
}
- if ( (tempz < vdata->left_edge[2]) && (tempx < vdata->right_edge[2]) ) {
+ if ( (tempz < vdata->left_edge[2]) && (tempz < vdata->right_edge[2]) ) {
tempz += vdata->period[2];
- } else if ( (tempz > vdata->left_edge[2]) && (tempx > vdata->right_edge[2]) ) {
+ } else if ( (tempz > vdata->left_edge[2]) && (tempz > vdata->right_edge[2]) ) {
tempz -= vdata->period[2];
}
if ( (tempx >= vdata->left_edge[0])
@@ -1350,14 +1350,14 @@
} else if ( (tempx > vdata->left_edge[0]) && (tempx > vdata->right_edge[0]) ) {
tempx -= vdata->period[0];
}
- if ( (tempy < vdata->left_edge[1]) && (tempx < vdata->right_edge[1]) ) {
+ if ( (tempy < vdata->left_edge[1]) && (tempy < vdata->right_edge[1]) ) {
tempy += vdata->period[1];
- } else if ( (tempy > vdata->left_edge[1]) && (tempx > vdata->right_edge[1]) ) {
+ } else if ( (tempy > vdata->left_edge[1]) && (tempy > vdata->right_edge[1]) ) {
tempy -= vdata->period[1];
}
- if ( (tempz < vdata->left_edge[2]) && (tempx < vdata->right_edge[2]) ) {
+ if ( (tempz < vdata->left_edge[2]) && (tempz < vdata->right_edge[2]) ) {
tempz += vdata->period[2];
- } else if ( (tempz > vdata->left_edge[2]) && (tempx > vdata->right_edge[2]) ) {
+ } else if ( (tempz > vdata->left_edge[2]) && (tempz > vdata->right_edge[2]) ) {
tempz -= vdata->period[2];
}
if ( (tempx >= vdata->left_edge[0])
@@ -1420,14 +1420,14 @@
} else if ( (tempx > vdata->left_edge[0]) && (tempx > vdata->right_edge[0]) ) {
tempx -= vdata->period[0];
}
- if ( (tempy < vdata->left_edge[1]) && (tempx < vdata->right_edge[1]) ) {
+ if ( (tempy < vdata->left_edge[1]) && (tempy < vdata->right_edge[1]) ) {
tempy += vdata->period[1];
- } else if ( (tempy > vdata->left_edge[1]) && (tempx > vdata->right_edge[1]) ) {
+ } else if ( (tempy > vdata->left_edge[1]) && (tempy > vdata->right_edge[1]) ) {
tempy -= vdata->period[1];
}
- if ( (tempz < vdata->left_edge[2]) && (tempx < vdata->right_edge[2]) ) {
+ if ( (tempz < vdata->left_edge[2]) && (tempz < vdata->right_edge[2]) ) {
tempz += vdata->period[2];
- } else if ( (tempz > vdata->left_edge[2]) && (tempx > vdata->right_edge[2]) ) {
+ } else if ( (tempz > vdata->left_edge[2]) && (tempz > vdata->right_edge[2]) ) {
tempz -= vdata->period[2];
}
if ( (tempx >= vdata->left_edge[0])
Modified: branches/yt-1.6/yt/lagos/HaloFinding.py
==============================================================================
--- branches/yt-1.6/yt/lagos/HaloFinding.py (original)
+++ branches/yt-1.6/yt/lagos/HaloFinding.py Thu Jan 28 18:01:28 2010
@@ -837,6 +837,9 @@
del self.densities
self.group_count = obj.group_count
self.group_sizes = obj.group_sizes
+ if self.group_count == 0:
+ mylog.info("There are no halos found.")
+ return
self.CoM = obj.CoM
self.Tot_M = obj.Tot_M * self.total_mass
self.max_dens_point = obj.max_dens_point
@@ -906,6 +909,9 @@
# We want arrays for parallel HOP
self._groups = na.empty(self.group_count, dtype='object')
self._max_dens = na.empty((self.group_count, 4), dtype='float64')
+ if self.group_count == 0:
+ mylog.info("There are no halos found.")
+ return
for i in unique_ids:
if i == -1:
cp += counts[i+1]
@@ -1183,6 +1189,9 @@
yt_counters("Final Grouping")
def _join_halolists(self):
+ if self.group_count == 0:
+ mylog.info("There are no halos found.")
+ return
ms = -self.Tot_M.copy()
del self.Tot_M
Cx = self.CoM[:,0].copy()
@@ -1207,10 +1216,8 @@
if dm_only:
select = self._get_dm_indices()
total_mass = self._mpi_allsum((self._data_source["ParticleMassMsun"][select]).sum())
- sub_mass = (self._data_source["ParticleMassMsun"][select]).sum()
else:
total_mass = self._mpi_allsum(self._data_source["ParticleMassMsun"].sum())
- sub_mass = self._data_source["ParticleMassMsun"].sum()
# MJT: Note that instead of this, if we are assuming that the particles
# are all on different processors, we should instead construct an
# object representing the entire domain and sum it "lazily" with
@@ -1220,7 +1227,11 @@
self.bounds = (LE, RE)
# reflect particles around the periodic boundary
#self._reposition_particles((LE, RE))
- #sub_mass = self._data_source["ParticleMassMsun"].sum()
+ if dm_only:
+ select = self._get_dm_indices()
+ sub_mass = self._data_source["ParticleMassMsun"][select].sum()
+ else:
+ sub_mass = self._data_source["ParticleMassMsun"].sum()
HOPHaloList.__init__(self, self._data_source, threshold*total_mass/sub_mass, dm_only)
self._parse_halolist(total_mass/sub_mass)
self._join_halolists()
Modified: branches/yt-1.6/yt/lagos/HierarchyType.py
==============================================================================
--- branches/yt-1.6/yt/lagos/HierarchyType.py (original)
+++ branches/yt-1.6/yt/lagos/HierarchyType.py Thu Jan 28 18:01:28 2010
@@ -135,16 +135,19 @@
def _initialize_data_storage(self):
if not ytcfg.getboolean('lagos','serialize'): return
- if os.path.isfile(os.path.join(self.directory,
- "%s.yt" % self.pf["CurrentTimeIdentifier"])):
- fn = os.path.join(self.directory,"%s.yt" % self.pf["CurrentTimeIdentifier"])
- else:
- fn = os.path.join(self.directory,
- "%s.yt" % self.parameter_file.basename)
+ fn = self.pf.storage_filename
+ if fn is None:
+ if os.path.isfile(os.path.join(self.directory,
+ "%s.yt" % self.pf["CurrentTimeIdentifier"])):
+ fn = os.path.join(self.directory,"%s.yt" % self.pf["CurrentTimeIdentifier"])
+ else:
+ fn = os.path.join(self.directory,
+ "%s.yt" % self.parameter_file.basename)
+ dir_to_check = os.path.dirname(fn)
if os.path.isfile(fn):
writable = os.access(fn, os.W_OK)
else:
- writable = os.access(self.directory, os.W_OK)
+ writable = os.access(dir_to_check, os.W_OK)
if ytcfg.getboolean('lagos','onlydeserialize') or not writable:
self._data_mode = mode = 'r'
else:
@@ -349,8 +352,11 @@
"%s.hierarchy" % (pf.parameter_filename))
harray_fn = self.hierarchy_filename[:-9] + "harrays"
if os.path.exists(harray_fn):
- harray_fp = h5py.File(harray_fn)
- self.num_grids = harray_fp["/Level"].len()
+ try:
+ harray_fp = h5py.File(harray_fn)
+ self.num_grids = harray_fp["/Level"].len()
+ except IOError:
+ pass
elif os.path.getsize(self.hierarchy_filename) == 0:
raise IOError(-1,"File empty", self.hierarchy_filename)
self.directory = os.path.dirname(self.hierarchy_filename)
@@ -489,7 +495,10 @@
def _parse_binary_hierarchy(self):
mylog.info("Getting the binary hierarchy")
- f = h5py.File(self.hierarchy_filename[:-9] + "harrays")
+ try:
+ f = h5py.File(self.hierarchy_filename[:-9] + "harrays")
+ except h5py.h5.H5Error:
+ return False
self.grid_dimensions[:] = f["/ActiveDimensions"][:]
self.grid_left_edge[:] = f["/LeftEdges"][:]
self.grid_right_edge[:] = f["/RightEdges"][:]
@@ -522,7 +531,10 @@
if self._data_mode == 'r': return
if self.data_style != "enzo_packed_3d": return
mylog.info("Storing the binary hierarchy")
- f = h5py.File(self.hierarchy_filename[:-9] + "harrays", "w")
+ try:
+ f = h5py.File(self.hierarchy_filename[:-9] + "harrays", "w")
+ except IOError:
+ return
f.create_dataset("/LeftEdges", data=self.grid_left_edge)
f.create_dataset("/RightEdges", data=self.grid_right_edge)
parents, procs, levels = [], [], []
Modified: branches/yt-1.6/yt/lagos/OutputTypes.py
==============================================================================
--- branches/yt-1.6/yt/lagos/OutputTypes.py (original)
+++ branches/yt-1.6/yt/lagos/OutputTypes.py Thu Jan 28 18:01:28 2010
@@ -168,7 +168,8 @@
_fieldinfo_class = EnzoFieldContainer
def __init__(self, filename, data_style=None,
parameter_override = None,
- conversion_override = None):
+ conversion_override = None,
+ storage_filename = None):
"""
This class is a stripped down class that simply reads and parses
*filename* without looking at the hierarchy. *data_style* gets passed
@@ -182,6 +183,7 @@
self.__parameter_override = parameter_override
if conversion_override is None: conversion_override = {}
self.__conversion_override = conversion_override
+ self.storage_filename = storage_filename
StaticOutput.__init__(self, filename, data_style)
if "InitialTime" not in self.parameters:
Modified: branches/yt-1.6/yt/lagos/ParallelTools.py
==============================================================================
--- branches/yt-1.6/yt/lagos/ParallelTools.py (original)
+++ branches/yt-1.6/yt/lagos/ParallelTools.py Thu Jan 28 18:01:28 2010
@@ -280,7 +280,7 @@
def _partition_hierarchy_3d(self, padding=0.0):
LE, RE = self.pf["DomainLeftEdge"].copy(), self.pf["DomainRightEdge"].copy()
if not self._distributed:
- return False, LE, RE, self.hierarchy.grid_collection(self.center, self.hierarchy.grids)
+ return False, LE, RE, self.hierarchy.all_data()
elif ytcfg.getboolean("yt", "inline"):
# At this point, we want to identify the root grid tile to which
# this processor is assigned.
More information about the yt-svn
mailing list