[Yt-svn] yt: Made a modification to HaloFinding that elminates an object ...
hg at spacepope.org
hg at spacepope.org
Sat Apr 3 12:56:04 PDT 2010
hg Repository: yt
details: yt/rev/c0d025027446
changeset: 1516:c0d025027446
user: Stephen Skory <stephenskory at yahoo.com>
date:
Sat Apr 03 12:55:43 2010 -0700
description:
Made a modification to HaloFinding that elminates an object reference
problem that manifested itself by an apparent memory leak.
The change might break other things, but I haven't seen any
problems.
Also a few changes to parallelHOP.
diffstat:
yt/lagos/HaloFinding.py | 60 ++++++++++++++++-------------
yt/lagos/parallelHOP/parallelHOP.py | 27 +++++++++++--
2 files changed, 55 insertions(+), 32 deletions(-)
diffs (283 lines):
diff -r fca725332306 -r c0d025027446 yt/lagos/HaloFinding.py
--- a/yt/lagos/HaloFinding.py Wed Mar 31 17:35:44 2010 -0700
+++ b/yt/lagos/HaloFinding.py Sat Apr 03 12:55:43 2010 -0700
@@ -40,7 +40,8 @@
from yt.performance_counters import yt_counters, time_function
from kd import *
-import math, sys, itertools
+from yt.funcs import *
+import math, sys, itertools, gc
from collections import defaultdict
class Halo(object):
@@ -59,7 +60,7 @@
def __init__(self, halo_list, id, indices = None, size=None, CoM=None,
max_dens_point=None, group_total_mass=None, max_radius=None, bulk_vel=None,
tasks=None, rms_vel=None):
- self.halo_list = halo_list
+ self._max_dens = halo_list._max_dens
self.id = id
self.data = halo_list._data_source
if indices is not None:
@@ -95,16 +96,16 @@
"""
Return the HOP-identified maximum density.
"""
- return self.halo_list._max_dens[self.id][0]
+ return self._max_dens[self.id][0]
def maximum_density_location(self):
"""
Return the location HOP identified as maximally dense.
"""
return na.array([
- self.halo_list._max_dens[self.id][1],
- self.halo_list._max_dens[self.id][2],
- self.halo_list._max_dens[self.id][3]])
+ self._max_dens[self.id][1],
+ self._max_dens[self.id][2],
+ self._max_dens[self.id][3]])
def total_mass(self):
"""
@@ -168,7 +169,7 @@
else: center = self.maximum_density_location()
radius = self.maximum_radius()
# A bit of a long-reach here...
- sphere = self.halo_list._data_source.hierarchy.sphere(
+ sphere = self.data.hierarchy.sphere(
center, radius=radius)
return sphere
@@ -241,9 +242,9 @@
return None
self.bin_count = bins
# Cosmology
- h = self.halo_list._data_source.pf['CosmologyHubbleConstantNow']
- Om_matter = self.halo_list._data_source.pf['CosmologyOmegaMatterNow']
- z = self.halo_list._data_source.pf['CosmologyCurrentRedshift']
+ h = self.data.pf['CosmologyHubbleConstantNow']
+ Om_matter = self.data.pf['CosmologyOmegaMatterNow']
+ z = self.data.pf['CosmologyCurrentRedshift']
rho_crit_now = 1.8788e-29 * h**2.0 * Om_matter # g cm^-3
Msun2g = 1.989e33
rho_crit = rho_crit_now * ((1.0 + z)**3.0)
@@ -252,8 +253,8 @@
self.mass_bins = na.zeros(self.bin_count+1, dtype='float64')
dist = na.empty(self.indices.size, dtype='float64')
cen = self.center_of_mass()
- period = self.halo_list._data_source.pf["DomainRightEdge"] - \
- self.halo_list._data_source.pf["DomainLeftEdge"]
+ period = self.data.pf["DomainRightEdge"] - \
+ self.data.pf["DomainLeftEdge"]
mark = 0
# Find the distances to the particles. I don't like this much, but I
# can't see a way to eliminate a loop like this, either here or in
@@ -277,7 +278,7 @@
# Calculate the over densities in the bins.
self.overdensity = self.mass_bins * Msun2g / \
(4./3. * math.pi * rho_crit * \
- (self.radial_bins * self.halo_list._data_source.pf["cm"])**3.0)
+ (self.radial_bins * self.data.pf["cm"])**3.0)
class HOPHalo(Halo):
@@ -295,8 +296,8 @@
Return the HOP-identified maximum density.
"""
if self.max_dens_point is not None:
- return self.halo_list._max_dens[self.id][0]
- max = self._mpi_allmax(self.halo_list._max_dens[self.id][0])
+ return self._max_dens[self.id][0]
+ max = self._mpi_allmax(self._max_dens[self.id][0])
return max
def maximum_density_location(self):
@@ -304,15 +305,15 @@
Return the location HOP identified as maximally dense.
"""
if self.max_dens_point is not None:
- return na.array([self.halo_list._max_dens[self.id][1], self.halo_list._max_dens[self.id][2],
- self.halo_list._max_dens[self.id][3]])
+ return na.array([self._max_dens[self.id][1], self._max_dens[self.id][2],
+ self._max_dens[self.id][3]])
# If I own the maximum density, my location is globally correct.
max_dens = self.maximum_density()
- if self.halo_list._max_dens[self.id][0] == max_dens:
+ if self._max_dens[self.id][0] == max_dens:
value = na.array([
- self.halo_list._max_dens[self.id][1],
- self.halo_list._max_dens[self.id][2],
- self.halo_list._max_dens[self.id][3]])
+ self._max_dens[self.id][1],
+ self._max_dens[self.id][2],
+ self._max_dens[self.id][3]])
else:
value = na.array([0,0,0])
# This works, and isn't appropriate but for now will be fine...
@@ -496,14 +497,14 @@
return None
# Do this for all because all will use it.
self.bin_count = bins
- period = self.halo_list._data_source.pf["DomainRightEdge"] - \
- self.halo_list._data_source.pf["DomainLeftEdge"]
+ period = self.data.pf["DomainRightEdge"] - \
+ self.data.pf["DomainLeftEdge"]
self.mass_bins = na.zeros(self.bin_count+1, dtype='float64')
cen = self.center_of_mass()
# Cosmology
- h = self.halo_list._data_source.pf['CosmologyHubbleConstantNow']
- Om_matter = self.halo_list._data_source.pf['CosmologyOmegaMatterNow']
- z = self.halo_list._data_source.pf['CosmologyCurrentRedshift']
+ h = self.data.pf['CosmologyHubbleConstantNow']
+ Om_matter = self.data.pf['CosmologyOmegaMatterNow']
+ z = self.data.pf['CosmologyCurrentRedshift']
rho_crit_now = 1.8788e-29 * h**2.0 * Om_matter # g cm^-3
Msun2g = 1.989e33
rho_crit = rho_crit_now * ((1.0 + z)**3.0)
@@ -546,7 +547,7 @@
# Calculate the over densities in the bins.
self.overdensity = self.mass_bins * Msun2g / \
(4./3. * math.pi * rho_crit * \
- (self.radial_bins * self.halo_list._data_source.pf["cm"])**3.0)
+ (self.radial_bins * self.data.pf["cm"])**3.0)
class FOFHalo(Halo):
@@ -1024,6 +1025,11 @@
# Clean up
del self.max_dens_point, self.max_radius, self.bulk_vel
del self.halo_taskmap, self.tags, self.rms_vel
+ del grab_indices, unique_ids, counts
+ try:
+ del group_indices
+ except UnboundLocalError:
+ pass
def __len__(self):
return self.group_count
diff -r fca725332306 -r c0d025027446 yt/lagos/parallelHOP/parallelHOP.py
--- a/yt/lagos/parallelHOP/parallelHOP.py Wed Mar 31 17:35:44 2010 -0700
+++ b/yt/lagos/parallelHOP/parallelHOP.py Sat Apr 03 12:55:43 2010 -0700
@@ -256,6 +256,7 @@
yt_counters("Picking padding data to send.")
# Communicate the sizes to send.
self.mine, global_send_count = self._mpi_info_dict(send_size)
+ del send_size
# Initialize the arrays to receive data.
yt_counters("Initalizing recv arrays.")
recv_real_indices = {}
@@ -392,6 +393,7 @@
# inside the real region, but within one padding of the boundary,
# and this will do it.
self.is_inside_annulus = na.bitwise_and(self.is_inside, inner)
+ del inner
# Below we make a mapping of real particle index->local ID
# Unf. this has to be a dict, because any task can have
# particles of any particle_index, which means that if it were an
@@ -594,6 +596,10 @@
# All our neighbors are in the same chain already, so
# we don't need to search again.
self.search_again[i] = False
+ try:
+ del NNtags
+ except UnboundLocalError:
+ pass
yt_counters("preconnect kd tree search.")
# Recursively jump links until we get to a chain whose densest
# link is to itself. At that point we've found the densest chain
@@ -632,7 +638,9 @@
self.chainID[i] = map[self.chainID[i]]
del map
self.densest_in_chain = dic_new.copy()
+ del dic_new
self.densest_in_chain_real_index = dicri_new.copy()
+ del dicri_new
self.__max_memory()
yt_counters("preconnect pregrouping.")
mylog.info("Preconnected %d chains." % removed)
@@ -731,6 +739,7 @@
continue
self.chainID[i] = new
del tolink, fix_map
+ del uniq
yt_counters("global chain hand-linking.")
yt_counters("create_global_densest_in_chain")
@@ -821,6 +830,7 @@
self.global_padded_count = self._mpi_joindict(self.global_padded_count)
# Send/receive 'em.
self._communicate_uphill_info()
+ del self.global_padded_count
self.__max_memory()
# Fix the IDs to localIDs.
for i,real_index in enumerate(self.recv_real_indices):
@@ -951,7 +961,7 @@
continue
# Clean up.
del recv_real_indices, recv_chainIDs, real_indices, chainIDs, select
- del hooks
+ del hooks, global_annulus_count
# We're done with this here.
del self.rev_index
yt_counters("communicate_annulus_chainIDs")
@@ -1027,6 +1037,10 @@
boundary_density
else:
continue
+ try:
+ del point, NNtags, results
+ except UnboundLocalError:
+ pass
self.__max_memory()
yt_counters("connect_chains")
@@ -1269,7 +1283,8 @@
if self.densest_in_group[groupID] < max_dens:
self.densest_in_group[groupID] = max_dens
self.densest_in_group_real_index[groupID] = self.densest_in_chain_real_index[chainID]
- del self.densest_in_chain, self.densest_in_chain_real_index
+ del self.densest_in_chain, self.densest_in_chain_real_index, self.reverse_map
+ del self.densest_in_group
yt_counters("translate_groupIDs")
def _precompute_group_info(self):
@@ -1306,6 +1321,7 @@
# some groups are on multiple tasks, there is only one densest_in_chain
# and only that task contributed above.
self.max_dens_point = self._mpi_Allsum_double(max_dens_point)
+ del max_dens_point
yt_counters("max dens point")
# Now CoM.
yt_counters("CoM")
@@ -1331,6 +1347,7 @@
if ms_u.size == 1:
single = True
Tot_M = size.astype('float64') * ms_u
+ del ms_u
else:
single = False
del ms_u
@@ -1383,8 +1400,9 @@
yt_counters("max radius")
yt_counters("Precomp.")
self.__max_memory()
+ del select, loc, subchain, CoM_M, Tot_M, size, max_radius
if calc:
- del loc, subchain, CoM_M, Tot_M, c_vec, max_radius, select
+ del c_vec
del sort_subchain, uniq_subchain, diff_subchain, marks, dist, sort
del rad, com
@@ -1429,9 +1447,8 @@
self._communicate_annulus_chainIDs()
mylog.info('Connecting %d chains into groups...' % self.nchains)
self._connect_chains()
- del fKD.dens, fKD.mass
+ del fKD.dens, fKD.mass, fKD.dens
del fKD.pos, fKD.chunk_tags
- del fKD.tags, fKD.dist
free_tree() # Frees the kdtree object.
del self.densestNN
mylog.info('Communicating group links globally...')
More information about the yt-svn
mailing list