[Yt-svn] yt: 1. Fixing a typo in HaloFinding for parallelHF rms_vel calcu...
hg at spacepope.org
hg at spacepope.org
Sat Mar 20 17:03:35 PDT 2010
hg Repository: yt
details: yt/rev/aea4a15ea12c
changeset: 1462:aea4a15ea12c
user: Stephen Skory <stephenskory at yahoo.com>
date:
Sat Mar 20 17:03:24 2010 -0700
description:
1. Fixing a typo in HaloFinding for parallelHF rms_vel calculations.
2. A couple cosmetic changes for the StructureFunction.
3. Some improvements to the MergerTree.
diffstat:
yt/extensions/MergerTree.py | 26 +++++++++++++++++++-------
yt/lagos/HaloFinding.py | 2 +-
yt/lagos/StructureFunctionGenerator.py | 9 +++++----
3 files changed, 25 insertions(+), 12 deletions(-)
diffs (131 lines):
diff -r f25078d03f66 -r aea4a15ea12c yt/extensions/MergerTree.py
--- a/yt/extensions/MergerTree.py Fri Mar 19 15:26:19 2010 -0700
+++ b/yt/extensions/MergerTree.py Sat Mar 20 17:03:24 2010 -0700
@@ -30,7 +30,7 @@
import yt.extensions.HaloProfiler as HP
import numpy as na
-import os, glob, md5, time
+import os, glob, md5, time, gc
import h5py
import types
import sqlite3 as sql
@@ -90,12 +90,14 @@
class MergerTree(DatabaseFunctions, lagos.ParallelAnalysisInterface):
def __init__(self, restart_files=[], database='halos.db',
halo_finder_function=HaloFinder, halo_finder_threshold=80.0,
- FOF_link_length=0.2):
+ FOF_link_length=0.2, dm_only=False, refresh=False):
self.restart_files = restart_files # list of enzo restart files
self.database = database # the sqlite database of haloes.
self.halo_finder_function = halo_finder_function # which halo finder to use
self.halo_finder_threshold = halo_finder_threshold # overdensity threshold
self.FOF_link_length= FOF_link_length # For FOF
+ self.dm_only = dm_only
+ self.refresh = refresh
# MPI stuff
self.mine = self._mpi_get_rank()
if self.mine is None:
@@ -104,6 +106,12 @@
if self.size is None:
self.size = 1
# Get to work.
+ if self.refresh and self.mine == 0:
+ try:
+ os.unlink(self.database)
+ except:
+ pass
+ self._barrier()
self._open_create_database()
self._create_halo_table()
self._run_halo_finder_add_to_db()
@@ -126,21 +134,24 @@
def _run_halo_finder_add_to_db(self):
for file in self.restart_files:
+ gc.collect()
pf = lagos.EnzoStaticOutput(file)
- # If the halos are already found, skip this one.
+ # If the halos are already found, skip this data step, unless
+ # refresh is True.
dir = os.path.dirname(file)
if os.path.exists(os.path.join(dir, 'MergerHalos.out')) and \
os.path.exists(os.path.join(dir, 'MergerHalos.txt')) and \
- glob.glob(os.path.join(dir, 'MergerHalos*h5')) is not []:
+ glob.glob(os.path.join(dir, 'MergerHalos*h5')) is not [] and \
+ not self.refresh:
pass
else:
# Run the halo finder.
if self.halo_finder_function == yt.lagos.HaloFinding.FOFHaloFinder:
halos = self.halo_finder_function(pf,
- link=self.FOF_link_length, dm_only=True)
+ link=self.FOF_link_length, dm_only=self.dm_only)
else:
halos = self.halo_finder_function(pf,
- threshold=self.halo_finder_threshold, dm_only=True)
+ threshold=self.halo_finder_threshold, dm_only=self.dm_only)
halos.write_out(os.path.join(dir, 'MergerHalos.out'))
halos.write_particle_lists(os.path.join(dir, 'MergerHalos'))
halos.write_particle_lists_txt(os.path.join(dir, 'MergerHalos'))
@@ -183,6 +194,7 @@
# doesn't already exist. Open it first on root, and then on the others.
if self.mine == 0:
self.conn = sql.connect(self.database)
+ self._barrier()
self._ensure_db_sync()
if self.mine != 0:
self.conn = sql.connect(self.database)
@@ -651,4 +663,4 @@
fp.write(line)
results = self.cursor.fetchone()
fp.close()
-
\ No newline at end of file
+
diff -r f25078d03f66 -r aea4a15ea12c yt/lagos/HaloFinding.py
--- a/yt/lagos/HaloFinding.py Fri Mar 19 15:26:19 2010 -0700
+++ b/yt/lagos/HaloFinding.py Sat Mar 20 17:03:24 2010 -0700
@@ -952,7 +952,7 @@
self.rms_vel[groupID] = \
na.sqrt(rms_vel_temp[groupID][0] / rms_vel_temp[groupID][1]) * \
self.group_sizes[groupID]
- del rms_vel_temp, ms
+ del rms_vel_temp
yt_counters("rms vel computing")
self.taskID = obj.mine
self.halo_taskmap = obj.halo_taskmap # A defaultdict.
diff -r f25078d03f66 -r aea4a15ea12c yt/lagos/StructureFunctionGenerator.py
--- a/yt/lagos/StructureFunctionGenerator.py Fri Mar 19 15:26:19 2010 -0700
+++ b/yt/lagos/StructureFunctionGenerator.py Sat Mar 20 17:03:24 2010 -0700
@@ -152,9 +152,10 @@
# Do all the startup tasks.
self._init_kd_tree()
self._build_fields_vals()
- self._build_points_array()
for bigloop, length in enumerate(self.lengths):
- #mylog.info("Doing length %1.5e" % length)
+ self._build_points_array()
+ if self.mine == 0:
+ mylog.info("Doing length %1.5e" % length)
# Things stop when this value below equals total_values.
self.generated_points = 0
self.gen_array = na.zeros(self.size, dtype='int64')
@@ -191,7 +192,8 @@
self._send_done_toall()
#print 'done!', self.mine
#self._barrier()
- mylog.info("Length (%d of %d) %1.5e took %d communication cycles to complete." % \
+ if self.mine == 0:
+ mylog.info("Length (%d of %d) %1.5e took %d communication cycles to complete." % \
(bigloop+1, len(self.lengths), length, self.comm_cycle_count))
self._allsum_bin_hits()
@@ -321,7 +323,6 @@
"""
Add up the hits to all the bins globally for all functions.
"""
- print 'here'
for fset in self.fsets:
fset.too_low = self._mpi_allsum(fset.too_low)
fset.too_high = self._mpi_allsum(fset.too_high)
More information about the yt-svn
mailing list