[yt-svn] commit/yt: 2 new changesets
Bitbucket
commits-noreply at bitbucket.org
Thu May 31 11:59:52 PDT 2012
2 new commits in yt:
https://bitbucket.org/yt_analysis/yt/changeset/4088e7857e24/
changeset: 4088e7857e24
branch: yt
user: brittonsmith
date: 2012-05-31 20:56:08
summary: Expanded parallel support for halo profiler. The make_profiles,
make_projections, and analyze_halo_spheres functions now all take
an njobs kwarg that is handed to the parallel_objects call. This
allows for all the halo profiler functionality to use more than
one processor per halo.
affected #: 3 files
diff -r 46608e61b743294c8a223362fe19910ab1f8a1f8 -r 4088e7857e24639c3c7d7edf013c5018f814862d yt/analysis_modules/halo_profiler/api.py
--- a/yt/analysis_modules/halo_profiler/api.py
+++ b/yt/analysis_modules/halo_profiler/api.py
@@ -34,5 +34,4 @@
from .multi_halo_profiler import \
HaloProfiler, \
FakeProfile, \
- get_halo_sphere, \
standard_fields
diff -r 46608e61b743294c8a223362fe19910ab1f8a1f8 -r 4088e7857e24639c3c7d7edf013c5018f814862d yt/analysis_modules/halo_profiler/centering_methods.py
--- a/yt/analysis_modules/halo_profiler/centering_methods.py
+++ b/yt/analysis_modules/halo_profiler/centering_methods.py
@@ -43,14 +43,14 @@
@add_function("Min_Dark_Matter_Density")
def find_minimum_dm_density(data):
ma, maxi, mx, my, mz, mg = data.quantities['MinLocation']('Dark_Matter_Density',
- lazy_reader=False,
+ lazy_reader=True,
preload=False)
return (mx, my, mz)
@add_function("Max_Dark_Matter_Density")
def find_maximum_dm_density(data):
ma, maxi, mx, my, mz, mg = data.quantities['MaxLocation']('Dark_Matter_Density',
- lazy_reader=False,
+ lazy_reader=True,
preload=False)
return (mx, my, mz)
@@ -58,7 +58,7 @@
def find_CoM_dm_density(data):
dc_x, dc_y, dc_z = data.quantities['CenterOfMass'](use_cells=False,
use_particles=True,
- lazy_reader=False,
+ lazy_reader=True,
preload=False)
return (dc_x, dc_y, dc_z)
@@ -67,14 +67,14 @@
@add_function("Min_Gas_Density")
def find_minimum_gas_density(data):
ma, maxi, mx, my, mz, mg = data.quantities['MinLocation']('Density',
- lazy_reader=False,
+ lazy_reader=True,
preload=False)
return (mx, my, mz)
@add_function("Max_Gas_Density")
def find_maximum_gas_density(data):
ma, maxi, mx, my, mz, mg = data.quantities['MaxLocation']('Density',
- lazy_reader=False,
+ lazy_reader=True,
preload=False)
return (mx, my, mz)
@@ -82,7 +82,7 @@
def find_CoM_gas_density(data):
dc_x, dc_y, dc_z = data.quantities['CenterOfMass'](use_cells=True,
use_particles=False,
- lazy_reader=False,
+ lazy_reader=True,
preload=False)
return (dc_x, dc_y, dc_z)
@@ -91,14 +91,14 @@
@add_function("Min_Total_Density")
def find_minimum_total_density(data):
ma, maxi, mx, my, mz, mg = data.quantities['MinLocation']('Matter_Density',
- lazy_reader=False,
+ lazy_reader=True,
preload=False)
return (mx, my, mz)
@add_function("Max_Total_Density")
def find_maximum_total_density(data):
ma, maxi, mx, my, mz, mg = data.quantities['MaxLocation']('Matter_Density',
- lazy_reader=False,
+ lazy_reader=True,
preload=False)
return (mx, my, mz)
@@ -106,7 +106,7 @@
def find_CoM_total_density(data):
dc_x, dc_y, dc_z = data.quantities['CenterOfMass'](use_cells=True,
use_particles=True,
- lazy_reader=False,
+ lazy_reader=True,
preload=False)
return (dc_x, dc_y, dc_z)
@@ -115,14 +115,14 @@
@add_function("Min_Temperature")
def find_minimum_temperature(data):
ma, mini, mx, my, mz, mg = data.quantities['MinLocation']('Temperature',
- lazy_reader=False,
+ lazy_reader=True,
preload=False)
return (mx, my, mz)
@add_function("Max_Temperature")
def find_maximum_temperature(data):
ma, maxi, mx, my, mz, mg = data.quantities['MaxLocation']('Temperature',
- lazy_reader=False,
+ lazy_reader=True,
preload=False)
return (mx, my, mz)
diff -r 46608e61b743294c8a223362fe19910ab1f8a1f8 -r 4088e7857e24639c3c7d7edf013c5018f814862d yt/analysis_modules/halo_profiler/multi_halo_profiler.py
--- a/yt/analysis_modules/halo_profiler/multi_halo_profiler.py
+++ b/yt/analysis_modules/halo_profiler/multi_halo_profiler.py
@@ -64,7 +64,7 @@
dm_only=False, resize=True,
fancy_padding=True, rearrange=True),
halo_radius=None, radius_units='1', n_profile_bins=50,
- recenter = None,
+ recenter=None,
profile_output_dir='radial_profiles', projection_output_dir='projections',
projection_width=8.0, projection_width_units='mpc', project_at_level='max',
velocity_center=['bulk', 'halo'], filter_quantities=['id', 'center', 'r_max'],
@@ -111,8 +111,32 @@
Args given with call to halo finder function. Default: None.
halo_finder_kwargs : dict
kwargs given with call to halo finder function. Default: None.
- recenter : {string, function
- The name of a function that recenters the halo for analysis.
+ recenter : {string, function}
+ The exact location of the sphere center can significantly affect
+ radial profiles. The halo center loaded by the HaloProfiler will
+ typically be the dark matter center of mass calculated by a halo
+ finder. However, this may not be the best location for centering
+ profiles of baryon quantities. For example, one may want to center
+ on the maximum density.
+ If recenter is given as a string, one of the existing recentering
+ functions will be used:
+ Min_Dark_Matter_Density : location of minimum dark matter density
+ Max_Dark_Matter_Density : location of maximum dark matter density
+ CoM_Dark_Matter_Density : dark matter center of mass
+ Min_Gas_Density : location of minimum gas density
+ Max_Gas_Density : location of maximum gas density
+ CoM_Gas_Density : gas center of mass
+ Min_Total_Density : location of minimum total density
+ Max_Total_Density : location of maximum total density
+ CoM_Total_Density : total center of mass
+ Min_Temperature : location of minimum temperature
+ Max_Temperature : location of maximum temperature
+ Alternately, a function can be supplied for custom recentering.
+ The function should take only one argument, a sphere object.
+ Example function:
+ def my_center_of_mass(data):
+ my_x, my_y, my_z = data.quantities['CenterOfMass']()
+ return (my_x, my_y, my_z)
Default: None.
halo_radius : float
If no halo radii are provided in the halo list file, this
@@ -148,8 +172,7 @@
* ["bulk", "sphere"]: the bulk velocity of the sphere
centered on the halo center.
* ["max", field]: the velocity of the cell that is the
- location of the maximum of the field
- specified (used only when halos set to single).
+ location of the maximum of the field specified.
filter_quantities : array_like
Quantities from the original halo list file to be written out in the
filtered list file. Default: ['id','center'].
@@ -161,8 +184,8 @@
Examples
--------
- >>> import yt.analysis_modules.halo_profiler.api as HP
- >>> hp = HP.halo_profiler("DD0242/DD0242")
+ >>> from yt.analysis_modules.halo_profiler.api import *
+ >>> hp = HaloProfiler("RedshiftOutput0005/RD0005")
"""
ParallelAnalysisInterface.__init__(self)
@@ -226,13 +249,9 @@
# Option to recenter sphere someplace else.
self.recenter = recenter
- # Look for any field that might need to have the bulk velocity set.
+ # Flag for whether calculating halo bulk velocity is necessary.
self._need_bulk_velocity = False
- for field in [hp['field'] for hp in self.profile_fields]:
- if 'Velocity' in field or 'Mach' in field:
- self._need_bulk_velocity = True
- break
-
+
# Check validity for VelocityCenter parameter which toggles how the
# velocity is zeroed out for radial velocity profiles.
self.velocity_center = velocity_center[:]
@@ -250,9 +269,7 @@
mylog.error("Second value of VelocityCenter must be either 'halo' or 'sphere' if first value is 'bulk'.")
return None
elif self.velocity_center[0] == 'max':
- if self.halos is 'multiple':
- mylog.error("Getting velocity center from a max field value only works with halos='single'.")
- return None
+ mylog.info('Using position of max %s for velocity center.' % self.velocity_center[1])
else:
mylog.error("First value of parameter, VelocityCenter, must be either 'bulk' or 'max'.")
return None
@@ -284,7 +301,7 @@
mylog.error("No halos loaded, there will be nothing to do.")
return None
else:
- mylog.error("I don't know whether to get halos from hop or from density maximum. This should not have happened.")
+ mylog.error("Keyword, halos, must be either 'single' or 'multiple'.")
return None
def add_halo_filter(self, function, *args, **kwargs):
@@ -351,6 +368,10 @@
"""
+ # Check for any field that might need to have the bulk velocity set.
+ if 'Velocity' in field or 'Mach' in field:
+ self._need_bulk_velocity = True
+
self.profile_fields.append({'field':field, 'weight_field':weight_field,
'accumulation':accumulation})
@@ -379,11 +400,15 @@
"""
+ # Check for any field that might need to have the bulk velocity set.
+ if 'Velocity' in field or 'Mach' in field:
+ self._need_bulk_velocity = True
+
self.projection_fields.append({'field':field, 'weight_field':weight_field,
'cmap': cmap})
@parallel_blocking_call
- def make_profiles(self, filename=None, prefilters=None, **kwargs):
+ def make_profiles(self, filename=None, prefilters=None, njobs=-1):
r"""Make radial profiles for all halos in the list.
After all the calls to `add_profile`, this will trigger the actual
@@ -394,7 +419,7 @@
filename : string
If set, a file will be written with all of the filtered halos
and the quantities returned by the filter functions.
- Default=None.
+ Default: None.
prefilters : array_like
A single dataset can contain thousands or tens of thousands of
halos. Significant time can be saved by not profiling halos
@@ -402,6 +427,11 @@
Simple filters based on quantities provided in the initial
halo list can be used to filter out unwanted halos using this
parameter.
+ Default: None.
+ njobs : int
+ The number of jobs over which to split the profiling. Set
+ to -1 so that each halo is done by a single processor.
+ Default: -1.
Examples
--------
@@ -454,7 +484,7 @@
# Profile all halos.
updated_halos = []
- for halo in parallel_objects(self.all_halos, -1):
+ for halo in parallel_objects(self.all_halos, njobs=njobs):
# Apply prefilters to avoid profiling unwanted halos.
filter_result = True
haloQuantities = {}
@@ -468,7 +498,8 @@
profile_filename = "%s/Halo_%04d_profile.dat" % (my_output_dir, halo['id'])
- profiledHalo = self._get_halo_profile(halo, profile_filename, virial_filter=virial_filter)
+ profiledHalo = self._get_halo_profile(halo, profile_filename,
+ virial_filter=virial_filter)
if profiledHalo is None:
continue
@@ -487,26 +518,26 @@
for quantity in self.filter_quantities:
if halo.has_key(quantity): haloQuantities[quantity] = halo[quantity]
- self.filtered_halos.append(haloQuantities)
+ only_on_root(self.filtered_halos.append, haloQuantities)
# If we've gotten this far down, this halo is good and we want
# to keep it. But we need to communicate the recentering changes
# to all processors (the root one in particular) without having
# one task clobber the other.
- updated_halos.append(halo)
-
+ only_on_root(updated_halos.append, halo)
+
# And here is where we bring it all together.
updated_halos = self.comm.par_combine_object(updated_halos,
datatype="list", op="cat")
- updated_halos.sort(key = lambda a:a['id'])
+ updated_halos.sort(key=lambda a:a['id'])
self.all_halos = updated_halos
self.filtered_halos = self.comm.par_combine_object(self.filtered_halos,
datatype="list", op="cat")
- self.filtered_halos.sort(key = lambda a:a['id'])
+ self.filtered_halos.sort(key=lambda a:a['id'])
if filename is not None:
- self._write_filtered_halo_list(filename, **kwargs)
+ self._write_filtered_halo_list(filename)
def _get_halo_profile(self, halo, filename, virial_filter=True,
force_write=False):
@@ -529,31 +560,13 @@
return None
# get a sphere object to profile
- sphere = get_halo_sphere(halo, self.pf, recenter=self.recenter)
+ sphere = self._get_halo_sphere(halo)
if sphere is None: return None
- if self._need_bulk_velocity:
- # Set bulk velocity to zero out radial velocity profiles.
- if self.velocity_center[0] == 'bulk':
- if self.velocity_center[1] == 'halo':
- sphere.set_field_parameter('bulk_velocity', halo['velocity'])
- elif self.velocity_center[1] == 'sphere':
- sphere.set_field_parameter('bulk_velocity',
- sphere.quantities['BulkVelocity'](lazy_reader=False,
- preload=False))
- else:
- mylog.error("Invalid parameter: VelocityCenter.")
- elif self.velocity_center[0] == 'max':
- max_grid, max_cell, max_value, max_location = \
- self.pf.h.find_max_cell_location(self.velocity_center[1])
- sphere.set_field_parameter('bulk_velocity', [max_grid['x-velocity'][max_cell],
- max_grid['y-velocity'][max_cell],
- max_grid['z-velocity'][max_cell]])
-
try:
profile = BinnedProfile1D(sphere, self.n_profile_bins, "RadiusMpc",
r_min, halo['r_max'],
- log_space=True, lazy_reader=False,
+ log_space=True, lazy_reader=True,
end_collect=True)
except EmptyProfileData:
mylog.error("Caught EmptyProfileData exception, returning None for this halo.")
@@ -586,9 +599,75 @@
return profile
+ def _get_halo_sphere(self, halo):
+ """
+ Returns a sphere object for a given halo, performs the recentering,
+ and calculates bulk velocities.
+ """
+
+ sphere = self.pf.h.sphere(halo['center'], halo['r_max']/self.pf.units['mpc'])
+ if len(sphere._grids) == 0: return None
+ new_sphere = False
+
+ if self.recenter:
+ old = halo['center']
+ if self.recenter in centering_registry:
+ new_x, new_y, new_z = \
+ centering_registry[self.recenter](sphere)
+ else:
+ # user supplied function
+ new_x, new_y, new_z = self.recenter(sphere)
+ if new_x < self.pf.domain_left_edge[0] or \
+ new_y < self.pf.domain_left_edge[1] or \
+ new_z < self.pf.domain_left_edge[2]:
+ mylog.info("Recentering rejected, skipping halo %d" % \
+ halo['id'])
+ return None
+ halo['center'] = [new_x, new_y, new_z]
+ d = self.pf['kpc'] * periodic_dist(old, halo['center'],
+ self.pf.domain_right_edge - self.pf.domain_left_edge)
+ mylog.info("Recentered halo %d %1.3e kpc away." % (halo['id'], d))
+ # Expand the halo to account for recentering.
+ halo['r_max'] += d / 1000. # d is in kpc -> want mpc
+ new_sphere = True
+
+ if new_sphere:
+ # Temporary solution to memory leak.
+ for g in self.pf.h.grids:
+ g.clear_data()
+ sphere.clear_data()
+ del sphere
+ sphere = self.pf.h.sphere(halo['center'], halo['r_max']/self.pf.units['mpc'])
+
+ if self._need_bulk_velocity:
+ # Set bulk velocity to zero out radial velocity profiles.
+ if self.velocity_center[0] == 'bulk':
+ if self.velocity_center[1] == 'halo':
+ sphere.set_field_parameter('bulk_velocity', halo['velocity'])
+ elif self.velocity_center[1] == 'sphere':
+ mylog.info('Calculating sphere bulk velocity.')
+ sphere.set_field_parameter('bulk_velocity',
+ sphere.quantities['BulkVelocity'](lazy_reader=True,
+ preload=False))
+ else:
+ mylog.error("Invalid parameter: velocity_center.")
+ return None
+ elif self.velocity_center[0] == 'max':
+ mylog.info('Setting bulk velocity with value at max %s.' % self.velocity_center[1])
+ max_val, maxi, mx, my, mz, mg = sphere.quantities['MaxLocation'](self.velocity_center[1],
+ lazy_reader=True)
+ max_grid = self.pf.h.grids[mg]
+ max_cell = na.unravel_index(maxi, max_grid.ActiveDimensions)
+ sphere.set_field_parameter('bulk_velocity', [max_grid['x-velocity'][max_cell],
+ max_grid['y-velocity'][max_cell],
+ max_grid['z-velocity'][max_cell]])
+ mylog.info('Bulk velocity set.')
+
+ return sphere
+
@parallel_blocking_call
def make_projections(self, axes=[0, 1, 2], halo_list='filtered',
- save_images=False, save_cube=True):
+ save_images=False, save_cube=True, njobs=-1):
r"""Make projections of all halos using specified fields.
After adding fields using `add_projection`, this starts the actual
@@ -608,6 +687,10 @@
save_cube : bool
Whether or not to save the HDF5 files of the halo projections.
Default=True.
+ njobs : int
+ The number of jobs over which to split the projections. Set
+ to -1 so that each halo is done by a single processor.
+ Default: -1.
Examples
--------
@@ -656,7 +739,7 @@
self.pf.parameters['DomainRightEdge'][w])
for w in range(self.pf.parameters['TopGridRank'])]
- for halo in parallel_objects(halo_projection_list, -1):
+ for halo in parallel_objects(halo_projection_list, njobs=njobs):
if halo is None:
continue
# Check if region will overlap domain edge.
@@ -745,7 +828,7 @@
@parallel_blocking_call
def analyze_halo_spheres(self, analysis_function, halo_list='filtered',
- analysis_output_dir=None):
+ analysis_output_dir=None, njobs=-1):
r"""Perform custom analysis on all halos.
This will loop through all halo on the HaloProfiler's list,
@@ -768,6 +851,10 @@
analysis_output_dir : string, optional
If specified, this directory will be created within the dataset to
contain any output from the analysis function. Default: None.
+ njobs : int
+ The number of jobs over which to split the analysis. Set
+ to -1 so that each halo is done by a single processor.
+ Default: -1.
Examples
--------
@@ -803,11 +890,11 @@
my_output_dir = "%s/%s" % (self.pf.fullpath, analysis_output_dir)
self.__check_directory(my_output_dir)
- for halo in parallel_objects(halo_analysis_list, -1):
+ for halo in parallel_objects(halo_analysis_list, njobs=njobs):
if halo is None: continue
# Get a sphere object to analze.
- sphere = get_halo_sphere(halo, self.pf, recenter=self.recenter)
+ sphere = self._get_halo_sphere(halo)
if sphere is None: continue
# Call the given analysis function.
@@ -1042,94 +1129,6 @@
else:
os.mkdir(my_output_dir)
-def get_halo_sphere(halo, pf, recenter=None):
- r"""Returns a sphere object for a given halo.
-
- With a dictionary containing halo properties, such as center
- and r_max, this creates a sphere object and optionally
- recenters and recreates the sphere using a recentering function.
- This is to be used primarily to make spheres for a set of halos
- loaded by the HaloProfiler.
-
- Parameters
- ----------
- halo : dict, required
- The dictionary containing halo properties used to make the sphere.
- Required entries:
- center : list with center coordinates.
- r_max : sphere radius in Mpc.
- pf : parameter file object, required
- The parameter file from which the sphere will be made.
- recenter : {None, string or function}
- The exact location of the sphere center can significantly affect
- radial profiles. The halo center loaded by the HaloProfiler will
- typically be the dark matter center of mass calculated by a halo
- finder. However, this may not be the best location for centering
- profiles of baryon quantities. For example, one may want to center
- on the maximum density.
- If recenter is given as a string, one of the existing recentering
- functions will be used:
- Min_Dark_Matter_Density : location of minimum dark matter density
- Max_Dark_Matter_Density : location of maximum dark matter density
- CoM_Dark_Matter_Density : dark matter center of mass
- Min_Gas_Density : location of minimum gas density
- Max_Gas_Density : location of maximum gas density
- CoM_Gas_Density : gas center of mass
- Min_Total_Density : location of minimum total density
- Max_Total_Density : location of maximum total density
- CoM_Total_Density : total center of mass
- Min_Temperature : location of minimum temperature
- Max_Temperature : location of maximum temperature
- Alternately, a function can be supplied for custom recentering.
- The function should take only one argument, a sphere object.
- Example function:
- def my_center_of_mass(data):
- my_x, my_y, my_z = data.quantities['CenterOfMass']()
- return (my_x, my_y, my_z)
-
- Examples: this should primarily be used with the halo list of the HaloProfiler.
- This is an example with an abstract halo asssuming a pre-defined pf.
- >>> halo = {'center': [0.5, 0.5, 0.5], 'r_max': 1.0}
- >>> my_sphere = get_halo_sphere(halo, pf, recenter='Max_Gas_Density')
- >>> # Assuming the above example function has been defined.
- >>> my_sphere = get_halo_sphere(halo, pf, recenter=my_center_of_mass)
- """
-
- sphere = pf.h.sphere(halo['center'], halo['r_max']/pf.units['mpc'])
- if len(sphere._grids) == 0: return None
- new_sphere = False
-
- if recenter:
- old = halo['center']
- if recenter in centering_registry:
- new_x, new_y, new_z = \
- centering_registry[recenter](sphere)
- else:
- # user supplied function
- new_x, new_y, new_z = recenter(sphere)
- if new_x < pf.domain_left_edge[0] or \
- new_y < pf.domain_left_edge[1] or \
- new_z < pf.domain_left_edge[2]:
- mylog.info("Recentering rejected, skipping halo %d" % \
- halo['id'])
- return None
- halo['center'] = [new_x, new_y, new_z]
- d = pf['kpc'] * periodic_dist(old, halo['center'],
- pf.domain_right_edge - pf.domain_left_edge)
- mylog.info("Recentered halo %d %1.3e kpc away." % (halo['id'], d))
- # Expand the halo to account for recentering.
- halo['r_max'] += d / 1000 # d is in kpc -> want mpc
- new_sphere = True
-
- if new_sphere:
- # Temporary solution to memory leak.
- for g in pf.h.grids:
- g.clear_data()
- sphere.clear_data()
- del sphere
- sphere = pf.h.sphere(halo['center'], halo['r_max']/pf.units['mpc'])
- return sphere
-
def _shift_projections(pf, projections, oldCenter, newCenter, axis):
"""
Shift projection data around.
https://bitbucket.org/yt_analysis/yt/changeset/3126bfa744b1/
changeset: 3126bfa744b1
branch: yt
user: MatthewTurk
date: 2012-05-31 20:59:51
summary: Merged in brittonsmith/yt (pull request #165)
affected #: 3 files
diff -r b71a07e830b674d89761f3e6019850855601c13b -r 3126bfa744b1e0eb13b3a99627efbe9015ffadac yt/analysis_modules/halo_profiler/api.py
--- a/yt/analysis_modules/halo_profiler/api.py
+++ b/yt/analysis_modules/halo_profiler/api.py
@@ -34,5 +34,4 @@
from .multi_halo_profiler import \
HaloProfiler, \
FakeProfile, \
- get_halo_sphere, \
standard_fields
diff -r b71a07e830b674d89761f3e6019850855601c13b -r 3126bfa744b1e0eb13b3a99627efbe9015ffadac yt/analysis_modules/halo_profiler/centering_methods.py
--- a/yt/analysis_modules/halo_profiler/centering_methods.py
+++ b/yt/analysis_modules/halo_profiler/centering_methods.py
@@ -43,14 +43,14 @@
@add_function("Min_Dark_Matter_Density")
def find_minimum_dm_density(data):
ma, maxi, mx, my, mz, mg = data.quantities['MinLocation']('Dark_Matter_Density',
- lazy_reader=False,
+ lazy_reader=True,
preload=False)
return (mx, my, mz)
@add_function("Max_Dark_Matter_Density")
def find_maximum_dm_density(data):
ma, maxi, mx, my, mz, mg = data.quantities['MaxLocation']('Dark_Matter_Density',
- lazy_reader=False,
+ lazy_reader=True,
preload=False)
return (mx, my, mz)
@@ -58,7 +58,7 @@
def find_CoM_dm_density(data):
dc_x, dc_y, dc_z = data.quantities['CenterOfMass'](use_cells=False,
use_particles=True,
- lazy_reader=False,
+ lazy_reader=True,
preload=False)
return (dc_x, dc_y, dc_z)
@@ -67,14 +67,14 @@
@add_function("Min_Gas_Density")
def find_minimum_gas_density(data):
ma, maxi, mx, my, mz, mg = data.quantities['MinLocation']('Density',
- lazy_reader=False,
+ lazy_reader=True,
preload=False)
return (mx, my, mz)
@add_function("Max_Gas_Density")
def find_maximum_gas_density(data):
ma, maxi, mx, my, mz, mg = data.quantities['MaxLocation']('Density',
- lazy_reader=False,
+ lazy_reader=True,
preload=False)
return (mx, my, mz)
@@ -82,7 +82,7 @@
def find_CoM_gas_density(data):
dc_x, dc_y, dc_z = data.quantities['CenterOfMass'](use_cells=True,
use_particles=False,
- lazy_reader=False,
+ lazy_reader=True,
preload=False)
return (dc_x, dc_y, dc_z)
@@ -91,14 +91,14 @@
@add_function("Min_Total_Density")
def find_minimum_total_density(data):
ma, maxi, mx, my, mz, mg = data.quantities['MinLocation']('Matter_Density',
- lazy_reader=False,
+ lazy_reader=True,
preload=False)
return (mx, my, mz)
@add_function("Max_Total_Density")
def find_maximum_total_density(data):
ma, maxi, mx, my, mz, mg = data.quantities['MaxLocation']('Matter_Density',
- lazy_reader=False,
+ lazy_reader=True,
preload=False)
return (mx, my, mz)
@@ -106,7 +106,7 @@
def find_CoM_total_density(data):
dc_x, dc_y, dc_z = data.quantities['CenterOfMass'](use_cells=True,
use_particles=True,
- lazy_reader=False,
+ lazy_reader=True,
preload=False)
return (dc_x, dc_y, dc_z)
@@ -115,14 +115,14 @@
@add_function("Min_Temperature")
def find_minimum_temperature(data):
ma, mini, mx, my, mz, mg = data.quantities['MinLocation']('Temperature',
- lazy_reader=False,
+ lazy_reader=True,
preload=False)
return (mx, my, mz)
@add_function("Max_Temperature")
def find_maximum_temperature(data):
ma, maxi, mx, my, mz, mg = data.quantities['MaxLocation']('Temperature',
- lazy_reader=False,
+ lazy_reader=True,
preload=False)
return (mx, my, mz)
diff -r b71a07e830b674d89761f3e6019850855601c13b -r 3126bfa744b1e0eb13b3a99627efbe9015ffadac yt/analysis_modules/halo_profiler/multi_halo_profiler.py
--- a/yt/analysis_modules/halo_profiler/multi_halo_profiler.py
+++ b/yt/analysis_modules/halo_profiler/multi_halo_profiler.py
@@ -64,7 +64,7 @@
dm_only=False, resize=True,
fancy_padding=True, rearrange=True),
halo_radius=None, radius_units='1', n_profile_bins=50,
- recenter = None,
+ recenter=None,
profile_output_dir='radial_profiles', projection_output_dir='projections',
projection_width=8.0, projection_width_units='mpc', project_at_level='max',
velocity_center=['bulk', 'halo'], filter_quantities=['id', 'center', 'r_max'],
@@ -111,8 +111,32 @@
Args given with call to halo finder function. Default: None.
halo_finder_kwargs : dict
kwargs given with call to halo finder function. Default: None.
- recenter : {string, function
- The name of a function that recenters the halo for analysis.
+ recenter : {string, function}
+ The exact location of the sphere center can significantly affect
+ radial profiles. The halo center loaded by the HaloProfiler will
+ typically be the dark matter center of mass calculated by a halo
+ finder. However, this may not be the best location for centering
+ profiles of baryon quantities. For example, one may want to center
+ on the maximum density.
+ If recenter is given as a string, one of the existing recentering
+ functions will be used:
+ Min_Dark_Matter_Density : location of minimum dark matter density
+ Max_Dark_Matter_Density : location of maximum dark matter density
+ CoM_Dark_Matter_Density : dark matter center of mass
+ Min_Gas_Density : location of minimum gas density
+ Max_Gas_Density : location of maximum gas density
+ CoM_Gas_Density : gas center of mass
+ Min_Total_Density : location of minimum total density
+ Max_Total_Density : location of maximum total density
+ CoM_Total_Density : total center of mass
+ Min_Temperature : location of minimum temperature
+ Max_Temperature : location of maximum temperature
+ Alternately, a function can be supplied for custom recentering.
+ The function should take only one argument, a sphere object.
+ Example function:
+ def my_center_of_mass(data):
+ my_x, my_y, my_z = data.quantities['CenterOfMass']()
+ return (my_x, my_y, my_z)
Default: None.
halo_radius : float
If no halo radii are provided in the halo list file, this
@@ -148,8 +172,7 @@
* ["bulk", "sphere"]: the bulk velocity of the sphere
centered on the halo center.
* ["max", field]: the velocity of the cell that is the
- location of the maximum of the field
- specified (used only when halos set to single).
+ location of the maximum of the field specified.
filter_quantities : array_like
Quantities from the original halo list file to be written out in the
filtered list file. Default: ['id','center'].
@@ -161,8 +184,8 @@
Examples
--------
- >>> import yt.analysis_modules.halo_profiler.api as HP
- >>> hp = HP.halo_profiler("DD0242/DD0242")
+ >>> from yt.analysis_modules.halo_profiler.api import *
+ >>> hp = HaloProfiler("RedshiftOutput0005/RD0005")
"""
ParallelAnalysisInterface.__init__(self)
@@ -226,13 +249,9 @@
# Option to recenter sphere someplace else.
self.recenter = recenter
- # Look for any field that might need to have the bulk velocity set.
+ # Flag for whether calculating halo bulk velocity is necessary.
self._need_bulk_velocity = False
- for field in [hp['field'] for hp in self.profile_fields]:
- if 'Velocity' in field or 'Mach' in field:
- self._need_bulk_velocity = True
- break
-
+
# Check validity for VelocityCenter parameter which toggles how the
# velocity is zeroed out for radial velocity profiles.
self.velocity_center = velocity_center[:]
@@ -250,9 +269,7 @@
mylog.error("Second value of VelocityCenter must be either 'halo' or 'sphere' if first value is 'bulk'.")
return None
elif self.velocity_center[0] == 'max':
- if self.halos is 'multiple':
- mylog.error("Getting velocity center from a max field value only works with halos='single'.")
- return None
+ mylog.info('Using position of max %s for velocity center.' % self.velocity_center[1])
else:
mylog.error("First value of parameter, VelocityCenter, must be either 'bulk' or 'max'.")
return None
@@ -284,7 +301,7 @@
mylog.error("No halos loaded, there will be nothing to do.")
return None
else:
- mylog.error("I don't know whether to get halos from hop or from density maximum. This should not have happened.")
+ mylog.error("Keyword, halos, must be either 'single' or 'multiple'.")
return None
def add_halo_filter(self, function, *args, **kwargs):
@@ -351,6 +368,10 @@
"""
+ # Check for any field that might need to have the bulk velocity set.
+ if 'Velocity' in field or 'Mach' in field:
+ self._need_bulk_velocity = True
+
self.profile_fields.append({'field':field, 'weight_field':weight_field,
'accumulation':accumulation})
@@ -379,11 +400,15 @@
"""
+ # Check for any field that might need to have the bulk velocity set.
+ if 'Velocity' in field or 'Mach' in field:
+ self._need_bulk_velocity = True
+
self.projection_fields.append({'field':field, 'weight_field':weight_field,
'cmap': cmap})
@parallel_blocking_call
- def make_profiles(self, filename=None, prefilters=None, **kwargs):
+ def make_profiles(self, filename=None, prefilters=None, njobs=-1):
r"""Make radial profiles for all halos in the list.
After all the calls to `add_profile`, this will trigger the actual
@@ -394,7 +419,7 @@
filename : string
If set, a file will be written with all of the filtered halos
and the quantities returned by the filter functions.
- Default=None.
+ Default: None.
prefilters : array_like
A single dataset can contain thousands or tens of thousands of
halos. Significant time can be saved by not profiling halos
@@ -402,6 +427,11 @@
Simple filters based on quantities provided in the initial
halo list can be used to filter out unwanted halos using this
parameter.
+ Default: None.
+ njobs : int
+ The number of jobs over which to split the profiling. Set
+ to -1 so that each halo is done by a single processor.
+ Default: -1.
Examples
--------
@@ -454,7 +484,7 @@
# Profile all halos.
updated_halos = []
- for halo in parallel_objects(self.all_halos, -1):
+ for halo in parallel_objects(self.all_halos, njobs=njobs):
# Apply prefilters to avoid profiling unwanted halos.
filter_result = True
haloQuantities = {}
@@ -468,7 +498,8 @@
profile_filename = "%s/Halo_%04d_profile.dat" % (my_output_dir, halo['id'])
- profiledHalo = self._get_halo_profile(halo, profile_filename, virial_filter=virial_filter)
+ profiledHalo = self._get_halo_profile(halo, profile_filename,
+ virial_filter=virial_filter)
if profiledHalo is None:
continue
@@ -487,26 +518,26 @@
for quantity in self.filter_quantities:
if halo.has_key(quantity): haloQuantities[quantity] = halo[quantity]
- self.filtered_halos.append(haloQuantities)
+ only_on_root(self.filtered_halos.append, haloQuantities)
# If we've gotten this far down, this halo is good and we want
# to keep it. But we need to communicate the recentering changes
# to all processors (the root one in particular) without having
# one task clobber the other.
- updated_halos.append(halo)
-
+ only_on_root(updated_halos.append, halo)
+
# And here is where we bring it all together.
updated_halos = self.comm.par_combine_object(updated_halos,
datatype="list", op="cat")
- updated_halos.sort(key = lambda a:a['id'])
+ updated_halos.sort(key=lambda a:a['id'])
self.all_halos = updated_halos
self.filtered_halos = self.comm.par_combine_object(self.filtered_halos,
datatype="list", op="cat")
- self.filtered_halos.sort(key = lambda a:a['id'])
+ self.filtered_halos.sort(key=lambda a:a['id'])
if filename is not None:
- self._write_filtered_halo_list(filename, **kwargs)
+ self._write_filtered_halo_list(filename)
def _get_halo_profile(self, halo, filename, virial_filter=True,
force_write=False):
@@ -529,31 +560,13 @@
return None
# get a sphere object to profile
- sphere = get_halo_sphere(halo, self.pf, recenter=self.recenter)
+ sphere = self._get_halo_sphere(halo)
if sphere is None: return None
- if self._need_bulk_velocity:
- # Set bulk velocity to zero out radial velocity profiles.
- if self.velocity_center[0] == 'bulk':
- if self.velocity_center[1] == 'halo':
- sphere.set_field_parameter('bulk_velocity', halo['velocity'])
- elif self.velocity_center[1] == 'sphere':
- sphere.set_field_parameter('bulk_velocity',
- sphere.quantities['BulkVelocity'](lazy_reader=False,
- preload=False))
- else:
- mylog.error("Invalid parameter: VelocityCenter.")
- elif self.velocity_center[0] == 'max':
- max_grid, max_cell, max_value, max_location = \
- self.pf.h.find_max_cell_location(self.velocity_center[1])
- sphere.set_field_parameter('bulk_velocity', [max_grid['x-velocity'][max_cell],
- max_grid['y-velocity'][max_cell],
- max_grid['z-velocity'][max_cell]])
-
try:
profile = BinnedProfile1D(sphere, self.n_profile_bins, "RadiusMpc",
r_min, halo['r_max'],
- log_space=True, lazy_reader=False,
+ log_space=True, lazy_reader=True,
end_collect=True)
except EmptyProfileData:
mylog.error("Caught EmptyProfileData exception, returning None for this halo.")
@@ -586,9 +599,75 @@
return profile
+ def _get_halo_sphere(self, halo):
+ """
+ Returns a sphere object for a given halo, performs the recentering,
+ and calculates bulk velocities.
+ """
+
+ sphere = self.pf.h.sphere(halo['center'], halo['r_max']/self.pf.units['mpc'])
+ if len(sphere._grids) == 0: return None
+ new_sphere = False
+
+ if self.recenter:
+ old = halo['center']
+ if self.recenter in centering_registry:
+ new_x, new_y, new_z = \
+ centering_registry[self.recenter](sphere)
+ else:
+ # user supplied function
+ new_x, new_y, new_z = self.recenter(sphere)
+ if new_x < self.pf.domain_left_edge[0] or \
+ new_y < self.pf.domain_left_edge[1] or \
+ new_z < self.pf.domain_left_edge[2]:
+ mylog.info("Recentering rejected, skipping halo %d" % \
+ halo['id'])
+ return None
+ halo['center'] = [new_x, new_y, new_z]
+ d = self.pf['kpc'] * periodic_dist(old, halo['center'],
+ self.pf.domain_right_edge - self.pf.domain_left_edge)
+ mylog.info("Recentered halo %d %1.3e kpc away." % (halo['id'], d))
+ # Expand the halo to account for recentering.
+ halo['r_max'] += d / 1000. # d is in kpc -> want mpc
+ new_sphere = True
+
+ if new_sphere:
+ # Temporary solution to memory leak.
+ for g in self.pf.h.grids:
+ g.clear_data()
+ sphere.clear_data()
+ del sphere
+ sphere = self.pf.h.sphere(halo['center'], halo['r_max']/self.pf.units['mpc'])
+
+ if self._need_bulk_velocity:
+ # Set bulk velocity to zero out radial velocity profiles.
+ if self.velocity_center[0] == 'bulk':
+ if self.velocity_center[1] == 'halo':
+ sphere.set_field_parameter('bulk_velocity', halo['velocity'])
+ elif self.velocity_center[1] == 'sphere':
+ mylog.info('Calculating sphere bulk velocity.')
+ sphere.set_field_parameter('bulk_velocity',
+ sphere.quantities['BulkVelocity'](lazy_reader=True,
+ preload=False))
+ else:
+ mylog.error("Invalid parameter: velocity_center.")
+ return None
+ elif self.velocity_center[0] == 'max':
+ mylog.info('Setting bulk velocity with value at max %s.' % self.velocity_center[1])
+ max_val, maxi, mx, my, mz, mg = sphere.quantities['MaxLocation'](self.velocity_center[1],
+ lazy_reader=True)
+ max_grid = self.pf.h.grids[mg]
+ max_cell = na.unravel_index(maxi, max_grid.ActiveDimensions)
+ sphere.set_field_parameter('bulk_velocity', [max_grid['x-velocity'][max_cell],
+ max_grid['y-velocity'][max_cell],
+ max_grid['z-velocity'][max_cell]])
+ mylog.info('Bulk velocity set.')
+
+ return sphere
+
@parallel_blocking_call
def make_projections(self, axes=[0, 1, 2], halo_list='filtered',
- save_images=False, save_cube=True):
+ save_images=False, save_cube=True, njobs=-1):
r"""Make projections of all halos using specified fields.
After adding fields using `add_projection`, this starts the actual
@@ -608,6 +687,10 @@
save_cube : bool
Whether or not to save the HDF5 files of the halo projections.
Default=True.
+ njobs : int
+ The number of jobs over which to split the projections. Set
+ to -1 so that each halo is done by a single processor.
+ Default: -1.
Examples
--------
@@ -656,7 +739,7 @@
self.pf.parameters['DomainRightEdge'][w])
for w in range(self.pf.parameters['TopGridRank'])]
- for halo in parallel_objects(halo_projection_list, -1):
+ for halo in parallel_objects(halo_projection_list, njobs=njobs):
if halo is None:
continue
# Check if region will overlap domain edge.
@@ -745,7 +828,7 @@
@parallel_blocking_call
def analyze_halo_spheres(self, analysis_function, halo_list='filtered',
- analysis_output_dir=None):
+ analysis_output_dir=None, njobs=-1):
r"""Perform custom analysis on all halos.
This will loop through all halo on the HaloProfiler's list,
@@ -768,6 +851,10 @@
analysis_output_dir : string, optional
If specified, this directory will be created within the dataset to
contain any output from the analysis function. Default: None.
+ njobs : int
+ The number of jobs over which to split the analysis. Set
+ to -1 so that each halo is done by a single processor.
+ Default: -1.
Examples
--------
@@ -803,11 +890,11 @@
my_output_dir = "%s/%s" % (self.pf.fullpath, analysis_output_dir)
self.__check_directory(my_output_dir)
- for halo in parallel_objects(halo_analysis_list, -1):
+ for halo in parallel_objects(halo_analysis_list, njobs=njobs):
if halo is None: continue
# Get a sphere object to analze.
- sphere = get_halo_sphere(halo, self.pf, recenter=self.recenter)
+ sphere = self._get_halo_sphere(halo)
if sphere is None: continue
# Call the given analysis function.
@@ -1042,94 +1129,6 @@
else:
os.mkdir(my_output_dir)
-def get_halo_sphere(halo, pf, recenter=None):
- r"""Returns a sphere object for a given halo.
-
- With a dictionary containing halo properties, such as center
- and r_max, this creates a sphere object and optionally
- recenters and recreates the sphere using a recentering function.
- This is to be used primarily to make spheres for a set of halos
- loaded by the HaloProfiler.
-
- Parameters
- ----------
- halo : dict, required
- The dictionary containing halo properties used to make the sphere.
- Required entries:
- center : list with center coordinates.
- r_max : sphere radius in Mpc.
- pf : parameter file object, required
- The parameter file from which the sphere will be made.
- recenter : {None, string or function}
- The exact location of the sphere center can significantly affect
- radial profiles. The halo center loaded by the HaloProfiler will
- typically be the dark matter center of mass calculated by a halo
- finder. However, this may not be the best location for centering
- profiles of baryon quantities. For example, one may want to center
- on the maximum density.
- If recenter is given as a string, one of the existing recentering
- functions will be used:
- Min_Dark_Matter_Density : location of minimum dark matter density
- Max_Dark_Matter_Density : location of maximum dark matter density
- CoM_Dark_Matter_Density : dark matter center of mass
- Min_Gas_Density : location of minimum gas density
- Max_Gas_Density : location of maximum gas density
- CoM_Gas_Density : gas center of mass
- Min_Total_Density : location of minimum total density
- Max_Total_Density : location of maximum total density
- CoM_Total_Density : total center of mass
- Min_Temperature : location of minimum temperature
- Max_Temperature : location of maximum temperature
- Alternately, a function can be supplied for custom recentering.
- The function should take only one argument, a sphere object.
- Example function:
- def my_center_of_mass(data):
- my_x, my_y, my_z = data.quantities['CenterOfMass']()
- return (my_x, my_y, my_z)
-
- Examples: this should primarily be used with the halo list of the HaloProfiler.
- This is an example with an abstract halo asssuming a pre-defined pf.
- >>> halo = {'center': [0.5, 0.5, 0.5], 'r_max': 1.0}
- >>> my_sphere = get_halo_sphere(halo, pf, recenter='Max_Gas_Density')
- >>> # Assuming the above example function has been defined.
- >>> my_sphere = get_halo_sphere(halo, pf, recenter=my_center_of_mass)
- """
-
- sphere = pf.h.sphere(halo['center'], halo['r_max']/pf.units['mpc'])
- if len(sphere._grids) == 0: return None
- new_sphere = False
-
- if recenter:
- old = halo['center']
- if recenter in centering_registry:
- new_x, new_y, new_z = \
- centering_registry[recenter](sphere)
- else:
- # user supplied function
- new_x, new_y, new_z = recenter(sphere)
- if new_x < pf.domain_left_edge[0] or \
- new_y < pf.domain_left_edge[1] or \
- new_z < pf.domain_left_edge[2]:
- mylog.info("Recentering rejected, skipping halo %d" % \
- halo['id'])
- return None
- halo['center'] = [new_x, new_y, new_z]
- d = pf['kpc'] * periodic_dist(old, halo['center'],
- pf.domain_right_edge - pf.domain_left_edge)
- mylog.info("Recentered halo %d %1.3e kpc away." % (halo['id'], d))
- # Expand the halo to account for recentering.
- halo['r_max'] += d / 1000 # d is in kpc -> want mpc
- new_sphere = True
-
- if new_sphere:
- # Temporary solution to memory leak.
- for g in pf.h.grids:
- g.clear_data()
- sphere.clear_data()
- del sphere
- sphere = pf.h.sphere(halo['center'], halo['r_max']/pf.units['mpc'])
- return sphere
-
def _shift_projections(pf, projections, oldCenter, newCenter, axis):
"""
Shift projection data around.
Repository URL: https://bitbucket.org/yt_analysis/yt/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the yt-svn
mailing list