[Yt-svn] yt-commit r949 - in branches/yt-non-3d: data yt yt/extensions/lightcone yt/lagos
mturk at wrangler.dreamhost.com
mturk at wrangler.dreamhost.com
Wed Nov 19 09:24:32 PST 2008
Author: mturk
Date: Wed Nov 19 09:24:29 2008
New Revision: 949
URL: http://yt.spacepope.org/changeset/949
Log:
Merging back from trunk, adding a fieldInfo lookalike in mods and lagos (if you
import from mods...)
Added:
branches/yt-non-3d/yt/extensions/lightcone/UniqueSolution.py
- copied unchanged from r948, /trunk/yt/extensions/lightcone/UniqueSolution.py
Modified:
branches/yt-non-3d/data/make_slice.py
branches/yt-non-3d/yt/extensions/lightcone/LightCone.py
branches/yt-non-3d/yt/extensions/lightcone/LightConeProjection.py
branches/yt-non-3d/yt/extensions/lightcone/__init__.py
branches/yt-non-3d/yt/lagos/BaseDataTypes.py
branches/yt-non-3d/yt/lagos/DataReadingFuncs.py
branches/yt-non-3d/yt/lagos/HierarchyType.py
branches/yt-non-3d/yt/lagos/ParallelTools.py
branches/yt-non-3d/yt/mods.py
Modified: branches/yt-non-3d/data/make_slice.py
==============================================================================
--- branches/yt-non-3d/data/make_slice.py (original)
+++ branches/yt-non-3d/data/make_slice.py Wed Nov 19 09:24:29 2008
@@ -1,5 +1,6 @@
from yt.mods import *
+
pf = EnzoStaticOutput("DD0003/sb_L2x2_0003")
EnzoFieldInfo["VelocityMagnitude"].take_log = True
Modified: branches/yt-non-3d/yt/extensions/lightcone/LightCone.py
==============================================================================
--- branches/yt-non-3d/yt/extensions/lightcone/LightCone.py (original)
+++ branches/yt-non-3d/yt/extensions/lightcone/LightCone.py Wed Nov 19 09:24:29 2008
@@ -25,13 +25,16 @@
from yt.extensions.lightcone import *
from yt.logger import lagosLogger as mylog
+import copy
+import os
import numpy as na
import random as rand
import tables as h5
from Common_nVolume import *
class LightCone(object):
- def __init__(self,EnzoParameterFile,LightConeParameterFile):
+ def __init__(self,EnzoParameterFile,LightConeParameterFile,verbose=True):
+ self.verbose = verbose
self.EnzoParameterFile = EnzoParameterFile
self.LightConeParameterFile = LightConeParameterFile
self.enzoParameters = {}
@@ -40,6 +43,7 @@
self.timeOutputs = []
self.allOutputs = []
self.lightConeSolution = []
+ self.masterSolution = [] # kept to compare with recycled solutions
self.projectionStack = []
self.projectionWeightFieldStack = []
@@ -54,6 +58,14 @@
self._ReadLightConeParameterFile()
self._ReadEnzoParameterFile()
+ # Create output directory.
+ if (os.path.exists(self.lightConeParameters['OutputDir'])):
+ if not(os.path.isdir(self.lightConeParameters['OutputDir'])):
+ mylog.error("Output directory exists, but is not a directory: %s." % self.lightConeParameters['OutputDir'])
+ self.lightConeParameters['OutputDir'] = './'
+ else:
+ os.mkdir(self.lightConeParameters['OutputDir'])
+
# Calculate redshifts for dt data dumps.
if (self.enzoParameters.has_key('dtDataDump')):
self._CalculateTimeDumpRedshifts()
@@ -71,7 +83,7 @@
self.recycleSolution = False
if seed is not None:
- self.lightConeParameters['RandomSeed'] = seed
+ self.lightConeParameters['RandomSeed'] = int(seed)
# Make light cone using minimum number of projections.
if (self.lightConeParameters['UseMinimumNumberOfProjections']):
@@ -94,14 +106,14 @@
while (z > output['redshift']):
output = output['previous']
if (output is None):
- mylog.error("CalculateLightConeSolution: search for data output went off the end the stack.")
- mylog.error("Could not calculate light cone solution.")
+ if self.verbose: mylog.error("CalculateLightConeSolution: search for data output went off the end the stack.")
+ if self.verbose: mylog.error("Could not calculate light cone solution.")
return
if (output['redshift'] == self.lightConeSolution[-1]['redshift']):
- mylog.error("CalculateLightConeSolution: No data dump between z = %f and %f." % \
+ if self.verbose: mylog.error("CalculateLightConeSolution: No data dump between z = %f and %f." % \
((self.lightConeSolution[-1]['redshift'] - self.lightConeSolution[-1]['deltaz']),
self.lightConeSolution[-1]['redshift']))
- mylog.error("Could not calculate light cone solution.")
+ if self.verbose: mylog.error("Could not calculate light cone solution.")
return
self.lightConeSolution.append(output)
z = self.lightConeSolution[-1]['redshift'] - self.lightConeSolution[-1]['deltaz']
@@ -123,7 +135,7 @@
self.lightConeSolution.append(nextOutput)
nextOutput = nextOutput['next']
- mylog.info("CalculateLightConeSolution: Used %d data dumps to get from z = %f to %f." % (len(self.lightConeSolution),
+ if self.verbose: mylog.info("CalculateLightConeSolution: Used %d data dumps to get from z = %f to %f." % (len(self.lightConeSolution),
self.lightConeParameters['InitialRedshift'],
self.lightConeParameters['FinalRedshift']))
@@ -145,9 +157,9 @@
self.enzoParameters['CosmologyHubbleConstantNow'] / self.enzoParameters['CosmologyComovingBoxSize']
# Simple error check to make sure more than 100% of box depth is never required.
if (self.lightConeSolution[q]['DepthBoxFraction'] > 1.0):
- mylog.error("Warning: box fraction required to go from z = %f to %f is %f" % (self.lightConeSolution[q]['redshift'],z_next,
+ if self.verbose: mylog.error("Warning: box fraction required to go from z = %f to %f is %f" % (self.lightConeSolution[q]['redshift'],z_next,
self.lightConeSolution[q]['DepthBoxFraction']))
- mylog.error("Full box delta z is %f, but it is %f to the next data dump." % (self.lightConeSolution[q]['deltaz'],
+ if self.verbose: mylog.error("Full box delta z is %f, but it is %f to the next data dump." % (self.lightConeSolution[q]['deltaz'],
self.lightConeSolution[q]['redshift']-z_next))
# Calculate fraction of box required for width corresponding to requested image size.
@@ -161,6 +173,9 @@
self.lightConeSolution[q]['ProjectionAxis'] = rand.randint(0,2)
self.lightConeSolution[q]['ProjectionCenter'] = [rand.random(),rand.random(),rand.random()]
+ # Store this as the mast solution.
+ self.masterSolution = [copy.deepcopy(q) for q in self.lightConeSolution]
+
# Clear out some stuff.
del self.allOutputs
del self.redshiftOutputs
@@ -170,6 +185,12 @@
def ProjectLightCone(self,field,weight_field=None,**kwargs):
"Create projections for light cone, then add them together."
+ # Clear projection stack.
+ self.projectionStack = []
+ self.projectionWeightFieldStack = []
+ if (self.lightConeSolution[-1].has_key('object')):
+ del self.lightConeSolution[-1]['object']
+
if not(self.lightConeParameters['OutputDir'].endswith("/")):
self.lightConeParameters['OutputDir'] += "/"
@@ -179,7 +200,7 @@
for q,output in enumerate(self.lightConeSolution):
name = "%s%s_%04d_%04d" % (self.lightConeParameters['OutputDir'],self.lightConeParameters['OutputPrefix'],
q,len(self.lightConeSolution))
- output['object'] = EnzoStaticOutput(output['filename'])
+ output['object'] = lagos.EnzoStaticOutput(output['filename'])
frb = LightConeProjection(output,field,pixels,weight_field=weight_field,
save_image=self.lightConeParameters['SaveLightConeSlices'],
name=name,**kwargs)
@@ -234,12 +255,12 @@
"""
if recycle:
- mylog.info("Recycling solution made with %s with new seed %s." % (self.lightConeParameters['RandomSeed'],
+ if self.verbose: mylog.info("Recycling solution made with %s with new seed %s." % (self.lightConeParameters['RandomSeed'],
newSeed))
- self.recycleRandomSeed = newSeed
+ self.recycleRandomSeed = int(newSeed)
else:
- mylog.info("Creating new solution with random seed %s." % newSeed)
- self.lightConeParameters['RandomSeed'] = newSeed
+ if self.verbose: mylog.info("Creating new solution with random seed %s." % newSeed)
+ self.lightConeParameters['RandomSeed'] = int(newSeed)
self.recycleRandomSeed = 0
self.recycleSolution = recycle
@@ -249,16 +270,18 @@
totalVolume = 0.0
# Seed random number generator with new seed.
- rand.seed(newSeed)
+ rand.seed(int(newSeed))
for q,output in enumerate(self.lightConeSolution):
# It is necessary to make the same number of calls to the random number generator
# so the original solution willbe produced if the same seed is given.
# Get random projection axis and center.
- # If recyclsing, axis will get thrown away since it is used in creating a unique projection object.
+ # If recycling, axis will get thrown away since it is used in creating a unique projection object.
newAxis = rand.randint(0,2)
- if not recycle:
+ if recycle:
+ output['ProjectionAxis'] = self.masterSolution[q]['ProjectionAxis']
+ else:
output['ProjectionAxis'] = newAxis
newCenter = [rand.random(),rand.random(),rand.random()]
@@ -267,20 +290,22 @@
newCube = na.zeros(shape=(len(newCenter),2))
oldCube = na.zeros(shape=(len(newCenter),2))
for w in range(len(newCenter)):
+ if (w == self.masterSolution[q]['ProjectionAxis']):
+ oldCube[w] = [self.masterSolution[q]['ProjectionCenter'][w] - 0.5 * self.masterSolution[q]['DepthBoxFraction'],
+ self.masterSolution[q]['ProjectionCenter'][w] + 0.5 * self.masterSolution[q]['DepthBoxFraction']]
+ else:
+ oldCube[w] = [self.masterSolution[q]['ProjectionCenter'][w] - 0.5 * self.masterSolution[q]['WidthBoxFraction'],
+ self.masterSolution[q]['ProjectionCenter'][w] + 0.5 * self.masterSolution[q]['WidthBoxFraction']]
+
if (w == output['ProjectionAxis']):
- oldCube[w] = [output['ProjectionCenter'][w] - 0.5 * output['DepthBoxFraction'],
- output['ProjectionCenter'][w] + 0.5 * output['DepthBoxFraction']]
- # If recycling old solution, then keep center for axis in line of sight.
if recycle:
newCube[w] = oldCube[w]
else:
- newCube[w] = [newCenter[w] - 0.5 * output['DepthBoxFraction'],
- newCenter[w] + 0.5 * output['DepthBoxFraction']]
+ newCube[w] = [newCenter[w] - 0.5 * self.masterSolution[q]['DepthBoxFraction'],
+ newCenter[w] + 0.5 * self.masterSolution[q]['DepthBoxFraction']]
else:
- oldCube[w] = [output['ProjectionCenter'][w] - 0.5 * output['WidthBoxFraction'],
- output['ProjectionCenter'][w] + 0.5 * output['WidthBoxFraction']]
- newCube[w] = [newCenter[w] - 0.5 * output['WidthBoxFraction'],
- newCenter[w] + 0.5 * output['WidthBoxFraction']]
+ newCube[w] = [newCenter[w] - 0.5 * self.masterSolution[q]['WidthBoxFraction'],
+ newCenter[w] + 0.5 * self.masterSolution[q]['WidthBoxFraction']]
commonVolume += commonNVolume(oldCube,newCube,periodic=na.array([[0,1],[0,1],[0,1]]))
totalVolume += output['DepthBoxFraction'] * output['WidthBoxFraction']**2
@@ -290,12 +315,20 @@
if not(recycle and (w == self.lightConeSolution[q]['ProjectionAxis'])):
self.lightConeSolution[q]['ProjectionCenter'][w] = newCenter[w]
- mylog.info("Fraction of total volume in common with old solution is %.2e." % (commonVolume/totalVolume))
+ if recycle:
+ if self.verbose: mylog.info("Fractional common volume between master and recycled solution is %.2e" % (commonVolume/totalVolume))
+ else:
+ if self.verbose: mylog.info("Fraction of total volume in common with old solution is %.2e." % (commonVolume/totalVolume))
+ self.masterSolution = [copy.deepcopy(q) for q in self.lightConeSolution]
+
+ def RestoreMasterSolution(self):
+ "Reset the active light cone solution to the master solution."
+ self.lightConeSolution = [copy.deepcopy(q) for q in self.masterSolution]
def SaveLightConeSolution(self,file="light_cone.dat"):
"Write out a text file with information on light cone solution."
- mylog.info("Saving light cone solution to %s." % file)
+ if self.verbose: mylog.info("Saving light cone solution to %s." % file)
f = open(file,'w')
if self.recycleSolution:
@@ -318,14 +351,15 @@
def SaveLightConeStack(self,filename=None):
"Save the light cone projection stack as a 3d array in and hdf5 file."
if (filename is None):
- filename = "%s%s_data" % (self.lightConeParameters['OutputDir'],self.lightConeParameters['OutputPrefix'])
- filename += ".h5"
+ filename = "%s/%s_data" % (self.lightConeParameters['OutputDir'],self.lightConeParameters['OutputPrefix'])
+ if not(filename.endswith('.h5')):
+ filename += ".h5"
if (len(self.projectionStack) == 0):
- mylog.error("SaveLightConeStack: no projection data loaded.")
+ if self.verbose: mylog.error("SaveLightConeStack: no projection data loaded.")
return
- mylog.info("Writing light cone data to %s." % filename)
+ if self.verbose: mylog.info("Writing light cone data to %s." % filename)
output = h5.openFile(filename, "a")
@@ -368,7 +402,7 @@
distance2 = co.ComovingRadialDistance(z2,z) * self.enzoParameters['CosmologyHubbleConstantNow']
iteration += 1
if (iteration > max_Iterations):
- mylog.error("CalculateDeltaZMax: Warning - max iterations exceeded for z = %f (delta z = %f)." % (z,na.fabs(z2-z)))
+ if self.verbose: mylog.error("CalculateDeltaZMax: Warning - max iterations exceeded for z = %f (delta z = %f)." % (z,na.fabs(z2-z)))
break
output['deltaz'] = na.fabs(z2-z)
@@ -427,7 +461,7 @@
param = param.strip()
vals = vals.strip()
except ValueError:
- mylog.error("Skipping line: %s" % line)
+ if self.verbose: mylog.error("Skipping line: %s" % line)
continue
if EnzoParameterDict.has_key(param):
t = map(EnzoParameterDict[param], vals.split())
@@ -462,7 +496,7 @@
param = param.strip()
vals = vals.strip()
except ValueError:
- mylog.error("Skipping line: %s" % line)
+ if self.verbose: mylog.error("Skipping line: %s" % line)
continue
if LightConeParameterDict.has_key(param):
t = map(LightConeParameterDict[param], vals.split())
Modified: branches/yt-non-3d/yt/extensions/lightcone/LightConeProjection.py
==============================================================================
--- branches/yt-non-3d/yt/extensions/lightcone/LightConeProjection.py (original)
+++ branches/yt-non-3d/yt/extensions/lightcone/LightConeProjection.py Wed Nov 19 09:24:29 2008
@@ -24,6 +24,11 @@
"""
from yt.extensions.lightcone import *
+from yt.logger import lagosLogger as mylog
+import yt.lagos as lagos
+import yt.raven as raven
+import numpy as na
+import copy
def LightConeProjection(lightConeSlice,field,pixels,weight_field=None,save_image=False,name="",**kwargs):
"Create a single projection to be added into the light cone stack."
@@ -106,7 +111,7 @@
# 3. The Shift Problem
# Shift projection by random x and y offsets.
- offset = lightConeSlice['ProjectionCenter']
+ offset = copy.deepcopy(lightConeSlice['ProjectionCenter'])
# Delete depth coordinate.
del offset[lightConeSlice['ProjectionAxis']]
Modified: branches/yt-non-3d/yt/extensions/lightcone/__init__.py
==============================================================================
--- branches/yt-non-3d/yt/extensions/lightcone/__init__.py (original)
+++ branches/yt-non-3d/yt/extensions/lightcone/__init__.py Wed Nov 19 09:24:29 2008
@@ -27,5 +27,5 @@
from LightConeProjection import *
from LightCone import *
-
+from UniqueSolution import *
Modified: branches/yt-non-3d/yt/lagos/BaseDataTypes.py
==============================================================================
--- branches/yt-non-3d/yt/lagos/BaseDataTypes.py (original)
+++ branches/yt-non-3d/yt/lagos/BaseDataTypes.py Wed Nov 19 09:24:29 2008
@@ -455,7 +455,7 @@
self.center, self.vec)
return mask
-class AMR2DData(AMRData, GridPropertiesMixin):
+class AMR2DData(AMRData, GridPropertiesMixin, ParallelAnalysisInterface):
_key_fields = ['px','py','pdx','pdy']
"""
Class to represent a set of :class:`AMRData` that's 2-D in nature, and
@@ -487,16 +487,24 @@
fields_to_get = self.fields
else:
fields_to_get = ensure_list(fields)
+ temp_data = {}
for field in fields_to_get:
- if self.data.has_key(field):
- continue
- rvs=[]
+ if self.data.has_key(field): continue
if field not in self.hierarchy.field_list:
if self._generate_field(field):
continue # A "True" return means we did it
- self[field] = na.concatenate(
+ # To ensure that we use data from this object as much as possible,
+ # we're going to have to set the same thing several times
+ temp_data[field] = na.concatenate(
[self._get_data_from_grid(grid, field)
- for grid in self._grids])
+ for grid in self._get_grids()])
+ # Now the next field can use this field
+ self[field] = temp_data[field]
+ # We finalize
+ temp_data = self._mpi_catdict(temp_data)
+ # And set, for the next group
+ for field in temp_data.keys():
+ self[field] = temp_data[field]
def _generate_field(self, field):
@@ -623,9 +631,9 @@
def _generate_coords(self):
points = []
- for grid in self._grids:
+ for grid in self._get_grids():
points.append(self._generate_grid_coords(grid))
- t = na.concatenate(points)
+ t = self._mpi_catarray(na.concatenate(points))
self['px'] = t[:,0]
self['py'] = t[:,1]
self['pz'] = t[:,2]
@@ -780,9 +788,9 @@
def _generate_coords(self):
points = []
- for grid in self._grids:
+ for grid in self._get_grids():
points.append(self._generate_grid_coords(grid))
- t = na.concatenate(points)
+ t = self._mpi_catarray(na.concatenate(points))
pos = (t[:,0:3] - self.center)
self['px'] = na.dot(pos, self._x_vec)
self['py'] = na.dot(pos, self._y_vec)
@@ -822,7 +830,7 @@
L_name = ("%s" % self._norm_vec).replace(" ","_")[1:-1]
return "%s_c%s_L%s" % (self.fields[0], cen_name, L_name)
-class AMRProjBase(AMR2DData, ParallelAnalysisInterface):
+class AMRProjBase(AMR2DData):
_top_node = "/Projections"
_key_fields = AMR2DData._key_fields + ['weight_field']
def __init__(self, axis, field, weight_field = None,
Modified: branches/yt-non-3d/yt/lagos/DataReadingFuncs.py
==============================================================================
--- branches/yt-non-3d/yt/lagos/DataReadingFuncs.py (original)
+++ branches/yt-non-3d/yt/lagos/DataReadingFuncs.py Wed Nov 19 09:24:29 2008
@@ -158,9 +158,10 @@
import enzo
sl = [slice(None), slice(None), slice(None)]
sl[axis] = slice(coord, coord + 1)
- sl = tuple(reversed(sl))
+ #sl = tuple(reversed(sl))
+ sl = tuple(sl)
bsl = (slice(3,-3), slice(3,-3), slice(3,-3))
- return enzo.grid_data[grid.id][field][bsl][sl].swapaxes(0,2)
+ return enzo.grid_data[grid.id][field][bsl][sl]#.swapaxes(0,2)
def getExceptionInMemory():
return KeyError
Modified: branches/yt-non-3d/yt/lagos/HierarchyType.py
==============================================================================
--- branches/yt-non-3d/yt/lagos/HierarchyType.py (original)
+++ branches/yt-non-3d/yt/lagos/HierarchyType.py Wed Nov 19 09:24:29 2008
@@ -830,6 +830,9 @@
if grid.Parent: grid._guess_properties_from_parent()
if pbar: pbar.finish()
+ def _join_field_lists(self, field_list):
+ return field_list
+
def _setup_field_lists(self):
field_list = self.get_data("/", "DataFields")
if field_list is None:
@@ -845,6 +848,7 @@
continue
mylog.debug("Grid %s has: %s", grid.id, gf)
field_list = field_list.union(sets.Set(gf))
+ field_list = self._join_field_lists(field_list)
self.save_data(list(field_list),"/","DataFields")
self.field_list = list(field_list)
for field in self.field_list:
@@ -888,6 +892,19 @@
def _initialize_data_file(self):
pass
+ def _join_field_lists(self, field_list):
+ from mpi4py import MPI
+ MPI.COMM_WORLD.Barrier()
+ data = list(field_list)
+ if MPI.COMM_WORLD.rank == 0:
+ for i in range(1, MPI.COMM_WORLD.size):
+ data += MPI.COMM_WORLD.Recv(source=i, tag=0)
+ data = list(set(data))
+ else:
+ MPI.COMM_WORLD.Send(data, dest=0, tag=0)
+ MPI.COMM_WORLD.Barrier()
+ return MPI.COMM_WORLD.Bcast(data, root=0)
+
def _populate_hierarchy(self):
self._copy_hierarchy_structure()
import enzo
@@ -907,6 +924,7 @@
mylog.debug("Preparing grids")
for i, grid in enumerate(self.grids):
if (i%1e4) == 0: mylog.debug("Prepared % 7i / % 7i grids", i, self.num_grids)
+ grid.filename = None
grid._prepare_grid()
grid.proc_num = self.gridProcs[i,0]
self._setup_grid_dxs()
Modified: branches/yt-non-3d/yt/lagos/ParallelTools.py
==============================================================================
--- branches/yt-non-3d/yt/lagos/ParallelTools.py (original)
+++ branches/yt-non-3d/yt/lagos/ParallelTools.py Wed Nov 19 09:24:29 2008
@@ -28,8 +28,8 @@
import yt.logger
import itertools, sys
-if os.path.basename(sys.executable) in ["mpi4py"] \
- or "--parallel" in sys.argv:
+if os.path.basename(sys.executable) in ["mpi4py", "embed_enzo"] \
+ or "--parallel" in sys.argv or '_parallel' in dir(sys):
from mpi4py import MPI
parallel_capable = (MPI.COMM_WORLD.size > 1)
if parallel_capable:
@@ -258,6 +258,7 @@
mylog.debug("Opening MPI Broadcast on %s", MPI.COMM_WORLD.rank)
data = MPI.COMM_WORLD.Bcast(data, root=0)
MPI.COMM_WORLD.Barrier()
+ return data
def _should_i_write(self):
if not parallel_capable: return True
Modified: branches/yt-non-3d/yt/mods.py
==============================================================================
--- branches/yt-non-3d/yt/mods.py (original)
+++ branches/yt-non-3d/yt/mods.py Wed Nov 19 09:24:29 2008
@@ -40,9 +40,14 @@
# Now individual component imports from lagos
from yt.lagos import EnzoStaticOutput, \
BinnedProfile1D, BinnedProfile2D, BinnedProfile3D, \
- add_field, EnzoFieldInfo, Enzo2DFieldInfo, OrionFieldInfo, \
+ add_field, FieldInfo, EnzoFieldInfo, Enzo2DFieldInfo, OrionFieldInfo, \
Clump, write_clump_hierarchy, find_clumps, write_clumps
+# This is a temporary solution -- in the future, we will allow the user to
+# select this via ytcfg.
+
+fieldInfo = lagos.fieldInfo = EnzoFieldInfo
+
# Now individual component imports from raven
from yt.raven import PlotCollection, PlotCollectionInteractive, \
QuiverCallback, ParticleCallback, ContourCallback, \
More information about the yt-svn
mailing list