[yt-svn] commit/yt-3.0: 135 new changesets
commits-noreply at bitbucket.org
commits-noreply at bitbucket.org
Mon Jul 1 14:40:45 PDT 2013
135 new commits in yt-3.0:
https://bitbucket.org/yt_analysis/yt-3.0/commits/20a074192f92/
Changeset: 20a074192f92
Branch: yt-3.0
User: sleitner
Date: 2013-04-08 05:32:11
Summary: some hacks and fixes for star formation histories
Affected #: 4 files
diff -r 382704af8a3f86e440e3d6a3bd25d8e4671e3412 -r 20a074192f9257cc0774b09d67a6921e04a9bcac yt/analysis_modules/star_analysis/sfr_spectrum.py
--- a/yt/analysis_modules/star_analysis/sfr_spectrum.py
+++ b/yt/analysis_modules/star_analysis/sfr_spectrum.py
@@ -105,16 +105,19 @@
"""
# Pick out the stars.
if self.mode == 'data_source':
- ct = self._data_source["creation_time"]
+ ct = self._data_source["stars","particle_age"]
+ if ct == None :
+ print 'data source must have particle_age!'
+ sys.exit(1)
ct_stars = ct[ct > 0]
- mass_stars = self._data_source["ParticleMassMsun"][ct > 0]
+ mass_stars = self._data_source["stars", "ParticleMassMsun"][ct > 0]
elif self.mode == 'provided':
ct_stars = self.star_creation_time
mass_stars = self.star_mass
# Find the oldest stars in units of code time.
tmin= min(ct_stars)
# Multiply the end to prevent numerical issues.
- self.time_bins = np.linspace(tmin*0.99, self._pf.current_time,
+ self.time_bins = np.linspace(tmin*1.01, self._pf.current_time,
num = self.bin_count + 1)
# Figure out which bins the stars go into.
inds = np.digitize(ct_stars, self.time_bins) - 1
@@ -127,7 +130,7 @@
for index in xrange(self.bin_count):
self.cum_mass_bins[index+1] += self.cum_mass_bins[index]
# We will want the time taken between bins.
- self.time_bins_dt = self.time_bins[1:] - self.time_bins[:-1]
+ self.time_bins_dt = self.time_bins[:-1] - self.time_bins[1:]
def attach_arrays(self):
"""
@@ -143,7 +146,7 @@
vol = ds.volume('mpc')
elif self.mode == 'provided':
vol = self.volume
- tc = self._pf["Time"]
+ tc = self._pf["Time"] #time to seconds?
self.time = []
self.lookback_time = []
self.redshift = []
diff -r 382704af8a3f86e440e3d6a3bd25d8e4671e3412 -r 20a074192f9257cc0774b09d67a6921e04a9bcac yt/data_objects/data_containers.py
--- a/yt/data_objects/data_containers.py
+++ b/yt/data_objects/data_containers.py
@@ -962,8 +962,10 @@
the container, so this may vary very slightly
from what might be expected from the geometric volume.
"""
- return self.quantities["TotalQuantity"]("CellVolume")[0] * \
- (self.pf[unit] / self.pf['cm']) ** 3.0
+# return self.quantities["TotalQuantity"]("CellVolume")[0] * \
+# (self.pf[unit] / self.pf['cm']) ** 3.0
+ print 'data_containers.py bad fix to volume to prevent data_size like issue for ARTIO'
+ return 1
def _reconstruct_object(*args, **kwargs):
pfid = args[0]
diff -r 382704af8a3f86e440e3d6a3bd25d8e4671e3412 -r 20a074192f9257cc0774b09d67a6921e04a9bcac yt/frontends/artio/_artio_caller.pyx
--- a/yt/frontends/artio/_artio_caller.pyx
+++ b/yt/frontends/artio/_artio_caller.pyx
@@ -341,7 +341,7 @@
if selected_mass[ispec] :
count = len(data[selected_mass[ispec]])
data[selected_mass[ispec]].resize(count+1)
- data[selected_mass[ispec]][count] = self.parameters["particle_species_mass"]
+ data[selected_mass[ispec]][count] = self.parameters["particle_species_mass"][0]
status = artio_particle_read_species_end( self.handle )
check_artio_status(status)
diff -r 382704af8a3f86e440e3d6a3bd25d8e4671e3412 -r 20a074192f9257cc0774b09d67a6921e04a9bcac yt/frontends/artio/fields.py
--- a/yt/frontends/artio/fields.py
+++ b/yt/frontends/artio/fields.py
@@ -295,12 +295,12 @@
#add_artio_field("creation_time", function=NullFunc, particle_type=True)
def _particle_age(field, data):
- pa = b2t(data['creation_time'])
+ pa = b2t(data['stars','creation_time'])*1e9*31556926
# tr = np.zeros(pa.shape,dtype='float')-1.0
# tr[pa>0] = pa[pa>0]
tr = pa
return tr
-add_field("particle_age", function=_particle_age, units=r"\rm{s}",
+add_field(("stars","particle_age"), function=_particle_age, units=r"\rm{s}",
particle_type=True)
@@ -416,9 +416,9 @@
def b2t(tb, n=1e2, logger=None, **kwargs):
tb = np.array(tb)
- if isinstance(tb, 1.1):
- return a2t(b2a(tb))
if tb.shape == ():
+ return None
+ if len(tb) == 1:
return a2t(b2a(tb))
if len(tb) < n:
n = len(tb)
https://bitbucket.org/yt_analysis/yt-3.0/commits/2314f9d71cc9/
Changeset: 2314f9d71cc9
Branch: yt-3.0
User: sleitner
Date: 2013-04-08 05:36:49
Summary: increase max hash table size (for isolated disk)
Affected #: 2 files
diff -r 20a074192f9257cc0774b09d67a6921e04a9bcac -r 2314f9d71cc900225418a03b4ec6a3842149d6de yt/analysis_modules/halo_finding/halo_objects.py
--- a/yt/analysis_modules/halo_finding/halo_objects.py
+++ b/yt/analysis_modules/halo_finding/halo_objects.py
@@ -2504,7 +2504,7 @@
if dm_only:
select = self._get_dm_indices()
total_mass = \
- self.comm.mpi_allreduce((self._data_source["ParticleMassMsun"][select]).sum(dtype='float64'), op='sum')
+ self.comm.mpi_allreduce((self._data_source['all', "ParticleMassMsun"][select]).sum(dtype='float64'), op='sum')
else:
total_mass = self.comm.mpi_allreduce(self._data_source.quantities["TotalQuantity"]("ParticleMassMsun")[0], op='sum')
# MJT: Note that instead of this, if we are assuming that the particles
diff -r 20a074192f9257cc0774b09d67a6921e04a9bcac -r 2314f9d71cc900225418a03b4ec6a3842149d6de yt/analysis_modules/halo_finding/hop/hop_hop.c
--- a/yt/analysis_modules/halo_finding/hop/hop_hop.c
+++ b/yt/analysis_modules/halo_finding/hop/hop_hop.c
@@ -443,7 +443,7 @@
/* Else, this slot was full, go to the next one */
hp++;
if (hp>=smx->hash+smx->nHashLength) hp = smx->hash;
- if (++count>1000) {
+ if (++count>1000000) {
fprintf(stderr,"Hash Table is too full.\n");
exit(1);
}
https://bitbucket.org/yt_analysis/yt-3.0/commits/c8df14c9eaea/
Changeset: c8df14c9eaea
Branch: yt-3.0
User: sleitner
Date: 2013-04-15 16:02:33
Summary: Merged yt_analysis/yt-3.0 into yt-3.0
Affected #: 3 files
diff -r 2314f9d71cc900225418a03b4ec6a3842149d6de -r c8df14c9eaea914a66426ec2063672bda16b5d3a yt/frontends/enzo/io.py
--- a/yt/frontends/enzo/io.py
+++ b/yt/frontends/enzo/io.py
@@ -55,7 +55,7 @@
ptypes = list(set([ftype for ftype, fname in fields]))
fields = list(set(fields))
if len(ptypes) > 1: raise NotImplementedError
- pfields = [(ptypes[0], "position_%s" % ax) for ax in 'xyz']
+ pfields = [(ptypes[0], "particle_position_%s" % ax) for ax in 'xyz']
size = 0
for chunk in chunks:
data = self._read_chunk_data(chunk, pfields, 'active',
diff -r 2314f9d71cc900225418a03b4ec6a3842149d6de -r c8df14c9eaea914a66426ec2063672bda16b5d3a yt/frontends/ramses/data_structures.py
--- a/yt/frontends/ramses/data_structures.py
+++ b/yt/frontends/ramses/data_structures.py
@@ -116,6 +116,9 @@
self.particle_field_offsets = {}
return
f = open(self.part_fn, "rb")
+ f.seek(0, os.SEEK_END)
+ flen = f.tell()
+ f.seek(0)
hvals = {}
attrs = ( ('ncpu', 1, 'I'),
('ndim', 1, 'I'),
@@ -143,12 +146,15 @@
if hvals["nstar_tot"] > 0:
particle_fields += [("particle_age", "d"),
("particle_metallicity", "d")]
- field_offsets = {particle_fields[0][0]: f.tell()}
- for field, vtype in particle_fields[1:]:
+ field_offsets = {}
+ _pfields = {}
+ for field, vtype in particle_fields:
+ if f.tell() >= flen: break
+ field_offsets[field] = f.tell()
+ _pfields[field] = vtype
fpu.skip(f, 1)
- field_offsets[field] = f.tell()
self.particle_field_offsets = field_offsets
- self.particle_field_types = dict(particle_fields)
+ self.particle_field_types = _pfields
def _read_amr_header(self):
hvals = {}
diff -r 2314f9d71cc900225418a03b4ec6a3842149d6de -r c8df14c9eaea914a66426ec2063672bda16b5d3a yt/frontends/ramses/fields.py
--- a/yt/frontends/ramses/fields.py
+++ b/yt/frontends/ramses/fields.py
@@ -91,6 +91,8 @@
"particle_mass",
"particle_identifier",
"particle_refinement_level",
+ "particle_age",
+ "particle_metallicity",
]
for f in known_ramses_particle_fields:
https://bitbucket.org/yt_analysis/yt-3.0/commits/5154fec5bf32/
Changeset: 5154fec5bf32
Branch: yt-3.0
User: sleitner
Date: 2013-04-15 16:20:32
Summary: current_time, smallest_dx, b2t fixes. b2t now in seconds
Affected #: 1 file
diff -r 2314f9d71cc900225418a03b4ec6a3842149d6de -r 5154fec5bf3280048fd25d5be4f094a0a03681d0 yt/frontends/artio/data_structures.py
--- a/yt/frontends/artio/data_structures.py
+++ b/yt/frontends/artio/data_structures.py
@@ -32,7 +32,7 @@
artio_is_valid, artio_fileset
from yt.utilities.definitions import \
mpc_conversion, sec_conversion
-from .fields import ARTIOFieldInfo, KnownARTIOFields
+from .fields import ARTIOFieldInfo, KnownARTIOFields, b2t
from yt.funcs import *
from yt.geometry.geometry_handler import \
@@ -145,7 +145,7 @@
"""
Returns (in code units) the smallest cell size in the simulation.
"""
- return (self.parameter_file.domain_width/(2**self.max_level)).min()
+ return 1.0/(2**self.max_level)
def convert(self, unit):
return self.parameter_file.conversion_factors[unit]
@@ -391,7 +391,7 @@
list(set(art_to_yt[s] for s in
self.artio_parameters["particle_species_labels"])))
- self.current_time = self.artio_parameters["tl"][0]
+ self.current_time = b2t(self.artio_parameters["tl"][0])
# detect cosmology
if "abox" in self.artio_parameters:
https://bitbucket.org/yt_analysis/yt-3.0/commits/ec3681a46d72/
Changeset: ec3681a46d72
Branch: yt-3.0
User: sleitner
Date: 2013-04-15 16:25:52
Summary: oops b2t fix
Affected #: 1 file
diff -r 5154fec5bf3280048fd25d5be4f094a0a03681d0 -r ec3681a46d72d80cf2b296032f292608dcc02052 yt/frontends/artio/fields.py
--- a/yt/frontends/artio/fields.py
+++ b/yt/frontends/artio/fields.py
@@ -295,7 +295,7 @@
#add_artio_field("creation_time", function=NullFunc, particle_type=True)
def _particle_age(field, data):
- pa = b2t(data['stars','creation_time'])*1e9*31556926
+ pa = b2t(data['stars','creation_time'])
# tr = np.zeros(pa.shape,dtype='float')-1.0
# tr[pa>0] = pa[pa>0]
tr = pa
@@ -416,10 +416,10 @@
def b2t(tb, n=1e2, logger=None, **kwargs):
tb = np.array(tb)
+ if len(np.atleast_1d(tb)) == 1:
+ return a2t(b2a(tb))
if tb.shape == ():
return None
- if len(tb) == 1:
- return a2t(b2a(tb))
if len(tb) < n:
n = len(tb)
age_min = a2t(b2a(tb.max(), **kwargs), **kwargs)
@@ -434,7 +434,7 @@
ages = np.array(ages)
fb2t = np.interp(tb, tbs, ages)
#fb2t = interp1d(tbs,ages)
- return fb2t
+ return fb2t*1e9*31556926
def spread_ages(ages, logger=None, spread=.0e7*365*24*3600):
https://bitbucket.org/yt_analysis/yt-3.0/commits/94eeca505250/
Changeset: 94eeca505250
Branch: yt-3.0
User: sleitner
Date: 2013-04-15 17:10:01
Summary: merge
Affected #: 3 files
diff -r ec3681a46d72d80cf2b296032f292608dcc02052 -r 94eeca5052506364a8db0afcc76d53ed2f36ff7b yt/frontends/enzo/io.py
--- a/yt/frontends/enzo/io.py
+++ b/yt/frontends/enzo/io.py
@@ -55,7 +55,7 @@
ptypes = list(set([ftype for ftype, fname in fields]))
fields = list(set(fields))
if len(ptypes) > 1: raise NotImplementedError
- pfields = [(ptypes[0], "position_%s" % ax) for ax in 'xyz']
+ pfields = [(ptypes[0], "particle_position_%s" % ax) for ax in 'xyz']
size = 0
for chunk in chunks:
data = self._read_chunk_data(chunk, pfields, 'active',
diff -r ec3681a46d72d80cf2b296032f292608dcc02052 -r 94eeca5052506364a8db0afcc76d53ed2f36ff7b yt/frontends/ramses/data_structures.py
--- a/yt/frontends/ramses/data_structures.py
+++ b/yt/frontends/ramses/data_structures.py
@@ -116,6 +116,9 @@
self.particle_field_offsets = {}
return
f = open(self.part_fn, "rb")
+ f.seek(0, os.SEEK_END)
+ flen = f.tell()
+ f.seek(0)
hvals = {}
attrs = ( ('ncpu', 1, 'I'),
('ndim', 1, 'I'),
@@ -143,12 +146,15 @@
if hvals["nstar_tot"] > 0:
particle_fields += [("particle_age", "d"),
("particle_metallicity", "d")]
- field_offsets = {particle_fields[0][0]: f.tell()}
- for field, vtype in particle_fields[1:]:
+ field_offsets = {}
+ _pfields = {}
+ for field, vtype in particle_fields:
+ if f.tell() >= flen: break
+ field_offsets[field] = f.tell()
+ _pfields[field] = vtype
fpu.skip(f, 1)
- field_offsets[field] = f.tell()
self.particle_field_offsets = field_offsets
- self.particle_field_types = dict(particle_fields)
+ self.particle_field_types = _pfields
def _read_amr_header(self):
hvals = {}
diff -r ec3681a46d72d80cf2b296032f292608dcc02052 -r 94eeca5052506364a8db0afcc76d53ed2f36ff7b yt/frontends/ramses/fields.py
--- a/yt/frontends/ramses/fields.py
+++ b/yt/frontends/ramses/fields.py
@@ -91,6 +91,8 @@
"particle_mass",
"particle_identifier",
"particle_refinement_level",
+ "particle_age",
+ "particle_metallicity",
]
for f in known_ramses_particle_fields:
https://bitbucket.org/yt_analysis/yt-3.0/commits/b75275862000/
Changeset: b75275862000
Branch: yt-3.0
User: drudd
Date: 2013-03-19 03:26:28
Summary: Removed size and shape parameters from YTSelectionContainer, made size a property, and stopped caching data_size
Affected #: 1 file
diff -r dba8e90381b5be3b028790fc00201cca02fd936c -r b752758620000799b9c38818ddc8be4301088aa0 yt/data_objects/data_containers.py
--- a/yt/data_objects/data_containers.py
+++ b/yt/data_objects/data_containers.py
@@ -424,8 +424,6 @@
_sort_by = None
_selector = None
_current_chunk = None
- size = None
- shape = None
def __init__(self, *args, **kwargs):
super(YTSelectionContainer, self).__init__(*args, **kwargs)
@@ -522,17 +520,19 @@
# There are several items that need to be swapped out
# field_data, size, shape
old_field_data, self.field_data = self.field_data, YTFieldData()
- old_size, self.size = self.size, chunk.data_size
old_chunk, self._current_chunk = self._current_chunk, chunk
old_locked, self._locked = self._locked, False
- #self.shape = (self.size,)
yield
self.field_data = old_field_data
- self.size = old_size
- #self.shape = (old_size,)
self._current_chunk = old_chunk
self._locked = old_locked
+ @property
+ def size(self) :
+ if self._current_chunk is None :
+ self.hierarchy._identify_base_chunk(self)
+ return self_current_chunk.data_size
+
@property
def icoords(self):
if self._current_chunk is None:
https://bitbucket.org/yt_analysis/yt-3.0/commits/fb14dd338289/
Changeset: fb14dd338289
Branch: yt-3.0
User: drudd
Date: 2013-03-19 06:16:02
Summary: Trying to remove accesses to data.shape, data.size
Affected #: 2 files
diff -r b752758620000799b9c38818ddc8be4301088aa0 -r fb14dd338289ba004f2d42bfe4f5e7f5160809c7 yt/data_objects/data_containers.py
--- a/yt/data_objects/data_containers.py
+++ b/yt/data_objects/data_containers.py
@@ -527,11 +527,12 @@
self._current_chunk = old_chunk
self._locked = old_locked
- @property
- def size(self) :
- if self._current_chunk is None :
- self.hierarchy._identify_base_chunk(self)
- return self_current_chunk.data_size
+# @property
+# def size(self) :
+# if self._current_chunk is None :
+## self.hierarchy._identify_base_chunk(self)
+# return 0
+# return self._current_chunk.data_size
@property
def icoords(self):
diff -r b752758620000799b9c38818ddc8be4301088aa0 -r fb14dd338289ba004f2d42bfe4f5e7f5160809c7 yt/data_objects/universal_fields.py
--- a/yt/data_objects/universal_fields.py
+++ b/yt/data_objects/universal_fields.py
@@ -95,7 +95,7 @@
display_field = False)
def _Ones(field, data):
- return np.ones(data.shape, dtype='float64')
+ return np.ones(data.ActiveDimensions, dtype='float64')
add_field("Ones", function=_Ones,
projection_conversion="unitary",
display_field = False)
https://bitbucket.org/yt_analysis/yt-3.0/commits/3a1a2a954578/
Changeset: 3a1a2a954578
Branch: yt-3.0
User: drudd
Date: 2013-04-04 22:43:48
Summary: Merged yt_analysis/yt-3.0 into yt-3.0
Affected #: 25 files
diff -r fb14dd338289ba004f2d42bfe4f5e7f5160809c7 -r 3a1a2a95457855b229396b14b67aeba15d2a3f27 .hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -5,11 +5,16 @@
hdf5.cfg
png.cfg
yt_updater.log
+yt/frontends/artio/_artio_caller.c
yt/frontends/ramses/_ramses_reader.cpp
+yt/frontends/sph/smoothing_kernel.c
+yt/geometry/oct_container.c
+yt/geometry/selection_routines.c
yt/utilities/amr_utils.c
yt/utilities/kdtree/forthonf2c.h
yt/utilities/libconfig_wrapper.c
yt/utilities/spatial/ckdtree.c
+yt/utilities/lib/alt_ray_tracers.c
yt/utilities/lib/CICDeposit.c
yt/utilities/lib/ContourFinding.c
yt/utilities/lib/DepthFirstOctree.c
diff -r fb14dd338289ba004f2d42bfe4f5e7f5160809c7 -r 3a1a2a95457855b229396b14b67aeba15d2a3f27 yt/analysis_modules/halo_finding/halo_objects.py
--- a/yt/analysis_modules/halo_finding/halo_objects.py
+++ b/yt/analysis_modules/halo_finding/halo_objects.py
@@ -1059,7 +1059,7 @@
_fields = ["particle_position_%s" % ax for ax in 'xyz']
- def __init__(self, data_source, dm_only=True):
+ def __init__(self, data_source, dm_only=True, redshift=-1):
"""
Run hop on *data_source* with a given density *threshold*. If
*dm_only* is set, only run it on the dark matter particles, otherwise
diff -r fb14dd338289ba004f2d42bfe4f5e7f5160809c7 -r 3a1a2a95457855b229396b14b67aeba15d2a3f27 yt/data_objects/data_containers.py
--- a/yt/data_objects/data_containers.py
+++ b/yt/data_objects/data_containers.py
@@ -210,6 +210,8 @@
"""
Deletes a field
"""
+ if key not in self.field_data:
+ key = self._determine_fields(key)[0]
del self.field_data[key]
def _generate_field(self, field):
diff -r fb14dd338289ba004f2d42bfe4f5e7f5160809c7 -r 3a1a2a95457855b229396b14b67aeba15d2a3f27 yt/data_objects/derived_quantities.py
--- a/yt/data_objects/derived_quantities.py
+++ b/yt/data_objects/derived_quantities.py
@@ -678,3 +678,37 @@
return [np.sum(totals[:,i]) for i in range(n_fields)]
add_quantity("TotalQuantity", function=_TotalQuantity,
combine_function=_combTotalQuantity, n_ret=2)
+
+def _ParticleDensityCenter(data,nbins=3,particle_type="all"):
+ """
+ Find the center of the particle density
+ by histogramming the particles iteratively.
+ """
+ pos = [data[(particle_type,"particle_position_%s"%ax)] for ax in "xyz"]
+ pos = np.array(pos).T
+ mas = data[(particle_type,"particle_mass")]
+ calc_radius= lambda x,y:np.sqrt(np.sum((x-y)**2.0,axis=1))
+ density = 0
+ if pos.shape[0]==0:
+ return -1.0,[-1.,-1.,-1.]
+ while pos.shape[0] > 1:
+ table,bins=np.histogramdd(pos,bins=nbins, weights=mas)
+ bin_size = min((np.max(bins,axis=1)-np.min(bins,axis=1))/nbins)
+ centeridx = np.where(table==table.max())
+ le = np.array([bins[0][centeridx[0][0]],
+ bins[1][centeridx[1][0]],
+ bins[2][centeridx[2][0]]])
+ re = np.array([bins[0][centeridx[0][0]+1],
+ bins[1][centeridx[1][0]+1],
+ bins[2][centeridx[2][0]+1]])
+ center = 0.5*(le+re)
+ idx = calc_radius(pos,center)<bin_size
+ pos, mas = pos[idx],mas[idx]
+ density = max(density,mas.sum()/bin_size**3.0)
+ return density, center
+def _combParticleDensityCenter(data,densities,centers):
+ i = np.argmax(densities)
+ return densities[i],centers[i]
+
+add_quantity("ParticleDensityCenter",function=_ParticleDensityCenter,
+ combine_function=_combParticleDensityCenter,n_ret=2)
diff -r fb14dd338289ba004f2d42bfe4f5e7f5160809c7 -r 3a1a2a95457855b229396b14b67aeba15d2a3f27 yt/data_objects/tests/test_fields.py
--- a/yt/data_objects/tests/test_fields.py
+++ b/yt/data_objects/tests/test_fields.py
@@ -33,6 +33,7 @@
pf.conversion_factors.update( dict((f, 1.0) for f in fields) )
pf.current_redshift = 0.0001
pf.hubble_constant = 0.7
+ pf.omega_matter = 0.27
for unit in mpc_conversion:
pf.units[unit+'h'] = pf.units[unit]
pf.units[unit+'cm'] = pf.units[unit]
@@ -72,7 +73,7 @@
if not field.particle_type:
assert_equal(v1, dd1["gas", self.field_name])
if not needs_spatial:
- assert_equal(v1, conv*field._function(field, dd2))
+ assert_array_almost_equal_nulp(v1, conv*field._function(field, dd2), 4)
if not skip_grids:
for g in pf.h.grids:
g.field_parameters.update(_sample_parameters)
@@ -80,7 +81,7 @@
v1 = g[self.field_name]
g.clear_data()
g.field_parameters.update(_sample_parameters)
- assert_equal(v1, conv*field._function(field, g))
+ assert_array_almost_equal_nulp(v1, conv*field._function(field, g), 4)
def test_all_fields():
for field in FieldInfo:
diff -r fb14dd338289ba004f2d42bfe4f5e7f5160809c7 -r 3a1a2a95457855b229396b14b67aeba15d2a3f27 yt/data_objects/universal_fields.py
--- a/yt/data_objects/universal_fields.py
+++ b/yt/data_objects/universal_fields.py
@@ -1,4 +1,5 @@
"""
+
The basic field info container resides here. These classes, code specific and
universal, are the means by which we access fields across YT, both derived and
native.
@@ -754,8 +755,9 @@
for i, ax in enumerate('xyz'):
np.subtract(data["%s%s" % (field_prefix, ax)], center[i], r)
if data.pf.periodicity[i] == True:
- np.subtract(DW[i], r, rdw)
np.abs(r, r)
+ np.subtract(r, DW[i], rdw)
+ np.abs(rdw, rdw)
np.minimum(r, rdw, r)
np.power(r, 2.0, r)
np.add(radius, r, radius)
@@ -946,7 +948,7 @@
data["particle_position_x"].size,
blank, np.array(data.LeftEdge).astype(np.float64),
np.array(data.ActiveDimensions).astype(np.int32),
- np.float64(data['dx']))
+ just_one(data['dx']))
return blank
add_field("particle_density", function=_pdensity,
validators=[ValidateGridType()], convert_function=_convertDensity,
diff -r fb14dd338289ba004f2d42bfe4f5e7f5160809c7 -r 3a1a2a95457855b229396b14b67aeba15d2a3f27 yt/frontends/art/data_structures.py
--- a/yt/frontends/art/data_structures.py
+++ b/yt/frontends/art/data_structures.py
@@ -30,6 +30,8 @@
import stat
import weakref
import cStringIO
+import difflib
+import glob
from yt.funcs import *
from yt.geometry.oct_geometry_handler import \
@@ -37,9 +39,9 @@
from yt.geometry.geometry_handler import \
GeometryHandler, YTDataChunk
from yt.data_objects.static_output import \
- StaticOutput
+ StaticOutput
from yt.geometry.oct_container import \
- RAMSESOctreeContainer
+ ARTOctreeContainer
from yt.data_objects.field_info_container import \
FieldInfoContainer, NullFunc
from .fields import \
@@ -52,20 +54,15 @@
get_box_grids_level
import yt.utilities.lib as amr_utils
-from .definitions import *
-from .io import _read_frecord
-from .io import _read_record
-from .io import _read_struct
+from yt.frontends.art.definitions import *
+from yt.utilities.fortran_utils import *
from .io import _read_art_level_info
from .io import _read_child_mask_level
from .io import _read_child_level
from .io import _read_root_level
-from .io import _read_record_size
-from .io import _skip_record
from .io import _count_art_octs
from .io import b2t
-
import yt.frontends.ramses._ramses_reader as _ramses_reader
from .fields import ARTFieldInfo, KnownARTFields
@@ -80,13 +77,9 @@
from yt.utilities.physical_constants import \
mass_hydrogen_cgs, sec_per_Gyr
+
class ARTGeometryHandler(OctreeGeometryHandler):
- def __init__(self,pf,data_style="art"):
- """
- Life is made simpler because we only have one AMR file
- and one domain. However, we are matching to the RAMSES
- multi-domain architecture.
- """
+ def __init__(self, pf, data_style="art"):
self.fluid_field_list = fluid_fields
self.data_style = data_style
self.parameter_file = weakref.proxy(pf)
@@ -94,7 +87,16 @@
self.directory = os.path.dirname(self.hierarchy_filename)
self.max_level = pf.max_level
self.float_type = np.float64
- super(ARTGeometryHandler,self).__init__(pf,data_style)
+ super(ARTGeometryHandler, self).__init__(pf, data_style)
+
+ def get_smallest_dx(self):
+ """
+ Returns (in code units) the smallest cell size in the simulation.
+ """
+ # Overloaded
+ pf = self.parameter_file
+ return (1.0/pf.domain_dimensions.astype('f8') /
+ (2**self.max_level)).min()
def _initialize_oct_handler(self):
"""
@@ -102,23 +104,37 @@
allocate the requisite memory in the oct tree
"""
nv = len(self.fluid_field_list)
- self.domains = [ARTDomainFile(self.parameter_file,1,nv)]
+ self.domains = [ARTDomainFile(self.parameter_file, l+1, nv, l)
+ for l in range(self.pf.max_level)]
self.octs_per_domain = [dom.level_count.sum() for dom in self.domains]
self.total_octs = sum(self.octs_per_domain)
- self.oct_handler = RAMSESOctreeContainer(
- self.parameter_file.domain_dimensions/2, #dd is # of root cells
+ self.oct_handler = ARTOctreeContainer(
+ self.parameter_file.domain_dimensions/2, # dd is # of root cells
self.parameter_file.domain_left_edge,
self.parameter_file.domain_right_edge)
mylog.debug("Allocating %s octs", self.total_octs)
self.oct_handler.allocate_domains(self.octs_per_domain)
for domain in self.domains:
- domain._read_amr(self.oct_handler)
+ if domain.domain_level == 0:
+ domain._read_amr_root(self.oct_handler)
+ else:
+ domain._read_amr_level(self.oct_handler)
def _detect_fields(self):
self.particle_field_list = particle_fields
- self.field_list = set(fluid_fields + particle_fields + particle_star_fields)
+ self.field_list = set(fluid_fields + particle_fields +
+ particle_star_fields)
self.field_list = list(self.field_list)
-
+ # now generate all of the possible particle fields
+ if "wspecies" in self.parameter_file.parameters.keys():
+ wspecies = self.parameter_file.parameters['wspecies']
+ nspecies = len(wspecies)
+ self.parameter_file.particle_types = ["all", "darkmatter", "stars"]
+ for specie in range(nspecies):
+ self.parameter_file.particle_types.append("specie%i" % specie)
+ else:
+ self.parameter_file.particle_types = []
+
def _setup_classes(self):
dd = self._get_data_reader_dict()
super(ARTGeometryHandler, self)._setup_classes(dd)
@@ -127,19 +143,23 @@
def _identify_base_chunk(self, dobj):
"""
Take the passed in data source dobj, and use its embedded selector
- to calculate the domain mask, build the reduced domain
+ to calculate the domain mask, build the reduced domain
subsets and oct counts. Attach this information to dobj.
"""
if getattr(dobj, "_chunk_info", None) is None:
- #Get all octs within this oct handler
+ # Get all octs within this oct handler
mask = dobj.selector.select_octs(self.oct_handler)
- if mask.sum()==0:
+ if mask.sum() == 0:
mylog.debug("Warning: selected zero octs")
counts = self.oct_handler.count_cells(dobj.selector, mask)
- #For all domains, figure out how many counts we have
- #and build a subset=mask of domains
- subsets = [ARTDomainSubset(d, mask, c)
- for d, c in zip(self.domains, counts) if c > 0]
+ # For all domains, figure out how many counts we have
+ # and build a subset=mask of domains
+ subsets = []
+ for d, c in zip(self.domains, counts):
+ if c < 1:
+ continue
+ subset = ARTDomainSubset(d, mask, c, d.domain_level)
+ subsets.append(subset)
dobj._chunk_info = subsets
dobj.size = sum(counts)
dobj.shape = (dobj.size,)
@@ -147,8 +167,8 @@
def _chunk_all(self, dobj):
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
- #We pass the chunk both the current chunk and list of chunks,
- #as well as the referring data source
+ # We pass the chunk both the current chunk and list of chunks,
+ # as well as the referring data source
yield YTDataChunk(dobj, "all", oobjs, dobj.size)
def _chunk_spatial(self, dobj, ngz):
@@ -157,7 +177,7 @@
def _chunk_io(self, dobj):
"""
Since subsets are calculated per domain,
- i.e. per file, yield each domain at a time to
+ i.e. per file, yield each domain at a time to
organize by IO. We will eventually chunk out NMSU ART
to be level-by-level.
"""
@@ -165,77 +185,66 @@
for subset in oobjs:
yield YTDataChunk(dobj, "io", [subset], subset.cell_count)
+
class ARTStaticOutput(StaticOutput):
_hierarchy_class = ARTGeometryHandler
_fieldinfo_fallback = ARTFieldInfo
_fieldinfo_known = KnownARTFields
- def __init__(self,filename,data_style='art',
- fields = None, storage_filename = None,
- skip_particles=False,skip_stars=False,
- limit_level=None,spread_age=True):
+ def __init__(self, filename, data_style='art',
+ fields=None, storage_filename=None,
+ skip_particles=False, skip_stars=False,
+ limit_level=None, spread_age=True,
+ force_max_level=None, file_particle_header=None,
+ file_particle_data=None, file_particle_stars=None):
if fields is None:
fields = fluid_fields
filename = os.path.abspath(filename)
self._fields_in_file = fields
+ self._file_amr = filename
+ self._file_particle_header = file_particle_header
+ self._file_particle_data = file_particle_data
+ self._file_particle_stars = file_particle_stars
self._find_files(filename)
- self.file_amr = filename
self.parameter_filename = filename
self.skip_particles = skip_particles
self.skip_stars = skip_stars
self.limit_level = limit_level
self.max_level = limit_level
+ self.force_max_level = force_max_level
self.spread_age = spread_age
- self.domain_left_edge = np.zeros(3,dtype='float')
- self.domain_right_edge = np.zeros(3,dtype='float')+1.0
- StaticOutput.__init__(self,filename,data_style)
+ self.domain_left_edge = np.zeros(3, dtype='float')
+ self.domain_right_edge = np.zeros(3, dtype='float')+1.0
+ StaticOutput.__init__(self, filename, data_style)
self.storage_filename = storage_filename
- def _find_files(self,file_amr):
+ def _find_files(self, file_amr):
"""
Given the AMR base filename, attempt to find the
particle header, star files, etc.
"""
- prefix,suffix = filename_pattern['amr'].split('%s')
- affix = os.path.basename(file_amr).replace(prefix,'')
- affix = affix.replace(suffix,'')
- affix = affix.replace('_','')
- full_affix = affix
- affix = affix[1:-1]
- dirname = os.path.dirname(file_amr)
- for fp in (filename_pattern_hf,filename_pattern):
- for filetype, pattern in fp.items():
- #if this attribute is already set skip it
- if getattr(self,"file_"+filetype,None) is not None:
- continue
- #sometimes the affix is surrounded by an extraneous _
- #so check for an extra character on either side
- check_filename = dirname+'/'+pattern%('?%s?'%affix)
- filenames = glob.glob(check_filename)
- if len(filenames)>1:
- check_filename_strict = \
- dirname+'/'+pattern%('?%s'%full_affix[1:])
- filenames = glob.glob(check_filename_strict)
-
- if len(filenames)==1:
- setattr(self,"file_"+filetype,filenames[0])
- mylog.info('discovered %s:%s',filetype,filenames[0])
- elif len(filenames)>1:
- setattr(self,"file_"+filetype,None)
- mylog.info("Ambiguous number of files found for %s",
- check_filename)
- for fn in filenames:
- faffix = float(affix)
- else:
- setattr(self,"file_"+filetype,None)
+ base_prefix, base_suffix = filename_pattern['amr']
+ possibles = glob.glob(os.path.dirname(file_amr)+"/*")
+ for filetype, (prefix, suffix) in filename_pattern.iteritems():
+ # if this attribute is already set skip it
+ if getattr(self, "_file_"+filetype, None) is not None:
+ continue
+ stripped = file_amr.replace(base_prefix, prefix)
+ stripped = stripped.replace(base_suffix, suffix)
+ match, = difflib.get_close_matches(stripped, possibles, 1, 0.6)
+ if match is not None:
+ mylog.info('discovered %s:%s', filetype, match)
+ setattr(self, "_file_"+filetype, match)
+ else:
+ setattr(self, "_file_"+filetype, None)
def __repr__(self):
- return self.file_amr.rsplit(".",1)[0]
+ return self._file_amr.split('/')[-1]
def _set_units(self):
"""
- Generates the conversion to various physical units based
- on the parameters from the header
+ Generates the conversion to various physical units based
+ on the parameters from the header
"""
self.units = {}
self.time_units = {}
@@ -243,9 +252,9 @@
self.units['1'] = 1.0
self.units['unitary'] = 1.0
- #spatial units
- z = self.current_redshift
- h = self.hubble_constant
+ # spatial units
+ z = self.current_redshift
+ h = self.hubble_constant
boxcm_cal = self.parameters["boxh"]
boxcm_uncal = boxcm_cal / h
box_proper = boxcm_uncal/(1+z)
@@ -256,55 +265,59 @@
self.units[unit+'cm'] = mpc_conversion[unit] * boxcm_uncal
self.units[unit+'hcm'] = mpc_conversion[unit] * boxcm_cal
- #all other units
+ # all other units
wmu = self.parameters["wmu"]
Om0 = self.parameters['Om0']
- ng = self.parameters['ng']
+ ng = self.parameters['ng']
wmu = self.parameters["wmu"]
- boxh = self.parameters['boxh']
- aexpn = self.parameters["aexpn"]
+ boxh = self.parameters['boxh']
+ aexpn = self.parameters["aexpn"]
hubble = self.parameters['hubble']
cf = defaultdict(lambda: 1.0)
r0 = boxh/ng
- P0= 4.697e-16 * Om0**2.0 * r0**2.0 * hubble**2.0
- T_0 = 3.03e5 * r0**2.0 * wmu * Om0 # [K]
+ P0 = 4.697e-16 * Om0**2.0 * r0**2.0 * hubble**2.0
+ T_0 = 3.03e5 * r0**2.0 * wmu * Om0 # [K]
S_0 = 52.077 * wmu**(5.0/3.0)
S_0 *= hubble**(-4.0/3.0)*Om0**(1.0/3.0)*r0**2.0
- #v0 = r0 * 50.0*1.0e5 * np.sqrt(self.omega_matter) #cm/s
+ # v0 = r0 * 50.0*1.0e5 * np.sqrt(self.omega_matter) #cm/s
v0 = 50.0*r0*np.sqrt(Om0)
t0 = r0/v0
rho1 = 1.8791e-29 * hubble**2.0 * self.omega_matter
rho0 = 2.776e11 * hubble**2.0 * Om0
- tr = 2./3. *(3.03e5*r0**2.0*wmu*self.omega_matter)*(1.0/(aexpn**2))
+ tr = 2./3. * (3.03e5*r0**2.0*wmu*self.omega_matter)*(1.0/(aexpn**2))
aM0 = rho0 * (boxh/hubble)**3.0 / ng**3.0
- cf['r0']=r0
- cf['P0']=P0
- cf['T_0']=T_0
- cf['S_0']=S_0
- cf['v0']=v0
- cf['t0']=t0
- cf['rho0']=rho0
- cf['rho1']=rho1
- cf['tr']=tr
- cf['aM0']=aM0
+ cf['r0'] = r0
+ cf['P0'] = P0
+ cf['T_0'] = T_0
+ cf['S_0'] = S_0
+ cf['v0'] = v0
+ cf['t0'] = t0
+ cf['rho0'] = rho0
+ cf['rho1'] = rho1
+ cf['tr'] = tr
+ cf['aM0'] = aM0
- #factors to multiply the native code units to CGS
- cf['Pressure'] = P0 #already cgs
- cf['Velocity'] = v0/aexpn*1.0e5 #proper cm/s
+ # factors to multiply the native code units to CGS
+ cf['Pressure'] = P0 # already cgs
+ cf['Velocity'] = v0/aexpn*1.0e5 # proper cm/s
cf["Mass"] = aM0 * 1.98892e33
cf["Density"] = rho1*(aexpn**-3.0)
cf["GasEnergy"] = rho0*v0**2*(aexpn**-5.0)
cf["Potential"] = 1.0
cf["Entropy"] = S_0
cf["Temperature"] = tr
+ cf["Time"] = 1.0
+ cf["particle_mass"] = cf['Mass']
+ cf["particle_mass_initial"] = cf['Mass']
self.cosmological_simulation = True
self.conversion_factors = cf
-
- for particle_field in particle_fields:
- self.conversion_factors[particle_field] = 1.0
+
for ax in 'xyz':
self.conversion_factors["%s-velocity" % ax] = 1.0
+ for pt in particle_fields:
+ if pt not in self.conversion_factors.keys():
+ self.conversion_factors[pt] = 1.0
for unit in sec_conversion.keys():
self.time_units[unit] = 1.0 / sec_conversion[unit]
@@ -320,72 +333,89 @@
self.unique_identifier = \
int(os.stat(self.parameter_filename)[stat.ST_CTIME])
self.parameters.update(constants)
- #read the amr header
- with open(self.file_amr,'rb') as f:
- amr_header_vals = _read_struct(f,amr_header_struct)
- for to_skip in ['tl','dtl','tlold','dtlold','iSO']:
- _skip_record(f)
- (self.ncell,) = struct.unpack('>l', _read_record(f))
+ self.parameters['Time'] = 1.0
+ # read the amr header
+ with open(self._file_amr, 'rb') as f:
+ amr_header_vals = read_attrs(f, amr_header_struct, '>')
+ for to_skip in ['tl', 'dtl', 'tlold', 'dtlold', 'iSO']:
+ skipped = skip(f, endian='>')
+ (self.ncell) = read_vector(f, 'i', '>')[0]
# Try to figure out the root grid dimensions
est = int(np.rint(self.ncell**(1.0/3.0)))
# Note here: this is the number of *cells* on the root grid.
# This is not the same as the number of Octs.
- #domain dimensions is the number of root *cells*
+ # domain dimensions is the number of root *cells*
self.domain_dimensions = np.ones(3, dtype='int64')*est
self.root_grid_mask_offset = f.tell()
self.root_nocts = self.domain_dimensions.prod()/8
self.root_ncells = self.root_nocts*8
- mylog.debug("Estimating %i cells on a root grid side,"+ \
- "%i root octs",est,self.root_nocts)
- self.root_iOctCh = _read_frecord(f,'>i')[:self.root_ncells]
+ mylog.debug("Estimating %i cells on a root grid side," +
+ "%i root octs", est, self.root_nocts)
+ self.root_iOctCh = read_vector(f, 'i', '>')[:self.root_ncells]
self.root_iOctCh = self.root_iOctCh.reshape(self.domain_dimensions,
- order='F')
+ order='F')
self.root_grid_offset = f.tell()
- #_skip_record(f) # hvar
- #_skip_record(f) # var
- self.root_nhvar = _read_frecord(f,'>f',size_only=True)
- self.root_nvar = _read_frecord(f,'>f',size_only=True)
- #make sure that the number of root variables is a multiple of rootcells
- assert self.root_nhvar%self.root_ncells==0
- assert self.root_nvar%self.root_ncells==0
- self.nhydro_variables = ((self.root_nhvar+self.root_nvar)/
- self.root_ncells)
- self.iOctFree, self.nOct = struct.unpack('>ii', _read_record(f))
+ self.root_nhvar = skip(f, endian='>')
+ self.root_nvar = skip(f, endian='>')
+ # make sure that the number of root variables is a multiple of
+ # rootcells
+ assert self.root_nhvar % self.root_ncells == 0
+ assert self.root_nvar % self.root_ncells == 0
+ self.nhydro_variables = ((self.root_nhvar+self.root_nvar) /
+ self.root_ncells)
+ self.iOctFree, self.nOct = read_vector(f, 'i', '>')
self.child_grid_offset = f.tell()
self.parameters.update(amr_header_vals)
self.parameters['ncell0'] = self.parameters['ng']**3
- #read the particle header
- if not self.skip_particles and self.file_particle_header:
- with open(self.file_particle_header,"rb") as fh:
- particle_header_vals = _read_struct(fh,particle_header_struct)
+ # estimate the root level
+ float_center, fl, iocts, nocts, root_level = _read_art_level_info(
+ f,
+ [0, self.child_grid_offset], 1,
+ coarse_grid=self.domain_dimensions[0])
+ del float_center, fl, iocts, nocts
+ self.root_level = root_level
+ mylog.info("Using root level of %02i", self.root_level)
+ # read the particle header
+ if not self.skip_particles and self._file_particle_header:
+ with open(self._file_particle_header, "rb") as fh:
+ particle_header_vals = read_attrs(
+ fh, particle_header_struct, '>')
fh.seek(seek_extras)
n = particle_header_vals['Nspecies']
- wspecies = np.fromfile(fh,dtype='>f',count=10)
- lspecies = np.fromfile(fh,dtype='>i',count=10)
+ wspecies = np.fromfile(fh, dtype='>f', count=10)
+ lspecies = np.fromfile(fh, dtype='>i', count=10)
self.parameters['wspecies'] = wspecies[:n]
self.parameters['lspecies'] = lspecies[:n]
ls_nonzero = np.diff(lspecies)[:n-1]
- mylog.info("Discovered %i species of particles",len(ls_nonzero))
+ self.star_type = len(ls_nonzero)
+ mylog.info("Discovered %i species of particles", len(ls_nonzero))
mylog.info("Particle populations: "+'%1.1e '*len(ls_nonzero),
- *ls_nonzero)
- for k,v in particle_header_vals.items():
+ *ls_nonzero)
+ for k, v in particle_header_vals.items():
if k in self.parameters.keys():
if not self.parameters[k] == v:
- mylog.info("Inconsistent parameter %s %1.1e %1.1e",k,v,
- self.parameters[k])
+ mylog.info(
+ "Inconsistent parameter %s %1.1e %1.1e", k, v,
+ self.parameters[k])
else:
- self.parameters[k]=v
+ self.parameters[k] = v
self.parameters_particles = particle_header_vals
-
- #setup standard simulation params yt expects to see
+
+ # setup standard simulation params yt expects to see
self.current_redshift = self.parameters["aexpn"]**-1.0 - 1.0
self.omega_lambda = amr_header_vals['Oml0']
self.omega_matter = amr_header_vals['Om0']
self.hubble_constant = amr_header_vals['hubble']
self.min_level = amr_header_vals['min_level']
self.max_level = amr_header_vals['max_level']
- self.hubble_time = 1.0/(self.hubble_constant*100/3.08568025e19)
+ if self.limit_level is not None:
+ self.max_level = min(
+ self.limit_level, amr_header_vals['max_level'])
+ if self.force_max_level is not None:
+ self.max_level = self.force_max_level
+ self.hubble_time = 1.0/(self.hubble_constant*100/3.08568025e19)
self.current_time = b2t(self.parameters['t']) * sec_per_Gyr
+ mylog.info("Max level is %02i", self.max_level)
@classmethod
def _is_valid(self, *args, **kwargs):
@@ -393,20 +423,24 @@
Defined for the NMSU file naming scheme.
This could differ for other formats.
"""
- fn = ("%s" % (os.path.basename(args[0])))
f = ("%s" % args[0])
- prefix, suffix = filename_pattern['amr'].split('%s')
- if fn.endswith(suffix) and fn.startswith(prefix) and\
- os.path.exists(f):
+ prefix, suffix = filename_pattern['amr']
+ with open(f, 'rb') as fh:
+ try:
+ amr_header_vals = read_attrs(fh, amr_header_struct, '>')
return True
+ except AssertionError:
+ return False
return False
+
class ARTDomainSubset(object):
- def __init__(self, domain, mask, cell_count):
+ def __init__(self, domain, mask, cell_count, domain_level):
self.mask = mask
self.domain = domain
self.oct_handler = domain.pf.h.oct_handler
self.cell_count = cell_count
+ self.domain_level = domain_level
level_counts = self.oct_handler.count_levels(
self.domain.pf.max_level, self.domain.domain_id, mask)
assert(level_counts.sum() == cell_count)
@@ -432,12 +466,12 @@
def select_fwidth(self, dobj):
base_dx = 1.0/self.domain.pf.domain_dimensions
widths = np.empty((self.cell_count, 3), dtype="float64")
- dds = (2**self.ires(dobj))
+ dds = (2**self.select_ires(dobj))
for i in range(3):
- widths[:,i] = base_dx[i] / dds
+ widths[:, i] = base_dx[i] / dds
return widths
- def fill(self, content, fields):
+ def fill_root(self, content, ftfields):
"""
This is called from IOHandler. It takes content
which is a binary stream, reads the requested field
@@ -446,135 +480,153 @@
the order they are in in the octhandler.
"""
oct_handler = self.oct_handler
- all_fields = self.domain.pf.h.fluid_field_list
- fields = [f for ft, f in fields]
- dest= {}
- filled = pos = level_offset = 0
+ all_fields = self.domain.pf.h.fluid_field_list
+ fields = [f for ft, f in ftfields]
+ level_offset = 0
field_idxs = [all_fields.index(f) for f in fields]
+ dest = {}
for field in fields:
- dest[field] = np.zeros(self.cell_count, 'float64')
- for level, offset in enumerate(self.domain.level_offsets):
- no = self.domain.level_count[level]
- if level==0:
- data = _read_root_level(content,self.domain.level_child_offsets,
- self.domain.level_count)
- data = data[field_idxs,:]
- else:
- data = _read_child_level(content,self.domain.level_child_offsets,
- self.domain.level_offsets,
- self.domain.level_count,level,fields,
- self.domain.pf.domain_dimensions,
- self.domain.pf.parameters['ncell0'])
- source= {}
- for i,field in enumerate(fields):
- source[field] = np.empty((no, 8), dtype="float64")
- source[field][:,:] = np.reshape(data[i,:],(no,8))
- level_offset += oct_handler.fill_level(self.domain.domain_id,
- level, dest, source, self.mask, level_offset)
+ dest[field] = np.zeros(self.cell_count, 'float64')-1.
+ level = self.domain_level
+ source = {}
+ data = _read_root_level(content, self.domain.level_child_offsets,
+ self.domain.level_count)
+ for field, i in zip(fields, field_idxs):
+ temp = np.reshape(data[i, :], self.domain.pf.domain_dimensions,
+ order='F').astype('float64').T
+ source[field] = temp
+ level_offset += oct_handler.fill_level_from_grid(
+ self.domain.domain_id,
+ level, dest, source, self.mask, level_offset)
return dest
+ def fill_level(self, content, ftfields):
+ oct_handler = self.oct_handler
+ fields = [f for ft, f in ftfields]
+ level_offset = 0
+ dest = {}
+ for field in fields:
+ dest[field] = np.zeros(self.cell_count, 'float64')-1.
+ level = self.domain_level
+ no = self.domain.level_count[level]
+ noct_range = [0, no]
+ source = _read_child_level(
+ content, self.domain.level_child_offsets,
+ self.domain.level_offsets,
+ self.domain.level_count, level, fields,
+ self.domain.pf.domain_dimensions,
+ self.domain.pf.parameters['ncell0'],
+ noct_range=noct_range)
+ nocts_filling = noct_range[1]-noct_range[0]
+ level_offset += oct_handler.fill_level(self.domain.domain_id,
+ level, dest, source,
+ self.mask, level_offset,
+ noct_range[0],
+ nocts_filling)
+ return dest
+
+
class ARTDomainFile(object):
"""
Read in the AMR, left/right edges, fill out the octhandler
"""
- #We already read in the header in static output,
- #and since these headers are defined in only a single file it's
- #best to leave them in the static output
+ # We already read in the header in static output,
+ # and since these headers are defined in only a single file it's
+ # best to leave them in the static output
_last_mask = None
_last_seletor_id = None
- def __init__(self,pf,domain_id,nvar):
+ def __init__(self, pf, domain_id, nvar, level):
self.nvar = nvar
self.pf = pf
self.domain_id = domain_id
+ self.domain_level = level
self._level_count = None
self._level_oct_offsets = None
self._level_child_offsets = None
@property
def level_count(self):
- #this is number of *octs*
- if self._level_count is not None: return self._level_count
+ # this is number of *octs*
+ if self._level_count is not None:
+ return self._level_count
self.level_offsets
- return self._level_count
+ return self._level_count[self.domain_level]
@property
def level_child_offsets(self):
- if self._level_count is not None: return self._level_child_offsets
+ if self._level_count is not None:
+ return self._level_child_offsets
self.level_offsets
return self._level_child_offsets
@property
- def level_offsets(self):
- #this is used by the IO operations to find the file offset,
- #and then start reading to fill values
- #note that this is called hydro_offset in ramses
- if self._level_oct_offsets is not None:
+ def level_offsets(self):
+ # this is used by the IO operations to find the file offset,
+ # and then start reading to fill values
+ # note that this is called hydro_offset in ramses
+ if self._level_oct_offsets is not None:
return self._level_oct_offsets
# We now have to open the file and calculate it
- f = open(self.pf.file_amr, "rb")
+ f = open(self.pf._file_amr, "rb")
nhydrovars, inoll, _level_oct_offsets, _level_child_offsets = \
_count_art_octs(f, self.pf.child_grid_offset, self.pf.min_level,
self.pf.max_level)
- #remember that the root grid is by itself; manually add it back in
+ # remember that the root grid is by itself; manually add it back in
inoll[0] = self.pf.domain_dimensions.prod()/8
_level_child_offsets[0] = self.pf.root_grid_offset
self.nhydrovars = nhydrovars
- self.inoll = inoll #number of octs
+ self.inoll = inoll # number of octs
self._level_oct_offsets = _level_oct_offsets
self._level_child_offsets = _level_child_offsets
self._level_count = inoll
return self._level_oct_offsets
-
- def _read_amr(self, oct_handler):
+
+ def _read_amr_level(self, oct_handler):
"""Open the oct file, read in octs level-by-level.
- For each oct, only the position, index, level and domain
+ For each oct, only the position, index, level and domain
are needed - its position in the octree is found automatically.
The most important is finding all the information to feed
oct_handler.add
"""
- #on the root level we typically have 64^3 octs
- #giving rise to 128^3 cells
- #but on level 1 instead of 128^3 octs, we have 256^3 octs
- #leave this code here instead of static output - it's memory intensive
self.level_offsets
- f = open(self.pf.file_amr, "rb")
- #add the root *cell* not *oct* mesh
+ f = open(self.pf._file_amr, "rb")
+ level = self.domain_level
+ unitary_center, fl, iocts, nocts, root_level = _read_art_level_info(
+ f,
+ self._level_oct_offsets, level,
+ coarse_grid=self.pf.domain_dimensions[0],
+ root_level=self.pf.root_level)
+ nocts_check = oct_handler.add(self.domain_id, level, nocts,
+ unitary_center, self.domain_id)
+ assert(nocts_check == nocts)
+ mylog.debug("Added %07i octs on level %02i, cumulative is %07i",
+ nocts, level, oct_handler.nocts)
+
+ def _read_amr_root(self, oct_handler):
+ self.level_offsets
+ f = open(self.pf._file_amr, "rb")
+ # add the root *cell* not *oct* mesh
+ level = self.domain_level
root_octs_side = self.pf.domain_dimensions[0]/2
NX = np.ones(3)*root_octs_side
+ octs_side = NX*2**level
LE = np.array([0.0, 0.0, 0.0], dtype='float64')
RE = np.array([1.0, 1.0, 1.0], dtype='float64')
root_dx = (RE - LE) / NX
LL = LE + root_dx/2.0
RL = RE - root_dx/2.0
- #compute floating point centers of root octs
- root_fc= np.mgrid[LL[0]:RL[0]:NX[0]*1j,
- LL[1]:RL[1]:NX[1]*1j,
- LL[2]:RL[2]:NX[2]*1j ]
- root_fc= np.vstack([p.ravel() for p in root_fc]).T
- nocts_check = oct_handler.add(1, 0, root_octs_side**3,
+ # compute floating point centers of root octs
+ root_fc = np.mgrid[LL[0]:RL[0]:NX[0]*1j,
+ LL[1]:RL[1]:NX[1]*1j,
+ LL[2]:RL[2]:NX[2]*1j]
+ root_fc = np.vstack([p.ravel() for p in root_fc]).T
+ nocts_check = oct_handler.add(self.domain_id, level,
+ root_octs_side**3,
root_fc, self.domain_id)
assert(oct_handler.nocts == root_fc.shape[0])
- nocts_added = root_fc.shape[0]
mylog.debug("Added %07i octs on level %02i, cumulative is %07i",
- root_octs_side**3, 0,nocts_added)
- for level in xrange(1, self.pf.max_level+1):
- left_index, fl, iocts, nocts,root_level = _read_art_level_info(f,
- self._level_oct_offsets,level,
- coarse_grid=self.pf.domain_dimensions[0])
- left_index/=2
- #at least one of the indices should be odd
- #assert np.sum(left_index[:,0]%2==1)>0
- octs_side = NX*2**level
- float_left_edge = left_index.astype("float64") / octs_side
- float_center = float_left_edge + 0.5*1.0/octs_side
- #all floatin unitary positions should fit inside the domain
- assert np.all(float_center<1.0)
- nocts_check = oct_handler.add(1,level, nocts, float_left_edge, self.domain_id)
- nocts_added += nocts
- assert(oct_handler.nocts == nocts_added)
- mylog.debug("Added %07i octs on level %02i, cumulative is %07i",
- nocts, level,nocts_added)
+ root_octs_side**3, 0, oct_handler.nocts)
def select(self, selector):
if id(selector) == self._last_selector_id:
@@ -585,8 +637,8 @@
def count(self, selector):
if id(selector) == self._last_selector_id:
- if self._last_mask is None: return 0
+ if self._last_mask is None:
+ return 0
return self._last_mask.sum()
self.select(selector)
return self.count(selector)
-
diff -r fb14dd338289ba004f2d42bfe4f5e7f5160809c7 -r 3a1a2a95457855b229396b14b67aeba15d2a3f27 yt/frontends/art/definitions.py
--- a/yt/frontends/art/definitions.py
+++ b/yt/frontends/art/definitions.py
@@ -25,7 +25,10 @@
"""
-fluid_fields= [
+# If not otherwise specified, we are big endian
+endian = '>'
+
+fluid_fields = [
'Density',
'TotalEnergy',
'XMomentumDensity',
@@ -40,32 +43,29 @@
'PotentialOld'
]
-hydro_struct = [('pad1','>i'),('idc','>i'),('iOctCh','>i')]
+hydro_struct = [('pad1', '>i'), ('idc', '>i'), ('iOctCh', '>i')]
for field in fluid_fields:
- hydro_struct += (field,'>f'),
-hydro_struct += ('pad2','>i'),
+ hydro_struct += (field, '>f'),
+hydro_struct += ('pad2', '>i'),
-particle_fields= [
- 'particle_age',
+particle_fields = [
+ 'particle_mass', # stars have variable mass
'particle_index',
- 'particle_mass',
- 'particle_mass_initial',
- 'particle_creation_time',
- 'particle_metallicity1',
- 'particle_metallicity2',
- 'particle_metallicity',
+ 'particle_type',
'particle_position_x',
'particle_position_y',
'particle_position_z',
'particle_velocity_x',
'particle_velocity_y',
'particle_velocity_z',
- 'particle_type',
- 'particle_index'
+ 'particle_mass_initial',
+ 'particle_creation_time',
+ 'particle_metallicity1',
+ 'particle_metallicity2',
+ 'particle_metallicity',
]
particle_star_fields = [
- 'particle_age',
'particle_mass',
'particle_mass_initial',
'particle_creation_time',
@@ -74,110 +74,65 @@
'particle_metallicity',
]
-filename_pattern = {
- 'amr':'10MpcBox_csf512_%s.d',
- 'particle_header':'PMcrd%s.DAT',
- 'particle_data':'PMcrs0%s.DAT',
- 'particle_stars':'stars_%s.dat'
-}
-filename_pattern_hf = {
- 'particle_header':'PMcrd_%s.DAT',
- 'particle_data':'PMcrs0_%s.DAT',
+filename_pattern = {
+ 'amr': ['10MpcBox_', '.d'],
+ 'particle_header': ['PMcrd', '.DAT'],
+ 'particle_data': ['PMcrs', '.DAT'],
+ 'particle_stars': ['stars', '.dat']
}
amr_header_struct = [
- ('>i','pad byte'),
- ('>256s','jname'),
- ('>i','pad byte'),
- ('>i','pad byte'),
- ('>i','istep'),
- ('>d','t'),
- ('>d','dt'),
- ('>f','aexpn'),
- ('>f','ainit'),
- ('>i','pad byte'),
- ('>i','pad byte'),
- ('>f','boxh'),
- ('>f','Om0'),
- ('>f','Oml0'),
- ('>f','Omb0'),
- ('>f','hubble'),
- ('>i','pad byte'),
- ('>i','pad byte'),
- ('>i','nextras'),
- ('>i','pad byte'),
- ('>i','pad byte'),
- ('>f','extra1'),
- ('>f','extra2'),
- ('>i','pad byte'),
- ('>i','pad byte'),
- ('>256s','lextra'),
- ('>256s','lextra'),
- ('>i','pad byte'),
- ('>i', 'pad byte'),
- ('>i', 'min_level'),
- ('>i', 'max_level'),
- ('>i', 'pad byte'),
+ ('jname', 1, '256s'),
+ (('istep', 't', 'dt', 'aexpn', 'ainit'), 1, 'iddff'),
+ (('boxh', 'Om0', 'Oml0', 'Omb0', 'hubble'), 5, 'f'),
+ ('nextras', 1, 'i'),
+ (('extra1', 'extra2'), 2, 'f'),
+ ('lextra', 1, '512s'),
+ (('min_level', 'max_level'), 2, 'i')
]
-particle_header_struct =[
- ('>i','pad'),
- ('45s','header'),
- ('>f','aexpn'),
- ('>f','aexp0'),
- ('>f','amplt'),
- ('>f','astep'),
- ('>i','istep'),
- ('>f','partw'),
- ('>f','tintg'),
- ('>f','Ekin'),
- ('>f','Ekin1'),
- ('>f','Ekin2'),
- ('>f','au0'),
- ('>f','aeu0'),
- ('>i','Nrow'),
- ('>i','Ngridc'),
- ('>i','Nspecies'),
- ('>i','Nseed'),
- ('>f','Om0'),
- ('>f','Oml0'),
- ('>f','hubble'),
- ('>f','Wp5'),
- ('>f','Ocurv'),
- ('>f','Omb0'),
- ('>%ds'%(396),'extras'),
- ('>f','unknown'),
- ('>i','pad')
+particle_header_struct = [
+ (('header',
+ 'aexpn', 'aexp0', 'amplt', 'astep',
+ 'istep',
+ 'partw', 'tintg',
+ 'Ekin', 'Ekin1', 'Ekin2',
+ 'au0', 'aeu0',
+ 'Nrow', 'Ngridc', 'Nspecies', 'Nseed',
+ 'Om0', 'Oml0', 'hubble', 'Wp5', 'Ocurv', 'Omb0',
+ 'extras', 'unknown'),
+ 1,
+ '45sffffi'+'fffffff'+'iiii'+'ffffff'+'396s'+'f')
]
star_struct = [
- ('>d',('tdum','adum')),
- ('>i','nstars'),
- ('>d',('ws_old','ws_oldi')),
- ('>f','mass'),
- ('>f','imass'),
- ('>f','tbirth'),
- ('>f','metallicity1'),
- ('>f','metallicity2')
- ]
+ ('>d', ('tdum', 'adum')),
+ ('>i', 'nstars'),
+ ('>d', ('ws_old', 'ws_oldi')),
+ ('>f', 'particle_mass'),
+ ('>f', 'particle_mass_initial'),
+ ('>f', 'particle_creation_time'),
+ ('>f', 'particle_metallicity1'),
+ ('>f', 'particle_metallicity2')
+]
star_name_map = {
- 'particle_mass':'mass',
- 'particle_mass_initial':'imass',
- 'particle_age':'tbirth',
- 'particle_metallicity1':'metallicity1',
- 'particle_metallicity2':'metallicity2',
- 'particle_metallicity':'metallicity',
- }
+ 'particle_mass': 'mass',
+ 'particle_mass_initial': 'imass',
+ 'particle_creation_time': 'tbirth',
+ 'particle_metallicity1': 'metallicity1',
+ 'particle_metallicity2': 'metallicity2',
+ 'particle_metallicity': 'metallicity',
+}
constants = {
- "Y_p":0.245,
- "gamma":5./3.,
- "T_CMB0":2.726,
- "T_min":300.,
- "ng":128,
- "wmu":4.0/(8.0-5.0*0.245)
+ "Y_p": 0.245,
+ "gamma": 5./3.,
+ "T_CMB0": 2.726,
+ "T_min": 300.,
+ "ng": 128,
+ "wmu": 4.0/(8.0-5.0*0.245)
}
seek_extras = 137
diff -r fb14dd338289ba004f2d42bfe4f5e7f5160809c7 -r 3a1a2a95457855b229396b14b67aeba15d2a3f27 yt/frontends/art/fields.py
--- a/yt/frontends/art/fields.py
+++ b/yt/frontends/art/fields.py
@@ -3,6 +3,8 @@
Author: Matthew Turk <matthewturk at gmail.com>
Affiliation: UCSD
+Author: Chris Moody <matthewturk at gmail.com>
+Affiliation: UCSC
Homepage: http://yt-project.org/
License:
Copyright (C) 2010-2011 Matthew Turk. All Rights Reserved.
@@ -22,7 +24,7 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
-
+import numpy as np
from yt.data_objects.field_info_container import \
FieldInfoContainer, \
FieldInfo, \
@@ -35,210 +37,221 @@
ValidateGridType
import yt.data_objects.universal_fields
import yt.utilities.lib as amr_utils
+from yt.utilities.physical_constants import mass_sun_cgs
+from yt.frontends.art.definitions import *
KnownARTFields = FieldInfoContainer()
add_art_field = KnownARTFields.add_field
-
ARTFieldInfo = FieldInfoContainer.create_with_fallback(FieldInfo)
add_field = ARTFieldInfo.add_field
-import numpy as np
+for f in fluid_fields:
+ add_art_field(f, function=NullFunc, take_log=True,
+ validators=[ValidateDataField(f)])
-#these are just the hydro fields
-known_art_fields = [ 'Density','TotalEnergy',
- 'XMomentumDensity','YMomentumDensity','ZMomentumDensity',
- 'Pressure','Gamma','GasEnergy',
- 'MetalDensitySNII', 'MetalDensitySNIa',
- 'PotentialNew','PotentialOld']
-
-#Add the fields, then later we'll individually defined units and names
-for f in known_art_fields:
+for f in particle_fields:
add_art_field(f, function=NullFunc, take_log=True,
- validators = [ValidateDataField(f)])
-
-#Hydro Fields that are verified to be OK unit-wise:
-#Density
-#Temperature
-#metallicities
-#MetalDensity SNII + SNia
-
-#Hydro Fields that need to be tested:
-#TotalEnergy
-#XYZMomentum
-#Pressure
-#Gamma
-#GasEnergy
-#Potentials
-#xyzvelocity
-
-#Particle fields that are tested:
-#particle_position_xyz
-#particle_type
-#particle_index
-#particle_mass
-#particle_mass_initial
-#particle_age
-#particle_velocity
-#particle_metallicity12
-
-#Particle fields that are untested:
-#NONE
-
-#Other checks:
-#CellMassMsun == Density * CellVolume
+ validators=[ValidateDataField(f)],
+ particle_type=True)
+add_art_field("particle_mass", function=NullFunc, take_log=True,
+ validators=[ValidateDataField(f)],
+ particle_type=True,
+ convert_function=lambda x: x.convert("particle_mass"))
+add_art_field("particle_mass_initial", function=NullFunc, take_log=True,
+ validators=[ValidateDataField(f)],
+ particle_type=True,
+ convert_function=lambda x: x.convert("particle_mass"))
def _convertDensity(data):
return data.convert("Density")
KnownARTFields["Density"]._units = r"\rm{g}/\rm{cm}^3"
KnownARTFields["Density"]._projected_units = r"\rm{g}/\rm{cm}^2"
-KnownARTFields["Density"]._convert_function=_convertDensity
+KnownARTFields["Density"]._convert_function = _convertDensity
def _convertTotalEnergy(data):
return data.convert("GasEnergy")
-KnownARTFields["TotalEnergy"]._units = r"\rm{g}/\rm{cm}^3"
-KnownARTFields["TotalEnergy"]._projected_units = r"\rm{K}"
-KnownARTFields["TotalEnergy"]._convert_function=_convertTotalEnergy
+KnownARTFields["TotalEnergy"]._units = r"\rm{g}\rm{cm}^2/\rm{s}^2"
+KnownARTFields["TotalEnergy"]._projected_units = r"\rm{g}\rm{cm}^3/\rm{s}^2"
+KnownARTFields["TotalEnergy"]._convert_function = _convertTotalEnergy
def _convertXMomentumDensity(data):
- tr = data.convert("Mass")*data.convert("Velocity")
+ tr = data.convert("Mass")*data.convert("Velocity")
tr *= (data.convert("Density")/data.convert("Mass"))
return tr
-KnownARTFields["XMomentumDensity"]._units = r"\rm{mg}/\rm{s}/\rm{cm}^3"
-KnownARTFields["XMomentumDensity"]._projected_units = r"\rm{K}"
-KnownARTFields["XMomentumDensity"]._convert_function=_convertXMomentumDensity
+KnownARTFields["XMomentumDensity"]._units = r"\rm{g}/\rm{s}/\rm{cm}^3"
+KnownARTFields["XMomentumDensity"]._projected_units = r"\rm{g}/\rm{s}/\rm{cm}^2"
+KnownARTFields["XMomentumDensity"]._convert_function = _convertXMomentumDensity
def _convertYMomentumDensity(data):
- tr = data.convert("Mass")*data.convert("Velocity")
+ tr = data.convert("Mass")*data.convert("Velocity")
tr *= (data.convert("Density")/data.convert("Mass"))
return tr
-KnownARTFields["YMomentumDensity"]._units = r"\rm{mg}/\rm{s}/\rm{cm}^3"
-KnownARTFields["YMomentumDensity"]._projected_units = r"\rm{K}"
-KnownARTFields["YMomentumDensity"]._convert_function=_convertYMomentumDensity
+KnownARTFields["YMomentumDensity"]._units = r"\rm{g}/\rm{s}/\rm{cm}^3"
+KnownARTFields["YMomentumDensity"]._projected_units = r"\rm{g}/\rm{s}/\rm{cm}^2"
+KnownARTFields["YMomentumDensity"]._convert_function = _convertYMomentumDensity
def _convertZMomentumDensity(data):
- tr = data.convert("Mass")*data.convert("Velocity")
+ tr = data.convert("Mass")*data.convert("Velocity")
tr *= (data.convert("Density")/data.convert("Mass"))
return tr
-KnownARTFields["ZMomentumDensity"]._units = r"\rm{mg}/\rm{s}/\rm{cm}^3"
-KnownARTFields["ZMomentumDensity"]._projected_units = r"\rm{K}"
-KnownARTFields["ZMomentumDensity"]._convert_function=_convertZMomentumDensity
+KnownARTFields["ZMomentumDensity"]._units = r"\rm{g}/\rm{s}/\rm{cm}^3"
+KnownARTFields["ZMomentumDensity"]._projected_units = r"\rm{g}/\rm{s}/\rm{cm}^2"
+KnownARTFields["ZMomentumDensity"]._convert_function = _convertZMomentumDensity
def _convertPressure(data):
return data.convert("Pressure")
-KnownARTFields["Pressure"]._units = r"\rm{g}/\rm{cm}/\rm{s}^2"
+KnownARTFields["Pressure"]._units = r"\rm{g}/\rm{s}^2/\rm{cm}^1"
KnownARTFields["Pressure"]._projected_units = r"\rm{g}/\rm{s}^2"
-KnownARTFields["Pressure"]._convert_function=_convertPressure
+KnownARTFields["Pressure"]._convert_function = _convertPressure
def _convertGamma(data):
return 1.0
KnownARTFields["Gamma"]._units = r""
KnownARTFields["Gamma"]._projected_units = r""
-KnownARTFields["Gamma"]._convert_function=_convertGamma
+KnownARTFields["Gamma"]._convert_function = _convertGamma
def _convertGasEnergy(data):
return data.convert("GasEnergy")
-KnownARTFields["GasEnergy"]._units = r"\rm{ergs}/\rm{g}"
-KnownARTFields["GasEnergy"]._projected_units = r""
-KnownARTFields["GasEnergy"]._convert_function=_convertGasEnergy
+KnownARTFields["GasEnergy"]._units = r"\rm{g}\rm{cm}^2/\rm{s}^2"
+KnownARTFields["GasEnergy"]._projected_units = r"\rm{g}\rm{cm}^3/\rm{s}^2"
+KnownARTFields["GasEnergy"]._convert_function = _convertGasEnergy
def _convertMetalDensitySNII(data):
return data.convert('Density')
KnownARTFields["MetalDensitySNII"]._units = r"\rm{g}/\rm{cm}^3"
KnownARTFields["MetalDensitySNII"]._projected_units = r"\rm{g}/\rm{cm}^2"
-KnownARTFields["MetalDensitySNII"]._convert_function=_convertMetalDensitySNII
+KnownARTFields["MetalDensitySNII"]._convert_function = _convertMetalDensitySNII
def _convertMetalDensitySNIa(data):
return data.convert('Density')
KnownARTFields["MetalDensitySNIa"]._units = r"\rm{g}/\rm{cm}^3"
KnownARTFields["MetalDensitySNIa"]._projected_units = r"\rm{g}/\rm{cm}^2"
-KnownARTFields["MetalDensitySNIa"]._convert_function=_convertMetalDensitySNIa
+KnownARTFields["MetalDensitySNIa"]._convert_function = _convertMetalDensitySNIa
def _convertPotentialNew(data):
return data.convert("Potential")
-KnownARTFields["PotentialNew"]._units = r"\rm{g}/\rm{cm}^3"
-KnownARTFields["PotentialNew"]._projected_units = r"\rm{g}/\rm{cm}^2"
-KnownARTFields["PotentialNew"]._convert_function=_convertPotentialNew
+KnownARTFields["PotentialNew"]._units = r"\rm{g}\rm{cm}^2/\rm{s}^2"
+KnownARTFields["PotentialNew"]._projected_units = r"\rm{g}\rm{cm}^3/\rm{s}^2"
+KnownARTFields["PotentialNew"]._convert_function = _convertPotentialNew
def _convertPotentialOld(data):
return data.convert("Potential")
-KnownARTFields["PotentialOld"]._units = r"\rm{g}/\rm{cm}^3"
-KnownARTFields["PotentialOld"]._projected_units = r"\rm{g}/\rm{cm}^2"
-KnownARTFields["PotentialOld"]._convert_function=_convertPotentialOld
+KnownARTFields["PotentialOld"]._units = r"\rm{g}\rm{cm}^2/\rm{s}^2"
+KnownARTFields["PotentialOld"]._projected_units = r"\rm{g}\rm{cm}^3/\rm{s}^2"
+KnownARTFields["PotentialOld"]._convert_function = _convertPotentialOld
####### Derived fields
+def _temperature(field, data):
+ tr = data["GasEnergy"]/data["Density"]
+ tr /= data.pf.conversion_factors["GasEnergy"]
+ tr *= data.pf.conversion_factors["Density"]
+ tr *= data.pf.conversion_factors['tr']
+ return tr
-def _temperature(field, data):
- dg = data["GasEnergy"] #.astype('float64')
- dg /= data.pf.conversion_factors["GasEnergy"]
- dd = data["Density"] #.astype('float64')
- dd /= data.pf.conversion_factors["Density"]
- tr = dg/dd*data.pf.conversion_factors['tr']
- #ghost cells have zero density?
- tr[np.isnan(tr)] = 0.0
- #dd[di] = -1.0
- #if data.id==460:
- #tr[di] = -1.0 #replace the zero-density points with zero temp
- #print tr.min()
- #assert np.all(np.isfinite(tr))
- return tr
def _converttemperature(data):
- #x = data.pf.conversion_factors["Temperature"]
- x = 1.0
- return x
-add_field("Temperature", function=_temperature, units = r"\mathrm{K}",take_log=True)
+ return 1.0
+add_field("Temperature", function=_temperature,
+ units=r"\mathrm{K}", take_log=True)
ARTFieldInfo["Temperature"]._units = r"\mathrm{K}"
ARTFieldInfo["Temperature"]._projected_units = r"\mathrm{K}"
-#ARTFieldInfo["Temperature"]._convert_function=_converttemperature
def _metallicity_snII(field, data):
- tr = data["MetalDensitySNII"] / data["Density"]
+ tr = data["MetalDensitySNII"] / data["Density"]
return tr
-add_field("Metallicity_SNII", function=_metallicity_snII, units = r"\mathrm{K}",take_log=True)
+add_field("Metallicity_SNII", function=_metallicity_snII,
+ units=r"\mathrm{K}", take_log=True)
ARTFieldInfo["Metallicity_SNII"]._units = r""
ARTFieldInfo["Metallicity_SNII"]._projected_units = r""
def _metallicity_snIa(field, data):
- tr = data["MetalDensitySNIa"] / data["Density"]
+ tr = data["MetalDensitySNIa"] / data["Density"]
return tr
-add_field("Metallicity_SNIa", function=_metallicity_snIa, units = r"\mathrm{K}",take_log=True)
+add_field("Metallicity_SNIa", function=_metallicity_snIa,
+ units=r"\mathrm{K}", take_log=True)
ARTFieldInfo["Metallicity_SNIa"]._units = r""
ARTFieldInfo["Metallicity_SNIa"]._projected_units = r""
def _metallicity(field, data):
- tr = data["Metal_Density"] / data["Density"]
+ tr = data["Metal_Density"] / data["Density"]
return tr
-add_field("Metallicity", function=_metallicity, units = r"\mathrm{K}",take_log=True)
+add_field("Metallicity", function=_metallicity,
+ units=r"\mathrm{K}", take_log=True)
ARTFieldInfo["Metallicity"]._units = r""
ARTFieldInfo["Metallicity"]._projected_units = r""
-def _x_velocity(field,data):
- tr = data["XMomentumDensity"]/data["Density"]
+def _x_velocity(field, data):
+ tr = data["XMomentumDensity"]/data["Density"]
return tr
-add_field("x-velocity", function=_x_velocity, units = r"\mathrm{cm/s}",take_log=False)
+add_field("x-velocity", function=_x_velocity,
+ units=r"\mathrm{cm/s}", take_log=False)
ARTFieldInfo["x-velocity"]._units = r"\rm{cm}/\rm{s}"
ARTFieldInfo["x-velocity"]._projected_units = r"\rm{cm}/\rm{s}"
-def _y_velocity(field,data):
- tr = data["YMomentumDensity"]/data["Density"]
+def _y_velocity(field, data):
+ tr = data["YMomentumDensity"]/data["Density"]
return tr
-add_field("y-velocity", function=_y_velocity, units = r"\mathrm{cm/s}",take_log=False)
+add_field("y-velocity", function=_y_velocity,
+ units=r"\mathrm{cm/s}", take_log=False)
ARTFieldInfo["y-velocity"]._units = r"\rm{cm}/\rm{s}"
ARTFieldInfo["y-velocity"]._projected_units = r"\rm{cm}/\rm{s}"
-def _z_velocity(field,data):
- tr = data["ZMomentumDensity"]/data["Density"]
+def _z_velocity(field, data):
+ tr = data["ZMomentumDensity"]/data["Density"]
return tr
-add_field("z-velocity", function=_z_velocity, units = r"\mathrm{cm/s}",take_log=False)
+add_field("z-velocity", function=_z_velocity,
+ units=r"\mathrm{cm/s}", take_log=False)
ARTFieldInfo["z-velocity"]._units = r"\rm{cm}/\rm{s}"
ARTFieldInfo["z-velocity"]._projected_units = r"\rm{cm}/\rm{s}"
def _metal_density(field, data):
- tr = data["MetalDensitySNIa"]
+ tr = data["MetalDensitySNIa"]
tr += data["MetalDensitySNII"]
return tr
-add_field("Metal_Density", function=_metal_density, units = r"\mathrm{K}",take_log=True)
-ARTFieldInfo["Metal_Density"]._units = r""
-ARTFieldInfo["Metal_Density"]._projected_units = r""
+add_field("Metal_Density", function=_metal_density,
+ units=r"\mathrm{K}", take_log=True)
+ARTFieldInfo["Metal_Density"]._units = r"\rm{g}/\rm{cm}^3"
+ARTFieldInfo["Metal_Density"]._projected_units = r"\rm{g}/\rm{cm}^2"
+# Particle fields
+def _particle_age(field, data):
+ tr = data["particle_creation_time"]
+ return data.pf.current_time - tr
+add_field("particle_age", function=_particle_age, units=r"\mathrm{s}",
+ take_log=True, particle_type=True)
-#Particle fields
+def spread_ages(ages, spread=1.0e7*365*24*3600):
+ # stars are formed in lumps; spread out the ages linearly
+ da = np.diff(ages)
+ assert np.all(da <= 0)
+ # ages should always be decreasing, and ordered so
+ agesd = np.zeros(ages.shape)
+ idx, = np.where(da < 0)
+ idx += 1 # mark the right edges
+ # spread this age evenly out to the next age
+ lidx = 0
+ lage = 0
+ for i in idx:
+ n = i-lidx # n stars affected
+ rage = ages[i]
+ lage = max(rage-spread, 0.0)
+ agesd[lidx:i] = np.linspace(lage, rage, n)
+ lidx = i
+ # lage=rage
+ # we didn't get the last iter
+ n = agesd.shape[0]-lidx
+ rage = ages[-1]
+ lage = max(rage-spread, 0.0)
+ agesd[lidx:] = np.linspace(lage, rage, n)
+ return agesd
+
+def _particle_age_spread(field, data):
+ tr = data["particle_creation_time"]
+ return spread_ages(data.pf.current_time - tr)
+
+add_field("particle_age_spread", function=_particle_age_spread,
+ particle_type=True, take_log=True, units=r"\rm{s}")
+
+def _ParticleMassMsun(field, data):
+ return data["particle_mass"]/mass_sun_cgs
+add_field("ParticleMassMsun", function=_ParticleMassMsun, particle_type=True,
+ take_log=True, units=r"\rm{Msun}")
This diff is so big that we needed to truncate the remainder.
https://bitbucket.org/yt_analysis/yt-3.0/commits/b521b6bb599c/
Changeset: b521b6bb599c
Branch: yt-3.0
User: MatthewTurk
Date: 2013-04-15 19:30:28
Summary: Merging in removal of shape and size from Doug.
Affected #: 2 files
diff -r 94eeca5052506364a8db0afcc76d53ed2f36ff7b -r b521b6bb599c1e8bfd36c85f88ca303a9b667546 yt/data_objects/data_containers.py
--- a/yt/data_objects/data_containers.py
+++ b/yt/data_objects/data_containers.py
@@ -426,8 +426,6 @@
_sort_by = None
_selector = None
_current_chunk = None
- size = None
- shape = None
def __init__(self, *args, **kwargs):
super(YTSelectionContainer, self).__init__(*args, **kwargs)
@@ -524,17 +522,20 @@
# There are several items that need to be swapped out
# field_data, size, shape
old_field_data, self.field_data = self.field_data, YTFieldData()
- old_size, self.size = self.size, chunk.data_size
old_chunk, self._current_chunk = self._current_chunk, chunk
old_locked, self._locked = self._locked, False
- #self.shape = (self.size,)
yield
self.field_data = old_field_data
- self.size = old_size
- #self.shape = (old_size,)
self._current_chunk = old_chunk
self._locked = old_locked
+# @property
+# def size(self) :
+# if self._current_chunk is None :
+## self.hierarchy._identify_base_chunk(self)
+# return 0
+# return self._current_chunk.data_size
+
@property
def icoords(self):
if self._current_chunk is None:
diff -r 94eeca5052506364a8db0afcc76d53ed2f36ff7b -r b521b6bb599c1e8bfd36c85f88ca303a9b667546 yt/data_objects/universal_fields.py
--- a/yt/data_objects/universal_fields.py
+++ b/yt/data_objects/universal_fields.py
@@ -96,7 +96,7 @@
display_field = False)
def _Ones(field, data):
- return np.ones(data.shape, dtype='float64')
+ return np.ones(data.ActiveDimensions, dtype='float64')
add_field("Ones", function=_Ones,
projection_conversion="unitary",
display_field = False)
https://bitbucket.org/yt_analysis/yt-3.0/commits/61b473b27a55/
Changeset: 61b473b27a55
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-12 04:30:31
Summary: Adding Morton indices function.
Affected #: 1 file
diff -r b8b8621258376d8a8e694fb58358f5d381639382 -r 61b473b27a558774070d4e00a450a2492c6e1398 yt/utilities/lib/geometry_utils.pyx
--- a/yt/utilities/lib/geometry_utils.pyx
+++ b/yt/utilities/lib/geometry_utils.pyx
@@ -300,6 +300,40 @@
positions[i, j] = p[j]
return positions
+cdef np.uint64_t _const20 = 0x000001FFC00003FF
+cdef np.uint64_t _const10 = 0x0007E007C00F801F
+cdef np.uint64_t _const04 = 0x00786070C0E181C3
+cdef np.uint64_t _const2a = 0x0199219243248649
+cdef np.uint64_t _const2b = 0x0649249249249249
+cdef np.uint64_t _const2c = 0x1249249249249249
+
+ at cython.cdivision(True)
+ at cython.boundscheck(False)
+ at cython.wraparound(False)
+cdef inline np.uint64_t spread_bits(np.uint64_t x):
+ # This magic comes from http://stackoverflow.com/questions/1024754/how-to-compute-a-3d-morton-number-interleave-the-bits-of-3-ints
+ x=(x|(x<<20))&_const20
+ x=(x|(x<<10))&_const10
+ x=(x|(x<<4))&_const04
+ x=(x|(x<<2))&_const2a
+ x=(x|(x<<2))&_const2b
+ x=(x|(x<<2))&_const2c
+ return x
+
+ at cython.cdivision(True)
+ at cython.boundscheck(False)
+ at cython.wraparound(False)
+def get_morton_indices(np.ndarray[np.int64_t, ndim=2] left_index):
+ cdef np.int64_t i, mi
+ cdef np.ndarray[np.uint64_t, ndim=1] morton_indices
+ morton_indices = np.zeros(left_index.shape[0], 'uint64')
+ for i in range(left_index.shape[0]):
+ mi = 0
+ mi |= spread_bits(left_index[i,0])<<0
+ mi |= spread_bits(left_index[i,1])<<1
+ mi |= spread_bits(left_index[i,2])<<2
+ morton_indices[i] = mi
+ return morton_indices
@cython.boundscheck(False)
@cython.wraparound(False)
https://bitbucket.org/yt_analysis/yt-3.0/commits/67ee5eb29734/
Changeset: 67ee5eb29734
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-12 22:15:18
Summary: Removing ParticleArrays object. Convert to using Morton Indices for ParticleOctrees.
This is a major change, and one that is not fully baked; additional changes
will need to be made to the particle frontends themselves, where we need to
convert to generating the Morton index, using a Barnes-Hut type indexing for
the particles themselves, and then doing IO in a different way based on that.
The octree generated for the OWLS dataset is identical to the one generated
previously.
Affected #: 2 files
diff -r 61b473b27a558774070d4e00a450a2492c6e1398 -r 67ee5eb297349e6e874cac0db007020dfa1aab08 yt/geometry/oct_container.pxd
--- a/yt/geometry/oct_container.pxd
+++ b/yt/geometry/oct_container.pxd
@@ -26,8 +26,6 @@
cimport numpy as np
from fp_utils cimport *
-cdef struct ParticleArrays
-
cdef struct Oct
cdef struct Oct:
np.int64_t file_ind # index with respect to the order in which it was
@@ -39,7 +37,6 @@
np.int64_t domain # (opt) addl int index
np.int64_t pos[3] # position in ints
np.int8_t level
- ParticleArrays *sd
Oct *children[2][2][2]
Oct *parent
@@ -73,9 +70,3 @@
cdef OctAllocationContainer **domains
cdef Oct *next_root(self, int domain_id, int ind[3])
cdef Oct *next_child(self, int domain_id, int ind[3], Oct *parent)
-
-cdef struct ParticleArrays:
- Oct *oct
- ParticleArrays *next
- np.float64_t **pos
- np.int64_t np
diff -r 61b473b27a558774070d4e00a450a2492c6e1398 -r 67ee5eb297349e6e874cac0db007020dfa1aab08 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -33,6 +33,8 @@
from selection_routines cimport SelectorObject
cimport cython
+DEF ORDER_MAX=20
+
cdef extern from "stdlib.h":
# NOTE that size_t might not be int
void *alloca(int)
@@ -815,11 +817,6 @@
elif o1.domain > o2.domain: return 1
cdef class ParticleOctreeContainer(OctreeContainer):
- #Each ParticleArrays contains an Oct
- #a reference to the next ParticleArrays
- #its index and the number of particles
- cdef ParticleArrays *first_sd
- cdef ParticleArrays *last_sd
cdef Oct** oct_list
#The starting oct index of each domain
cdef np.int64_t *dom_offsets
@@ -860,11 +857,6 @@
for k in range(2):
if o.children[i][j][k] == NULL: continue
self.visit_free(o.children[i][j][k])
- if o.sd.np >= 0:
- if o.sd.pos != NULL:
- for i in range(3):
- free(o.sd.pos[i])
- free(o.sd.pos)
free(o)
def __iter__(self):
@@ -984,36 +976,29 @@
#every domain
cdef int max_level = 0
self.oct_list = <Oct**> malloc(sizeof(Oct*)*self.nocts)
- cdef np.int64_t i = 0
- cdef np.int64_t dom_ind
- cdef ParticleArrays *c = self.first_sd
- while c != NULL:
- self.oct_list[i] = c.oct
- max_level = imax(max_level, c.oct.level)
- if c.np >= 0:
- for j in range(3):
- free(c.pos[j])
- free(c.pos)
- c.pos = NULL
- c = c.next
- i += 1
+ cdef np.int64_t i = 0, lpos = 0
self.max_level = max_level
- qsort(self.oct_list, self.nocts, sizeof(Oct*), &compare_octs)
cdef int cur_dom = -1
# We always need at least 2, and if max_domain is 0, we need 3.
- self.dom_offsets = <np.int64_t *>malloc(sizeof(np.int64_t) *
- (self.max_domain + 3))
- self.dom_offsets[0] = 0
- dom_ind = 0
+ for i in range(self.nn[0]):
+ for j in range(self.nn[1]):
+ for k in range(self.nn[2]):
+ self.visit_assign(self.root_mesh[i][j][k], &lpos)
+ assert(lpos == self.nocts)
for i in range(self.nocts):
self.oct_list[i].domain_ind = i
- self.oct_list[i].file_ind = dom_ind
- dom_ind += 1
- if self.oct_list[i].domain > cur_dom:
- cur_dom = self.oct_list[i].domain
- self.dom_offsets[cur_dom + 1] = i
- dom_ind = 0
- self.dom_offsets[cur_dom + 2] = self.nocts
+ self.oct_list[i].file_ind = -1
+
+ cdef visit_assign(self, Oct *o, np.int64_t *lpos):
+ cdef int i, j, k
+ self.oct_list[lpos[0]] = o
+ lpos[0] += 1
+ for i in range(2):
+ for j in range(2):
+ for k in range(2):
+ if o.children[i][j][k] != NULL:
+ self.visit_assign(o.children[i][j][k], lpos)
+ return
cdef np.int64_t get_domain_offset(self, int domain_id):
return self.dom_offsets[domain_id + 1]
@@ -1024,45 +1009,19 @@
#track of how many are used with np initially 0
self.nocts += 1
cdef Oct *my_oct = <Oct*> malloc(sizeof(Oct))
- cdef ParticleArrays *sd = <ParticleArrays*> \
- malloc(sizeof(ParticleArrays))
cdef int i, j, k
- my_oct.file_ind = my_oct.domain = -1
+ my_oct.domain = -1
+ my_oct.file_ind = 0
my_oct.domain_ind = self.nocts - 1
my_oct.pos[0] = my_oct.pos[1] = my_oct.pos[2] = -1
my_oct.level = -1
- my_oct.sd = sd
for i in range(2):
for j in range(2):
for k in range(2):
my_oct.children[i][j][k] = NULL
my_oct.parent = NULL
- if self.first_sd == NULL:
- self.first_sd = sd
- if self.last_sd != NULL:
- self.last_sd.next = sd
- self.last_sd = sd
- sd.oct = my_oct
- sd.next = NULL
- sd.pos = <np.float64_t **> malloc(sizeof(np.float64_t*) * 3)
- for i in range(3):
- sd.pos[i] = <np.float64_t *> malloc(sizeof(np.float64_t) * self.n_ref)
- for i in range(self.n_ref):
- sd.pos[0][i] = sd.pos[1][i] = sd.pos[2][i] = 0.0
- sd.np = 0
return my_oct
- def linearly_count(self):
- #Without visiting oct and cells
- #jump from particle arrays to the next one
- #counting the total # of particles en route
- cdef np.int64_t total = 0
- cdef ParticleArrays *c = self.first_sd
- while c != NULL:
- total += 1
- c = c.next
- return total
-
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
@@ -1083,105 +1042,73 @@
level_count[o.level] += 1
return level_count
- def add(self, np.ndarray[np.float64_t, ndim=2] pos, np.int64_t domain_id):
+ def add(self, np.ndarray[np.uint64_t, ndim=1] indices, np.int64_t domain_id):
#Add this particle to the root oct
#Then if that oct has children, add it to them recursively
#If the child needs to be refined because of max particles, do so
- cdef int no = pos.shape[0]
- cdef int p, i, level
- cdef np.float64_t dds[3], cp[3], pp[3]
- cdef int ind[3]
- self.max_domain = max(self.max_domain, domain_id)
- cdef int mid, mad
+ cdef np.int64_t no = indices.shape[0], p, index
+ cdef int i, level, ind[3]
if self.root_mesh[0][0][0] == NULL: self.allocate_root()
+ cdef np.uint64_t *data = <np.uint64_t *> indices.data
for p in range(no):
+ # We have morton indices, which means we choose left and right by
+ # looking at (MAX_ORDER - level) & with the values 1, 2, 8.
level = 0
+ index = indices[p]
for i in range(3):
- #PP Calculate the unitary position,
- #DDS Domain dimensions
- #IND Corresponding integer index on the root octs
- #CP Center point of that oct
- pp[i] = pos[p, i]
- dds[i] = (self.DRE[i] - self.DLE[i])/self.nn[i]
- ind[i] = <np.int64_t> ((pp[i] - self.DLE[i])/dds[i])
- cp[i] = (ind[i] + 0.5) * dds[i] + self.DLE[i]
+ ind[i] = (index >> ((ORDER_MAX - level)*3 + (2 - i))) & 1
+ assert(ind[i] < self.nn[i])
cur = self.root_mesh[ind[0]][ind[1]][ind[2]]
if cur == NULL:
raise RuntimeError
- if self._check_refine(cur, cp, domain_id) == 1:
- self.refine_oct(cur, cp)
- while cur.sd.np < 0:
- if level > 100:
- raise RuntimeError
+ while cur.file_ind >= self.n_ref:
+ if level >= ORDER_MAX: break # Just dump it here.
+ level += 1
for i in range(3):
- dds[i] = dds[i] / 2.0
- if cp[i] > pp[i]:
- ind[i] = 0
- cp[i] -= dds[i] / 2.0
- else:
- ind[i] = 1
- cp[i] += dds[i]/2.0
+ ind[i] = (index >> ((ORDER_MAX - level)*3 + (2 - i))) & 1
+ if cur.children[ind[0]][ind[1]][ind[2]] == NULL:
+ self.refine_oct(cur, data, p)
cur = cur.children[ind[0]][ind[1]][ind[2]]
- level += 1
- if self._check_refine(cur, cp, domain_id) == 1:
- self.refine_oct(cur, cp)
- # Now we copy in our particle
- cur.level = level
- for i in range(3):
- cur.sd.pos[i][cur.sd.np] = pp[i]
- cur.domain = domain_id
- cur.sd.np += 1
+ cur.file_ind += 1
- cdef int _check_refine(self, Oct *cur, np.float64_t cp[3], int domain_id):
- #Answers: should we refine this oct?
- #False if refined,
- #False if not refined, but doesn't need refinement
- #True if particles need refinement,
- #True if not in domain
- if cur.children[0][0][0] != NULL:
- return 0
- elif cur.sd.np == 0:
- return 0
- elif cur.sd.np >= self.n_ref:
- return 1
- elif cur.domain >= 0 and cur.domain != domain_id:
- return 1
- return 0
-
- cdef void refine_oct(self, Oct *o, np.float64_t pos[3]):
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ cdef void refine_oct(self, Oct *o, np.uint64_t *data, np.int64_t p):
#Allocate and initialize child octs
#Attach particles to child octs
#Remove particles from this oct entirely
cdef int i, j, k, m, ind[3]
cdef Oct *noct
+ cdef np.uint64_t prefix2
for i in range(2):
for j in range(2):
for k in range(2):
noct = self.allocate_oct()
noct.domain = o.domain
+ noct.file_ind = 0
noct.level = o.level + 1
noct.pos[0] = (o.pos[0] << 1) + i
noct.pos[1] = (o.pos[1] << 1) + j
noct.pos[2] = (o.pos[2] << 1) + k
noct.parent = o
o.children[i][j][k] = noct
- for m in range(o.sd.np):
- for i in range(3):
- if o.sd.pos[i][m] < pos[i]:
- ind[i] = 0
- else:
- ind[i] = 1
- noct = o.children[ind[0]][ind[1]][ind[2]]
- k = noct.sd.np
- for i in range(3):
- noct.sd.pos[i][k] = o.sd.pos[i][m]
- noct.domain = o.domain
- noct.sd.np += 1
- o.sd.np = -1
- o.domain = -1
+ o.file_ind = self.n_ref + 1
for i in range(3):
- free(o.sd.pos[i])
- free(o.sd.pos)
+ ind[i] = (data[p] >> ((ORDER_MAX - (o.level + 1))*3 + (2 - i))) & 1
+ noct = o.children[ind[0]][ind[1]][ind[2]]
+ # Now we look at the last nref particles to decide where they go.
+ n = imin(p, self.n_ref)
+ cdef np.uint64_t *arr = data + imax(p - self.n_ref, 0)
+ # Now we figure out our prefix, which is the oct address at this level.
+ # As long as we're actually in Morton order, we do not need to worry
+ # about *any* of the other children of the oct.
+ prefix1 = data[p] >> (ORDER_MAX - noct.level)*3
+ for i in range(n):
+ prefix2 = arr[i] >> (ORDER_MAX - noct.level)*3
+ if (prefix1 == prefix2):
+ noct.file_ind += 1
+ #print ind[0], ind[1], ind[2], noct.file_ind, noct.level
def recursively_count(self):
#Visit every cell, accumulate the # of cells per level
@@ -1220,7 +1147,7 @@
for oi in range(self.nocts):
m = 0
o = self.oct_list[oi]
- if o.sd.np <= 0 or o.domain == -1: continue
+ #if o.sd.np <= 0 or o.domain == -1: continue
for i in range(8):
if mask[oi, i] == 1:
m = 1
@@ -1232,24 +1159,6 @@
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def count_neighbor_particles(self, oppos):
- #How many particles are in my neighborhood
- cdef int i, ni, dl, tnp
- cdef np.float64_t ppos[3]
- for i in range(3):
- ppos[i] = oppos[i]
- cdef Oct *main = self.get(ppos)
- cdef Oct* neighbors[27]
- self.neighbors(main, neighbors)
- tnp = 0
- for i in range(27):
- if neighbors[i].sd != NULL:
- tnp += neighbors[i].sd.np
- return tnp
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
def count_cells(self, SelectorObject selector,
np.ndarray[np.uint8_t, ndim=2, cast=True] mask):
#Count how many cells per level there are
https://bitbucket.org/yt_analysis/yt-3.0/commits/e77db2f9573c/
Changeset: e77db2f9573c
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-12 23:33:09
Summary: Adding function to help with testing parallelism strategies.
Affected #: 1 file
diff -r 67ee5eb297349e6e874cac0db007020dfa1aab08 -r e77db2f9573cce9efe444e0d94a61190e7e769af yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -859,6 +859,23 @@
self.visit_free(o.children[i][j][k])
free(o)
+ def clear_fileind(self):
+ cdef i, j, k
+ for i in range(self.nn[0]):
+ for j in range(self.nn[1]):
+ for k in range(self.nn[2]):
+ self.visit_clear(self.root_mesh[i][j][k])
+
+ cdef void visit_clear(self, Oct *o):
+ #Free the memory for this oct recursively
+ cdef int i, j, k
+ o.file_ind = 0
+ for i in range(2):
+ for j in range(2):
+ for k in range(2):
+ if o.children[i][j][k] == NULL: continue
+ self.visit_clear(o.children[i][j][k])
+
def __iter__(self):
#Get the next oct, will traverse domains
#Note that oct containers can be sorted
https://bitbucket.org/yt_analysis/yt-3.0/commits/528783b895c8/
Changeset: 528783b895c8
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-13 03:24:41
Summary: Minor optimizations.
Affected #: 1 file
diff -r e77db2f9573cce9efe444e0d94a61190e7e769af -r 528783b895c83b00ae9a515722e4a5c2edfef15d yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -1059,6 +1059,9 @@
level_count[o.level] += 1
return level_count
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
def add(self, np.ndarray[np.uint64_t, ndim=1] indices, np.int64_t domain_id):
#Add this particle to the root oct
#Then if that oct has children, add it to them recursively
@@ -1069,12 +1072,11 @@
cdef np.uint64_t *data = <np.uint64_t *> indices.data
for p in range(no):
# We have morton indices, which means we choose left and right by
- # looking at (MAX_ORDER - level) & with the values 1, 2, 8.
+ # looking at (MAX_ORDER - level) & with the values 1, 2, 4.
level = 0
index = indices[p]
for i in range(3):
ind[i] = (index >> ((ORDER_MAX - level)*3 + (2 - i))) & 1
- assert(ind[i] < self.nn[i])
cur = self.root_mesh[ind[0]][ind[1]][ind[2]]
if cur == NULL:
raise RuntimeError
@@ -1095,9 +1097,9 @@
#Allocate and initialize child octs
#Attach particles to child octs
#Remove particles from this oct entirely
- cdef int i, j, k, m, ind[3]
+ cdef int i, j, k, m, n, ind[3]
cdef Oct *noct
- cdef np.uint64_t prefix2
+ cdef np.uint64_t prefix1, prefix2
for i in range(2):
for j in range(2):
for k in range(2):
https://bitbucket.org/yt_analysis/yt-3.0/commits/960000d1afa9/
Changeset: 960000d1afa9
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-13 03:55:04
Summary: Resetting particle filtering lets this now work with NREF buffer particles.
Affected #: 1 file
diff -r 528783b895c83b00ae9a515722e4a5c2edfef15d -r 960000d1afa968b92400bb198f05d851416e8692 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -1080,20 +1080,26 @@
cur = self.root_mesh[ind[0]][ind[1]][ind[2]]
if cur == NULL:
raise RuntimeError
- while cur.file_ind >= self.n_ref:
+ while (cur.file_ind + 1) > self.n_ref:
if level >= ORDER_MAX: break # Just dump it here.
level += 1
for i in range(3):
ind[i] = (index >> ((ORDER_MAX - level)*3 + (2 - i))) & 1
if cur.children[ind[0]][ind[1]][ind[2]] == NULL:
- self.refine_oct(cur, data, p)
- cur = cur.children[ind[0]][ind[1]][ind[2]]
+ cur = self.refine_oct(cur, index)
+ self.filter_particles(cur, data, p)
+ else:
+ cur = cur.children[ind[0]][ind[1]][ind[2]]
+ if domain_id > 0:
+ cur.file_ind = 0
+ self.filter_particles(cur, data, p)
+ if p >= self.n_ref: domain_id = 0
cur.file_ind += 1
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- cdef void refine_oct(self, Oct *o, np.uint64_t *data, np.int64_t p):
+ cdef Oct *refine_oct(self, Oct *o, np.uint64_t index):
#Allocate and initialize child octs
#Attach particles to child octs
#Remove particles from this oct entirely
@@ -1114,20 +1120,23 @@
o.children[i][j][k] = noct
o.file_ind = self.n_ref + 1
for i in range(3):
- ind[i] = (data[p] >> ((ORDER_MAX - (o.level + 1))*3 + (2 - i))) & 1
+ ind[i] = (index >> ((ORDER_MAX - (o.level + 1))*3 + (2 - i))) & 1
noct = o.children[ind[0]][ind[1]][ind[2]]
+ return noct
+
+ cdef void filter_particles(self, Oct *o, np.uint64_t *data, np.int64_t p):
# Now we look at the last nref particles to decide where they go.
- n = imin(p, self.n_ref)
+ cdef int n = imin(p, self.n_ref)
cdef np.uint64_t *arr = data + imax(p - self.n_ref, 0)
# Now we figure out our prefix, which is the oct address at this level.
# As long as we're actually in Morton order, we do not need to worry
# about *any* of the other children of the oct.
- prefix1 = data[p] >> (ORDER_MAX - noct.level)*3
+ prefix1 = data[p] >> (ORDER_MAX - o.level)*3
for i in range(n):
- prefix2 = arr[i] >> (ORDER_MAX - noct.level)*3
+ prefix2 = arr[i] >> (ORDER_MAX - o.level)*3
if (prefix1 == prefix2):
- noct.file_ind += 1
- #print ind[0], ind[1], ind[2], noct.file_ind, noct.level
+ o.file_ind += 1
+ #print ind[0], ind[1], ind[2], o.file_ind, o.level
def recursively_count(self):
#Visit every cell, accumulate the # of cells per level
https://bitbucket.org/yt_analysis/yt-3.0/commits/8f9c4f153c91/
Changeset: 8f9c4f153c91
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-13 22:55:06
Summary: First set of steps to rework particle handlers.
This has begun the process of moving away from Octree subsets.
Affected #: 5 files
diff -r 960000d1afa968b92400bb198f05d851416e8692 -r 8f9c4f153c91be9d4e4291d73ed8f9fe798b161a yt/frontends/sph/data_structures.py
--- a/yt/frontends/sph/data_structures.py
+++ b/yt/frontends/sph/data_structures.py
@@ -32,10 +32,8 @@
from yt.utilities.fortran_utils import read_record
from yt.funcs import *
-from yt.geometry.oct_geometry_handler import \
- OctreeGeometryHandler
-from yt.geometry.oct_container import \
- ParticleOctreeContainer
+from yt.geometry.particle_geometry_handler import \
+ ParticleGeometryHandler
from yt.geometry.geometry_handler import \
GeometryHandler, YTDataChunk
from yt.data_objects.static_output import \
@@ -61,12 +59,12 @@
from yt.data_objects.field_info_container import \
FieldInfoContainer, NullFunc
-class ParticleDomainFile(object):
- def __init__(self, pf, io, domain_filename, domain_id):
+class ParticleFile(object):
+ def __init__(self, pf, io, filename, file_id):
self.pf = pf
self.io = weakref.proxy(io)
- self.domain_filename = domain_filename
- self.domain_id = domain_id
+ self.filename = filename
+ self.file_id = file_id
self.total_particles = self.io._count_particles(self)
def select(self, selector):
@@ -78,99 +76,14 @@
def _calculate_offsets(self, fields):
pass
-class ParticleDomainSubset(OctreeSubset):
- pass
-
-class ParticleGeometryHandler(OctreeGeometryHandler):
-
- def __init__(self, pf, data_style):
- self.data_style = data_style
- self.parameter_file = weakref.proxy(pf)
- # for now, the hierarchy file is the parameter file!
- self.hierarchy_filename = self.parameter_file.parameter_filename
- self.directory = os.path.dirname(self.hierarchy_filename)
- self.float_type = np.float64
- super(ParticleGeometryHandler, self).__init__(pf, data_style)
-
- def _initialize_oct_handler(self):
- self._setup_data_io()
- template = self.parameter_file.domain_template
- ndoms = self.parameter_file.domain_count
- cls = self.parameter_file._domain_class
- self.domains = [cls(self.parameter_file, self.io, template % {'num':i}, i)
- for i in range(ndoms)]
- total_particles = sum(sum(d.total_particles.values())
- for d in self.domains)
- self.oct_handler = ParticleOctreeContainer(
- self.parameter_file.domain_dimensions/2,
- self.parameter_file.domain_left_edge,
- self.parameter_file.domain_right_edge)
- self.oct_handler.n_ref = 64
- mylog.info("Allocating for %0.3e particles", total_particles)
- for dom in self.domains:
- self.io._initialize_octree(dom, self.oct_handler)
- self.oct_handler.finalize()
- self.max_level = self.oct_handler.max_level
- tot = self.oct_handler.linearly_count()
- mylog.info("Identified %0.3e octs", tot)
-
- def _detect_fields(self):
- # TODO: Add additional fields
- pfl = []
- for dom in self.domains:
- fl = self.io._identify_fields(dom)
- dom._calculate_offsets(fl)
- for f in fl:
- if f not in pfl: pfl.append(f)
- self.field_list = pfl
- pf = self.parameter_file
- pf.particle_types = tuple(set(pt for pt, pf in pfl))
- pf.particle_types += ('all',)
-
- def _setup_classes(self):
- dd = self._get_data_reader_dict()
- super(ParticleGeometryHandler, self)._setup_classes(dd)
- self.object_types.sort()
-
- def _identify_base_chunk(self, dobj):
- if getattr(dobj, "_chunk_info", None) is None:
- mask = dobj.selector.select_octs(self.oct_handler)
- counts = self.oct_handler.count_cells(dobj.selector, mask)
- subsets = [ParticleDomainSubset(d, mask, c)
- for d, c in zip(self.domains, counts) if c > 0]
- dobj._chunk_info = subsets
- dobj.size = sum(counts)
- dobj.shape = (dobj.size,)
- dobj._current_chunk = list(self._chunk_all(dobj))[0]
-
- def _chunk_all(self, dobj):
- oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
- yield YTDataChunk(dobj, "all", oobjs, dobj.size)
-
- def _chunk_spatial(self, dobj, ngz, sort = None):
- sobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
- for i,og in enumerate(sobjs):
- if ngz > 0:
- g = og.retrieve_ghost_zones(ngz, [], smoothed=True)
- else:
- g = og
- size = og.cell_count
- if size == 0: continue
- yield YTDataChunk(dobj, "spatial", [g], size)
-
- def _chunk_io(self, dobj):
- oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
- for subset in oobjs:
- yield YTDataChunk(dobj, "io", [subset], subset.cell_count)
-
-class GadgetBinaryDomainFile(ParticleDomainFile):
- def __init__(self, pf, io, domain_filename, domain_id):
- with open(domain_filename, "rb") as f:
+class GadgetBinaryFile(ParticleFile):
+ def __init__(self, pf, io, filename, file_id):
+ with open(filename, "rb") as f:
self.header = read_record(f, pf._header_spec)
self._position_offset = f.tell()
- super(GadgetBinaryDomainFile, self).__init__(pf, io,
- domain_filename, domain_id)
+ super(GadgetBinaryFile, self).__init__(pf, io,
+ filename, file_id)
def _calculate_offsets(self, field_list):
self.field_offsets = self.io._calculate_field_offsets(
@@ -210,7 +123,7 @@
class GadgetStaticOutput(ParticleStaticOutput):
_hierarchy_class = ParticleGeometryHandler
- _domain_class = GadgetBinaryDomainFile
+ _file_class = GadgetBinaryFile
_fieldinfo_fallback = GadgetFieldInfo
_fieldinfo_known = KnownGadgetFields
_header_spec = (('Npart', 6, 'i'),
@@ -235,7 +148,6 @@
additional_fields = (), root_dimensions = 64,
unit_base = None):
self._root_dimensions = root_dimensions
- # Set up the template for domain files
self.storage_filename = None
if unit_base is not None and "UnitLength_in_cm" in unit_base:
# We assume this is comoving, because in the absence of comoving
@@ -306,11 +218,11 @@
suffix = self.parameter_filename.rsplit(".", 1)[-1]
if hvals["NumFiles"] > 1:
- self.domain_template = "%s.%%(num)s" % (prefix)
+ self.filename_template = "%s.%%(num)s" % (prefix)
else:
- self.domain_template = self.parameter_filename
+ self.filename_template = self.parameter_filename
- self.domain_count = hvals["NumFiles"]
+ self.file_count = hvals["NumFiles"]
f.close()
@@ -341,14 +253,13 @@
class OWLSStaticOutput(GadgetStaticOutput):
_hierarchy_class = ParticleGeometryHandler
- _domain_class = ParticleDomainFile
+ _file_class = ParticleFile
_fieldinfo_fallback = OWLSFieldInfo # For now we have separate from Gadget
_fieldinfo_known = KnownOWLSFields
_header_spec = None # Override so that there's no confusion
def __init__(self, filename, data_style="OWLS", root_dimensions = 64):
self._root_dimensions = root_dimensions
- # Set up the template for domain files
self.storage_filename = None
super(OWLSStaticOutput, self).__init__(filename, data_style,
root_dimensions,
@@ -383,8 +294,8 @@
prefix = self.parameter_filename.split(".", 1)[0]
suffix = self.parameter_filename.rsplit(".", 1)[-1]
- self.domain_template = "%s.%%(num)i.%s" % (prefix, suffix)
- self.domain_count = hvals["NumFilesPerSnapshot"]
+ self.filename_template = "%s.%%(num)i.%s" % (prefix, suffix)
+ self.file_count = hvals["NumFilesPerSnapshot"]
# To avoid having to open files twice
self._unit_base = {}
@@ -407,23 +318,23 @@
pass
return False
-class TipsyDomainFile(ParticleDomainFile):
+class TipsyFile(ParticleFile):
def _calculate_offsets(self, field_list):
self.field_offsets = self.io._calculate_particle_offsets(self)
- def __init__(self, pf, io, domain_filename, domain_id):
+ def __init__(self, pf, io, filename, file_id):
# To go above 1 domain, we need to include an indexing step in the
# IOHandler, rather than simply reading from a single file.
- assert domain_id == 0
- super(TipsyDomainFile, self).__init__(pf, io,
- domain_filename, domain_id)
+ assert file_id == 0
+ super(TipsyFile, self).__init__(pf, io,
+ filename, file_id)
io._create_dtypes(self)
class TipsyStaticOutput(ParticleStaticOutput):
_hierarchy_class = ParticleGeometryHandler
- _domain_class = TipsyDomainFile
+ _file_class = TipsyFile
_fieldinfo_fallback = TipsyFieldInfo
_fieldinfo_known = KnownTipsyFields
_header_spec = (('time', 'd'),
@@ -443,7 +354,6 @@
cosmology_parameters = None):
self.endian = endian
self._root_dimensions = root_dimensions
- # Set up the template for domain files
self.storage_filename = None
if domain_left_edge is None:
domain_left_edge = np.zeros(3, "float64") - 0.5
@@ -506,8 +416,8 @@
self.parameters = hvals
- self.domain_template = self.parameter_filename
- self.domain_count = 1
+ self.filename_template = self.parameter_filename
+ self.file_count = 1
f.close()
diff -r 960000d1afa968b92400bb198f05d851416e8692 -r 8f9c4f153c91be9d4e4291d73ed8f9fe798b161a yt/frontends/sph/io.py
--- a/yt/frontends/sph/io.py
+++ b/yt/frontends/sph/io.py
@@ -32,6 +32,9 @@
BaseIOHandler
from yt.utilities.fortran_utils import read_record
+from yt.utilities.lib.geometry_utils import get_morton_indices
+
+from yt.geometry.oct_container import _ORDER_MAX as ORDER_MAX
_vector_fields = ("Coordinates", "Velocity", "Velocities")
@@ -373,34 +376,44 @@
f.close()
return rv
- def _initialize_octree(self, domain, octree):
- pf = domain.pf
- with open(domain.domain_filename, "rb") as f:
- f.seek(domain.pf._header_offset)
+ def _initialize_octree(self, data_file, octree):
+ pf = data_file.pf
+ morton = np.empty(sum(data_file.total_particles.values()),
+ dtype="uint64")
+ ind = 0
+ DLE, DRE = pf.domain_right_edge, pf.domain_left_edge
+ dx = (DRE - DLE) / (2**ORDER_MAX)
+ with open(data_file.filename, "rb") as f:
+ f.seek(pf._header_offset)
for ptype in self._ptypes:
# We'll just add the individual types separately
- count = domain.total_particles[ptype]
+ count = data_file.total_particles[ptype]
if count == 0: continue
pp = np.fromfile(f, dtype = self._pdtypes[ptype],
count = count)
- pos = np.empty((count, 3), dtype="float64")
- mylog.info("Adding %0.3e %s particles", count, ptype)
- pos[:,0] = pp['Coordinates']['x']
- pos[:,1] = pp['Coordinates']['y']
- pos[:,2] = pp['Coordinates']['z']
- mylog.debug("Spanning: %0.3e .. %0.3e in x",
- pos[:,0].min(), pos[:,0].max())
- mylog.debug("Spanning: %0.3e .. %0.3e in y",
- pos[:,1].min(), pos[:,1].max())
- mylog.debug("Spanning: %0.3e .. %0.3e in z",
- pos[:,2].min(), pos[:,2].max())
- if np.any(pos.min(axis=0) < pf.domain_left_edge) or \
- np.any(pos.max(axis=0) > pf.domain_right_edge):
- raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0),
+ mis = np.empty(3, dtype="float64")
+ mas = np.empty(3, dtype="float64")
+ for axi, ax in enumerate('xyz'):
+ mi = pp["Coordinates"][ax].min()
+ ma = pp["Coordinates"][ax].max()
+ mylog.debug("Spanning: %0.3e .. %0.3e in %s", mi, ma, ax)
+ mis[axi] = mi
+ mas[axi] = ma
+ if np.any(mis < pf.domain_left_edge) or \
+ np.any(mas > pf.domain_right_edge):
+ raise YTDomainOverflow(mis, mas,
pf.domain_left_edge,
pf.domain_right_edge)
+ pos = np.empty((count, 3), dtype="uint64")
+ mylog.info("Adding %0.3e %s particles", count, ptype)
+ pos[:,0] = np.floor((pp['Coordinates']['x'] - DLE[0])/dx[0])
+ pos[:,1] = np.floor((pp['Coordinates']['y'] - DLE[1])/dx[1])
+ pos[:,2] = np.floor((pp['Coordinates']['z'] - DLE[2])/dx[2])
del pp
- octree.add(pos, domain.domain_id)
+ morton[ind:ind+count] = get_morton_indices(pos)
+ morton.sort()
+ octree.add(morton, data_file.file_id)
+ print octree.recursively_count()
def _count_particles(self, domain):
npart = {
diff -r 960000d1afa968b92400bb198f05d851416e8692 -r 8f9c4f153c91be9d4e4291d73ed8f9fe798b161a yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -35,6 +35,8 @@
DEF ORDER_MAX=20
+_ORDER_MAX = ORDER_MAX
+
cdef extern from "stdlib.h":
# NOTE that size_t might not be int
void *alloca(int)
diff -r 960000d1afa968b92400bb198f05d851416e8692 -r 8f9c4f153c91be9d4e4291d73ed8f9fe798b161a yt/geometry/particle_geometry_handler.py
--- /dev/null
+++ b/yt/geometry/particle_geometry_handler.py
@@ -0,0 +1,143 @@
+"""
+Particle-only geometry handler
+
+Author: Matthew Turk <matthewturk at gmail.com>
+Affiliation: Columbia University
+Homepage: http://yt-project.org/
+License:
+ Copyright (C) 2012 Matthew Turk. All Rights Reserved.
+
+ This file is part of yt.
+
+ yt is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+
+import h5py
+import numpy as na
+import string, re, gc, time, cPickle
+import weakref
+
+from itertools import chain, izip
+
+from yt.funcs import *
+from yt.utilities.logger import ytLogger as mylog
+from yt.arraytypes import blankRecordArray
+from yt.config import ytcfg
+from yt.data_objects.field_info_container import NullFunc
+from yt.geometry.geometry_handler import GeometryHandler, YTDataChunk
+from yt.geometry.oct_container import \
+ ParticleOctreeContainer
+from yt.utilities.definitions import MAXLEVEL
+from yt.utilities.io_handler import io_registry
+from yt.utilities.parallel_tools.parallel_analysis_interface import \
+ ParallelAnalysisInterface, parallel_splitter
+
+from yt.data_objects.data_containers import data_object_registry
+
+class ParticleGeometryHandler(GeometryHandler):
+
+ def __init__(self, pf, data_style):
+ self.data_style = data_style
+ self.parameter_file = weakref.proxy(pf)
+ # for now, the hierarchy file is the parameter file!
+ self.hierarchy_filename = self.parameter_file.parameter_filename
+ self.directory = os.path.dirname(self.hierarchy_filename)
+ self.float_type = np.float64
+ super(ParticleGeometryHandler, self).__init__(pf, data_style)
+
+ def _setup_geometry(self):
+ mylog.debug("Initializing Particle Geometry Handler.")
+ self._initialize_particle_handler()
+
+
+ def get_smallest_dx(self):
+ """
+ Returns (in code units) the smallest cell size in the simulation.
+ """
+ raise NotImplementedError
+
+ def convert(self, unit):
+ return self.parameter_file.conversion_factors[unit]
+
+ def _initialize_particle_handler(self):
+ self._setup_data_io()
+ template = self.parameter_file.filename_template
+ ndoms = self.parameter_file.file_count
+ cls = self.parameter_file._file_class
+ self.data_files = [cls(self.parameter_file, self.io, template % {'num':i}, i)
+ for i in range(ndoms)]
+ total_particles = sum(sum(d.total_particles.values())
+ for d in self.data_files)
+ self.oct_handler = ParticleOctreeContainer(
+ [1, 1, 1],
+ self.parameter_file.domain_left_edge,
+ self.parameter_file.domain_right_edge)
+ self.oct_handler.n_ref = 64
+ mylog.info("Allocating for %0.3e particles", total_particles)
+ for dom in self.data_files:
+ self.io._initialize_octree(dom, self.oct_handler)
+ self.oct_handler.finalize()
+ self.max_level = self.oct_handler.max_level
+ tot = self.oct_handler.linearly_count()
+ mylog.info("Identified %0.3e octs", tot)
+
+ def _detect_fields(self):
+ # TODO: Add additional fields
+ pfl = []
+ for dom in self.data_files:
+ fl = self.io._identify_fields(dom)
+ dom._calculate_offsets(fl)
+ for f in fl:
+ if f not in pfl: pfl.append(f)
+ self.field_list = pfl
+ pf = self.parameter_file
+ pf.particle_types = tuple(set(pt for pt, pf in pfl))
+ pf.particle_types += ('all',)
+
+ def _setup_classes(self):
+ dd = self._get_data_reader_dict()
+ super(ParticleGeometryHandler, self)._setup_classes(dd)
+ self.object_types.sort()
+
+ def _identify_base_chunk(self, dobj):
+ if getattr(dobj, "_chunk_info", None) is None:
+ mask = dobj.selector.select_octs(self.oct_handler)
+ counts = self.oct_handler.count_cells(dobj.selector, mask)
+ subsets = [ParticleDomainSubset(d, mask, c)
+ for d, c in zip(self.domains, counts) if c > 0]
+ dobj._chunk_info = subsets
+ dobj.size = sum(counts)
+ dobj.shape = (dobj.size,)
+ dobj._current_chunk = list(self._chunk_all(dobj))[0]
+
+ def _chunk_all(self, dobj):
+ oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
+ yield YTDataChunk(dobj, "all", oobjs, dobj.size)
+
+ def _chunk_spatial(self, dobj, ngz, sort = None):
+ sobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
+ for i,og in enumerate(sobjs):
+ if ngz > 0:
+ g = og.retrieve_ghost_zones(ngz, [], smoothed=True)
+ else:
+ g = og
+ size = og.cell_count
+ if size == 0: continue
+ yield YTDataChunk(dobj, "spatial", [g], size)
+
+ def _chunk_io(self, dobj):
+ oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
+ for subset in oobjs:
+ yield YTDataChunk(dobj, "io", [subset], subset.cell_count)
+
diff -r 960000d1afa968b92400bb198f05d851416e8692 -r 8f9c4f153c91be9d4e4291d73ed8f9fe798b161a yt/utilities/lib/geometry_utils.pyx
--- a/yt/utilities/lib/geometry_utils.pyx
+++ b/yt/utilities/lib/geometry_utils.pyx
@@ -323,7 +323,7 @@
@cython.cdivision(True)
@cython.boundscheck(False)
@cython.wraparound(False)
-def get_morton_indices(np.ndarray[np.int64_t, ndim=2] left_index):
+def get_morton_indices(np.ndarray[np.uint64_t, ndim=2] left_index):
cdef np.int64_t i, mi
cdef np.ndarray[np.uint64_t, ndim=1] morton_indices
morton_indices = np.zeros(left_index.shape[0], 'uint64')
https://bitbucket.org/yt_analysis/yt-3.0/commits/61f889ee4959/
Changeset: 61f889ee4959
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-13 23:33:38
Summary: Correct calculation of DLE,DRE and coordinates.
Affected #: 1 file
diff -r 8f9c4f153c91be9d4e4291d73ed8f9fe798b161a -r 61f889ee49592863806236960e27175571c43c78 yt/frontends/sph/io.py
--- a/yt/frontends/sph/io.py
+++ b/yt/frontends/sph/io.py
@@ -381,7 +381,7 @@
morton = np.empty(sum(data_file.total_particles.values()),
dtype="uint64")
ind = 0
- DLE, DRE = pf.domain_right_edge, pf.domain_left_edge
+ DLE, DRE = pf.domain_left_edge, pf.domain_right_edge
dx = (DRE - DLE) / (2**ORDER_MAX)
with open(data_file.filename, "rb") as f:
f.seek(pf._header_offset)
@@ -406,14 +406,14 @@
pf.domain_right_edge)
pos = np.empty((count, 3), dtype="uint64")
mylog.info("Adding %0.3e %s particles", count, ptype)
- pos[:,0] = np.floor((pp['Coordinates']['x'] - DLE[0])/dx[0])
- pos[:,1] = np.floor((pp['Coordinates']['y'] - DLE[1])/dx[1])
- pos[:,2] = np.floor((pp['Coordinates']['z'] - DLE[2])/dx[2])
- del pp
+ for axi, ax in enumerate("xyz"):
+ coords = pp['Coordinates'][ax].astype("float64")
+ coords = np.floor((coords - DLE[axi])/dx[axi])
+ pos[:,axi] = coords
morton[ind:ind+count] = get_morton_indices(pos)
+ del pp, pos
morton.sort()
octree.add(morton, data_file.file_id)
- print octree.recursively_count()
def _count_particles(self, domain):
npart = {
https://bitbucket.org/yt_analysis/yt-3.0/commits/fbed8ce438cc/
Changeset: fbed8ce438cc
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-13 23:41:45
Summary: Continuing to convert Tipsy.
Affected #: 2 files
diff -r 61f889ee49592863806236960e27175571c43c78 -r fbed8ce438cc9bc6a6a56f3017a6500a4b4a5f9e yt/frontends/sph/io.py
--- a/yt/frontends/sph/io.py
+++ b/yt/frontends/sph/io.py
@@ -405,13 +405,13 @@
pf.domain_left_edge,
pf.domain_right_edge)
pos = np.empty((count, 3), dtype="uint64")
- mylog.info("Adding %0.3e %s particles", count, ptype)
for axi, ax in enumerate("xyz"):
coords = pp['Coordinates'][ax].astype("float64")
coords = np.floor((coords - DLE[axi])/dx[axi])
pos[:,axi] = coords
morton[ind:ind+count] = get_morton_indices(pos)
del pp, pos
+ mylog.info("Adding %0.3e particles", morton.size)
morton.sort()
octree.add(morton, data_file.file_id)
diff -r 61f889ee49592863806236960e27175571c43c78 -r fbed8ce438cc9bc6a6a56f3017a6500a4b4a5f9e yt/geometry/particle_geometry_handler.py
--- a/yt/geometry/particle_geometry_handler.py
+++ b/yt/geometry/particle_geometry_handler.py
@@ -89,7 +89,7 @@
self.io._initialize_octree(dom, self.oct_handler)
self.oct_handler.finalize()
self.max_level = self.oct_handler.max_level
- tot = self.oct_handler.linearly_count()
+ tot = sum(self.oct_handler.recursively_count().values())
mylog.info("Identified %0.3e octs", tot)
def _detect_fields(self):
https://bitbucket.org/yt_analysis/yt-3.0/commits/8dcd631cef53/
Changeset: 8dcd631cef53
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-14 00:30:37
Summary: Initial implementation of coarse particle region identifier.
Affected #: 2 files
diff -r fbed8ce438cc9bc6a6a56f3017a6500a4b4a5f9e -r 8dcd631cef53f6e80852d4668adda1d859c5647f yt/frontends/sph/io.py
--- a/yt/frontends/sph/io.py
+++ b/yt/frontends/sph/io.py
@@ -34,7 +34,7 @@
from yt.utilities.fortran_utils import read_record
from yt.utilities.lib.geometry_utils import get_morton_indices
-from yt.geometry.oct_container import _ORDER_MAX as ORDER_MAX
+from yt.geometry.oct_container import _ORDER_MAX, ParticleRegions
_vector_fields = ("Coordinates", "Velocity", "Velocities")
@@ -382,10 +382,13 @@
dtype="uint64")
ind = 0
DLE, DRE = pf.domain_left_edge, pf.domain_right_edge
- dx = (DRE - DLE) / (2**ORDER_MAX)
+ dx = (DRE - DLE) / (2**_ORDER_MAX)
+ self.regions = ParticleRegions(
+ pf.domain_left_edge, pf.domain_right_edge,
+ [64, 64, 64], len(self._ptypes))
with open(data_file.filename, "rb") as f:
f.seek(pf._header_offset)
- for ptype in self._ptypes:
+ for iptype, ptype in enumerate(self._ptypes):
# We'll just add the individual types separately
count = data_file.total_particles[ptype]
if count == 0: continue
@@ -404,6 +407,12 @@
raise YTDomainOverflow(mis, mas,
pf.domain_left_edge,
pf.domain_right_edge)
+ fpos = np.empty((count, 3), dtype="float64")
+ fpos[:,0] = pp["Coordinates"]["x"]
+ fpos[:,1] = pp["Coordinates"]["y"]
+ fpos[:,2] = pp["Coordinates"]["z"]
+ self.regions.add_data_file(fpos, iptype)
+ del fpos
pos = np.empty((count, 3), dtype="uint64")
for axi, ax in enumerate("xyz"):
coords = pp['Coordinates'][ax].astype("float64")
diff -r fbed8ce438cc9bc6a6a56f3017a6500a4b4a5f9e -r 8dcd631cef53f6e80852d4668adda1d859c5647f yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -1286,3 +1286,40 @@
ind[oi] = nm
nm += use
return ind
+
+cdef class ParticleRegions:
+ cdef np.float64_t left_edge[3]
+ cdef np.float64_t dds[3]
+ cdef np.float64_t idds[3]
+ cdef np.int32_t dims[3]
+ cdef public int nfiles
+ cdef public object masks
+
+ def __init__(self, left_edge, right_edge, dims, nfiles):
+ cdef int i
+ self.nfiles = nfiles
+ for i in range(3):
+ self.left_edge[i] = left_edge[i]
+ self.dims[i] = dims[i]
+ self.dds[i] = (right_edge[i] - left_edge[i])/dims[i]
+ self.idds[i] = 1.0/self.dds[i]
+ # We use 64-bit masks
+ self.masks = []
+ for i in range(nfiles/64 + 1):
+ self.masks.append(np.zeros(dims, dtype="uint64"))
+
+ def add_data_file(self, np.ndarray[np.float64_t, ndim=2] pos, int file_id):
+ cdef np.int64_t no = pos.shape[0]
+ cdef np.int64_t p
+ cdef int ind[3], i
+ cdef np.ndarray[np.uint64_t, ndim=3] mask
+ mask = self.masks[file_id/64]
+ val = 1 << (file_id - (file_id/64)*64)
+ for p in range(no):
+ # Now we locate the particle
+ for i in range(3):
+ ind[i] = <int> ((pos[p, i] - self.left_edge[i])*self.idds[i])
+ mask[ind[0],ind[1],ind[2]] |= val
+
+ def identify_data_files(self, SelectorObject selector):
+ pass
https://bitbucket.org/yt_analysis/yt-3.0/commits/b26e0a320e6f/
Changeset: b26e0a320e6f
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-14 03:20:57
Summary: Extending ParticleRegions. Now lives on the geometry handler and can identify
coarse data file inclusion.
Affected #: 3 files
diff -r 8dcd631cef53f6e80852d4668adda1d859c5647f -r b26e0a320e6ff4a019f4447fb16ba034c0d5e91a yt/frontends/sph/io.py
--- a/yt/frontends/sph/io.py
+++ b/yt/frontends/sph/io.py
@@ -92,7 +92,7 @@
f.close()
return rv
- def _initialize_octree(self, domain, octree):
+ def _initialize_index(self, domain, octree):
f = h5py.File(domain.domain_filename, "r")
for key in f.keys():
if not key.startswith("PartType"): continue
@@ -236,7 +236,7 @@
arr = arr.reshape((count/3, 3), order="C")
return arr.astype("float64")
- def _initialize_octree(self, domain, octree):
+ def _initialize_index(self, domain, octree):
count = sum(domain.total_particles.values())
dt = [("px", "float32"), ("py", "float32"), ("pz", "float32")]
with open(domain.domain_filename, "rb") as f:
@@ -376,16 +376,13 @@
f.close()
return rv
- def _initialize_octree(self, data_file, octree):
+ def _initialize_index(self, data_file, octree, regions):
pf = data_file.pf
morton = np.empty(sum(data_file.total_particles.values()),
dtype="uint64")
ind = 0
DLE, DRE = pf.domain_left_edge, pf.domain_right_edge
dx = (DRE - DLE) / (2**_ORDER_MAX)
- self.regions = ParticleRegions(
- pf.domain_left_edge, pf.domain_right_edge,
- [64, 64, 64], len(self._ptypes))
with open(data_file.filename, "rb") as f:
f.seek(pf._header_offset)
for iptype, ptype in enumerate(self._ptypes):
@@ -411,7 +408,7 @@
fpos[:,0] = pp["Coordinates"]["x"]
fpos[:,1] = pp["Coordinates"]["y"]
fpos[:,2] = pp["Coordinates"]["z"]
- self.regions.add_data_file(fpos, iptype)
+ regions.add_data_file(fpos, data_file.file_id)
del fpos
pos = np.empty((count, 3), dtype="uint64")
for axi, ax in enumerate("xyz"):
diff -r 8dcd631cef53f6e80852d4668adda1d859c5647f -r b26e0a320e6ff4a019f4447fb16ba034c0d5e91a yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -1322,4 +1322,34 @@
mask[ind[0],ind[1],ind[2]] |= val
def identify_data_files(self, SelectorObject selector):
- pass
+ # This is relatively cheap to iterate over.
+ cdef int i, j, k, n
+ cdef np.uint64_t fmask, offset
+ cdef np.float64_t LE[3], RE[3]
+ cdef np.ndarray[np.uint64_t, ndim=3] mask
+ files = []
+ for n in range(len(self.masks)):
+ fmask = 0
+ mask = self.masks[n]
+ LE[0] = self.left_edge[0]
+ RE[0] = LE[0] + self.dds[0]
+ for i in range(self.dims[0]):
+ LE[1] = self.left_edge[1]
+ RE[1] = LE[1] + self.dds[1]
+ for j in range(self.dims[1]):
+ LE[2] = self.left_edge[2]
+ RE[2] = LE[2] + self.dds[2]
+ for k in range(self.dims[2]):
+ if selector.select_grid(LE, RE, 0) == 0: continue
+ fmask |= mask[i,j,k]
+ LE[2] += self.dds[2]
+ RE[2] += self.dds[2]
+ LE[1] += self.dds[1]
+ RE[1] += self.dds[1]
+ LE[0] += self.dds[0]
+ RE[0] += self.dds[0]
+ # Now we iterate through...
+ for i in range(64):
+ if ((fmask >> i) & 1) == 1:
+ files.append(i + n * 64)
+ return files
diff -r 8dcd631cef53f6e80852d4668adda1d859c5647f -r b26e0a320e6ff4a019f4447fb16ba034c0d5e91a yt/geometry/particle_geometry_handler.py
--- a/yt/geometry/particle_geometry_handler.py
+++ b/yt/geometry/particle_geometry_handler.py
@@ -37,7 +37,7 @@
from yt.data_objects.field_info_container import NullFunc
from yt.geometry.geometry_handler import GeometryHandler, YTDataChunk
from yt.geometry.oct_container import \
- ParticleOctreeContainer
+ ParticleOctreeContainer, ParticleRegions
from yt.utilities.definitions import MAXLEVEL
from yt.utilities.io_handler import io_registry
from yt.utilities.parallel_tools.parallel_analysis_interface import \
@@ -79,14 +79,17 @@
for i in range(ndoms)]
total_particles = sum(sum(d.total_particles.values())
for d in self.data_files)
+ pf = self.parameter_file
self.oct_handler = ParticleOctreeContainer(
- [1, 1, 1],
- self.parameter_file.domain_left_edge,
- self.parameter_file.domain_right_edge)
+ [1, 1, 1], pf.domain_left_edge, pf.domain_right_edge)
self.oct_handler.n_ref = 64
mylog.info("Allocating for %0.3e particles", total_particles)
+ N = len(self.data_files)
+ self.regions = ParticleRegions(
+ pf.domain_left_edge, pf.domain_right_edge,
+ [N, N, N], N)
for dom in self.data_files:
- self.io._initialize_octree(dom, self.oct_handler)
+ self.io._initialize_index(dom, self.oct_handler, self.regions)
self.oct_handler.finalize()
self.max_level = self.oct_handler.max_level
tot = sum(self.oct_handler.recursively_count().values())
@@ -113,12 +116,10 @@
def _identify_base_chunk(self, dobj):
if getattr(dobj, "_chunk_info", None) is None:
mask = dobj.selector.select_octs(self.oct_handler)
- counts = self.oct_handler.count_cells(dobj.selector, mask)
- subsets = [ParticleDomainSubset(d, mask, c)
- for d, c in zip(self.domains, counts) if c > 0]
- dobj._chunk_info = subsets
- dobj.size = sum(counts)
- dobj.shape = (dobj.size,)
+ file_ids = self.regions.identify_data_files(dobj.selector)
+ dobj._chunk_info = [self.data_files[i] for i in file_ids]
+ #dobj.size = sum(counts)
+ #dobj.shape = (dobj.size,)
dobj._current_chunk = list(self._chunk_all(dobj))[0]
def _chunk_all(self, dobj):
https://bitbucket.org/yt_analysis/yt-3.0/commits/d6198c6c1a9a/
Changeset: d6198c6c1a9a
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-14 03:32:16
Summary: Starting to convert the OWLS format, which will test multi-file support.
Affected #: 1 file
diff -r b26e0a320e6ff4a019f4447fb16ba034c0d5e91a -r d6198c6c1a9a79e05acece5a7f4ae12d45b15a66 yt/frontends/sph/io.py
--- a/yt/frontends/sph/io.py
+++ b/yt/frontends/sph/io.py
@@ -92,23 +92,33 @@
f.close()
return rv
- def _initialize_index(self, domain, octree):
- f = h5py.File(domain.domain_filename, "r")
+ def _initialize_index(self, data_file, octree, regions):
+ f = h5py.File(data_file.filename, "r")
+ pcount = f["/Header"].attrs["NumPart_ThisFile"][:].sum()
+ morton = np.empty(pcount, dtype='uint64')
+ DLE = data_file.pf.domain_left_edge
+ DRE = data_file.pf.domain_right_edge
+ dx = (DRE - DLE) / 2**_ORDER_MAX
+ ind = 0
for key in f.keys():
if not key.startswith("PartType"): continue
pos = f[key]["Coordinates"][:].astype("float64")
- octree.add(pos, domain.domain_id)
+ regions.add_data_file(pos, data_file.file_id)
+ pos = np.floor((pos - DLE)/dx).astype("uint64")
+ morton[ind:ind+pos.shape[0]] = get_morton_indices(pos)
f.close()
+ morton.sort()
+ #octree.add(morton, data_file.file_id)
- def _count_particles(self, domain):
- f = h5py.File(domain.domain_filename, "r")
- np = f["/Header"].attrs["NumPart_ThisFile"][:]
+ def _count_particles(self, data_file):
+ f = h5py.File(data_file.filename, "r")
+ pcount = f["/Header"].attrs["NumPart_ThisFile"][:]
f.close()
- npart = dict(("PartType%s" % (i), v) for i, v in enumerate(np))
+ npart = dict(("PartType%s" % (i), v) for i, v in enumerate(pcount))
return npart
- def _identify_fields(self, domain):
- f = h5py.File(domain.domain_filename, "r")
+ def _identify_fields(self, data_file):
+ f = h5py.File(data_file.filename, "r")
fields = []
for key in f.keys():
if not key.startswith("PartType"): continue
https://bitbucket.org/yt_analysis/yt-3.0/commits/bd419ea73d70/
Changeset: bd419ea73d70
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-14 03:40:07
Summary: Converted some OWLS, Tipsy and OWLS now generate regions correctly.
Affected #: 3 files
diff -r d6198c6c1a9a79e05acece5a7f4ae12d45b15a66 -r bd419ea73d701a908c3e4583afff894f8da568f7 yt/frontends/sph/io.py
--- a/yt/frontends/sph/io.py
+++ b/yt/frontends/sph/io.py
@@ -92,7 +92,7 @@
f.close()
return rv
- def _initialize_index(self, data_file, octree, regions):
+ def _initialize_index(self, data_file, regions):
f = h5py.File(data_file.filename, "r")
pcount = f["/Header"].attrs["NumPart_ThisFile"][:].sum()
morton = np.empty(pcount, dtype='uint64')
@@ -107,8 +107,7 @@
pos = np.floor((pos - DLE)/dx).astype("uint64")
morton[ind:ind+pos.shape[0]] = get_morton_indices(pos)
f.close()
- morton.sort()
- #octree.add(morton, data_file.file_id)
+ return morton
def _count_particles(self, data_file):
f = h5py.File(data_file.filename, "r")
@@ -386,7 +385,7 @@
f.close()
return rv
- def _initialize_index(self, data_file, octree, regions):
+ def _initialize_index(self, data_file, regions):
pf = data_file.pf
morton = np.empty(sum(data_file.total_particles.values()),
dtype="uint64")
@@ -428,8 +427,7 @@
morton[ind:ind+count] = get_morton_indices(pos)
del pp, pos
mylog.info("Adding %0.3e particles", morton.size)
- morton.sort()
- octree.add(morton, data_file.file_id)
+ return morton
def _count_particles(self, domain):
npart = {
diff -r d6198c6c1a9a79e05acece5a7f4ae12d45b15a66 -r bd419ea73d701a908c3e4583afff894f8da568f7 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -1064,7 +1064,7 @@
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def add(self, np.ndarray[np.uint64_t, ndim=1] indices, np.int64_t domain_id):
+ def add(self, np.ndarray[np.uint64_t, ndim=1] indices):
#Add this particle to the root oct
#Then if that oct has children, add it to them recursively
#If the child needs to be refined because of max particles, do so
@@ -1092,10 +1092,6 @@
self.filter_particles(cur, data, p)
else:
cur = cur.children[ind[0]][ind[1]][ind[2]]
- if domain_id > 0:
- cur.file_ind = 0
- self.filter_particles(cur, data, p)
- if p >= self.n_ref: domain_id = 0
cur.file_ind += 1
@cython.boundscheck(False)
diff -r d6198c6c1a9a79e05acece5a7f4ae12d45b15a66 -r bd419ea73d701a908c3e4583afff894f8da568f7 yt/geometry/particle_geometry_handler.py
--- a/yt/geometry/particle_geometry_handler.py
+++ b/yt/geometry/particle_geometry_handler.py
@@ -77,24 +77,45 @@
cls = self.parameter_file._file_class
self.data_files = [cls(self.parameter_file, self.io, template % {'num':i}, i)
for i in range(ndoms)]
- total_particles = sum(sum(d.total_particles.values())
- for d in self.data_files)
+ self.total_particles = sum(
+ sum(d.total_particles.values()) for d in self.data_files)
pf = self.parameter_file
self.oct_handler = ParticleOctreeContainer(
[1, 1, 1], pf.domain_left_edge, pf.domain_right_edge)
self.oct_handler.n_ref = 64
- mylog.info("Allocating for %0.3e particles", total_particles)
+ mylog.info("Allocating for %0.3e particles", self.total_particles)
N = len(self.data_files)
self.regions = ParticleRegions(
pf.domain_left_edge, pf.domain_right_edge,
[N, N, N], N)
- for dom in self.data_files:
- self.io._initialize_index(dom, self.oct_handler, self.regions)
+ self._initialize_indices()
self.oct_handler.finalize()
self.max_level = self.oct_handler.max_level
tot = sum(self.oct_handler.recursively_count().values())
mylog.info("Identified %0.3e octs", tot)
+ def _initialize_indices(self):
+ # This will be replaced with a parallel-aware iteration step.
+ # Roughly outlined, what we will do is:
+ # * Generate Morton indices on each set of files that belong to
+ # an individual processor
+ # * Create a global, accumulated histogram
+ # * Cut based on estimated load balancing
+ # * Pass particles to specific processors, along with NREF buffer
+ # * Broadcast back a serialized octree to join
+ #
+ # For now we will do this in serial.
+ morton = np.empty(self.total_particles, dtype="uint64")
+ ind = 0
+ for data_file in self.data_files:
+ npart = sum(data_file.total_particles.values())
+ morton[ind:ind + npart] = \
+ self.io._initialize_index(data_file, self.regions)
+ morton.sort()
+ # Now we add them all at once.
+ self.oct_handler.add(morton)
+
+
def _detect_fields(self):
# TODO: Add additional fields
pfl = []
https://bitbucket.org/yt_analysis/yt-3.0/commits/e011bb3d32d2/
Changeset: e011bb3d32d2
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-14 17:14:07
Summary: Critical error -- need to increment LE/RE regardless of selector.
Affected #: 1 file
diff -r bd419ea73d701a908c3e4583afff894f8da568f7 -r e011bb3d32d21379a4a7bc6c82767eba69263c62 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -1336,8 +1336,8 @@
LE[2] = self.left_edge[2]
RE[2] = LE[2] + self.dds[2]
for k in range(self.dims[2]):
- if selector.select_grid(LE, RE, 0) == 0: continue
- fmask |= mask[i,j,k]
+ if selector.select_grid(LE, RE, 0) == 1:
+ fmask |= mask[i,j,k]
LE[2] += self.dds[2]
RE[2] += self.dds[2]
LE[1] += self.dds[1]
https://bitbucket.org/yt_analysis/yt-3.0/commits/31ed51b8ccb0/
Changeset: 31ed51b8ccb0
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-14 17:25:02
Summary: Region selection works for OWLS.
Affected #: 2 files
diff -r e011bb3d32d21379a4a7bc6c82767eba69263c62 -r 31ed51b8ccb0f32a4f66bc542e48a9a623d8850a yt/frontends/sph/io.py
--- a/yt/frontends/sph/io.py
+++ b/yt/frontends/sph/io.py
@@ -53,8 +53,8 @@
for ftype, fname in fields:
ptf[ftype].append(fname)
for chunk in chunks: # Will be OWLS domains
- for subset in chunk.objs:
- f = h5py.File(subset.domain.domain_filename, "r")
+ for data_file in chunk.objs:
+ f = h5py.File(data_file.filename, "r")
# This double-reads
for ptype, field_list in sorted(ptf.items()):
coords = f["/%s/Coordinates" % ptype][:].astype("float64")
@@ -73,8 +73,8 @@
rv[field] = np.empty(shape, dtype="float64")
ind[field] = 0
for chunk in chunks: # Will be OWLS domains
- for subset in chunk.objs:
- f = h5py.File(subset.domain.domain_filename, "r")
+ for data_file in chunk.objs:
+ f = h5py.File(data_file.filename, "r")
for ptype, field_list in sorted(ptf.items()):
g = f["/%s" % ptype]
coords = g["Coordinates"][:].astype("float64")
diff -r e011bb3d32d21379a4a7bc6c82767eba69263c62 -r 31ed51b8ccb0f32a4f66bc542e48a9a623d8850a yt/geometry/particle_geometry_handler.py
--- a/yt/geometry/particle_geometry_handler.py
+++ b/yt/geometry/particle_geometry_handler.py
@@ -84,10 +84,11 @@
[1, 1, 1], pf.domain_left_edge, pf.domain_right_edge)
self.oct_handler.n_ref = 64
mylog.info("Allocating for %0.3e particles", self.total_particles)
- N = len(self.data_files)
+ # No more than 256^3 in the region finder.
+ N = min(len(self.data_files), 256)
self.regions = ParticleRegions(
pf.domain_left_edge, pf.domain_right_edge,
- [N, N, N], N)
+ [N, N, N], len(self.data_files))
self._initialize_indices()
self.oct_handler.finalize()
self.max_level = self.oct_handler.max_level
@@ -139,8 +140,6 @@
mask = dobj.selector.select_octs(self.oct_handler)
file_ids = self.regions.identify_data_files(dobj.selector)
dobj._chunk_info = [self.data_files[i] for i in file_ids]
- #dobj.size = sum(counts)
- #dobj.shape = (dobj.size,)
dobj._current_chunk = list(self._chunk_all(dobj))[0]
def _chunk_all(self, dobj):
@@ -161,5 +160,5 @@
def _chunk_io(self, dobj):
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
for subset in oobjs:
- yield YTDataChunk(dobj, "io", [subset], subset.cell_count)
+ yield YTDataChunk(dobj, "io", [subset], -1)
https://bitbucket.org/yt_analysis/yt-3.0/commits/52cc6544d609/
Changeset: 52cc6544d609
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-14 18:36:46
Summary: Merging in noglobalmesh, to start removing size and shape.
Affected #: 8 files
diff -r 31ed51b8ccb0f32a4f66bc542e48a9a623d8850a -r 52cc6544d609a36cf117a5bde14206e7ced903da yt/analysis_modules/halo_finding/halo_objects.py
--- a/yt/analysis_modules/halo_finding/halo_objects.py
+++ b/yt/analysis_modules/halo_finding/halo_objects.py
@@ -2495,7 +2495,7 @@
if dm_only:
select = self._get_dm_indices()
total_mass = \
- self.comm.mpi_allreduce((self._data_source["ParticleMassMsun"][select]).sum(dtype='float64'), op='sum')
+ self.comm.mpi_allreduce((self._data_source['all', "ParticleMassMsun"][select]).sum(dtype='float64'), op='sum')
else:
total_mass = self.comm.mpi_allreduce(self._data_source.quantities["TotalQuantity"]("ParticleMassMsun")[0], op='sum')
# MJT: Note that instead of this, if we are assuming that the particles
diff -r 31ed51b8ccb0f32a4f66bc542e48a9a623d8850a -r 52cc6544d609a36cf117a5bde14206e7ced903da yt/analysis_modules/halo_finding/hop/hop_hop.c
--- a/yt/analysis_modules/halo_finding/hop/hop_hop.c
+++ b/yt/analysis_modules/halo_finding/hop/hop_hop.c
@@ -443,7 +443,7 @@
/* Else, this slot was full, go to the next one */
hp++;
if (hp>=smx->hash+smx->nHashLength) hp = smx->hash;
- if (++count>1000) {
+ if (++count>1000000) {
fprintf(stderr,"Hash Table is too full.\n");
exit(1);
}
diff -r 31ed51b8ccb0f32a4f66bc542e48a9a623d8850a -r 52cc6544d609a36cf117a5bde14206e7ced903da yt/analysis_modules/star_analysis/sfr_spectrum.py
--- a/yt/analysis_modules/star_analysis/sfr_spectrum.py
+++ b/yt/analysis_modules/star_analysis/sfr_spectrum.py
@@ -109,16 +109,19 @@
"""
# Pick out the stars.
if self.mode == 'data_source':
- ct = self._data_source["creation_time"]
+ ct = self._data_source["stars","particle_age"]
+ if ct == None :
+ print 'data source must have particle_age!'
+ sys.exit(1)
ct_stars = ct[ct > 0]
- mass_stars = self._data_source["ParticleMassMsun"][ct > 0]
+ mass_stars = self._data_source["stars", "ParticleMassMsun"][ct > 0]
elif self.mode == 'provided':
ct_stars = self.star_creation_time
mass_stars = self.star_mass
# Find the oldest stars in units of code time.
tmin= min(ct_stars)
# Multiply the end to prevent numerical issues.
- self.time_bins = np.linspace(tmin*0.99, self._pf.current_time,
+ self.time_bins = np.linspace(tmin*1.01, self._pf.current_time,
num = self.bin_count + 1)
# Figure out which bins the stars go into.
inds = np.digitize(ct_stars, self.time_bins) - 1
@@ -131,7 +134,7 @@
for index in xrange(self.bin_count):
self.cum_mass_bins[index+1] += self.cum_mass_bins[index]
# We will want the time taken between bins.
- self.time_bins_dt = self.time_bins[1:] - self.time_bins[:-1]
+ self.time_bins_dt = self.time_bins[:-1] - self.time_bins[1:]
def attach_arrays(self):
"""
@@ -147,7 +150,7 @@
vol = ds.volume('mpc')
elif self.mode == 'provided':
vol = self.volume
- tc = self._pf["Time"]
+ tc = self._pf["Time"] #time to seconds?
self.time = []
self.lookback_time = []
self.redshift = []
diff -r 31ed51b8ccb0f32a4f66bc542e48a9a623d8850a -r 52cc6544d609a36cf117a5bde14206e7ced903da yt/data_objects/data_containers.py
--- a/yt/data_objects/data_containers.py
+++ b/yt/data_objects/data_containers.py
@@ -432,8 +432,6 @@
_sort_by = None
_selector = None
_current_chunk = None
- size = None
- shape = None
def __init__(self, *args, **kwargs):
super(YTSelectionContainer, self).__init__(*args, **kwargs)
@@ -551,16 +549,10 @@
# There are several items that need to be swapped out
# field_data, size, shape
old_field_data, self.field_data = self.field_data, YTFieldData()
- old_size, self.size = self.size, chunk.data_size
old_chunk, self._current_chunk = self._current_chunk, chunk
old_locked, self._locked = self._locked, False
- if not self._spatial:
- self.shape = (self.size,)
yield
self.field_data = old_field_data
- self.size = old_size
- if not self._spatial:
- self.shape = (old_size,)
self._current_chunk = old_chunk
self._locked = old_locked
diff -r 31ed51b8ccb0f32a4f66bc542e48a9a623d8850a -r 52cc6544d609a36cf117a5bde14206e7ced903da yt/data_objects/universal_fields.py
--- a/yt/data_objects/universal_fields.py
+++ b/yt/data_objects/universal_fields.py
@@ -96,7 +96,7 @@
display_field = False)
def _Ones(field, data):
- return np.ones(data.shape, dtype='float64')
+ return np.ones(data.ActiveDimensions, dtype='float64')
add_field("Ones", function=_Ones,
projection_conversion="unitary",
display_field = False)
diff -r 31ed51b8ccb0f32a4f66bc542e48a9a623d8850a -r 52cc6544d609a36cf117a5bde14206e7ced903da yt/frontends/artio/_artio_caller.pyx
--- a/yt/frontends/artio/_artio_caller.pyx
+++ b/yt/frontends/artio/_artio_caller.pyx
@@ -341,7 +341,7 @@
if selected_mass[ispec] :
count = len(data[selected_mass[ispec]])
data[selected_mass[ispec]].resize(count+1)
- data[selected_mass[ispec]][count] = self.parameters["particle_species_mass"]
+ data[selected_mass[ispec]][count] = self.parameters["particle_species_mass"][0]
status = artio_particle_read_species_end( self.handle )
check_artio_status(status)
diff -r 31ed51b8ccb0f32a4f66bc542e48a9a623d8850a -r 52cc6544d609a36cf117a5bde14206e7ced903da yt/frontends/artio/data_structures.py
--- a/yt/frontends/artio/data_structures.py
+++ b/yt/frontends/artio/data_structures.py
@@ -32,7 +32,7 @@
artio_is_valid, artio_fileset
from yt.utilities.definitions import \
mpc_conversion, sec_conversion
-from .fields import ARTIOFieldInfo, KnownARTIOFields
+from .fields import ARTIOFieldInfo, KnownARTIOFields, b2t
from yt.funcs import *
from yt.geometry.geometry_handler import \
@@ -145,7 +145,7 @@
"""
Returns (in code units) the smallest cell size in the simulation.
"""
- return (self.parameter_file.domain_width/(2**self.max_level)).min()
+ return 1.0/(2**self.max_level)
def convert(self, unit):
return self.parameter_file.conversion_factors[unit]
@@ -391,7 +391,7 @@
list(set(art_to_yt[s] for s in
self.artio_parameters["particle_species_labels"])))
- self.current_time = self.artio_parameters["tl"][0]
+ self.current_time = b2t(self.artio_parameters["tl"][0])
# detect cosmology
if "abox" in self.artio_parameters:
diff -r 31ed51b8ccb0f32a4f66bc542e48a9a623d8850a -r 52cc6544d609a36cf117a5bde14206e7ced903da yt/frontends/artio/fields.py
--- a/yt/frontends/artio/fields.py
+++ b/yt/frontends/artio/fields.py
@@ -295,12 +295,12 @@
#add_artio_field("creation_time", function=NullFunc, particle_type=True)
def _particle_age(field, data):
- pa = b2t(data['creation_time'])
+ pa = b2t(data['stars','creation_time'])
# tr = np.zeros(pa.shape,dtype='float')-1.0
# tr[pa>0] = pa[pa>0]
tr = pa
return tr
-add_field("particle_age", function=_particle_age, units=r"\rm{s}",
+add_field(("stars","particle_age"), function=_particle_age, units=r"\rm{s}",
particle_type=True)
@@ -416,10 +416,10 @@
def b2t(tb, n=1e2, logger=None, **kwargs):
tb = np.array(tb)
- if isinstance(tb, 1.1):
+ if len(np.atleast_1d(tb)) == 1:
return a2t(b2a(tb))
if tb.shape == ():
- return a2t(b2a(tb))
+ return None
if len(tb) < n:
n = len(tb)
age_min = a2t(b2a(tb.max(), **kwargs), **kwargs)
@@ -434,7 +434,7 @@
ages = np.array(ages)
fb2t = np.interp(tb, tbs, ages)
#fb2t = interp1d(tbs,ages)
- return fb2t
+ return fb2t*1e9*31556926
def spread_ages(ages, logger=None, spread=.0e7*365*24*3600):
https://bitbucket.org/yt_analysis/yt-3.0/commits/a23b3276b87d/
Changeset: a23b3276b87d
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-14 19:16:29
Summary: Starting process of removing global mesh.
Note that this turns off global mesh for particle codes, which we may revert.
They do in fact have a global mesh, so they may not be applicable here. It
also sets up the proposed change of an on-demand iteration for coordinates.
Affected #: 3 files
diff -r 52cc6544d609a36cf117a5bde14206e7ced903da -r a23b3276b87d63bf7d4f39965fc645f97f22355e yt/data_objects/construction_data_containers.py
--- a/yt/data_objects/construction_data_containers.py
+++ b/yt/data_objects/construction_data_containers.py
@@ -281,8 +281,9 @@
chunk_fields.append(self.weight_field)
tree = self._get_tree(len(fields))
# We do this once
- for chunk in self.data_source.chunks(None, "io"):
- self._initialize_chunk(chunk, tree)
+ if self.pf.h._global_mesh:
+ for chunk in self.data_source.chunks(None, "io"):
+ self._initialize_chunk(chunk, tree)
# This needs to be parallel_objects-ified
for chunk in parallel_objects(self.data_source.chunks(
chunk_fields, "io")):
diff -r 52cc6544d609a36cf117a5bde14206e7ced903da -r a23b3276b87d63bf7d4f39965fc645f97f22355e yt/geometry/geometry_handler.py
--- a/yt/geometry/geometry_handler.py
+++ b/yt/geometry/geometry_handler.py
@@ -44,6 +44,7 @@
ParallelAnalysisInterface, parallel_splitter
class GeometryHandler(ParallelAnalysisInterface):
+ _global_mesh = True
def __init__(self, pf, data_style):
ParallelAnalysisInterface.__init__(self)
@@ -449,7 +450,7 @@
class YTDataChunk(object):
- def __init__(self, dobj, chunk_type, objs, data_size, field_type = None):
+ def __init__(self, dobj, chunk_type, objs, data_size = None, field_type = None):
self.dobj = dobj
self.chunk_type = chunk_type
self.objs = objs
@@ -462,9 +463,23 @@
self._data_size = self._data_size(self.dobj, self.objs)
return self._data_size
+ def _accumulate_values(self, method):
+ # We call this generically. It's somewhat slower, since we're doing
+ # costly getattr functions, but this allows us to generalize.
+ mname = "select_%s" % method
+ arrs = []
+ for obj in self.objs:
+ f = getattr(obj, mname)
+ arrs.append(f(self.dobj))
+ arrs = np.concatenate(arrs)
+ self._data_size = arrs.shape[0]
+ return arrs
+
_fcoords = None
@property
def fcoords(self):
+ if self.data_size is None:
+ self._fcoords = self._accumulate_values("fcoords")
if self._fcoords is not None: return self._fcoords
ci = np.empty((self.data_size, 3), dtype='float64')
self._fcoords = ci
@@ -480,6 +495,8 @@
_icoords = None
@property
def icoords(self):
+ if self.data_size is None:
+ self._icoords = self._accumulate_values("icoords")
if self._icoords is not None: return self._icoords
ci = np.empty((self.data_size, 3), dtype='int64')
self._icoords = ci
@@ -495,6 +512,8 @@
_fwidth = None
@property
def fwidth(self):
+ if self.data_size is None:
+ self._fwidth = self._accumulate_values("fwidth")
if self._fwidth is not None: return self._fwidth
ci = np.empty((self.data_size, 3), dtype='float64')
self._fwidth = ci
@@ -510,6 +529,8 @@
_ires = None
@property
def ires(self):
+ if self.data_size is None:
+ self._ires = self._accumulate_values("ires")
if self._ires is not None: return self._ires
ci = np.empty(self.data_size, dtype='int64')
self._ires = ci
diff -r 52cc6544d609a36cf117a5bde14206e7ced903da -r a23b3276b87d63bf7d4f39965fc645f97f22355e yt/geometry/particle_geometry_handler.py
--- a/yt/geometry/particle_geometry_handler.py
+++ b/yt/geometry/particle_geometry_handler.py
@@ -46,6 +46,7 @@
from yt.data_objects.data_containers import data_object_registry
class ParticleGeometryHandler(GeometryHandler):
+ _global_mesh = False
def __init__(self, pf, data_style):
self.data_style = data_style
@@ -144,7 +145,7 @@
def _chunk_all(self, dobj):
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
- yield YTDataChunk(dobj, "all", oobjs, dobj.size)
+ yield YTDataChunk(dobj, "all", oobjs, None)
def _chunk_spatial(self, dobj, ngz, sort = None):
sobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
@@ -160,5 +161,5 @@
def _chunk_io(self, dobj):
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
for subset in oobjs:
- yield YTDataChunk(dobj, "io", [subset], -1)
+ yield YTDataChunk(dobj, "io", [subset], None)
https://bitbucket.org/yt_analysis/yt-3.0/commits/777c8f708205/
Changeset: 777c8f708205
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-14 19:54:24
Summary: Splitting up the oct_container module, starting work on particle selection.
Affected #: 8 files
diff -r a23b3276b87d63bf7d4f39965fc645f97f22355e -r 777c8f7082052eb202f9bd61e119c0520e98b19a yt/data_objects/data_containers.py
--- a/yt/data_objects/data_containers.py
+++ b/yt/data_objects/data_containers.py
@@ -243,7 +243,7 @@
return rv
def _generate_spatial_fluid(self, field, ngz):
- rv = np.empty(self.size, dtype="float64")
+ rv = np.empty(self.ires.size, dtype="float64")
ind = 0
if ngz == 0:
for io_chunk in self.chunks([], "io"):
diff -r a23b3276b87d63bf7d4f39965fc645f97f22355e -r 777c8f7082052eb202f9bd61e119c0520e98b19a yt/frontends/sph/data_structures.py
--- a/yt/frontends/sph/data_structures.py
+++ b/yt/frontends/sph/data_structures.py
@@ -59,6 +59,10 @@
from yt.data_objects.field_info_container import \
FieldInfoContainer, NullFunc
+class ParticleOctreeSubset(object):
+ def __init__(self):
+ pass
+
class ParticleFile(object):
def __init__(self, pf, io, filename, file_id):
self.pf = pf
diff -r a23b3276b87d63bf7d4f39965fc645f97f22355e -r 777c8f7082052eb202f9bd61e119c0520e98b19a yt/frontends/sph/io.py
--- a/yt/frontends/sph/io.py
+++ b/yt/frontends/sph/io.py
@@ -34,7 +34,7 @@
from yt.utilities.fortran_utils import read_record
from yt.utilities.lib.geometry_utils import get_morton_indices
-from yt.geometry.oct_container import _ORDER_MAX, ParticleRegions
+from yt.geometry.oct_container import _ORDER_MAX
_vector_fields = ("Coordinates", "Velocity", "Velocities")
diff -r a23b3276b87d63bf7d4f39965fc645f97f22355e -r 777c8f7082052eb202f9bd61e119c0520e98b19a yt/geometry/oct_container.pxd
--- a/yt/geometry/oct_container.pxd
+++ b/yt/geometry/oct_container.pxd
@@ -26,6 +26,8 @@
cimport numpy as np
from fp_utils cimport *
+cdef int ORDER_MAX=20
+
cdef struct Oct
cdef struct Oct:
np.int64_t file_ind # index with respect to the order in which it was
diff -r a23b3276b87d63bf7d4f39965fc645f97f22355e -r 777c8f7082052eb202f9bd61e119c0520e98b19a yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -29,12 +29,11 @@
from libc.math cimport floor
cimport numpy as np
import numpy as np
-from oct_container cimport Oct, OctAllocationContainer, OctreeContainer
+from oct_container cimport Oct, OctAllocationContainer, \
+ OctreeContainer, ORDER_MAX
from selection_routines cimport SelectorObject
cimport cython
-DEF ORDER_MAX=20
-
_ORDER_MAX = ORDER_MAX
cdef extern from "stdlib.h":
@@ -818,534 +817,3 @@
else: return 0
elif o1.domain > o2.domain: return 1
-cdef class ParticleOctreeContainer(OctreeContainer):
- cdef Oct** oct_list
- #The starting oct index of each domain
- cdef np.int64_t *dom_offsets
- cdef public int max_level
- #How many particles do we keep befor refining
- cdef public int n_ref
-
- def allocate_root(self):
- cdef int i, j, k
- cdef Oct *cur
- for i in range(self.nn[0]):
- for j in range(self.nn[1]):
- for k in range(self.nn[2]):
- cur = self.allocate_oct()
- cur.level = 0
- cur.pos[0] = i
- cur.pos[1] = j
- cur.pos[2] = k
- cur.parent = NULL
- self.root_mesh[i][j][k] = cur
-
- def __dealloc__(self):
- #Call the freemem ops on every ocy
- #of the root mesh recursively
- cdef i, j, k
- for i in range(self.nn[0]):
- for j in range(self.nn[1]):
- for k in range(self.nn[2]):
- self.visit_free(self.root_mesh[i][j][k])
- free(self.oct_list)
- free(self.dom_offsets)
-
- cdef void visit_free(self, Oct *o):
- #Free the memory for this oct recursively
- cdef int i, j, k
- for i in range(2):
- for j in range(2):
- for k in range(2):
- if o.children[i][j][k] == NULL: continue
- self.visit_free(o.children[i][j][k])
- free(o)
-
- def clear_fileind(self):
- cdef i, j, k
- for i in range(self.nn[0]):
- for j in range(self.nn[1]):
- for k in range(self.nn[2]):
- self.visit_clear(self.root_mesh[i][j][k])
-
- cdef void visit_clear(self, Oct *o):
- #Free the memory for this oct recursively
- cdef int i, j, k
- o.file_ind = 0
- for i in range(2):
- for j in range(2):
- for k in range(2):
- if o.children[i][j][k] == NULL: continue
- self.visit_clear(o.children[i][j][k])
-
- def __iter__(self):
- #Get the next oct, will traverse domains
- #Note that oct containers can be sorted
- #so that consecutive octs are on the same domain
- cdef int oi
- cdef Oct *o
- for oi in range(self.nocts):
- o = self.oct_list[oi]
- yield (o.file_ind, o.domain_ind, o.domain)
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
- def icoords(self, int domain_id,
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
- np.int64_t cell_count,
- np.ndarray[np.int64_t, ndim=1] level_counts):
- #Return the integer positions of the cells
- #Limited to this domain and within the mask
- #Positions are binary; aside from the root mesh
- #to each digit we just add a << 1 and a 0 or 1
- #for each child recursively
- cdef np.ndarray[np.int64_t, ndim=2] coords
- coords = np.empty((cell_count, 3), dtype="int64")
- cdef int oi, i, ci, ii
- ci = 0
- for oi in range(self.nocts):
- o = self.oct_list[oi]
- if o.domain != domain_id: continue
- for i in range(2):
- for j in range(2):
- for k in range(2):
- ii = ((k*2)+j)*2+i
- if mask[oi, ii] == 1:
- coords[ci, 0] = (o.pos[0] << 1) + i
- coords[ci, 1] = (o.pos[1] << 1) + j
- coords[ci, 2] = (o.pos[2] << 1) + k
- ci += 1
- return coords
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
- def ires(self, int domain_id,
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
- np.int64_t cell_count,
- np.ndarray[np.int64_t, ndim=1] level_counts):
- #Return the 'resolution' of each cell; ie the level
- cdef np.ndarray[np.int64_t, ndim=1] res
- res = np.empty(cell_count, dtype="int64")
- cdef int oi, i, ci
- ci = 0
- for oi in range(self.nocts):
- o = self.oct_list[oi]
- if o.domain != domain_id: continue
- for i in range(8):
- if mask[oi, i] == 1:
- res[ci] = o.level
- ci += 1
- return res
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
- def fcoords(self, int domain_id,
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
- np.int64_t cell_count,
- np.ndarray[np.int64_t, ndim=1] level_counts):
- #Return the floating point unitary position of every cell
- cdef np.ndarray[np.float64_t, ndim=2] coords
- coords = np.empty((cell_count, 3), dtype="float64")
- cdef int oi, i, ci
- cdef np.float64_t base_dx[3], dx[3], pos[3]
- for i in range(3):
- # This is the base_dx, but not the base distance from the center
- # position. Note that the positions will also all be offset by
- # dx/2.0. This is also for *oct grids*, not cells.
- base_dx[i] = (self.DRE[i] - self.DLE[i])/self.nn[i]
- ci = 0
- cdef int proc
- for oi in range(self.nocts):
- proc = 0
- for i in range(8):
- if mask[oi, i] == 1:
- proc = 1
- break
- if proc == 0: continue
- o = self.oct_list[oi]
- if o.domain != domain_id: continue
- for i in range(3):
- # This gives the *grid* width for this level
- dx[i] = base_dx[i] / (1 << o.level)
- # o.pos is the *grid* index, so pos[i] is the center of the
- # first cell in the grid
- pos[i] = self.DLE[i] + o.pos[i]*dx[i] + dx[i]/4.0
- dx[i] = dx[i] / 2.0 # This is now the *offset*
- for i in range(2):
- for j in range(2):
- for k in range(2):
- ii = ((k*2)+j)*2+i
- if mask[oi, ii] == 0: continue
- coords[ci, 0] = pos[0] + dx[0] * i
- coords[ci, 1] = pos[1] + dx[1] * j
- coords[ci, 2] = pos[2] + dx[2] * k
- ci += 1
- return coords
-
- def allocate_domains(self, domain_counts):
- pass
-
- def finalize(self):
- #This will sort the octs in the oct list
- #so that domains appear consecutively
- #And then find the oct index/offset for
- #every domain
- cdef int max_level = 0
- self.oct_list = <Oct**> malloc(sizeof(Oct*)*self.nocts)
- cdef np.int64_t i = 0, lpos = 0
- self.max_level = max_level
- cdef int cur_dom = -1
- # We always need at least 2, and if max_domain is 0, we need 3.
- for i in range(self.nn[0]):
- for j in range(self.nn[1]):
- for k in range(self.nn[2]):
- self.visit_assign(self.root_mesh[i][j][k], &lpos)
- assert(lpos == self.nocts)
- for i in range(self.nocts):
- self.oct_list[i].domain_ind = i
- self.oct_list[i].file_ind = -1
-
- cdef visit_assign(self, Oct *o, np.int64_t *lpos):
- cdef int i, j, k
- self.oct_list[lpos[0]] = o
- lpos[0] += 1
- for i in range(2):
- for j in range(2):
- for k in range(2):
- if o.children[i][j][k] != NULL:
- self.visit_assign(o.children[i][j][k], lpos)
- return
-
- cdef np.int64_t get_domain_offset(self, int domain_id):
- return self.dom_offsets[domain_id + 1]
-
- cdef Oct* allocate_oct(self):
- #Allocate the memory, set to NULL or -1
- #We reserve space for n_ref particles, but keep
- #track of how many are used with np initially 0
- self.nocts += 1
- cdef Oct *my_oct = <Oct*> malloc(sizeof(Oct))
- cdef int i, j, k
- my_oct.domain = -1
- my_oct.file_ind = 0
- my_oct.domain_ind = self.nocts - 1
- my_oct.pos[0] = my_oct.pos[1] = my_oct.pos[2] = -1
- my_oct.level = -1
- for i in range(2):
- for j in range(2):
- for k in range(2):
- my_oct.children[i][j][k] = NULL
- my_oct.parent = NULL
- return my_oct
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
- def count_levels(self, int max_level, int domain_id,
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask):
- cdef np.ndarray[np.int64_t, ndim=1] level_count
- cdef Oct *o
- cdef int oi, i
- level_count = np.zeros(max_level+1, 'int64')
- cdef np.int64_t ndo, doff
- ndo = self.dom_offsets[domain_id + 2] \
- - self.dom_offsets[domain_id + 1]
- doff = self.dom_offsets[domain_id + 1]
- for oi in range(ndo):
- o = self.oct_list[oi + doff]
- for i in range(8):
- if mask[o.domain_ind, i] == 0: continue
- level_count[o.level] += 1
- return level_count
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
- def add(self, np.ndarray[np.uint64_t, ndim=1] indices):
- #Add this particle to the root oct
- #Then if that oct has children, add it to them recursively
- #If the child needs to be refined because of max particles, do so
- cdef np.int64_t no = indices.shape[0], p, index
- cdef int i, level, ind[3]
- if self.root_mesh[0][0][0] == NULL: self.allocate_root()
- cdef np.uint64_t *data = <np.uint64_t *> indices.data
- for p in range(no):
- # We have morton indices, which means we choose left and right by
- # looking at (MAX_ORDER - level) & with the values 1, 2, 4.
- level = 0
- index = indices[p]
- for i in range(3):
- ind[i] = (index >> ((ORDER_MAX - level)*3 + (2 - i))) & 1
- cur = self.root_mesh[ind[0]][ind[1]][ind[2]]
- if cur == NULL:
- raise RuntimeError
- while (cur.file_ind + 1) > self.n_ref:
- if level >= ORDER_MAX: break # Just dump it here.
- level += 1
- for i in range(3):
- ind[i] = (index >> ((ORDER_MAX - level)*3 + (2 - i))) & 1
- if cur.children[ind[0]][ind[1]][ind[2]] == NULL:
- cur = self.refine_oct(cur, index)
- self.filter_particles(cur, data, p)
- else:
- cur = cur.children[ind[0]][ind[1]][ind[2]]
- cur.file_ind += 1
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
- cdef Oct *refine_oct(self, Oct *o, np.uint64_t index):
- #Allocate and initialize child octs
- #Attach particles to child octs
- #Remove particles from this oct entirely
- cdef int i, j, k, m, n, ind[3]
- cdef Oct *noct
- cdef np.uint64_t prefix1, prefix2
- for i in range(2):
- for j in range(2):
- for k in range(2):
- noct = self.allocate_oct()
- noct.domain = o.domain
- noct.file_ind = 0
- noct.level = o.level + 1
- noct.pos[0] = (o.pos[0] << 1) + i
- noct.pos[1] = (o.pos[1] << 1) + j
- noct.pos[2] = (o.pos[2] << 1) + k
- noct.parent = o
- o.children[i][j][k] = noct
- o.file_ind = self.n_ref + 1
- for i in range(3):
- ind[i] = (index >> ((ORDER_MAX - (o.level + 1))*3 + (2 - i))) & 1
- noct = o.children[ind[0]][ind[1]][ind[2]]
- return noct
-
- cdef void filter_particles(self, Oct *o, np.uint64_t *data, np.int64_t p):
- # Now we look at the last nref particles to decide where they go.
- cdef int n = imin(p, self.n_ref)
- cdef np.uint64_t *arr = data + imax(p - self.n_ref, 0)
- # Now we figure out our prefix, which is the oct address at this level.
- # As long as we're actually in Morton order, we do not need to worry
- # about *any* of the other children of the oct.
- prefix1 = data[p] >> (ORDER_MAX - o.level)*3
- for i in range(n):
- prefix2 = arr[i] >> (ORDER_MAX - o.level)*3
- if (prefix1 == prefix2):
- o.file_ind += 1
- #print ind[0], ind[1], ind[2], o.file_ind, o.level
-
- def recursively_count(self):
- #Visit every cell, accumulate the # of cells per level
- cdef int i, j, k
- cdef np.int64_t counts[128]
- for i in range(128): counts[i] = 0
- for i in range(self.nn[0]):
- for j in range(self.nn[1]):
- for k in range(self.nn[2]):
- if self.root_mesh[i][j][k] != NULL:
- self.visit(self.root_mesh[i][j][k], counts)
- level_counts = {}
- for i in range(128):
- if counts[i] == 0: break
- level_counts[i] = counts[i]
- return level_counts
-
- cdef visit(self, Oct *o, np.int64_t *counts, level = 0):
- cdef int i, j, k
- counts[level] += 1
- for i in range(2):
- for j in range(2):
- for k in range(2):
- if o.children[i][j][k] != NULL:
- self.visit(o.children[i][j][k], counts, level + 1)
- return
-
- def domain_identify(self, np.ndarray[np.uint8_t, ndim=2, cast=True] mask):
- #Return an array of length # of domains
- #Every element is True if there is at least one
- #fully refined *cell* in that domain that isn't masked out
- cdef int i, oi, m
- cdef Oct *o
- cdef np.ndarray[np.uint8_t, ndim=1, cast=True] dmask
- dmask = np.zeros(self.max_domain+1, dtype='uint8')
- for oi in range(self.nocts):
- m = 0
- o = self.oct_list[oi]
- #if o.sd.np <= 0 or o.domain == -1: continue
- for i in range(8):
- if mask[oi, i] == 1:
- m = 1
- break
- if m == 0: continue
- dmask[o.domain] = 1
- return dmask.astype("bool")
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
- def count_cells(self, SelectorObject selector,
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask):
- #Count how many cells per level there are
- cdef int i, j, k, oi
- # pos here is CELL center, not OCT center.
- cdef np.float64_t pos[3]
- cdef int n = mask.shape[0]
- cdef int eterm[3]
- cdef np.ndarray[np.int64_t, ndim=1] count
- count = np.zeros(self.max_domain + 1, 'int64')
- for oi in range(n):
- o = self.oct_list[oi]
- if o.domain == -1: continue
- for i in range(8):
- count[o.domain] += mask[oi,i]
- return count
-
- def domain_and(self, np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
- int domain_id):
- cdef np.int64_t i, oi, n, use
- cdef Oct *o
- cdef np.ndarray[np.uint8_t, ndim=2] m2 = \
- np.zeros((mask.shape[0], 8), 'uint8')
- n = mask.shape[0]
- for oi in range(n):
- o = self.oct_list[oi]
- if o.domain != domain_id: continue
- use = 0
- for i in range(8):
- m2[o.domain_ind, i] = mask[o.domain_ind, i]
- return m2
-
- def domain_mask(self,
- # mask is the base selector's *global* mask
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
- int domain_id):
- # What distinguishes this one from domain_and is that we have a mask,
- # which covers the whole domain, but our output will only be of a much
- # smaller subset of octs that belong to a given domain *and* the mask.
- # Note also that typically when something calls domain_and, they will
- # use a logical_any along the oct axis. Here we don't do that.
- # Note also that we change the shape of the returned array.
- cdef np.int64_t i, j, k, oi, n, nm, use
- cdef Oct *o
- n = mask.shape[0]
- nm = 0
- # This could perhaps be faster if we
- for oi in range(n):
- o = self.oct_list[oi]
- if o.domain != domain_id: continue
- use = 0
- for i in range(8):
- if mask[o.domain_ind, i] == 1: use = 1
- nm += use
- cdef np.ndarray[np.uint8_t, ndim=4] m2 = \
- np.zeros((2, 2, 2, nm), 'uint8')
- nm = 0
- for oi in range(n):
- o = self.oct_list[oi]
- if o.domain != domain_id: continue
- use = 0
- for i in range(2):
- for j in range(2):
- for k in range(2):
- ii = ((k*2)+j)*2+i
- if mask[o.domain_ind, ii] == 0: continue
- use = m2[i, j, k, nm] = 1
- nm += use
- return m2.astype("bool")
-
- def domain_ind(self,
- # mask is the base selector's *global* mask
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
- int domain_id):
- # Here we once again do something similar to the other functions. We
- # need a set of indices into the final reduced, masked values. The
- # indices will be domain.n long, and will be of type int64. This way,
- # we can get the Oct through a .get() call, then use Oct.file_ind as an
- # index into this newly created array, then finally use the returned
- # index into the domain subset array for deposition.
- cdef np.int64_t i, j, k, oi, noct, n, nm, use, offset
- cdef Oct *o
- # For particle octrees, domain 0 is special and means non-leaf nodes.
- offset = self.dom_offsets[domain_id + 1]
- noct = self.dom_offsets[domain_id + 2] - offset
- cdef np.ndarray[np.int64_t, ndim=1] ind = np.zeros(noct, 'int64')
- nm = 0
- for oi in range(noct):
- ind[oi] = -1
- o = self.oct_list[oi + offset]
- use = 0
- for i in range(8):
- if mask[o.domain_ind, i] == 1: use = 1
- if use == 1:
- ind[oi] = nm
- nm += use
- return ind
-
-cdef class ParticleRegions:
- cdef np.float64_t left_edge[3]
- cdef np.float64_t dds[3]
- cdef np.float64_t idds[3]
- cdef np.int32_t dims[3]
- cdef public int nfiles
- cdef public object masks
-
- def __init__(self, left_edge, right_edge, dims, nfiles):
- cdef int i
- self.nfiles = nfiles
- for i in range(3):
- self.left_edge[i] = left_edge[i]
- self.dims[i] = dims[i]
- self.dds[i] = (right_edge[i] - left_edge[i])/dims[i]
- self.idds[i] = 1.0/self.dds[i]
- # We use 64-bit masks
- self.masks = []
- for i in range(nfiles/64 + 1):
- self.masks.append(np.zeros(dims, dtype="uint64"))
-
- def add_data_file(self, np.ndarray[np.float64_t, ndim=2] pos, int file_id):
- cdef np.int64_t no = pos.shape[0]
- cdef np.int64_t p
- cdef int ind[3], i
- cdef np.ndarray[np.uint64_t, ndim=3] mask
- mask = self.masks[file_id/64]
- val = 1 << (file_id - (file_id/64)*64)
- for p in range(no):
- # Now we locate the particle
- for i in range(3):
- ind[i] = <int> ((pos[p, i] - self.left_edge[i])*self.idds[i])
- mask[ind[0],ind[1],ind[2]] |= val
-
- def identify_data_files(self, SelectorObject selector):
- # This is relatively cheap to iterate over.
- cdef int i, j, k, n
- cdef np.uint64_t fmask, offset
- cdef np.float64_t LE[3], RE[3]
- cdef np.ndarray[np.uint64_t, ndim=3] mask
- files = []
- for n in range(len(self.masks)):
- fmask = 0
- mask = self.masks[n]
- LE[0] = self.left_edge[0]
- RE[0] = LE[0] + self.dds[0]
- for i in range(self.dims[0]):
- LE[1] = self.left_edge[1]
- RE[1] = LE[1] + self.dds[1]
- for j in range(self.dims[1]):
- LE[2] = self.left_edge[2]
- RE[2] = LE[2] + self.dds[2]
- for k in range(self.dims[2]):
- if selector.select_grid(LE, RE, 0) == 1:
- fmask |= mask[i,j,k]
- LE[2] += self.dds[2]
- RE[2] += self.dds[2]
- LE[1] += self.dds[1]
- RE[1] += self.dds[1]
- LE[0] += self.dds[0]
- RE[0] += self.dds[0]
- # Now we iterate through...
- for i in range(64):
- if ((fmask >> i) & 1) == 1:
- files.append(i + n * 64)
- return files
diff -r a23b3276b87d63bf7d4f39965fc645f97f22355e -r 777c8f7082052eb202f9bd61e119c0520e98b19a yt/geometry/particle_geometry_handler.py
--- a/yt/geometry/particle_geometry_handler.py
+++ b/yt/geometry/particle_geometry_handler.py
@@ -36,7 +36,7 @@
from yt.config import ytcfg
from yt.data_objects.field_info_container import NullFunc
from yt.geometry.geometry_handler import GeometryHandler, YTDataChunk
-from yt.geometry.oct_container import \
+from yt.geometry.particle_oct_container import \
ParticleOctreeContainer, ParticleRegions
from yt.utilities.definitions import MAXLEVEL
from yt.utilities.io_handler import io_registry
@@ -145,7 +145,7 @@
def _chunk_all(self, dobj):
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
- yield YTDataChunk(dobj, "all", oobjs, None)
+ yield ParticleDataChunk(self.oct_handler, self.regions, dobj, "all", oobjs, None)
def _chunk_spatial(self, dobj, ngz, sort = None):
sobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
@@ -156,10 +156,21 @@
g = og
size = og.cell_count
if size == 0: continue
- yield YTDataChunk(dobj, "spatial", [g], size)
+ yield ParticleDataChunk(self.oct_handler, self.regions,
+ dobj, "spatial", [g], size)
def _chunk_io(self, dobj):
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
for subset in oobjs:
- yield YTDataChunk(dobj, "io", [subset], None)
+ yield ParticleDataChunk(self.oct_handler, self.regions,
+ dobj, "io", [subset], None)
+class ParticleDataChunk(YTDataChunk):
+ def __init__(self, oct_handler, regions, *args, **kwargs):
+ self.oct_handler = oct_handler
+ self.regions = regions
+ super(ParticleDataChunk, self).__init__(*args, **kwargs)
+
+ def _accumulate_values(self, method):
+ mfunc = getattr(self.oct_handler, "select_%s" % method)
+ return mfunc(self.dobj)
diff -r a23b3276b87d63bf7d4f39965fc645f97f22355e -r 777c8f7082052eb202f9bd61e119c0520e98b19a yt/geometry/particle_oct_container.pyx
--- /dev/null
+++ b/yt/geometry/particle_oct_container.pyx
@@ -0,0 +1,569 @@
+"""
+Oct container tuned for Particles
+
+Author: Matthew Turk <matthewturk at gmail.com>
+Affiliation: Columbia University
+Author: Christopher Moody <chris.e.moody at gmail.com>
+Affiliation: UC Santa Cruz
+Homepage: http://yt.enzotools.org/
+License:
+ Copyright (C) 2013 Matthew Turk. All Rights Reserved.
+
+ This file is part of yt.
+
+ yt is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+
+from oct_container cimport OctreeContainer, Oct, OctInfo
+from libc.stdlib cimport malloc, free, qsort
+from libc.math cimport floor
+from fp_utils cimport *
+cimport numpy as np
+import numpy as np
+from oct_container cimport Oct, OctAllocationContainer, \
+ OctreeContainer, ORDER_MAX
+from selection_routines cimport SelectorObject
+cimport cython
+
+cdef class ParticleOctreeContainer(OctreeContainer):
+ cdef Oct** oct_list
+ #The starting oct index of each domain
+ cdef np.int64_t *dom_offsets
+ cdef public int max_level
+ #How many particles do we keep befor refining
+ cdef public int n_ref
+
+ def allocate_root(self):
+ cdef int i, j, k
+ cdef Oct *cur
+ for i in range(self.nn[0]):
+ for j in range(self.nn[1]):
+ for k in range(self.nn[2]):
+ cur = self.allocate_oct()
+ cur.level = 0
+ cur.pos[0] = i
+ cur.pos[1] = j
+ cur.pos[2] = k
+ cur.parent = NULL
+ self.root_mesh[i][j][k] = cur
+
+ def __dealloc__(self):
+ #Call the freemem ops on every ocy
+ #of the root mesh recursively
+ cdef i, j, k
+ for i in range(self.nn[0]):
+ for j in range(self.nn[1]):
+ for k in range(self.nn[2]):
+ self.visit_free(self.root_mesh[i][j][k])
+ free(self.oct_list)
+ free(self.dom_offsets)
+
+ cdef void visit_free(self, Oct *o):
+ #Free the memory for this oct recursively
+ cdef int i, j, k
+ for i in range(2):
+ for j in range(2):
+ for k in range(2):
+ if o.children[i][j][k] == NULL: continue
+ self.visit_free(o.children[i][j][k])
+ free(o)
+
+ def clear_fileind(self):
+ cdef i, j, k
+ for i in range(self.nn[0]):
+ for j in range(self.nn[1]):
+ for k in range(self.nn[2]):
+ self.visit_clear(self.root_mesh[i][j][k])
+
+ cdef void visit_clear(self, Oct *o):
+ #Free the memory for this oct recursively
+ cdef int i, j, k
+ o.file_ind = 0
+ for i in range(2):
+ for j in range(2):
+ for k in range(2):
+ if o.children[i][j][k] == NULL: continue
+ self.visit_clear(o.children[i][j][k])
+
+ def __iter__(self):
+ #Get the next oct, will traverse domains
+ #Note that oct containers can be sorted
+ #so that consecutive octs are on the same domain
+ cdef int oi
+ cdef Oct *o
+ for oi in range(self.nocts):
+ o = self.oct_list[oi]
+ yield (o.file_ind, o.domain_ind, o.domain)
+
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ def icoords(self, int domain_id,
+ np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
+ np.int64_t cell_count,
+ np.ndarray[np.int64_t, ndim=1] level_counts):
+ #Return the integer positions of the cells
+ #Limited to this domain and within the mask
+ #Positions are binary; aside from the root mesh
+ #to each digit we just add a << 1 and a 0 or 1
+ #for each child recursively
+ cdef np.ndarray[np.int64_t, ndim=2] coords
+ coords = np.empty((cell_count, 3), dtype="int64")
+ cdef int oi, i, ci, ii
+ ci = 0
+ for oi in range(self.nocts):
+ o = self.oct_list[oi]
+ if o.domain != domain_id: continue
+ for i in range(2):
+ for j in range(2):
+ for k in range(2):
+ ii = ((k*2)+j)*2+i
+ if mask[oi, ii] == 1:
+ coords[ci, 0] = (o.pos[0] << 1) + i
+ coords[ci, 1] = (o.pos[1] << 1) + j
+ coords[ci, 2] = (o.pos[2] << 1) + k
+ ci += 1
+ return coords
+
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ def ires(self, int domain_id,
+ np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
+ np.int64_t cell_count,
+ np.ndarray[np.int64_t, ndim=1] level_counts):
+ #Return the 'resolution' of each cell; ie the level
+ cdef np.ndarray[np.int64_t, ndim=1] res
+ res = np.empty(cell_count, dtype="int64")
+ cdef int oi, i, ci
+ ci = 0
+ for oi in range(self.nocts):
+ o = self.oct_list[oi]
+ if o.domain != domain_id: continue
+ for i in range(8):
+ if mask[oi, i] == 1:
+ res[ci] = o.level
+ ci += 1
+ return res
+
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ def fcoords(self, int domain_id,
+ np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
+ np.int64_t cell_count,
+ np.ndarray[np.int64_t, ndim=1] level_counts):
+ #Return the floating point unitary position of every cell
+ cdef np.ndarray[np.float64_t, ndim=2] coords
+ coords = np.empty((cell_count, 3), dtype="float64")
+ cdef int oi, i, ci
+ cdef np.float64_t base_dx[3], dx[3], pos[3]
+ for i in range(3):
+ # This is the base_dx, but not the base distance from the center
+ # position. Note that the positions will also all be offset by
+ # dx/2.0. This is also for *oct grids*, not cells.
+ base_dx[i] = (self.DRE[i] - self.DLE[i])/self.nn[i]
+ ci = 0
+ cdef int proc
+ for oi in range(self.nocts):
+ proc = 0
+ for i in range(8):
+ if mask[oi, i] == 1:
+ proc = 1
+ break
+ if proc == 0: continue
+ o = self.oct_list[oi]
+ if o.domain != domain_id: continue
+ for i in range(3):
+ # This gives the *grid* width for this level
+ dx[i] = base_dx[i] / (1 << o.level)
+ # o.pos is the *grid* index, so pos[i] is the center of the
+ # first cell in the grid
+ pos[i] = self.DLE[i] + o.pos[i]*dx[i] + dx[i]/4.0
+ dx[i] = dx[i] / 2.0 # This is now the *offset*
+ for i in range(2):
+ for j in range(2):
+ for k in range(2):
+ ii = ((k*2)+j)*2+i
+ if mask[oi, ii] == 0: continue
+ coords[ci, 0] = pos[0] + dx[0] * i
+ coords[ci, 1] = pos[1] + dx[1] * j
+ coords[ci, 2] = pos[2] + dx[2] * k
+ ci += 1
+ return coords
+
+ def allocate_domains(self, domain_counts):
+ pass
+
+ def finalize(self):
+ #This will sort the octs in the oct list
+ #so that domains appear consecutively
+ #And then find the oct index/offset for
+ #every domain
+ cdef int max_level = 0
+ self.oct_list = <Oct**> malloc(sizeof(Oct*)*self.nocts)
+ cdef np.int64_t i = 0, lpos = 0
+ self.max_level = max_level
+ cdef int cur_dom = -1
+ # We always need at least 2, and if max_domain is 0, we need 3.
+ for i in range(self.nn[0]):
+ for j in range(self.nn[1]):
+ for k in range(self.nn[2]):
+ self.visit_assign(self.root_mesh[i][j][k], &lpos)
+ assert(lpos == self.nocts)
+ for i in range(self.nocts):
+ self.oct_list[i].domain_ind = i
+ self.oct_list[i].file_ind = -1
+
+ cdef visit_assign(self, Oct *o, np.int64_t *lpos):
+ cdef int i, j, k
+ self.oct_list[lpos[0]] = o
+ lpos[0] += 1
+ for i in range(2):
+ for j in range(2):
+ for k in range(2):
+ if o.children[i][j][k] != NULL:
+ self.visit_assign(o.children[i][j][k], lpos)
+ return
+
+ cdef np.int64_t get_domain_offset(self, int domain_id):
+ return self.dom_offsets[domain_id + 1]
+
+ cdef Oct* allocate_oct(self):
+ #Allocate the memory, set to NULL or -1
+ #We reserve space for n_ref particles, but keep
+ #track of how many are used with np initially 0
+ self.nocts += 1
+ cdef Oct *my_oct = <Oct*> malloc(sizeof(Oct))
+ cdef int i, j, k
+ my_oct.domain = -1
+ my_oct.file_ind = 0
+ my_oct.domain_ind = self.nocts - 1
+ my_oct.pos[0] = my_oct.pos[1] = my_oct.pos[2] = -1
+ my_oct.level = -1
+ for i in range(2):
+ for j in range(2):
+ for k in range(2):
+ my_oct.children[i][j][k] = NULL
+ my_oct.parent = NULL
+ return my_oct
+
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ def count_levels(self, int max_level, int domain_id,
+ np.ndarray[np.uint8_t, ndim=2, cast=True] mask):
+ cdef np.ndarray[np.int64_t, ndim=1] level_count
+ cdef Oct *o
+ cdef int oi, i
+ level_count = np.zeros(max_level+1, 'int64')
+ cdef np.int64_t ndo, doff
+ ndo = self.dom_offsets[domain_id + 2] \
+ - self.dom_offsets[domain_id + 1]
+ doff = self.dom_offsets[domain_id + 1]
+ for oi in range(ndo):
+ o = self.oct_list[oi + doff]
+ for i in range(8):
+ if mask[o.domain_ind, i] == 0: continue
+ level_count[o.level] += 1
+ return level_count
+
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ def add(self, np.ndarray[np.uint64_t, ndim=1] indices):
+ #Add this particle to the root oct
+ #Then if that oct has children, add it to them recursively
+ #If the child needs to be refined because of max particles, do so
+ cdef np.int64_t no = indices.shape[0], p, index
+ cdef int i, level, ind[3]
+ if self.root_mesh[0][0][0] == NULL: self.allocate_root()
+ cdef np.uint64_t *data = <np.uint64_t *> indices.data
+ for p in range(no):
+ # We have morton indices, which means we choose left and right by
+ # looking at (MAX_ORDER - level) & with the values 1, 2, 4.
+ level = 0
+ index = indices[p]
+ for i in range(3):
+ ind[i] = (index >> ((ORDER_MAX - level)*3 + (2 - i))) & 1
+ cur = self.root_mesh[ind[0]][ind[1]][ind[2]]
+ if cur == NULL:
+ raise RuntimeError
+ while (cur.file_ind + 1) > self.n_ref:
+ if level >= ORDER_MAX: break # Just dump it here.
+ level += 1
+ for i in range(3):
+ ind[i] = (index >> ((ORDER_MAX - level)*3 + (2 - i))) & 1
+ if cur.children[ind[0]][ind[1]][ind[2]] == NULL:
+ cur = self.refine_oct(cur, index)
+ self.filter_particles(cur, data, p)
+ else:
+ cur = cur.children[ind[0]][ind[1]][ind[2]]
+ cur.file_ind += 1
+
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ cdef Oct *refine_oct(self, Oct *o, np.uint64_t index):
+ #Allocate and initialize child octs
+ #Attach particles to child octs
+ #Remove particles from this oct entirely
+ cdef int i, j, k, m, n, ind[3]
+ cdef Oct *noct
+ cdef np.uint64_t prefix1, prefix2
+ for i in range(2):
+ for j in range(2):
+ for k in range(2):
+ noct = self.allocate_oct()
+ noct.domain = o.domain
+ noct.file_ind = 0
+ noct.level = o.level + 1
+ noct.pos[0] = (o.pos[0] << 1) + i
+ noct.pos[1] = (o.pos[1] << 1) + j
+ noct.pos[2] = (o.pos[2] << 1) + k
+ noct.parent = o
+ o.children[i][j][k] = noct
+ o.file_ind = self.n_ref + 1
+ for i in range(3):
+ ind[i] = (index >> ((ORDER_MAX - (o.level + 1))*3 + (2 - i))) & 1
+ noct = o.children[ind[0]][ind[1]][ind[2]]
+ return noct
+
+ cdef void filter_particles(self, Oct *o, np.uint64_t *data, np.int64_t p):
+ # Now we look at the last nref particles to decide where they go.
+ cdef int n = imin(p, self.n_ref)
+ cdef np.uint64_t *arr = data + imax(p - self.n_ref, 0)
+ # Now we figure out our prefix, which is the oct address at this level.
+ # As long as we're actually in Morton order, we do not need to worry
+ # about *any* of the other children of the oct.
+ prefix1 = data[p] >> (ORDER_MAX - o.level)*3
+ for i in range(n):
+ prefix2 = arr[i] >> (ORDER_MAX - o.level)*3
+ if (prefix1 == prefix2):
+ o.file_ind += 1
+ #print ind[0], ind[1], ind[2], o.file_ind, o.level
+
+ def recursively_count(self):
+ #Visit every cell, accumulate the # of cells per level
+ cdef int i, j, k
+ cdef np.int64_t counts[128]
+ for i in range(128): counts[i] = 0
+ for i in range(self.nn[0]):
+ for j in range(self.nn[1]):
+ for k in range(self.nn[2]):
+ if self.root_mesh[i][j][k] != NULL:
+ self.visit(self.root_mesh[i][j][k], counts)
+ level_counts = {}
+ for i in range(128):
+ if counts[i] == 0: break
+ level_counts[i] = counts[i]
+ return level_counts
+
+ cdef visit(self, Oct *o, np.int64_t *counts, level = 0):
+ cdef int i, j, k
+ counts[level] += 1
+ for i in range(2):
+ for j in range(2):
+ for k in range(2):
+ if o.children[i][j][k] != NULL:
+ self.visit(o.children[i][j][k], counts, level + 1)
+ return
+
+ def domain_identify(self, np.ndarray[np.uint8_t, ndim=2, cast=True] mask):
+ #Return an array of length # of domains
+ #Every element is True if there is at least one
+ #fully refined *cell* in that domain that isn't masked out
+ cdef int i, oi, m
+ cdef Oct *o
+ cdef np.ndarray[np.uint8_t, ndim=1, cast=True] dmask
+ dmask = np.zeros(self.max_domain+1, dtype='uint8')
+ for oi in range(self.nocts):
+ m = 0
+ o = self.oct_list[oi]
+ #if o.sd.np <= 0 or o.domain == -1: continue
+ for i in range(8):
+ if mask[oi, i] == 1:
+ m = 1
+ break
+ if m == 0: continue
+ dmask[o.domain] = 1
+ return dmask.astype("bool")
+
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ def count_cells(self, SelectorObject selector,
+ np.ndarray[np.uint8_t, ndim=2, cast=True] mask):
+ #Count how many cells per level there are
+ cdef int i, j, k, oi
+ # pos here is CELL center, not OCT center.
+ cdef np.float64_t pos[3]
+ cdef int n = mask.shape[0]
+ cdef int eterm[3]
+ cdef np.ndarray[np.int64_t, ndim=1] count
+ count = np.zeros(self.max_domain + 1, 'int64')
+ for oi in range(n):
+ o = self.oct_list[oi]
+ if o.domain == -1: continue
+ for i in range(8):
+ count[o.domain] += mask[oi,i]
+ return count
+
+ def domain_and(self, np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
+ int domain_id):
+ cdef np.int64_t i, oi, n, use
+ cdef Oct *o
+ cdef np.ndarray[np.uint8_t, ndim=2] m2 = \
+ np.zeros((mask.shape[0], 8), 'uint8')
+ n = mask.shape[0]
+ for oi in range(n):
+ o = self.oct_list[oi]
+ if o.domain != domain_id: continue
+ use = 0
+ for i in range(8):
+ m2[o.domain_ind, i] = mask[o.domain_ind, i]
+ return m2
+
+ def domain_mask(self,
+ # mask is the base selector's *global* mask
+ np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
+ int domain_id):
+ # What distinguishes this one from domain_and is that we have a mask,
+ # which covers the whole domain, but our output will only be of a much
+ # smaller subset of octs that belong to a given domain *and* the mask.
+ # Note also that typically when something calls domain_and, they will
+ # use a logical_any along the oct axis. Here we don't do that.
+ # Note also that we change the shape of the returned array.
+ cdef np.int64_t i, j, k, oi, n, nm, use
+ cdef Oct *o
+ n = mask.shape[0]
+ nm = 0
+ # This could perhaps be faster if we
+ for oi in range(n):
+ o = self.oct_list[oi]
+ if o.domain != domain_id: continue
+ use = 0
+ for i in range(8):
+ if mask[o.domain_ind, i] == 1: use = 1
+ nm += use
+ cdef np.ndarray[np.uint8_t, ndim=4] m2 = \
+ np.zeros((2, 2, 2, nm), 'uint8')
+ nm = 0
+ for oi in range(n):
+ o = self.oct_list[oi]
+ if o.domain != domain_id: continue
+ use = 0
+ for i in range(2):
+ for j in range(2):
+ for k in range(2):
+ ii = ((k*2)+j)*2+i
+ if mask[o.domain_ind, ii] == 0: continue
+ use = m2[i, j, k, nm] = 1
+ nm += use
+ return m2.astype("bool")
+
+ def domain_ind(self,
+ # mask is the base selector's *global* mask
+ np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
+ int domain_id):
+ # Here we once again do something similar to the other functions. We
+ # need a set of indices into the final reduced, masked values. The
+ # indices will be domain.n long, and will be of type int64. This way,
+ # we can get the Oct through a .get() call, then use Oct.file_ind as an
+ # index into this newly created array, then finally use the returned
+ # index into the domain subset array for deposition.
+ cdef np.int64_t i, j, k, oi, noct, n, nm, use, offset
+ cdef Oct *o
+ # For particle octrees, domain 0 is special and means non-leaf nodes.
+ offset = self.dom_offsets[domain_id + 1]
+ noct = self.dom_offsets[domain_id + 2] - offset
+ cdef np.ndarray[np.int64_t, ndim=1] ind = np.zeros(noct, 'int64')
+ nm = 0
+ for oi in range(noct):
+ ind[oi] = -1
+ o = self.oct_list[oi + offset]
+ use = 0
+ for i in range(8):
+ if mask[o.domain_ind, i] == 1: use = 1
+ if use == 1:
+ ind[oi] = nm
+ nm += use
+ return ind
+
+cdef class ParticleRegions:
+ cdef np.float64_t left_edge[3]
+ cdef np.float64_t dds[3]
+ cdef np.float64_t idds[3]
+ cdef np.int32_t dims[3]
+ cdef public int nfiles
+ cdef public object masks
+
+ def __init__(self, left_edge, right_edge, dims, nfiles):
+ cdef int i
+ self.nfiles = nfiles
+ for i in range(3):
+ self.left_edge[i] = left_edge[i]
+ self.dims[i] = dims[i]
+ self.dds[i] = (right_edge[i] - left_edge[i])/dims[i]
+ self.idds[i] = 1.0/self.dds[i]
+ # We use 64-bit masks
+ self.masks = []
+ for i in range(nfiles/64 + 1):
+ self.masks.append(np.zeros(dims, dtype="uint64"))
+
+ def add_data_file(self, np.ndarray[np.float64_t, ndim=2] pos, int file_id):
+ cdef np.int64_t no = pos.shape[0]
+ cdef np.int64_t p
+ cdef int ind[3], i
+ cdef np.ndarray[np.uint64_t, ndim=3] mask
+ mask = self.masks[file_id/64]
+ val = 1 << (file_id - (file_id/64)*64)
+ for p in range(no):
+ # Now we locate the particle
+ for i in range(3):
+ ind[i] = <int> ((pos[p, i] - self.left_edge[i])*self.idds[i])
+ mask[ind[0],ind[1],ind[2]] |= val
+
+ def identify_data_files(self, SelectorObject selector):
+ # This is relatively cheap to iterate over.
+ cdef int i, j, k, n
+ cdef np.uint64_t fmask, offset
+ cdef np.float64_t LE[3], RE[3]
+ cdef np.ndarray[np.uint64_t, ndim=3] mask
+ files = []
+ for n in range(len(self.masks)):
+ fmask = 0
+ mask = self.masks[n]
+ LE[0] = self.left_edge[0]
+ RE[0] = LE[0] + self.dds[0]
+ for i in range(self.dims[0]):
+ LE[1] = self.left_edge[1]
+ RE[1] = LE[1] + self.dds[1]
+ for j in range(self.dims[1]):
+ LE[2] = self.left_edge[2]
+ RE[2] = LE[2] + self.dds[2]
+ for k in range(self.dims[2]):
+ if selector.select_grid(LE, RE, 0) == 1:
+ fmask |= mask[i,j,k]
+ LE[2] += self.dds[2]
+ RE[2] += self.dds[2]
+ LE[1] += self.dds[1]
+ RE[1] += self.dds[1]
+ LE[0] += self.dds[0]
+ RE[0] += self.dds[0]
+ # Now we iterate through...
+ for i in range(64):
+ if ((fmask >> i) & 1) == 1:
+ files.append(i + n * 64)
+ return files
diff -r a23b3276b87d63bf7d4f39965fc645f97f22355e -r 777c8f7082052eb202f9bd61e119c0520e98b19a yt/geometry/setup.py
--- a/yt/geometry/setup.py
+++ b/yt/geometry/setup.py
@@ -14,6 +14,13 @@
depends=["yt/utilities/lib/fp_utils.pxd",
"yt/geometry/oct_container.pxd",
"yt/geometry/selection_routines.pxd"])
+ config.add_extension("particle_oct_container",
+ ["yt/geometry/particle_oct_container.pyx"],
+ include_dirs=["yt/utilities/lib/"],
+ libraries=["m"],
+ depends=["yt/utilities/lib/fp_utils.pxd",
+ "yt/geometry/oct_container.pxd",
+ "yt/geometry/selection_routines.pxd"])
config.add_extension("selection_routines",
["yt/geometry/selection_routines.pyx"],
include_dirs=["yt/utilities/lib/"],
https://bitbucket.org/yt_analysis/yt-3.0/commits/120a06e24932/
Changeset: 120a06e24932
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-14 20:25:46
Summary: Adding get_smallest_dx() to particle geometry handler.
Affected #: 3 files
diff -r 777c8f7082052eb202f9bd61e119c0520e98b19a -r 120a06e24932cdf28c6538774a88d03924403107 yt/geometry/oct_container.pxd
--- a/yt/geometry/oct_container.pxd
+++ b/yt/geometry/oct_container.pxd
@@ -26,7 +26,7 @@
cimport numpy as np
from fp_utils cimport *
-cdef int ORDER_MAX=20
+cdef int ORDER_MAX
cdef struct Oct
cdef struct Oct:
diff -r 777c8f7082052eb202f9bd61e119c0520e98b19a -r 120a06e24932cdf28c6538774a88d03924403107 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -34,6 +34,7 @@
from selection_routines cimport SelectorObject
cimport cython
+ORDER_MAX = 20
_ORDER_MAX = ORDER_MAX
cdef extern from "stdlib.h":
diff -r 777c8f7082052eb202f9bd61e119c0520e98b19a -r 120a06e24932cdf28c6538774a88d03924403107 yt/geometry/particle_geometry_handler.py
--- a/yt/geometry/particle_geometry_handler.py
+++ b/yt/geometry/particle_geometry_handler.py
@@ -66,7 +66,10 @@
"""
Returns (in code units) the smallest cell size in the simulation.
"""
- raise NotImplementedError
+ dx = 1.0/(2**self.oct_handler.max_level)
+ dx *= (self.parameter_file.domain_right_edge -
+ self.parameter_file.domain_left_edge)
+ return dx.min()
def convert(self, unit):
return self.parameter_file.conversion_factors[unit]
https://bitbucket.org/yt_analysis/yt-3.0/commits/3c28775ee450/
Changeset: 3c28775ee450
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-14 20:30:26
Summary: Adding an oct counting function. This is just a stopgap for now, as we will
want to provide bounds or some method of cutting into sub-sequences.
Affected #: 2 files
diff -r 120a06e24932cdf28c6538774a88d03924403107 -r 3c28775ee450fe57f7a3c15d51f023bde3d186d8 yt/geometry/selection_routines.pxd
--- a/yt/geometry/selection_routines.pxd
+++ b/yt/geometry/selection_routines.pxd
@@ -35,6 +35,9 @@
np.float64_t pos[3], np.float64_t dds[3],
np.ndarray[np.uint8_t, ndim=2] mask,
int level = ?)
+ cdef void recursively_count_octs(self, Oct *root,
+ np.float64_t pos[3], np.float64_t dds[3],
+ int level, np.int64_t *count)
cdef int select_grid(self, np.float64_t left_edge[3],
np.float64_t right_edge[3],
np.int32_t level) nogil
diff -r 120a06e24932cdf28c6538774a88d03924403107 -r 3c28775ee450fe57f7a3c15d51f023bde3d186d8 yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -226,6 +226,79 @@
spos[1] += sdds[1]
spos[0] += sdds[0]
+ def count_octs(self, OctreeContainer octree):
+ cdef int i, j, k, n
+ cdef np.float64_t pos[3], dds[3]
+ cdef np.int64_t count = 0
+ # This dds is the oct-width
+ for i in range(3):
+ dds[i] = (octree.DRE[i] - octree.DLE[i]) / octree.nn[i]
+ # Pos is the center of the octs
+ pos[0] = octree.DLE[0] + dds[0]/2.0
+ for i in range(octree.nn[0]):
+ pos[1] = octree.DLE[1] + dds[1]/2.0
+ for j in range(octree.nn[1]):
+ pos[2] = octree.DLE[2] + dds[2]/2.0
+ for k in range(octree.nn[2]):
+ if octree.root_mesh[i][j][k] == NULL: continue
+ self.recursively_count_octs(
+ octree.root_mesh[i][j][k],
+ pos, dds, 0, &count)
+ pos[2] += dds[2]
+ pos[1] += dds[1]
+ pos[0] += dds[0]
+ return count
+
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ cdef void recursively_count_octs(self, Oct *root,
+ np.float64_t pos[3], np.float64_t dds[3],
+ int level, np.int64_t *count):
+ cdef np.float64_t LE[3], RE[3], sdds[3], spos[3]
+ cdef int i, j, k, res, ii
+ cdef Oct *ch
+ # Remember that pos is the *center* of the oct, and dds is the oct
+ # width. So to get to the edges, we add/subtract half of dds.
+ for i in range(3):
+ # sdds is the cell width
+ sdds[i] = dds[i]/2.0
+ LE[i] = pos[i] - dds[i]/2.0
+ RE[i] = pos[i] + dds[i]/2.0
+ #print LE[0], RE[0], LE[1], RE[1], LE[2], RE[2]
+ res = self.select_grid(LE, RE, level)
+ cdef int eterm[3]
+ eterm[0] = eterm[1] = eterm[2] = 0
+ cdef int next_level, this_level
+ # next_level: an int that says whether or not we can progress to children
+ # this_level: an int that says whether or not we can select from this
+ # level
+ next_level = this_level = 1
+ if level == self.max_level:
+ next_level = 0
+ if level < self.min_level or level > self.max_level:
+ this_level = 0
+ if res == 0 and this_level == 1:
+ return
+ # Now we visit all our children. We subtract off sdds for the first
+ # pass because we center it on the first cell.
+ spos[0] = pos[0] - sdds[0]/2.0
+ for i in range(2):
+ spos[1] = pos[1] - sdds[1]/2.0
+ for j in range(2):
+ spos[2] = pos[2] - sdds[2]/2.0
+ for k in range(2):
+ ii = ((k*2)+j)*2+i
+ ch = root.children[i][j][k]
+ if next_level == 1 and ch != NULL:
+ self.recursively_count_octs(
+ ch, spos, sdds, level + 1, count)
+ elif this_level == 1:
+ count[0] += self.select_cell(spos, sdds, eterm)
+ spos[2] += sdds[2]
+ spos[1] += sdds[1]
+ spos[0] += sdds[0]
+
cdef int select_grid(self, np.float64_t left_edge[3],
np.float64_t right_edge[3],
np.int32_t level) nogil:
https://bitbucket.org/yt_analysis/yt-3.0/commits/f73078da0b91/
Changeset: f73078da0b91
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-14 20:58:19
Summary: Fix up oct icoords. Will soon change to a visit function.
Affected #: 2 files
diff -r 3c28775ee450fe57f7a3c15d51f023bde3d186d8 -r f73078da0b9129fd7c8e9a599fedffd59808fdca yt/geometry/particle_geometry_handler.py
--- a/yt/geometry/particle_geometry_handler.py
+++ b/yt/geometry/particle_geometry_handler.py
@@ -120,7 +120,6 @@
# Now we add them all at once.
self.oct_handler.add(morton)
-
def _detect_fields(self):
# TODO: Add additional fields
pfl = []
diff -r 3c28775ee450fe57f7a3c15d51f023bde3d186d8 -r f73078da0b9129fd7c8e9a599fedffd59808fdca yt/geometry/particle_oct_container.pyx
--- a/yt/geometry/particle_oct_container.pyx
+++ b/yt/geometry/particle_oct_container.pyx
@@ -106,34 +106,41 @@
o = self.oct_list[oi]
yield (o.file_ind, o.domain_ind, o.domain)
- @cython.boundscheck(False)
+ #@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def icoords(self, int domain_id,
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
- np.int64_t cell_count,
- np.ndarray[np.int64_t, ndim=1] level_counts):
- #Return the integer positions of the cells
- #Limited to this domain and within the mask
- #Positions are binary; aside from the root mesh
- #to each digit we just add a << 1 and a 0 or 1
- #for each child recursively
+ def icoords(self, SelectorObject selector, np.uint64_t num_cells = -1):
+ if num_cells == -1:
+ num_cells = selector.count_octs(self)
cdef np.ndarray[np.int64_t, ndim=2] coords
- coords = np.empty((cell_count, 3), dtype="int64")
- cdef int oi, i, ci, ii
+ coords = np.empty((num_cells, 3), dtype="int64")
+ cdef int oi, i, ci, ii, eterm[3]
ci = 0
+ cdef np.float64_t left_edge[3], right_edge[3], dds[3]
for oi in range(self.nocts):
o = self.oct_list[oi]
- if o.domain != domain_id: continue
+ #if o.domain != domain_id: continue
+ if o.children[0][0][0] != NULL: continue
+ self.oct_bounds(o, left_edge, dds)
+ for i in range(3): right_edge[i] = left_edge[i] + dds[i]
+ if not selector.select_grid(left_edge, right_edge, o.level):
+ continue
+ for i in range(3): # Set up cell info
+ dds[i] /= 2.0
+ right_edge[0] = left_edge[0] + dds[0]/2.0
for i in range(2):
+ right_edge[1] = left_edge[1] + dds[1]/2.0
for j in range(2):
+ right_edge[2] = left_edge[2] + dds[2]/2.0
for k in range(2):
- ii = ((k*2)+j)*2+i
- if mask[oi, ii] == 1:
+ if selector.select_cell(right_edge, dds, eterm) == 1:
coords[ci, 0] = (o.pos[0] << 1) + i
coords[ci, 1] = (o.pos[1] << 1) + j
coords[ci, 2] = (o.pos[2] << 1) + k
ci += 1
+ right_edge[2] += dds[2]
+ right_edge[1] += dds[1]
+ right_edge[0] += dds[0]
return coords
@cython.boundscheck(False)
@@ -214,7 +221,6 @@
cdef int max_level = 0
self.oct_list = <Oct**> malloc(sizeof(Oct*)*self.nocts)
cdef np.int64_t i = 0, lpos = 0
- self.max_level = max_level
cdef int cur_dom = -1
# We always need at least 2, and if max_domain is 0, we need 3.
for i in range(self.nn[0]):
@@ -225,6 +231,8 @@
for i in range(self.nocts):
self.oct_list[i].domain_ind = i
self.oct_list[i].file_ind = -1
+ max_level = imax(max_level, self.oct_list[i].level)
+ self.max_level = max_level
cdef visit_assign(self, Oct *o, np.int64_t *lpos):
cdef int i, j, k
@@ -400,26 +408,6 @@
dmask[o.domain] = 1
return dmask.astype("bool")
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
- def count_cells(self, SelectorObject selector,
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask):
- #Count how many cells per level there are
- cdef int i, j, k, oi
- # pos here is CELL center, not OCT center.
- cdef np.float64_t pos[3]
- cdef int n = mask.shape[0]
- cdef int eterm[3]
- cdef np.ndarray[np.int64_t, ndim=1] count
- count = np.zeros(self.max_domain + 1, 'int64')
- for oi in range(n):
- o = self.oct_list[oi]
- if o.domain == -1: continue
- for i in range(8):
- count[o.domain] += mask[oi,i]
- return count
-
def domain_and(self, np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
int domain_id):
cdef np.int64_t i, oi, n, use
https://bitbucket.org/yt_analysis/yt-3.0/commits/fc0e492c0265/
Changeset: fc0e492c0265
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-14 21:09:03
Summary: Convert generic counting function into visitor function.
Affected #: 2 files
diff -r f73078da0b9129fd7c8e9a599fedffd59808fdca -r fc0e492c026567482fc6d20f786bfbe0255db89e yt/geometry/selection_routines.pxd
--- a/yt/geometry/selection_routines.pxd
+++ b/yt/geometry/selection_routines.pxd
@@ -26,6 +26,11 @@
cimport numpy as np
cdef struct Oct
+cdef struct OctVisitorData:
+ np.uint64_t index
+ void *array
+
+ctypedef void oct_visitor_function(Oct *, OctVisitorData *visitor)
cdef class SelectorObject:
cdef public np.int32_t min_level
@@ -35,9 +40,11 @@
np.float64_t pos[3], np.float64_t dds[3],
np.ndarray[np.uint8_t, ndim=2] mask,
int level = ?)
- cdef void recursively_count_octs(self, Oct *root,
+ cdef void recursively_visit_octs(self, Oct *root,
np.float64_t pos[3], np.float64_t dds[3],
- int level, np.int64_t *count)
+ int level,
+ oct_visitor_function *func,
+ OctVisitorData *data)
cdef int select_grid(self, np.float64_t left_edge[3],
np.float64_t right_edge[3],
np.int32_t level) nogil
@@ -46,3 +53,4 @@
cdef void set_bounds(self,
np.float64_t left_edge[3], np.float64_t right_edge[3],
np.float64_t dds[3], int ind[3][2], int *check)
+
diff -r f73078da0b9129fd7c8e9a599fedffd59808fdca -r fc0e492c026567482fc6d20f786bfbe0255db89e yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -113,7 +113,10 @@
else:
raise RuntimeError
-# Inclined Box
+# Now our visitor functions
+
+cdef void visit_count_octs(Oct *o, OctVisitorData *data):
+ data.index += 1
cdef class SelectorObject:
@@ -229,11 +232,12 @@
def count_octs(self, OctreeContainer octree):
cdef int i, j, k, n
cdef np.float64_t pos[3], dds[3]
- cdef np.int64_t count = 0
# This dds is the oct-width
for i in range(3):
dds[i] = (octree.DRE[i] - octree.DLE[i]) / octree.nn[i]
# Pos is the center of the octs
+ cdef OctVisitorData data
+ data.index = 0
pos[0] = octree.DLE[0] + dds[0]/2.0
for i in range(octree.nn[0]):
pos[1] = octree.DLE[1] + dds[1]/2.0
@@ -241,20 +245,22 @@
pos[2] = octree.DLE[2] + dds[2]/2.0
for k in range(octree.nn[2]):
if octree.root_mesh[i][j][k] == NULL: continue
- self.recursively_count_octs(
+ self.recursively_visit_octs(
octree.root_mesh[i][j][k],
- pos, dds, 0, &count)
+ pos, dds, 0, visit_count_octs, &data)
pos[2] += dds[2]
pos[1] += dds[1]
pos[0] += dds[0]
- return count
+ return data.index
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- cdef void recursively_count_octs(self, Oct *root,
+ cdef void recursively_visit_octs(self, Oct *root,
np.float64_t pos[3], np.float64_t dds[3],
- int level, np.int64_t *count):
+ int level,
+ oct_visitor_function *func,
+ OctVisitorData *data):
cdef np.float64_t LE[3], RE[3], sdds[3], spos[3]
cdef int i, j, k, res, ii
cdef Oct *ch
@@ -291,10 +297,11 @@
ii = ((k*2)+j)*2+i
ch = root.children[i][j][k]
if next_level == 1 and ch != NULL:
- self.recursively_count_octs(
- ch, spos, sdds, level + 1, count)
- elif this_level == 1:
- count[0] += self.select_cell(spos, sdds, eterm)
+ self.recursively_visit_octs(
+ ch, spos, sdds, level + 1, func, data)
+ elif this_level == 1 and self.select_cell(
+ spos, sdds, eterm):
+ func(ch, data)
spos[2] += sdds[2]
spos[1] += sdds[1]
spos[0] += sdds[0]
https://bitbucket.org/yt_analysis/yt-3.0/commits/c29ef8057b0d/
Changeset: c29ef8057b0d
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-14 21:33:06
Summary: Convert icoords to visitor function.
Affected #: 5 files
diff -r fc0e492c026567482fc6d20f786bfbe0255db89e -r c29ef8057b0def006df90b6d629a2d27cc0baca8 yt/geometry/oct_container.pxd
--- a/yt/geometry/oct_container.pxd
+++ b/yt/geometry/oct_container.pxd
@@ -25,6 +25,8 @@
cimport numpy as np
from fp_utils cimport *
+from selection_routines cimport SelectorObject, \
+ OctVisitorData, oct_visitor_function
cdef int ORDER_MAX
@@ -67,8 +69,15 @@
# This function must return the offset from global-to-local domains; i.e.,
# OctAllocationContainer.offset if such a thing exists.
cdef np.int64_t get_domain_offset(self, int domain_id)
+ cdef void visit_all_octs(self, SelectorObject selector,
+ oct_visitor_function *func,
+ OctVisitorData *data)
cdef class RAMSESOctreeContainer(OctreeContainer):
cdef OctAllocationContainer **domains
cdef Oct *next_root(self, int domain_id, int ind[3])
cdef Oct *next_child(self, int domain_id, int ind[3], Oct *parent)
+
+# Now some visitor functions
+
+cdef void visit_icoords_octs(Oct *o, OctVisitorData *data)
diff -r fc0e492c026567482fc6d20f786bfbe0255db89e -r c29ef8057b0def006df90b6d629a2d27cc0baca8 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -31,7 +31,8 @@
import numpy as np
from oct_container cimport Oct, OctAllocationContainer, \
OctreeContainer, ORDER_MAX
-from selection_routines cimport SelectorObject
+from selection_routines cimport SelectorObject, \
+ OctVisitorData, oct_visitor_function
cimport cython
ORDER_MAX = 20
@@ -137,6 +138,29 @@
yield (this.file_ind, this.domain_ind, this.domain)
cur = cur.next
+ cdef void visit_all_octs(self, SelectorObject selector,
+ oct_visitor_function *func,
+ OctVisitorData *data):
+ cdef int i, j, k, n
+ cdef np.float64_t pos[3], dds[3]
+ # This dds is the oct-width
+ for i in range(3):
+ dds[i] = (self.DRE[i] - self.DLE[i]) / self.nn[i]
+ # Pos is the center of the octs
+ pos[0] = self.DLE[0] + dds[0]/2.0
+ for i in range(self.nn[0]):
+ pos[1] = self.DLE[1] + dds[1]/2.0
+ for j in range(self.nn[1]):
+ pos[2] = self.DLE[2] + dds[2]/2.0
+ for k in range(self.nn[2]):
+ if self.root_mesh[i][j][k] == NULL: continue
+ selector.recursively_visit_octs(
+ self.root_mesh[i][j][k],
+ pos, dds, 0, func, data)
+ pos[2] += dds[2]
+ pos[1] += dds[1]
+ pos[0] += dds[0]
+
cdef void oct_bounds(self, Oct *o, np.float64_t *corner, np.float64_t *size):
cdef int i
for i in range(3):
@@ -804,17 +828,11 @@
local_filled += 1
return local_filled
+# Now some visitor functions
-cdef int compare_octs(void *vo1, void *vo2) nogil:
- #This only compares if the octs live on the
- #domain, not if they are actually equal
- #Used to sort octs into consecutive domains
- cdef Oct *o1 = (<Oct**> vo1)[0]
- cdef Oct *o2 = (<Oct**> vo2)[0]
- if o1.domain < o2.domain: return -1
- elif o1.domain == o2.domain:
- if o1.level < o2.level: return -1
- if o1.level > o2.level: return 1
- else: return 0
- elif o1.domain > o2.domain: return 1
-
+cdef void visit_icoords_octs(Oct *o, OctVisitorData *data):
+ cdef np.int64_t *coords = <np.int64_t*> data.array
+ cdef int i
+ for i in range(3):
+ coords[data.index * 3 + i] = (o.pos[i] << 1) + data.ind[i]
+ data.index += 1
diff -r fc0e492c026567482fc6d20f786bfbe0255db89e -r c29ef8057b0def006df90b6d629a2d27cc0baca8 yt/geometry/particle_oct_container.pyx
--- a/yt/geometry/particle_oct_container.pyx
+++ b/yt/geometry/particle_oct_container.pyx
@@ -25,15 +25,15 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
-from oct_container cimport OctreeContainer, Oct, OctInfo
+from oct_container cimport OctreeContainer, Oct, OctInfo, \
+ visit_icoords_octs, ORDER_MAX
from libc.stdlib cimport malloc, free, qsort
from libc.math cimport floor
from fp_utils cimport *
cimport numpy as np
import numpy as np
-from oct_container cimport Oct, OctAllocationContainer, \
- OctreeContainer, ORDER_MAX
-from selection_routines cimport SelectorObject
+from selection_routines cimport SelectorObject, \
+ OctVisitorData, oct_visitor_function
cimport cython
cdef class ParticleOctreeContainer(OctreeContainer):
@@ -114,33 +114,11 @@
num_cells = selector.count_octs(self)
cdef np.ndarray[np.int64_t, ndim=2] coords
coords = np.empty((num_cells, 3), dtype="int64")
- cdef int oi, i, ci, ii, eterm[3]
- ci = 0
+ cdef OctVisitorData data
+ data.array = <void *> coords.data
+ data.index = 0
cdef np.float64_t left_edge[3], right_edge[3], dds[3]
- for oi in range(self.nocts):
- o = self.oct_list[oi]
- #if o.domain != domain_id: continue
- if o.children[0][0][0] != NULL: continue
- self.oct_bounds(o, left_edge, dds)
- for i in range(3): right_edge[i] = left_edge[i] + dds[i]
- if not selector.select_grid(left_edge, right_edge, o.level):
- continue
- for i in range(3): # Set up cell info
- dds[i] /= 2.0
- right_edge[0] = left_edge[0] + dds[0]/2.0
- for i in range(2):
- right_edge[1] = left_edge[1] + dds[1]/2.0
- for j in range(2):
- right_edge[2] = left_edge[2] + dds[2]/2.0
- for k in range(2):
- if selector.select_cell(right_edge, dds, eterm) == 1:
- coords[ci, 0] = (o.pos[0] << 1) + i
- coords[ci, 1] = (o.pos[1] << 1) + j
- coords[ci, 2] = (o.pos[2] << 1) + k
- ci += 1
- right_edge[2] += dds[2]
- right_edge[1] += dds[1]
- right_edge[0] += dds[0]
+ self.visit_all_octs(selector, visit_icoords_octs, &data)
return coords
@cython.boundscheck(False)
diff -r fc0e492c026567482fc6d20f786bfbe0255db89e -r c29ef8057b0def006df90b6d629a2d27cc0baca8 yt/geometry/selection_routines.pxd
--- a/yt/geometry/selection_routines.pxd
+++ b/yt/geometry/selection_routines.pxd
@@ -28,6 +28,7 @@
cdef struct Oct
cdef struct OctVisitorData:
np.uint64_t index
+ int ind[3]
void *array
ctypedef void oct_visitor_function(Oct *, OctVisitorData *visitor)
diff -r fc0e492c026567482fc6d20f786bfbe0255db89e -r c29ef8057b0def006df90b6d629a2d27cc0baca8 yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -230,27 +230,9 @@
spos[0] += sdds[0]
def count_octs(self, OctreeContainer octree):
- cdef int i, j, k, n
- cdef np.float64_t pos[3], dds[3]
- # This dds is the oct-width
- for i in range(3):
- dds[i] = (octree.DRE[i] - octree.DLE[i]) / octree.nn[i]
- # Pos is the center of the octs
cdef OctVisitorData data
data.index = 0
- pos[0] = octree.DLE[0] + dds[0]/2.0
- for i in range(octree.nn[0]):
- pos[1] = octree.DLE[1] + dds[1]/2.0
- for j in range(octree.nn[1]):
- pos[2] = octree.DLE[2] + dds[2]/2.0
- for k in range(octree.nn[2]):
- if octree.root_mesh[i][j][k] == NULL: continue
- self.recursively_visit_octs(
- octree.root_mesh[i][j][k],
- pos, dds, 0, visit_count_octs, &data)
- pos[2] += dds[2]
- pos[1] += dds[1]
- pos[0] += dds[0]
+ octree.visit_all_octs(self, visit_count_octs, &data)
return data.index
@cython.boundscheck(False)
@@ -301,7 +283,10 @@
ch, spos, sdds, level + 1, func, data)
elif this_level == 1 and self.select_cell(
spos, sdds, eterm):
- func(ch, data)
+ data.ind[0] = i
+ data.ind[1] = j
+ data.ind[2] = k
+ func(root, data)
spos[2] += sdds[2]
spos[1] += sdds[1]
spos[0] += sdds[0]
https://bitbucket.org/yt_analysis/yt-3.0/commits/8181e28e4064/
Changeset: 8181e28e4064
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-14 22:05:21
Summary: Converted ires and fcoords to visitor functions.
Affected #: 5 files
diff -r c29ef8057b0def006df90b6d629a2d27cc0baca8 -r 8181e28e4064a64a688e345e469f842d6423e646 yt/frontends/sph/io.py
--- a/yt/frontends/sph/io.py
+++ b/yt/frontends/sph/io.py
@@ -106,6 +106,7 @@
regions.add_data_file(pos, data_file.file_id)
pos = np.floor((pos - DLE)/dx).astype("uint64")
morton[ind:ind+pos.shape[0]] = get_morton_indices(pos)
+ ind += pos.shape[0]
f.close()
return morton
diff -r c29ef8057b0def006df90b6d629a2d27cc0baca8 -r 8181e28e4064a64a688e345e469f842d6423e646 yt/geometry/oct_container.pxd
--- a/yt/geometry/oct_container.pxd
+++ b/yt/geometry/oct_container.pxd
@@ -81,3 +81,5 @@
# Now some visitor functions
cdef void visit_icoords_octs(Oct *o, OctVisitorData *data)
+cdef void visit_ires_octs(Oct *o, OctVisitorData *data)
+cdef void visit_fcoords_octs(Oct *o, OctVisitorData *data)
diff -r c29ef8057b0def006df90b6d629a2d27cc0baca8 -r 8181e28e4064a64a688e345e469f842d6423e646 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -836,3 +836,21 @@
for i in range(3):
coords[data.index * 3 + i] = (o.pos[i] << 1) + data.ind[i]
data.index += 1
+
+cdef void visit_ires_octs(Oct *o, OctVisitorData *data):
+ cdef np.int64_t *ires = <np.int64_t*> data.array
+ ires[data.index] = o.level
+ data.index += 1
+
+cdef void visit_fcoords_octs(Oct *o, OctVisitorData *data):
+ # Note that this does not actually give the correct floating point
+ # coordinates. It gives them in some unit system where the domain is 1.0
+ # in all directions, and assumes that they will be scaled later.
+ cdef np.float64_t *fcoords = <np.float64_t*> data.array
+ cdef int i
+ cdef np.float64_t c, dx
+ dx = 1.0 / (2 << o.level)
+ for i in range(3):
+ c = <np.float64_t> ((o.pos[i] << 1 ) + data.ind[i])
+ fcoords[data.index * 3 + i] = (c + 0.5) * dx
+ data.index += 1
diff -r c29ef8057b0def006df90b6d629a2d27cc0baca8 -r 8181e28e4064a64a688e345e469f842d6423e646 yt/geometry/particle_geometry_handler.py
--- a/yt/geometry/particle_geometry_handler.py
+++ b/yt/geometry/particle_geometry_handler.py
@@ -116,6 +116,7 @@
npart = sum(data_file.total_particles.values())
morton[ind:ind + npart] = \
self.io._initialize_index(data_file, self.regions)
+ ind += npart
morton.sort()
# Now we add them all at once.
self.oct_handler.add(morton)
diff -r c29ef8057b0def006df90b6d629a2d27cc0baca8 -r 8181e28e4064a64a688e345e469f842d6423e646 yt/geometry/particle_oct_container.pyx
--- a/yt/geometry/particle_oct_container.pyx
+++ b/yt/geometry/particle_oct_container.pyx
@@ -25,8 +25,8 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
-from oct_container cimport OctreeContainer, Oct, OctInfo, \
- visit_icoords_octs, ORDER_MAX
+from oct_container cimport OctreeContainer, Oct, OctInfo, ORDER_MAX, \
+ visit_icoords_octs, visit_ires_octs, visit_fcoords_octs
from libc.stdlib cimport malloc, free, qsort
from libc.math cimport floor
from fp_utils cimport *
@@ -117,75 +117,43 @@
cdef OctVisitorData data
data.array = <void *> coords.data
data.index = 0
- cdef np.float64_t left_edge[3], right_edge[3], dds[3]
self.visit_all_octs(selector, visit_icoords_octs, &data)
return coords
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def ires(self, int domain_id,
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
- np.int64_t cell_count,
- np.ndarray[np.int64_t, ndim=1] level_counts):
+ def ires(self, SelectorObject selector, np.uint64_t num_cells = -1):
+ if num_cells == -1:
+ num_cells = selector.count_octs(self)
#Return the 'resolution' of each cell; ie the level
cdef np.ndarray[np.int64_t, ndim=1] res
- res = np.empty(cell_count, dtype="int64")
- cdef int oi, i, ci
- ci = 0
- for oi in range(self.nocts):
- o = self.oct_list[oi]
- if o.domain != domain_id: continue
- for i in range(8):
- if mask[oi, i] == 1:
- res[ci] = o.level
- ci += 1
+ res = np.empty(num_cells, dtype="int64")
+ cdef OctVisitorData data
+ data.array = <void *> res.data
+ data.index = 0
+ self.visit_all_octs(selector, visit_ires_octs, &data)
return res
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def fcoords(self, int domain_id,
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
- np.int64_t cell_count,
- np.ndarray[np.int64_t, ndim=1] level_counts):
+ def fcoords(self, SelectorObject selector, np.uint64_t num_cells = -1):
+ if num_cells == -1:
+ num_cells = selector.count_octs(self)
#Return the floating point unitary position of every cell
cdef np.ndarray[np.float64_t, ndim=2] coords
- coords = np.empty((cell_count, 3), dtype="float64")
- cdef int oi, i, ci
- cdef np.float64_t base_dx[3], dx[3], pos[3]
+ coords = np.empty((num_cells, 3), dtype="float64")
+ cdef OctVisitorData data
+ data.array = <void *> coords.data
+ data.index = 0
+ self.visit_all_octs(selector, visit_fcoords_octs, &data)
+ cdef int i
+ cdef np.float64_t base_dx
for i in range(3):
- # This is the base_dx, but not the base distance from the center
- # position. Note that the positions will also all be offset by
- # dx/2.0. This is also for *oct grids*, not cells.
- base_dx[i] = (self.DRE[i] - self.DLE[i])/self.nn[i]
- ci = 0
- cdef int proc
- for oi in range(self.nocts):
- proc = 0
- for i in range(8):
- if mask[oi, i] == 1:
- proc = 1
- break
- if proc == 0: continue
- o = self.oct_list[oi]
- if o.domain != domain_id: continue
- for i in range(3):
- # This gives the *grid* width for this level
- dx[i] = base_dx[i] / (1 << o.level)
- # o.pos is the *grid* index, so pos[i] is the center of the
- # first cell in the grid
- pos[i] = self.DLE[i] + o.pos[i]*dx[i] + dx[i]/4.0
- dx[i] = dx[i] / 2.0 # This is now the *offset*
- for i in range(2):
- for j in range(2):
- for k in range(2):
- ii = ((k*2)+j)*2+i
- if mask[oi, ii] == 0: continue
- coords[ci, 0] = pos[0] + dx[0] * i
- coords[ci, 1] = pos[1] + dx[1] * j
- coords[ci, 2] = pos[2] + dx[2] * k
- ci += 1
+ base_dx = (self.DRE[i] - self.DLE[i])/self.nn[i]
+ coords[:,i] *= base_dx
+ coords[:,i] += self.DLE[i]
return coords
def allocate_domains(self, domain_counts):
https://bitbucket.org/yt_analysis/yt-3.0/commits/e4f6828237f1/
Changeset: e4f6828237f1
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-14 22:09:34
Summary: Remove currently-unused functions.
Affected #: 1 file
diff -r 8181e28e4064a64a688e345e469f842d6423e646 -r e4f6828237f1dc49c3331b7415e4df0099d89af7 yt/geometry/particle_oct_container.pyx
--- a/yt/geometry/particle_oct_container.pyx
+++ b/yt/geometry/particle_oct_container.pyx
@@ -216,26 +216,6 @@
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def count_levels(self, int max_level, int domain_id,
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask):
- cdef np.ndarray[np.int64_t, ndim=1] level_count
- cdef Oct *o
- cdef int oi, i
- level_count = np.zeros(max_level+1, 'int64')
- cdef np.int64_t ndo, doff
- ndo = self.dom_offsets[domain_id + 2] \
- - self.dom_offsets[domain_id + 1]
- doff = self.dom_offsets[domain_id + 1]
- for oi in range(ndo):
- o = self.oct_list[oi + doff]
- for i in range(8):
- if mask[o.domain_ind, i] == 0: continue
- level_count[o.level] += 1
- return level_count
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
def add(self, np.ndarray[np.uint64_t, ndim=1] indices):
#Add this particle to the root oct
#Then if that oct has children, add it to them recursively
@@ -334,107 +314,6 @@
self.visit(o.children[i][j][k], counts, level + 1)
return
- def domain_identify(self, np.ndarray[np.uint8_t, ndim=2, cast=True] mask):
- #Return an array of length # of domains
- #Every element is True if there is at least one
- #fully refined *cell* in that domain that isn't masked out
- cdef int i, oi, m
- cdef Oct *o
- cdef np.ndarray[np.uint8_t, ndim=1, cast=True] dmask
- dmask = np.zeros(self.max_domain+1, dtype='uint8')
- for oi in range(self.nocts):
- m = 0
- o = self.oct_list[oi]
- #if o.sd.np <= 0 or o.domain == -1: continue
- for i in range(8):
- if mask[oi, i] == 1:
- m = 1
- break
- if m == 0: continue
- dmask[o.domain] = 1
- return dmask.astype("bool")
-
- def domain_and(self, np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
- int domain_id):
- cdef np.int64_t i, oi, n, use
- cdef Oct *o
- cdef np.ndarray[np.uint8_t, ndim=2] m2 = \
- np.zeros((mask.shape[0], 8), 'uint8')
- n = mask.shape[0]
- for oi in range(n):
- o = self.oct_list[oi]
- if o.domain != domain_id: continue
- use = 0
- for i in range(8):
- m2[o.domain_ind, i] = mask[o.domain_ind, i]
- return m2
-
- def domain_mask(self,
- # mask is the base selector's *global* mask
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
- int domain_id):
- # What distinguishes this one from domain_and is that we have a mask,
- # which covers the whole domain, but our output will only be of a much
- # smaller subset of octs that belong to a given domain *and* the mask.
- # Note also that typically when something calls domain_and, they will
- # use a logical_any along the oct axis. Here we don't do that.
- # Note also that we change the shape of the returned array.
- cdef np.int64_t i, j, k, oi, n, nm, use
- cdef Oct *o
- n = mask.shape[0]
- nm = 0
- # This could perhaps be faster if we
- for oi in range(n):
- o = self.oct_list[oi]
- if o.domain != domain_id: continue
- use = 0
- for i in range(8):
- if mask[o.domain_ind, i] == 1: use = 1
- nm += use
- cdef np.ndarray[np.uint8_t, ndim=4] m2 = \
- np.zeros((2, 2, 2, nm), 'uint8')
- nm = 0
- for oi in range(n):
- o = self.oct_list[oi]
- if o.domain != domain_id: continue
- use = 0
- for i in range(2):
- for j in range(2):
- for k in range(2):
- ii = ((k*2)+j)*2+i
- if mask[o.domain_ind, ii] == 0: continue
- use = m2[i, j, k, nm] = 1
- nm += use
- return m2.astype("bool")
-
- def domain_ind(self,
- # mask is the base selector's *global* mask
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
- int domain_id):
- # Here we once again do something similar to the other functions. We
- # need a set of indices into the final reduced, masked values. The
- # indices will be domain.n long, and will be of type int64. This way,
- # we can get the Oct through a .get() call, then use Oct.file_ind as an
- # index into this newly created array, then finally use the returned
- # index into the domain subset array for deposition.
- cdef np.int64_t i, j, k, oi, noct, n, nm, use, offset
- cdef Oct *o
- # For particle octrees, domain 0 is special and means non-leaf nodes.
- offset = self.dom_offsets[domain_id + 1]
- noct = self.dom_offsets[domain_id + 2] - offset
- cdef np.ndarray[np.int64_t, ndim=1] ind = np.zeros(noct, 'int64')
- nm = 0
- for oi in range(noct):
- ind[oi] = -1
- o = self.oct_list[oi + offset]
- use = 0
- for i in range(8):
- if mask[o.domain_ind, i] == 1: use = 1
- if use == 1:
- ind[oi] = nm
- nm += use
- return ind
-
cdef class ParticleRegions:
cdef np.float64_t left_edge[3]
cdef np.float64_t dds[3]
https://bitbucket.org/yt_analysis/yt-3.0/commits/db7f3802fc2e/
Changeset: db7f3802fc2e
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-14 23:41:57
Summary: Starting process of creating ParticleOctreeSubset selector.
This is the start of re-enabling spatial fields for particle codes.
Affected #: 6 files
diff -r e4f6828237f1dc49c3331b7415e4df0099d89af7 -r db7f3802fc2eafc45bbd99b1b0f3095ff05d42ee yt/data_objects/octree_subset.py
--- a/yt/data_objects/octree_subset.py
+++ b/yt/data_objects/octree_subset.py
@@ -168,3 +168,74 @@
def select_particles(self, selector, x, y, z):
mask = selector.select_points(x,y,z)
return mask
+
+class ParticleOctreeSubset(OctreeSubset):
+ # Subclassing OctreeSubset is somewhat dubious.
+ # This is some subset of an octree. Note that the sum of subsets of an
+ # octree may multiply include data files. While we can attempt to mitigate
+ # this, it's unavoidable for many types of data storage on disk.
+ _type_name = 'particle_octree_subset'
+ _con_args = ('data_files', 'pf', 'min_ind', 'max_ind')
+ def __init__(self, data_files, pf, min_ind = 0, max_ind = 0):
+ # The first attempt at this will not work in parallel.
+ self.data_files = data_files
+ self.field_data = YTFieldData()
+ self.field_parameters = {}
+ self.pf = pf
+ self.hierarchy = self.pf.hierarchy
+ self.oct_handler = pf.h.oct_handler
+ self.min_ind = min_ind
+ self.max_ind = max_ind
+ if max_ind == 0: max_ind = (1 << 63)
+ self._last_mask = None
+ self._last_selector_id = None
+ self._current_particle_type = 'all'
+ self._current_fluid_type = self.pf.default_fluid_type
+
+ def select_icoords(self, dobj):
+ return self.oct_handler.icoords(dobj)
+
+ def select_fcoords(self, dobj):
+ return self.oct_handler.fcoords(dobj)
+
+ def select_fwidth(self, dobj):
+ # Recall domain_dimensions is the number of cells, not octs
+ base_dx = (self.domain.pf.domain_width /
+ self.domain.pf.domain_dimensions)
+ widths = np.empty((self.cell_count, 3), dtype="float64")
+ dds = (2**self.select_ires(dobj))
+ for i in range(3):
+ widths[:,i] = base_dx[i] / dds
+ return widths
+
+ def select_ires(self, dobj):
+ return self.oct_handler.ires(dobj)
+
+ def select(self, selector):
+ if id(selector) == self._last_selector_id:
+ return self._last_mask
+ m1 = self.selector.select_octs(self.oct_handler)
+ m2 = selector.select_octs(self.oct_handler)
+ np.logical_and(m1, m2, m1)
+ del m2
+ self._last_mask = m1
+ if self._last_mask.sum() == 0: return None
+ self._last_selector_id = id(selector)
+ return self._last_mask
+
+ def count(self, selector):
+ if id(selector) == self._last_selector_id:
+ if self._last_mask is None: return 0
+ return self._last_mask.sum()
+ self.select(selector)
+ return self.count(selector)
+
+ def count_particles(self, selector, x, y, z):
+ # We don't cache the selector results
+ count = selector.count_points(x,y,z)
+ return count
+
+ def select_particles(self, selector, x, y, z):
+ mask = selector.select_points(x,y,z)
+ return mask
+
diff -r e4f6828237f1dc49c3331b7415e4df0099d89af7 -r db7f3802fc2eafc45bbd99b1b0f3095ff05d42ee yt/data_objects/universal_fields.py
--- a/yt/data_objects/universal_fields.py
+++ b/yt/data_objects/universal_fields.py
@@ -96,7 +96,7 @@
display_field = False)
def _Ones(field, data):
- return np.ones(data.ActiveDimensions, dtype='float64')
+ return np.ones(data.ires.shape, dtype='float64')
add_field("Ones", function=_Ones,
projection_conversion="unitary",
display_field = False)
diff -r e4f6828237f1dc49c3331b7415e4df0099d89af7 -r db7f3802fc2eafc45bbd99b1b0f3095ff05d42ee yt/frontends/sph/data_structures.py
--- a/yt/frontends/sph/data_structures.py
+++ b/yt/frontends/sph/data_structures.py
@@ -59,10 +59,6 @@
from yt.data_objects.field_info_container import \
FieldInfoContainer, NullFunc
-class ParticleOctreeSubset(object):
- def __init__(self):
- pass
-
class ParticleFile(object):
def __init__(self, pf, io, filename, file_id):
self.pf = pf
diff -r e4f6828237f1dc49c3331b7415e4df0099d89af7 -r db7f3802fc2eafc45bbd99b1b0f3095ff05d42ee yt/frontends/sph/io.py
--- a/yt/frontends/sph/io.py
+++ b/yt/frontends/sph/io.py
@@ -52,16 +52,24 @@
chunks = list(chunks)
for ftype, fname in fields:
ptf[ftype].append(fname)
- for chunk in chunks: # Will be OWLS domains
- for data_file in chunk.objs:
- f = h5py.File(data_file.filename, "r")
- # This double-reads
- for ptype, field_list in sorted(ptf.items()):
- coords = f["/%s/Coordinates" % ptype][:].astype("float64")
- psize[ptype] += selector.count_points(
- coords[:,0], coords[:,1], coords[:,2])
- del coords
- f.close()
+ # For this type of file, we actually have something slightly different.
+ # We are given a list of ParticleDataChunks, which is composed of
+ # individual ParticleOctreeSubsets. The data_files attribute on these
+ # may in fact overlap. So we will iterate over a union of all the
+ # data_files.
+ data_files = set([])
+ for chunk in chunks:
+ for obj in chunk.objs:
+ data_files.update(obj.data_files)
+ for data_file in data_files:
+ f = h5py.File(data_file.filename, "r")
+ # This double-reads
+ for ptype, field_list in sorted(ptf.items()):
+ coords = f["/%s/Coordinates" % ptype][:].astype("float64")
+ psize[ptype] += selector.count_points(
+ coords[:,0], coords[:,1], coords[:,2])
+ del coords
+ f.close()
# Now we have all the sizes, and we can allocate
ind = {}
for field in fields:
@@ -72,24 +80,23 @@
shape = psize[field[0]]
rv[field] = np.empty(shape, dtype="float64")
ind[field] = 0
- for chunk in chunks: # Will be OWLS domains
- for data_file in chunk.objs:
- f = h5py.File(data_file.filename, "r")
- for ptype, field_list in sorted(ptf.items()):
- g = f["/%s" % ptype]
- coords = g["Coordinates"][:].astype("float64")
- mask = selector.select_points(
- coords[:,0], coords[:,1], coords[:,2])
- del coords
- if mask is None: continue
- for field in field_list:
- data = g[field][:][mask,...]
- my_ind = ind[ptype, field]
- mylog.debug("Filling from %s to %s with %s",
- my_ind, my_ind+data.shape[0], field)
- rv[ptype, field][my_ind:my_ind + data.shape[0],...] = data
- ind[ptype, field] += data.shape[0]
- f.close()
+ for data_file in data_files:
+ f = h5py.File(data_file.filename, "r")
+ for ptype, field_list in sorted(ptf.items()):
+ g = f["/%s" % ptype]
+ coords = g["Coordinates"][:].astype("float64")
+ mask = selector.select_points(
+ coords[:,0], coords[:,1], coords[:,2])
+ del coords
+ if mask is None: continue
+ for field in field_list:
+ data = g[field][:][mask,...]
+ my_ind = ind[ptype, field]
+ mylog.debug("Filling from %s to %s with %s",
+ my_ind, my_ind+data.shape[0], field)
+ rv[ptype, field][my_ind:my_ind + data.shape[0],...] = data
+ ind[ptype, field] += data.shape[0]
+ f.close()
return rv
def _initialize_index(self, data_file, regions):
diff -r e4f6828237f1dc49c3331b7415e4df0099d89af7 -r db7f3802fc2eafc45bbd99b1b0f3095ff05d42ee yt/geometry/particle_geometry_handler.py
--- a/yt/geometry/particle_geometry_handler.py
+++ b/yt/geometry/particle_geometry_handler.py
@@ -44,6 +44,7 @@
ParallelAnalysisInterface, parallel_splitter
from yt.data_objects.data_containers import data_object_registry
+from yt.data_objects.octree_subset import ParticleOctreeSubset
class ParticleGeometryHandler(GeometryHandler):
_global_mesh = False
@@ -141,9 +142,10 @@
def _identify_base_chunk(self, dobj):
if getattr(dobj, "_chunk_info", None) is None:
- mask = dobj.selector.select_octs(self.oct_handler)
file_ids = self.regions.identify_data_files(dobj.selector)
- dobj._chunk_info = [self.data_files[i] for i in file_ids]
+ subset = [ParticleOctreeSubset([self.data_files[i] for i in file_ids],
+ self.parameter_file)]
+ dobj._chunk_info = subset
dobj._current_chunk = list(self._chunk_all(dobj))[0]
def _chunk_all(self, dobj):
@@ -152,15 +154,17 @@
def _chunk_spatial(self, dobj, ngz, sort = None):
sobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
- for i,og in enumerate(sobjs):
- if ngz > 0:
- g = og.retrieve_ghost_zones(ngz, [], smoothed=True)
- else:
- g = og
- size = og.cell_count
- if size == 0: continue
- yield ParticleDataChunk(self.oct_handler, self.regions,
- dobj, "spatial", [g], size)
+ # We actually do not really use the data files except as input to the
+ # ParticleOctreeSubset.
+ # This is where we will perform cutting of the Octree and
+ # load-balancing. That may require a specialized selector object to
+ # cut based on some space-filling curve index.
+ osubset = ParticleOctreeSubset(sobjs, self.parameter_file)
+
+ if ngz > 0:
+ raise NotImplementedError
+ yield ParticleDataChunk(self.oct_handler, self.regions,
+ dobj, "spatial", [osubset])
def _chunk_io(self, dobj):
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
@@ -175,5 +179,6 @@
super(ParticleDataChunk, self).__init__(*args, **kwargs)
def _accumulate_values(self, method):
- mfunc = getattr(self.oct_handler, "select_%s" % method)
- return mfunc(self.dobj)
+ mfunc = getattr(self.oct_handler, method)
+ return mfunc(self.dobj.selector)
+
diff -r e4f6828237f1dc49c3331b7415e4df0099d89af7 -r db7f3802fc2eafc45bbd99b1b0f3095ff05d42ee yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -1213,3 +1213,50 @@
octree_subset_selector = OctreeSubsetSelector
+cdef class ParticleOctreeSubsetSelector(SelectorObject):
+ # This is a numpy array, which will be a bool of ndim 1
+ cdef np.uint64_t min_ind
+ cdef np.uint64_t max_ind
+
+ def __init__(self, dobj):
+ self.min_ind = dobj.min_ind
+ self.max_ind = dobj.max_ind
+
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ def select_octs(self, OctreeContainer octree):
+ # There has to be a better way to do this.
+ cdef np.ndarray[np.uint8_t, ndim=2, cast=True] m2
+ m2 = np.ones((octree.nocts, 8), dtype="uint8")
+ # This is where we'll -- in the future -- cut up based on indices of
+ # the octs.
+ return m2.astype("bool")
+
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ cdef void set_bounds(self,
+ np.float64_t left_edge[3], np.float64_t right_edge[3],
+ np.float64_t dds[3], int ind[3][2], int *check):
+ check[0] = 0
+ return
+
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ def select_grids(self,
+ np.ndarray[np.float64_t, ndim=2] left_edges,
+ np.ndarray[np.float64_t, ndim=2] right_edges,
+ np.ndarray[np.int32_t, ndim=2] levels):
+ raise RuntimeError
+
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ cdef int select_cell(self, np.float64_t pos[3], np.float64_t dds[3],
+ int eterm[3]) nogil:
+ return 1
+
+particle_octree_subset_selector = ParticleOctreeSubsetSelector
+
https://bitbucket.org/yt_analysis/yt-3.0/commits/7ce103b71f24/
Changeset: 7ce103b71f24
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-15 00:02:11
Summary: Getting closer -- but data.ires in Ones is breaking.
Affected #: 1 file
diff -r db7f3802fc2eafc45bbd99b1b0f3095ff05d42ee -r 7ce103b71f24ae606469fc050269296fee81a599 yt/geometry/particle_geometry_handler.py
--- a/yt/geometry/particle_geometry_handler.py
+++ b/yt/geometry/particle_geometry_handler.py
@@ -142,9 +142,8 @@
def _identify_base_chunk(self, dobj):
if getattr(dobj, "_chunk_info", None) is None:
- file_ids = self.regions.identify_data_files(dobj.selector)
- subset = [ParticleOctreeSubset([self.data_files[i] for i in file_ids],
- self.parameter_file)]
+ data_files = getattr(dobj, "data_files", self.data_files)
+ subset = [ParticleOctreeSubset(data_files, self.parameter_file)]
dobj._chunk_info = subset
dobj._current_chunk = list(self._chunk_all(dobj))[0]
@@ -159,12 +158,13 @@
# This is where we will perform cutting of the Octree and
# load-balancing. That may require a specialized selector object to
# cut based on some space-filling curve index.
- osubset = ParticleOctreeSubset(sobjs, self.parameter_file)
-
- if ngz > 0:
- raise NotImplementedError
- yield ParticleDataChunk(self.oct_handler, self.regions,
- dobj, "spatial", [osubset])
+ for i,og in enumerate(sobjs):
+ if ngz > 0:
+ g = og.retrieve_ghost_zones(ngz, [], smoothed=True)
+ else:
+ g = og
+ yield ParticleDataChunk(self.oct_handler, self.regions, dobj,
+ "spatial", [g])
def _chunk_io(self, dobj):
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
https://bitbucket.org/yt_analysis/yt-3.0/commits/209dd30d499e/
Changeset: 209dd30d499e
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-15 00:12:24
Summary: Adding an "fwidth" function.
Affected #: 3 files
diff -r 7ce103b71f24ae606469fc050269296fee81a599 -r 209dd30d499e9c59bd06e3c2f924591a575c9fa7 yt/geometry/oct_container.pxd
--- a/yt/geometry/oct_container.pxd
+++ b/yt/geometry/oct_container.pxd
@@ -83,3 +83,4 @@
cdef void visit_icoords_octs(Oct *o, OctVisitorData *data)
cdef void visit_ires_octs(Oct *o, OctVisitorData *data)
cdef void visit_fcoords_octs(Oct *o, OctVisitorData *data)
+cdef void visit_fwidth_octs(Oct *o, OctVisitorData *data)
diff -r 7ce103b71f24ae606469fc050269296fee81a599 -r 209dd30d499e9c59bd06e3c2f924591a575c9fa7 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -854,3 +854,15 @@
c = <np.float64_t> ((o.pos[i] << 1 ) + data.ind[i])
fcoords[data.index * 3 + i] = (c + 0.5) * dx
data.index += 1
+
+cdef void visit_fwidth_octs(Oct *o, OctVisitorData *data):
+ # Note that this does not actually give the correct floating point
+ # coordinates. It gives them in some unit system where the domain is 1.0
+ # in all directions, and assumes that they will be scaled later.
+ cdef np.float64_t *fwidth = <np.float64_t*> data.array
+ cdef int i
+ cdef np.float64_t dx
+ dx = 1.0 / (2 << o.level)
+ for i in range(3):
+ fwidth[data.index * 3 + i] = dx
+ data.index += 1
diff -r 7ce103b71f24ae606469fc050269296fee81a599 -r 209dd30d499e9c59bd06e3c2f924591a575c9fa7 yt/geometry/particle_oct_container.pyx
--- a/yt/geometry/particle_oct_container.pyx
+++ b/yt/geometry/particle_oct_container.pyx
@@ -26,7 +26,8 @@
"""
from oct_container cimport OctreeContainer, Oct, OctInfo, ORDER_MAX, \
- visit_icoords_octs, visit_ires_octs, visit_fcoords_octs
+ visit_icoords_octs, visit_ires_octs, \
+ visit_fcoords_octs, visit_fwidth_octs
from libc.stdlib cimport malloc, free, qsort
from libc.math cimport floor
from fp_utils cimport *
@@ -138,6 +139,24 @@
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
+ def fwidth(self, SelectorObject selector, np.uint64_t num_cells = -1):
+ if num_cells == -1:
+ num_cells = selector.count_octs(self)
+ cdef np.ndarray[np.float64_t, ndim=2] fwidth
+ fwidth = np.empty((num_cells, 3), dtype="float64")
+ cdef OctVisitorData data
+ data.array = <void *> fwidth.data
+ data.index = 0
+ self.visit_all_octs(selector, visit_fwidth_octs, &data)
+ cdef np.float64_t base_dx
+ for i in range(3):
+ base_dx = (self.DRE[i] - self.DLE[i])/self.nn[i]
+ fwidth[:,i] *= base_dx
+ return fwidth
+
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
def fcoords(self, SelectorObject selector, np.uint64_t num_cells = -1):
if num_cells == -1:
num_cells = selector.count_octs(self)
https://bitbucket.org/yt_analysis/yt-3.0/commits/ca46ea9c4089/
Changeset: ca46ea9c4089
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-15 00:20:54
Summary: Adding a select_grid function and fixing accumulate_values.
This nearly gets spatial chunking working for particle codes. A few more
lingering issues to deal with still, for instance how to get the Octs selected
correctly now that the visit_oct function only looks at the extents.
Affected #: 2 files
diff -r 209dd30d499e9c59bd06e3c2f924591a575c9fa7 -r ca46ea9c40892e92d9999a365e03ead8c8075dcc yt/geometry/particle_geometry_handler.py
--- a/yt/geometry/particle_geometry_handler.py
+++ b/yt/geometry/particle_geometry_handler.py
@@ -180,5 +180,7 @@
def _accumulate_values(self, method):
mfunc = getattr(self.oct_handler, method)
- return mfunc(self.dobj.selector)
+ rv = mfunc(self.dobj.selector)
+ self._data_size = rv.shape[0]
+ return rv
diff -r 209dd30d499e9c59bd06e3c2f924591a575c9fa7 -r ca46ea9c40892e92d9999a365e03ead8c8075dcc yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -1258,5 +1258,10 @@
int eterm[3]) nogil:
return 1
+ cdef int select_grid(self, np.float64_t left_edge[3],
+ np.float64_t right_edge[3], np.int32_t level) nogil:
+ # This is where we'll want to add a check for the min/max index.
+ return 1
+
particle_octree_subset_selector = ParticleOctreeSubsetSelector
https://bitbucket.org/yt_analysis/yt-3.0/commits/5e52753237df/
Changeset: 5e52753237df
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-15 03:53:09
Summary: Spatial fields now generate, but all zeros.
Affected #: 8 files
diff -r ca46ea9c40892e92d9999a365e03ead8c8075dcc -r 5e52753237df033ac87c2acb04b21e489ca86779 yt/data_objects/construction_data_containers.py
--- a/yt/data_objects/construction_data_containers.py
+++ b/yt/data_objects/construction_data_containers.py
@@ -287,7 +287,7 @@
# This needs to be parallel_objects-ified
for chunk in parallel_objects(self.data_source.chunks(
chunk_fields, "io")):
- mylog.debug("Adding chunk (%s) to tree", chunk.size)
+ mylog.debug("Adding chunk (%s) to tree", chunk.ires.size)
self._handle_chunk(chunk, fields, tree)
# Note that this will briefly double RAM usage
if self.proj_style == "mip":
@@ -346,7 +346,7 @@
dl = 1.0
else:
dl = chunk.fwidth[:, self.axis]
- v = np.empty((chunk.size, len(fields)), dtype="float64")
+ v = np.empty((chunk.ires.size, len(fields)), dtype="float64")
for i in range(len(fields)):
v[:,i] = chunk[fields[i]] * dl
if self.weight_field is not None:
@@ -354,7 +354,7 @@
np.multiply(v, w[:,None], v)
np.multiply(w, dl, w)
else:
- w = np.ones(chunk.size, dtype="float64")
+ w = np.ones(chunk.ires.size, dtype="float64")
icoords = chunk.icoords
i1 = icoords[:,x_dict[self.axis]]
i2 = icoords[:,y_dict[self.axis]]
diff -r ca46ea9c40892e92d9999a365e03ead8c8075dcc -r 5e52753237df033ac87c2acb04b21e489ca86779 yt/data_objects/octree_subset.py
--- a/yt/data_objects/octree_subset.py
+++ b/yt/data_objects/octree_subset.py
@@ -176,7 +176,7 @@
# this, it's unavoidable for many types of data storage on disk.
_type_name = 'particle_octree_subset'
_con_args = ('data_files', 'pf', 'min_ind', 'max_ind')
- def __init__(self, data_files, pf, min_ind = 0, max_ind = 0):
+ def __init__(self, base_selector, data_files, pf, min_ind = 0, max_ind = 0):
# The first attempt at this will not work in parallel.
self.data_files = data_files
self.field_data = YTFieldData()
@@ -191,34 +191,54 @@
self._last_selector_id = None
self._current_particle_type = 'all'
self._current_fluid_type = self.pf.default_fluid_type
+ self.base_selector = base_selector
+
+ _domain_ind = None
+ @property
+ def domain_ind(self):
+ if self._domain_ind is None:
+ mask = self.selector.select_octs(self.oct_handler)
+ di = self.oct_handler.domain_ind(mask)
+ self._domain_ind = di
+ return self._domain_ind
+
+ def deposit(self, positions, fields = None, method = None):
+ # Here we perform our particle deposition.
+ cls = getattr(particle_deposit, "deposit_%s" % method, None)
+ if cls is None:
+ raise YTParticleDepositionNotImplemented(method)
+ nvals = (self.domain_ind >= 0).sum() * 8
+ op = cls(nvals) # We allocate number of zones, not number of octs
+ op.initialize()
+ op.process_octree(self.oct_handler, self.domain_ind, positions, fields, 0)
+ vals = op.finalize()
+ return self._reshape_vals(vals)
+
def select_icoords(self, dobj):
- return self.oct_handler.icoords(dobj)
+ return self.oct_handler.icoords(dobj.selector)
def select_fcoords(self, dobj):
- return self.oct_handler.fcoords(dobj)
+ return self.oct_handler.fcoords(dobj.selector)
def select_fwidth(self, dobj):
# Recall domain_dimensions is the number of cells, not octs
- base_dx = (self.domain.pf.domain_width /
- self.domain.pf.domain_dimensions)
- widths = np.empty((self.cell_count, 3), dtype="float64")
+ base_dx = (self.pf.domain_width /
+ self.pf.domain_dimensions)
dds = (2**self.select_ires(dobj))
+ widths = np.empty((dds.shape[0], 3), dtype="float64")
for i in range(3):
widths[:,i] = base_dx[i] / dds
return widths
def select_ires(self, dobj):
- return self.oct_handler.ires(dobj)
+ return self.oct_handler.ires(dobj.selector)
def select(self, selector):
if id(selector) == self._last_selector_id:
return self._last_mask
- m1 = self.selector.select_octs(self.oct_handler)
- m2 = selector.select_octs(self.oct_handler)
- np.logical_and(m1, m2, m1)
- del m2
- self._last_mask = m1
+ self._last_mask = self.oct_handler.domain_mask(
+ self.selector)
if self._last_mask.sum() == 0: return None
self._last_selector_id = id(selector)
return self._last_mask
diff -r ca46ea9c40892e92d9999a365e03ead8c8075dcc -r 5e52753237df033ac87c2acb04b21e489ca86779 yt/geometry/oct_container.pxd
--- a/yt/geometry/oct_container.pxd
+++ b/yt/geometry/oct_container.pxd
@@ -80,6 +80,9 @@
# Now some visitor functions
+cdef void visit_count_octs(Oct *o, OctVisitorData *data)
+cdef void visit_count_total_octs(Oct *o, OctVisitorData *data)
+cdef void visit_mark_octs(Oct *o, OctVisitorData *data)
cdef void visit_icoords_octs(Oct *o, OctVisitorData *data)
cdef void visit_ires_octs(Oct *o, OctVisitorData *data)
cdef void visit_fcoords_octs(Oct *o, OctVisitorData *data)
diff -r ca46ea9c40892e92d9999a365e03ead8c8075dcc -r 5e52753237df033ac87c2acb04b21e489ca86779 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -830,6 +830,26 @@
# Now some visitor functions
+cdef void visit_count_octs(Oct *o, OctVisitorData *data):
+ # Number of cells visited
+ data.index += 1
+
+cdef void visit_count_total_octs(Oct *o, OctVisitorData *data):
+ # Number of *octs* visited.
+ if data.last != o.domain_ind:
+ data.index += 1
+ data.last = o.domain_ind
+
+cdef void visit_mark_octs(Oct *o, OctVisitorData *data):
+ cdef int i
+ cdef np.uint8_t *arr
+ if data.last != o.domain_ind:
+ data.last = o.domain_ind
+ arr = <np.uint8_t *> data.array
+ for i in range(8):
+ arr[data.index * 8 + i] = 1
+ data.index += 1
+
cdef void visit_icoords_octs(Oct *o, OctVisitorData *data):
cdef np.int64_t *coords = <np.int64_t*> data.array
cdef int i
diff -r ca46ea9c40892e92d9999a365e03ead8c8075dcc -r 5e52753237df033ac87c2acb04b21e489ca86779 yt/geometry/particle_geometry_handler.py
--- a/yt/geometry/particle_geometry_handler.py
+++ b/yt/geometry/particle_geometry_handler.py
@@ -142,8 +142,12 @@
def _identify_base_chunk(self, dobj):
if getattr(dobj, "_chunk_info", None) is None:
- data_files = getattr(dobj, "data_files", self.data_files)
- subset = [ParticleOctreeSubset(data_files, self.parameter_file)]
+ data_files = getattr(dobj, "data_files", None)
+ if data_files is None:
+ data_files = [self.data_files[i] for i in
+ self.regions.identify_data_files(dobj.selector)]
+ subset = [ParticleOctreeSubset(dobj.selector, data_files,
+ self.parameter_file)]
dobj._chunk_info = subset
dobj._current_chunk = list(self._chunk_all(dobj))[0]
diff -r ca46ea9c40892e92d9999a365e03ead8c8075dcc -r 5e52753237df033ac87c2acb04b21e489ca86779 yt/geometry/particle_oct_container.pyx
--- a/yt/geometry/particle_oct_container.pyx
+++ b/yt/geometry/particle_oct_container.pyx
@@ -27,7 +27,9 @@
from oct_container cimport OctreeContainer, Oct, OctInfo, ORDER_MAX, \
visit_icoords_octs, visit_ires_octs, \
- visit_fcoords_octs, visit_fwidth_octs
+ visit_fcoords_octs, visit_fwidth_octs, \
+ visit_count_octs, visit_count_total_octs, \
+ visit_mark_octs
from libc.stdlib cimport malloc, free, qsort
from libc.math cimport floor
from fp_utils cimport *
@@ -187,7 +189,8 @@
self.oct_list = <Oct**> malloc(sizeof(Oct*)*self.nocts)
cdef np.int64_t i = 0, lpos = 0
cdef int cur_dom = -1
- # We always need at least 2, and if max_domain is 0, we need 3.
+ # Note that we now assign them in the same order they will be visited
+ # by recursive visitors.
for i in range(self.nn[0]):
for j in range(self.nn[1]):
for k in range(self.nn[2]):
@@ -195,6 +198,8 @@
assert(lpos == self.nocts)
for i in range(self.nocts):
self.oct_list[i].domain_ind = i
+ if self.oct_list[i].children[0][0][0] != NULL:
+ self.oct_list[i].domain = -1
self.oct_list[i].file_ind = -1
max_level = imax(max_level, self.oct_list[i].level)
self.max_level = max_level
@@ -211,7 +216,7 @@
return
cdef np.int64_t get_domain_offset(self, int domain_id):
- return self.dom_offsets[domain_id + 1]
+ return 0
cdef Oct* allocate_oct(self):
#Allocate the memory, set to NULL or -1
@@ -333,6 +338,34 @@
self.visit(o.children[i][j][k], counts, level + 1)
return
+ def domain_ind(self, np.ndarray[np.uint8_t, ndim=2, cast=True] mask):
+ cdef np.ndarray[np.int64_t, ndim=1] ind
+ ind = np.empty(mask.shape[0], 'int64')
+ # Here's where we grab the masked items.
+ nm = 0
+ for oi in range(mask.shape[0]):
+ ind[oi] = -1
+ use = 0
+ for i in range(8):
+ if mask[oi, i] == 1: use = 1
+ if use == 1:
+ ind[oi] = nm
+ nm += use
+ return ind
+
+ def domain_mask(self, SelectorObject selector):
+ cdef OctVisitorData data
+ data.index = 0
+ data.last = -1
+ self.visit_all_octs(selector, visit_count_total_octs, &data)
+ cdef np.ndarray[np.uint8_t, ndim=4] m2 = \
+ np.zeros((2, 2, 2, data.index), 'uint8')
+ data.index = 0
+ data.last = -1
+ data.array = m2.data
+ self.visit_all_octs(selector, visit_mark_octs, &data)
+ return m2.astype("bool")
+
cdef class ParticleRegions:
cdef np.float64_t left_edge[3]
cdef np.float64_t dds[3]
@@ -399,3 +432,4 @@
if ((fmask >> i) & 1) == 1:
files.append(i + n * 64)
return files
+
diff -r ca46ea9c40892e92d9999a365e03ead8c8075dcc -r 5e52753237df033ac87c2acb04b21e489ca86779 yt/geometry/selection_routines.pxd
--- a/yt/geometry/selection_routines.pxd
+++ b/yt/geometry/selection_routines.pxd
@@ -28,6 +28,7 @@
cdef struct Oct
cdef struct OctVisitorData:
np.uint64_t index
+ np.uint64_t last
int ind[3]
void *array
diff -r ca46ea9c40892e92d9999a365e03ead8c8075dcc -r 5e52753237df033ac87c2acb04b21e489ca86779 yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -1217,10 +1217,12 @@
# This is a numpy array, which will be a bool of ndim 1
cdef np.uint64_t min_ind
cdef np.uint64_t max_ind
+ cdef SelectorObject base_selector
def __init__(self, dobj):
self.min_ind = dobj.min_ind
self.max_ind = dobj.max_ind
+ self.base_selector = dobj.base_selector
@cython.boundscheck(False)
@cython.wraparound(False)
@@ -1228,9 +1230,18 @@
def select_octs(self, OctreeContainer octree):
# There has to be a better way to do this.
cdef np.ndarray[np.uint8_t, ndim=2, cast=True] m2
- m2 = np.ones((octree.nocts, 8), dtype="uint8")
+ m2 = self.base_selector.select_octs(octree)
# This is where we'll -- in the future -- cut up based on indices of
# the octs.
+ cdef np.int64_t nm, i
+ cdef np.uint8_t use, k
+ nm = m2.shape[0]
+ for i in range(nm):
+ use = 0
+ for k in range(8):
+ if m2[i,k] == 1: use = 1
+ for k in range(8):
+ m2[i,k] = use
return m2.astype("bool")
@cython.boundscheck(False)
https://bitbucket.org/yt_analysis/yt-3.0/commits/d17da4886813/
Changeset: d17da4886813
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-15 04:03:08
Summary: Uncertain why this change is necessary to get non-zero results.
Affected #: 3 files
diff -r 5e52753237df033ac87c2acb04b21e489ca86779 -r d17da48868139c69d0f5ee3e86c0952e2ce0f912 yt/data_objects/octree_subset.py
--- a/yt/data_objects/octree_subset.py
+++ b/yt/data_objects/octree_subset.py
@@ -223,13 +223,7 @@
def select_fwidth(self, dobj):
# Recall domain_dimensions is the number of cells, not octs
- base_dx = (self.pf.domain_width /
- self.pf.domain_dimensions)
- dds = (2**self.select_ires(dobj))
- widths = np.empty((dds.shape[0], 3), dtype="float64")
- for i in range(3):
- widths[:,i] = base_dx[i] / dds
- return widths
+ return self.oct_handler.fwidth(dobj.selector)
def select_ires(self, dobj):
return self.oct_handler.ires(dobj.selector)
diff -r 5e52753237df033ac87c2acb04b21e489ca86779 -r d17da48868139c69d0f5ee3e86c0952e2ce0f912 yt/geometry/particle_deposit.pyx
--- a/yt/geometry/particle_deposit.pyx
+++ b/yt/geometry/particle_deposit.pyx
@@ -75,10 +75,11 @@
oct = octree.get(pos, &oi)
# This next line is unfortunate. Basically it says, sometimes we
# might have particles that belong to octs outside our domain.
+ #print oct.domain, domain_id
if oct.domain != domain_id: continue
- #print domain_id, oct.local_ind, oct.ind, oct.domain, oct.pos[0], oct.pos[1], oct.pos[2]
# Note that this has to be our local index, not our in-file index.
offset = dom_ind[oct.domain_ind - moff] * 8
+ #print domain_id, offset, oct.domain_ind, oct.file_ind, oct.domain, oct.pos[0], oct.pos[1], oct.pos[2]
if offset < 0: continue
# Check that we found the oct ...
self.process(dims, oi.left_edge, oi.dds,
diff -r 5e52753237df033ac87c2acb04b21e489ca86779 -r d17da48868139c69d0f5ee3e86c0952e2ce0f912 yt/geometry/particle_oct_container.pyx
--- a/yt/geometry/particle_oct_container.pyx
+++ b/yt/geometry/particle_oct_container.pyx
@@ -199,7 +199,9 @@
for i in range(self.nocts):
self.oct_list[i].domain_ind = i
if self.oct_list[i].children[0][0][0] != NULL:
- self.oct_list[i].domain = -1
+ self.oct_list[i].domain = 0
+ else:
+ self.oct_list[i].domain = 0
self.oct_list[i].file_ind = -1
max_level = imax(max_level, self.oct_list[i].level)
self.max_level = max_level
https://bitbucket.org/yt_analysis/yt-3.0/commits/d8b0bf0e6a73/
Changeset: d8b0bf0e6a73
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-15 16:33:11
Summary: Spatial for particle datasets now works.
The Morton-ordering was backwards, causing octs to be missed. Additionally,
the root_dimensions argument (now removed, as it is not relevant, but it will
return for the ParticleRegions) was causing the size of the projection tree to
shrink to occupy only a tiny portion of the corner.
Affected #: 4 files
diff -r d17da48868139c69d0f5ee3e86c0952e2ce0f912 -r d8b0bf0e6a737bd8a236cd71169c42a4474e2764 yt/data_objects/construction_data_containers.py
--- a/yt/data_objects/construction_data_containers.py
+++ b/yt/data_objects/construction_data_containers.py
@@ -311,7 +311,6 @@
np.multiply(py, self.pf.domain_width[y_dict[self.axis]], py)
np.add(py, oy, py)
np.multiply(pdy, self.pf.domain_width[y_dict[self.axis]], pdy)
-
if self.weight_field is not None:
np.divide(nvals, nwvals[:,None], nvals)
if self.weight_field is None:
@@ -336,8 +335,8 @@
def _initialize_chunk(self, chunk, tree):
icoords = chunk.icoords
- i1 = icoords[:,0]
- i2 = icoords[:,1]
+ i1 = icoords[:,x_dict[self.axis]]
+ i2 = icoords[:,y_dict[self.axis]]
ilevel = chunk.ires
tree.initialize_chunk(i1, i2, ilevel)
diff -r d17da48868139c69d0f5ee3e86c0952e2ce0f912 -r d8b0bf0e6a737bd8a236cd71169c42a4474e2764 yt/frontends/sph/data_structures.py
--- a/yt/frontends/sph/data_structures.py
+++ b/yt/frontends/sph/data_structures.py
@@ -145,9 +145,8 @@
('unused', 16, 'i') )
def __init__(self, filename, data_style="gadget_binary",
- additional_fields = (), root_dimensions = 64,
+ additional_fields = (),
unit_base = None):
- self._root_dimensions = root_dimensions
self.storage_filename = None
if unit_base is not None and "UnitLength_in_cm" in unit_base:
# We assume this is comoving, because in the absence of comoving
@@ -180,7 +179,7 @@
self.domain_left_edge = np.zeros(3, "float64")
self.domain_right_edge = np.ones(3, "float64") * hvals["BoxSize"]
- self.domain_dimensions = np.ones(3, "int32") * self._root_dimensions
+ self.domain_dimensions = np.ones(3, "int32") * 2
self.periodicity = (True, True, True)
self.cosmological_simulation = 1
@@ -258,11 +257,9 @@
_fieldinfo_known = KnownOWLSFields
_header_spec = None # Override so that there's no confusion
- def __init__(self, filename, data_style="OWLS", root_dimensions = 64):
- self._root_dimensions = root_dimensions
+ def __init__(self, filename, data_style="OWLS"):
self.storage_filename = None
super(OWLSStaticOutput, self).__init__(filename, data_style,
- root_dimensions,
unit_base = None)
def __repr__(self):
@@ -283,7 +280,7 @@
self.current_time = hvals["Time_GYR"] * sec_conversion["Gyr"]
self.domain_left_edge = np.zeros(3, "float64")
self.domain_right_edge = np.ones(3, "float64") * hvals["BoxSize"]
- self.domain_dimensions = np.ones(3, "int32") * self._root_dimensions
+ self.domain_dimensions = np.ones(3, "int32") * 2
self.cosmological_simulation = 1
self.periodicity = (True, True, True)
self.current_redshift = hvals["Redshift"]
@@ -346,14 +343,13 @@
('dummy', 'i'))
def __init__(self, filename, data_style="tipsy",
- root_dimensions = 64, endian = ">",
+ endian = ">",
field_dtypes = None,
domain_left_edge = None,
domain_right_edge = None,
unit_base = None,
cosmology_parameters = None):
self.endian = endian
- self._root_dimensions = root_dimensions
self.storage_filename = None
if domain_left_edge is None:
domain_left_edge = np.zeros(3, "float64") - 0.5
@@ -399,7 +395,7 @@
# NOTE: These are now set in the main initializer.
#self.domain_left_edge = np.zeros(3, "float64") - 0.5
#self.domain_right_edge = np.ones(3, "float64") + 0.5
- self.domain_dimensions = np.ones(3, "int32") * self._root_dimensions
+ self.domain_dimensions = np.ones(3, "int32") * 2
self.periodicity = (True, True, True)
self.cosmological_simulation = 1
diff -r d17da48868139c69d0f5ee3e86c0952e2ce0f912 -r d8b0bf0e6a737bd8a236cd71169c42a4474e2764 yt/geometry/particle_oct_container.pyx
--- a/yt/geometry/particle_oct_container.pyx
+++ b/yt/geometry/particle_oct_container.pyx
@@ -198,10 +198,7 @@
assert(lpos == self.nocts)
for i in range(self.nocts):
self.oct_list[i].domain_ind = i
- if self.oct_list[i].children[0][0][0] != NULL:
- self.oct_list[i].domain = 0
- else:
- self.oct_list[i].domain = 0
+ self.oct_list[i].domain = 0
self.oct_list[i].file_ind = -1
max_level = imax(max_level, self.oct_list[i].level)
self.max_level = max_level
diff -r d17da48868139c69d0f5ee3e86c0952e2ce0f912 -r d8b0bf0e6a737bd8a236cd71169c42a4474e2764 yt/utilities/lib/geometry_utils.pyx
--- a/yt/utilities/lib/geometry_utils.pyx
+++ b/yt/utilities/lib/geometry_utils.pyx
@@ -329,9 +329,9 @@
morton_indices = np.zeros(left_index.shape[0], 'uint64')
for i in range(left_index.shape[0]):
mi = 0
- mi |= spread_bits(left_index[i,0])<<0
+ mi |= spread_bits(left_index[i,2])<<0
mi |= spread_bits(left_index[i,1])<<1
- mi |= spread_bits(left_index[i,2])<<2
+ mi |= spread_bits(left_index[i,0])<<2
morton_indices[i] = mi
return morton_indices
https://bitbucket.org/yt_analysis/yt-3.0/commits/8880d16e1caf/
Changeset: 8880d16e1caf
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-15 16:57:53
Summary: Adding some comments about how to move forward.
Affected #: 1 file
diff -r d8b0bf0e6a737bd8a236cd71169c42a4474e2764 -r 8880d16e1caf0b86c7fb0885a874f7bb5193ea42 yt/geometry/particle_deposit.pyx
--- a/yt/geometry/particle_deposit.pyx
+++ b/yt/geometry/particle_deposit.pyx
@@ -72,14 +72,18 @@
field_vals[j] = field_pointers[j][i]
for j in range(3):
pos[j] = positions[i, j]
+ # This line should be modified to have it return the index into an
+ # array based on whatever cutting of the domain we have done. This
+ # may or may not include the domain indices that we have
+ # previously generated. This way we can support not knowing the
+ # full octree structure. All we *really* care about is some
+ # arbitrary offset into a field value for deposition.
oct = octree.get(pos, &oi)
# This next line is unfortunate. Basically it says, sometimes we
# might have particles that belong to octs outside our domain.
- #print oct.domain, domain_id
if oct.domain != domain_id: continue
# Note that this has to be our local index, not our in-file index.
offset = dom_ind[oct.domain_ind - moff] * 8
- #print domain_id, offset, oct.domain_ind, oct.file_ind, oct.domain, oct.pos[0], oct.pos[1], oct.pos[2]
if offset < 0: continue
# Check that we found the oct ...
self.process(dims, oi.left_edge, oi.dds,
https://bitbucket.org/yt_analysis/yt-3.0/commits/f6c7b1949be8/
Changeset: f6c7b1949be8
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-17 16:07:27
Summary: Shorthand commit for a smoothing kernel test field.
Affected #: 1 file
diff -r 8880d16e1caf0b86c7fb0885a874f7bb5193ea42 -r f6c7b1949be82e3bbd735b843fc0049856d0d8a0 yt/frontends/sph/fields.py
--- a/yt/frontends/sph/fields.py
+++ b/yt/frontends/sph/fields.py
@@ -234,3 +234,13 @@
func = _field_concat_slice(iname, axi)
OWLSFieldInfo.add_field(("all", oname + ax), function=func,
particle_type = True)
+
+def SmoothedGas(field, data):
+ pos = data["PartType0", "Coordinates"]
+ sml = data["PartType0", "SmoothingLength"]
+ dens = data["PartType0", "Density"]
+ rv = data.deposit(pos, [sml, dens], method="simple_smooth")
+ return rv
+OWLSFieldInfo.add_field(("deposit", "PartType0_simple_smooth"),
+ function = SmoothedGas, validators = [ValidateSpatial()])
+
https://bitbucket.org/yt_analysis/yt-3.0/commits/c58d372d1e36/
Changeset: c58d372d1e36
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-17 16:28:56
Summary: Beginning update of Gadget IO.
Affected #: 2 files
diff -r f6c7b1949be82e3bbd735b843fc0049856d0d8a0 -r c58d372d1e36c4ea6b360ee5e0a8f579b4d0180f yt/frontends/sph/io.py
--- a/yt/frontends/sph/io.py
+++ b/yt/frontends/sph/io.py
@@ -32,7 +32,8 @@
BaseIOHandler
from yt.utilities.fortran_utils import read_record
-from yt.utilities.lib.geometry_utils import get_morton_indices
+from yt.utilities.lib.geometry_utils import get_morton_indices, \
+ get_morton_indices_unravel
from yt.geometry.oct_container import _ORDER_MAX
@@ -186,26 +187,28 @@
ptall = []
psize = defaultdict(lambda: 0)
chunks = list(chunks)
- pf = chunks[0].objs[0].domain.pf
ptypes = set()
for ftype, fname in fields:
ptf[ftype].append(fname)
ptypes.add(ftype)
ptypes = list(ptypes)
ptypes.sort(key = lambda a: self._ptypes.index(a))
+ data_files = set([])
for chunk in chunks:
- for subset in chunk.objs:
- poff = subset.domain.field_offsets
- tp = subset.domain.total_particles
- f = open(subset.domain.domain_filename, "rb")
- for ptype in ptypes:
- f.seek(poff[ptype, "Coordinates"], os.SEEK_SET)
- pos = self._read_field_from_file(f,
- tp[ptype], "Coordinates")
- psize[ptype] += selector.count_points(
- pos[:,0], pos[:,1], pos[:,2])
- del pos
- f.close()
+ for obj in chunk.objs:
+ data_files.update(obj.data_files)
+ for data_file in data_files:
+ poff = data_file.field_offsets
+ tp = data_file.total_particles
+ f = open(data_file.filename, "rb")
+ for ptype in ptypes:
+ f.seek(poff[ptype, "Coordinates"], os.SEEK_SET)
+ pos = self._read_field_from_file(f,
+ tp[ptype], "Coordinates")
+ psize[ptype] += selector.count_points(
+ pos[:,0], pos[:,1], pos[:,2])
+ del pos
+ f.close()
ind = {}
for field in fields:
mylog.debug("Allocating %s values for %s", psize[field[0]], field)
@@ -215,29 +218,28 @@
shape = psize[field[0]]
rv[field] = np.empty(shape, dtype="float64")
ind[field] = 0
- for chunk in chunks:
- for subset in chunk.objs:
- poff = subset.domain.field_offsets
- tp = subset.domain.total_particles
- f = open(subset.domain.domain_filename, "rb")
- for ptype, field_list in sorted(ptf.items()):
- f.seek(poff[ptype, "Coordinates"], os.SEEK_SET)
- pos = self._read_field_from_file(f,
- tp[ptype], "Coordinates")
- mask = selector.select_points(
- pos[:,0], pos[:,1], pos[:,2])
- del pos
- if mask is None: continue
- for field in field_list:
- f.seek(poff[ptype, field], os.SEEK_SET)
- data = self._read_field_from_file(f, tp[ptype], field)
- data = data[mask]
- my_ind = ind[ptype, field]
- mylog.debug("Filling from %s to %s with %s",
- my_ind, my_ind+data.shape[0], field)
- rv[ptype, field][my_ind:my_ind + data.shape[0],...] = data
- ind[ptype, field] += data.shape[0]
- f.close()
+ for data_file in data_files:
+ poff = data_file.field_offsets
+ tp = data_file.total_particles
+ f = open(data_file.filename, "rb")
+ for ptype, field_list in sorted(ptf.items()):
+ f.seek(poff[ptype, "Coordinates"], os.SEEK_SET)
+ pos = self._read_field_from_file(f,
+ tp[ptype], "Coordinates")
+ mask = selector.select_points(
+ pos[:,0], pos[:,1], pos[:,2])
+ del pos
+ if mask is None: continue
+ for field in field_list:
+ f.seek(poff[ptype, field], os.SEEK_SET)
+ data = self._read_field_from_file(f, tp[ptype], field)
+ data = data[mask]
+ my_ind = ind[ptype, field]
+ mylog.debug("Filling from %s to %s with %s",
+ my_ind, my_ind+data.shape[0], field)
+ rv[ptype, field][my_ind:my_ind + data.shape[0],...] = data
+ ind[ptype, field] += data.shape[0]
+ f.close()
return rv
def _read_field_from_file(self, f, count, name):
@@ -253,19 +255,27 @@
arr = arr.reshape((count/3, 3), order="C")
return arr.astype("float64")
- def _initialize_index(self, domain, octree):
- count = sum(domain.total_particles.values())
+ def _initialize_index(self, data_file, regions):
+ count = sum(data_file.total_particles.values())
dt = [("px", "float32"), ("py", "float32"), ("pz", "float32")]
- with open(domain.domain_filename, "rb") as f:
+ DLE = data_file.pf.domain_left_edge
+ DRE = data_file.pf.domain_right_edge
+ dx = (DRE - DLE) / 2**_ORDER_MAX
+ with open(data_file.filename, "rb") as f:
f.seek(self._header_offset)
# The first total_particles * 3 values are positions
pp = np.fromfile(f, dtype = dt, count = count)
- pos = np.empty((count, 3), dtype="float64")
- pos[:,0] = pp['px']
- pos[:,1] = pp['py']
- pos[:,2] = pp['pz']
+ pos = np.column_stack([pp['px'], pp['py'], pp['pz']]).astype("float64")
del pp
- octree.add(pos, domain.domain_id)
+ regions.add_data_file(pos, data_file.file_id)
+ lx = np.floor((pos[:,0] - DLE[0])/dx[0]).astype("uint64")
+ ly = np.floor((pos[:,1] - DLE[1])/dx[1]).astype("uint64")
+ lz = np.floor((pos[:,2] - DLE[2])/dx[2]).astype("uint64")
+ del pos
+ morton = get_morton_indices_unravel(lx, ly, lz)
+ del lx, ly, lz
+ return morton
+
def _count_particles(self, domain):
npart = dict((self._ptypes[i], v)
diff -r f6c7b1949be82e3bbd735b843fc0049856d0d8a0 -r c58d372d1e36c4ea6b360ee5e0a8f579b4d0180f yt/utilities/lib/geometry_utils.pyx
--- a/yt/utilities/lib/geometry_utils.pyx
+++ b/yt/utilities/lib/geometry_utils.pyx
@@ -335,6 +335,23 @@
morton_indices[i] = mi
return morton_indices
+ at cython.cdivision(True)
+ at cython.boundscheck(False)
+ at cython.wraparound(False)
+def get_morton_indices_unravel(np.ndarray[np.uint64_t, ndim=1] left_x,
+ np.ndarray[np.uint64_t, ndim=1] left_y,
+ np.ndarray[np.uint64_t, ndim=1] left_z,):
+ cdef np.int64_t i, mi
+ cdef np.ndarray[np.uint64_t, ndim=1] morton_indices
+ morton_indices = np.zeros(left_x.shape[0], 'uint64')
+ for i in range(left_x.shape[0]):
+ mi = 0
+ mi |= spread_bits(left_z[i])<<0
+ mi |= spread_bits(left_y[i])<<1
+ mi |= spread_bits(left_x[i])<<2
+ morton_indices[i] = mi
+ return morton_indices
+
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
https://bitbucket.org/yt_analysis/yt-3.0/commits/4a332ffbe8a9/
Changeset: 4a332ffbe8a9
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-17 18:26:29
Summary: Eliminate final mask usage. Grid intersection seems somewhat off and
sub-region selection is currently broken.
Affected #: 5 files
diff -r c58d372d1e36c4ea6b360ee5e0a8f579b4d0180f -r 4a332ffbe8a9c8acce22ce7c78342666c99b8f5b yt/data_objects/octree_subset.py
--- a/yt/data_objects/octree_subset.py
+++ b/yt/data_objects/octree_subset.py
@@ -198,8 +198,7 @@
@property
def domain_ind(self):
if self._domain_ind is None:
- mask = self.selector.select_octs(self.oct_handler)
- di = self.oct_handler.domain_ind(mask)
+ di = self.oct_handler.domain_ind(self.base_selector)
self._domain_ind = di
return self._domain_ind
@@ -214,7 +213,7 @@
op.process_octree(self.oct_handler, self.domain_ind, positions, fields, 0)
vals = op.finalize()
return self._reshape_vals(vals)
-
+
def select_icoords(self, dobj):
return self.oct_handler.icoords(dobj.selector)
diff -r c58d372d1e36c4ea6b360ee5e0a8f579b4d0180f -r 4a332ffbe8a9c8acce22ce7c78342666c99b8f5b yt/geometry/oct_container.pxd
--- a/yt/geometry/oct_container.pxd
+++ b/yt/geometry/oct_container.pxd
@@ -83,6 +83,7 @@
cdef void visit_count_octs(Oct *o, OctVisitorData *data)
cdef void visit_count_total_octs(Oct *o, OctVisitorData *data)
cdef void visit_mark_octs(Oct *o, OctVisitorData *data)
+cdef void visit_index_octs(Oct *o, OctVisitorData *data)
cdef void visit_icoords_octs(Oct *o, OctVisitorData *data)
cdef void visit_ires_octs(Oct *o, OctVisitorData *data)
cdef void visit_fcoords_octs(Oct *o, OctVisitorData *data)
diff -r c58d372d1e36c4ea6b360ee5e0a8f579b4d0180f -r 4a332ffbe8a9c8acce22ce7c78342666c99b8f5b yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -850,6 +850,15 @@
arr[data.index * 8 + i] = 1
data.index += 1
+cdef void visit_index_octs(Oct *o, OctVisitorData *data):
+ cdef int i
+ cdef np.int64_t *arr
+ if data.last != o.domain_ind:
+ data.last = o.domain_ind
+ arr = <np.int64_t *> data.array
+ arr[o.domain_ind] = data.index
+ data.index += 1
+
cdef void visit_icoords_octs(Oct *o, OctVisitorData *data):
cdef np.int64_t *coords = <np.int64_t*> data.array
cdef int i
diff -r c58d372d1e36c4ea6b360ee5e0a8f579b4d0180f -r 4a332ffbe8a9c8acce22ce7c78342666c99b8f5b yt/geometry/particle_oct_container.pyx
--- a/yt/geometry/particle_oct_container.pyx
+++ b/yt/geometry/particle_oct_container.pyx
@@ -29,7 +29,7 @@
visit_icoords_octs, visit_ires_octs, \
visit_fcoords_octs, visit_fwidth_octs, \
visit_count_octs, visit_count_total_octs, \
- visit_mark_octs
+ visit_mark_octs, visit_index_octs
from libc.stdlib cimport malloc, free, qsort
from libc.math cimport floor
from fp_utils cimport *
@@ -337,19 +337,15 @@
self.visit(o.children[i][j][k], counts, level + 1)
return
- def domain_ind(self, np.ndarray[np.uint8_t, ndim=2, cast=True] mask):
+ def domain_ind(self, selector):
cdef np.ndarray[np.int64_t, ndim=1] ind
- ind = np.empty(mask.shape[0], 'int64')
# Here's where we grab the masked items.
- nm = 0
- for oi in range(mask.shape[0]):
- ind[oi] = -1
- use = 0
- for i in range(8):
- if mask[oi, i] == 1: use = 1
- if use == 1:
- ind[oi] = nm
- nm += use
+ ind = np.zeros(self.nocts, 'int64') - 1
+ cdef OctVisitorData data
+ data.array = ind.data
+ data.last = -1
+ data.index = 0
+ self.visit_all_octs(selector, visit_index_octs, &data)
return ind
def domain_mask(self, SelectorObject selector):
diff -r c58d372d1e36c4ea6b360ee5e0a8f579b4d0180f -r 4a332ffbe8a9c8acce22ce7c78342666c99b8f5b yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -1271,8 +1271,9 @@
cdef int select_grid(self, np.float64_t left_edge[3],
np.float64_t right_edge[3], np.int32_t level) nogil:
- # This is where we'll want to add a check for the min/max index.
- return 1
+ # Because visitors now use select_grid, we should be explicitly
+ # checking this.
+ return self.base_selector.select_grid(left_edge, right_edge, level)
particle_octree_subset_selector = ParticleOctreeSubsetSelector
https://bitbucket.org/yt_analysis/yt-3.0/commits/10befbee6f52/
Changeset: 10befbee6f52
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-17 18:45:08
Summary: Use traversal everywhere and fix selection.
Affected #: 3 files
diff -r 4a332ffbe8a9c8acce22ce7c78342666c99b8f5b -r 10befbee6f52858268c7f4c4310a403bc15eeb23 yt/data_objects/octree_subset.py
--- a/yt/data_objects/octree_subset.py
+++ b/yt/data_objects/octree_subset.py
@@ -198,7 +198,7 @@
@property
def domain_ind(self):
if self._domain_ind is None:
- di = self.oct_handler.domain_ind(self.base_selector)
+ di = self.oct_handler.domain_ind(self.selector)
self._domain_ind = di
return self._domain_ind
@@ -231,7 +231,7 @@
if id(selector) == self._last_selector_id:
return self._last_mask
self._last_mask = self.oct_handler.domain_mask(
- self.selector)
+ self.base_selector)
if self._last_mask.sum() == 0: return None
self._last_selector_id = id(selector)
return self._last_mask
diff -r 4a332ffbe8a9c8acce22ce7c78342666c99b8f5b -r 10befbee6f52858268c7f4c4310a403bc15eeb23 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -842,13 +842,13 @@
cdef void visit_mark_octs(Oct *o, OctVisitorData *data):
cdef int i
- cdef np.uint8_t *arr
+ cdef np.uint8_t *arr = <np.uint8_t *> data.array
if data.last != o.domain_ind:
data.last = o.domain_ind
- arr = <np.uint8_t *> data.array
- for i in range(8):
- arr[data.index * 8 + i] = 1
data.index += 1
+ cdef np.int64_t index = data.index * 8
+ index += ((data.ind[2]*2)+data.ind[1])*2+data.ind[0]
+ arr[index] = 1
cdef void visit_index_octs(Oct *o, OctVisitorData *data):
cdef int i
diff -r 4a332ffbe8a9c8acce22ce7c78342666c99b8f5b -r 10befbee6f52858268c7f4c4310a403bc15eeb23 yt/geometry/particle_oct_container.pyx
--- a/yt/geometry/particle_oct_container.pyx
+++ b/yt/geometry/particle_oct_container.pyx
@@ -355,7 +355,7 @@
self.visit_all_octs(selector, visit_count_total_octs, &data)
cdef np.ndarray[np.uint8_t, ndim=4] m2 = \
np.zeros((2, 2, 2, data.index), 'uint8')
- data.index = 0
+ data.index = -1
data.last = -1
data.array = m2.data
self.visit_all_octs(selector, visit_mark_octs, &data)
https://bitbucket.org/yt_analysis/yt-3.0/commits/52a4eaee2343/
Changeset: 52a4eaee2343
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-18 16:50:18
Summary: A variety of changes.
* Change the ParticleOctree to use base_region.
* Add a new masking visitor.
* Change how select_octs works.
Affected #: 7 files
diff -r 10befbee6f52858268c7f4c4310a403bc15eeb23 -r 52a4eaee2343e6e716a7f2fc210d47e2265077c0 yt/data_objects/octree_subset.py
--- a/yt/data_objects/octree_subset.py
+++ b/yt/data_objects/octree_subset.py
@@ -176,7 +176,7 @@
# this, it's unavoidable for many types of data storage on disk.
_type_name = 'particle_octree_subset'
_con_args = ('data_files', 'pf', 'min_ind', 'max_ind')
- def __init__(self, base_selector, data_files, pf, min_ind = 0, max_ind = 0):
+ def __init__(self, base_region, data_files, pf, min_ind = 0, max_ind = 0):
# The first attempt at this will not work in parallel.
self.data_files = data_files
self.field_data = YTFieldData()
@@ -191,8 +191,9 @@
self._last_selector_id = None
self._current_particle_type = 'all'
self._current_fluid_type = self.pf.default_fluid_type
- self.base_selector = base_selector
-
+ self.base_region = base_region
+ self.base_selector = base_region.selector
+
_domain_ind = None
@property
@@ -230,6 +231,8 @@
def select(self, selector):
if id(selector) == self._last_selector_id:
return self._last_mask
+ # This is where things get confused. I believe the data is differently
+ # ordered than the mask.
self._last_mask = self.oct_handler.domain_mask(
self.base_selector)
if self._last_mask.sum() == 0: return None
diff -r 10befbee6f52858268c7f4c4310a403bc15eeb23 -r 52a4eaee2343e6e716a7f2fc210d47e2265077c0 yt/geometry/oct_container.pxd
--- a/yt/geometry/oct_container.pxd
+++ b/yt/geometry/oct_container.pxd
@@ -83,6 +83,7 @@
cdef void visit_count_octs(Oct *o, OctVisitorData *data)
cdef void visit_count_total_octs(Oct *o, OctVisitorData *data)
cdef void visit_mark_octs(Oct *o, OctVisitorData *data)
+cdef void visit_mask_octs(Oct *o, OctVisitorData *data)
cdef void visit_index_octs(Oct *o, OctVisitorData *data)
cdef void visit_icoords_octs(Oct *o, OctVisitorData *data)
cdef void visit_ires_octs(Oct *o, OctVisitorData *data)
diff -r 10befbee6f52858268c7f4c4310a403bc15eeb23 -r 52a4eaee2343e6e716a7f2fc210d47e2265077c0 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -142,6 +142,7 @@
oct_visitor_function *func,
OctVisitorData *data):
cdef int i, j, k, n
+ data.global_index = -1
cdef np.float64_t pos[3], dds[3]
# This dds is the oct-width
for i in range(3):
@@ -850,6 +851,13 @@
index += ((data.ind[2]*2)+data.ind[1])*2+data.ind[0]
arr[index] = 1
+cdef void visit_mask_octs(Oct *o, OctVisitorData *data):
+ cdef int i
+ cdef np.uint8_t *arr = <np.uint8_t *> data.array
+ cdef np.int64_t index = data.index * 8
+ index += ((data.ind[2]*2)+data.ind[1])*2+data.ind[0]
+ arr[index] = 1
+
cdef void visit_index_octs(Oct *o, OctVisitorData *data):
cdef int i
cdef np.int64_t *arr
diff -r 10befbee6f52858268c7f4c4310a403bc15eeb23 -r 52a4eaee2343e6e716a7f2fc210d47e2265077c0 yt/geometry/particle_geometry_handler.py
--- a/yt/geometry/particle_geometry_handler.py
+++ b/yt/geometry/particle_geometry_handler.py
@@ -146,7 +146,7 @@
if data_files is None:
data_files = [self.data_files[i] for i in
self.regions.identify_data_files(dobj.selector)]
- subset = [ParticleOctreeSubset(dobj.selector, data_files,
+ subset = [ParticleOctreeSubset(dobj, data_files,
self.parameter_file)]
dobj._chunk_info = subset
dobj._current_chunk = list(self._chunk_all(dobj))[0]
@@ -181,10 +181,3 @@
self.oct_handler = oct_handler
self.regions = regions
super(ParticleDataChunk, self).__init__(*args, **kwargs)
-
- def _accumulate_values(self, method):
- mfunc = getattr(self.oct_handler, method)
- rv = mfunc(self.dobj.selector)
- self._data_size = rv.shape[0]
- return rv
-
diff -r 10befbee6f52858268c7f4c4310a403bc15eeb23 -r 52a4eaee2343e6e716a7f2fc210d47e2265077c0 yt/geometry/particle_oct_container.pyx
--- a/yt/geometry/particle_oct_container.pyx
+++ b/yt/geometry/particle_oct_container.pyx
@@ -343,18 +343,21 @@
ind = np.zeros(self.nocts, 'int64') - 1
cdef OctVisitorData data
data.array = ind.data
+ data.index = 0
data.last = -1
- data.index = 0
self.visit_all_octs(selector, visit_index_octs, &data)
return ind
def domain_mask(self, SelectorObject selector):
+ # This is actually not correct. The hard part is that we need to
+ # iterate the same way visit_all_octs does, but we need to track the
+ # number of octs total visited.
cdef OctVisitorData data
data.index = 0
data.last = -1
self.visit_all_octs(selector, visit_count_total_octs, &data)
cdef np.ndarray[np.uint8_t, ndim=4] m2 = \
- np.zeros((2, 2, 2, data.index), 'uint8')
+ np.zeros((2, 2, 2, data.index), 'uint8', order='F')
data.index = -1
data.last = -1
data.array = m2.data
diff -r 10befbee6f52858268c7f4c4310a403bc15eeb23 -r 52a4eaee2343e6e716a7f2fc210d47e2265077c0 yt/geometry/selection_routines.pxd
--- a/yt/geometry/selection_routines.pxd
+++ b/yt/geometry/selection_routines.pxd
@@ -29,6 +29,7 @@
cdef struct OctVisitorData:
np.uint64_t index
np.uint64_t last
+ np.uint64_t global_index
int ind[3]
void *array
diff -r 10befbee6f52858268c7f4c4310a403bc15eeb23 -r 52a4eaee2343e6e716a7f2fc210d47e2265077c0 yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -29,7 +29,8 @@
from libc.stdlib cimport malloc, free
from fp_utils cimport fclip, iclip
from selection_routines cimport SelectorObject
-from oct_container cimport OctreeContainer, OctAllocationContainer, Oct
+from oct_container cimport OctreeContainer, OctAllocationContainer, Oct, \
+ visit_mark_octs, visit_count_total_octs
#from geometry_utils cimport point_to_hilbert
from yt.utilities.lib.grid_traversal cimport \
VolumeContainer, sample_function, walk_volume
@@ -178,6 +179,7 @@
np.float64_t pos[3], np.float64_t dds[3],
np.ndarray[np.uint8_t, ndim=2] mask,
int level = 0):
+
cdef np.float64_t LE[3], RE[3], sdds[3], spos[3]
cdef int i, j, k, res, ii
cdef Oct *ch
@@ -268,6 +270,7 @@
this_level = 0
if res == 0 and this_level == 1:
return
+ data.global_index += 1
# Now we visit all our children. We subtract off sdds for the first
# pass because we center it on the first cell.
spos[0] = pos[0] - sdds[0]/2.0
@@ -1229,19 +1232,18 @@
@cython.cdivision(True)
def select_octs(self, OctreeContainer octree):
# There has to be a better way to do this.
- cdef np.ndarray[np.uint8_t, ndim=2, cast=True] m2
- m2 = self.base_selector.select_octs(octree)
+ cdef OctVisitorData data
+ data.index = 0
+ data.last = -1
+ octree.visit_all_octs(self, visit_count_total_octs, &data)
+ cdef np.ndarray[np.uint8_t, ndim=4] m2 = \
+ np.zeros((2, 2, 2, data.index), 'uint8', order='C')
# This is where we'll -- in the future -- cut up based on indices of
# the octs.
- cdef np.int64_t nm, i
- cdef np.uint8_t use, k
- nm = m2.shape[0]
- for i in range(nm):
- use = 0
- for k in range(8):
- if m2[i,k] == 1: use = 1
- for k in range(8):
- m2[i,k] = use
+ data.index = -1
+ data.last = -1
+ data.array = m2.data
+ octree.visit_all_octs(self, visit_mark_octs, &data)
return m2.astype("bool")
@cython.boundscheck(False)
https://bitbucket.org/yt_analysis/yt-3.0/commits/e3a9b69f4b0c/
Changeset: e3a9b69f4b0c
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-18 19:21:40
Summary: Attempting to push selection deeper, to avoid a few more mask constructions.
Spatial/non-spatial ordering is still incorrect.
Affected #: 6 files
diff -r 52a4eaee2343e6e716a7f2fc210d47e2265077c0 -r e3a9b69f4b0c766138f7249fe05c70576b044907 yt/data_objects/data_containers.py
--- a/yt/data_objects/data_containers.py
+++ b/yt/data_objects/data_containers.py
@@ -248,28 +248,19 @@
if ngz == 0:
for io_chunk in self.chunks([], "io"):
for i,chunk in enumerate(self.chunks(field, "spatial", ngz = 0)):
- mask = self._current_chunk.objs[0].select(self.selector)
- if mask is None: continue
- data = self[field]
- if len(data.shape) == 4:
- # This is how we keep it consistent between oct ordering
- # and grid ordering.
- data = data.T[mask.T]
- else:
- data = data[mask]
- rv[ind:ind+data.size] = data
- ind += data.size
+ print self.selector
+ ind += self._current_chunk.objs[0].select(
+ self.selector, self[field], rv, ind)
else:
chunks = self.hierarchy._chunk(self, "spatial", ngz = ngz)
for i, chunk in enumerate(chunks):
with self._chunked_read(chunk):
gz = self._current_chunk.objs[0]
wogz = gz._base_grid
- mask = wogz.select(self.selector)
- if mask is None: continue
- data = gz[field][ngz:-ngz, ngz:-ngz, ngz:-ngz][mask]
- rv[ind:ind+data.size] = data
- ind += data.size
+ ind += wogz.select(
+ self.selector,
+ gz[field][ngz:-ngz, ngz:-ngz, ngz:-ngz],
+ rv, ind)
return rv
def _generate_particle_field(self, field):
diff -r 52a4eaee2343e6e716a7f2fc210d47e2265077c0 -r e3a9b69f4b0c766138f7249fe05c70576b044907 yt/data_objects/octree_subset.py
--- a/yt/data_objects/octree_subset.py
+++ b/yt/data_objects/octree_subset.py
@@ -75,6 +75,8 @@
return self._current_chunk.fwidth[:,1]
elif field == "dz":
return self._current_chunk.fwidth[:,2]
+ else:
+ raise RuntimeError
def select_icoords(self, dobj):
return self.oct_handler.icoords(self.domain.domain_id, self.mask,
@@ -117,9 +119,10 @@
return tr
def _reshape_vals(self, arr):
+ if len(arr.shape) == 4: return arr
nz = self._num_zones + 2*self._num_ghost_zones
n_oct = arr.shape[0] / (nz**3.0)
- arr = arr.reshape((nz, nz, nz, n_oct), order="F")
+ arr = arr.reshape((nz, nz, nz, n_oct), order="C")
return arr
_domain_ind = None
@@ -185,8 +188,8 @@
self.hierarchy = self.pf.hierarchy
self.oct_handler = pf.h.oct_handler
self.min_ind = min_ind
+ if max_ind == 0: max_ind = (1 << 63)
self.max_ind = max_ind
- if max_ind == 0: max_ind = (1 << 63)
self._last_mask = None
self._last_selector_id = None
self._current_particle_type = 'all'
@@ -228,16 +231,9 @@
def select_ires(self, dobj):
return self.oct_handler.ires(dobj.selector)
- def select(self, selector):
- if id(selector) == self._last_selector_id:
- return self._last_mask
- # This is where things get confused. I believe the data is differently
- # ordered than the mask.
- self._last_mask = self.oct_handler.domain_mask(
- self.base_selector)
- if self._last_mask.sum() == 0: return None
- self._last_selector_id = id(selector)
- return self._last_mask
+ def select(self, selector, source, dest, offset):
+ n = self.oct_handler.selector_fill(selector, source, dest, offset)
+ return n
def count(self, selector):
if id(selector) == self._last_selector_id:
@@ -254,4 +250,3 @@
def select_particles(self, selector, x, y, z):
mask = selector.select_points(x,y,z)
return mask
-
diff -r 52a4eaee2343e6e716a7f2fc210d47e2265077c0 -r e3a9b69f4b0c766138f7249fe05c70576b044907 yt/geometry/oct_container.pxd
--- a/yt/geometry/oct_container.pxd
+++ b/yt/geometry/oct_container.pxd
@@ -80,12 +80,13 @@
# Now some visitor functions
-cdef void visit_count_octs(Oct *o, OctVisitorData *data)
-cdef void visit_count_total_octs(Oct *o, OctVisitorData *data)
-cdef void visit_mark_octs(Oct *o, OctVisitorData *data)
-cdef void visit_mask_octs(Oct *o, OctVisitorData *data)
-cdef void visit_index_octs(Oct *o, OctVisitorData *data)
-cdef void visit_icoords_octs(Oct *o, OctVisitorData *data)
-cdef void visit_ires_octs(Oct *o, OctVisitorData *data)
-cdef void visit_fcoords_octs(Oct *o, OctVisitorData *data)
-cdef void visit_fwidth_octs(Oct *o, OctVisitorData *data)
+cdef oct_visitor_function visit_count_octs
+cdef oct_visitor_function visit_count_total_octs
+cdef oct_visitor_function visit_mark_octs
+cdef oct_visitor_function visit_mask_octs
+cdef oct_visitor_function visit_index_octs
+cdef oct_visitor_function visit_icoords_octs
+cdef oct_visitor_function visit_ires_octs
+cdef oct_visitor_function visit_fcoords_octs
+cdef oct_visitor_function visit_fwidth_octs
+cdef oct_visitor_function visit_copy_array
diff -r 52a4eaee2343e6e716a7f2fc210d47e2265077c0 -r e3a9b69f4b0c766138f7249fe05c70576b044907 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -831,6 +831,14 @@
# Now some visitor functions
+cdef void visit_copy_array(Oct *o, OctVisitorData *data):
+ # We should always have global_index less than our source.
+ cdef np.int64_t index = data.global_index * 8
+ cdef np.float64_t **p = <np.float64_t**> data.array
+ index += ((data.ind[2]*2)+data.ind[1])*2+data.ind[0]
+ p[1][data.index] = p[0][index]
+ data.index += 1
+
cdef void visit_count_octs(Oct *o, OctVisitorData *data):
# Number of cells visited
data.index += 1
diff -r 52a4eaee2343e6e716a7f2fc210d47e2265077c0 -r e3a9b69f4b0c766138f7249fe05c70576b044907 yt/geometry/particle_deposit.pxd
--- a/yt/geometry/particle_deposit.pxd
+++ b/yt/geometry/particle_deposit.pxd
@@ -38,7 +38,7 @@
void *alloca(int)
cdef inline int gind(int i, int j, int k, int dims[3]):
- return ((k*dims[1])+j)*dims[0]+i
+ return ((i*dims[1])+j)*dims[2]+k
####################################################
diff -r 52a4eaee2343e6e716a7f2fc210d47e2265077c0 -r e3a9b69f4b0c766138f7249fe05c70576b044907 yt/geometry/particle_oct_container.pyx
--- a/yt/geometry/particle_oct_container.pyx
+++ b/yt/geometry/particle_oct_container.pyx
@@ -29,7 +29,7 @@
visit_icoords_octs, visit_ires_octs, \
visit_fcoords_octs, visit_fwidth_octs, \
visit_count_octs, visit_count_total_octs, \
- visit_mark_octs, visit_index_octs
+ visit_mark_octs, visit_index_octs, visit_copy_array
from libc.stdlib cimport malloc, free, qsort
from libc.math cimport floor
from fp_utils cimport *
@@ -348,21 +348,23 @@
self.visit_all_octs(selector, visit_index_octs, &data)
return ind
- def domain_mask(self, SelectorObject selector):
+ def selector_fill(self, SelectorObject selector,
+ np.ndarray[np.float64_t, ndim=4] source,
+ np.ndarray[np.float64_t, ndim=1] dest,
+ np.int64_t offset):
# This is actually not correct. The hard part is that we need to
# iterate the same way visit_all_octs does, but we need to track the
# number of octs total visited.
cdef OctVisitorData data
- data.index = 0
+ data.index = offset
+ # We only need this so we can continue calculating the offset
data.last = -1
- self.visit_all_octs(selector, visit_count_total_octs, &data)
- cdef np.ndarray[np.uint8_t, ndim=4] m2 = \
- np.zeros((2, 2, 2, data.index), 'uint8', order='F')
- data.index = -1
- data.last = -1
- data.array = m2.data
- self.visit_all_octs(selector, visit_mark_octs, &data)
- return m2.astype("bool")
+ cdef void *p[2]
+ p[0] = source.data
+ p[1] = dest.data
+ data.array = &p
+ self.visit_all_octs(selector, visit_copy_array, &data)
+ return data.index - offset
cdef class ParticleRegions:
cdef np.float64_t left_edge[3]
https://bitbucket.org/yt_analysis/yt-3.0/commits/14f5ff9a4421/
Changeset: 14f5ff9a4421
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-18 23:07:59
Summary: Some functions need to be called even if no cells are selected.
This also brings the spatial fields back in line with the non-spatial fields
for particle octrees.
However, I think it breaks most of the other Octree frontends.
Affected #: 6 files
diff -r e3a9b69f4b0c766138f7249fe05c70576b044907 -r 14f5ff9a442112c858c84f8274bd72cef19fc66c yt/data_objects/octree_subset.py
--- a/yt/data_objects/octree_subset.py
+++ b/yt/data_objects/octree_subset.py
@@ -122,7 +122,8 @@
if len(arr.shape) == 4: return arr
nz = self._num_zones + 2*self._num_ghost_zones
n_oct = arr.shape[0] / (nz**3.0)
- arr = arr.reshape((nz, nz, nz, n_oct), order="C")
+ arr = arr.reshape((nz, nz, nz, n_oct), order="F")
+ arr = np.asfortranarray(arr)
return arr
_domain_ind = None
diff -r e3a9b69f4b0c766138f7249fe05c70576b044907 -r 14f5ff9a442112c858c84f8274bd72cef19fc66c yt/geometry/oct_container.pxd
--- a/yt/geometry/oct_container.pxd
+++ b/yt/geometry/oct_container.pxd
@@ -80,7 +80,6 @@
# Now some visitor functions
-cdef oct_visitor_function visit_count_octs
cdef oct_visitor_function visit_count_total_octs
cdef oct_visitor_function visit_mark_octs
cdef oct_visitor_function visit_mask_octs
diff -r e3a9b69f4b0c766138f7249fe05c70576b044907 -r 14f5ff9a442112c858c84f8274bd72cef19fc66c yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -831,25 +831,24 @@
# Now some visitor functions
-cdef void visit_copy_array(Oct *o, OctVisitorData *data):
+cdef void visit_copy_array(Oct *o, OctVisitorData *data, np.uint8_t selected):
# We should always have global_index less than our source.
+ if selected == 0: return
cdef np.int64_t index = data.global_index * 8
cdef np.float64_t **p = <np.float64_t**> data.array
index += ((data.ind[2]*2)+data.ind[1])*2+data.ind[0]
p[1][data.index] = p[0][index]
data.index += 1
-cdef void visit_count_octs(Oct *o, OctVisitorData *data):
- # Number of cells visited
- data.index += 1
-
-cdef void visit_count_total_octs(Oct *o, OctVisitorData *data):
+cdef void visit_count_total_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
+ # Count even if not selected.
# Number of *octs* visited.
if data.last != o.domain_ind:
data.index += 1
data.last = o.domain_ind
-cdef void visit_mark_octs(Oct *o, OctVisitorData *data):
+cdef void visit_mark_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
+ # We mark them even if they are not selected
cdef int i
cdef np.uint8_t *arr = <np.uint8_t *> data.array
if data.last != o.domain_ind:
@@ -859,14 +858,16 @@
index += ((data.ind[2]*2)+data.ind[1])*2+data.ind[0]
arr[index] = 1
-cdef void visit_mask_octs(Oct *o, OctVisitorData *data):
+cdef void visit_mask_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
+ if selected == 0: return
cdef int i
cdef np.uint8_t *arr = <np.uint8_t *> data.array
- cdef np.int64_t index = data.index * 8
+ cdef np.int64_t index = data.global_index * 8
index += ((data.ind[2]*2)+data.ind[1])*2+data.ind[0]
arr[index] = 1
-cdef void visit_index_octs(Oct *o, OctVisitorData *data):
+cdef void visit_index_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
+ # Note that we provide an index even if the cell is not selected.
cdef int i
cdef np.int64_t *arr
if data.last != o.domain_ind:
@@ -875,22 +876,25 @@
arr[o.domain_ind] = data.index
data.index += 1
-cdef void visit_icoords_octs(Oct *o, OctVisitorData *data):
+cdef void visit_icoords_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
+ if selected == 0: return
cdef np.int64_t *coords = <np.int64_t*> data.array
cdef int i
for i in range(3):
coords[data.index * 3 + i] = (o.pos[i] << 1) + data.ind[i]
data.index += 1
-cdef void visit_ires_octs(Oct *o, OctVisitorData *data):
+cdef void visit_ires_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
+ if selected == 0: return
cdef np.int64_t *ires = <np.int64_t*> data.array
ires[data.index] = o.level
data.index += 1
-cdef void visit_fcoords_octs(Oct *o, OctVisitorData *data):
+cdef void visit_fcoords_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
# Note that this does not actually give the correct floating point
# coordinates. It gives them in some unit system where the domain is 1.0
# in all directions, and assumes that they will be scaled later.
+ if selected == 0: return
cdef np.float64_t *fcoords = <np.float64_t*> data.array
cdef int i
cdef np.float64_t c, dx
@@ -900,10 +904,11 @@
fcoords[data.index * 3 + i] = (c + 0.5) * dx
data.index += 1
-cdef void visit_fwidth_octs(Oct *o, OctVisitorData *data):
+cdef void visit_fwidth_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
# Note that this does not actually give the correct floating point
# coordinates. It gives them in some unit system where the domain is 1.0
# in all directions, and assumes that they will be scaled later.
+ if selected == 0: return
cdef np.float64_t *fwidth = <np.float64_t*> data.array
cdef int i
cdef np.float64_t dx
diff -r e3a9b69f4b0c766138f7249fe05c70576b044907 -r 14f5ff9a442112c858c84f8274bd72cef19fc66c yt/geometry/particle_oct_container.pyx
--- a/yt/geometry/particle_oct_container.pyx
+++ b/yt/geometry/particle_oct_container.pyx
@@ -28,7 +28,7 @@
from oct_container cimport OctreeContainer, Oct, OctInfo, ORDER_MAX, \
visit_icoords_octs, visit_ires_octs, \
visit_fcoords_octs, visit_fwidth_octs, \
- visit_count_octs, visit_count_total_octs, \
+ visit_count_total_octs, \
visit_mark_octs, visit_index_octs, visit_copy_array
from libc.stdlib cimport malloc, free, qsort
from libc.math cimport floor
diff -r e3a9b69f4b0c766138f7249fe05c70576b044907 -r 14f5ff9a442112c858c84f8274bd72cef19fc66c yt/geometry/selection_routines.pxd
--- a/yt/geometry/selection_routines.pxd
+++ b/yt/geometry/selection_routines.pxd
@@ -33,7 +33,8 @@
int ind[3]
void *array
-ctypedef void oct_visitor_function(Oct *, OctVisitorData *visitor)
+ctypedef void oct_visitor_function(Oct *, OctVisitorData *visitor,
+ np.uint8_t selected)
cdef class SelectorObject:
cdef public np.int32_t min_level
diff -r e3a9b69f4b0c766138f7249fe05c70576b044907 -r 14f5ff9a442112c858c84f8274bd72cef19fc66c yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -30,7 +30,7 @@
from fp_utils cimport fclip, iclip
from selection_routines cimport SelectorObject
from oct_container cimport OctreeContainer, OctAllocationContainer, Oct, \
- visit_mark_octs, visit_count_total_octs
+ visit_mark_octs, visit_count_total_octs, visit_mask_octs
#from geometry_utils cimport point_to_hilbert
from yt.utilities.lib.grid_traversal cimport \
VolumeContainer, sample_function, walk_volume
@@ -116,8 +116,8 @@
# Now our visitor functions
-cdef void visit_count_octs(Oct *o, OctVisitorData *data):
- data.index += 1
+cdef void visit_count_cells(Oct *o, OctVisitorData *data, np.uint8_t selected):
+ data.index += selected
cdef class SelectorObject:
@@ -150,27 +150,21 @@
@cython.wraparound(False)
@cython.cdivision(True)
def select_octs(self, OctreeContainer octree):
- cdef int i, j, k, n
- cdef np.ndarray[np.uint8_t, ndim=2] mask = np.zeros((octree.nocts, 8), dtype='uint8')
- cdef np.float64_t pos[3], dds[3]
- # This dds is the oct-width
- for i in range(3):
- dds[i] = (octree.DRE[i] - octree.DLE[i]) / octree.nn[i]
- # Pos is the center of the octs
- pos[0] = octree.DLE[0] + dds[0]/2.0
- for i in range(octree.nn[0]):
- pos[1] = octree.DLE[1] + dds[1]/2.0
- for j in range(octree.nn[1]):
- pos[2] = octree.DLE[2] + dds[2]/2.0
- for k in range(octree.nn[2]):
- if octree.root_mesh[i][j][k] == NULL: continue
- self.recursively_select_octs(
- octree.root_mesh[i][j][k],
- pos, dds, mask, 0)
- pos[2] += dds[2]
- pos[1] += dds[1]
- pos[0] += dds[0]
- return mask.astype("bool")
+ # There has to be a better way to do this.
+ cdef OctVisitorData data
+ data.index = 0
+ data.last = -1
+ data.global_index = -1
+ octree.visit_all_octs(self, visit_count_total_octs, &data)
+ cdef np.ndarray[np.uint8_t, ndim=4] m2 = \
+ np.zeros((2, 2, 2, data.index), 'uint8', order='C')
+ # This is where we'll -- in the future -- cut up based on indices of
+ # the octs.
+ data.index = -1
+ data.last = -1
+ data.array = m2.data
+ octree.visit_all_octs(self, visit_mask_octs, &data)
+ return m2.astype("bool")
@cython.boundscheck(False)
@cython.wraparound(False)
@@ -234,7 +228,7 @@
def count_octs(self, OctreeContainer octree):
cdef OctVisitorData data
data.index = 0
- octree.visit_all_octs(self, visit_count_octs, &data)
+ octree.visit_all_octs(self, visit_count_cells, &data)
return data.index
@cython.boundscheck(False)
@@ -248,6 +242,7 @@
cdef np.float64_t LE[3], RE[3], sdds[3], spos[3]
cdef int i, j, k, res, ii
cdef Oct *ch
+ cdef np.uint8_t selected
# Remember that pos is the *center* of the oct, and dds is the oct
# width. So to get to the edges, we add/subtract half of dds.
for i in range(3):
@@ -270,7 +265,7 @@
this_level = 0
if res == 0 and this_level == 1:
return
- data.global_index += 1
+ cdef int increment = 1
# Now we visit all our children. We subtract off sdds for the first
# pass because we center it on the first cell.
spos[0] = pos[0] - sdds[0]/2.0
@@ -284,12 +279,14 @@
if next_level == 1 and ch != NULL:
self.recursively_visit_octs(
ch, spos, sdds, level + 1, func, data)
- elif this_level == 1 and self.select_cell(
- spos, sdds, eterm):
+ elif this_level == 1:
+ data.global_index += increment
+ increment = 0
+ selected = self.select_cell(spos, sdds, eterm)
data.ind[0] = i
data.ind[1] = j
data.ind[2] = k
- func(root, data)
+ func(root, data, selected)
spos[2] += sdds[2]
spos[1] += sdds[1]
spos[0] += sdds[0]
https://bitbucket.org/yt_analysis/yt-3.0/commits/3ffaeb736bef/
Changeset: 3ffaeb736bef
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-18 23:23:37
Summary: Removing print statement.
Affected #: 1 file
diff -r 14f5ff9a442112c858c84f8274bd72cef19fc66c -r 3ffaeb736bef058752f2b4d9f15f03a171b76ebf yt/data_objects/data_containers.py
--- a/yt/data_objects/data_containers.py
+++ b/yt/data_objects/data_containers.py
@@ -248,7 +248,6 @@
if ngz == 0:
for io_chunk in self.chunks([], "io"):
for i,chunk in enumerate(self.chunks(field, "spatial", ngz = 0)):
- print self.selector
ind += self._current_chunk.objs[0].select(
self.selector, self[field], rv, ind)
else:
https://bitbucket.org/yt_analysis/yt-3.0/commits/c051307ad745/
Changeset: c051307ad745
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-18 23:45:28
Summary: A few more changes to try to get chunking working properly.
Affected #: 2 files
diff -r 3ffaeb736bef058752f2b4d9f15f03a171b76ebf -r c051307ad74525a6e6a5f48713c79cb42c72c925 yt/geometry/particle_geometry_handler.py
--- a/yt/geometry/particle_geometry_handler.py
+++ b/yt/geometry/particle_geometry_handler.py
@@ -146,7 +146,8 @@
if data_files is None:
data_files = [self.data_files[i] for i in
self.regions.identify_data_files(dobj.selector)]
- subset = [ParticleOctreeSubset(dobj, data_files,
+ base_region = getattr(dobj, "base_region", dobj)
+ subset = [ParticleOctreeSubset(base_region, data_files,
self.parameter_file)]
dobj._chunk_info = subset
dobj._current_chunk = list(self._chunk_all(dobj))[0]
@@ -181,3 +182,11 @@
self.oct_handler = oct_handler
self.regions = regions
super(ParticleDataChunk, self).__init__(*args, **kwargs)
+
+ def _accumulate_values(self, method):
+ mfunc = getattr(self.oct_handler, method)
+ rv = mfunc(self.dobj.selector)
+ self._data_size = rv.shape[0]
+ return rv
+
+
diff -r 3ffaeb736bef058752f2b4d9f15f03a171b76ebf -r c051307ad74525a6e6a5f48713c79cb42c72c925 yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -1267,6 +1267,7 @@
cdef int select_cell(self, np.float64_t pos[3], np.float64_t dds[3],
int eterm[3]) nogil:
return 1
+ return self.base_selector.select_cell(pos, dds, eterm)
cdef int select_grid(self, np.float64_t left_edge[3],
np.float64_t right_edge[3], np.int32_t level) nogil:
https://bitbucket.org/yt_analysis/yt-3.0/commits/725fe2106c30/
Changeset: 725fe2106c30
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-19 00:44:00
Summary: Refactoring oct visitor functions into their own module.
Affected #: 7 files
diff -r c051307ad74525a6e6a5f48713c79cb42c72c925 -r 725fe2106c30074adcbf993e89081186d717c110 yt/geometry/oct_container.pxd
--- a/yt/geometry/oct_container.pxd
+++ b/yt/geometry/oct_container.pxd
@@ -27,6 +27,7 @@
from fp_utils cimport *
from selection_routines cimport SelectorObject, \
OctVisitorData, oct_visitor_function
+from oct_visitors cimport *
cdef int ORDER_MAX
@@ -78,14 +79,3 @@
cdef Oct *next_root(self, int domain_id, int ind[3])
cdef Oct *next_child(self, int domain_id, int ind[3], Oct *parent)
-# Now some visitor functions
-
-cdef oct_visitor_function visit_count_total_octs
-cdef oct_visitor_function visit_mark_octs
-cdef oct_visitor_function visit_mask_octs
-cdef oct_visitor_function visit_index_octs
-cdef oct_visitor_function visit_icoords_octs
-cdef oct_visitor_function visit_ires_octs
-cdef oct_visitor_function visit_fcoords_octs
-cdef oct_visitor_function visit_fwidth_octs
-cdef oct_visitor_function visit_copy_array
diff -r c051307ad74525a6e6a5f48713c79cb42c72c925 -r 725fe2106c30074adcbf993e89081186d717c110 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -31,8 +31,7 @@
import numpy as np
from oct_container cimport Oct, OctAllocationContainer, \
OctreeContainer, ORDER_MAX
-from selection_routines cimport SelectorObject, \
- OctVisitorData, oct_visitor_function
+cimport oct_visitors
cimport cython
ORDER_MAX = 20
@@ -829,90 +828,3 @@
local_filled += 1
return local_filled
-# Now some visitor functions
-
-cdef void visit_copy_array(Oct *o, OctVisitorData *data, np.uint8_t selected):
- # We should always have global_index less than our source.
- if selected == 0: return
- cdef np.int64_t index = data.global_index * 8
- cdef np.float64_t **p = <np.float64_t**> data.array
- index += ((data.ind[2]*2)+data.ind[1])*2+data.ind[0]
- p[1][data.index] = p[0][index]
- data.index += 1
-
-cdef void visit_count_total_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
- # Count even if not selected.
- # Number of *octs* visited.
- if data.last != o.domain_ind:
- data.index += 1
- data.last = o.domain_ind
-
-cdef void visit_mark_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
- # We mark them even if they are not selected
- cdef int i
- cdef np.uint8_t *arr = <np.uint8_t *> data.array
- if data.last != o.domain_ind:
- data.last = o.domain_ind
- data.index += 1
- cdef np.int64_t index = data.index * 8
- index += ((data.ind[2]*2)+data.ind[1])*2+data.ind[0]
- arr[index] = 1
-
-cdef void visit_mask_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
- if selected == 0: return
- cdef int i
- cdef np.uint8_t *arr = <np.uint8_t *> data.array
- cdef np.int64_t index = data.global_index * 8
- index += ((data.ind[2]*2)+data.ind[1])*2+data.ind[0]
- arr[index] = 1
-
-cdef void visit_index_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
- # Note that we provide an index even if the cell is not selected.
- cdef int i
- cdef np.int64_t *arr
- if data.last != o.domain_ind:
- data.last = o.domain_ind
- arr = <np.int64_t *> data.array
- arr[o.domain_ind] = data.index
- data.index += 1
-
-cdef void visit_icoords_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
- if selected == 0: return
- cdef np.int64_t *coords = <np.int64_t*> data.array
- cdef int i
- for i in range(3):
- coords[data.index * 3 + i] = (o.pos[i] << 1) + data.ind[i]
- data.index += 1
-
-cdef void visit_ires_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
- if selected == 0: return
- cdef np.int64_t *ires = <np.int64_t*> data.array
- ires[data.index] = o.level
- data.index += 1
-
-cdef void visit_fcoords_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
- # Note that this does not actually give the correct floating point
- # coordinates. It gives them in some unit system where the domain is 1.0
- # in all directions, and assumes that they will be scaled later.
- if selected == 0: return
- cdef np.float64_t *fcoords = <np.float64_t*> data.array
- cdef int i
- cdef np.float64_t c, dx
- dx = 1.0 / (2 << o.level)
- for i in range(3):
- c = <np.float64_t> ((o.pos[i] << 1 ) + data.ind[i])
- fcoords[data.index * 3 + i] = (c + 0.5) * dx
- data.index += 1
-
-cdef void visit_fwidth_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
- # Note that this does not actually give the correct floating point
- # coordinates. It gives them in some unit system where the domain is 1.0
- # in all directions, and assumes that they will be scaled later.
- if selected == 0: return
- cdef np.float64_t *fwidth = <np.float64_t*> data.array
- cdef int i
- cdef np.float64_t dx
- dx = 1.0 / (2 << o.level)
- for i in range(3):
- fwidth[data.index * 3 + i] = dx
- data.index += 1
diff -r c051307ad74525a6e6a5f48713c79cb42c72c925 -r 725fe2106c30074adcbf993e89081186d717c110 yt/geometry/oct_visitors.pxd
--- /dev/null
+++ b/yt/geometry/oct_visitors.pxd
@@ -0,0 +1,42 @@
+"""
+Oct visitor definitions file
+
+Author: Matthew Turk <matthewturk at gmail.com>
+Affiliation: Columbia University
+Homepage: http://yt.enzotools.org/
+License:
+ Copyright (C) 2013 Matthew Turk. All Rights Reserved.
+
+ This file is part of yt.
+
+ yt is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+
+cimport numpy as np
+from selection_routines cimport \
+ OctVisitorData, oct_visitor_function
+from oct_container cimport \
+ Oct
+
+cdef oct_visitor_function count_total_octs
+cdef oct_visitor_function count_total_cells
+cdef oct_visitor_function mark_octs
+cdef oct_visitor_function mask_octs
+cdef oct_visitor_function index_octs
+cdef oct_visitor_function icoords_octs
+cdef oct_visitor_function ires_octs
+cdef oct_visitor_function fcoords_octs
+cdef oct_visitor_function fwidth_octs
+cdef oct_visitor_function copy_array_f64
+cdef oct_visitor_function copy_array_i64
diff -r c051307ad74525a6e6a5f48713c79cb42c72c925 -r 725fe2106c30074adcbf993e89081186d717c110 yt/geometry/oct_visitors.pyx
--- /dev/null
+++ b/yt/geometry/oct_visitors.pyx
@@ -0,0 +1,134 @@
+"""
+Oct visitor functions
+
+Author: Matthew Turk <matthewturk at gmail.com>
+Affiliation: Columbia University
+Author: Christopher Moody <chris.e.moody at gmail.com>
+Affiliation: UC Santa Cruz
+Homepage: http://yt.enzotools.org/
+License:
+ Copyright (C) 2013 Matthew Turk. All Rights Reserved.
+
+ This file is part of yt.
+
+ yt is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+
+cimport numpy
+import numpy
+from fp_utils cimport *
+
+# Now some visitor functions
+
+cdef void copy_array_f64(Oct *o, OctVisitorData *data, np.uint8_t selected):
+ # We should always have global_index less than our source.
+ # "last" here tells us the dimensionality of the array.
+ if selected == 0: return
+ cdef np.int64_t index = (data.global_index * 8)*data.last
+ cdef np.float64_t **p = <np.float64_t**> data.array
+ index += ((data.ind[2]*2)+data.ind[1])*2+data.ind[0]
+ p[1][data.index] = p[0][index]
+ data.index += 1
+
+cdef void copy_array_i64(Oct *o, OctVisitorData *data, np.uint8_t selected):
+ # We should always have global_index less than our source.
+ # "last" here tells us the dimensionality of the array.
+ if selected == 0: return
+ cdef np.int64_t index = (data.global_index * 8)*data.last
+ cdef np.int64_t **p = <np.int64_t**> data.array
+ index += ((data.ind[2]*2)+data.ind[1])*2+data.ind[0]
+ p[1][data.index] = p[0][index]
+ data.index += 1
+
+cdef void count_total_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
+ # Count even if not selected.
+ # Number of *octs* visited.
+ if data.last != o.domain_ind:
+ data.index += 1
+ data.last = o.domain_ind
+
+cdef void count_total_cells(Oct *o, OctVisitorData *data, np.uint8_t selected):
+ # Count even if not selected.
+ # Number of *octs* visited.
+ data.index += selected
+
+cdef void mark_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
+ # We mark them even if they are not selected
+ cdef int i
+ cdef np.uint8_t *arr = <np.uint8_t *> data.array
+ if data.last != o.domain_ind:
+ data.last = o.domain_ind
+ data.index += 1
+ cdef np.int64_t index = data.index * 8
+ index += ((data.ind[2]*2)+data.ind[1])*2+data.ind[0]
+ arr[index] = 1
+
+cdef void mask_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
+ if selected == 0: return
+ cdef int i
+ cdef np.uint8_t *arr = <np.uint8_t *> data.array
+ cdef np.int64_t index = data.global_index * 8
+ index += ((data.ind[2]*2)+data.ind[1])*2+data.ind[0]
+ arr[index] = 1
+
+cdef void index_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
+ # Note that we provide an index even if the cell is not selected.
+ cdef int i
+ cdef np.int64_t *arr
+ if data.last != o.domain_ind:
+ data.last = o.domain_ind
+ arr = <np.int64_t *> data.array
+ arr[o.domain_ind] = data.index
+ data.index += 1
+
+cdef void icoords_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
+ if selected == 0: return
+ cdef np.int64_t *coords = <np.int64_t*> data.array
+ cdef int i
+ for i in range(3):
+ coords[data.index * 3 + i] = (o.pos[i] << 1) + data.ind[i]
+ data.index += 1
+
+cdef void ires_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
+ if selected == 0: return
+ cdef np.int64_t *ires = <np.int64_t*> data.array
+ ires[data.index] = o.level
+ data.index += 1
+
+cdef void fcoords_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
+ # Note that this does not actually give the correct floating point
+ # coordinates. It gives them in some unit system where the domain is 1.0
+ # in all directions, and assumes that they will be scaled later.
+ if selected == 0: return
+ cdef np.float64_t *fcoords = <np.float64_t*> data.array
+ cdef int i
+ cdef np.float64_t c, dx
+ dx = 1.0 / (2 << o.level)
+ for i in range(3):
+ c = <np.float64_t> ((o.pos[i] << 1 ) + data.ind[i])
+ fcoords[data.index * 3 + i] = (c + 0.5) * dx
+ data.index += 1
+
+cdef void fwidth_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
+ # Note that this does not actually give the correct floating point
+ # coordinates. It gives them in some unit system where the domain is 1.0
+ # in all directions, and assumes that they will be scaled later.
+ if selected == 0: return
+ cdef np.float64_t *fwidth = <np.float64_t*> data.array
+ cdef int i
+ cdef np.float64_t dx
+ dx = 1.0 / (2 << o.level)
+ for i in range(3):
+ fwidth[data.index * 3 + i] = dx
+ data.index += 1
diff -r c051307ad74525a6e6a5f48713c79cb42c72c925 -r 725fe2106c30074adcbf993e89081186d717c110 yt/geometry/particle_oct_container.pyx
--- a/yt/geometry/particle_oct_container.pyx
+++ b/yt/geometry/particle_oct_container.pyx
@@ -25,11 +25,8 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
-from oct_container cimport OctreeContainer, Oct, OctInfo, ORDER_MAX, \
- visit_icoords_octs, visit_ires_octs, \
- visit_fcoords_octs, visit_fwidth_octs, \
- visit_count_total_octs, \
- visit_mark_octs, visit_index_octs, visit_copy_array
+from oct_container cimport OctreeContainer, Oct, OctInfo, ORDER_MAX
+cimport oct_visitors
from libc.stdlib cimport malloc, free, qsort
from libc.math cimport floor
from fp_utils cimport *
@@ -120,7 +117,7 @@
cdef OctVisitorData data
data.array = <void *> coords.data
data.index = 0
- self.visit_all_octs(selector, visit_icoords_octs, &data)
+ self.visit_all_octs(selector, oct_visitors.icoords_octs, &data)
return coords
@cython.boundscheck(False)
@@ -135,7 +132,7 @@
cdef OctVisitorData data
data.array = <void *> res.data
data.index = 0
- self.visit_all_octs(selector, visit_ires_octs, &data)
+ self.visit_all_octs(selector, oct_visitors.ires_octs, &data)
return res
@cython.boundscheck(False)
@@ -149,7 +146,7 @@
cdef OctVisitorData data
data.array = <void *> fwidth.data
data.index = 0
- self.visit_all_octs(selector, visit_fwidth_octs, &data)
+ self.visit_all_octs(selector, oct_visitors.fwidth_octs, &data)
cdef np.float64_t base_dx
for i in range(3):
base_dx = (self.DRE[i] - self.DLE[i])/self.nn[i]
@@ -168,7 +165,7 @@
cdef OctVisitorData data
data.array = <void *> coords.data
data.index = 0
- self.visit_all_octs(selector, visit_fcoords_octs, &data)
+ self.visit_all_octs(selector, oct_visitors.fcoords_octs, &data)
cdef int i
cdef np.float64_t base_dx
for i in range(3):
@@ -345,25 +342,34 @@
data.array = ind.data
data.index = 0
data.last = -1
- self.visit_all_octs(selector, visit_index_octs, &data)
+ self.visit_all_octs(selector, oct_visitors.index_octs, &data)
return ind
def selector_fill(self, SelectorObject selector,
- np.ndarray[np.float64_t, ndim=4] source,
- np.ndarray[np.float64_t, ndim=1] dest,
- np.int64_t offset):
+ np.ndarray source,
+ np.ndarray dest = None,
+ np.int64_t offset = 0, int dims = 1):
# This is actually not correct. The hard part is that we need to
# iterate the same way visit_all_octs does, but we need to track the
# number of octs total visited.
cdef OctVisitorData data
data.index = offset
# We only need this so we can continue calculating the offset
- data.last = -1
+ data.last = dims
cdef void *p[2]
p[0] = source.data
p[1] = dest.data
data.array = &p
- self.visit_all_octs(selector, visit_copy_array, &data)
+ cdef oct_visitor_function *func
+ if source.dtype != dest.dtype:
+ raise RuntimeError
+ if source.dtype == np.int64:
+ func = oct_visitors.copy_array_i64
+ elif source.dtype == np.float64:
+ func = oct_visitors.copy_array_f64
+ else:
+ raise NotImplementedError
+ self.visit_all_octs(selector, func, &data)
return data.index - offset
cdef class ParticleRegions:
diff -r c051307ad74525a6e6a5f48713c79cb42c72c925 -r 725fe2106c30074adcbf993e89081186d717c110 yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -29,8 +29,8 @@
from libc.stdlib cimport malloc, free
from fp_utils cimport fclip, iclip
from selection_routines cimport SelectorObject
-from oct_container cimport OctreeContainer, OctAllocationContainer, Oct, \
- visit_mark_octs, visit_count_total_octs, visit_mask_octs
+from oct_container cimport OctreeContainer, OctAllocationContainer, Oct
+cimport oct_visitors
#from geometry_utils cimport point_to_hilbert
from yt.utilities.lib.grid_traversal cimport \
VolumeContainer, sample_function, walk_volume
@@ -114,11 +114,6 @@
else:
raise RuntimeError
-# Now our visitor functions
-
-cdef void visit_count_cells(Oct *o, OctVisitorData *data, np.uint8_t selected):
- data.index += selected
-
cdef class SelectorObject:
def __cinit__(self, dobj):
@@ -155,7 +150,7 @@
data.index = 0
data.last = -1
data.global_index = -1
- octree.visit_all_octs(self, visit_count_total_octs, &data)
+ octree.visit_all_octs(self, oct_visitors.count_total_octs, &data)
cdef np.ndarray[np.uint8_t, ndim=4] m2 = \
np.zeros((2, 2, 2, data.index), 'uint8', order='C')
# This is where we'll -- in the future -- cut up based on indices of
@@ -163,7 +158,7 @@
data.index = -1
data.last = -1
data.array = m2.data
- octree.visit_all_octs(self, visit_mask_octs, &data)
+ octree.visit_all_octs(self, oct_visitors.mask_octs, &data)
return m2.astype("bool")
@cython.boundscheck(False)
@@ -228,7 +223,7 @@
def count_octs(self, OctreeContainer octree):
cdef OctVisitorData data
data.index = 0
- octree.visit_all_octs(self, visit_count_cells, &data)
+ octree.visit_all_octs(self, oct_visitors.count_total_cells, &data)
return data.index
@cython.boundscheck(False)
@@ -1232,7 +1227,7 @@
cdef OctVisitorData data
data.index = 0
data.last = -1
- octree.visit_all_octs(self, visit_count_total_octs, &data)
+ octree.visit_all_octs(self, oct_visitors.count_total_octs, &data)
cdef np.ndarray[np.uint8_t, ndim=4] m2 = \
np.zeros((2, 2, 2, data.index), 'uint8', order='C')
# This is where we'll -- in the future -- cut up based on indices of
@@ -1240,7 +1235,7 @@
data.index = -1
data.last = -1
data.array = m2.data
- octree.visit_all_octs(self, visit_mark_octs, &data)
+ octree.visit_all_octs(self, oct_visitors.mark_octs, &data)
return m2.astype("bool")
@cython.boundscheck(False)
diff -r c051307ad74525a6e6a5f48713c79cb42c72c925 -r 725fe2106c30074adcbf993e89081186d717c110 yt/geometry/setup.py
--- a/yt/geometry/setup.py
+++ b/yt/geometry/setup.py
@@ -14,6 +14,13 @@
depends=["yt/utilities/lib/fp_utils.pxd",
"yt/geometry/oct_container.pxd",
"yt/geometry/selection_routines.pxd"])
+ config.add_extension("oct_visitors",
+ ["yt/geometry/oct_visitors.pyx"],
+ include_dirs=["yt/utilities/lib/"],
+ libraries=["m"],
+ depends=["yt/utilities/lib/fp_utils.pxd",
+ "yt/geometry/oct_container.pxd",
+ "yt/geometry/selection_routines.pxd"])
config.add_extension("particle_oct_container",
["yt/geometry/particle_oct_container.pyx"],
include_dirs=["yt/utilities/lib/"],
https://bitbucket.org/yt_analysis/yt-3.0/commits/1d2a9b997cf8/
Changeset: 1d2a9b997cf8
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-19 04:23:16
Summary: Initial attempt at implementing multi-dim coord selection.
Affected #: 4 files
diff -r 725fe2106c30074adcbf993e89081186d717c110 -r 1d2a9b997cf8df9e2e636fc536fc8754ec56f552 yt/data_objects/octree_subset.py
--- a/yt/data_objects/octree_subset.py
+++ b/yt/data_objects/octree_subset.py
@@ -220,17 +220,25 @@
return self._reshape_vals(vals)
def select_icoords(self, dobj):
- return self.oct_handler.icoords(dobj.selector)
+ d = self.oct_handler.icoords(self.selector)
+ tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 3)
+ return tr
def select_fcoords(self, dobj):
- return self.oct_handler.fcoords(dobj.selector)
+ d = self.oct_handler.fcoords(self.selector)
+ tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 3)
+ return tr
def select_fwidth(self, dobj):
# Recall domain_dimensions is the number of cells, not octs
- return self.oct_handler.fwidth(dobj.selector)
+ d = self.oct_handler.fwidth(self.selector)
+ tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 3)
+ return tr
def select_ires(self, dobj):
- return self.oct_handler.ires(dobj.selector)
+ d = self.oct_handler.ires(self.selector)
+ tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 1)
+ return tr
def select(self, selector, source, dest, offset):
n = self.oct_handler.selector_fill(selector, source, dest, offset)
diff -r 725fe2106c30074adcbf993e89081186d717c110 -r 1d2a9b997cf8df9e2e636fc536fc8754ec56f552 yt/geometry/oct_visitors.pyx
--- a/yt/geometry/oct_visitors.pyx
+++ b/yt/geometry/oct_visitors.pyx
@@ -35,21 +35,25 @@
# We should always have global_index less than our source.
# "last" here tells us the dimensionality of the array.
if selected == 0: return
+ cdef int i
cdef np.int64_t index = (data.global_index * 8)*data.last
cdef np.float64_t **p = <np.float64_t**> data.array
index += ((data.ind[2]*2)+data.ind[1])*2+data.ind[0]
- p[1][data.index] = p[0][index]
- data.index += 1
+ for i in range(data.last):
+ p[1][data.index + i] = p[0][index + i]
+ data.index += data.last
cdef void copy_array_i64(Oct *o, OctVisitorData *data, np.uint8_t selected):
# We should always have global_index less than our source.
# "last" here tells us the dimensionality of the array.
if selected == 0: return
+ cdef int i
cdef np.int64_t index = (data.global_index * 8)*data.last
cdef np.int64_t **p = <np.int64_t**> data.array
index += ((data.ind[2]*2)+data.ind[1])*2+data.ind[0]
- p[1][data.index] = p[0][index]
- data.index += 1
+ for i in range(data.last):
+ p[1][data.index + i] = p[0][index + i]
+ data.index += data.last
cdef void count_total_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
# Count even if not selected.
diff -r 725fe2106c30074adcbf993e89081186d717c110 -r 1d2a9b997cf8df9e2e636fc536fc8754ec56f552 yt/geometry/particle_geometry_handler.py
--- a/yt/geometry/particle_geometry_handler.py
+++ b/yt/geometry/particle_geometry_handler.py
@@ -182,11 +182,3 @@
self.oct_handler = oct_handler
self.regions = regions
super(ParticleDataChunk, self).__init__(*args, **kwargs)
-
- def _accumulate_values(self, method):
- mfunc = getattr(self.oct_handler, method)
- rv = mfunc(self.dobj.selector)
- self._data_size = rv.shape[0]
- return rv
-
-
diff -r 725fe2106c30074adcbf993e89081186d717c110 -r 1d2a9b997cf8df9e2e636fc536fc8754ec56f552 yt/geometry/particle_oct_container.pyx
--- a/yt/geometry/particle_oct_container.pyx
+++ b/yt/geometry/particle_oct_container.pyx
@@ -352,6 +352,15 @@
# This is actually not correct. The hard part is that we need to
# iterate the same way visit_all_octs does, but we need to track the
# number of octs total visited.
+ cdef np.int64_t num_cells = -1
+ if dest is None:
+ num_cells = selector.count_octs(self)
+ if dims > 1:
+ dest = np.zeros((num_cells, dims), dtype=source.dtype,
+ order='C')
+ else:
+ dest = np.zeros(num_cells, dtype=source.dtype, order='C')
+ dest = dest - 10000
cdef OctVisitorData data
data.index = offset
# We only need this so we can continue calculating the offset
@@ -370,6 +379,8 @@
else:
raise NotImplementedError
self.visit_all_octs(selector, func, &data)
+ if num_cells >= 0:
+ return dest
return data.index - offset
cdef class ParticleRegions:
https://bitbucket.org/yt_analysis/yt-3.0/commits/886547e04cbc/
Changeset: 886547e04cbc
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-19 05:03:51
Summary: Need to increment by 'data.last' between copies.
Affected #: 2 files
diff -r 1d2a9b997cf8df9e2e636fc536fc8754ec56f552 -r 886547e04cbc4537419fa1f1689c1d378195a0f0 yt/data_objects/octree_subset.py
--- a/yt/data_objects/octree_subset.py
+++ b/yt/data_objects/octree_subset.py
@@ -230,7 +230,6 @@
return tr
def select_fwidth(self, dobj):
- # Recall domain_dimensions is the number of cells, not octs
d = self.oct_handler.fwidth(self.selector)
tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 3)
return tr
diff -r 1d2a9b997cf8df9e2e636fc536fc8754ec56f552 -r 886547e04cbc4537419fa1f1689c1d378195a0f0 yt/geometry/oct_visitors.pyx
--- a/yt/geometry/oct_visitors.pyx
+++ b/yt/geometry/oct_visitors.pyx
@@ -38,7 +38,7 @@
cdef int i
cdef np.int64_t index = (data.global_index * 8)*data.last
cdef np.float64_t **p = <np.float64_t**> data.array
- index += ((data.ind[2]*2)+data.ind[1])*2+data.ind[0]
+ index += (((data.ind[2]*2)+data.ind[1])*2+data.ind[0])*data.last
for i in range(data.last):
p[1][data.index + i] = p[0][index + i]
data.index += data.last
@@ -50,7 +50,7 @@
cdef int i
cdef np.int64_t index = (data.global_index * 8)*data.last
cdef np.int64_t **p = <np.int64_t**> data.array
- index += ((data.ind[2]*2)+data.ind[1])*2+data.ind[0]
+ index += (((data.ind[2]*2)+data.ind[1])*2+data.ind[0])*data.last
for i in range(data.last):
p[1][data.index + i] = p[0][index + i]
data.index += data.last
https://bitbucket.org/yt_analysis/yt-3.0/commits/c78f757d2bb4/
Changeset: c78f757d2bb4
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-19 05:06:18
Summary: Ordering fix.
Note that we still seem to be getting some spurious cell/oct selection that is
visible in projections using spherical data sources.
Affected #: 1 file
diff -r 886547e04cbc4537419fa1f1689c1d378195a0f0 -r c78f757d2bb415f5d00648421f4e021f882b4ccc yt/geometry/oct_visitors.pyx
--- a/yt/geometry/oct_visitors.pyx
+++ b/yt/geometry/oct_visitors.pyx
@@ -38,7 +38,7 @@
cdef int i
cdef np.int64_t index = (data.global_index * 8)*data.last
cdef np.float64_t **p = <np.float64_t**> data.array
- index += (((data.ind[2]*2)+data.ind[1])*2+data.ind[0])*data.last
+ index += (((data.ind[0]*2)+data.ind[1])*2+data.ind[2])*data.last
for i in range(data.last):
p[1][data.index + i] = p[0][index + i]
data.index += data.last
@@ -50,7 +50,7 @@
cdef int i
cdef np.int64_t index = (data.global_index * 8)*data.last
cdef np.int64_t **p = <np.int64_t**> data.array
- index += (((data.ind[2]*2)+data.ind[1])*2+data.ind[0])*data.last
+ index += (((data.ind[0]*2)+data.ind[1])*2+data.ind[2])*data.last
for i in range(data.last):
p[1][data.index + i] = p[0][index + i]
data.index += data.last
https://bitbucket.org/yt_analysis/yt-3.0/commits/049580841736/
Changeset: 049580841736
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-19 23:39:02
Summary: Some refactoring of indexing functions.
Also allowing nvals to be a tuple.
Affected #: 4 files
diff -r c78f757d2bb415f5d00648421f4e021f882b4ccc -r 049580841736fe75201f1acbafd7b147b9eb381b yt/geometry/oct_visitors.pxd
--- a/yt/geometry/oct_visitors.pxd
+++ b/yt/geometry/oct_visitors.pxd
@@ -40,3 +40,10 @@
cdef oct_visitor_function fwidth_octs
cdef oct_visitor_function copy_array_f64
cdef oct_visitor_function copy_array_i64
+
+cdef inline int oind(OctVisitorData *data):
+ return (((data.ind[0]*2)+data.ind[1])*2+data.ind[2])
+
+cdef inline int rind(OctVisitorData *data):
+ #return (((data.ind[0]*2)+data.ind[1])*2+data.ind[2])
+ return (((data.ind[2]*2)+data.ind[1])*2+data.ind[0])
diff -r c78f757d2bb415f5d00648421f4e021f882b4ccc -r 049580841736fe75201f1acbafd7b147b9eb381b yt/geometry/oct_visitors.pyx
--- a/yt/geometry/oct_visitors.pyx
+++ b/yt/geometry/oct_visitors.pyx
@@ -38,7 +38,7 @@
cdef int i
cdef np.int64_t index = (data.global_index * 8)*data.last
cdef np.float64_t **p = <np.float64_t**> data.array
- index += (((data.ind[0]*2)+data.ind[1])*2+data.ind[2])*data.last
+ index += oind(data)*data.last
for i in range(data.last):
p[1][data.index + i] = p[0][index + i]
data.index += data.last
@@ -50,7 +50,7 @@
cdef int i
cdef np.int64_t index = (data.global_index * 8)*data.last
cdef np.int64_t **p = <np.int64_t**> data.array
- index += (((data.ind[0]*2)+data.ind[1])*2+data.ind[2])*data.last
+ index += oind(data)*data.last
for i in range(data.last):
p[1][data.index + i] = p[0][index + i]
data.index += data.last
@@ -75,7 +75,7 @@
data.last = o.domain_ind
data.index += 1
cdef np.int64_t index = data.index * 8
- index += ((data.ind[2]*2)+data.ind[1])*2+data.ind[0]
+ index += oind(data)
arr[index] = 1
cdef void mask_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
@@ -83,11 +83,12 @@
cdef int i
cdef np.uint8_t *arr = <np.uint8_t *> data.array
cdef np.int64_t index = data.global_index * 8
- index += ((data.ind[2]*2)+data.ind[1])*2+data.ind[0]
+ index += oind(data)
arr[index] = 1
cdef void index_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
# Note that we provide an index even if the cell is not selected.
+ if selected == 0: return
cdef int i
cdef np.int64_t *arr
if data.last != o.domain_ind:
diff -r c78f757d2bb415f5d00648421f4e021f882b4ccc -r 049580841736fe75201f1acbafd7b147b9eb381b yt/geometry/particle_deposit.pxd
--- a/yt/geometry/particle_deposit.pxd
+++ b/yt/geometry/particle_deposit.pxd
@@ -39,6 +39,7 @@
cdef inline int gind(int i, int j, int k, int dims[3]):
return ((i*dims[1])+j)*dims[2]+k
+ #return ((k*dims[1])+j)*dims[0]+i
####################################################
@@ -57,7 +58,8 @@
cdef class ParticleDepositOperation:
# We assume each will allocate and define their own temporary storage
- cdef np.int64_t nvals
+ cdef public object nvals
+ cdef public int bad_indices
cdef void process(self, int dim[3], np.float64_t left_edge[3],
np.float64_t dds[3], np.int64_t offset,
np.float64_t ppos[3], np.float64_t *fields)
diff -r c78f757d2bb415f5d00648421f4e021f882b4ccc -r 049580841736fe75201f1acbafd7b147b9eb381b yt/geometry/particle_deposit.pyx
--- a/yt/geometry/particle_deposit.pyx
+++ b/yt/geometry/particle_deposit.pyx
@@ -50,6 +50,7 @@
np.ndarray[np.float64_t, ndim=2] positions,
fields = None, int domain_id = -1):
cdef int nf, i, j
+ self.bad_indices = 0
if fields is None:
fields = []
nf = len(fields)
@@ -127,7 +128,7 @@
cdef public object ocount
def initialize(self):
# Create a numpy array accessible to python
- self.ocount = np.zeros(self.nvals, dtype="int64")
+ self.ocount = np.zeros(self.nvals, dtype="int64", order='F')
cdef np.ndarray arr = self.ocount
# alias the C-view for use in cython
self.count = <np.int64_t*> arr.data
@@ -161,10 +162,10 @@
cdef public object otemp
def initialize(self):
- self.odata = np.zeros(self.nvals, dtype="float64")
+ self.odata = np.zeros(self.nvals, dtype="float64", order='F')
cdef np.ndarray arr = self.odata
self.data = <np.float64_t*> arr.data
- self.otemp = np.zeros(self.nvals, dtype="float64")
+ self.otemp = np.zeros(self.nvals, dtype="float64", order='F')
arr = self.otemp
self.temp = <np.float64_t*> arr.data
@@ -217,7 +218,7 @@
cdef np.float64_t *sum
cdef public object osum
def initialize(self):
- self.osum = np.zeros(self.nvals, dtype="float64")
+ self.osum = np.zeros(self.nvals, dtype="float64", order='F')
cdef np.ndarray arr = self.osum
self.sum = <np.float64_t*> arr.data
https://bitbucket.org/yt_analysis/yt-3.0/commits/847fd9cd34ff/
Changeset: 847fd9cd34ff
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-20 12:53:21
Summary: A few more attempted fixes.
The failure of spheres to project correctly suggests to me that what is
happening is a mis-copying during the selector_fill operation. I am unable to
determine where this is going wrong; I initially thought that having the i,j,k
indices backwards could cause this, but this no longer seems to be the case to
me.
Affected #: 4 files
diff -r 049580841736fe75201f1acbafd7b147b9eb381b -r 847fd9cd34ff2ec1bb34d9ba34301d1e1433f05e yt/data_objects/octree_subset.py
--- a/yt/data_objects/octree_subset.py
+++ b/yt/data_objects/octree_subset.py
@@ -212,12 +212,12 @@
cls = getattr(particle_deposit, "deposit_%s" % method, None)
if cls is None:
raise YTParticleDepositionNotImplemented(method)
- nvals = (self.domain_ind >= 0).sum() * 8
+ nvals = (2, 2, 2, (self.domain_ind >= 0).sum())
op = cls(nvals) # We allocate number of zones, not number of octs
op.initialize()
op.process_octree(self.oct_handler, self.domain_ind, positions, fields, 0)
vals = op.finalize()
- return self._reshape_vals(vals)
+ return np.asfortranarray(vals)
def select_icoords(self, dobj):
d = self.oct_handler.icoords(self.selector)
diff -r 049580841736fe75201f1acbafd7b147b9eb381b -r 847fd9cd34ff2ec1bb34d9ba34301d1e1433f05e yt/geometry/oct_visitors.pxd
--- a/yt/geometry/oct_visitors.pxd
+++ b/yt/geometry/oct_visitors.pxd
@@ -45,5 +45,4 @@
return (((data.ind[0]*2)+data.ind[1])*2+data.ind[2])
cdef inline int rind(OctVisitorData *data):
- #return (((data.ind[0]*2)+data.ind[1])*2+data.ind[2])
return (((data.ind[2]*2)+data.ind[1])*2+data.ind[0])
diff -r 049580841736fe75201f1acbafd7b147b9eb381b -r 847fd9cd34ff2ec1bb34d9ba34301d1e1433f05e yt/geometry/oct_visitors.pyx
--- a/yt/geometry/oct_visitors.pyx
+++ b/yt/geometry/oct_visitors.pyx
@@ -36,6 +36,7 @@
# "last" here tells us the dimensionality of the array.
if selected == 0: return
cdef int i
+ # There are this many records between "octs"
cdef np.int64_t index = (data.global_index * 8)*data.last
cdef np.float64_t **p = <np.float64_t**> data.array
index += oind(data)*data.last
@@ -88,7 +89,6 @@
cdef void index_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
# Note that we provide an index even if the cell is not selected.
- if selected == 0: return
cdef int i
cdef np.int64_t *arr
if data.last != o.domain_ind:
diff -r 049580841736fe75201f1acbafd7b147b9eb381b -r 847fd9cd34ff2ec1bb34d9ba34301d1e1433f05e yt/geometry/particle_deposit.pyx
--- a/yt/geometry/particle_deposit.pyx
+++ b/yt/geometry/particle_deposit.pyx
@@ -234,6 +234,7 @@
for i in range(3):
ii[i] = <int>((ppos[i] - left_edge[i]) / dds[i])
self.sum[gind(ii[0], ii[1], ii[2], dim) + offset] += fields[0]
+ return
def finalize(self):
return self.osum
@@ -305,10 +306,11 @@
cdef np.float64_t *field
cdef public object ofield
def initialize(self):
- self.ofield = np.zeros(self.nvals, dtype="float64")
+ self.ofield = np.zeros(self.nvals, dtype="float64", order='F')
cdef np.ndarray arr = self.ofield
self.field = <np.float64_t *> arr.data
+ @cython.cdivision(True)
cdef void process(self, int dim[3],
np.float64_t left_edge[3],
np.float64_t dds[3],
https://bitbucket.org/yt_analysis/yt-3.0/commits/abf193b22ae2/
Changeset: abf193b22ae2
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-20 13:26:37
Summary: Remove deprecated routine recursively_select_octs.
Affected #: 2 files
diff -r 847fd9cd34ff2ec1bb34d9ba34301d1e1433f05e -r abf193b22ae2cc50255264b13d7b4ee3d2924e6d yt/geometry/selection_routines.pxd
--- a/yt/geometry/selection_routines.pxd
+++ b/yt/geometry/selection_routines.pxd
@@ -40,10 +40,6 @@
cdef public np.int32_t min_level
cdef public np.int32_t max_level
- cdef void recursively_select_octs(self, Oct *root,
- np.float64_t pos[3], np.float64_t dds[3],
- np.ndarray[np.uint8_t, ndim=2] mask,
- int level = ?)
cdef void recursively_visit_octs(self, Oct *root,
np.float64_t pos[3], np.float64_t dds[3],
int level,
diff -r 847fd9cd34ff2ec1bb34d9ba34301d1e1433f05e -r abf193b22ae2cc50255264b13d7b4ee3d2924e6d yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -161,65 +161,6 @@
octree.visit_all_octs(self, oct_visitors.mask_octs, &data)
return m2.astype("bool")
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
- cdef void recursively_select_octs(self, Oct *root,
- np.float64_t pos[3], np.float64_t dds[3],
- np.ndarray[np.uint8_t, ndim=2] mask,
- int level = 0):
-
- cdef np.float64_t LE[3], RE[3], sdds[3], spos[3]
- cdef int i, j, k, res, ii
- cdef Oct *ch
- # Remember that pos is the *center* of the oct, and dds is the oct
- # width. So to get to the edges, we add/subtract half of dds.
- for i in range(3):
- # sdds is the cell width
- sdds[i] = dds[i]/2.0
- LE[i] = pos[i] - dds[i]/2.0
- RE[i] = pos[i] + dds[i]/2.0
- #print LE[0], RE[0], LE[1], RE[1], LE[2], RE[2]
- res = self.select_grid(LE, RE, level)
- cdef int eterm[3]
- eterm[0] = eterm[1] = eterm[2] = 0
- cdef int next_level, this_level
- # next_level: an int that says whether or not we can progress to children
- # this_level: an int that says whether or not we can select from this
- # level
- next_level = this_level = 1
- if level == self.max_level:
- next_level = 0
- if level < self.min_level or level > self.max_level:
- this_level = 0
- if res == 0:
- for i in range(8):
- mask[root.domain_ind,i] = 0
- # If this level *is* being selected (i.e., no early termination)
- # then we know no child zones will be selected.
- if this_level == 1:
- return
- # Now we visit all our children. We subtract off sdds for the first
- # pass because we center it on the first cell.
- spos[0] = pos[0] - sdds[0]/2.0
- for i in range(2):
- spos[1] = pos[1] - sdds[1]/2.0
- for j in range(2):
- spos[2] = pos[2] - sdds[2]/2.0
- for k in range(2):
- ii = ((k*2)+j)*2+i
- ch = root.children[i][j][k]
- if next_level == 1 and ch != NULL:
- mask[root.domain_ind, ii] = 0
- self.recursively_select_octs(
- ch, spos, sdds, mask, level + 1)
- elif this_level == 1:
- mask[root.domain_ind, ii] = \
- self.select_cell(spos, sdds, eterm)
- spos[2] += sdds[2]
- spos[1] += sdds[1]
- spos[0] += sdds[0]
-
def count_octs(self, OctreeContainer octree):
cdef OctVisitorData data
data.index = 0
https://bitbucket.org/yt_analysis/yt-3.0/commits/bd162c69f9a4/
Changeset: bd162c69f9a4
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-20 14:42:26
Summary: Mark out a few more potential problems.
Affected #: 2 files
diff -r abf193b22ae2cc50255264b13d7b4ee3d2924e6d -r bd162c69f9a44ee62e6252d0dccd9bade282ef18 yt/geometry/particle_oct_container.pyx
--- a/yt/geometry/particle_oct_container.pyx
+++ b/yt/geometry/particle_oct_container.pyx
@@ -379,6 +379,8 @@
else:
raise NotImplementedError
self.visit_all_octs(selector, func, &data)
+ assert ((data.global_index + 1)*8*dims == source.size)
+ assert (dest.size == data.index)
if num_cells >= 0:
return dest
return data.index - offset
diff -r abf193b22ae2cc50255264b13d7b4ee3d2924e6d -r bd162c69f9a44ee62e6252d0dccd9bade282ef18 yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -1165,19 +1165,7 @@
@cython.cdivision(True)
def select_octs(self, OctreeContainer octree):
# There has to be a better way to do this.
- cdef OctVisitorData data
- data.index = 0
- data.last = -1
- octree.visit_all_octs(self, oct_visitors.count_total_octs, &data)
- cdef np.ndarray[np.uint8_t, ndim=4] m2 = \
- np.zeros((2, 2, 2, data.index), 'uint8', order='C')
- # This is where we'll -- in the future -- cut up based on indices of
- # the octs.
- data.index = -1
- data.last = -1
- data.array = m2.data
- octree.visit_all_octs(self, oct_visitors.mark_octs, &data)
- return m2.astype("bool")
+ raise RuntimeError
@cython.boundscheck(False)
@cython.wraparound(False)
@@ -1203,7 +1191,6 @@
cdef int select_cell(self, np.float64_t pos[3], np.float64_t dds[3],
int eterm[3]) nogil:
return 1
- return self.base_selector.select_cell(pos, dds, eterm)
cdef int select_grid(self, np.float64_t left_edge[3],
np.float64_t right_edge[3], np.int32_t level) nogil:
https://bitbucket.org/yt_analysis/yt-3.0/commits/b769f0f4ca5c/
Changeset: b769f0f4ca5c
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-21 00:35:57
Summary: Merging with other line of development, including QuadTree bugfix.
Affected #: 4 files
diff -r bd162c69f9a44ee62e6252d0dccd9bade282ef18 -r b769f0f4ca5c20ff636a68576a02fdd260ab4b23 yt/frontends/enzo/data_structures.py
--- a/yt/frontends/enzo/data_structures.py
+++ b/yt/frontends/enzo/data_structures.py
@@ -492,7 +492,11 @@
field_list = None
field_list = self.comm.mpi_bcast(field_list)
self.field_list = []
- # Now we will, avoiding the problem of particle types not having names.
+ # Now we will iterate over all fields, trying to avoid the problem of
+ # particle types not having names. This should convert all known
+ # particle fields that exist in Enzo outputs into the construction
+ # ("all", field) and should not otherwise affect ActiveParticle
+ # simulations.
for field in field_list:
if ("all", field) in KnownEnzoFields:
self.field_list.append(("all", field))
diff -r bd162c69f9a44ee62e6252d0dccd9bade282ef18 -r b769f0f4ca5c20ff636a68576a02fdd260ab4b23 yt/frontends/ramses/data_structures.py
--- a/yt/frontends/ramses/data_structures.py
+++ b/yt/frontends/ramses/data_structures.py
@@ -91,6 +91,7 @@
hydro_offset = np.zeros(n_levels, dtype='int64')
hydro_offset -= 1
level_count = np.zeros(n_levels, dtype='int64')
+ skipped = []
for level in range(self.amr_header['nlevelmax']):
for cpu in range(self.amr_header['nboundary'] +
self.amr_header['ncpu']):
@@ -101,13 +102,15 @@
except AssertionError:
print "You are running with the wrong number of fields."
print "Please specify these in the load command."
+ print "We are looking for %s fields." % self.nvar
+ print "The last set of field sizes was: %s" % skipped
raise
if hvals['file_ncache'] == 0: continue
assert(hvals['file_ilevel'] == level+1)
if cpu + 1 == self.domain_id and level >= min_level:
hydro_offset[level - min_level] = f.tell()
level_count[level - min_level] = hvals['file_ncache']
- fpu.skip(f, 8 * self.nvar)
+ skipped = fpu.skip(f, 8 * self.nvar)
self._hydro_offset = hydro_offset
self._level_count = level_count
return self._hydro_offset
diff -r bd162c69f9a44ee62e6252d0dccd9bade282ef18 -r b769f0f4ca5c20ff636a68576a02fdd260ab4b23 yt/utilities/fortran_utils.py
--- a/yt/utilities/fortran_utils.py
+++ b/yt/utilities/fortran_utils.py
@@ -158,7 +158,7 @@
>>> f = open("fort.3", "rb")
>>> skip(f, 3)
"""
- skipped = 0
+ skipped = []
pos = f.tell()
for i in range(n):
fmt = endian+"I"
@@ -167,7 +167,7 @@
f.seek(s1+ struct.calcsize(fmt), os.SEEK_CUR)
s2= struct.unpack(fmt, size)[0]
assert s1==s2
- skipped += s1/struct.calcsize(fmt)
+ skipped.append(s1/struct.calcsize(fmt))
return skipped
def peek_record_size(f,endian='='):
diff -r bd162c69f9a44ee62e6252d0dccd9bade282ef18 -r b769f0f4ca5c20ff636a68576a02fdd260ab4b23 yt/utilities/lib/QuadTree.pyx
--- a/yt/utilities/lib/QuadTree.pyx
+++ b/yt/utilities/lib/QuadTree.pyx
@@ -342,6 +342,7 @@
@cython.boundscheck(False)
@cython.wraparound(False)
+ @cython.cdivision(True)
def get_all(self, int count_only = 0, int style = 1):
cdef int i, j, vi
cdef int total = 0
@@ -391,6 +392,7 @@
count += self.count(node.children[i][j])
return count
+ @cython.cdivision(True)
cdef int fill(self, QuadTreeNode *node,
np.int64_t curpos,
np.float64_t *px,
@@ -403,6 +405,8 @@
np.float64_t wtoadd,
np.int64_t level):
cdef int i, j, n
+ cdef np.float64_t *vorig
+ vorig = <np.float64_t *> alloca(sizeof(np.float64_t) * self.nvals)
if node.children[0][0] == NULL:
if self.merged == -1:
for i in range(self.nvals):
@@ -422,6 +426,7 @@
cdef np.int64_t added = 0
if self.merged == 1:
for i in range(self.nvals):
+ vorig[i] = vtoadd[i]
vtoadd[i] += node.val[i]
wtoadd += node.weight_val
elif self.merged == -1:
@@ -437,7 +442,7 @@
vtoadd, wtoadd, level + 1)
if self.merged == 1:
for i in range(self.nvals):
- vtoadd[i] -= node.val[i]
+ vtoadd[i] = vorig[i]
wtoadd -= node.weight_val
return added
https://bitbucket.org/yt_analysis/yt-3.0/commits/ac97077f2e47/
Changeset: ac97077f2e47
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-21 00:51:37
Summary: Adding a brief note about the index order in particle_deposit.pxd.
Affected #: 1 file
diff -r b769f0f4ca5c20ff636a68576a02fdd260ab4b23 -r ac97077f2e471c40a2ac3dae0d31bb94543d4b83 yt/geometry/particle_deposit.pxd
--- a/yt/geometry/particle_deposit.pxd
+++ b/yt/geometry/particle_deposit.pxd
@@ -38,8 +38,11 @@
void *alloca(int)
cdef inline int gind(int i, int j, int k, int dims[3]):
+ # The ordering is such that we want i to vary the slowest in this instance,
+ # even though in other instances it varies the fastest. To see this in
+ # action, try looking at the results of an n_ref=256 particle CIC plot,
+ # which shows it the most clearly.
return ((i*dims[1])+j)*dims[2]+k
- #return ((k*dims[1])+j)*dims[0]+i
####################################################
https://bitbucket.org/yt_analysis/yt-3.0/commits/e3cc788c3d2f/
Changeset: e3cc788c3d2f
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-21 16:15:38
Summary: Updated Tipsy IO for new Morton-ordering.
Affected #: 2 files
diff -r ac97077f2e471c40a2ac3dae0d31bb94543d4b83 -r e3cc788c3d2f00c52c8c55b134a8547e4656b585 yt/frontends/sph/io.py
--- a/yt/frontends/sph/io.py
+++ b/yt/frontends/sph/io.py
@@ -275,11 +275,10 @@
morton = get_morton_indices_unravel(lx, ly, lz)
del lx, ly, lz
return morton
-
- def _count_particles(self, domain):
+ def _count_particles(self, data_file):
npart = dict((self._ptypes[i], v)
- for i, v in enumerate(domain.header["Npart"]))
+ for i, v in enumerate(data_file.header["Npart"]))
return npart
_header_offset = 256
@@ -384,23 +383,26 @@
ptypes.add(ftype)
ptypes = list(ptypes)
ptypes.sort(key = lambda a: self._ptypes.index(a))
+ data_files = set([])
for chunk in chunks:
- for subset in chunk.objs:
- poff = subset.domain.field_offsets
- tp = subset.domain.total_particles
- f = open(subset.domain.domain_filename, "rb")
- for ptype in ptypes:
- f.seek(poff[ptype], os.SEEK_SET)
- p = np.fromfile(f, self._pdtypes[ptype], count=tp[ptype])
- mask = selector.select_points(
- p['Coordinates']['x'].astype("float64"),
- p['Coordinates']['y'].astype("float64"),
- p['Coordinates']['z'].astype("float64"))
- tf = self._fill_fields(ptf[ptype], p, mask)
- for field in tf:
- rv[ptype, field] = tf[field]
- del p, tf
- f.close()
+ for obj in chunk.objs:
+ data_files.update(obj.data_files)
+ for data_file in data_files:
+ poff = data_file.field_offsets
+ tp = data_file.total_particles
+ f = open(data_file.filename, "rb")
+ for ptype in ptypes:
+ f.seek(poff[ptype], os.SEEK_SET)
+ p = np.fromfile(f, self._pdtypes[ptype], count=tp[ptype])
+ mask = selector.select_points(
+ p['Coordinates']['x'].astype("float64"),
+ p['Coordinates']['y'].astype("float64"),
+ p['Coordinates']['z'].astype("float64"))
+ tf = self._fill_fields(ptf[ptype], p, mask)
+ for field in tf:
+ rv[ptype, field] = tf[field]
+ del p, tf
+ f.close()
return rv
def _initialize_index(self, data_file, regions):
@@ -447,26 +449,26 @@
mylog.info("Adding %0.3e particles", morton.size)
return morton
- def _count_particles(self, domain):
+ def _count_particles(self, data_file):
npart = {
- "Gas": domain.pf.parameters['nsph'],
- "Stars": domain.pf.parameters['nstar'],
- "DarkMatter": domain.pf.parameters['ndark']
+ "Gas": data_file.pf.parameters['nsph'],
+ "Stars": data_file.pf.parameters['nstar'],
+ "DarkMatter": data_file.pf.parameters['ndark']
}
return npart
- def _create_dtypes(self, domain):
+ def _create_dtypes(self, data_file):
# We can just look at the particle counts.
- self._header_offset = domain.pf._header_offset
+ self._header_offset = data_file.pf._header_offset
self._pdtypes = {}
pds = {}
field_list = []
- tp = domain.total_particles
+ tp = data_file.total_particles
for ptype, field in self._fields:
pfields = []
if tp[ptype] == 0: continue
- dtbase = domain.pf._field_dtypes.get(field, 'f')
- ff = "%s%s" % (domain.pf.endian, dtbase)
+ dtbase = data_file.pf._field_dtypes.get(field, 'f')
+ ff = "%s%s" % (data_file.pf.endian, dtbase)
if field in _vector_fields:
dt = (field, [('x', ff), ('y', ff), ('z', ff)])
else:
@@ -478,15 +480,15 @@
self._field_list = field_list
return self._field_list
- def _identify_fields(self, domain):
+ def _identify_fields(self, data_file):
return self._field_list
- def _calculate_particle_offsets(self, domain):
+ def _calculate_particle_offsets(self, data_file):
field_offsets = {}
- pos = domain.pf._header_offset
+ pos = data_file.pf._header_offset
for ptype in self._ptypes:
field_offsets[ptype] = pos
- if domain.total_particles[ptype] == 0: continue
+ if data_file.total_particles[ptype] == 0: continue
size = self._pdtypes[ptype].itemsize
- pos += domain.total_particles[ptype] * size
+ pos += data_file.total_particles[ptype] * size
return field_offsets
diff -r ac97077f2e471c40a2ac3dae0d31bb94543d4b83 -r e3cc788c3d2f00c52c8c55b134a8547e4656b585 yt/geometry/particle_oct_container.pyx
--- a/yt/geometry/particle_oct_container.pyx
+++ b/yt/geometry/particle_oct_container.pyx
@@ -412,7 +412,7 @@
cdef int ind[3], i
cdef np.ndarray[np.uint64_t, ndim=3] mask
mask = self.masks[file_id/64]
- val = 1 << (file_id - (file_id/64)*64)
+ cdef np.int64_t val = 1 << (file_id - (file_id/64)*64)
for p in range(no):
# Now we locate the particle
for i in range(3):
https://bitbucket.org/yt_analysis/yt-3.0/commits/2c7066b27099/
Changeset: 2c7066b27099
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-21 16:32:48
Summary: Profiling revealed a few instances of easy Cython speedups.
We're now dominated by IO and casting (via astype) from float32/float64.
Eventually we may be able to eliminate this by either using 32 bit internall or
figuring out a better IO system for Tipsy/Gadget.
Affected #: 3 files
diff -r e3cc788c3d2f00c52c8c55b134a8547e4656b585 -r 2c7066b270997d000070dd7636e37d6343083df1 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -137,6 +137,7 @@
yield (this.file_ind, this.domain_ind, this.domain)
cur = cur.next
+ @cython.cdivision(True)
cdef void visit_all_octs(self, SelectorObject selector,
oct_visitor_function *func,
OctVisitorData *data):
diff -r e3cc788c3d2f00c52c8c55b134a8547e4656b585 -r 2c7066b270997d000070dd7636e37d6343083df1 yt/geometry/oct_visitors.pyx
--- a/yt/geometry/oct_visitors.pyx
+++ b/yt/geometry/oct_visitors.pyx
@@ -25,6 +25,7 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
+cimport cython
cimport numpy
import numpy
from fp_utils cimport *
@@ -111,6 +112,7 @@
ires[data.index] = o.level
data.index += 1
+ at cython.cdivision(True)
cdef void fcoords_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
# Note that this does not actually give the correct floating point
# coordinates. It gives them in some unit system where the domain is 1.0
@@ -125,6 +127,7 @@
fcoords[data.index * 3 + i] = (c + 0.5) * dx
data.index += 1
+ at cython.cdivision(True)
cdef void fwidth_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
# Note that this does not actually give the correct floating point
# coordinates. It gives them in some unit system where the domain is 1.0
diff -r e3cc788c3d2f00c52c8c55b134a8547e4656b585 -r 2c7066b270997d000070dd7636e37d6343083df1 yt/geometry/particle_deposit.pyx
--- a/yt/geometry/particle_deposit.pyx
+++ b/yt/geometry/particle_deposit.pyx
@@ -45,6 +45,8 @@
def finalize(self, *args):
raise NotImplementedError
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
def process_octree(self, OctreeContainer octree,
np.ndarray[np.int64_t, ndim=1] dom_ind,
np.ndarray[np.float64_t, ndim=2] positions,
@@ -90,6 +92,8 @@
self.process(dims, oi.left_edge, oi.dds,
offset, pos, field_vals)
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
def process_grid(self, gobj,
np.ndarray[np.float64_t, ndim=2] positions,
fields = None):
https://bitbucket.org/yt_analysis/yt-3.0/commits/299fb9fa8915/
Changeset: 299fb9fa8915
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-21 18:12:19
Summary: Update grid class with new select() function call.
Affected #: 1 file
diff -r 2c7066b270997d000070dd7636e37d6343083df1 -r 299fb9fa8915f54575ca3841ce1caf92aebaef87 yt/data_objects/grid_patch.py
--- a/yt/data_objects/grid_patch.py
+++ b/yt/data_objects/grid_patch.py
@@ -441,14 +441,14 @@
return new_field
def select_icoords(self, dobj):
- mask = self.select(dobj.selector)
+ mask = self._get_selector_mask(dobj.selector)
if mask is None: return np.empty((0,3), dtype='int64')
coords = convert_mask_to_indices(mask, mask.sum())
coords += self.get_global_startindex()[None, :]
return coords
def select_fcoords(self, dobj):
- mask = self.select(dobj.selector)
+ mask = self._get_selector_mask(dobj.selector)
if mask is None: return np.empty((0,3), dtype='float64')
coords = convert_mask_to_indices(mask, mask.sum()).astype("float64")
coords += 0.5
@@ -457,15 +457,15 @@
return coords
def select_fwidth(self, dobj):
- mask = self.select(dobj.selector)
- if mask is None: return np.empty((0,3), dtype='float64')
- coords = np.empty((mask.sum(), 3), dtype='float64')
+ count = self.count(dobj.selector)
+ if count == 0: return np.empty((0,3), dtype='float64')
+ coords = np.empty((count, 3), dtype='float64')
for axis in range(3):
coords[:,axis] = self.dds[axis]
return coords
def select_ires(self, dobj):
- mask = self.select(dobj.selector)
+ mask = self._get_selector_mask(dobj.selector)
if mask is None: return np.empty(0, dtype='int64')
coords = np.empty(mask.sum(), dtype='int64')
coords[:] = self.Level
@@ -484,21 +484,27 @@
op.initialize()
op.process_grid(self, positions, fields)
vals = op.finalize()
- return vals.reshape(self.ActiveDimensions, order="F")
+ return vals.reshape(self.ActiveDimensions, order="C")
- def select(self, selector):
+ def _get_selector_mask(self, selector):
if id(selector) == self._last_selector_id:
- return self._last_mask
- self._last_mask = selector.fill_mask(self)
- self._last_selector_id = id(selector)
- return self._last_mask
+ mask = self._last_mask
+ else:
+ self._last_mask = mask = selector.fill_mask(self)
+ self._last_selector_id = id(selector)
+ return mask
+
+ def select(self, selector, source, dest, offset):
+ mask = self._get_selector_mask(selector)
+ count = self.count(selector)
+ if count == 0: return
+ dest[offset:offset+count] = source[mask]
+ return count
def count(self, selector):
- if id(selector) == self._last_selector_id:
- if self._last_mask is None: return 0
- return self._last_mask.sum()
- self.select(selector)
- return self.count(selector)
+ mask = self._get_selector_mask(selector)
+ if mask is None: return 0
+ return mask.sum()
def count_particles(self, selector, x, y, z):
# We don't cache the selector results
https://bitbucket.org/yt_analysis/yt-3.0/commits/c7b78242f38c/
Changeset: c7b78242f38c
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-21 20:43:19
Summary: Beginning consolidation of Oct/ParticleOct code. Breaks currently.
Affected #: 10 files
diff -r 299fb9fa8915f54575ca3841ce1caf92aebaef87 -r c7b78242f38ceced42d4c05d8eb95a254a67daf8 yt/data_objects/construction_data_containers.py
--- a/yt/data_objects/construction_data_containers.py
+++ b/yt/data_objects/construction_data_containers.py
@@ -281,9 +281,8 @@
chunk_fields.append(self.weight_field)
tree = self._get_tree(len(fields))
# We do this once
- if self.pf.h._global_mesh:
- for chunk in self.data_source.chunks(None, "io"):
- self._initialize_chunk(chunk, tree)
+ for chunk in self.data_source.chunks(None, "io"):
+ self._initialize_chunk(chunk, tree)
# This needs to be parallel_objects-ified
for chunk in parallel_objects(self.data_source.chunks(
chunk_fields, "io")):
diff -r 299fb9fa8915f54575ca3841ce1caf92aebaef87 -r c7b78242f38ceced42d4c05d8eb95a254a67daf8 yt/data_objects/octree_subset.py
--- a/yt/data_objects/octree_subset.py
+++ b/yt/data_objects/octree_subset.py
@@ -43,28 +43,23 @@
_num_zones = 2
_type_name = 'octree_subset'
_skip_add = True
- _con_args = ('domain', 'mask', 'cell_count')
+ _con_args = ('base_region', 'domain', 'pf')
_container_fields = ("dx", "dy", "dz")
- def __init__(self, domain, mask, cell_count):
+ def __init__(self, base_region, domain, pf):
self.field_data = YTFieldData()
self.field_parameters = {}
- self.mask = mask
self.domain = domain
+ self.domain_id = domain.domain_id
self.pf = domain.pf
self.hierarchy = self.pf.hierarchy
self.oct_handler = domain.pf.h.oct_handler
- self.cell_count = cell_count
- level_counts = self.oct_handler.count_levels(
- self.domain.pf.max_level, self.domain.domain_id, mask)
- assert(level_counts.sum() == cell_count)
- level_counts[1:] = level_counts[:-1]
- level_counts[0] = 0
- self.level_counts = np.add.accumulate(level_counts)
self._last_mask = None
self._last_selector_id = None
self._current_particle_type = 'all'
self._current_fluid_type = self.pf.default_fluid_type
+ self.base_region = base_region
+ self.base_selector = base_region.selector
def _generate_container_field(self, field):
if self._current_chunk is None:
@@ -78,31 +73,6 @@
else:
raise RuntimeError
- def select_icoords(self, dobj):
- return self.oct_handler.icoords(self.domain.domain_id, self.mask,
- self.cell_count,
- self.level_counts.copy())
-
- def select_fcoords(self, dobj):
- return self.oct_handler.fcoords(self.domain.domain_id, self.mask,
- self.cell_count,
- self.level_counts.copy())
-
- def select_fwidth(self, dobj):
- # Recall domain_dimensions is the number of cells, not octs
- base_dx = (self.domain.pf.domain_width /
- self.domain.pf.domain_dimensions)
- widths = np.empty((self.cell_count, 3), dtype="float64")
- dds = (2**self.select_ires(dobj))
- for i in range(3):
- widths[:,i] = base_dx[i] / dds
- return widths
-
- def select_ires(self, dobj):
- return self.oct_handler.ires(self.domain.domain_id, self.mask,
- self.cell_count,
- self.level_counts.copy())
-
def __getitem__(self, key):
tr = super(OctreeSubset, self).__getitem__(key)
try:
@@ -140,22 +110,41 @@
cls = getattr(particle_deposit, "deposit_%s" % method, None)
if cls is None:
raise YTParticleDepositionNotImplemented(method)
- nvals = (self.domain_ind >= 0).sum() * 8
+ nvals = (2, 2, 2, (self.domain_ind >= 0).sum())
op = cls(nvals) # We allocate number of zones, not number of octs
op.initialize()
- op.process_octree(self.oct_handler, self.domain_ind, positions, fields,
- self.domain.domain_id)
+ op.process_octree(self.oct_handler, self.domain_ind, positions, fields, 0)
vals = op.finalize()
- return self._reshape_vals(vals)
+ return np.asfortranarray(vals)
- def select(self, selector):
- if id(selector) == self._last_selector_id:
- return self._last_mask
- self._last_mask = self.oct_handler.domain_mask(
- self.mask, self.domain.domain_id)
- if self._last_mask.sum() == 0: return None
- self._last_selector_id = id(selector)
- return self._last_mask
+ def select_icoords(self, dobj):
+ d = self.oct_handler.icoords(self.selector)
+ tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 3,
+ domain_id = self.domain_id)
+ return tr
+
+ def select_fcoords(self, dobj):
+ d = self.oct_handler.fcoords(self.selector)
+ tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 3,
+ domain_id = self.domain_id)
+ return tr
+
+ def select_fwidth(self, dobj):
+ d = self.oct_handler.fwidth(self.selector)
+ tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 3,
+ domain_id = self.domain_id)
+ return tr
+
+ def select_ires(self, dobj):
+ d = self.oct_handler.ires(self.selector)
+ tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 1,
+ domain_id = self.domain_id)
+ return tr
+
+ def select(self, selector, source, dest, offset):
+ n = self.oct_handler.selector_fill(selector, source, dest, offset,
+ domain_id = self.domain_id)
+ return n
def count(self, selector):
if id(selector) == self._last_selector_id:
@@ -180,6 +169,7 @@
# this, it's unavoidable for many types of data storage on disk.
_type_name = 'particle_octree_subset'
_con_args = ('data_files', 'pf', 'min_ind', 'max_ind')
+ domain_id = -1
def __init__(self, base_region, data_files, pf, min_ind = 0, max_ind = 0):
# The first attempt at this will not work in parallel.
self.data_files = data_files
@@ -206,55 +196,3 @@
di = self.oct_handler.domain_ind(self.selector)
self._domain_ind = di
return self._domain_ind
-
- def deposit(self, positions, fields = None, method = None):
- # Here we perform our particle deposition.
- cls = getattr(particle_deposit, "deposit_%s" % method, None)
- if cls is None:
- raise YTParticleDepositionNotImplemented(method)
- nvals = (2, 2, 2, (self.domain_ind >= 0).sum())
- op = cls(nvals) # We allocate number of zones, not number of octs
- op.initialize()
- op.process_octree(self.oct_handler, self.domain_ind, positions, fields, 0)
- vals = op.finalize()
- return np.asfortranarray(vals)
-
- def select_icoords(self, dobj):
- d = self.oct_handler.icoords(self.selector)
- tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 3)
- return tr
-
- def select_fcoords(self, dobj):
- d = self.oct_handler.fcoords(self.selector)
- tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 3)
- return tr
-
- def select_fwidth(self, dobj):
- d = self.oct_handler.fwidth(self.selector)
- tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 3)
- return tr
-
- def select_ires(self, dobj):
- d = self.oct_handler.ires(self.selector)
- tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 1)
- return tr
-
- def select(self, selector, source, dest, offset):
- n = self.oct_handler.selector_fill(selector, source, dest, offset)
- return n
-
- def count(self, selector):
- if id(selector) == self._last_selector_id:
- if self._last_mask is None: return 0
- return self._last_mask.sum()
- self.select(selector)
- return self.count(selector)
-
- def count_particles(self, selector, x, y, z):
- # We don't cache the selector results
- count = selector.count_points(x,y,z)
- return count
-
- def select_particles(self, selector, x, y, z):
- mask = selector.select_points(x,y,z)
- return mask
diff -r 299fb9fa8915f54575ca3841ce1caf92aebaef87 -r c7b78242f38ceced42d4c05d8eb95a254a67daf8 yt/frontends/ramses/data_structures.py
--- a/yt/frontends/ramses/data_structures.py
+++ b/yt/frontends/ramses/data_structures.py
@@ -346,17 +346,18 @@
def _identify_base_chunk(self, dobj):
if getattr(dobj, "_chunk_info", None) is None:
mask = dobj.selector.select_octs(self.oct_handler)
- counts = self.oct_handler.count_cells(dobj.selector, mask)
- subsets = [RAMSESDomainSubset(d, mask, c)
- for d, c in zip(self.domains, counts) if c > 0]
+ base_region = getattr(dobj, "base_region", dobj)
+ # Note that domain_ids will be ONE INDEXED
+ domain_ids = self.oct_handler.domain_identify(dobj.selector)
+ subsets = [RAMSESDomainSubset(base_region, self.domains[did - 1],
+ self.parameter_file)
+ for did in domain_ids]
dobj._chunk_info = subsets
- dobj.size = sum(counts)
- dobj.shape = (dobj.size,)
dobj._current_chunk = list(self._chunk_all(dobj))[0]
def _chunk_all(self, dobj):
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
- yield YTDataChunk(dobj, "all", oobjs, dobj.size)
+ yield YTDataChunk(dobj, "all", oobjs, None)
def _chunk_spatial(self, dobj, ngz, sort = None):
sobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
@@ -372,7 +373,7 @@
def _chunk_io(self, dobj):
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
for subset in oobjs:
- yield YTDataChunk(dobj, "io", [subset], subset.cell_count)
+ yield YTDataChunk(dobj, "io", [subset], None)
class RAMSESStaticOutput(StaticOutput):
_hierarchy_class = RAMSESGeometryHandler
diff -r 299fb9fa8915f54575ca3841ce1caf92aebaef87 -r c7b78242f38ceced42d4c05d8eb95a254a67daf8 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -31,6 +31,8 @@
import numpy as np
from oct_container cimport Oct, OctAllocationContainer, \
OctreeContainer, ORDER_MAX
+from selection_routines cimport SelectorObject, \
+ OctVisitorData, oct_visitor_function
cimport oct_visitors
cimport cython
@@ -215,61 +217,6 @@
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def count_cells(self, SelectorObject selector,
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask):
- cdef int i, j, k
- cdef np.int64_t oi
- # pos here is CELL center, not OCT center.
- cdef np.float64_t pos[3]
- cdef int n = mask.shape[0]
- cdef np.ndarray[np.int64_t, ndim=1] count
- count = np.zeros(self.max_domain, 'int64')
- #
- cur = self.cont
- for oi in range(n):
- if oi - cur.offset >= cur.n_assigned:
- cur = cur.next
- o = &cur.my_octs[oi - cur.offset]
- for i in range(8):
- count[o.domain - 1] += mask[o.domain_ind,i]
- return count
-
- @cython.boundscheck(True)
- @cython.wraparound(False)
- @cython.cdivision(True)
- def count_leaves(self, np.ndarray[np.uint8_t, ndim=2, cast=True] mask):
- # Modified to work when not all octs are assigned
- cdef int i, j, k, ii
- cdef np.int64_t oi
- # pos here is CELL center, not OCT center.
- cdef np.float64_t pos[3]
- cdef int n = mask.shape[0]
- cdef np.ndarray[np.int64_t, ndim=1] count
- count = np.zeros(self.max_domain, 'int64')
- #
- cur = self.cont
- for oi in range(n):
- if oi - cur.offset >= cur.n_assigned:
- cur = cur.next
- if cur == NULL:
- break
- o = &cur.my_octs[oi - cur.offset]
- # skip if unassigned
- if o == NULL:
- continue
- if o.domain == -1:
- continue
- for i in range(2):
- for j in range(2):
- for k in range(2):
- if o.children[i][j][k] == NULL:
- ii = ((k*2)+j)*2+i
- count[o.domain - 1] += mask[o.domain_ind,ii]
- return count
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
cdef void neighbors(self, Oct* o, Oct* neighbors[27]):
#Get 3x3x3 neighbors, although the 1,1,1 oct is the
#central one.
@@ -357,8 +304,129 @@
bounds[i, 3+ii] = size[ii]
return bounds
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ def icoords(self, SelectorObject selector, np.uint64_t num_cells = -1):
+ if num_cells == -1:
+ num_cells = selector.count_octs(self)
+ cdef np.ndarray[np.int64_t, ndim=2] coords
+ coords = np.empty((num_cells, 3), dtype="int64")
+ cdef OctVisitorData data
+ data.array = <void *> coords.data
+ data.index = 0
+ self.visit_all_octs(selector, oct_visitors.icoords_octs, &data)
+ return coords
+
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ def ires(self, SelectorObject selector, np.uint64_t num_cells = -1):
+ if num_cells == -1:
+ num_cells = selector.count_octs(self)
+ #Return the 'resolution' of each cell; ie the level
+ cdef np.ndarray[np.int64_t, ndim=1] res
+ res = np.empty(num_cells, dtype="int64")
+ cdef OctVisitorData data
+ data.array = <void *> res.data
+ data.index = 0
+ self.visit_all_octs(selector, oct_visitors.ires_octs, &data)
+ return res
+
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ def fwidth(self, SelectorObject selector, np.uint64_t num_cells = -1):
+ if num_cells == -1:
+ num_cells = selector.count_octs(self)
+ cdef np.ndarray[np.float64_t, ndim=2] fwidth
+ fwidth = np.empty((num_cells, 3), dtype="float64")
+ cdef OctVisitorData data
+ data.array = <void *> fwidth.data
+ data.index = 0
+ self.visit_all_octs(selector, oct_visitors.fwidth_octs, &data)
+ cdef np.float64_t base_dx
+ for i in range(3):
+ base_dx = (self.DRE[i] - self.DLE[i])/self.nn[i]
+ fwidth[:,i] *= base_dx
+ return fwidth
+
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ def fcoords(self, SelectorObject selector, np.uint64_t num_cells = -1):
+ if num_cells == -1:
+ num_cells = selector.count_octs(self)
+ #Return the floating point unitary position of every cell
+ cdef np.ndarray[np.float64_t, ndim=2] coords
+ coords = np.empty((num_cells, 3), dtype="float64")
+ cdef OctVisitorData data
+ data.array = <void *> coords.data
+ data.index = 0
+ self.visit_all_octs(selector, oct_visitors.fcoords_octs, &data)
+ cdef int i
+ cdef np.float64_t base_dx
+ for i in range(3):
+ base_dx = (self.DRE[i] - self.DLE[i])/self.nn[i]
+ coords[:,i] *= base_dx
+ coords[:,i] += self.DLE[i]
+ return coords
+
+ def selector_fill(self, SelectorObject selector,
+ np.ndarray source,
+ np.ndarray dest = None,
+ np.int64_t offset = 0, int dims = 1,
+ int domain_id = -1):
+ # This is actually not correct. The hard part is that we need to
+ # iterate the same way visit_all_octs does, but we need to track the
+ # number of octs total visited.
+ cdef np.int64_t num_cells = -1
+ if dest is None:
+ num_cells = selector.count_octs(self)
+ if dims > 1:
+ dest = np.zeros((num_cells, dims), dtype=source.dtype,
+ order='C')
+ else:
+ dest = np.zeros(num_cells, dtype=source.dtype, order='C')
+ cdef OctVisitorData data
+ data.index = offset
+ data.domain = domain_id
+ # We only need this so we can continue calculating the offset
+ data.dims = dims
+ cdef void *p[2]
+ p[0] = source.data
+ p[1] = dest.data
+ data.array = &p
+ cdef oct_visitor_function *func
+ if source.dtype != dest.dtype:
+ raise RuntimeError
+ if source.dtype == np.int64:
+ func = oct_visitors.copy_array_i64
+ elif source.dtype == np.float64:
+ func = oct_visitors.copy_array_f64
+ else:
+ raise NotImplementedError
+ self.visit_all_octs(selector, func, &data)
+ if num_cells >= 0:
+ return dest
+ return data.index - offset
+
cdef class RAMSESOctreeContainer(OctreeContainer):
+ def domain_identify(self, SelectorObject selector):
+ cdef np.ndarray[np.uint8_t, ndim=1] domain_mask
+ domain_mask = np.zeros(self.max_domain, dtype="uint8")
+ cdef OctVisitorData data
+ data.array = domain_mask.data
+ self.visit_all_octs(selector, oct_visitors.identify_octs, &data)
+ cdef int i
+ domain_ids = []
+ for i in range(self.max_domain):
+ if domain_mask[i] == 1:
+ domain_ids.append(i+1)
+ return domain_ids
+
+
cdef np.int64_t get_domain_offset(self, int domain_id):
cdef OctAllocationContainer *cont = self.domains[domain_id - 1]
return cont.offset
@@ -413,149 +481,6 @@
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def count(self, np.ndarray[np.uint8_t, ndim=1, cast=True] mask,
- split = False):
- cdef int n = mask.shape[0]
- cdef int i, dom
- cdef OctAllocationContainer *cur
- cdef np.ndarray[np.int64_t, ndim=1] count
- count = np.zeros(self.max_domain, 'int64')
- # This is the idiom for iterating over many containers.
- cur = self.cont
- for i in range(n):
- if i - cur.offset >= cur.n_assigned: cur = cur.next
- if mask[i] == 1:
- count[cur.my_octs[i - cur.offset].domain - 1] += 1
- return count
-
- def domain_and(self, np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
- int domain_id):
- cdef np.int64_t i, oi, n, use
- cdef OctAllocationContainer *cur = self.domains[domain_id - 1]
- cdef Oct *o
- cdef np.ndarray[np.uint8_t, ndim=2] m2 = \
- np.zeros((mask.shape[0], 8), 'uint8')
- n = mask.shape[0]
- for oi in range(cur.n_assigned):
- o = &cur.my_octs[oi]
- use = 0
- for i in range(8):
- m2[o.domain_ind, i] = mask[o.domain_ind, i]
- return m2 # NOTE: This is uint8_t
-
- def domain_mask(self,
- # mask is the base selector's *global* mask
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
- int domain_id):
- # What distinguishes this one from domain_and is that we have a mask,
- # which covers the whole domain, but our output will only be of a much
- # smaller subset of octs that belong to a given domain *and* the mask.
- # Note also that typically when something calls domain_and, they will
- # use a logical_any along the oct axis. Here we don't do that.
- # Note also that we change the shape of the returned array.
- cdef np.int64_t i, j, k, oi, n, nm, use
- cdef OctAllocationContainer *cur = self.domains[domain_id - 1]
- cdef Oct *o
- n = mask.shape[0]
- nm = 0
- for oi in range(cur.n_assigned):
- o = &cur.my_octs[oi]
- use = 0
- for i in range(8):
- if mask[o.domain_ind, i] == 1: use = 1
- nm += use
- cdef np.ndarray[np.uint8_t, ndim=4] m2 = \
- np.zeros((2, 2, 2, nm), 'uint8')
- nm = 0
- for oi in range(cur.n_assigned):
- o = &cur.my_octs[oi]
- use = 0
- for i in range(2):
- for j in range(2):
- for k in range(2):
- ii = ((k*2)+j)*2+i
- if mask[o.domain_ind, ii] == 0: continue
- use = m2[i, j, k, nm] = 1
- nm += use
- return m2.astype("bool")
-
- def domain_ind(self,
- # mask is the base selector's *global* mask
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
- int domain_id):
- cdef np.int64_t i, j, k, oi, noct, n, nm, use, offset
- cdef OctAllocationContainer *cur = self.domains[domain_id - 1]
- cdef Oct *o
- cdef np.ndarray[np.int64_t, ndim=1] ind = np.zeros(cur.n, 'int64') - 1
- nm = 0
- for oi in range(cur.n):
- o = &cur.my_octs[oi]
- use = 0
- for i in range(8):
- if mask[o.domain_ind, i] == 1: use = 1
- if use == 1:
- ind[o.domain_ind - cur.offset] = nm
- nm += use
- return ind
-
- def check(self, int curdom, int print_all = 0):
- cdef int dind, pi
- cdef Oct oct
- cdef OctAllocationContainer *cont = self.domains[curdom - 1]
- cdef int nbad = 0
- cdef int nmissed = 0
- cdef int unassigned = 0
- for pi in range(cont.n_assigned):
- oct = cont.my_octs[pi]
- if print_all==1:
- print pi, oct.level, oct.domain,
- print oct.pos[0],oct.pos[1],oct.pos[2]
- for i in range(2):
- for j in range(2):
- for k in range(2):
- if oct.children[i][j][k] != NULL and \
- oct.children[i][j][k].level != oct.level + 1:
- nbad += 1
- if oct.domain != curdom:
- print curdom, oct.domain
- nmissed += 1
- if oct.domain == -1:
- unassigned += 1
- print "DOMAIN % 3i HAS % 9i BAD OCTS (%s / %s / %s)" % (curdom, nbad,
- cont.n - cont.n_assigned, cont.n_assigned, cont.n)
- print "DOMAIN % 3i HAS % 9i MISSED OCTS" % (curdom, nmissed)
- print "DOMAIN % 3i HAS % 9i UNASSIGNED OCTS" % (curdom, unassigned)
-
- def check_refinement(self, int curdom):
- cdef int pi, i, j, k, some_refined, some_unrefined
- cdef Oct *oct
- cdef int bad = 0
- cdef OctAllocationContainer *cont = self.domains[curdom - 1]
- for pi in range(cont.n_assigned):
- oct = &cont.my_octs[pi]
- some_unrefined = 0
- some_refined = 0
- for i in range(2):
- for j in range(2):
- for k in range(2):
- if oct.children[i][j][k] == NULL:
- some_unrefined = 1
- else:
- some_refined = 1
- if some_unrefined == some_refined == 1:
- #print "BAD", oct.file_ind, oct.domain_ind
- bad += 1
- if curdom == 10 or curdom == 72:
- for i in range(2):
- for j in range(2):
- for k in range(2):
- print (oct.children[i][j][k] == NULL),
- print
- print "BAD TOTAL", curdom, bad, cont.n_assigned
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
def add(self, int curdom, int curlevel, int ng,
np.ndarray[np.float64_t, ndim=2] pos,
int local_domain, int skip_boundary = 1):
@@ -608,116 +533,6 @@
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def icoords(self, int domain_id,
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
- np.int64_t cell_count,
- np.ndarray[np.int64_t, ndim=1] level_counts):
- # Wham, bam, it's a scam
- cdef np.int64_t i, j, k, oi, ci, n, ii, level
- cdef OctAllocationContainer *cur = self.domains[domain_id - 1]
- cdef Oct *o
- n = mask.shape[0]
- cdef np.ndarray[np.int64_t, ndim=2] coords
- coords = np.empty((cell_count, 3), dtype="int64")
- ci = 0
- for oi in range(cur.n_assigned):
- o = &cur.my_octs[oi]
- for i in range(2):
- for j in range(2):
- for k in range(2):
- ii = ((k*2)+j)*2+i
- if mask[o.domain_ind, ii] == 0: continue
- coords[ci, 0] = (o.pos[0] << 1) + i
- coords[ci, 1] = (o.pos[1] << 1) + j
- coords[ci, 2] = (o.pos[2] << 1) + k
- ci += 1
- return coords
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
- def ires(self, int domain_id,
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
- np.int64_t cell_count,
- np.ndarray[np.int64_t, ndim=1] level_counts):
- # Wham, bam, it's a scam
- cdef np.int64_t i, j, k, oi, ci, n
- cdef OctAllocationContainer *cur = self.domains[domain_id - 1]
- cdef Oct *o
- n = mask.shape[0]
- cdef np.ndarray[np.int64_t, ndim=1] levels
- levels = np.empty(cell_count, dtype="int64")
- ci = 0
- for oi in range(cur.n):
- o = &cur.my_octs[oi]
- for i in range(8):
- if mask[oi + cur.offset, i] == 0: continue
- levels[ci] = o.level
- ci += 1
- return levels
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
- def count_levels(self, int max_level, int domain_id,
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask):
- cdef np.ndarray[np.int64_t, ndim=1] level_count
- cdef OctAllocationContainer *cur = self.domains[domain_id - 1]
- cdef Oct *o
- cdef int oi, i
- level_count = np.zeros(max_level+1, 'int64')
- for oi in range(cur.n_assigned):
- o = &cur.my_octs[oi]
- for i in range(8):
- if mask[o.domain_ind, i] == 0: continue
- level_count[o.level] += 1
- return level_count
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
- def fcoords(self, int domain_id,
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
- np.int64_t cell_count,
- np.ndarray[np.int64_t, ndim=1] level_counts):
- # Wham, bam, it's a scam
- cdef np.int64_t i, j, k, oi, ci, n, ii
- cdef OctAllocationContainer *cur = self.domains[domain_id - 1]
- cdef Oct *o
- cdef np.float64_t pos[3]
- cdef np.float64_t base_dx[3], dx[3]
- n = mask.shape[0]
- cdef np.ndarray[np.float64_t, ndim=2] coords
- coords = np.empty((cell_count, 3), dtype="float64")
- for i in range(3):
- # This is the base_dx, but not the base distance from the center
- # position. Note that the positions will also all be offset by
- # dx/2.0. This is also for *oct grids*, not cells.
- base_dx[i] = (self.DRE[i] - self.DLE[i])/self.nn[i]
- ci = 0
- for oi in range(cur.n):
- o = &cur.my_octs[oi]
- for i in range(3):
- # This gives the *grid* width for this level
- dx[i] = base_dx[i] / (1 << o.level)
- # o.pos is the *grid* index, so pos[i] is the center of the
- # first cell in the grid
- pos[i] = self.DLE[i] + o.pos[i]*dx[i] + dx[i]/4.0
- dx[i] = dx[i] / 2.0 # This is now the *offset*
- for i in range(2):
- for j in range(2):
- for k in range(2):
- ii = ((k*2)+j)*2+i
- if mask[o.domain_ind, ii] == 0: continue
- coords[ci, 0] = pos[0] + dx[0] * i
- coords[ci, 1] = pos[1] + dx[1] * j
- coords[ci, 2] = pos[2] + dx[2] * k
- ci += 1
- return coords
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
def fill_level(self, int domain, int level, dest_fields, source_fields,
np.ndarray[np.uint8_t, ndim=2, cast=True] mask, int offset):
cdef np.ndarray[np.float64_t, ndim=2] source
@@ -745,6 +560,17 @@
local_filled += 1
return local_filled
+ def domain_ind(self, selector):
+ cdef np.ndarray[np.int64_t, ndim=1] ind
+ # Here's where we grab the masked items.
+ ind = np.zeros(self.nocts, 'int64') - 1
+ cdef OctVisitorData data
+ data.array = ind.data
+ data.index = 0
+ data.last = -1
+ self.visit_all_octs(selector, oct_visitors.index_octs, &data)
+ return ind
+
cdef class ARTOctreeContainer(RAMSESOctreeContainer):
@cython.boundscheck(True)
diff -r 299fb9fa8915f54575ca3841ce1caf92aebaef87 -r c7b78242f38ceced42d4c05d8eb95a254a67daf8 yt/geometry/oct_visitors.pxd
--- a/yt/geometry/oct_visitors.pxd
+++ b/yt/geometry/oct_visitors.pxd
@@ -40,6 +40,7 @@
cdef oct_visitor_function fwidth_octs
cdef oct_visitor_function copy_array_f64
cdef oct_visitor_function copy_array_i64
+cdef oct_visitor_function identify_octs
cdef inline int oind(OctVisitorData *data):
return (((data.ind[0]*2)+data.ind[1])*2+data.ind[2])
diff -r 299fb9fa8915f54575ca3841ce1caf92aebaef87 -r c7b78242f38ceced42d4c05d8eb95a254a67daf8 yt/geometry/oct_visitors.pyx
--- a/yt/geometry/oct_visitors.pyx
+++ b/yt/geometry/oct_visitors.pyx
@@ -36,26 +36,28 @@
# We should always have global_index less than our source.
# "last" here tells us the dimensionality of the array.
if selected == 0: return
+ if data.domain > 0 and o.domain != data.domain: return
cdef int i
# There are this many records between "octs"
- cdef np.int64_t index = (data.global_index * 8)*data.last
+ cdef np.int64_t index = (data.global_index * 8)*data.dims
cdef np.float64_t **p = <np.float64_t**> data.array
- index += oind(data)*data.last
- for i in range(data.last):
+ index += oind(data)*data.dims
+ for i in range(data.dims):
p[1][data.index + i] = p[0][index + i]
- data.index += data.last
+ data.index += data.dims
cdef void copy_array_i64(Oct *o, OctVisitorData *data, np.uint8_t selected):
# We should always have global_index less than our source.
# "last" here tells us the dimensionality of the array.
if selected == 0: return
+ if data.domain > 0 and o.domain != data.domain: return
cdef int i
- cdef np.int64_t index = (data.global_index * 8)*data.last
+ cdef np.int64_t index = (data.global_index * 8)*data.dims
cdef np.int64_t **p = <np.int64_t**> data.array
- index += oind(data)*data.last
- for i in range(data.last):
+ index += oind(data)*data.dims
+ for i in range(data.dims):
p[1][data.index + i] = p[0][index + i]
- data.index += data.last
+ data.index += data.dims
cdef void count_total_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
# Count even if not selected.
@@ -140,3 +142,11 @@
for i in range(3):
fwidth[data.index * 3 + i] = dx
data.index += 1
+
+cdef void identify_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
+ # We assume that our domain has *already* been selected by, which means
+ # we'll get all cells within the domain for a by-domain selector and all
+ # cells within the domain *and* selector for the selector itself.
+ if selected == 0: return
+ cdef np.uint8_t *arr = <np.uint8_t *> data.array
+ arr[o.domain - 1] = 1
diff -r 299fb9fa8915f54575ca3841ce1caf92aebaef87 -r c7b78242f38ceced42d4c05d8eb95a254a67daf8 yt/geometry/particle_geometry_handler.py
--- a/yt/geometry/particle_geometry_handler.py
+++ b/yt/geometry/particle_geometry_handler.py
@@ -154,7 +154,7 @@
def _chunk_all(self, dobj):
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
- yield ParticleDataChunk(self.oct_handler, self.regions, dobj, "all", oobjs, None)
+ yield YTDataChunk(dobj, "all", oobjs, None)
def _chunk_spatial(self, dobj, ngz, sort = None):
sobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
@@ -168,14 +168,12 @@
g = og.retrieve_ghost_zones(ngz, [], smoothed=True)
else:
g = og
- yield ParticleDataChunk(self.oct_handler, self.regions, dobj,
- "spatial", [g])
+ yield YTDataChunk(dobj, "spatial", [g])
def _chunk_io(self, dobj):
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
for subset in oobjs:
- yield ParticleDataChunk(self.oct_handler, self.regions,
- dobj, "io", [subset], None)
+ yield YTDataChunk(dobj, "io", [subset], None)
class ParticleDataChunk(YTDataChunk):
def __init__(self, oct_handler, regions, *args, **kwargs):
diff -r 299fb9fa8915f54575ca3841ce1caf92aebaef87 -r c7b78242f38ceced42d4c05d8eb95a254a67daf8 yt/geometry/particle_oct_container.pyx
--- a/yt/geometry/particle_oct_container.pyx
+++ b/yt/geometry/particle_oct_container.pyx
@@ -106,74 +106,6 @@
o = self.oct_list[oi]
yield (o.file_ind, o.domain_ind, o.domain)
- #@cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
- def icoords(self, SelectorObject selector, np.uint64_t num_cells = -1):
- if num_cells == -1:
- num_cells = selector.count_octs(self)
- cdef np.ndarray[np.int64_t, ndim=2] coords
- coords = np.empty((num_cells, 3), dtype="int64")
- cdef OctVisitorData data
- data.array = <void *> coords.data
- data.index = 0
- self.visit_all_octs(selector, oct_visitors.icoords_octs, &data)
- return coords
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
- def ires(self, SelectorObject selector, np.uint64_t num_cells = -1):
- if num_cells == -1:
- num_cells = selector.count_octs(self)
- #Return the 'resolution' of each cell; ie the level
- cdef np.ndarray[np.int64_t, ndim=1] res
- res = np.empty(num_cells, dtype="int64")
- cdef OctVisitorData data
- data.array = <void *> res.data
- data.index = 0
- self.visit_all_octs(selector, oct_visitors.ires_octs, &data)
- return res
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
- def fwidth(self, SelectorObject selector, np.uint64_t num_cells = -1):
- if num_cells == -1:
- num_cells = selector.count_octs(self)
- cdef np.ndarray[np.float64_t, ndim=2] fwidth
- fwidth = np.empty((num_cells, 3), dtype="float64")
- cdef OctVisitorData data
- data.array = <void *> fwidth.data
- data.index = 0
- self.visit_all_octs(selector, oct_visitors.fwidth_octs, &data)
- cdef np.float64_t base_dx
- for i in range(3):
- base_dx = (self.DRE[i] - self.DLE[i])/self.nn[i]
- fwidth[:,i] *= base_dx
- return fwidth
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
- def fcoords(self, SelectorObject selector, np.uint64_t num_cells = -1):
- if num_cells == -1:
- num_cells = selector.count_octs(self)
- #Return the floating point unitary position of every cell
- cdef np.ndarray[np.float64_t, ndim=2] coords
- coords = np.empty((num_cells, 3), dtype="float64")
- cdef OctVisitorData data
- data.array = <void *> coords.data
- data.index = 0
- self.visit_all_octs(selector, oct_visitors.fcoords_octs, &data)
- cdef int i
- cdef np.float64_t base_dx
- for i in range(3):
- base_dx = (self.DRE[i] - self.DLE[i])/self.nn[i]
- coords[:,i] *= base_dx
- coords[:,i] += self.DLE[i]
- return coords
-
def allocate_domains(self, domain_counts):
pass
@@ -334,57 +266,6 @@
self.visit(o.children[i][j][k], counts, level + 1)
return
- def domain_ind(self, selector):
- cdef np.ndarray[np.int64_t, ndim=1] ind
- # Here's where we grab the masked items.
- ind = np.zeros(self.nocts, 'int64') - 1
- cdef OctVisitorData data
- data.array = ind.data
- data.index = 0
- data.last = -1
- self.visit_all_octs(selector, oct_visitors.index_octs, &data)
- return ind
-
- def selector_fill(self, SelectorObject selector,
- np.ndarray source,
- np.ndarray dest = None,
- np.int64_t offset = 0, int dims = 1):
- # This is actually not correct. The hard part is that we need to
- # iterate the same way visit_all_octs does, but we need to track the
- # number of octs total visited.
- cdef np.int64_t num_cells = -1
- if dest is None:
- num_cells = selector.count_octs(self)
- if dims > 1:
- dest = np.zeros((num_cells, dims), dtype=source.dtype,
- order='C')
- else:
- dest = np.zeros(num_cells, dtype=source.dtype, order='C')
- dest = dest - 10000
- cdef OctVisitorData data
- data.index = offset
- # We only need this so we can continue calculating the offset
- data.last = dims
- cdef void *p[2]
- p[0] = source.data
- p[1] = dest.data
- data.array = &p
- cdef oct_visitor_function *func
- if source.dtype != dest.dtype:
- raise RuntimeError
- if source.dtype == np.int64:
- func = oct_visitors.copy_array_i64
- elif source.dtype == np.float64:
- func = oct_visitors.copy_array_f64
- else:
- raise NotImplementedError
- self.visit_all_octs(selector, func, &data)
- assert ((data.global_index + 1)*8*dims == source.size)
- assert (dest.size == data.index)
- if num_cells >= 0:
- return dest
- return data.index - offset
-
cdef class ParticleRegions:
cdef np.float64_t left_edge[3]
cdef np.float64_t dds[3]
diff -r 299fb9fa8915f54575ca3841ce1caf92aebaef87 -r c7b78242f38ceced42d4c05d8eb95a254a67daf8 yt/geometry/selection_routines.pxd
--- a/yt/geometry/selection_routines.pxd
+++ b/yt/geometry/selection_routines.pxd
@@ -32,6 +32,8 @@
np.uint64_t global_index
int ind[3]
void *array
+ int dims
+ int domain
ctypedef void oct_visitor_function(Oct *, OctVisitorData *visitor,
np.uint8_t selected)
@@ -47,7 +49,7 @@
OctVisitorData *data)
cdef int select_grid(self, np.float64_t left_edge[3],
np.float64_t right_edge[3],
- np.int32_t level) nogil
+ np.int32_t level, Oct *o = ?) nogil
cdef int select_cell(self, np.float64_t pos[3], np.float64_t dds[3],
int eterm[3]) nogil
cdef void set_bounds(self,
diff -r 299fb9fa8915f54575ca3841ce1caf92aebaef87 -r c7b78242f38ceced42d4c05d8eb95a254a67daf8 yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -187,7 +187,7 @@
LE[i] = pos[i] - dds[i]/2.0
RE[i] = pos[i] + dds[i]/2.0
#print LE[0], RE[0], LE[1], RE[1], LE[2], RE[2]
- res = self.select_grid(LE, RE, level)
+ res = self.select_grid(LE, RE, level, root)
cdef int eterm[3]
eterm[0] = eterm[1] = eterm[2] = 0
cdef int next_level, this_level
@@ -201,7 +201,15 @@
this_level = 0
if res == 0 and this_level == 1:
return
+ if res == -1:
+ # This happens when we do domain selection but the oct has
+ # children. This would allow an oct to pass to its children but
+ # not get accessed itself.
+ next_level = 1
+ this_level = 0
cdef int increment = 1
+ if data.domain > 0 and root.domain != data.domain:
+ increment = 0
# Now we visit all our children. We subtract off sdds for the first
# pass because we center it on the first cell.
spos[0] = pos[0] - sdds[0]/2.0
@@ -229,7 +237,7 @@
cdef int select_grid(self, np.float64_t left_edge[3],
np.float64_t right_edge[3],
- np.int32_t level) nogil:
+ np.int32_t level, Oct *o = NULL) nogil:
return 0
cdef int select_cell(self, np.float64_t pos[3], np.float64_t dds[3],
@@ -425,7 +433,7 @@
@cython.cdivision(True)
cdef int select_grid(self, np.float64_t left_edge[3],
np.float64_t right_edge[3],
- np.int32_t level) nogil:
+ np.int32_t level, Oct *o = NULL) nogil:
cdef np.float64_t box_center, relcenter, closest, dist, edge
cdef int i
if (left_edge[0] <= self.center[0] <= right_edge[0] and
@@ -491,7 +499,7 @@
@cython.cdivision(True)
cdef int select_grid(self, np.float64_t left_edge[3],
np.float64_t right_edge[3],
- np.int32_t level) nogil:
+ np.int32_t level, Oct *o = NULL) nogil:
if level < self.min_level or level > self.max_level: return 0
for i in range(3):
if left_edge[i] >= self.right_edge[i]: return 0
@@ -562,7 +570,7 @@
@cython.cdivision(True)
cdef int select_grid(self, np.float64_t left_edge[3],
np.float64_t right_edge[3],
- np.int32_t level) nogil:
+ np.int32_t level, Oct *o = NULL) nogil:
cdef np.float64_t *arr[2]
cdef np.float64_t pos[3], H, D, R2, temp
cdef int i, j, k, n
@@ -630,7 +638,7 @@
@cython.cdivision(True)
cdef int select_grid(self, np.float64_t left_edge[3],
np.float64_t right_edge[3],
- np.int32_t level) nogil:
+ np.int32_t level, Oct *o = NULL) nogil:
cdef int i, j, k, n
cdef np.float64_t *arr[2]
cdef np.float64_t pos[3]
@@ -698,7 +706,7 @@
@cython.cdivision(True)
cdef int select_grid(self, np.float64_t left_edge[3],
np.float64_t right_edge[3],
- np.int32_t level) nogil:
+ np.int32_t level, Oct *o = NULL) nogil:
if right_edge[self.axis] > self.coord \
and left_edge[self.axis] <= self.coord:
return 1
@@ -736,7 +744,7 @@
@cython.cdivision(True)
cdef int select_grid(self, np.float64_t left_edge[3],
np.float64_t right_edge[3],
- np.int32_t level) nogil:
+ np.int32_t level, Oct *o = NULL) nogil:
if ( (self.px >= left_edge[self.px_ax])
and (self.px < right_edge[self.px_ax])
and (self.py >= left_edge[self.py_ax])
@@ -815,7 +823,7 @@
@cython.cdivision(True)
cdef int select_grid(self, np.float64_t left_edge[3],
np.float64_t right_edge[3],
- np.int32_t level) nogil:
+ np.int32_t level, Oct *o = NULL) nogil:
cdef int i, ax
cdef int i1, i2
cdef np.float64_t vs[3], t, v[3]
@@ -1006,7 +1014,7 @@
@cython.cdivision(True)
cdef int select_grid(self, np.float64_t left_edge[3],
np.float64_t right_edge[3],
- np.int32_t level) nogil:
+ np.int32_t level, Oct *o = NULL) nogil:
# This is the sphere selection
cdef np.float64_t radius2, box_center, relcenter, closest, dist, edge
return 1
@@ -1099,28 +1107,18 @@
grid_selector = GridSelector
cdef class OctreeSubsetSelector(SelectorObject):
- # This is a numpy array, which will be a bool of ndim 1
- cdef object oct_mask
cdef int domain_id
+ cdef SelectorObject base_selector
def __init__(self, dobj):
- self.oct_mask = dobj.mask
self.domain_id = dobj.domain.domain_id
+ self.base_selector = dobj.base_selector
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
def select_octs(self, OctreeContainer octree):
- cdef np.ndarray[np.uint8_t, ndim=2] m2
- m2 = octree.domain_and(self.oct_mask, self.domain_id)
- cdef int oi, i, a
- for oi in range(m2.shape[0]):
- a = 0
- for i in range(8):
- if m2[oi, i] == 1: a = 1
- for i in range(8):
- m2[oi, i] = a
- return m2.astype("bool")
+ raise RuntimeError
@cython.boundscheck(False)
@cython.wraparound(False)
@@ -1147,6 +1145,20 @@
int eterm[3]) nogil:
return 1
+ cdef int select_grid(self, np.float64_t left_edge[3],
+ np.float64_t right_edge[3], np.int32_t level,
+ Oct *o = NULL) nogil:
+ # Because visitors now use select_grid, we should be explicitly
+ # checking this.
+ cdef int any_children = 0
+ cdef int i, j, k
+ if o == NULL:
+ return 0
+ cdef int res
+ res = self.base_selector.select_grid(left_edge, right_edge, level, o)
+ if res == 1 and o.domain != self.domain_id: return -1
+ return 1
+
octree_subset_selector = OctreeSubsetSelector
cdef class ParticleOctreeSubsetSelector(SelectorObject):
@@ -1193,10 +1205,11 @@
return 1
cdef int select_grid(self, np.float64_t left_edge[3],
- np.float64_t right_edge[3], np.int32_t level) nogil:
+ np.float64_t right_edge[3], np.int32_t level,
+ Oct *o = NULL) nogil:
# Because visitors now use select_grid, we should be explicitly
# checking this.
- return self.base_selector.select_grid(left_edge, right_edge, level)
+ return self.base_selector.select_grid(left_edge, right_edge, level, o)
particle_octree_subset_selector = ParticleOctreeSubsetSelector
https://bitbucket.org/yt_analysis/yt-3.0/commits/e9ab67bfd569/
Changeset: e9ab67bfd569
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-21 20:57:37
Summary: Attempting to speed up traversal.
Affected #: 2 files
diff -r c7b78242f38ceced42d4c05d8eb95a254a67daf8 -r e9ab67bfd5695e0f0b27f320388854120c4ced7c yt/geometry/oct_visitors.pyx
--- a/yt/geometry/oct_visitors.pyx
+++ b/yt/geometry/oct_visitors.pyx
@@ -69,6 +69,7 @@
cdef void count_total_cells(Oct *o, OctVisitorData *data, np.uint8_t selected):
# Count even if not selected.
# Number of *octs* visited.
+ if data.domain > 0 and o.domain != data.domain: return
data.index += selected
cdef void mark_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
@@ -94,6 +95,7 @@
# Note that we provide an index even if the cell is not selected.
cdef int i
cdef np.int64_t *arr
+ if data.domain > 0 and data.domain != o.domain: return
if data.last != o.domain_ind:
data.last = o.domain_ind
arr = <np.int64_t *> data.array
diff -r c7b78242f38ceced42d4c05d8eb95a254a67daf8 -r e9ab67bfd5695e0f0b27f320388854120c4ced7c yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -1152,8 +1152,7 @@
# checking this.
cdef int any_children = 0
cdef int i, j, k
- if o == NULL:
- return 0
+ if o == NULL: return 0
cdef int res
res = self.base_selector.select_grid(left_edge, right_edge, level, o)
if res == 1 and o.domain != self.domain_id: return -1
https://bitbucket.org/yt_analysis/yt-3.0/commits/3959497fa6a3/
Changeset: 3959497fa6a3
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-21 22:32:18
Summary: Attempting to thread domain info through the selectors.
This (hopefully) will unify the oct containers and make it viable to distribute
the RAMSES oct container over multiple processors.
Affected #: 5 files
diff -r e9ab67bfd5695e0f0b27f320388854120c4ced7c -r 3959497fa6a3f90e117788dcd5f570296387ad7e yt/data_objects/octree_subset.py
--- a/yt/data_objects/octree_subset.py
+++ b/yt/data_objects/octree_subset.py
@@ -118,25 +118,25 @@
return np.asfortranarray(vals)
def select_icoords(self, dobj):
- d = self.oct_handler.icoords(self.selector)
+ d = self.oct_handler.icoords(self.selector, domain_id = self.domain_id)
tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 3,
domain_id = self.domain_id)
return tr
def select_fcoords(self, dobj):
- d = self.oct_handler.fcoords(self.selector)
+ d = self.oct_handler.fcoords(self.selector, domain_id = self.domain_id)
tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 3,
domain_id = self.domain_id)
return tr
def select_fwidth(self, dobj):
- d = self.oct_handler.fwidth(self.selector)
+ d = self.oct_handler.fwidth(self.selector, domain_id = self.domain_id)
tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 3,
domain_id = self.domain_id)
return tr
def select_ires(self, dobj):
- d = self.oct_handler.ires(self.selector)
+ d = self.oct_handler.ires(self.selector, domain_id = self.domain_id)
tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 1,
domain_id = self.domain_id)
return tr
diff -r e9ab67bfd5695e0f0b27f320388854120c4ced7c -r 3959497fa6a3f90e117788dcd5f570296387ad7e yt/frontends/ramses/data_structures.py
--- a/yt/frontends/ramses/data_structures.py
+++ b/yt/frontends/ramses/data_structures.py
@@ -345,10 +345,10 @@
def _identify_base_chunk(self, dobj):
if getattr(dobj, "_chunk_info", None) is None:
- mask = dobj.selector.select_octs(self.oct_handler)
base_region = getattr(dobj, "base_region", dobj)
# Note that domain_ids will be ONE INDEXED
domain_ids = self.oct_handler.domain_identify(dobj.selector)
+ mylog.debug("Identified %s intersecting domains", len(domain_ids))
subsets = [RAMSESDomainSubset(base_region, self.domains[did - 1],
self.parameter_file)
for did in domain_ids]
diff -r e9ab67bfd5695e0f0b27f320388854120c4ced7c -r 3959497fa6a3f90e117788dcd5f570296387ad7e yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -307,43 +307,49 @@
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def icoords(self, SelectorObject selector, np.uint64_t num_cells = -1):
+ def icoords(self, SelectorObject selector, np.uint64_t num_cells = -1,
+ int domain_id = -1):
if num_cells == -1:
- num_cells = selector.count_octs(self)
+ num_cells = selector.count_octs(self, domain_id)
cdef np.ndarray[np.int64_t, ndim=2] coords
coords = np.empty((num_cells, 3), dtype="int64")
cdef OctVisitorData data
data.array = <void *> coords.data
data.index = 0
+ data.domain = domain_id
self.visit_all_octs(selector, oct_visitors.icoords_octs, &data)
return coords
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def ires(self, SelectorObject selector, np.uint64_t num_cells = -1):
+ def ires(self, SelectorObject selector, np.uint64_t num_cells = -1,
+ int domain_id = -1):
if num_cells == -1:
- num_cells = selector.count_octs(self)
+ num_cells = selector.count_octs(self, domain_id)
#Return the 'resolution' of each cell; ie the level
cdef np.ndarray[np.int64_t, ndim=1] res
res = np.empty(num_cells, dtype="int64")
cdef OctVisitorData data
data.array = <void *> res.data
data.index = 0
+ data.domain = domain_id
self.visit_all_octs(selector, oct_visitors.ires_octs, &data)
return res
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def fwidth(self, SelectorObject selector, np.uint64_t num_cells = -1):
+ def fwidth(self, SelectorObject selector, np.uint64_t num_cells = -1,
+ int domain_id = -1):
if num_cells == -1:
- num_cells = selector.count_octs(self)
+ num_cells = selector.count_octs(self, domain_id)
cdef np.ndarray[np.float64_t, ndim=2] fwidth
fwidth = np.empty((num_cells, 3), dtype="float64")
cdef OctVisitorData data
data.array = <void *> fwidth.data
data.index = 0
+ data.domain = domain_id
self.visit_all_octs(selector, oct_visitors.fwidth_octs, &data)
cdef np.float64_t base_dx
for i in range(3):
@@ -354,15 +360,17 @@
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def fcoords(self, SelectorObject selector, np.uint64_t num_cells = -1):
+ def fcoords(self, SelectorObject selector, np.uint64_t num_cells = -1,
+ int domain_id = -1):
if num_cells == -1:
- num_cells = selector.count_octs(self)
+ num_cells = selector.count_octs(self, domain_id)
#Return the floating point unitary position of every cell
cdef np.ndarray[np.float64_t, ndim=2] coords
coords = np.empty((num_cells, 3), dtype="float64")
cdef OctVisitorData data
data.array = <void *> coords.data
data.index = 0
+ data.domain = domain_id
self.visit_all_octs(selector, oct_visitors.fcoords_octs, &data)
cdef int i
cdef np.float64_t base_dx
@@ -382,7 +390,7 @@
# number of octs total visited.
cdef np.int64_t num_cells = -1
if dest is None:
- num_cells = selector.count_octs(self)
+ num_cells = selector.count_octs(self, domain_id)
if dims > 1:
dest = np.zeros((num_cells, dims), dtype=source.dtype,
order='C')
@@ -407,6 +415,14 @@
else:
raise NotImplementedError
self.visit_all_octs(selector, func, &data)
+ if (data.global_index + 1) * 8 * data.dims > source.size:
+ print "GLOBAL INDEX RAN AHEAD.",
+ print (data.global_index + 1) * 8 * data.dims - source.size
+ raise RuntimeError
+ if data.index > dest.size:
+ print "DEST INDEX RAN AHEAD.",
+ print data.index - dest.size
+ raise RuntimeError
if num_cells >= 0:
return dest
return data.index - offset
@@ -418,6 +434,7 @@
domain_mask = np.zeros(self.max_domain, dtype="uint8")
cdef OctVisitorData data
data.array = domain_mask.data
+ data.domain = -1
self.visit_all_octs(selector, oct_visitors.identify_octs, &data)
cdef int i
domain_ids = []
@@ -426,7 +443,6 @@
domain_ids.append(i+1)
return domain_ids
-
cdef np.int64_t get_domain_offset(self, int domain_id):
cdef OctAllocationContainer *cont = self.domains[domain_id - 1]
return cont.offset
@@ -560,11 +576,12 @@
local_filled += 1
return local_filled
- def domain_ind(self, selector):
+ def domain_ind(self, selector, int domain_id = -1):
cdef np.ndarray[np.int64_t, ndim=1] ind
# Here's where we grab the masked items.
ind = np.zeros(self.nocts, 'int64') - 1
cdef OctVisitorData data
+ data.domain = domain_id
data.array = ind.data
data.index = 0
data.last = -1
diff -r e9ab67bfd5695e0f0b27f320388854120c4ced7c -r 3959497fa6a3f90e117788dcd5f570296387ad7e yt/geometry/oct_visitors.pyx
--- a/yt/geometry/oct_visitors.pyx
+++ b/yt/geometry/oct_visitors.pyx
@@ -62,6 +62,7 @@
cdef void count_total_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
# Count even if not selected.
# Number of *octs* visited.
+ if data.domain > 0 and o.domain != data.domain: return
if data.last != o.domain_ind:
data.index += 1
data.last = o.domain_ind
diff -r e9ab67bfd5695e0f0b27f320388854120c4ced7c -r 3959497fa6a3f90e117788dcd5f570296387ad7e yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -146,6 +146,7 @@
@cython.cdivision(True)
def select_octs(self, OctreeContainer octree):
# There has to be a better way to do this.
+ raise RuntimeError
cdef OctVisitorData data
data.index = 0
data.last = -1
@@ -161,9 +162,10 @@
octree.visit_all_octs(self, oct_visitors.mask_octs, &data)
return m2.astype("bool")
- def count_octs(self, OctreeContainer octree):
+ def count_octs(self, OctreeContainer octree, int domain_id = -1):
cdef OctVisitorData data
data.index = 0
+ data.domain = domain_id
octree.visit_all_octs(self, oct_visitors.count_total_cells, &data)
return data.index
@@ -188,28 +190,29 @@
RE[i] = pos[i] + dds[i]/2.0
#print LE[0], RE[0], LE[1], RE[1], LE[2], RE[2]
res = self.select_grid(LE, RE, level, root)
+ if res == 1 and data.domain > 0 and root.domain != data.domain:
+ res = -1
cdef int eterm[3]
+ cdef int increment = 1
eterm[0] = eterm[1] = eterm[2] = 0
cdef int next_level, this_level
# next_level: an int that says whether or not we can progress to children
# this_level: an int that says whether or not we can select from this
# level
next_level = this_level = 1
+ if res == -1:
+ # This happens when we do domain selection but the oct has
+ # children. This would allow an oct to pass to its children but
+ # not get accessed itself.
+ next_level = 1
+ this_level = 0
+ increment = 0
if level == self.max_level:
next_level = 0
if level < self.min_level or level > self.max_level:
this_level = 0
if res == 0 and this_level == 1:
return
- if res == -1:
- # This happens when we do domain selection but the oct has
- # children. This would allow an oct to pass to its children but
- # not get accessed itself.
- next_level = 1
- this_level = 0
- cdef int increment = 1
- if data.domain > 0 and root.domain != data.domain:
- increment = 0
# Now we visit all our children. We subtract off sdds for the first
# pass because we center it on the first cell.
spos[0] = pos[0] - sdds[0]/2.0
@@ -1150,13 +1153,12 @@
Oct *o = NULL) nogil:
# Because visitors now use select_grid, we should be explicitly
# checking this.
- cdef int any_children = 0
- cdef int i, j, k
+ cdef int res
if o == NULL: return 0
- cdef int res
res = self.base_selector.select_grid(left_edge, right_edge, level, o)
- if res == 1 and o.domain != self.domain_id: return -1
- return 1
+ if res != 0 and o.domain != self.domain_id:
+ res = -1
+ return res
octree_subset_selector = OctreeSubsetSelector
https://bitbucket.org/yt_analysis/yt-3.0/commits/ff8d85b3c294/
Changeset: ff8d85b3c294
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-21 23:50:22
Summary: Beginning conversion of RAMSES to distributed memory.
Affected #: 4 files
diff -r 3959497fa6a3f90e117788dcd5f570296387ad7e -r ff8d85b3c294be269af13380e3aa881c1df0adcc yt/data_objects/octree_subset.py
--- a/yt/data_objects/octree_subset.py
+++ b/yt/data_objects/octree_subset.py
@@ -53,7 +53,7 @@
self.domain_id = domain.domain_id
self.pf = domain.pf
self.hierarchy = self.pf.hierarchy
- self.oct_handler = domain.pf.h.oct_handler
+ self.oct_handler = domain.oct_handler
self._last_mask = None
self._last_selector_id = None
self._current_particle_type = 'all'
diff -r 3959497fa6a3f90e117788dcd5f570296387ad7e -r ff8d85b3c294be269af13380e3aa881c1df0adcc yt/frontends/ramses/data_structures.py
--- a/yt/frontends/ramses/data_structures.py
+++ b/yt/frontends/ramses/data_structures.py
@@ -69,6 +69,7 @@
setattr(self, "%s_fn" % t, basename % t)
self._read_amr_header()
self._read_particle_header()
+ self._read_amr()
_hydro_offset = None
_level_count = None
@@ -183,21 +184,26 @@
self.amr_header = hvals
self.amr_offset = f.tell()
self.local_oct_count = hvals['numbl'][self.pf.min_level:, self.domain_id - 1].sum()
+ self.total_oct_count = hvals['numbl'][self.pf.min_level:,:].sum(axis=0)
- def _read_amr(self, oct_handler):
+ def _read_amr(self):
"""Open the oct file, read in octs level-by-level.
For each oct, only the position, index, level and domain
are needed - its position in the octree is found automatically.
The most important is finding all the information to feed
oct_handler.add
"""
+ self.oct_handler = RAMSESOctreeContainer(self.pf.domain_dimensions/2,
+ self.pf.domain_left_edge, self.pf.domain_right_edge)
+ root_nodes = self.amr_header['numbl'][self.pf.min_level,:].sum()
+ self.oct_handler.allocate_domains(self.total_oct_count, root_nodes)
fb = open(self.amr_fn, "rb")
fb.seek(self.amr_offset)
f = cStringIO.StringIO()
f.write(fb.read())
f.seek(0)
mylog.debug("Reading domain AMR % 4i (%0.3e, %0.3e)",
- self.domain_id, self.local_oct_count, self.ngridbound.sum())
+ self.domain_id, self.total_oct_count.sum(), self.ngridbound.sum())
def _ng(c, l):
if c < self.amr_header['ncpu']:
ng = self.amr_header['numbl'][l, c]
@@ -236,26 +242,18 @@
# rmap[:,i] = fpu.read_vector(f, "I")
# We don't want duplicate grids.
# Note that we're adding *grids*, not individual cells.
- if level >= min_level and cpu + 1 >= self.domain_id:
+ if level >= min_level:
assert(pos.shape[0] == ng)
if cpu + 1 == self.domain_id:
total += ng
- oct_handler.add(cpu + 1, level - min_level, ng, pos,
- self.domain_id)
+ self.oct_handler.add(cpu + 1, level - min_level,
+ pos, self.domain_id)
+ self.oct_handler.finalize()
+ #raise RuntimeError
- def select(self, selector):
- if id(selector) == self._last_selector_id:
- return self._last_mask
- self._last_mask = selector.fill_mask(self)
- self._last_selector_id = id(selector)
- return self._last_mask
-
- def count(self, selector):
- if id(selector) == self._last_selector_id:
- if self._last_mask is None: return 0
- return self._last_mask.sum()
- self.select(selector)
- return self.count(selector)
+ def included(self, selector):
+ domain_ids = self.oct_handler.domain_identify(selector)
+ return self.domain_id in domain_ids
class RAMSESDomainSubset(OctreeSubset):
@@ -314,21 +312,6 @@
total_octs = sum(dom.local_oct_count #+ dom.ngridbound.sum()
for dom in self.domains)
self.num_grids = total_octs
- #this merely allocates space for the oct tree
- #and nothing else
- self.oct_handler = RAMSESOctreeContainer(
- self.parameter_file.domain_dimensions/2,
- self.parameter_file.domain_left_edge,
- self.parameter_file.domain_right_edge)
- mylog.debug("Allocating %s octs", total_octs)
- self.oct_handler.allocate_domains(
- [dom.local_oct_count #+ dom.ngridbound.sum()
- for dom in self.domains])
- #this actually reads every oct and loads it into the octree
- for dom in self.domains:
- dom._read_amr(self.oct_handler)
- #for dom in self.domains:
- # self.oct_handler.check(dom.domain_id)
def _detect_fields(self):
# TODO: Add additional fields
@@ -347,11 +330,11 @@
if getattr(dobj, "_chunk_info", None) is None:
base_region = getattr(dobj, "base_region", dobj)
# Note that domain_ids will be ONE INDEXED
- domain_ids = self.oct_handler.domain_identify(dobj.selector)
+ domains = [dom for dom in self.domains if
+ dom.included(dobj.selector)]
mylog.debug("Identified %s intersecting domains", len(domain_ids))
- subsets = [RAMSESDomainSubset(base_region, self.domains[did - 1],
- self.parameter_file)
- for did in domain_ids]
+ subsets = [RAMSESDomainSubset(base_region, domain, self.parameter_file)
+ for domain in domains]
dobj._chunk_info = subsets
dobj._current_chunk = list(self._chunk_all(dobj))[0]
diff -r 3959497fa6a3f90e117788dcd5f570296387ad7e -r ff8d85b3c294be269af13380e3aa881c1df0adcc yt/geometry/oct_container.pxd
--- a/yt/geometry/oct_container.pxd
+++ b/yt/geometry/oct_container.pxd
@@ -28,6 +28,7 @@
from selection_routines cimport SelectorObject, \
OctVisitorData, oct_visitor_function
from oct_visitors cimport *
+from libc.stdlib cimport bsearch, qsort
cdef int ORDER_MAX
@@ -65,6 +66,7 @@
cdef public int nocts
cdef public int max_domain
cdef Oct* get(self, np.float64_t ppos[3], OctInfo *oinfo = ?)
+ cdef Oct *get_root(self, int ind[3])
cdef void neighbors(self, Oct *, Oct **)
cdef void oct_bounds(self, Oct *, np.float64_t *, np.float64_t *)
# This function must return the offset from global-to-local domains; i.e.,
@@ -76,6 +78,8 @@
cdef class RAMSESOctreeContainer(OctreeContainer):
cdef OctAllocationContainer **domains
+ cdef Oct **root_nodes
+ cdef int num_root
cdef Oct *next_root(self, int domain_id, int ind[3])
cdef Oct *next_child(self, int domain_id, int ind[3], Oct *parent)
diff -r 3959497fa6a3f90e117788dcd5f570296387ad7e -r ff8d85b3c294be269af13380e3aa881c1df0adcc yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -173,13 +173,16 @@
cdef np.int64_t get_domain_offset(self, int domain_id):
return 0
+ cdef Oct *get_root(self, int ind[3]):
+ return self.root_mesh[ind[0]][ind[1]][ind[2]]
+
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
cdef Oct *get(self, np.float64_t ppos[3], OctInfo *oinfo = NULL):
#Given a floating point position, retrieve the most
#refined oct at that time
- cdef np.int64_t ind[3]
+ cdef int ind[3]
cdef np.float64_t dds[3], cp[3], pp[3]
cdef Oct *cur
cdef int i
@@ -187,7 +190,7 @@
dds[i] = (self.DRE[i] - self.DLE[i])/self.nn[i]
ind[i] = <np.int64_t> ((ppos[i] - self.DLE[i])/dds[i])
cp[i] = (ind[i] + 0.5) * dds[i] + self.DLE[i]
- next = self.root_mesh[ind[0]][ind[1]][ind[2]]
+ next = self.get_root(ind)
# We want to stop recursing when there's nowhere else to go
while next != NULL:
cur = next
@@ -427,8 +430,86 @@
return dest
return data.index - offset
+cdef int root_key_compare(void *key, void *member) nogil:
+ cdef int i
+ cdef np.int64_t vkey = 0
+ cdef Oct *o = <Oct*>member
+ cdef np.int64_t fkey = (<np.int64_t *>key)[0]
+ for i in range(3):
+ vkey |= (o.pos[i] << 20 * (2 - i))
+ if vkey < fkey:
+ return -1
+ elif vkey == fkey:
+ return 0
+ else:
+ return 1
+
+cdef int root_node_compare(void *a, void *b) nogil:
+ cdef int i
+ cdef np.int64_t akey, bkey
+ cdef Oct *ao, *bo
+ ao = <Oct *>a
+ bo = <Oct *>b
+ for i in range(3):
+ akey |= (ao.pos[i] << 20 * (2 - i))
+ bkey |= (bo.pos[i] << 20 * (2 - i))
+ if akey < bkey:
+ return -1
+ elif akey == bkey:
+ return 0
+ else:
+ return 1
+
cdef class RAMSESOctreeContainer(OctreeContainer):
+ def __init__(self, domain_dimensions, domain_left_edge, domain_right_edge):
+ cdef int i, j, k, p
+ for i in range(3):
+ self.nn[i] = domain_dimensions[i]
+ self.max_domain = -1
+ self.nocts = 0 # Increment when initialized
+ self.root_mesh = NULL
+ self.root_nodes = NULL
+ self.num_root = 0
+ # We don't initialize the octs yet
+ for i in range(3):
+ self.DLE[i] = domain_left_edge[i] #0
+ self.DRE[i] = domain_right_edge[i] #num_grid
+
+ def __dealloc__(self):
+ free_octs(self.cont)
+
+ def finalize(self):
+ return
+
+ cdef Oct *get_root(self, int ind[3]):
+ cdef int i
+ cdef np.int64_t key = 0
+ for i in range(3):
+ key |= (ind[i] << 20 * (2 - i))
+ cdef Oct *o = <Oct *> bsearch(&key, self.root_nodes[0],
+ self.num_root, sizeof(Oct *), root_key_compare)
+ return o
+
+ @cython.cdivision(True)
+ cdef void visit_all_octs(self, SelectorObject selector,
+ oct_visitor_function *func,
+ OctVisitorData *data):
+ cdef int i, j, k, n
+ data.global_index = -1
+ cdef np.float64_t pos[3], dds[3]
+ # This dds is the oct-width
+ for i in range(3):
+ dds[i] = (self.DRE[i] - self.DLE[i]) / self.nn[i]
+ # Pos is the center of the octs
+ cdef Oct *o
+ for i in range(self.num_root):
+ o = self.root_nodes[i]
+ for j in range(3):
+ pos[0] = self.DLE[0] + (o.pos[0] + 0.5) * dds[0]
+ selector.recursively_visit_octs(
+ o, pos, dds, 0, func, data)
+
def domain_identify(self, SelectorObject selector):
cdef np.ndarray[np.uint8_t, ndim=1] domain_mask
domain_mask = np.zeros(self.max_domain, dtype="uint8")
@@ -448,18 +529,24 @@
return cont.offset
cdef Oct* next_root(self, int domain_id, int ind[3]):
- cdef Oct *next = self.root_mesh[ind[0]][ind[1]][ind[2]]
+ # We assume that 20 bits is enough for each index.
+ cdef int i
+ next = self.get_root(ind)
if next != NULL: return next
+ # Otherwise, we'll have to insert and then qsort
cdef OctAllocationContainer *cont = self.domains[domain_id - 1]
if cont.n_assigned >= cont.n: raise RuntimeError
next = &cont.my_octs[cont.n_assigned]
cont.n_assigned += 1
- self.root_mesh[ind[0]][ind[1]][ind[2]] = next
+ self.root_nodes[self.num_root] = next
+ self.num_root += 1
next.parent = NULL
next.level = 0
for i in range(3):
next.pos[i] = ind[i]
self.nocts += 1
+ qsort(self.root_nodes, self.num_root, sizeof(Oct *),
+ root_node_compare)
return next
cdef Oct* next_child(self, int domain_id, int ind[3], Oct *parent):
@@ -477,7 +564,7 @@
self.nocts += 1
return next
- def allocate_domains(self, domain_counts):
+ def allocate_domains(self, domain_counts, int root_nodes):
cdef int count, i
cdef OctAllocationContainer *cur = self.cont
assert(cur == NULL)
@@ -488,20 +575,23 @@
cur = allocate_octs(count, cur)
if self.cont == NULL: self.cont = cur
self.domains[i] = cur
+ self.root_nodes = <Oct**> malloc(sizeof(Oct*) * root_nodes)
+ for i in range(root_nodes):
+ self.root_nodes[i] = NULL
def __dealloc__(self):
# This gets called BEFORE the superclass deallocation. But, both get
# called.
+ if self.root_nodes != NULL: free(self.root_nodes)
if self.domains != NULL: free(self.domains)
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def add(self, int curdom, int curlevel, int ng,
+ def add(self, int curdom, int curlevel,
np.ndarray[np.float64_t, ndim=2] pos,
- int local_domain, int skip_boundary = 1):
+ int skip_boundary = 1):
cdef int level, no, p, i, j, k, ind[3]
- cdef int local = (local_domain == curdom)
cdef Oct *cur, *next = NULL
cdef np.float64_t pp[3], cp[3], dds[3]
no = pos.shape[0] #number of octs
@@ -541,8 +631,7 @@
cur = self.next_child(curdom, ind, cur)
# Now we should be at the right level
cur.domain = curdom
- if local == 1:
- cur.file_ind = p
+ cur.file_ind = p
cur.level = curlevel
return cont.n_assigned - initial
https://bitbucket.org/yt_analysis/yt-3.0/commits/eafad7983797/
Changeset: eafad7983797
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-22 02:27:43
Summary: This allows the RAMSES oct container to run.
Affected #: 1 file
diff -r ff8d85b3c294be269af13380e3aa881c1df0adcc -r eafad7983797c4f5f7a1733b6e2a53f54dad0e60 yt/frontends/ramses/data_structures.py
--- a/yt/frontends/ramses/data_structures.py
+++ b/yt/frontends/ramses/data_structures.py
@@ -212,7 +212,6 @@
self.amr_header['nboundary']*l]
return ng
min_level = self.pf.min_level
- total = 0
nx, ny, nz = (((i-1.0)/2.0) for i in self.amr_header['nx'])
for level in range(self.amr_header['nlevelmax']):
# Easier if do this 1-indexed
@@ -244,10 +243,7 @@
# Note that we're adding *grids*, not individual cells.
if level >= min_level:
assert(pos.shape[0] == ng)
- if cpu + 1 == self.domain_id:
- total += ng
- self.oct_handler.add(cpu + 1, level - min_level,
- pos, self.domain_id)
+ self.oct_handler.add(cpu + 1, level - min_level, pos)
self.oct_handler.finalize()
#raise RuntimeError
https://bitbucket.org/yt_analysis/yt-3.0/commits/01540ae0e5fd/
Changeset: 01540ae0e5fd
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-22 05:21:20
Summary: Switching to using tsearch. Still lingering issues.
Affected #: 3 files
diff -r eafad7983797c4f5f7a1733b6e2a53f54dad0e60 -r 01540ae0e5fd94ff886d88616ade01c5e0f34f84 yt/frontends/ramses/data_structures.py
--- a/yt/frontends/ramses/data_structures.py
+++ b/yt/frontends/ramses/data_structures.py
@@ -245,7 +245,6 @@
assert(pos.shape[0] == ng)
self.oct_handler.add(cpu + 1, level - min_level, pos)
self.oct_handler.finalize()
- #raise RuntimeError
def included(self, selector):
domain_ids = self.oct_handler.domain_identify(selector)
@@ -459,7 +458,7 @@
self.omega_lambda = rheader["omega_l"]
self.omega_matter = rheader["omega_m"]
self.hubble_constant = rheader["H0"] / 100.0 # This is H100
- self.max_level = rheader['levelmax'] - rheader['levelmin']
+ self.max_level = rheader['levelmax'] - self.min_level
@classmethod
def _is_valid(self, *args, **kwargs):
diff -r eafad7983797c4f5f7a1733b6e2a53f54dad0e60 -r 01540ae0e5fd94ff886d88616ade01c5e0f34f84 yt/geometry/oct_container.pxd
--- a/yt/geometry/oct_container.pxd
+++ b/yt/geometry/oct_container.pxd
@@ -46,6 +46,10 @@
Oct *children[2][2][2]
Oct *parent
+cdef struct OctKey:
+ np.int64_t key
+ Oct *node
+
cdef struct OctInfo:
np.float64_t left_edge[3]
np.float64_t dds[3]
@@ -65,8 +69,8 @@
cdef np.float64_t DLE[3], DRE[3]
cdef public int nocts
cdef public int max_domain
- cdef Oct* get(self, np.float64_t ppos[3], OctInfo *oinfo = ?)
- cdef Oct *get_root(self, int ind[3])
+ cdef Oct *get(self, np.float64_t ppos[3], OctInfo *oinfo = ?)
+ cdef int get_root(self, int ind[3], Oct **o)
cdef void neighbors(self, Oct *, Oct **)
cdef void oct_bounds(self, Oct *, np.float64_t *, np.float64_t *)
# This function must return the offset from global-to-local domains; i.e.,
@@ -78,8 +82,25 @@
cdef class RAMSESOctreeContainer(OctreeContainer):
cdef OctAllocationContainer **domains
- cdef Oct **root_nodes
+ cdef OctKey *root_nodes
+ cdef void *tree_root
cdef int num_root
+ cdef int max_root
cdef Oct *next_root(self, int domain_id, int ind[3])
cdef Oct *next_child(self, int domain_id, int ind[3], Oct *parent)
+cdef extern from "search.h" nogil:
+ void *tsearch(const void *key, void **rootp,
+ int (*compar)(const void *, const void *))
+ void *tfind(const void *key, const void **rootp,
+ int (*compar)(const void *, const void *))
+ void *tdelete(const void *key, void **rootp,
+ int (*compar)(const void *, const void *))
+
+cdef inline np.int64_t oct_key(Oct *o):
+ cdef int i
+ if o.level != 0: return -1
+ cdef np.int64_t key = 0
+ for i in range(3):
+ key |= (o.pos[i] << 20 * (2 - i))
+ return key
diff -r eafad7983797c4f5f7a1733b6e2a53f54dad0e60 -r 01540ae0e5fd94ff886d88616ade01c5e0f34f84 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -173,8 +173,9 @@
cdef np.int64_t get_domain_offset(self, int domain_id):
return 0
- cdef Oct *get_root(self, int ind[3]):
- return self.root_mesh[ind[0]][ind[1]][ind[2]]
+ cdef int get_root(self, int ind[3], Oct **o):
+ o[0] = self.root_mesh[ind[0]][ind[1]][ind[2]]
+ return 1
@cython.boundscheck(False)
@cython.wraparound(False)
@@ -184,13 +185,13 @@
#refined oct at that time
cdef int ind[3]
cdef np.float64_t dds[3], cp[3], pp[3]
- cdef Oct *cur
+ cdef Oct *cur, *next
cdef int i
for i in range(3):
dds[i] = (self.DRE[i] - self.DLE[i])/self.nn[i]
ind[i] = <np.int64_t> ((ppos[i] - self.DLE[i])/dds[i])
cp[i] = (ind[i] + 0.5) * dds[i] + self.DLE[i]
- next = self.get_root(ind)
+ self.get_root(ind, &next)
# We want to stop recursing when there's nowhere else to go
while next != NULL:
cur = next
@@ -430,32 +431,13 @@
return dest
return data.index - offset
-cdef int root_key_compare(void *key, void *member) nogil:
- cdef int i
- cdef np.int64_t vkey = 0
- cdef Oct *o = <Oct*>member
- cdef np.int64_t fkey = (<np.int64_t *>key)[0]
- for i in range(3):
- vkey |= (o.pos[i] << 20 * (2 - i))
- if vkey < fkey:
+cdef int root_node_compare(void *a, void *b) nogil:
+ cdef OctKey *ao, *bo
+ ao = <OctKey *>a
+ bo = <OctKey *>b
+ if ao.key < bo.key:
return -1
- elif vkey == fkey:
- return 0
- else:
- return 1
-
-cdef int root_node_compare(void *a, void *b) nogil:
- cdef int i
- cdef np.int64_t akey, bkey
- cdef Oct *ao, *bo
- ao = <Oct *>a
- bo = <Oct *>b
- for i in range(3):
- akey |= (ao.pos[i] << 20 * (2 - i))
- bkey |= (bo.pos[i] << 20 * (2 - i))
- if akey < bkey:
- return -1
- elif akey == bkey:
+ elif ao.key == bo.key:
return 0
else:
return 1
@@ -470,26 +452,29 @@
self.nocts = 0 # Increment when initialized
self.root_mesh = NULL
self.root_nodes = NULL
+ self.tree_root = NULL
self.num_root = 0
# We don't initialize the octs yet
for i in range(3):
self.DLE[i] = domain_left_edge[i] #0
self.DRE[i] = domain_right_edge[i] #num_grid
- def __dealloc__(self):
- free_octs(self.cont)
-
def finalize(self):
return
- cdef Oct *get_root(self, int ind[3]):
+ cdef int get_root(self, int ind[3], Oct **o):
+ o[0] = NULL
cdef int i
cdef np.int64_t key = 0
for i in range(3):
key |= (ind[i] << 20 * (2 - i))
- cdef Oct *o = <Oct *> bsearch(&key, self.root_nodes[0],
- self.num_root, sizeof(Oct *), root_key_compare)
- return o
+ cdef OctKey okey, *oresult
+ okey.key = key
+ okey.node = NULL
+ oresult = <OctKey *> tfind(<void*>&okey,
+ &self.tree_root, root_node_compare)
+ if oresult != NULL:
+ o[0] = oresult.node
@cython.cdivision(True)
cdef void visit_all_octs(self, SelectorObject selector,
@@ -504,7 +489,7 @@
# Pos is the center of the octs
cdef Oct *o
for i in range(self.num_root):
- o = self.root_nodes[i]
+ o = self.root_nodes[i].node
for j in range(3):
pos[0] = self.DLE[0] + (o.pos[0] + 0.5) * dds[0]
selector.recursively_visit_octs(
@@ -531,22 +516,23 @@
cdef Oct* next_root(self, int domain_id, int ind[3]):
# We assume that 20 bits is enough for each index.
cdef int i
- next = self.get_root(ind)
+ cdef Oct *next
+ self.get_root(ind, &next)
if next != NULL: return next
- # Otherwise, we'll have to insert and then qsort
cdef OctAllocationContainer *cont = self.domains[domain_id - 1]
- if cont.n_assigned >= cont.n: raise RuntimeError
+ if cont.n_assigned >= cont.n: return NULL
next = &cont.my_octs[cont.n_assigned]
cont.n_assigned += 1
- self.root_nodes[self.num_root] = next
- self.num_root += 1
next.parent = NULL
next.level = 0
for i in range(3):
next.pos[i] = ind[i]
self.nocts += 1
- qsort(self.root_nodes, self.num_root, sizeof(Oct *),
- root_node_compare)
+ self.root_nodes[self.num_root].key = oct_key(next)
+ self.root_nodes[self.num_root].node = next
+ tsearch(<void*>&self.root_nodes[self.num_root], &self.tree_root,
+ root_node_compare)
+ self.num_root += 1
return next
cdef Oct* next_child(self, int domain_id, int ind[3], Oct *parent):
@@ -575,13 +561,16 @@
cur = allocate_octs(count, cur)
if self.cont == NULL: self.cont = cur
self.domains[i] = cur
- self.root_nodes = <Oct**> malloc(sizeof(Oct*) * root_nodes)
+ self.root_nodes = <OctKey*> malloc(sizeof(OctKey) * root_nodes)
+ self.max_root = root_nodes
for i in range(root_nodes):
- self.root_nodes[i] = NULL
+ self.root_nodes[i].key = -1
+ self.root_nodes[i].node = NULL
def __dealloc__(self):
# This gets called BEFORE the superclass deallocation. But, both get
# called.
+ free_octs(self.cont)
if self.root_nodes != NULL: free(self.root_nodes)
if self.domains != NULL: free(self.domains)
@@ -613,6 +602,7 @@
in_boundary = 1
if skip_boundary == in_boundary == 1: continue
cur = self.next_root(curdom, ind)
+ if cur == NULL: raise RuntimeError
# Now we find the location we want
# Note that RAMSES I think 1-findiceses levels, but we don't.
for level in range(curlevel):
https://bitbucket.org/yt_analysis/yt-3.0/commits/fc1946874090/
Changeset: fc1946874090
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-22 05:58:11
Summary: Inserting and sorting now works for disconnecte RAMSES octree subsets.
Affected #: 3 files
diff -r 01540ae0e5fd94ff886d88616ade01c5e0f34f84 -r fc1946874090f5c420f98c457e83d80977c03ee4 yt/frontends/ramses/data_structures.py
--- a/yt/frontends/ramses/data_structures.py
+++ b/yt/frontends/ramses/data_structures.py
@@ -327,7 +327,7 @@
# Note that domain_ids will be ONE INDEXED
domains = [dom for dom in self.domains if
dom.included(dobj.selector)]
- mylog.debug("Identified %s intersecting domains", len(domain_ids))
+ mylog.debug("Identified %s intersecting domains", len(domains))
subsets = [RAMSESDomainSubset(base_region, domain, self.parameter_file)
for domain in domains]
dobj._chunk_info = subsets
diff -r 01540ae0e5fd94ff886d88616ade01c5e0f34f84 -r fc1946874090f5c420f98c457e83d80977c03ee4 yt/geometry/oct_container.pxd
--- a/yt/geometry/oct_container.pxd
+++ b/yt/geometry/oct_container.pxd
@@ -96,11 +96,3 @@
int (*compar)(const void *, const void *))
void *tdelete(const void *key, void **rootp,
int (*compar)(const void *, const void *))
-
-cdef inline np.int64_t oct_key(Oct *o):
- cdef int i
- if o.level != 0: return -1
- cdef np.int64_t key = 0
- for i in range(3):
- key |= (o.pos[i] << 20 * (2 - i))
- return key
diff -r 01540ae0e5fd94ff886d88616ade01c5e0f34f84 -r fc1946874090f5c420f98c457e83d80977c03ee4 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -117,13 +117,13 @@
def __dealloc__(self):
free_octs(self.cont)
+ if self.root_mesh == NULL: return
for i in range(self.nn[0]):
for j in range(self.nn[1]):
if self.root_mesh[i][j] == NULL: continue
free(self.root_mesh[i][j])
if self.root_mesh[i] == NULL: continue
free(self.root_mesh[i])
- if self.root_mesh == NULL: return
free(self.root_mesh)
def __iter__(self):
@@ -468,13 +468,13 @@
cdef np.int64_t key = 0
for i in range(3):
key |= (ind[i] << 20 * (2 - i))
- cdef OctKey okey, *oresult
+ cdef OctKey okey, **oresult
okey.key = key
okey.node = NULL
- oresult = <OctKey *> tfind(<void*>&okey,
+ oresult = <OctKey **> tfind(<void*>&okey,
&self.tree_root, root_node_compare)
if oresult != NULL:
- o[0] = oresult.node
+ o[0] = oresult[0].node
@cython.cdivision(True)
cdef void visit_all_octs(self, SelectorObject selector,
@@ -521,18 +521,22 @@
if next != NULL: return next
cdef OctAllocationContainer *cont = self.domains[domain_id - 1]
if cont.n_assigned >= cont.n: return NULL
+ if self.num_root >= self.max_root:
+ return NULL
next = &cont.my_octs[cont.n_assigned]
cont.n_assigned += 1
next.parent = NULL
next.level = 0
+ cdef np.int64_t key = 0
+ cdef OctKey *ikey = &self.root_nodes[self.num_root]
for i in range(3):
next.pos[i] = ind[i]
+ key |= (ind[i] << 20 * (2 - i))
+ self.root_nodes[self.num_root].key = key
+ self.root_nodes[self.num_root].node = next
+ tsearch(<void*>ikey, &self.tree_root, root_node_compare)
+ self.num_root += 1
self.nocts += 1
- self.root_nodes[self.num_root].key = oct_key(next)
- self.root_nodes[self.num_root].node = next
- tsearch(<void*>&self.root_nodes[self.num_root], &self.tree_root,
- root_node_compare)
- self.num_root += 1
return next
cdef Oct* next_child(self, int domain_id, int ind[3], Oct *parent):
@@ -570,7 +574,6 @@
def __dealloc__(self):
# This gets called BEFORE the superclass deallocation. But, both get
# called.
- free_octs(self.cont)
if self.root_nodes != NULL: free(self.root_nodes)
if self.domains != NULL: free(self.domains)
https://bitbucket.org/yt_analysis/yt-3.0/commits/b0f74baf53c3/
Changeset: b0f74baf53c3
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-22 06:02:29
Summary: Minor fix to grid selection.
Affected #: 1 file
diff -r fc1946874090f5c420f98c457e83d80977c03ee4 -r b0f74baf53c3a667957cd09dbce56bf032362cd4 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -491,9 +491,9 @@
for i in range(self.num_root):
o = self.root_nodes[i].node
for j in range(3):
- pos[0] = self.DLE[0] + (o.pos[0] + 0.5) * dds[0]
- selector.recursively_visit_octs(
- o, pos, dds, 0, func, data)
+ pos[j] = self.DLE[j] + (o.pos[j] + 0.5) * dds[j]
+ selector.recursively_visit_octs(
+ o, pos, dds, 0, func, data)
def domain_identify(self, SelectorObject selector):
cdef np.ndarray[np.uint8_t, ndim=1] domain_mask
https://bitbucket.org/yt_analysis/yt-3.0/commits/f20de60b7af3/
Changeset: f20de60b7af3
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-22 15:22:11
Summary: This change fixes things for O2 and above in dist-mem RAMSES.
Copying arrays is still broken, which is unfortunate since it's the exact thing
I was trying to address when I started down the path of breaking up the Octree
containers!
Affected #: 1 file
diff -r b0f74baf53c3a667957cd09dbce56bf032362cd4 -r f20de60b7af3f9a0c716187292450fccd72d7d07 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -467,7 +467,7 @@
cdef int i
cdef np.int64_t key = 0
for i in range(3):
- key |= (ind[i] << 20 * (2 - i))
+ key |= ((<np.int64_t>ind[i]) << 20 * (2 - i))
cdef OctKey okey, **oresult
okey.key = key
okey.node = NULL
@@ -520,8 +520,11 @@
self.get_root(ind, &next)
if next != NULL: return next
cdef OctAllocationContainer *cont = self.domains[domain_id - 1]
- if cont.n_assigned >= cont.n: return NULL
+ if cont.n_assigned >= cont.n:
+ print "Too many assigned."
+ return NULL
if self.num_root >= self.max_root:
+ print "Too many roots."
return NULL
next = &cont.my_octs[cont.n_assigned]
cont.n_assigned += 1
@@ -531,7 +534,7 @@
cdef OctKey *ikey = &self.root_nodes[self.num_root]
for i in range(3):
next.pos[i] = ind[i]
- key |= (ind[i] << 20 * (2 - i))
+ key |= ((<np.int64_t>ind[i]) << 20 * (2 - i))
self.root_nodes[self.num_root].key = key
self.root_nodes[self.num_root].node = next
tsearch(<void*>ikey, &self.tree_root, root_node_compare)
https://bitbucket.org/yt_analysis/yt-3.0/commits/01bc11c244dd/
Changeset: 01bc11c244dd
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-22 16:10:55
Summary: This should be signed so we can have it be -1 without trouble.
Affected #: 1 file
diff -r f20de60b7af3f9a0c716187292450fccd72d7d07 -r 01bc11c244ddcd4a8013219a4f9e6dbd63727ffa yt/geometry/selection_routines.pxd
--- a/yt/geometry/selection_routines.pxd
+++ b/yt/geometry/selection_routines.pxd
@@ -29,7 +29,7 @@
cdef struct OctVisitorData:
np.uint64_t index
np.uint64_t last
- np.uint64_t global_index
+ np.int64_t global_index
int ind[3]
void *array
int dims
https://bitbucket.org/yt_analysis/yt-3.0/commits/c8f4e92a99e5/
Changeset: c8f4e92a99e5
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-23 20:58:29
Summary: Domain checking should only show up in the visitation function.
Affected #: 2 files
diff -r 01bc11c244ddcd4a8013219a4f9e6dbd63727ffa -r c8f4e92a99e5c981e6cbf32e08486de472d20b75 yt/geometry/oct_visitors.pyx
--- a/yt/geometry/oct_visitors.pyx
+++ b/yt/geometry/oct_visitors.pyx
@@ -36,7 +36,6 @@
# We should always have global_index less than our source.
# "last" here tells us the dimensionality of the array.
if selected == 0: return
- if data.domain > 0 and o.domain != data.domain: return
cdef int i
# There are this many records between "octs"
cdef np.int64_t index = (data.global_index * 8)*data.dims
@@ -50,7 +49,6 @@
# We should always have global_index less than our source.
# "last" here tells us the dimensionality of the array.
if selected == 0: return
- if data.domain > 0 and o.domain != data.domain: return
cdef int i
cdef np.int64_t index = (data.global_index * 8)*data.dims
cdef np.int64_t **p = <np.int64_t**> data.array
@@ -62,7 +60,6 @@
cdef void count_total_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
# Count even if not selected.
# Number of *octs* visited.
- if data.domain > 0 and o.domain != data.domain: return
if data.last != o.domain_ind:
data.index += 1
data.last = o.domain_ind
@@ -70,7 +67,6 @@
cdef void count_total_cells(Oct *o, OctVisitorData *data, np.uint8_t selected):
# Count even if not selected.
# Number of *octs* visited.
- if data.domain > 0 and o.domain != data.domain: return
data.index += selected
cdef void mark_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
@@ -96,7 +92,6 @@
# Note that we provide an index even if the cell is not selected.
cdef int i
cdef np.int64_t *arr
- if data.domain > 0 and data.domain != o.domain: return
if data.last != o.domain_ind:
data.last = o.domain_ind
arr = <np.int64_t *> data.array
diff -r 01bc11c244ddcd4a8013219a4f9e6dbd63727ffa -r c8f4e92a99e5c981e6cbf32e08486de472d20b75 yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -207,9 +207,9 @@
next_level = 1
this_level = 0
increment = 0
- if level == self.max_level:
+ elif level == self.max_level:
next_level = 0
- if level < self.min_level or level > self.max_level:
+ elif level < self.min_level or level > self.max_level:
this_level = 0
if res == 0 and this_level == 1:
return
@@ -227,9 +227,9 @@
self.recursively_visit_octs(
ch, spos, sdds, level + 1, func, data)
elif this_level == 1:
+ selected = self.select_cell(spos, sdds, eterm)
data.global_index += increment
increment = 0
- selected = self.select_cell(spos, sdds, eterm)
data.ind[0] = i
data.ind[1] = j
data.ind[2] = k
@@ -1110,11 +1110,9 @@
grid_selector = GridSelector
cdef class OctreeSubsetSelector(SelectorObject):
- cdef int domain_id
cdef SelectorObject base_selector
def __init__(self, dobj):
- self.domain_id = dobj.domain.domain_id
self.base_selector = dobj.base_selector
@cython.boundscheck(False)
@@ -1153,12 +1151,7 @@
Oct *o = NULL) nogil:
# Because visitors now use select_grid, we should be explicitly
# checking this.
- cdef int res
- if o == NULL: return 0
- res = self.base_selector.select_grid(left_edge, right_edge, level, o)
- if res != 0 and o.domain != self.domain_id:
- res = -1
- return res
+ return self.base_selector.select_grid(left_edge, right_edge, level, o)
octree_subset_selector = OctreeSubsetSelector
https://bitbucket.org/yt_analysis/yt-3.0/commits/c841f83d89f4/
Changeset: c841f83d89f4
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-24 16:00:38
Summary: Ditch the 'size' for spatial chunking.
Affected #: 1 file
diff -r c8f4e92a99e5c981e6cbf32e08486de472d20b75 -r c841f83d89f4b0964d879e1f4487a541571b1a0b yt/frontends/ramses/data_structures.py
--- a/yt/frontends/ramses/data_structures.py
+++ b/yt/frontends/ramses/data_structures.py
@@ -344,9 +344,7 @@
g = og.retrieve_ghost_zones(ngz, [], smoothed=True)
else:
g = og
- size = og.cell_count
- if size == 0: continue
- yield YTDataChunk(dobj, "spatial", [g], size)
+ yield YTDataChunk(dobj, "spatial", [g], None)
def _chunk_io(self, dobj):
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
https://bitbucket.org/yt_analysis/yt-3.0/commits/2089e4129601/
Changeset: 2089e4129601
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-24 16:53:57
Summary: First step toward partial coverage of domains/octs for RAMSES.
Affected #: 7 files
diff -r c841f83d89f4b0964d879e1f4487a541571b1a0b -r 2089e412960103a0ca76966220d285eac65a1f32 yt/data_objects/octree_subset.py
--- a/yt/data_objects/octree_subset.py
+++ b/yt/data_objects/octree_subset.py
@@ -101,7 +101,7 @@
@property
def domain_ind(self):
if self._domain_ind is None:
- di = self.oct_handler.domain_ind(self.mask, self.domain.domain_id)
+ di = self.oct_handler.domain_ind(self.selector)
self._domain_ind = di
return self._domain_ind
@@ -188,11 +188,3 @@
self.base_region = base_region
self.base_selector = base_region.selector
- _domain_ind = None
-
- @property
- def domain_ind(self):
- if self._domain_ind is None:
- di = self.oct_handler.domain_ind(self.selector)
- self._domain_ind = di
- return self._domain_ind
diff -r c841f83d89f4b0964d879e1f4487a541571b1a0b -r 2089e412960103a0ca76966220d285eac65a1f32 yt/frontends/ramses/data_structures.py
--- a/yt/frontends/ramses/data_structures.py
+++ b/yt/frontends/ramses/data_structures.py
@@ -243,7 +243,8 @@
# Note that we're adding *grids*, not individual cells.
if level >= min_level:
assert(pos.shape[0] == ng)
- self.oct_handler.add(cpu + 1, level - min_level, pos)
+ n = self.oct_handler.add(cpu + 1, level - min_level, pos)
+ assert(n == ng)
self.oct_handler.finalize()
def included(self, selector):
diff -r c841f83d89f4b0964d879e1f4487a541571b1a0b -r 2089e412960103a0ca76966220d285eac65a1f32 yt/geometry/oct_container.pxd
--- a/yt/geometry/oct_container.pxd
+++ b/yt/geometry/oct_container.pxd
@@ -65,6 +65,7 @@
cdef class OctreeContainer:
cdef OctAllocationContainer *cont
cdef Oct ****root_mesh
+ cdef int partial_coverage
cdef int nn[3]
cdef np.float64_t DLE[3], DRE[3]
cdef public int nocts
diff -r c841f83d89f4b0964d879e1f4487a541571b1a0b -r 2089e412960103a0ca76966220d285eac65a1f32 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -97,6 +97,7 @@
def __init__(self, oct_domain_dimensions, domain_left_edge, domain_right_edge):
# This will just initialize the root mesh octs
+ self.partial_coverage = 0
cdef int i, j, k, p
for i in range(3):
self.nn[i] = oct_domain_dimensions[i]
@@ -143,7 +144,8 @@
cdef void visit_all_octs(self, SelectorObject selector,
oct_visitor_function *func,
OctVisitorData *data):
- cdef int i, j, k, n
+ cdef int i, j, k, n, vc
+ vc = self.partial_coverage
data.global_index = -1
cdef np.float64_t pos[3], dds[3]
# This dds is the oct-width
@@ -159,7 +161,7 @@
if self.root_mesh[i][j][k] == NULL: continue
selector.recursively_visit_octs(
self.root_mesh[i][j][k],
- pos, dds, 0, func, data)
+ pos, dds, 0, func, data, vc)
pos[2] += dds[2]
pos[1] += dds[1]
pos[0] += dds[0]
@@ -314,7 +316,7 @@
def icoords(self, SelectorObject selector, np.uint64_t num_cells = -1,
int domain_id = -1):
if num_cells == -1:
- num_cells = selector.count_octs(self, domain_id)
+ num_cells = selector.count_oct_cells(self, domain_id)
cdef np.ndarray[np.int64_t, ndim=2] coords
coords = np.empty((num_cells, 3), dtype="int64")
cdef OctVisitorData data
@@ -330,7 +332,7 @@
def ires(self, SelectorObject selector, np.uint64_t num_cells = -1,
int domain_id = -1):
if num_cells == -1:
- num_cells = selector.count_octs(self, domain_id)
+ num_cells = selector.count_octs(self, domain_id) * 8
#Return the 'resolution' of each cell; ie the level
cdef np.ndarray[np.int64_t, ndim=1] res
res = np.empty(num_cells, dtype="int64")
@@ -347,7 +349,7 @@
def fwidth(self, SelectorObject selector, np.uint64_t num_cells = -1,
int domain_id = -1):
if num_cells == -1:
- num_cells = selector.count_octs(self, domain_id)
+ num_cells = selector.count_octs(self, domain_id) * 8
cdef np.ndarray[np.float64_t, ndim=2] fwidth
fwidth = np.empty((num_cells, 3), dtype="float64")
cdef OctVisitorData data
@@ -367,7 +369,7 @@
def fcoords(self, SelectorObject selector, np.uint64_t num_cells = -1,
int domain_id = -1):
if num_cells == -1:
- num_cells = selector.count_octs(self, domain_id)
+ num_cells = selector.count_octs(self, domain_id) * 8
#Return the floating point unitary position of every cell
cdef np.ndarray[np.float64_t, ndim=2] coords
coords = np.empty((num_cells, 3), dtype="float64")
@@ -392,14 +394,17 @@
# This is actually not correct. The hard part is that we need to
# iterate the same way visit_all_octs does, but we need to track the
# number of octs total visited.
- cdef np.int64_t num_cells = -1
+ cdef np.int64_t num_octs = -1
if dest is None:
- num_cells = selector.count_octs(self, domain_id)
+ # Note that RAMSES can have partial refinement inside an Oct. This
+ # means we actually do want the number of Octs, not the number of
+ # cells.
+ num_cells = selector.count_oct_cells(self, domain_id)
if dims > 1:
dest = np.zeros((num_cells, dims), dtype=source.dtype,
order='C')
else:
- dest = np.zeros(num_cells, dtype=source.dtype, order='C')
+ dest = np.zeros(num_cells * 8, dtype=source.dtype, order='C')
cdef OctVisitorData data
data.index = offset
data.domain = domain_id
@@ -422,6 +427,7 @@
if (data.global_index + 1) * 8 * data.dims > source.size:
print "GLOBAL INDEX RAN AHEAD.",
print (data.global_index + 1) * 8 * data.dims - source.size
+ print dest.size, source.size, num_cells
raise RuntimeError
if data.index > dest.size:
print "DEST INDEX RAN AHEAD.",
@@ -446,6 +452,7 @@
def __init__(self, domain_dimensions, domain_left_edge, domain_right_edge):
cdef int i, j, k, p
+ self.partial_coverage = 1
for i in range(3):
self.nn[i] = domain_dimensions[i]
self.max_domain = -1
@@ -480,8 +487,9 @@
cdef void visit_all_octs(self, SelectorObject selector,
oct_visitor_function *func,
OctVisitorData *data):
- cdef int i, j, k, n
+ cdef int i, j, k, n, vc
data.global_index = -1
+ vc = self.partial_coverage
cdef np.float64_t pos[3], dds[3]
# This dds is the oct-width
for i in range(3):
diff -r c841f83d89f4b0964d879e1f4487a541571b1a0b -r 2089e412960103a0ca76966220d285eac65a1f32 yt/geometry/oct_visitors.pyx
--- a/yt/geometry/oct_visitors.pyx
+++ b/yt/geometry/oct_visitors.pyx
@@ -65,8 +65,7 @@
data.last = o.domain_ind
cdef void count_total_cells(Oct *o, OctVisitorData *data, np.uint8_t selected):
- # Count even if not selected.
- # Number of *octs* visited.
+ # Number of *cells* visited and selected.
data.index += selected
cdef void mark_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
diff -r c841f83d89f4b0964d879e1f4487a541571b1a0b -r 2089e412960103a0ca76966220d285eac65a1f32 yt/geometry/selection_routines.pxd
--- a/yt/geometry/selection_routines.pxd
+++ b/yt/geometry/selection_routines.pxd
@@ -46,7 +46,8 @@
np.float64_t pos[3], np.float64_t dds[3],
int level,
oct_visitor_function *func,
- OctVisitorData *data)
+ OctVisitorData *data,
+ int visit_covered = ?)
cdef int select_grid(self, np.float64_t left_edge[3],
np.float64_t right_edge[3],
np.int32_t level, Oct *o = ?) nogil
diff -r c841f83d89f4b0964d879e1f4487a541571b1a0b -r 2089e412960103a0ca76966220d285eac65a1f32 yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -166,6 +166,13 @@
cdef OctVisitorData data
data.index = 0
data.domain = domain_id
+ octree.visit_all_octs(self, oct_visitors.count_total_octs, &data)
+ return data.index
+
+ def count_oct_cells(self, OctreeContainer octree, int domain_id = -1):
+ cdef OctVisitorData data
+ data.index = 0
+ data.domain = domain_id
octree.visit_all_octs(self, oct_visitors.count_total_cells, &data)
return data.index
@@ -176,7 +183,8 @@
np.float64_t pos[3], np.float64_t dds[3],
int level,
oct_visitor_function *func,
- OctVisitorData *data):
+ OctVisitorData *data,
+ int visit_covered = 0):
cdef np.float64_t LE[3], RE[3], sdds[3], spos[3]
cdef int i, j, k, res, ii
cdef Oct *ch
@@ -206,7 +214,6 @@
# not get accessed itself.
next_level = 1
this_level = 0
- increment = 0
elif level == self.max_level:
next_level = 0
elif level < self.min_level or level > self.max_level:
@@ -223,7 +230,7 @@
for k in range(2):
ii = ((k*2)+j)*2+i
ch = root.children[i][j][k]
- if next_level == 1 and ch != NULL:
+ if visit_covered == 0 and next_level == 1 and ch != NULL:
self.recursively_visit_octs(
ch, spos, sdds, level + 1, func, data)
elif this_level == 1:
@@ -237,6 +244,11 @@
spos[2] += sdds[2]
spos[1] += sdds[1]
spos[0] += sdds[0]
+ if visit_covered == 1:
+ # On our first pass through, we always only look at the current
+ # level. So we make a second pass to go downwards.
+ self.recursively_visit_octs(root, pos, dds, level, func,
+ data, 0)
cdef int select_grid(self, np.float64_t left_edge[3],
np.float64_t right_edge[3],
https://bitbucket.org/yt_analysis/yt-3.0/commits/0421a6418d61/
Changeset: 0421a6418d61
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-24 20:23:10
Summary: Try another tactic for filling partially-covered Octrees.
Affected #: 2 files
diff -r 2089e412960103a0ca76966220d285eac65a1f32 -r 0421a6418d618c59f9e065a6666376b4d1eb4a4d yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -316,7 +316,7 @@
def icoords(self, SelectorObject selector, np.uint64_t num_cells = -1,
int domain_id = -1):
if num_cells == -1:
- num_cells = selector.count_oct_cells(self, domain_id)
+ num_cells = selector.count_octs(self, domain_id) * 8
cdef np.ndarray[np.int64_t, ndim=2] coords
coords = np.empty((num_cells, 3), dtype="int64")
cdef OctVisitorData data
@@ -404,7 +404,7 @@
dest = np.zeros((num_cells, dims), dtype=source.dtype,
order='C')
else:
- dest = np.zeros(num_cells * 8, dtype=source.dtype, order='C')
+ dest = np.zeros(num_cells, dtype=source.dtype, order='C')
cdef OctVisitorData data
data.index = offset
data.domain = domain_id
@@ -501,7 +501,7 @@
for j in range(3):
pos[j] = self.DLE[j] + (o.pos[j] + 0.5) * dds[j]
selector.recursively_visit_octs(
- o, pos, dds, 0, func, data)
+ o, pos, dds, 0, func, data, vc)
def domain_identify(self, SelectorObject selector):
cdef np.ndarray[np.uint8_t, ndim=1] domain_mask
diff -r 2089e412960103a0ca76966220d285eac65a1f32 -r 0421a6418d618c59f9e065a6666376b4d1eb4a4d yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -185,6 +185,10 @@
oct_visitor_function *func,
OctVisitorData *data,
int visit_covered = 0):
+ # visit_covered tells us whether this octree supports partial
+ # refinement. If it does, we need to handle this specially -- first
+ # we visit *this* oct, then we make a second pass to check any child
+ # octs.
cdef np.float64_t LE[3], RE[3], sdds[3], spos[3]
cdef int i, j, k, res, ii
cdef Oct *ch
@@ -222,33 +226,33 @@
return
# Now we visit all our children. We subtract off sdds for the first
# pass because we center it on the first cell.
- spos[0] = pos[0] - sdds[0]/2.0
- for i in range(2):
- spos[1] = pos[1] - sdds[1]/2.0
- for j in range(2):
- spos[2] = pos[2] - sdds[2]/2.0
- for k in range(2):
- ii = ((k*2)+j)*2+i
- ch = root.children[i][j][k]
- if visit_covered == 0 and next_level == 1 and ch != NULL:
- self.recursively_visit_octs(
- ch, spos, sdds, level + 1, func, data)
- elif this_level == 1:
- selected = self.select_cell(spos, sdds, eterm)
- data.global_index += increment
- increment = 0
- data.ind[0] = i
- data.ind[1] = j
- data.ind[2] = k
- func(root, data, selected)
- spos[2] += sdds[2]
- spos[1] += sdds[1]
- spos[0] += sdds[0]
- if visit_covered == 1:
- # On our first pass through, we always only look at the current
- # level. So we make a second pass to go downwards.
- self.recursively_visit_octs(root, pos, dds, level, func,
- data, 0)
+ cdef int iter = 1 - visit_covered # 2 if 1, 1 if 0.
+ while iter < 2:
+ spos[0] = pos[0] - sdds[0]/2.0
+ for i in range(2):
+ spos[1] = pos[1] - sdds[1]/2.0
+ for j in range(2):
+ spos[2] = pos[2] - sdds[2]/2.0
+ for k in range(2):
+ ii = ((k*2)+j)*2+i
+ ch = root.children[i][j][k]
+ if iter == 1 and next_level == 1 and ch != NULL:
+ self.recursively_visit_octs(
+ ch, spos, sdds, level + 1, func, data,
+ visit_covered)
+ elif this_level == 1:
+ selected = self.select_cell(spos, sdds, eterm)
+ data.global_index += increment
+ increment = 0
+ data.ind[0] = i
+ data.ind[1] = j
+ data.ind[2] = k
+ func(root, data, selected)
+ spos[2] += sdds[2]
+ spos[1] += sdds[1]
+ spos[0] += sdds[0]
+ this_level = 0 # We turn this off for the second pass.
+ iter += 1
cdef int select_grid(self, np.float64_t left_edge[3],
np.float64_t right_edge[3],
https://bitbucket.org/yt_analysis/yt-3.0/commits/6a4182f19d47/
Changeset: 6a4182f19d47
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-24 20:31:03
Summary: Adding a _domain_offset, like _id_offset for grids.
Affected #: 3 files
diff -r 0421a6418d618c59f9e065a6666376b4d1eb4a4d -r 6a4182f19d47a1fdd4f977d7490773851587d8d5 yt/data_objects/octree_subset.py
--- a/yt/data_objects/octree_subset.py
+++ b/yt/data_objects/octree_subset.py
@@ -45,6 +45,7 @@
_skip_add = True
_con_args = ('base_region', 'domain', 'pf')
_container_fields = ("dx", "dy", "dz")
+ _domain_offset = 0
def __init__(self, base_region, domain, pf):
self.field_data = YTFieldData()
@@ -113,7 +114,8 @@
nvals = (2, 2, 2, (self.domain_ind >= 0).sum())
op = cls(nvals) # We allocate number of zones, not number of octs
op.initialize()
- op.process_octree(self.oct_handler, self.domain_ind, positions, fields, 0)
+ op.process_octree(self.oct_handler, self.domain_ind, positions, fields,
+ self.domain_id, self._domain_offset)
vals = op.finalize()
return np.asfortranarray(vals)
diff -r 0421a6418d618c59f9e065a6666376b4d1eb4a4d -r 6a4182f19d47a1fdd4f977d7490773851587d8d5 yt/frontends/ramses/data_structures.py
--- a/yt/frontends/ramses/data_structures.py
+++ b/yt/frontends/ramses/data_structures.py
@@ -253,6 +253,8 @@
class RAMSESDomainSubset(OctreeSubset):
+ _domain_offset = 1
+
def fill(self, content, fields):
# Here we get a copy of the file, which we skip through and read the
# bits we want.
diff -r 0421a6418d618c59f9e065a6666376b4d1eb4a4d -r 6a4182f19d47a1fdd4f977d7490773851587d8d5 yt/geometry/particle_deposit.pyx
--- a/yt/geometry/particle_deposit.pyx
+++ b/yt/geometry/particle_deposit.pyx
@@ -50,7 +50,8 @@
def process_octree(self, OctreeContainer octree,
np.ndarray[np.int64_t, ndim=1] dom_ind,
np.ndarray[np.float64_t, ndim=2] positions,
- fields = None, int domain_id = -1):
+ fields = None, int domain_id = -1,
+ int domain_offset = 0):
cdef int nf, i, j
self.bad_indices = 0
if fields is None:
@@ -68,7 +69,7 @@
cdef OctInfo oi
cdef np.int64_t offset, moff
cdef Oct *oct
- moff = octree.get_domain_offset(domain_id)
+ moff = octree.get_domain_offset(domain_id + domain_offset)
for i in range(positions.shape[0]):
# We should check if particle remains inside the Oct here
for j in range(nf):
https://bitbucket.org/yt_analysis/yt-3.0/commits/5dcf99cf0493/
Changeset: 5dcf99cf0493
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-24 20:55:24
Summary: Adding AlwaysSelector and fixing a few things from the multi/single domain
transition.
Affected #: 6 files
diff -r 6a4182f19d47a1fdd4f977d7490773851587d8d5 -r 5dcf99cf04933f874b840409288051360cdd5899 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -33,6 +33,7 @@
OctreeContainer, ORDER_MAX
from selection_routines cimport SelectorObject, \
OctVisitorData, oct_visitor_function
+import selection_routines
cimport oct_visitors
cimport cython
@@ -188,6 +189,7 @@
cdef int ind[3]
cdef np.float64_t dds[3], cp[3], pp[3]
cdef Oct *cur, *next
+ cur = next = NULL
cdef int i
for i in range(3):
dds[i] = (self.DRE[i] - self.DLE[i])/self.nn[i]
@@ -467,7 +469,12 @@
self.DRE[i] = domain_right_edge[i] #num_grid
def finalize(self):
- return
+ cdef SelectorObject selector = selection_routines.AlwaysSelector(None)
+ cdef OctVisitorData data
+ data.index = 0
+
+ self.visit_all_octs(selector, oct_visitors.assign_domain_ind, &data)
+ assert ((data.global_index+1)*8 == data.index)
cdef int get_root(self, int ind[3], Oct **o):
o[0] = NULL
@@ -518,8 +525,7 @@
return domain_ids
cdef np.int64_t get_domain_offset(self, int domain_id):
- cdef OctAllocationContainer *cont = self.domains[domain_id - 1]
- return cont.offset
+ return 0 # We no longer have a domain offset.
cdef Oct* next_root(self, int domain_id, int ind[3]):
# We assume that 20 bits is enough for each index.
diff -r 6a4182f19d47a1fdd4f977d7490773851587d8d5 -r 5dcf99cf04933f874b840409288051360cdd5899 yt/geometry/oct_visitors.pxd
--- a/yt/geometry/oct_visitors.pxd
+++ b/yt/geometry/oct_visitors.pxd
@@ -41,6 +41,7 @@
cdef oct_visitor_function copy_array_f64
cdef oct_visitor_function copy_array_i64
cdef oct_visitor_function identify_octs
+cdef oct_visitor_function assign_domain_ind
cdef inline int oind(OctVisitorData *data):
return (((data.ind[0]*2)+data.ind[1])*2+data.ind[2])
diff -r 6a4182f19d47a1fdd4f977d7490773851587d8d5 -r 5dcf99cf04933f874b840409288051360cdd5899 yt/geometry/oct_visitors.pyx
--- a/yt/geometry/oct_visitors.pyx
+++ b/yt/geometry/oct_visitors.pyx
@@ -147,3 +147,7 @@
if selected == 0: return
cdef np.uint8_t *arr = <np.uint8_t *> data.array
arr[o.domain - 1] = 1
+
+cdef void assign_domain_ind(Oct *o, OctVisitorData *data, np.uint8_t selected):
+ o.domain_ind = data.global_index
+ data.index += 1
diff -r 6a4182f19d47a1fdd4f977d7490773851587d8d5 -r 5dcf99cf04933f874b840409288051360cdd5899 yt/geometry/particle_deposit.pyx
--- a/yt/geometry/particle_deposit.pyx
+++ b/yt/geometry/particle_deposit.pyx
@@ -67,8 +67,9 @@
cdef int dims[3]
dims[0] = dims[1] = dims[2] = 2
cdef OctInfo oi
- cdef np.int64_t offset, moff
+ cdef np.int64_t offset, moff, missed = 0
cdef Oct *oct
+ cdef np.int64_t numpart = positions.shape[0]
moff = octree.get_domain_offset(domain_id + domain_offset)
for i in range(positions.shape[0]):
# We should check if particle remains inside the Oct here
@@ -83,6 +84,9 @@
# full octree structure. All we *really* care about is some
# arbitrary offset into a field value for deposition.
oct = octree.get(pos, &oi)
+ if oct == NULL:
+ missed += 1
+ continue
# This next line is unfortunate. Basically it says, sometimes we
# might have particles that belong to octs outside our domain.
if oct.domain != domain_id: continue
@@ -92,6 +96,9 @@
# Check that we found the oct ...
self.process(dims, oi.left_edge, oi.dds,
offset, pos, field_vals)
+ if missed > 0:
+ print "MISSED %s out of %s on domain %s" % (
+ missed, positions.shape[0], domain_id)
@cython.boundscheck(False)
@cython.wraparound(False)
diff -r 6a4182f19d47a1fdd4f977d7490773851587d8d5 -r 5dcf99cf04933f874b840409288051360cdd5899 yt/geometry/selection_routines.pxd
--- a/yt/geometry/selection_routines.pxd
+++ b/yt/geometry/selection_routines.pxd
@@ -56,4 +56,3 @@
cdef void set_bounds(self,
np.float64_t left_edge[3], np.float64_t right_edge[3],
np.float64_t dds[3], int ind[3][2], int *check)
-
diff -r 6a4182f19d47a1fdd4f977d7490773851587d8d5 -r 5dcf99cf04933f874b840409288051360cdd5899 yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -1223,3 +1223,47 @@
particle_octree_subset_selector = ParticleOctreeSubsetSelector
+cdef class AlwaysSelector(SelectorObject):
+
+ def __init__(self, dobj):
+ pass
+
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ def select_octs(self, OctreeContainer octree):
+ raise RuntimeError
+
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ cdef void set_bounds(self,
+ np.float64_t left_edge[3], np.float64_t right_edge[3],
+ np.float64_t dds[3], int ind[3][2], int *check):
+ check[0] = 0
+ return
+
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ def select_grids(self,
+ np.ndarray[np.float64_t, ndim=2] left_edges,
+ np.ndarray[np.float64_t, ndim=2] right_edges,
+ np.ndarray[np.int32_t, ndim=2] levels):
+ cdef int ng = left_edges.shape[0]
+ cdef np.ndarray[np.uint8_t, ndim=1] gridi = np.ones(ng, dtype='uint8')
+ return gridi.astype("bool")
+
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ cdef int select_cell(self, np.float64_t pos[3], np.float64_t dds[3],
+ int eterm[3]) nogil:
+ return 1
+
+ cdef int select_grid(self, np.float64_t left_edge[3],
+ np.float64_t right_edge[3], np.int32_t level,
+ Oct *o = NULL) nogil:
+ return 1
+
+always_selector = AlwaysSelector
https://bitbucket.org/yt_analysis/yt-3.0/commits/cdf4fdbb839d/
Changeset: cdf4fdbb839d
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-24 21:37:54
Summary: Enable sub-domain on sub-domain selection.
RAMSES is now working as before for particle deposition. Baryon field
deposition will require further work.
Affected #: 5 files
diff -r 5dcf99cf04933f874b840409288051360cdd5899 -r cdf4fdbb839d408c05f7c80b17b583c32f5f3ae4 yt/data_objects/octree_subset.py
--- a/yt/data_objects/octree_subset.py
+++ b/yt/data_objects/octree_subset.py
@@ -36,6 +36,7 @@
NeedsProperty, \
NeedsParameter
import yt.geometry.particle_deposit as particle_deposit
+from yt.funcs import *
class OctreeSubset(YTSelectionContainer):
_spatial = True
@@ -114,6 +115,8 @@
nvals = (2, 2, 2, (self.domain_ind >= 0).sum())
op = cls(nvals) # We allocate number of zones, not number of octs
op.initialize()
+ mylog.debug("Depositing %s particles into %s Octs",
+ positions.shape[0], nvals[-1])
op.process_octree(self.oct_handler, self.domain_ind, positions, fields,
self.domain_id, self._domain_offset)
vals = op.finalize()
diff -r 5dcf99cf04933f874b840409288051360cdd5899 -r cdf4fdbb839d408c05f7c80b17b583c32f5f3ae4 yt/frontends/ramses/data_structures.py
--- a/yt/frontends/ramses/data_structures.py
+++ b/yt/frontends/ramses/data_structures.py
@@ -248,6 +248,8 @@
self.oct_handler.finalize()
def included(self, selector):
+ if getattr(selector, "domain_id", -1) == self.domain_id:
+ return [self.domain_id]
domain_ids = self.oct_handler.domain_identify(selector)
return self.domain_id in domain_ids
@@ -326,11 +328,11 @@
def _identify_base_chunk(self, dobj):
if getattr(dobj, "_chunk_info", None) is None:
- base_region = getattr(dobj, "base_region", dobj)
- # Note that domain_ids will be ONE INDEXED
domains = [dom for dom in self.domains if
dom.included(dobj.selector)]
- mylog.debug("Identified %s intersecting domains", len(domains))
+ base_region = getattr(dobj, "base_region", dobj)
+ if len(domains) > 1:
+ mylog.debug("Identified %s intersecting domains", len(domains))
subsets = [RAMSESDomainSubset(base_region, domain, self.parameter_file)
for domain in domains]
dobj._chunk_info = subsets
diff -r 5dcf99cf04933f874b840409288051360cdd5899 -r cdf4fdbb839d408c05f7c80b17b583c32f5f3ae4 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -396,7 +396,7 @@
# This is actually not correct. The hard part is that we need to
# iterate the same way visit_all_octs does, but we need to track the
# number of octs total visited.
- cdef np.int64_t num_octs = -1
+ cdef np.int64_t num_cells = -1
if dest is None:
# Note that RAMSES can have partial refinement inside an Oct. This
# means we actually do want the number of Octs, not the number of
diff -r 5dcf99cf04933f874b840409288051360cdd5899 -r cdf4fdbb839d408c05f7c80b17b583c32f5f3ae4 yt/geometry/particle_deposit.pyx
--- a/yt/geometry/particle_deposit.pyx
+++ b/yt/geometry/particle_deposit.pyx
@@ -67,7 +67,7 @@
cdef int dims[3]
dims[0] = dims[1] = dims[2] = 2
cdef OctInfo oi
- cdef np.int64_t offset, moff, missed = 0
+ cdef np.int64_t offset, moff
cdef Oct *oct
cdef np.int64_t numpart = positions.shape[0]
moff = octree.get_domain_offset(domain_id + domain_offset)
@@ -84,21 +84,23 @@
# full octree structure. All we *really* care about is some
# arbitrary offset into a field value for deposition.
oct = octree.get(pos, &oi)
- if oct == NULL:
- missed += 1
- continue
# This next line is unfortunate. Basically it says, sometimes we
# might have particles that belong to octs outside our domain.
- if oct.domain != domain_id: continue
+ # For the distributed-memory octrees, this will manifest as a NULL
+ # oct. For the non-distributed memory octrees, we'll simply see
+ # this as a domain_id that is not the current domain id. Note that
+ # this relies on the idea that all the particles in a region are
+ # all fed to sequential domain subsets, which will not be true with
+ # RAMSES, where we *will* miss particles that live in ghost
+ # regions on other processors. Addressing this is on the TODO
+ # list.
+ if oct == NULL or oct.domain != domain_id: continue
# Note that this has to be our local index, not our in-file index.
offset = dom_ind[oct.domain_ind - moff] * 8
if offset < 0: continue
# Check that we found the oct ...
self.process(dims, oi.left_edge, oi.dds,
offset, pos, field_vals)
- if missed > 0:
- print "MISSED %s out of %s on domain %s" % (
- missed, positions.shape[0], domain_id)
@cython.boundscheck(False)
@cython.wraparound(False)
diff -r 5dcf99cf04933f874b840409288051360cdd5899 -r cdf4fdbb839d408c05f7c80b17b583c32f5f3ae4 yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -1127,9 +1127,11 @@
cdef class OctreeSubsetSelector(SelectorObject):
cdef SelectorObject base_selector
+ cdef public np.int64_t domain_id
def __init__(self, dobj):
self.base_selector = dobj.base_selector
+ self.domain_id = dobj.domain_id
@cython.boundscheck(False)
@cython.wraparound(False)
@@ -1167,7 +1169,11 @@
Oct *o = NULL) nogil:
# Because visitors now use select_grid, we should be explicitly
# checking this.
- return self.base_selector.select_grid(left_edge, right_edge, level, o)
+ cdef int res
+ res = self.base_selector.select_grid(left_edge, right_edge, level, o)
+ if res == 1 and o != NULL and o.domain != self.domain_id:
+ return -1
+ return res
octree_subset_selector = OctreeSubsetSelector
https://bitbucket.org/yt_analysis/yt-3.0/commits/0f5a5c53607b/
Changeset: 0f5a5c53607b
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-24 21:43:11
Summary: This line, which was *incorrect*, actually doubled runtime.
It's now fixed to be correct *and* faster.
Affected #: 1 file
diff -r cdf4fdbb839d408c05f7c80b17b583c32f5f3ae4 -r 0f5a5c53607b7e1d221551c0e000bf2027e2589b yt/frontends/ramses/data_structures.py
--- a/yt/frontends/ramses/data_structures.py
+++ b/yt/frontends/ramses/data_structures.py
@@ -248,8 +248,8 @@
self.oct_handler.finalize()
def included(self, selector):
- if getattr(selector, "domain_id", -1) == self.domain_id:
- return [self.domain_id]
+ if getattr(selector, "domain_id", None) is not None:
+ return selector.domain_id == self.domain_id
domain_ids = self.oct_handler.domain_identify(selector)
return self.domain_id in domain_ids
https://bitbucket.org/yt_analysis/yt-3.0/commits/4d5fcc4699c5/
Changeset: 4d5fcc4699c5
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-24 21:47:12
Summary: Starting to remove obsolete routines.
Affected #: 1 file
diff -r 0f5a5c53607b7e1d221551c0e000bf2027e2589b -r 4d5fcc4699c538125574d861c913557d0fb8102c yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -141,27 +141,6 @@
gridi[n] = self.select_grid(LE, RE, levels[n,0])
return gridi.astype("bool")
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
- def select_octs(self, OctreeContainer octree):
- # There has to be a better way to do this.
- raise RuntimeError
- cdef OctVisitorData data
- data.index = 0
- data.last = -1
- data.global_index = -1
- octree.visit_all_octs(self, oct_visitors.count_total_octs, &data)
- cdef np.ndarray[np.uint8_t, ndim=4] m2 = \
- np.zeros((2, 2, 2, data.index), 'uint8', order='C')
- # This is where we'll -- in the future -- cut up based on indices of
- # the octs.
- data.index = -1
- data.last = -1
- data.array = m2.data
- octree.visit_all_octs(self, oct_visitors.mask_octs, &data)
- return m2.astype("bool")
-
def count_octs(self, OctreeContainer octree, int domain_id = -1):
cdef OctVisitorData data
data.index = 0
@@ -1136,12 +1115,6 @@
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def select_octs(self, OctreeContainer octree):
- raise RuntimeError
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
cdef void set_bounds(self,
np.float64_t left_edge[3], np.float64_t right_edge[3],
np.float64_t dds[3], int ind[3][2], int *check):
@@ -1191,13 +1164,6 @@
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def select_octs(self, OctreeContainer octree):
- # There has to be a better way to do this.
- raise RuntimeError
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
cdef void set_bounds(self,
np.float64_t left_edge[3], np.float64_t right_edge[3],
np.float64_t dds[3], int ind[3][2], int *check):
@@ -1237,12 +1203,6 @@
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def select_octs(self, OctreeContainer octree):
- raise RuntimeError
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
cdef void set_bounds(self,
np.float64_t left_edge[3], np.float64_t right_edge[3],
np.float64_t dds[3], int ind[3][2], int *check):
https://bitbucket.org/yt_analysis/yt-3.0/commits/40184b22d3e4/
Changeset: 40184b22d3e4
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-24 22:06:26
Summary: Change static allocation of 8 pointers to dynamic allocation on Octs.
Affected #: 6 files
diff -r 4d5fcc4699c538125574d861c913557d0fb8102c -r 40184b22d3e4e15cee3a0b1efa82c495d426f900 yt/geometry/fake_octree.pyx
--- a/yt/geometry/fake_octree.pyx
+++ b/yt/geometry/fake_octree.pyx
@@ -27,6 +27,7 @@
from libc.stdlib cimport malloc, free, rand, RAND_MAX
cimport numpy as np
+from oct_visitors cimport cind
import numpy as np
cimport cython
@@ -68,7 +69,7 @@
long max_noct, long max_level, float fsubdivide,
np.ndarray[np.uint8_t, ndim=2] mask):
print "child", parent.file_ind, ind[0], ind[1], ind[2], cur_leaf, cur_level
- cdef int ddr[3]
+ cdef int ddr[3], ii
cdef long i,j,k
cdef float rf #random float from 0-1
if cur_level >= max_level:
@@ -80,7 +81,8 @@
ddr[i] = 2
rf = rand() * 1.0 / RAND_MAX
if rf > fsubdivide:
- if parent.children[ind[0]][ind[1]][ind[2]] == NULL:
+ ii = cind(ind[0], ind[1], ind[2])
+ if parent.children[ii] == NULL:
cur_leaf += 7
oct = oct_handler.next_child(1, ind, parent)
oct.domain = 1
diff -r 4d5fcc4699c538125574d861c913557d0fb8102c -r 40184b22d3e4e15cee3a0b1efa82c495d426f900 yt/geometry/oct_container.pxd
--- a/yt/geometry/oct_container.pxd
+++ b/yt/geometry/oct_container.pxd
@@ -43,7 +43,7 @@
np.int64_t domain # (opt) addl int index
np.int64_t pos[3] # position in ints
np.int8_t level
- Oct *children[2][2][2]
+ Oct **children # Up to 8 long
Oct *parent
cdef struct OctKey:
diff -r 4d5fcc4699c538125574d861c913557d0fb8102c -r 40184b22d3e4e15cee3a0b1efa82c495d426f900 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -35,6 +35,7 @@
OctVisitorData, oct_visitor_function
import selection_routines
cimport oct_visitors
+from oct_visitors cimport cind
cimport cython
ORDER_MAX = 20
@@ -66,10 +67,7 @@
oct.file_ind = oct.domain = -1
oct.domain_ind = n + n_cont.offset
oct.level = -1
- for i in range(2):
- for j in range(2):
- for k in range(2):
- oct.children[i][j][k] = NULL
+ oct.children = NULL
if prev != NULL:
prev.next = n_cont
n_cont.next = NULL
@@ -80,6 +78,9 @@
cdef OctAllocationContainer *cur
while first != NULL:
cur = first
+ for i in range(cur.n):
+ if cur.my_octs[i].children != NULL:
+ free(cur.my_octs[i].children)
free(first.my_octs)
first = cur.next
free(cur)
@@ -207,7 +208,10 @@
else:
ind[i] = 1
cp[i] += dds[i]/2.0
- next = cur.children[ind[0]][ind[1]][ind[2]]
+ if cur.children != NULL:
+ next = cur.children[cind(ind[0],ind[1],ind[2])]
+ else:
+ next = NULL
if oinfo == NULL: return cur
for i in range(3):
# This will happen *after* we quit out, so we need to back out the
@@ -285,8 +289,10 @@
dl = o.level - (candidate.level + 1)
for i in range(3):
ind[i] = (npos[i] >> dl) & 1
- if candidate.children[0][0][0] == NULL: break
- candidate = candidate.children[ind[0]][ind[1]][ind[2]]
+ if candidate.children[cind(ind[0],ind[1],ind[2])] \
+ == NULL:
+ break
+ candidate = candidate.children[cind(ind[0],ind[1],ind[2])]
neighbors[nn] = candidate
nn += 1
@@ -557,13 +563,20 @@
return next
cdef Oct* next_child(self, int domain_id, int ind[3], Oct *parent):
- cdef Oct *next = parent.children[ind[0]][ind[1]][ind[2]]
+ cdef int i
+ cdef Oct *next = NULL
+ if parent.children != NULL:
+ next = parent.children[cind(ind[0],ind[1],ind[2])]
+ else:
+ parent.children = <Oct **> malloc(sizeof(Oct *) * 8)
+ for i in range(8):
+ parent.children[i] = NULL
if next != NULL: return next
cdef OctAllocationContainer *cont = self.domains[domain_id - 1]
if cont.n_assigned >= cont.n: raise RuntimeError
next = &cont.my_octs[cont.n_assigned]
cont.n_assigned += 1
- parent.children[ind[0]][ind[1]][ind[2]] = next
+ parent.children[cind(ind[0],ind[1],ind[2])] = next
next.parent = parent
next.level = parent.level + 1
for i in range(3):
diff -r 4d5fcc4699c538125574d861c913557d0fb8102c -r 40184b22d3e4e15cee3a0b1efa82c495d426f900 yt/geometry/oct_visitors.pxd
--- a/yt/geometry/oct_visitors.pxd
+++ b/yt/geometry/oct_visitors.pxd
@@ -43,6 +43,9 @@
cdef oct_visitor_function identify_octs
cdef oct_visitor_function assign_domain_ind
+cdef inline int cind(int i, int j, int k):
+ return (((i*2)+j)*2+k)
+
cdef inline int oind(OctVisitorData *data):
return (((data.ind[0]*2)+data.ind[1])*2+data.ind[2])
diff -r 4d5fcc4699c538125574d861c913557d0fb8102c -r 40184b22d3e4e15cee3a0b1efa82c495d426f900 yt/geometry/particle_oct_container.pyx
--- a/yt/geometry/particle_oct_container.pyx
+++ b/yt/geometry/particle_oct_container.pyx
@@ -27,6 +27,7 @@
from oct_container cimport OctreeContainer, Oct, OctInfo, ORDER_MAX
cimport oct_visitors
+from oct_visitors cimport cind
from libc.stdlib cimport malloc, free, qsort
from libc.math cimport floor
from fp_utils cimport *
@@ -75,8 +76,10 @@
for i in range(2):
for j in range(2):
for k in range(2):
- if o.children[i][j][k] == NULL: continue
- self.visit_free(o.children[i][j][k])
+ if o.children != NULL \
+ and o.children[cind(i,j,k)] != NULL:
+ self.visit_free(o.children[cind(i,j,k)])
+ free(o.children)
free(o)
def clear_fileind(self):
@@ -93,8 +96,9 @@
for i in range(2):
for j in range(2):
for k in range(2):
- if o.children[i][j][k] == NULL: continue
- self.visit_clear(o.children[i][j][k])
+ if o.children != NULL \
+ and o.children[cind(i,j,k)] != NULL:
+ self.visit_clear(o.children[cind(i,j,k)])
def __iter__(self):
#Get the next oct, will traverse domains
@@ -139,8 +143,9 @@
for i in range(2):
for j in range(2):
for k in range(2):
- if o.children[i][j][k] != NULL:
- self.visit_assign(o.children[i][j][k], lpos)
+ if o.children != NULL \
+ and o.children[cind(i,j,k)] != NULL:
+ self.visit_assign(o.children[cind(i,j,k)], lpos)
return
cdef np.int64_t get_domain_offset(self, int domain_id):
@@ -158,10 +163,7 @@
my_oct.domain_ind = self.nocts - 1
my_oct.pos[0] = my_oct.pos[1] = my_oct.pos[2] = -1
my_oct.level = -1
- for i in range(2):
- for j in range(2):
- for k in range(2):
- my_oct.children[i][j][k] = NULL
+ my_oct.children = NULL
my_oct.parent = NULL
return my_oct
@@ -191,11 +193,12 @@
level += 1
for i in range(3):
ind[i] = (index >> ((ORDER_MAX - level)*3 + (2 - i))) & 1
- if cur.children[ind[0]][ind[1]][ind[2]] == NULL:
+ if cur.children == NULL or \
+ cur.children[cind(ind[0],ind[1],ind[2])] == NULL:
cur = self.refine_oct(cur, index)
self.filter_particles(cur, data, p)
else:
- cur = cur.children[ind[0]][ind[1]][ind[2]]
+ cur = cur.children[cind(ind[0],ind[1],ind[2])]
cur.file_ind += 1
@cython.boundscheck(False)
@@ -208,6 +211,7 @@
cdef int i, j, k, m, n, ind[3]
cdef Oct *noct
cdef np.uint64_t prefix1, prefix2
+ o.children = <Oct **> malloc(sizeof(Oct *)*8)
for i in range(2):
for j in range(2):
for k in range(2):
@@ -219,11 +223,11 @@
noct.pos[1] = (o.pos[1] << 1) + j
noct.pos[2] = (o.pos[2] << 1) + k
noct.parent = o
- o.children[i][j][k] = noct
+ o.children[cind(i,j,k)] = noct
o.file_ind = self.n_ref + 1
for i in range(3):
ind[i] = (index >> ((ORDER_MAX - (o.level + 1))*3 + (2 - i))) & 1
- noct = o.children[ind[0]][ind[1]][ind[2]]
+ noct = o.children[cind(ind[0],ind[1],ind[2])]
return noct
cdef void filter_particles(self, Oct *o, np.uint64_t *data, np.int64_t p):
@@ -262,8 +266,9 @@
for i in range(2):
for j in range(2):
for k in range(2):
- if o.children[i][j][k] != NULL:
- self.visit(o.children[i][j][k], counts, level + 1)
+ if o.children != NULL \
+ and o.children[cind(i,j,k)] != NULL:
+ self.visit(o.children[cind(i,j,k)], counts, level + 1)
return
cdef class ParticleRegions:
diff -r 4d5fcc4699c538125574d861c913557d0fb8102c -r 40184b22d3e4e15cee3a0b1efa82c495d426f900 yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -31,6 +31,7 @@
from selection_routines cimport SelectorObject
from oct_container cimport OctreeContainer, OctAllocationContainer, Oct
cimport oct_visitors
+from oct_visitors cimport cind
#from geometry_utils cimport point_to_hilbert
from yt.utilities.lib.grid_traversal cimport \
VolumeContainer, sample_function, walk_volume
@@ -214,7 +215,9 @@
spos[2] = pos[2] - sdds[2]/2.0
for k in range(2):
ii = ((k*2)+j)*2+i
- ch = root.children[i][j][k]
+ ch = NULL
+ if root.children != NULL:
+ ch = root.children[cind(i,j,k)]
if iter == 1 and next_level == 1 and ch != NULL:
self.recursively_visit_octs(
ch, spos, sdds, level + 1, func, data,
https://bitbucket.org/yt_analysis/yt-3.0/commits/ae57aa843411/
Changeset: ae57aa843411
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-24 22:32:05
Summary: Removing 'parent' attribute, which will require cleverness to get the neighbors.
Affected #: 3 files
diff -r 40184b22d3e4e15cee3a0b1efa82c495d426f900 -r ae57aa843411ef00c7f66f5c2d3508d388cfced8 yt/geometry/oct_container.pxd
--- a/yt/geometry/oct_container.pxd
+++ b/yt/geometry/oct_container.pxd
@@ -37,14 +37,11 @@
np.int64_t file_ind # index with respect to the order in which it was
# added
np.int64_t domain_ind # index within the global set of domains
- # note that moving to a local index will require
- # moving to split-up masks, which is part of a
- # bigger refactor
np.int64_t domain # (opt) addl int index
np.int64_t pos[3] # position in ints
np.int8_t level
Oct **children # Up to 8 long
- Oct *parent
+ #Oct *parent
cdef struct OctKey:
np.int64_t key
diff -r 40184b22d3e4e15cee3a0b1efa82c495d426f900 -r ae57aa843411ef00c7f66f5c2d3508d388cfced8 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -63,7 +63,7 @@
n_cont.n_assigned = 0
for n in range(n_octs):
oct = &n_cont.my_octs[n]
- oct.parent = NULL
+ #oct.parent = NULL
oct.file_ind = oct.domain = -1
oct.domain_ind = n + n_cont.offset
oct.level = -1
@@ -278,7 +278,10 @@
curnpos[i] = (curnpos[i] >> 1)
# Now we update to the candidate's parent, which should
# have a matching position to curopos[]
- candidate = candidate.parent
+ # TODO: This has not survived the transition to
+ # mostly-stateless Octs!
+ raise RuntimeError
+ #candidate = candidate.parent
if candidate == NULL:
# Worst case scenario
for i in range(3):
@@ -548,7 +551,7 @@
return NULL
next = &cont.my_octs[cont.n_assigned]
cont.n_assigned += 1
- next.parent = NULL
+ #next.parent = NULL
next.level = 0
cdef np.int64_t key = 0
cdef OctKey *ikey = &self.root_nodes[self.num_root]
@@ -577,7 +580,7 @@
next = &cont.my_octs[cont.n_assigned]
cont.n_assigned += 1
parent.children[cind(ind[0],ind[1],ind[2])] = next
- next.parent = parent
+ #next.parent = parent
next.level = parent.level + 1
for i in range(3):
next.pos[i] = ind[i] + (parent.pos[i] << 1)
diff -r 40184b22d3e4e15cee3a0b1efa82c495d426f900 -r ae57aa843411ef00c7f66f5c2d3508d388cfced8 yt/geometry/particle_oct_container.pyx
--- a/yt/geometry/particle_oct_container.pyx
+++ b/yt/geometry/particle_oct_container.pyx
@@ -56,7 +56,7 @@
cur.pos[0] = i
cur.pos[1] = j
cur.pos[2] = k
- cur.parent = NULL
+ #cur.parent = NULL
self.root_mesh[i][j][k] = cur
def __dealloc__(self):
@@ -164,7 +164,7 @@
my_oct.pos[0] = my_oct.pos[1] = my_oct.pos[2] = -1
my_oct.level = -1
my_oct.children = NULL
- my_oct.parent = NULL
+ #my_oct.parent = NULL
return my_oct
@cython.boundscheck(False)
@@ -222,7 +222,7 @@
noct.pos[0] = (o.pos[0] << 1) + i
noct.pos[1] = (o.pos[1] << 1) + j
noct.pos[2] = (o.pos[2] << 1) + k
- noct.parent = o
+ #noct.parent = o
o.children[cind(i,j,k)] = noct
o.file_ind = self.n_ref + 1
for i in range(3):
https://bitbucket.org/yt_analysis/yt-3.0/commits/00d2f58f5f57/
Changeset: 00d2f58f5f57
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-24 22:57:02
Summary: Removing pos, level and parent pointers.
Affected #: 6 files
diff -r ae57aa843411ef00c7f66f5c2d3508d388cfced8 -r 00d2f58f5f57f7e9d9b73bc37376e5f64d697be5 yt/geometry/oct_container.pxd
--- a/yt/geometry/oct_container.pxd
+++ b/yt/geometry/oct_container.pxd
@@ -38,10 +38,7 @@
# added
np.int64_t domain_ind # index within the global set of domains
np.int64_t domain # (opt) addl int index
- np.int64_t pos[3] # position in ints
- np.int8_t level
Oct **children # Up to 8 long
- #Oct *parent
cdef struct OctKey:
np.int64_t key
diff -r ae57aa843411ef00c7f66f5c2d3508d388cfced8 -r 00d2f58f5f57f7e9d9b73bc37376e5f64d697be5 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -63,10 +63,8 @@
n_cont.n_assigned = 0
for n in range(n_octs):
oct = &n_cont.my_octs[n]
- #oct.parent = NULL
oct.file_ind = oct.domain = -1
oct.domain_ind = n + n_cont.offset
- oct.level = -1
oct.children = NULL
if prev != NULL:
prev.next = n_cont
@@ -149,6 +147,7 @@
cdef int i, j, k, n, vc
vc = self.partial_coverage
data.global_index = -1
+ data.level = 0
cdef np.float64_t pos[3], dds[3]
# This dds is the oct-width
for i in range(3):
@@ -161,6 +160,9 @@
pos[2] = self.DLE[2] + dds[2]/2.0
for k in range(self.nn[2]):
if self.root_mesh[i][j][k] == NULL: continue
+ data.pos[0] = i
+ data.pos[1] = j
+ data.pos[2] = k
selector.recursively_visit_octs(
self.root_mesh[i][j][k],
pos, dds, 0, func, data, vc)
@@ -170,9 +172,9 @@
cdef void oct_bounds(self, Oct *o, np.float64_t *corner, np.float64_t *size):
cdef int i
- for i in range(3):
- size[i] = (self.DRE[i] - self.DLE[i]) / (self.nn[i] << o.level)
- corner[i] = o.pos[i] * size[i] + self.DLE[i]
+ #for i in range(3):
+ # size[i] = (self.DRE[i] - self.DLE[i]) / (self.nn[i] << o.level)
+ # corner[i] = o.pos[i] * size[i] + self.DLE[i]
cdef np.int64_t get_domain_offset(self, int domain_id):
return 0
@@ -241,63 +243,64 @@
cdef Oct* candidate
for i in range(27): neighbors[i] = NULL
nn = 0
- for ni in range(3):
- for nj in range(3):
- for nk in range(3):
- if ni == nj == nk == 1:
- neighbors[nn] = o
- nn += 1
- continue
- npos[0] = o.pos[0] + (ni - 1)
- npos[1] = o.pos[1] + (nj - 1)
- npos[2] = o.pos[2] + (nk - 1)
- for i in range(3):
- # Periodicity
- if npos[i] == -1:
- npos[i] = (self.nn[i] << o.level) - 1
- elif npos[i] == (self.nn[i] << o.level):
- npos[i] = 0
- curopos[i] = o.pos[i]
- curnpos[i] = npos[i]
- # Now we have our neighbor position and a safe place to
- # keep it. curnpos will be the root index of the neighbor
- # at a given level, and npos will be constant. curopos is
- # the candidate root at a level.
- candidate = o
- while candidate != NULL:
- if ((curopos[0] == curnpos[0]) and
- (curopos[1] == curnpos[1]) and
- (curopos[2] == curnpos[2])):
- break
- # This one doesn't meet it, so we pop up a level.
- # First we update our positions, then we update our
- # candidate.
- for i in range(3):
- # We strip a digit off the right
- curopos[i] = (curopos[i] >> 1)
- curnpos[i] = (curnpos[i] >> 1)
- # Now we update to the candidate's parent, which should
- # have a matching position to curopos[]
- # TODO: This has not survived the transition to
- # mostly-stateless Octs!
- raise RuntimeError
- #candidate = candidate.parent
- if candidate == NULL:
- # Worst case scenario
- for i in range(3):
- ind[i] = (npos[i] >> (o.level))
- candidate = self.root_mesh[ind[0]][ind[1]][ind[2]]
- # Now we have the common root, which may be NULL
- while candidate.level < o.level:
- dl = o.level - (candidate.level + 1)
- for i in range(3):
- ind[i] = (npos[i] >> dl) & 1
- if candidate.children[cind(ind[0],ind[1],ind[2])] \
- == NULL:
- break
- candidate = candidate.children[cind(ind[0],ind[1],ind[2])]
- neighbors[nn] = candidate
- nn += 1
+ raise RuntimeError
+ #for ni in range(3):
+ # for nj in range(3):
+ # for nk in range(3):
+ # if ni == nj == nk == 1:
+ # neighbors[nn] = o
+ # nn += 1
+ # continue
+ # npos[0] = o.pos[0] + (ni - 1)
+ # npos[1] = o.pos[1] + (nj - 1)
+ # npos[2] = o.pos[2] + (nk - 1)
+ # for i in range(3):
+ # # Periodicity
+ # if npos[i] == -1:
+ # npos[i] = (self.nn[i] << o.level) - 1
+ # elif npos[i] == (self.nn[i] << o.level):
+ # npos[i] = 0
+ # curopos[i] = o.pos[i]
+ # curnpos[i] = npos[i]
+ # # Now we have our neighbor position and a safe place to
+ # # keep it. curnpos will be the root index of the neighbor
+ # # at a given level, and npos will be constant. curopos is
+ # # the candidate root at a level.
+ # candidate = o
+ # while candidate != NULL:
+ # if ((curopos[0] == curnpos[0]) and
+ # (curopos[1] == curnpos[1]) and
+ # (curopos[2] == curnpos[2])):
+ # break
+ # # This one doesn't meet it, so we pop up a level.
+ # # First we update our positions, then we update our
+ # # candidate.
+ # for i in range(3):
+ # # We strip a digit off the right
+ # curopos[i] = (curopos[i] >> 1)
+ # curnpos[i] = (curnpos[i] >> 1)
+ # # Now we update to the candidate's parent, which should
+ # # have a matching position to curopos[]
+ # # TODO: This has not survived the transition to
+ # # mostly-stateless Octs!
+ # raise RuntimeError
+ # candidate = candidate.parent
+ # if candidate == NULL:
+ # # Worst case scenario
+ # for i in range(3):
+ # ind[i] = (npos[i] >> (o.level))
+ # candidate = self.root_mesh[ind[0]][ind[1]][ind[2]]
+ # # Now we have the common root, which may be NULL
+ # while candidate.level < o.level:
+ # dl = o.level - (candidate.level + 1)
+ # for i in range(3):
+ # ind[i] = (npos[i] >> dl) & 1
+ # if candidate.children[cind(ind[0],ind[1],ind[2])] \
+ # == NULL:
+ # break
+ # candidate = candidate.children[cind(ind[0],ind[1],ind[2])]
+ # neighbors[nn] = candidate
+ # nn += 1
@cython.boundscheck(False)
@cython.wraparound(False)
@@ -314,6 +317,7 @@
cdef np.float64_t corner[3], size[3]
bounds = np.zeros((27,6), dtype="float64")
tnp = 0
+ raise RuntimeError
for i in range(27):
self.oct_bounds(neighbors[i], corner, size)
for ii in range(3):
@@ -504,7 +508,9 @@
oct_visitor_function *func,
OctVisitorData *data):
cdef int i, j, k, n, vc
+ cdef np.int64_t key, ukey
data.global_index = -1
+ data.level = 0
vc = self.partial_coverage
cdef np.float64_t pos[3], dds[3]
# This dds is the oct-width
@@ -512,10 +518,15 @@
dds[i] = (self.DRE[i] - self.DLE[i]) / self.nn[i]
# Pos is the center of the octs
cdef Oct *o
+ ukey = 0
+ for i in range(20):
+ ukey |= (1 << i)
for i in range(self.num_root):
o = self.root_nodes[i].node
+ key = self.root_nodes[i].key
for j in range(3):
- pos[j] = self.DLE[j] + (o.pos[j] + 0.5) * dds[j]
+ pos[j] = self.DLE[j] + \
+ ((key >> 20 * (2 - j) & ukey) + 0.5) * dds[j]
selector.recursively_visit_octs(
o, pos, dds, 0, func, data, vc)
@@ -551,12 +562,9 @@
return NULL
next = &cont.my_octs[cont.n_assigned]
cont.n_assigned += 1
- #next.parent = NULL
- next.level = 0
cdef np.int64_t key = 0
cdef OctKey *ikey = &self.root_nodes[self.num_root]
for i in range(3):
- next.pos[i] = ind[i]
key |= ((<np.int64_t>ind[i]) << 20 * (2 - i))
self.root_nodes[self.num_root].key = key
self.root_nodes[self.num_root].node = next
@@ -580,10 +588,6 @@
next = &cont.my_octs[cont.n_assigned]
cont.n_assigned += 1
parent.children[cind(ind[0],ind[1],ind[2])] = next
- #next.parent = parent
- next.level = parent.level + 1
- for i in range(3):
- next.pos[i] = ind[i] + (parent.pos[i] << 1)
self.nocts += 1
return next
@@ -658,7 +662,6 @@
# Now we should be at the right level
cur.domain = curdom
cur.file_ind = p
- cur.level = curlevel
return cont.n_assigned - initial
@cython.boundscheck(False)
@@ -685,9 +688,10 @@
for k in range(2):
ii = ((k*2)+j)*2+i
if mask[o.domain_ind, ii] == 0: continue
- if o.level == level:
- dest[local_filled] = \
- source[o.file_ind, ii]
+ # TODO: Uncomment this!
+ #if o.level == level:
+ # dest[local_filled] = \
+ # source[o.file_ind, ii]
local_filled += 1
return local_filled
@@ -729,7 +733,8 @@
for n in range(dom.n):
o = &dom.my_octs[n]
index = o.file_ind-subchunk_offset
- if o.level != level: continue
+ # TODO: Uncomment this!
+ #if o.level != level: continue
if index < 0: continue
if index >= subchunk_max:
#if we hit the end of the array,
@@ -759,6 +764,9 @@
#As a result, source is 3D grid with 8 times as many
#elements as the number of octs on this level in this domain
#and with the shape of an equal-sided cube
+ #
+ # TODO: Convert to a recrusive function.
+ # Note that the .pos[0] etc calls need to be uncommented.
cdef np.ndarray[np.float64_t, ndim=3] source
cdef np.ndarray[np.float64_t, ndim=1] dest
cdef OctAllocationContainer *dom = self.domains[domain - 1]
@@ -774,15 +782,16 @@
source = source_fields[key]
for n in range(dom.n):
o = &dom.my_octs[n]
- if o.level != level: continue
+ # TODO: Uncomment this!
+ #if o.level != level: continue
for i in range(2):
for j in range(2):
for k in range(2):
ii = ((k*2)+j)*2+i
if mask[o.domain_ind, ii] == 0: continue
- ox = (o.pos[0] << 1) + i
- oy = (o.pos[1] << 1) + j
- oz = (o.pos[2] << 1) + k
+ #ox = (o.pos[0] << 1) + i
+ #oy = (o.pos[1] << 1) + j
+ #oz = (o.pos[2] << 1) + k
dest[local_filled + offset] = source[ox,oy,oz]
local_filled += 1
return local_filled
diff -r ae57aa843411ef00c7f66f5c2d3508d388cfced8 -r 00d2f58f5f57f7e9d9b73bc37376e5f64d697be5 yt/geometry/oct_visitors.pyx
--- a/yt/geometry/oct_visitors.pyx
+++ b/yt/geometry/oct_visitors.pyx
@@ -102,13 +102,13 @@
cdef np.int64_t *coords = <np.int64_t*> data.array
cdef int i
for i in range(3):
- coords[data.index * 3 + i] = (o.pos[i] << 1) + data.ind[i]
+ coords[data.index * 3 + i] = (data.pos[i] << 1) + data.ind[i]
data.index += 1
cdef void ires_octs(Oct *o, OctVisitorData *data, np.uint8_t selected):
if selected == 0: return
cdef np.int64_t *ires = <np.int64_t*> data.array
- ires[data.index] = o.level
+ ires[data.index] = data.level
data.index += 1
@cython.cdivision(True)
@@ -120,9 +120,9 @@
cdef np.float64_t *fcoords = <np.float64_t*> data.array
cdef int i
cdef np.float64_t c, dx
- dx = 1.0 / (2 << o.level)
+ dx = 1.0 / (2 << data.level)
for i in range(3):
- c = <np.float64_t> ((o.pos[i] << 1 ) + data.ind[i])
+ c = <np.float64_t> ((data.pos[i] << 1 ) + data.ind[i])
fcoords[data.index * 3 + i] = (c + 0.5) * dx
data.index += 1
@@ -135,7 +135,7 @@
cdef np.float64_t *fwidth = <np.float64_t*> data.array
cdef int i
cdef np.float64_t dx
- dx = 1.0 / (2 << o.level)
+ dx = 1.0 / (2 << data.level)
for i in range(3):
fwidth[data.index * 3 + i] = dx
data.index += 1
diff -r ae57aa843411ef00c7f66f5c2d3508d388cfced8 -r 00d2f58f5f57f7e9d9b73bc37376e5f64d697be5 yt/geometry/particle_oct_container.pyx
--- a/yt/geometry/particle_oct_container.pyx
+++ b/yt/geometry/particle_oct_container.pyx
@@ -52,11 +52,6 @@
for j in range(self.nn[1]):
for k in range(self.nn[2]):
cur = self.allocate_oct()
- cur.level = 0
- cur.pos[0] = i
- cur.pos[1] = j
- cur.pos[2] = k
- #cur.parent = NULL
self.root_mesh[i][j][k] = cur
def __dealloc__(self):
@@ -127,25 +122,27 @@
for i in range(self.nn[0]):
for j in range(self.nn[1]):
for k in range(self.nn[2]):
- self.visit_assign(self.root_mesh[i][j][k], &lpos)
+ self.visit_assign(self.root_mesh[i][j][k], &lpos,
+ 0, &max_level)
assert(lpos == self.nocts)
for i in range(self.nocts):
self.oct_list[i].domain_ind = i
self.oct_list[i].domain = 0
self.oct_list[i].file_ind = -1
- max_level = imax(max_level, self.oct_list[i].level)
self.max_level = max_level
- cdef visit_assign(self, Oct *o, np.int64_t *lpos):
+ cdef visit_assign(self, Oct *o, np.int64_t *lpos, int level, int *max_level):
cdef int i, j, k
self.oct_list[lpos[0]] = o
lpos[0] += 1
+ max_level[0] = imax(max_level[0], level)
for i in range(2):
for j in range(2):
for k in range(2):
if o.children != NULL \
and o.children[cind(i,j,k)] != NULL:
- self.visit_assign(o.children[cind(i,j,k)], lpos)
+ self.visit_assign(o.children[cind(i,j,k)], lpos,
+ level + 1, max_level)
return
cdef np.int64_t get_domain_offset(self, int domain_id):
@@ -161,10 +158,7 @@
my_oct.domain = -1
my_oct.file_ind = 0
my_oct.domain_ind = self.nocts - 1
- my_oct.pos[0] = my_oct.pos[1] = my_oct.pos[2] = -1
- my_oct.level = -1
my_oct.children = NULL
- #my_oct.parent = NULL
return my_oct
@cython.boundscheck(False)
@@ -195,8 +189,8 @@
ind[i] = (index >> ((ORDER_MAX - level)*3 + (2 - i))) & 1
if cur.children == NULL or \
cur.children[cind(ind[0],ind[1],ind[2])] == NULL:
- cur = self.refine_oct(cur, index)
- self.filter_particles(cur, data, p)
+ cur = self.refine_oct(cur, index, level)
+ self.filter_particles(cur, data, p, level + 1)
else:
cur = cur.children[cind(ind[0],ind[1],ind[2])]
cur.file_ind += 1
@@ -204,7 +198,7 @@
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- cdef Oct *refine_oct(self, Oct *o, np.uint64_t index):
+ cdef Oct *refine_oct(self, Oct *o, np.uint64_t index, int level):
#Allocate and initialize child octs
#Attach particles to child octs
#Remove particles from this oct entirely
@@ -218,31 +212,27 @@
noct = self.allocate_oct()
noct.domain = o.domain
noct.file_ind = 0
- noct.level = o.level + 1
- noct.pos[0] = (o.pos[0] << 1) + i
- noct.pos[1] = (o.pos[1] << 1) + j
- noct.pos[2] = (o.pos[2] << 1) + k
- #noct.parent = o
o.children[cind(i,j,k)] = noct
o.file_ind = self.n_ref + 1
for i in range(3):
- ind[i] = (index >> ((ORDER_MAX - (o.level + 1))*3 + (2 - i))) & 1
+ ind[i] = (index >> ((ORDER_MAX - (level + 1))*3 + (2 - i))) & 1
noct = o.children[cind(ind[0],ind[1],ind[2])]
return noct
- cdef void filter_particles(self, Oct *o, np.uint64_t *data, np.int64_t p):
+ cdef void filter_particles(self, Oct *o, np.uint64_t *data, np.int64_t p,
+ int level):
# Now we look at the last nref particles to decide where they go.
cdef int n = imin(p, self.n_ref)
cdef np.uint64_t *arr = data + imax(p - self.n_ref, 0)
# Now we figure out our prefix, which is the oct address at this level.
# As long as we're actually in Morton order, we do not need to worry
# about *any* of the other children of the oct.
- prefix1 = data[p] >> (ORDER_MAX - o.level)*3
+ prefix1 = data[p] >> (ORDER_MAX - level)*3
for i in range(n):
- prefix2 = arr[i] >> (ORDER_MAX - o.level)*3
+ prefix2 = arr[i] >> (ORDER_MAX - level)*3
if (prefix1 == prefix2):
o.file_ind += 1
- #print ind[0], ind[1], ind[2], o.file_ind, o.level
+ #print ind[0], ind[1], ind[2], o.file_ind, level
def recursively_count(self):
#Visit every cell, accumulate the # of cells per level
diff -r ae57aa843411ef00c7f66f5c2d3508d388cfced8 -r 00d2f58f5f57f7e9d9b73bc37376e5f64d697be5 yt/geometry/selection_routines.pxd
--- a/yt/geometry/selection_routines.pxd
+++ b/yt/geometry/selection_routines.pxd
@@ -30,10 +30,12 @@
np.uint64_t index
np.uint64_t last
np.int64_t global_index
- int ind[3]
+ np.int64_t pos[3] # position in ints
+ np.uint8_t ind[3] # cell position
void *array
int dims
- int domain
+ np.int32_t domain
+ np.int8_t level
ctypedef void oct_visitor_function(Oct *, OctVisitorData *visitor,
np.uint8_t selected)
diff -r ae57aa843411ef00c7f66f5c2d3508d388cfced8 -r 00d2f58f5f57f7e9d9b73bc37376e5f64d697be5 yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -170,7 +170,7 @@
# we visit *this* oct, then we make a second pass to check any child
# octs.
cdef np.float64_t LE[3], RE[3], sdds[3], spos[3]
- cdef int i, j, k, res, ii
+ cdef int i, j, k, res, ii, mi
cdef Oct *ch
cdef np.uint8_t selected
# Remember that pos is the *center* of the oct, and dds is the oct
@@ -219,9 +219,17 @@
if root.children != NULL:
ch = root.children[cind(i,j,k)]
if iter == 1 and next_level == 1 and ch != NULL:
+ data.pos[0] = (data.pos[0] << 1) + i
+ data.pos[1] = (data.pos[1] << 1) + j
+ data.pos[2] = (data.pos[2] << 1) + k
+ data.level += 1
self.recursively_visit_octs(
ch, spos, sdds, level + 1, func, data,
visit_covered)
+ data.pos[0] = (data.pos[0] >> 1)
+ data.pos[1] = (data.pos[1] >> 1)
+ data.pos[2] = (data.pos[2] >> 1)
+ data.level -= 1
elif this_level == 1:
selected = self.select_cell(spos, sdds, eterm)
data.global_index += increment
https://bitbucket.org/yt_analysis/yt-3.0/commits/aef64c0bb95c/
Changeset: aef64c0bb95c
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-25 19:37:11
Summary: Switching order of key bit shifting.
Affected #: 1 file
diff -r 00d2f58f5f57f7e9d9b73bc37376e5f64d697be5 -r aef64c0bb95c9725e36da890bedceb5d79972691 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -525,8 +525,10 @@
o = self.root_nodes[i].node
key = self.root_nodes[i].key
for j in range(3):
- pos[j] = self.DLE[j] + \
- ((key >> 20 * (2 - j) & ukey) + 0.5) * dds[j]
+ data.pos[2 - j] = (key & ukey)
+ key = key >> 20
+ for j in range(3):
+ pos[j] = self.DLE[j] + (data.pos[j] + 0.5) * dds[j]
selector.recursively_visit_octs(
o, pos, dds, 0, func, data, vc)
https://bitbucket.org/yt_analysis/yt-3.0/commits/c693384fdeb0/
Changeset: c693384fdeb0
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-25 19:53:03
Summary: Re-arranging Oct definitions.
Affected #: 4 files
diff -r aef64c0bb95c9725e36da890bedceb5d79972691 -r c693384fdeb083faf9429d687b928c371f970477 yt/geometry/oct_container.pxd
--- a/yt/geometry/oct_container.pxd
+++ b/yt/geometry/oct_container.pxd
@@ -25,20 +25,13 @@
cimport numpy as np
from fp_utils cimport *
-from selection_routines cimport SelectorObject, \
- OctVisitorData, oct_visitor_function
-from oct_visitors cimport *
+from selection_routines cimport SelectorObject
+from oct_visitors cimport \
+ OctVisitorData, oct_visitor_function, Oct
from libc.stdlib cimport bsearch, qsort
cdef int ORDER_MAX
-cdef struct Oct
-cdef struct Oct:
- np.int64_t file_ind # index with respect to the order in which it was
- # added
- np.int64_t domain_ind # index within the global set of domains
- np.int64_t domain # (opt) addl int index
- Oct **children # Up to 8 long
cdef struct OctKey:
np.int64_t key
diff -r aef64c0bb95c9725e36da890bedceb5d79972691 -r c693384fdeb083faf9429d687b928c371f970477 yt/geometry/oct_visitors.pxd
--- a/yt/geometry/oct_visitors.pxd
+++ b/yt/geometry/oct_visitors.pxd
@@ -24,10 +24,28 @@
"""
cimport numpy as np
-from selection_routines cimport \
- OctVisitorData, oct_visitor_function
-from oct_container cimport \
- Oct
+
+cdef struct Oct
+cdef struct Oct:
+ np.int64_t file_ind # index with respect to the order in which it was
+ # added
+ np.int64_t domain_ind # index within the global set of domains
+ np.int64_t domain # (opt) addl int index
+ Oct **children # Up to 8 long
+
+cdef struct OctVisitorData:
+ np.uint64_t index
+ np.uint64_t last
+ np.int64_t global_index
+ np.int64_t pos[3] # position in ints
+ np.uint8_t ind[3] # cell position
+ void *array
+ int dims
+ np.int32_t domain
+ np.int8_t level
+
+ctypedef void oct_visitor_function(Oct *, OctVisitorData *visitor,
+ np.uint8_t selected)
cdef oct_visitor_function count_total_octs
cdef oct_visitor_function count_total_cells
diff -r aef64c0bb95c9725e36da890bedceb5d79972691 -r c693384fdeb083faf9429d687b928c371f970477 yt/geometry/selection_routines.pxd
--- a/yt/geometry/selection_routines.pxd
+++ b/yt/geometry/selection_routines.pxd
@@ -24,21 +24,8 @@
"""
cimport numpy as np
-
-cdef struct Oct
-cdef struct OctVisitorData:
- np.uint64_t index
- np.uint64_t last
- np.int64_t global_index
- np.int64_t pos[3] # position in ints
- np.uint8_t ind[3] # cell position
- void *array
- int dims
- np.int32_t domain
- np.int8_t level
-
-ctypedef void oct_visitor_function(Oct *, OctVisitorData *visitor,
- np.uint8_t selected)
+from oct_visitors cimport Oct, OctVisitorData, \
+ oct_visitor_function
cdef class SelectorObject:
cdef public np.int32_t min_level
diff -r aef64c0bb95c9725e36da890bedceb5d79972691 -r c693384fdeb083faf9429d687b928c371f970477 yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -170,7 +170,7 @@
# we visit *this* oct, then we make a second pass to check any child
# octs.
cdef np.float64_t LE[3], RE[3], sdds[3], spos[3]
- cdef int i, j, k, res, ii, mi
+ cdef int i, j, k, res, mi
cdef Oct *ch
cdef np.uint8_t selected
# Remember that pos is the *center* of the oct, and dds is the oct
@@ -214,7 +214,6 @@
for j in range(2):
spos[2] = pos[2] - sdds[2]/2.0
for k in range(2):
- ii = ((k*2)+j)*2+i
ch = NULL
if root.children != NULL:
ch = root.children[cind(i,j,k)]
https://bitbucket.org/yt_analysis/yt-3.0/commits/d7db8bf1f2a1/
Changeset: d7db8bf1f2a1
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-25 21:32:51
Summary: This fixes an error that showed up at O0, but not O3. data.last was uninitialized.
Affected #: 2 files
diff -r c693384fdeb083faf9429d687b928c371f970477 -r d7db8bf1f2a13d7ec9550662789d7e677048c18b yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -331,7 +331,7 @@
def icoords(self, SelectorObject selector, np.uint64_t num_cells = -1,
int domain_id = -1):
if num_cells == -1:
- num_cells = selector.count_octs(self, domain_id) * 8
+ num_cells = selector.count_octs(self, domain_id)
cdef np.ndarray[np.int64_t, ndim=2] coords
coords = np.empty((num_cells, 3), dtype="int64")
cdef OctVisitorData data
@@ -347,7 +347,7 @@
def ires(self, SelectorObject selector, np.uint64_t num_cells = -1,
int domain_id = -1):
if num_cells == -1:
- num_cells = selector.count_octs(self, domain_id) * 8
+ num_cells = selector.count_octs(self, domain_id)
#Return the 'resolution' of each cell; ie the level
cdef np.ndarray[np.int64_t, ndim=1] res
res = np.empty(num_cells, dtype="int64")
@@ -364,7 +364,7 @@
def fwidth(self, SelectorObject selector, np.uint64_t num_cells = -1,
int domain_id = -1):
if num_cells == -1:
- num_cells = selector.count_octs(self, domain_id) * 8
+ num_cells = selector.count_octs(self, domain_id)
cdef np.ndarray[np.float64_t, ndim=2] fwidth
fwidth = np.empty((num_cells, 3), dtype="float64")
cdef OctVisitorData data
@@ -384,7 +384,7 @@
def fcoords(self, SelectorObject selector, np.uint64_t num_cells = -1,
int domain_id = -1):
if num_cells == -1:
- num_cells = selector.count_octs(self, domain_id) * 8
+ num_cells = selector.count_octs(self, domain_id)
#Return the floating point unitary position of every cell
cdef np.ndarray[np.float64_t, ndim=2] coords
coords = np.empty((num_cells, 3), dtype="float64")
diff -r c693384fdeb083faf9429d687b928c371f970477 -r d7db8bf1f2a13d7ec9550662789d7e677048c18b yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -145,9 +145,10 @@
def count_octs(self, OctreeContainer octree, int domain_id = -1):
cdef OctVisitorData data
data.index = 0
+ data.last = -1
data.domain = domain_id
octree.visit_all_octs(self, oct_visitors.count_total_octs, &data)
- return data.index
+ return data.index * 8
def count_oct_cells(self, OctreeContainer octree, int domain_id = -1):
cdef OctVisitorData data
https://bitbucket.org/yt_analysis/yt-3.0/commits/96ff7315a44a/
Changeset: 96ff7315a44a
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-25 21:41:01
Summary: Add oct-count caching.
Affected #: 2 files
diff -r d7db8bf1f2a13d7ec9550662789d7e677048c18b -r 96ff7315a44a131c07cabb0580a212ff198fe556 yt/data_objects/octree_subset.py
--- a/yt/data_objects/octree_subset.py
+++ b/yt/data_objects/octree_subset.py
@@ -47,6 +47,7 @@
_con_args = ('base_region', 'domain', 'pf')
_container_fields = ("dx", "dy", "dz")
_domain_offset = 0
+ _num_octs = -1
def __init__(self, base_region, domain, pf):
self.field_data = YTFieldData()
@@ -123,25 +124,33 @@
return np.asfortranarray(vals)
def select_icoords(self, dobj):
- d = self.oct_handler.icoords(self.selector, domain_id = self.domain_id)
+ d = self.oct_handler.icoords(self.selector, domain_id = self.domain_id,
+ num_octs = self._num_octs)
+ self._num_octs = d.shape[0]
tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 3,
domain_id = self.domain_id)
return tr
def select_fcoords(self, dobj):
- d = self.oct_handler.fcoords(self.selector, domain_id = self.domain_id)
+ d = self.oct_handler.fcoords(self.selector, domain_id = self.domain_id,
+ num_octs = self._num_octs)
+ self._num_octs = d.shape[0]
tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 3,
domain_id = self.domain_id)
return tr
def select_fwidth(self, dobj):
- d = self.oct_handler.fwidth(self.selector, domain_id = self.domain_id)
+ d = self.oct_handler.fwidth(self.selector, domain_id = self.domain_id,
+ num_octs = self._num_octs)
+ self._num_octs = d.shape[0]
tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 3,
domain_id = self.domain_id)
return tr
def select_ires(self, dobj):
- d = self.oct_handler.ires(self.selector, domain_id = self.domain_id)
+ d = self.oct_handler.ires(self.selector, domain_id = self.domain_id,
+ num_octs = self._num_octs)
+ self._num_octs = d.shape[0]
tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 1,
domain_id = self.domain_id)
return tr
diff -r d7db8bf1f2a13d7ec9550662789d7e677048c18b -r 96ff7315a44a131c07cabb0580a212ff198fe556 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -328,12 +328,12 @@
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def icoords(self, SelectorObject selector, np.uint64_t num_cells = -1,
+ def icoords(self, SelectorObject selector, np.int64_t num_octs = -1,
int domain_id = -1):
- if num_cells == -1:
- num_cells = selector.count_octs(self, domain_id)
+ if num_octs == -1:
+ num_octs = selector.count_octs(self, domain_id)
cdef np.ndarray[np.int64_t, ndim=2] coords
- coords = np.empty((num_cells, 3), dtype="int64")
+ coords = np.empty((num_octs, 3), dtype="int64")
cdef OctVisitorData data
data.array = <void *> coords.data
data.index = 0
@@ -344,13 +344,13 @@
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def ires(self, SelectorObject selector, np.uint64_t num_cells = -1,
+ def ires(self, SelectorObject selector, np.int64_t num_octs = -1,
int domain_id = -1):
- if num_cells == -1:
- num_cells = selector.count_octs(self, domain_id)
+ if num_octs == -1:
+ num_octs = selector.count_octs(self, domain_id)
#Return the 'resolution' of each cell; ie the level
cdef np.ndarray[np.int64_t, ndim=1] res
- res = np.empty(num_cells, dtype="int64")
+ res = np.empty(num_octs, dtype="int64")
cdef OctVisitorData data
data.array = <void *> res.data
data.index = 0
@@ -361,12 +361,12 @@
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def fwidth(self, SelectorObject selector, np.uint64_t num_cells = -1,
+ def fwidth(self, SelectorObject selector, np.int64_t num_octs = -1,
int domain_id = -1):
- if num_cells == -1:
- num_cells = selector.count_octs(self, domain_id)
+ if num_octs == -1:
+ num_octs = selector.count_octs(self, domain_id)
cdef np.ndarray[np.float64_t, ndim=2] fwidth
- fwidth = np.empty((num_cells, 3), dtype="float64")
+ fwidth = np.empty((num_octs, 3), dtype="float64")
cdef OctVisitorData data
data.array = <void *> fwidth.data
data.index = 0
@@ -381,13 +381,13 @@
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def fcoords(self, SelectorObject selector, np.uint64_t num_cells = -1,
+ def fcoords(self, SelectorObject selector, np.int64_t num_octs = -1,
int domain_id = -1):
- if num_cells == -1:
- num_cells = selector.count_octs(self, domain_id)
+ if num_octs == -1:
+ num_octs = selector.count_octs(self, domain_id)
#Return the floating point unitary position of every cell
cdef np.ndarray[np.float64_t, ndim=2] coords
- coords = np.empty((num_cells, 3), dtype="float64")
+ coords = np.empty((num_octs, 3), dtype="float64")
cdef OctVisitorData data
data.array = <void *> coords.data
data.index = 0
https://bitbucket.org/yt_analysis/yt-3.0/commits/d17386ccee37/
Changeset: d17386ccee37
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-26 14:41:22
Summary: Switching to count_octs meaning what it says and moving domain_ind into baseclass.
Affected #: 3 files
diff -r 96ff7315a44a131c07cabb0580a212ff198fe556 -r d17386ccee37a9818ecd6ce25b04f66bb80983f3 yt/data_objects/octree_subset.py
--- a/yt/data_objects/octree_subset.py
+++ b/yt/data_objects/octree_subset.py
@@ -126,7 +126,7 @@
def select_icoords(self, dobj):
d = self.oct_handler.icoords(self.selector, domain_id = self.domain_id,
num_octs = self._num_octs)
- self._num_octs = d.shape[0]
+ self._num_octs = d.shape[0] / 8
tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 3,
domain_id = self.domain_id)
return tr
@@ -134,7 +134,7 @@
def select_fcoords(self, dobj):
d = self.oct_handler.fcoords(self.selector, domain_id = self.domain_id,
num_octs = self._num_octs)
- self._num_octs = d.shape[0]
+ self._num_octs = d.shape[0] / 8
tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 3,
domain_id = self.domain_id)
return tr
@@ -142,7 +142,7 @@
def select_fwidth(self, dobj):
d = self.oct_handler.fwidth(self.selector, domain_id = self.domain_id,
num_octs = self._num_octs)
- self._num_octs = d.shape[0]
+ self._num_octs = d.shape[0] / 8
tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 3,
domain_id = self.domain_id)
return tr
@@ -150,7 +150,7 @@
def select_ires(self, dobj):
d = self.oct_handler.ires(self.selector, domain_id = self.domain_id,
num_octs = self._num_octs)
- self._num_octs = d.shape[0]
+ self._num_octs = d.shape[0] / 8
tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 1,
domain_id = self.domain_id)
return tr
diff -r 96ff7315a44a131c07cabb0580a212ff198fe556 -r d17386ccee37a9818ecd6ce25b04f66bb80983f3 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -333,7 +333,7 @@
if num_octs == -1:
num_octs = selector.count_octs(self, domain_id)
cdef np.ndarray[np.int64_t, ndim=2] coords
- coords = np.empty((num_octs, 3), dtype="int64")
+ coords = np.empty((num_octs * 8, 3), dtype="int64")
cdef OctVisitorData data
data.array = <void *> coords.data
data.index = 0
@@ -350,7 +350,7 @@
num_octs = selector.count_octs(self, domain_id)
#Return the 'resolution' of each cell; ie the level
cdef np.ndarray[np.int64_t, ndim=1] res
- res = np.empty(num_octs, dtype="int64")
+ res = np.empty(num_octs * 8, dtype="int64")
cdef OctVisitorData data
data.array = <void *> res.data
data.index = 0
@@ -366,7 +366,7 @@
if num_octs == -1:
num_octs = selector.count_octs(self, domain_id)
cdef np.ndarray[np.float64_t, ndim=2] fwidth
- fwidth = np.empty((num_octs, 3), dtype="float64")
+ fwidth = np.empty((num_octs * 8, 3), dtype="float64")
cdef OctVisitorData data
data.array = <void *> fwidth.data
data.index = 0
@@ -387,7 +387,7 @@
num_octs = selector.count_octs(self, domain_id)
#Return the floating point unitary position of every cell
cdef np.ndarray[np.float64_t, ndim=2] coords
- coords = np.empty((num_octs, 3), dtype="float64")
+ coords = np.empty((num_octs * 8, 3), dtype="float64")
cdef OctVisitorData data
data.array = <void *> coords.data
data.index = 0
@@ -452,6 +452,18 @@
return dest
return data.index - offset
+ def domain_ind(self, selector, int domain_id = -1):
+ cdef np.ndarray[np.int64_t, ndim=1] ind
+ # Here's where we grab the masked items.
+ ind = np.zeros(self.nocts, 'int64') - 1
+ cdef OctVisitorData data
+ data.domain = domain_id
+ data.array = ind.data
+ data.index = 0
+ data.last = -1
+ self.visit_all_octs(selector, oct_visitors.index_octs, &data)
+ return ind
+
cdef int root_node_compare(void *a, void *b) nogil:
cdef OctKey *ao, *bo
ao = <OctKey *>a
@@ -697,18 +709,6 @@
local_filled += 1
return local_filled
- def domain_ind(self, selector, int domain_id = -1):
- cdef np.ndarray[np.int64_t, ndim=1] ind
- # Here's where we grab the masked items.
- ind = np.zeros(self.nocts, 'int64') - 1
- cdef OctVisitorData data
- data.domain = domain_id
- data.array = ind.data
- data.index = 0
- data.last = -1
- self.visit_all_octs(selector, oct_visitors.index_octs, &data)
- return ind
-
cdef class ARTOctreeContainer(RAMSESOctreeContainer):
@cython.boundscheck(True)
diff -r 96ff7315a44a131c07cabb0580a212ff198fe556 -r d17386ccee37a9818ecd6ce25b04f66bb80983f3 yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -148,7 +148,7 @@
data.last = -1
data.domain = domain_id
octree.visit_all_octs(self, oct_visitors.count_total_octs, &data)
- return data.index * 8
+ return data.index
def count_oct_cells(self, OctreeContainer octree, int domain_id = -1):
cdef OctVisitorData data
https://bitbucket.org/yt_analysis/yt-3.0/commits/08d4ff8c0204/
Changeset: 08d4ff8c0204
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-26 14:47:15
Summary: This fixes problems with particle code depositions.
Affected #: 1 file
diff -r d17386ccee37a9818ecd6ce25b04f66bb80983f3 -r 08d4ff8c02043994d944f8bcd36f03c1d38aca45 yt/geometry/particle_deposit.pyx
--- a/yt/geometry/particle_deposit.pyx
+++ b/yt/geometry/particle_deposit.pyx
@@ -94,7 +94,8 @@
# RAMSES, where we *will* miss particles that live in ghost
# regions on other processors. Addressing this is on the TODO
# list.
- if oct == NULL or oct.domain != domain_id: continue
+ if oct == NULL or (domain_id > 0 and oct.domain != domain_id):
+ continue
# Note that this has to be our local index, not our in-file index.
offset = dom_ind[oct.domain_ind - moff] * 8
if offset < 0: continue
https://bitbucket.org/yt_analysis/yt-3.0/commits/3f01744f5cbf/
Changeset: 3f01744f5cbf
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-27 04:27:09
Summary: NMSU-ART loads up again, does not read data yet.
Affected #: 2 files
diff -r 08d4ff8c02043994d944f8bcd36f03c1d38aca45 -r 3f01744f5cbf9cb07ce0b8ffc05dbc01be7527af yt/frontends/art/data_structures.py
--- a/yt/frontends/art/data_structures.py
+++ b/yt/frontends/art/data_structures.py
@@ -20,7 +20,7 @@
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
-.
+
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
@@ -575,8 +575,7 @@
self._level_oct_offsets, level,
coarse_grid=self.pf.domain_dimensions[0],
root_level=self.pf.root_level)
- nocts_check = oct_handler.add(self.domain_id, level, nocts,
- unitary_center, self.domain_id)
+ nocts_check = oct_handler.add(self.domain_id, level, unitary_center)
assert(nocts_check == nocts)
mylog.debug("Added %07i octs on level %02i, cumulative is %07i",
nocts, level, oct_handler.nocts)
@@ -600,8 +599,7 @@
LL[2]:RL[2]:NX[2]*1j]
root_fc = np.vstack([p.ravel() for p in root_fc]).T
nocts_check = oct_handler.add(self.domain_id, level,
- root_octs_side**3,
- root_fc, self.domain_id)
+ root_fc)
assert(oct_handler.nocts == root_fc.shape[0])
mylog.debug("Added %07i octs on level %02i, cumulative is %07i",
root_octs_side**3, 0, oct_handler.nocts)
diff -r 08d4ff8c02043994d944f8bcd36f03c1d38aca45 -r 3f01744f5cbf9cb07ce0b8ffc05dbc01be7527af yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -104,6 +104,12 @@
self.max_domain = -1
p = 0
self.nocts = 0 # Increment when initialized
+ for i in range(3):
+ self.DLE[i] = domain_left_edge[i] #0
+ self.DRE[i] = domain_right_edge[i] #num_grid
+ self._initialize_root_mesh()
+
+ def _initialize_root_mesh(self):
self.root_mesh = <Oct****> malloc(sizeof(void*) * self.nn[0])
for i in range(self.nn[0]):
self.root_mesh[i] = <Oct ***> malloc(sizeof(void*) * self.nn[1])
@@ -111,10 +117,6 @@
self.root_mesh[i][j] = <Oct **> malloc(sizeof(void*) * self.nn[2])
for k in range(self.nn[2]):
self.root_mesh[i][j][k] = NULL
- # We don't initialize the octs yet
- for i in range(3):
- self.DLE[i] = domain_left_edge[i] #0
- self.DRE[i] = domain_right_edge[i] #num_grid
def __dealloc__(self):
free_octs(self.cont)
@@ -621,7 +623,7 @@
for i in range(root_nodes):
self.root_nodes[i].key = -1
self.root_nodes[i].node = NULL
-
+
def __dealloc__(self):
# This gets called BEFORE the superclass deallocation. But, both get
# called.
@@ -711,6 +713,10 @@
cdef class ARTOctreeContainer(RAMSESOctreeContainer):
+ def __init__(self, domain_dimensions, domain_left_edge, domain_right_edge):
+ OctreeContainer.__init__(self, domain_dimensions,
+ domain_left_edge, domain_right_edge)
+
@cython.boundscheck(True)
@cython.wraparound(False)
@cython.cdivision(True)
@@ -798,3 +804,25 @@
local_filled += 1
return local_filled
+ def allocate_domains(self, domain_counts):
+ cdef int count, i
+ cdef OctAllocationContainer *cur = self.cont
+ assert(cur == NULL)
+ self.max_domain = len(domain_counts) # 1-indexed
+ self.domains = <OctAllocationContainer **> malloc(
+ sizeof(OctAllocationContainer *) * len(domain_counts))
+ for i, count in enumerate(domain_counts):
+ cur = allocate_octs(count, cur)
+ if self.cont == NULL: self.cont = cur
+ self.domains[i] = cur
+
+ cdef Oct* next_root(self, int domain_id, int ind[3]):
+ cdef Oct *next = self.root_mesh[ind[0]][ind[1]][ind[2]]
+ if next != NULL: return next
+ cdef OctAllocationContainer *cont = self.domains[domain_id - 1]
+ if cont.n_assigned >= cont.n: raise RuntimeError
+ next = &cont.my_octs[cont.n_assigned]
+ cont.n_assigned += 1
+ self.root_mesh[ind[0]][ind[1]][ind[2]] = next
+ self.nocts += 1
+ return next
https://bitbucket.org/yt_analysis/yt-3.0/commits/eadc1ed9c34a/
Changeset: eadc1ed9c34a
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-27 04:56:15
Summary: Continuing to port NMSU-ART
Affected #: 2 files
diff -r 3f01744f5cbf9cb07ce0b8ffc05dbc01be7527af -r eadc1ed9c34a9069e9121b59d3b70d2b7156f26a yt/frontends/art/data_structures.py
--- a/yt/frontends/art/data_structures.py
+++ b/yt/frontends/art/data_structures.py
@@ -106,14 +106,15 @@
allocate the requisite memory in the oct tree
"""
nv = len(self.fluid_field_list)
- self.domains = [ARTDomainFile(self.parameter_file, l+1, nv, l)
- for l in range(self.pf.max_level)]
- self.octs_per_domain = [dom.level_count.sum() for dom in self.domains]
- self.total_octs = sum(self.octs_per_domain)
self.oct_handler = ARTOctreeContainer(
self.parameter_file.domain_dimensions/2, # dd is # of root cells
self.parameter_file.domain_left_edge,
self.parameter_file.domain_right_edge)
+ self.domains = [ARTDomainFile(self.parameter_file, l+1, nv, l,
+ self.oct_handler)
+ for l in range(self.pf.max_level)]
+ self.octs_per_domain = [dom.level_count.sum() for dom in self.domains]
+ self.total_octs = sum(self.octs_per_domain)
mylog.debug("Allocating %s octs", self.total_octs)
self.oct_handler.allocate_domains(self.octs_per_domain)
for domain in self.domains:
@@ -150,28 +151,21 @@
"""
if getattr(dobj, "_chunk_info", None) is None:
# Get all octs within this oct handler
- mask = dobj.selector.select_octs(self.oct_handler)
- if mask.sum() == 0:
- mylog.debug("Warning: selected zero octs")
- counts = self.oct_handler.count_cells(dobj.selector, mask)
- # For all domains, figure out how many counts we have
- # and build a subset=mask of domains
- subsets = []
- for d, c in zip(self.domains, counts):
- if c < 1:
- continue
- subset = ARTDomainSubset(d, mask, c, d.domain_level)
- subsets.append(subset)
+ domains = [dom for dom in self.domains if
+ dom.included(dobj.selector)]
+ base_region = getattr(dobj, "base_region", dobj)
+ if len(domains) > 1:
+ mylog.debug("Identified %s intersecting domains", len(domains))
+ subsets = [ARTDomainSubset(base_region, domain, self.parameter_file)
+ for domain in domains]
dobj._chunk_info = subsets
- dobj.size = sum(counts)
- dobj.shape = (dobj.size,)
dobj._current_chunk = list(self._chunk_all(dobj))[0]
def _chunk_all(self, dobj):
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
# We pass the chunk both the current chunk and list of chunks,
# as well as the referring data source
- yield YTDataChunk(dobj, "all", oobjs, dobj.size)
+ yield YTDataChunk(dobj, "all", oobjs, None)
def _chunk_spatial(self, dobj, ngz, sort = None):
sobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
@@ -180,9 +174,7 @@
g = og.retrieve_ghost_zones(ngz, [], smoothed=True)
else:
g = og
- size = og.cell_count
- if size == 0: continue
- yield YTDataChunk(dobj, "spatial", [g], size)
+ yield YTDataChunk(dobj, "spatial", [g], None)
def _chunk_io(self, dobj):
"""
@@ -193,7 +185,7 @@
"""
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
for subset in oobjs:
- yield YTDataChunk(dobj, "io", [subset], subset.cell_count)
+ yield YTDataChunk(dobj, "io", [subset], None)
class ARTStaticOutput(StaticOutput):
@@ -445,9 +437,9 @@
return False
class ARTDomainSubset(OctreeSubset):
- def __init__(self, domain, mask, cell_count, domain_level):
- super(ARTDomainSubset, self).__init__(domain, mask, cell_count)
- self.domain_level = domain_level
+ def __init__(self, base_region, domain, pf):
+ super(ARTDomainSubset, self).__init__(base_region, domain, pf)
+ self.domain_level = domain.domain_level
def fill_root(self, content, ftfields):
"""
@@ -503,7 +495,6 @@
nocts_filling)
return dest
-
class ARTDomainFile(object):
"""
Read in the AMR, left/right edges, fill out the octhandler
@@ -514,7 +505,7 @@
_last_mask = None
_last_seletor_id = None
- def __init__(self, pf, domain_id, nvar, level):
+ def __init__(self, pf, domain_id, nvar, level, oct_handler):
self.nvar = nvar
self.pf = pf
self.domain_id = domain_id
@@ -522,6 +513,7 @@
self._level_count = None
self._level_oct_offsets = None
self._level_child_offsets = None
+ self.oct_handler = oct_handler
@property
def level_count(self):
@@ -618,3 +610,10 @@
return self._last_mask.sum()
self.select(selector)
return self.count(selector)
+
+ def included(self, selector):
+ return True
+ if getattr(selector, "domain_id", None) is not None:
+ return selector.domain_id == self.domain_id
+ domain_ids = self.pf.h.oct_handler.domain_identify(selector)
+ return self.domain_id in domain_ids
diff -r 3f01744f5cbf9cb07ce0b8ffc05dbc01be7527af -r eadc1ed9c34a9069e9121b59d3b70d2b7156f26a yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -230,6 +230,20 @@
oinfo.left_edge[i] = cp[i] - dds[i] # Center minus dds
return cur
+ def domain_identify(self, SelectorObject selector):
+ cdef np.ndarray[np.uint8_t, ndim=1] domain_mask
+ domain_mask = np.zeros(self.max_domain, dtype="uint8")
+ cdef OctVisitorData data
+ data.array = domain_mask.data
+ data.domain = -1
+ self.visit_all_octs(selector, oct_visitors.identify_octs, &data)
+ cdef int i
+ domain_ids = []
+ for i in range(self.max_domain):
+ if domain_mask[i] == 1:
+ domain_ids.append(i+1)
+ return domain_ids
+
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
@@ -546,20 +560,6 @@
selector.recursively_visit_octs(
o, pos, dds, 0, func, data, vc)
- def domain_identify(self, SelectorObject selector):
- cdef np.ndarray[np.uint8_t, ndim=1] domain_mask
- domain_mask = np.zeros(self.max_domain, dtype="uint8")
- cdef OctVisitorData data
- data.array = domain_mask.data
- data.domain = -1
- self.visit_all_octs(selector, oct_visitors.identify_octs, &data)
- cdef int i
- domain_ids = []
- for i in range(self.max_domain):
- if domain_mask[i] == 1:
- domain_ids.append(i+1)
- return domain_ids
-
cdef np.int64_t get_domain_offset(self, int domain_id):
return 0 # We no longer have a domain offset.
https://bitbucket.org/yt_analysis/yt-3.0/commits/6f78ec8a3a59/
Changeset: 6f78ec8a3a59
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-27 06:00:32
Summary: Refactoring to re-enable NMSU-ART.
Affected #: 6 files
diff -r eadc1ed9c34a9069e9121b59d3b70d2b7156f26a -r 6f78ec8a3a59b676d3fe3de5a6f45ef5798e9e37 yt/frontends/art/data_structures.py
--- a/yt/frontends/art/data_structures.py
+++ b/yt/frontends/art/data_structures.py
@@ -452,22 +452,18 @@
oct_handler = self.oct_handler
all_fields = self.domain.pf.h.fluid_field_list
fields = [f for ft, f in ftfields]
- level_offset = 0
field_idxs = [all_fields.index(f) for f in fields]
- dest = {}
- for field in fields:
- dest[field] = np.zeros(self.cell_count, 'float64')-1.
level = self.domain_level
source = {}
data = _read_root_level(content, self.domain.level_child_offsets,
self.domain.level_count)
+
for field, i in zip(fields, field_idxs):
temp = np.reshape(data[i, :], self.domain.pf.domain_dimensions,
order='F').astype('float64').T
source[field] = temp
- level_offset += oct_handler.fill_level_from_grid(
- self.domain.domain_id,
- level, dest, source, self.mask, level_offset)
+ dest = oct_handler.fill_level_from_grid(
+ self.selector, self.domain_id, source)
return dest
def fill_level(self, content, ftfields):
@@ -596,21 +592,6 @@
mylog.debug("Added %07i octs on level %02i, cumulative is %07i",
root_octs_side**3, 0, oct_handler.nocts)
- def select(self, selector):
- if id(selector) == self._last_selector_id:
- return self._last_mask
- self._last_mask = selector.fill_mask(self)
- self._last_selector_id = id(selector)
- return self._last_mask
-
- def count(self, selector):
- if id(selector) == self._last_selector_id:
- if self._last_mask is None:
- return 0
- return self._last_mask.sum()
- self.select(selector)
- return self.count(selector)
-
def included(self, selector):
return True
if getattr(selector, "domain_id", None) is not None:
diff -r eadc1ed9c34a9069e9121b59d3b70d2b7156f26a -r 6f78ec8a3a59b676d3fe3de5a6f45ef5798e9e37 yt/frontends/art/io.py
--- a/yt/frontends/art/io.py
+++ b/yt/frontends/art/io.py
@@ -30,6 +30,7 @@
import os
import os.path
+from yt.funcs import *
from yt.utilities.io_handler import \
BaseIOHandler
import yt.utilities.lib as au
@@ -47,7 +48,7 @@
# Chunks in this case will have affiliated domain subset objects
# Each domain subset will contain a hydro_offset array, which gives
# pointers to level-by-level hydro information
- tr = dict((f, np.empty(size, dtype='float64')) for f in fields)
+ tr = defaultdict(list)
cp = 0
for chunk in chunks:
for subset in chunk.objs:
@@ -60,13 +61,16 @@
else:
rv = subset.fill_level(f, fields)
for ft, f in fields:
+ d = rv.pop(f)
mylog.debug("Filling L%i %s with %s (%0.3e %0.3e) (%s:%s)",
- subset.domain_level,
- f, subset.cell_count, rv[f].min(), rv[f].max(),
- cp, cp+subset.cell_count)
- tr[(ft, f)][cp:cp+subset.cell_count] = rv.pop(f)
+ subset.domain_level, f, d.size, d.min(), d.max(),
+ cp, cp+d.size)
+ tr[(ft, f)].append(d)
cp += subset.cell_count
- return tr
+ d = {}
+ for k in tr.keys():
+ d[k] = np.concatenate(tr.pop(k))
+ return d
def _read_particle_selection(self, chunks, selector, fields):
tr = {}
diff -r eadc1ed9c34a9069e9121b59d3b70d2b7156f26a -r 6f78ec8a3a59b676d3fe3de5a6f45ef5798e9e37 yt/geometry/oct_container.pxd
--- a/yt/geometry/oct_container.pxd
+++ b/yt/geometry/oct_container.pxd
@@ -51,6 +51,7 @@
cdef class OctreeContainer:
cdef OctAllocationContainer *cont
+ cdef OctAllocationContainer **domains
cdef Oct ****root_mesh
cdef int partial_coverage
cdef int nn[3]
@@ -67,15 +68,14 @@
cdef void visit_all_octs(self, SelectorObject selector,
oct_visitor_function *func,
OctVisitorData *data)
+ cdef Oct *next_root(self, int domain_id, int ind[3])
+ cdef Oct *next_child(self, int domain_id, int ind[3], Oct *parent)
cdef class RAMSESOctreeContainer(OctreeContainer):
- cdef OctAllocationContainer **domains
cdef OctKey *root_nodes
cdef void *tree_root
cdef int num_root
cdef int max_root
- cdef Oct *next_root(self, int domain_id, int ind[3])
- cdef Oct *next_child(self, int domain_id, int ind[3], Oct *parent)
cdef extern from "search.h" nogil:
void *tsearch(const void *key, void **rootp,
diff -r eadc1ed9c34a9069e9121b59d3b70d2b7156f26a -r 6f78ec8a3a59b676d3fe3de5a6f45ef5798e9e37 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -480,6 +480,97 @@
self.visit_all_octs(selector, oct_visitors.index_octs, &data)
return ind
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ def add(self, int curdom, int curlevel,
+ np.ndarray[np.float64_t, ndim=2] pos,
+ int skip_boundary = 1):
+ cdef int level, no, p, i, j, k, ind[3]
+ cdef Oct *cur, *next = NULL
+ cdef np.float64_t pp[3], cp[3], dds[3]
+ no = pos.shape[0] #number of octs
+ if curdom > self.max_domain: return 0
+ cdef OctAllocationContainer *cont = self.domains[curdom - 1]
+ cdef int initial = cont.n_assigned
+ cdef int in_boundary = 0
+ # How do we bootstrap ourselves?
+ for p in range(no):
+ #for every oct we're trying to add find the
+ #floating point unitary position on this level
+ in_boundary = 0
+ for i in range(3):
+ pp[i] = pos[p, i]
+ dds[i] = (self.DRE[i] - self.DLE[i])/self.nn[i]
+ ind[i] = <np.int64_t> ((pp[i] - self.DLE[i])/dds[i])
+ cp[i] = (ind[i] + 0.5) * dds[i] + self.DLE[i]
+ if ind[i] < 0 or ind[i] >= self.nn[i]:
+ in_boundary = 1
+ if skip_boundary == in_boundary == 1: continue
+ cur = self.next_root(curdom, ind)
+ if cur == NULL: raise RuntimeError
+ # Now we find the location we want
+ # Note that RAMSES I think 1-findiceses levels, but we don't.
+ for level in range(curlevel):
+ # At every level, find the cell this oct
+ # lives inside
+ for i in range(3):
+ #as we get deeper, oct size halves
+ dds[i] = dds[i] / 2.0
+ if cp[i] > pp[i]:
+ ind[i] = 0
+ cp[i] -= dds[i]/2.0
+ else:
+ ind[i] = 1
+ cp[i] += dds[i]/2.0
+ # Check if it has not been allocated
+ cur = self.next_child(curdom, ind, cur)
+ # Now we should be at the right level
+ cur.domain = curdom
+ cur.file_ind = p
+ return cont.n_assigned - initial
+
+ def allocate_domains(self, domain_counts):
+ cdef int count, i
+ cdef OctAllocationContainer *cur = self.cont
+ assert(cur == NULL)
+ self.max_domain = len(domain_counts) # 1-indexed
+ self.domains = <OctAllocationContainer **> malloc(
+ sizeof(OctAllocationContainer *) * len(domain_counts))
+ for i, count in enumerate(domain_counts):
+ cur = allocate_octs(count, cur)
+ if self.cont == NULL: self.cont = cur
+ self.domains[i] = cur
+
+ cdef Oct* next_root(self, int domain_id, int ind[3]):
+ cdef Oct *next = self.root_mesh[ind[0]][ind[1]][ind[2]]
+ if next != NULL: return next
+ cdef OctAllocationContainer *cont = self.domains[domain_id - 1]
+ if cont.n_assigned >= cont.n: raise RuntimeError
+ next = &cont.my_octs[cont.n_assigned]
+ cont.n_assigned += 1
+ self.root_mesh[ind[0]][ind[1]][ind[2]] = next
+ self.nocts += 1
+ return next
+
+ cdef Oct* next_child(self, int domain_id, int ind[3], Oct *parent):
+ cdef int i
+ cdef Oct *next = NULL
+ if parent.children != NULL:
+ next = parent.children[cind(ind[0],ind[1],ind[2])]
+ else:
+ parent.children = <Oct **> malloc(sizeof(Oct *) * 8)
+ for i in range(8):
+ parent.children[i] = NULL
+ if next != NULL: return next
+ cdef OctAllocationContainer *cont = self.domains[domain_id - 1]
+ if cont.n_assigned >= cont.n: raise RuntimeError
+ next = &cont.my_octs[cont.n_assigned]
+ cont.n_assigned += 1
+ parent.children[cind(ind[0],ind[1],ind[2])] = next
+ self.nocts += 1
+ return next
+
cdef int root_node_compare(void *a, void *b) nogil:
cdef OctKey *ao, *bo
ao = <OctKey *>a
@@ -589,35 +680,8 @@
self.nocts += 1
return next
- cdef Oct* next_child(self, int domain_id, int ind[3], Oct *parent):
- cdef int i
- cdef Oct *next = NULL
- if parent.children != NULL:
- next = parent.children[cind(ind[0],ind[1],ind[2])]
- else:
- parent.children = <Oct **> malloc(sizeof(Oct *) * 8)
- for i in range(8):
- parent.children[i] = NULL
- if next != NULL: return next
- cdef OctAllocationContainer *cont = self.domains[domain_id - 1]
- if cont.n_assigned >= cont.n: raise RuntimeError
- next = &cont.my_octs[cont.n_assigned]
- cont.n_assigned += 1
- parent.children[cind(ind[0],ind[1],ind[2])] = next
- self.nocts += 1
- return next
-
def allocate_domains(self, domain_counts, int root_nodes):
- cdef int count, i
- cdef OctAllocationContainer *cur = self.cont
- assert(cur == NULL)
- self.max_domain = len(domain_counts) # 1-indexed
- self.domains = <OctAllocationContainer **> malloc(
- sizeof(OctAllocationContainer *) * len(domain_counts))
- for i, count in enumerate(domain_counts):
- cur = allocate_octs(count, cur)
- if self.cont == NULL: self.cont = cur
- self.domains[i] = cur
+ OctreeContainer.allocate_domains(domain_counts)
self.root_nodes = <OctKey*> malloc(sizeof(OctKey) * root_nodes)
self.max_root = root_nodes
for i in range(root_nodes):
@@ -633,56 +697,6 @@
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def add(self, int curdom, int curlevel,
- np.ndarray[np.float64_t, ndim=2] pos,
- int skip_boundary = 1):
- cdef int level, no, p, i, j, k, ind[3]
- cdef Oct *cur, *next = NULL
- cdef np.float64_t pp[3], cp[3], dds[3]
- no = pos.shape[0] #number of octs
- if curdom > self.max_domain: return 0
- cdef OctAllocationContainer *cont = self.domains[curdom - 1]
- cdef int initial = cont.n_assigned
- cdef int in_boundary = 0
- # How do we bootstrap ourselves?
- for p in range(no):
- #for every oct we're trying to add find the
- #floating point unitary position on this level
- in_boundary = 0
- for i in range(3):
- pp[i] = pos[p, i]
- dds[i] = (self.DRE[i] - self.DLE[i])/self.nn[i]
- ind[i] = <np.int64_t> ((pp[i] - self.DLE[i])/dds[i])
- cp[i] = (ind[i] + 0.5) * dds[i] + self.DLE[i]
- if ind[i] < 0 or ind[i] >= self.nn[i]:
- in_boundary = 1
- if skip_boundary == in_boundary == 1: continue
- cur = self.next_root(curdom, ind)
- if cur == NULL: raise RuntimeError
- # Now we find the location we want
- # Note that RAMSES I think 1-findiceses levels, but we don't.
- for level in range(curlevel):
- # At every level, find the cell this oct
- # lives inside
- for i in range(3):
- #as we get deeper, oct size halves
- dds[i] = dds[i] / 2.0
- if cp[i] > pp[i]:
- ind[i] = 0
- cp[i] -= dds[i]/2.0
- else:
- ind[i] = 1
- cp[i] += dds[i]/2.0
- # Check if it has not been allocated
- cur = self.next_child(curdom, ind, cur)
- # Now we should be at the right level
- cur.domain = curdom
- cur.file_ind = p
- return cont.n_assigned - initial
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
def fill_level(self, int domain, int level, dest_fields, source_fields,
np.ndarray[np.uint8_t, ndim=2, cast=True] mask, int offset):
cdef np.ndarray[np.float64_t, ndim=2] source
@@ -711,11 +725,7 @@
local_filled += 1
return local_filled
-cdef class ARTOctreeContainer(RAMSESOctreeContainer):
-
- def __init__(self, domain_dimensions, domain_left_edge, domain_right_edge):
- OctreeContainer.__init__(self, domain_dimensions,
- domain_left_edge, domain_right_edge)
+cdef class ARTOctreeContainer(OctreeContainer):
@cython.boundscheck(True)
@cython.wraparound(False)
@@ -758,14 +768,11 @@
local_filled += 1
return local_filled
-
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def fill_level_from_grid(self, int domain, int level, dest_fields,
- source_fields,
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
- int offset):
+ def fill_level_from_grid(self, SelectorObject selector,
+ int domain_id, source_fields):
#Fill level, but instead of assuming that the source
#order is that of the oct order, we look up the oct position
#and fill its children from the the source field
@@ -777,32 +784,21 @@
# Note that the .pos[0] etc calls need to be uncommented.
cdef np.ndarray[np.float64_t, ndim=3] source
cdef np.ndarray[np.float64_t, ndim=1] dest
- cdef OctAllocationContainer *dom = self.domains[domain - 1]
- cdef Oct *o
- cdef int n
- cdef int i, j, k, ii
- cdef int local_pos, local_filled
- cdef np.float64_t val
- cdef np.int64_t ox,oy,oz
- for key in dest_fields:
+ cdef OctVisitorData data
+ cdef void *p[2]
+ num_cells = selector.count_oct_cells(self, domain_id)
+ dest_fields = {}
+ for key in source_fields:
+ dest_fields[key] = dest = \
+ np.zeros(num_cells, dtype="float64")
+ data.index = 0
local_filled = 0
- dest = dest_fields[key]
source = source_fields[key]
- for n in range(dom.n):
- o = &dom.my_octs[n]
- # TODO: Uncomment this!
- #if o.level != level: continue
- for i in range(2):
- for j in range(2):
- for k in range(2):
- ii = ((k*2)+j)*2+i
- if mask[o.domain_ind, ii] == 0: continue
- #ox = (o.pos[0] << 1) + i
- #oy = (o.pos[1] << 1) + j
- #oz = (o.pos[2] << 1) + k
- dest[local_filled + offset] = source[ox,oy,oz]
- local_filled += 1
- return local_filled
+ p[0] = source.data
+ p[1] = dest.data
+ data.array = &p
+ self.visit_all_octs(selector, oct_visitors.fill_from_file, &data)
+ return dest_fields
def allocate_domains(self, domain_counts):
cdef int count, i
@@ -816,13 +812,3 @@
if self.cont == NULL: self.cont = cur
self.domains[i] = cur
- cdef Oct* next_root(self, int domain_id, int ind[3]):
- cdef Oct *next = self.root_mesh[ind[0]][ind[1]][ind[2]]
- if next != NULL: return next
- cdef OctAllocationContainer *cont = self.domains[domain_id - 1]
- if cont.n_assigned >= cont.n: raise RuntimeError
- next = &cont.my_octs[cont.n_assigned]
- cont.n_assigned += 1
- self.root_mesh[ind[0]][ind[1]][ind[2]] = next
- self.nocts += 1
- return next
diff -r eadc1ed9c34a9069e9121b59d3b70d2b7156f26a -r 6f78ec8a3a59b676d3fe3de5a6f45ef5798e9e37 yt/geometry/oct_visitors.pxd
--- a/yt/geometry/oct_visitors.pxd
+++ b/yt/geometry/oct_visitors.pxd
@@ -60,6 +60,7 @@
cdef oct_visitor_function copy_array_i64
cdef oct_visitor_function identify_octs
cdef oct_visitor_function assign_domain_ind
+cdef oct_visitor_function fill_from_file
cdef inline int cind(int i, int j, int k):
return (((i*2)+j)*2+k)
diff -r eadc1ed9c34a9069e9121b59d3b70d2b7156f26a -r 6f78ec8a3a59b676d3fe3de5a6f45ef5798e9e37 yt/geometry/oct_visitors.pyx
--- a/yt/geometry/oct_visitors.pyx
+++ b/yt/geometry/oct_visitors.pyx
@@ -151,3 +151,11 @@
cdef void assign_domain_ind(Oct *o, OctVisitorData *data, np.uint8_t selected):
o.domain_ind = data.global_index
data.index += 1
+
+cdef void fill_from_file(Oct *o, OctVisitorData *data, np.uint8_t selected):
+ if selected == 0: return
+ # There are this many records between "octs"
+ cdef np.float64_t **p = <np.float64_t**> data.array
+ p[1][data.index] = p[0][o.file_ind + oind(data)]
+ data.index += 1
+
https://bitbucket.org/yt_analysis/yt-3.0/commits/9e27d3a8e379/
Changeset: 9e27d3a8e379
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-27 06:20:18
Summary: Specifying min/max level fixes the ART selection.
Affected #: 2 files
diff -r 6f78ec8a3a59b676d3fe3de5a6f45ef5798e9e37 -r 9e27d3a8e379927a92e4787119bad3c29c37b061 yt/frontends/art/data_structures.py
--- a/yt/frontends/art/data_structures.py
+++ b/yt/frontends/art/data_structures.py
@@ -439,6 +439,8 @@
class ARTDomainSubset(OctreeSubset):
def __init__(self, base_region, domain, pf):
super(ARTDomainSubset, self).__init__(base_region, domain, pf)
+ self.min_level = domain.domain_level
+ self.max_level = domain.domain_level
self.domain_level = domain.domain_level
def fill_root(self, content, ftfields):
diff -r 6f78ec8a3a59b676d3fe3de5a6f45ef5798e9e37 -r 9e27d3a8e379927a92e4787119bad3c29c37b061 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -727,6 +727,10 @@
cdef class ARTOctreeContainer(OctreeContainer):
+ def __init__(self, *args, **kwargs):
+ self.partial_coverage = 1
+ OctreeContainer.__init__(self, *args, **kwargs)
+
@cython.boundscheck(True)
@cython.wraparound(False)
@cython.cdivision(True)
https://bitbucket.org/yt_analysis/yt-3.0/commits/9d53024f1c04/
Changeset: 9d53024f1c04
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-27 14:25:04
Summary: Beginning unification of NMSU-ART domains.
Affected #: 3 files
diff -r 9e27d3a8e379927a92e4787119bad3c29c37b061 -r 9d53024f1c049fc9e151e068fecc039cf42290c4 yt/frontends/art/data_structures.py
--- a/yt/frontends/art/data_structures.py
+++ b/yt/frontends/art/data_structures.py
@@ -110,9 +110,8 @@
self.parameter_file.domain_dimensions/2, # dd is # of root cells
self.parameter_file.domain_left_edge,
self.parameter_file.domain_right_edge)
- self.domains = [ARTDomainFile(self.parameter_file, l+1, nv, l,
- self.oct_handler)
- for l in range(self.pf.max_level)]
+ self.domains = [ARTDomainFile(self.parameter_file, 0, nv, l,
+ self.oct_handler)]
self.octs_per_domain = [dom.level_count.sum() for dom in self.domains]
self.total_octs = sum(self.octs_per_domain)
mylog.debug("Allocating %s octs", self.total_octs)
@@ -439,8 +438,6 @@
class ARTDomainSubset(OctreeSubset):
def __init__(self, base_region, domain, pf):
super(ARTDomainSubset, self).__init__(base_region, domain, pf)
- self.min_level = domain.domain_level
- self.max_level = domain.domain_level
self.domain_level = domain.domain_level
def fill_root(self, content, ftfields):
@@ -455,7 +452,6 @@
all_fields = self.domain.pf.h.fluid_field_list
fields = [f for ft, f in ftfields]
field_idxs = [all_fields.index(f) for f in fields]
- level = self.domain_level
source = {}
data = _read_root_level(content, self.domain.level_child_offsets,
self.domain.level_count)
@@ -503,11 +499,10 @@
_last_mask = None
_last_seletor_id = None
- def __init__(self, pf, domain_id, nvar, level, oct_handler):
+ def __init__(self, pf, nvar, level, oct_handler):
self.nvar = nvar
self.pf = pf
self.domain_id = domain_id
- self.domain_level = level
self._level_count = None
self._level_oct_offsets = None
self._level_child_offsets = None
@@ -519,7 +514,7 @@
if self._level_count is not None:
return self._level_count
self.level_offsets
- return self._level_count[self.domain_level]
+ return self._level_count
@property
def level_child_offsets(self):
@@ -559,25 +554,25 @@
"""
self.level_offsets
f = open(self.pf._file_amr, "rb")
- level = self.domain_level
- unitary_center, fl, iocts, nocts, root_level = _read_art_level_info(
- f,
- self._level_oct_offsets, level,
- coarse_grid=self.pf.domain_dimensions[0],
- root_level=self.pf.root_level)
- nocts_check = oct_handler.add(self.domain_id, level, unitary_center)
- assert(nocts_check == nocts)
- mylog.debug("Added %07i octs on level %02i, cumulative is %07i",
- nocts, level, oct_handler.nocts)
+ for level in range(self.pf.max_level + 1):
+ unitary_center, fl, iocts, nocts, root_level = \
+ _read_art_level_info( f,
+ self._level_oct_offsets, level,
+ coarse_grid=self.pf.domain_dimensions[0],
+ root_level=self.pf.root_level)
+ nocts_check = oct_handler.add(self.domain_id, level,
+ unitary_center)
+ assert(nocts_check == nocts)
+ mylog.debug("Added %07i octs on level %02i, cumulative is %07i",
+ nocts, level, oct_handler.nocts)
def _read_amr_root(self, oct_handler):
self.level_offsets
f = open(self.pf._file_amr, "rb")
# add the root *cell* not *oct* mesh
- level = self.domain_level
root_octs_side = self.pf.domain_dimensions[0]/2
NX = np.ones(3)*root_octs_side
- octs_side = NX*2**level
+ octs_side = NX*2 # Level == 0
LE = np.array([0.0, 0.0, 0.0], dtype='float64')
RE = np.array([1.0, 1.0, 1.0], dtype='float64')
root_dx = (RE - LE) / NX
@@ -588,8 +583,7 @@
LL[1]:RL[1]:NX[1]*1j,
LL[2]:RL[2]:NX[2]*1j]
root_fc = np.vstack([p.ravel() for p in root_fc]).T
- nocts_check = oct_handler.add(self.domain_id, level,
- root_fc)
+ nocts_check = oct_handler.add(self.domain_id, 0, root_fc)
assert(oct_handler.nocts == root_fc.shape[0])
mylog.debug("Added %07i octs on level %02i, cumulative is %07i",
root_octs_side**3, 0, oct_handler.nocts)
diff -r 9e27d3a8e379927a92e4787119bad3c29c37b061 -r 9d53024f1c049fc9e151e068fecc039cf42290c4 yt/frontends/art/io.py
--- a/yt/frontends/art/io.py
+++ b/yt/frontends/art/io.py
@@ -66,7 +66,7 @@
subset.domain_level, f, d.size, d.min(), d.max(),
cp, cp+d.size)
tr[(ft, f)].append(d)
- cp += subset.cell_count
+ cp += d.size
d = {}
for k in tr.keys():
d[k] = np.concatenate(tr.pop(k))
diff -r 9e27d3a8e379927a92e4787119bad3c29c37b061 -r 9d53024f1c049fc9e151e068fecc039cf42290c4 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -783,9 +783,6 @@
#As a result, source is 3D grid with 8 times as many
#elements as the number of octs on this level in this domain
#and with the shape of an equal-sided cube
- #
- # TODO: Convert to a recrusive function.
- # Note that the .pos[0] etc calls need to be uncommented.
cdef np.ndarray[np.float64_t, ndim=3] source
cdef np.ndarray[np.float64_t, ndim=1] dest
cdef OctVisitorData data
https://bitbucket.org/yt_analysis/yt-3.0/commits/642acd1552bd/
Changeset: 642acd1552bd
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-28 05:09:37
Summary: Attempt at a file-index function.
Affected #: 2 files
diff -r 9d53024f1c049fc9e151e068fecc039cf42290c4 -r 642acd1552bd49a8498995949eefb0b5ccf6262c yt/geometry/oct_visitors.pxd
--- a/yt/geometry/oct_visitors.pxd
+++ b/yt/geometry/oct_visitors.pxd
@@ -60,7 +60,7 @@
cdef oct_visitor_function copy_array_i64
cdef oct_visitor_function identify_octs
cdef oct_visitor_function assign_domain_ind
-cdef oct_visitor_function fill_from_file
+cdef oct_visitor_function fill_file_indices
cdef inline int cind(int i, int j, int k):
return (((i*2)+j)*2+k)
diff -r 9d53024f1c049fc9e151e068fecc039cf42290c4 -r 642acd1552bd49a8498995949eefb0b5ccf6262c yt/geometry/oct_visitors.pyx
--- a/yt/geometry/oct_visitors.pyx
+++ b/yt/geometry/oct_visitors.pyx
@@ -152,10 +152,13 @@
o.domain_ind = data.global_index
data.index += 1
-cdef void fill_from_file(Oct *o, OctVisitorData *data, np.uint8_t selected):
+cdef void fill_file_indices(Oct *o, OctVisitorData *data, np.uint8_t selected):
+ # We fill these arrays, then inside the level filler we use these as
+ # indices as we fill a second array from the data.
if selected == 0: return
- # There are this many records between "octs"
- cdef np.float64_t **p = <np.float64_t**> data.array
- p[1][data.index] = p[0][o.file_ind + oind(data)]
- data.index += 1
-
+ cdef void **p = data.array
+ cdef np.uint8_t *level_arr = <np.uint8_t *> p[0]
+ cdef np.int64_t *find_arr = <np.int64_t *> p[1]
+ level_arr[data.index] = data.level
+ level_arr[data.index] = o.file_ind * 8 + oind(data)
+ data.index +=1
https://bitbucket.org/yt_analysis/yt-3.0/commits/a63a8aa7dcea/
Changeset: a63a8aa7dcea
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-27 14:44:37
Summary: Fix to RAMSES oct container.
Affected #: 1 file
diff -r 9d53024f1c049fc9e151e068fecc039cf42290c4 -r a63a8aa7dcead07db35d0cee61e7c6611ba2fd64 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -681,7 +681,7 @@
return next
def allocate_domains(self, domain_counts, int root_nodes):
- OctreeContainer.allocate_domains(domain_counts)
+ OctreeContainer.allocate_domains(self, domain_counts)
self.root_nodes = <OctKey*> malloc(sizeof(OctKey) * root_nodes)
self.max_root = root_nodes
for i in range(root_nodes):
https://bitbucket.org/yt_analysis/yt-3.0/commits/e8efda0bf32c/
Changeset: e8efda0bf32c
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-28 05:11:16
Summary: Merging
Affected #: 2 files
diff -r a63a8aa7dcead07db35d0cee61e7c6611ba2fd64 -r e8efda0bf32c8250a1f0ebc06704c3ea3c449b1d yt/geometry/oct_visitors.pxd
--- a/yt/geometry/oct_visitors.pxd
+++ b/yt/geometry/oct_visitors.pxd
@@ -60,7 +60,7 @@
cdef oct_visitor_function copy_array_i64
cdef oct_visitor_function identify_octs
cdef oct_visitor_function assign_domain_ind
-cdef oct_visitor_function fill_from_file
+cdef oct_visitor_function fill_file_indices
cdef inline int cind(int i, int j, int k):
return (((i*2)+j)*2+k)
diff -r a63a8aa7dcead07db35d0cee61e7c6611ba2fd64 -r e8efda0bf32c8250a1f0ebc06704c3ea3c449b1d yt/geometry/oct_visitors.pyx
--- a/yt/geometry/oct_visitors.pyx
+++ b/yt/geometry/oct_visitors.pyx
@@ -152,10 +152,13 @@
o.domain_ind = data.global_index
data.index += 1
-cdef void fill_from_file(Oct *o, OctVisitorData *data, np.uint8_t selected):
+cdef void fill_file_indices(Oct *o, OctVisitorData *data, np.uint8_t selected):
+ # We fill these arrays, then inside the level filler we use these as
+ # indices as we fill a second array from the data.
if selected == 0: return
- # There are this many records between "octs"
- cdef np.float64_t **p = <np.float64_t**> data.array
- p[1][data.index] = p[0][o.file_ind + oind(data)]
- data.index += 1
-
+ cdef void **p = data.array
+ cdef np.uint8_t *level_arr = <np.uint8_t *> p[0]
+ cdef np.int64_t *find_arr = <np.int64_t *> p[1]
+ level_arr[data.index] = data.level
+ level_arr[data.index] = o.file_ind * 8 + oind(data)
+ data.index +=1
https://bitbucket.org/yt_analysis/yt-3.0/commits/fe4a5753149b/
Changeset: fe4a5753149b
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-28 06:09:51
Summary: First pass at re-doing RAMSES fluid IO.
Affected #: 4 files
diff -r e8efda0bf32c8250a1f0ebc06704c3ea3c449b1d -r fe4a5753149b223dbf07d03a0d0d6efa6acab2dc yt/frontends/ramses/data_structures.py
--- a/yt/frontends/ramses/data_structures.py
+++ b/yt/frontends/ramses/data_structures.py
@@ -257,17 +257,18 @@
_domain_offset = 1
- def fill(self, content, fields):
+ def fill(self, content, fields, selector):
# Here we get a copy of the file, which we skip through and read the
# bits we want.
oct_handler = self.oct_handler
all_fields = self.domain.pf.h.fluid_field_list
fields = [f for ft, f in fields]
tr = {}
- filled = pos = level_offset = 0
- min_level = self.domain.pf.min_level
+ cell_count = selector.count_oct_cells(self.oct_handler, self.domain_id)
+ levels, cell_inds, file_inds = self.oct_handler.file_index_octs(
+ selector, self.domain_id, cell_count)
for field in fields:
- tr[field] = np.zeros(self.cell_count, 'float64')
+ tr[field] = np.zeros(cell_count, 'float64')
for level, offset in enumerate(self.domain.hydro_offset):
if offset == -1: continue
content.seek(offset)
@@ -278,17 +279,10 @@
for i in range(8):
for field in all_fields:
if field not in fields:
- #print "Skipping %s in %s : %s" % (field, level,
- # self.domain.domain_id)
fpu.skip(content)
else:
- #print "Reading %s in %s : %s" % (field, level,
- # self.domain.domain_id)
temp[field][:,i] = fpu.read_vector(content, 'd') # cell 1
- level_offset += oct_handler.fill_level(self.domain.domain_id, level,
- tr, temp, self.mask, level_offset)
- #print "FILL (%s : %s) %s" % (self.domain.domain_id, level, level_offset)
- #print "DONE (%s) %s of %s" % (self.domain.domain_id, level_offset, self.cell_count)
+ oct_handler.fill_level(level, levels, cell_inds, file_inds, tr, temp)
return tr
class RAMSESGeometryHandler(OctreeGeometryHandler):
diff -r e8efda0bf32c8250a1f0ebc06704c3ea3c449b1d -r fe4a5753149b223dbf07d03a0d0d6efa6acab2dc yt/frontends/ramses/io.py
--- a/yt/frontends/ramses/io.py
+++ b/yt/frontends/ramses/io.py
@@ -39,7 +39,7 @@
# Chunks in this case will have affiliated domain subset objects
# Each domain subset will contain a hydro_offset array, which gives
# pointers to level-by-level hydro information
- tr = dict((f, np.empty(size, dtype='float64')) for f in fields)
+ tr = defaultdict(list)
cp = 0
for chunk in chunks:
for subset in chunk.objs:
@@ -48,14 +48,16 @@
# This contains the boundary information, so we skim through
# and pick off the right vectors
content = cStringIO.StringIO(f.read())
- rv = subset.fill(content, fields)
+ rv = subset.fill(content, fields, selector)
for ft, f in fields:
- mylog.debug("Filling %s with %s (%0.3e %0.3e) (%s:%s)",
- f, subset.cell_count, rv[f].min(), rv[f].max(),
- cp, cp+subset.cell_count)
- tr[(ft, f)][cp:cp+subset.cell_count] = rv.pop(f)
- cp += subset.cell_count
- return tr
+ d = rv.pop(f)
+ mylog.debug("Filling %s with %s (%0.3e %0.3e) (%s zones)",
+ f, d.size, d.min(), d.max(), d.size)
+ tr[(ft, f)].append(d)
+ d = {}
+ for field in fields:
+ d[field] = np.concatenate(tr.pop(field))
+ return d
def _read_particle_selection(self, chunks, selector, fields):
size = 0
diff -r e8efda0bf32c8250a1f0ebc06704c3ea3c449b1d -r fe4a5753149b223dbf07d03a0d0d6efa6acab2dc yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -694,36 +694,53 @@
if self.root_nodes != NULL: free(self.root_nodes)
if self.domains != NULL: free(self.domains)
+ def file_index_octs(self, SelectorObject selector, int domain_id,
+ num_cells = -1):
+ # We create oct arrays of the correct size
+ cdef np.int64_t i
+ cdef np.ndarray[np.uint8_t, ndim=1] levels
+ cdef np.ndarray[np.uint8_t, ndim=1] cell_inds
+ cdef np.ndarray[np.int64_t, ndim=1] file_inds
+ if num_cells < 0:
+ num_cells = selector.count_oct_cells(self, domain_id)
+ levels = np.zeros(num_cells, dtype="uint8")
+ file_inds = np.zeros(num_cells, dtype="int64")
+ cell_inds = np.zeros(num_cells, dtype="uint8")
+ for i in range(num_cells):
+ levels[i] = 100
+ file_inds[i] = -1
+ cell_inds[i] = 9
+ cdef OctVisitorData data
+ data.index = 0
+ cdef void *p[3]
+ p[0] = levels.data
+ p[1] = file_inds.data
+ p[2] = cell_inds.data
+ data.array = p
+ data.domain = domain_id
+ self.visit_all_octs(selector, oct_visitors.fill_file_indices, &data)
+ return levels, cell_inds, file_inds
+
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
- def fill_level(self, int domain, int level, dest_fields, source_fields,
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask, int offset):
+ def fill_level(self, int level,
+ np.ndarray[np.uint8_t, ndim=1] levels,
+ np.ndarray[np.uint8_t, ndim=1] cell_inds,
+ np.ndarray[np.int64_t, ndim=1] file_inds,
+ dest_fields, source_fields):
cdef np.ndarray[np.float64_t, ndim=2] source
cdef np.ndarray[np.float64_t, ndim=1] dest
- cdef OctAllocationContainer *dom = self.domains[domain - 1]
- cdef Oct *o
cdef int n
- cdef int i, j, k, ii
+ cdef int i, di
cdef int local_pos, local_filled
cdef np.float64_t val
for key in dest_fields:
- local_filled = 0
dest = dest_fields[key]
source = source_fields[key]
- for n in range(dom.n):
- o = &dom.my_octs[n]
- for i in range(2):
- for j in range(2):
- for k in range(2):
- ii = ((k*2)+j)*2+i
- if mask[o.domain_ind, ii] == 0: continue
- # TODO: Uncomment this!
- #if o.level == level:
- # dest[local_filled] = \
- # source[o.file_ind, ii]
- local_filled += 1
- return local_filled
+ for i in range(levels.shape[0]):
+ if levels[i] != level: continue
+ dest[i] = source[file_inds[i], cell_inds[i]]
cdef class ARTOctreeContainer(OctreeContainer):
@@ -798,7 +815,7 @@
p[0] = source.data
p[1] = dest.data
data.array = &p
- self.visit_all_octs(selector, oct_visitors.fill_from_file, &data)
+ #self.visit_all_octs(selector, oct_visitors.fill_from_file, &data)
return dest_fields
def allocate_domains(self, domain_counts):
diff -r e8efda0bf32c8250a1f0ebc06704c3ea3c449b1d -r fe4a5753149b223dbf07d03a0d0d6efa6acab2dc yt/geometry/oct_visitors.pyx
--- a/yt/geometry/oct_visitors.pyx
+++ b/yt/geometry/oct_visitors.pyx
@@ -156,9 +156,11 @@
# We fill these arrays, then inside the level filler we use these as
# indices as we fill a second array from the data.
if selected == 0: return
- cdef void **p = data.array
+ cdef void **p = <void **> data.array
cdef np.uint8_t *level_arr = <np.uint8_t *> p[0]
cdef np.int64_t *find_arr = <np.int64_t *> p[1]
+ cdef np.uint8_t *cell_arr = <np.uint8_t *> p[2]
level_arr[data.index] = data.level
- level_arr[data.index] = o.file_ind * 8 + oind(data)
+ find_arr[data.index] = o.file_ind
+ cell_arr[data.index] = rind(data)
data.index +=1
https://bitbucket.org/yt_analysis/yt-3.0/commits/cd025de87f35/
Changeset: cd025de87f35
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-28 06:32:40
Summary: This allows overlapping cell descent to be specified.
Affected #: 2 files
diff -r fe4a5753149b223dbf07d03a0d0d6efa6acab2dc -r cd025de87f35468f467633e2afe81f7f666896dd yt/geometry/selection_routines.pxd
--- a/yt/geometry/selection_routines.pxd
+++ b/yt/geometry/selection_routines.pxd
@@ -30,6 +30,7 @@
cdef class SelectorObject:
cdef public np.int32_t min_level
cdef public np.int32_t max_level
+ cdef int overlap_cells
cdef void recursively_visit_octs(self, Oct *root,
np.float64_t pos[3], np.float64_t dds[3],
diff -r fe4a5753149b223dbf07d03a0d0d6efa6acab2dc -r cd025de87f35468f467633e2afe81f7f666896dd yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -120,6 +120,7 @@
def __cinit__(self, dobj):
self.min_level = getattr(dobj, "min_level", 0)
self.max_level = getattr(dobj, "max_level", 99)
+ self.overlap_cells = 0
@cython.boundscheck(False)
@cython.wraparound(False)
@@ -232,6 +233,8 @@
data.level -= 1
elif this_level == 1:
selected = self.select_cell(spos, sdds, eterm)
+ if ch != NULL:
+ selected *= self.overlap_cells
data.global_index += increment
increment = 0
data.ind[0] = i
@@ -1122,6 +1125,7 @@
def __init__(self, dobj):
self.base_selector = dobj.base_selector
self.domain_id = dobj.domain_id
+ self.overlap_cells = 1
@cython.boundscheck(False)
@cython.wraparound(False)
@@ -1209,7 +1213,7 @@
cdef class AlwaysSelector(SelectorObject):
def __init__(self, dobj):
- pass
+ self.overlap_cells = 1
@cython.boundscheck(False)
@cython.wraparound(False)
https://bitbucket.org/yt_analysis/yt-3.0/commits/dc1968091184/
Changeset: dc1968091184
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-28 18:23:05
Summary: Adding some dependencies, because we changed selection_routines.pxd.
Affected #: 1 file
diff -r cd025de87f35468f467633e2afe81f7f666896dd -r dc1968091184d01910f90916a1c6530617c30480 yt/frontends/artio/setup.py
--- a/yt/frontends/artio/setup.py
+++ b/yt/frontends/artio/setup.py
@@ -16,7 +16,10 @@
include_dirs=["yt/frontends/artio/artio_headers/",
"yt/geometry/",
"yt/utilities/lib/"],
- depends=artio_sources)
+ depends=artio_sources +
+ ["yt/utilities/lib/fp_utils.pxd",
+ "yt/geometry/oct_container.pxd",
+ "yt/geometry/selection_routines.pxd"])
config.make_config_py() # installs __config__.py
#config.make_svn_version_py()
return config
https://bitbucket.org/yt_analysis/yt-3.0/commits/e2dbd707505b/
Changeset: e2dbd707505b
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-28 19:01:23
Summary: Further development of NMSU-ART file reading and IO.
Affected #: 3 files
diff -r dc1968091184d01910f90916a1c6530617c30480 -r e2dbd707505b656f4c9e81ca942d81bb80e99f43 yt/frontends/art/data_structures.py
--- a/yt/frontends/art/data_structures.py
+++ b/yt/frontends/art/data_structures.py
@@ -110,17 +110,16 @@
self.parameter_file.domain_dimensions/2, # dd is # of root cells
self.parameter_file.domain_left_edge,
self.parameter_file.domain_right_edge)
- self.domains = [ARTDomainFile(self.parameter_file, 0, nv, l,
- self.oct_handler)]
+ # The 1 here refers to domain_id == 1 always for ARTIO.
+ self.domains = [ARTDomainFile(self.parameter_file, nv,
+ self.oct_handler, 1)]
self.octs_per_domain = [dom.level_count.sum() for dom in self.domains]
self.total_octs = sum(self.octs_per_domain)
mylog.debug("Allocating %s octs", self.total_octs)
self.oct_handler.allocate_domains(self.octs_per_domain)
- for domain in self.domains:
- if domain.domain_level == 0:
- domain._read_amr_root(self.oct_handler)
- else:
- domain._read_amr_level(self.oct_handler)
+ domain = self.domains[0]
+ domain._read_amr_root(self.oct_handler)
+ domain._read_amr_level(self.oct_handler)
def _detect_fields(self):
self.particle_field_list = particle_fields
@@ -436,11 +435,8 @@
return False
class ARTDomainSubset(OctreeSubset):
- def __init__(self, base_region, domain, pf):
- super(ARTDomainSubset, self).__init__(base_region, domain, pf)
- self.domain_level = domain.domain_level
- def fill_root(self, content, ftfields):
+ def fill(self, content, ftfields, selector):
"""
This is called from IOHandler. It takes content
which is a binary stream, reads the requested field
@@ -452,42 +448,34 @@
all_fields = self.domain.pf.h.fluid_field_list
fields = [f for ft, f in ftfields]
field_idxs = [all_fields.index(f) for f in fields]
- source = {}
+ tr = {}
+ cell_count = selector.count_oct_cells(self.oct_handler, self.domain_id)
+ levels, cell_inds, file_inds = self.oct_handler.file_index_octs(
+ selector, self.domain_id, cell_count)
+ for field in fields:
+ tr[field] = np.zeros(cell_count, 'float64')
data = _read_root_level(content, self.domain.level_child_offsets,
self.domain.level_count)
-
for field, i in zip(fields, field_idxs):
temp = np.reshape(data[i, :], self.domain.pf.domain_dimensions,
- order='F').astype('float64').T
- source[field] = temp
- dest = oct_handler.fill_level_from_grid(
- self.selector, self.domain_id, source)
- return dest
-
- def fill_level(self, content, ftfields):
- oct_handler = self.oct_handler
- fields = [f for ft, f in ftfields]
- level_offset = 0
- dest = {}
- for field in fields:
- dest[field] = np.zeros(self.cell_count, 'float64')-1.
- level = self.domain_level
- no = self.domain.level_count[level]
- noct_range = [0, no]
- source = _read_child_level(
- content, self.domain.level_child_offsets,
- self.domain.level_offsets,
- self.domain.level_count, level, fields,
- self.domain.pf.domain_dimensions,
- self.domain.pf.parameters['ncell0'],
- noct_range=noct_range)
- nocts_filling = noct_range[1]-noct_range[0]
- level_offset += oct_handler.fill_level(self.domain.domain_id,
- level, dest, source,
- self.mask, level_offset,
- noct_range[0],
- nocts_filling)
- return dest
+ order='F')
+ # Need it to be ordered correctly; this is expensive, though ...
+ oct_handler.fill_level(0, levels, cell_inds, file_inds, tr, temp)
+ # Now we continue with the additional levels.
+ for level in range(1, self.pf.max_level + 1):
+ no = self.domain.level_count[level]
+ noct_range = [0, no]
+ source = _read_child_level(
+ content, self.domain.level_child_offsets,
+ self.domain.level_offsets,
+ self.domain.level_count, level, fields,
+ self.domain.pf.domain_dimensions,
+ self.domain.pf.parameters['ncell0'],
+ noct_range=noct_range)
+ for field, i in zip(fields, field_idxs):
+ # Need it to be ordered correctly; this is expensive, though ...
+ oct_handler.fill_level(level, levels, cell_inds, file_inds, tr, temp)
+ return tr
class ARTDomainFile(object):
"""
@@ -499,7 +487,7 @@
_last_mask = None
_last_seletor_id = None
- def __init__(self, pf, nvar, level, oct_handler):
+ def __init__(self, pf, nvar, oct_handler, domain_id):
self.nvar = nvar
self.pf = pf
self.domain_id = domain_id
@@ -554,7 +542,7 @@
"""
self.level_offsets
f = open(self.pf._file_amr, "rb")
- for level in range(self.pf.max_level + 1):
+ for level in range(1, self.pf.max_level + 1):
unitary_center, fl, iocts, nocts, root_level = \
_read_art_level_info( f,
self._level_oct_offsets, level,
diff -r dc1968091184d01910f90916a1c6530617c30480 -r e2dbd707505b656f4c9e81ca942d81bb80e99f43 yt/frontends/art/io.py
--- a/yt/frontends/art/io.py
+++ b/yt/frontends/art/io.py
@@ -56,10 +56,7 @@
f = open(subset.domain.pf._file_amr, "rb")
# This contains the boundary information, so we skim through
# and pick off the right vectors
- if subset.domain_level == 0:
- rv = subset.fill_root(f, fields)
- else:
- rv = subset.fill_level(f, fields)
+ rv = subset.fill(f, fields, selector)
for ft, f in fields:
d = rv.pop(f)
mylog.debug("Filling L%i %s with %s (%0.3e %0.3e) (%s:%s)",
@@ -68,8 +65,8 @@
tr[(ft, f)].append(d)
cp += d.size
d = {}
- for k in tr.keys():
- d[k] = np.concatenate(tr.pop(k))
+ for field in fields:
+ d[field] = np.concatenate(tr.pop(field))
return d
def _read_particle_selection(self, chunks, selector, fields):
diff -r dc1968091184d01910f90916a1c6530617c30480 -r e2dbd707505b656f4c9e81ca942d81bb80e99f43 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -571,6 +571,55 @@
self.nocts += 1
return next
+ def file_index_octs(self, SelectorObject selector, int domain_id,
+ num_cells = -1):
+ # We create oct arrays of the correct size
+ cdef np.int64_t i
+ cdef np.ndarray[np.uint8_t, ndim=1] levels
+ cdef np.ndarray[np.uint8_t, ndim=1] cell_inds
+ cdef np.ndarray[np.int64_t, ndim=1] file_inds
+ if num_cells < 0:
+ num_cells = selector.count_oct_cells(self, domain_id)
+ levels = np.zeros(num_cells, dtype="uint8")
+ file_inds = np.zeros(num_cells, dtype="int64")
+ cell_inds = np.zeros(num_cells, dtype="uint8")
+ for i in range(num_cells):
+ levels[i] = 100
+ file_inds[i] = -1
+ cell_inds[i] = 9
+ cdef OctVisitorData data
+ data.index = 0
+ cdef void *p[3]
+ p[0] = levels.data
+ p[1] = file_inds.data
+ p[2] = cell_inds.data
+ data.array = p
+ data.domain = domain_id
+ self.visit_all_octs(selector, oct_visitors.fill_file_indices, &data)
+ return levels, cell_inds, file_inds
+
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ def fill_level(self, int level,
+ np.ndarray[np.uint8_t, ndim=1] levels,
+ np.ndarray[np.uint8_t, ndim=1] cell_inds,
+ np.ndarray[np.int64_t, ndim=1] file_inds,
+ dest_fields, source_fields):
+ cdef np.ndarray[np.float64_t, ndim=2] source
+ cdef np.ndarray[np.float64_t, ndim=1] dest
+ cdef int n
+ cdef int i, di
+ cdef int local_pos, local_filled
+ cdef np.float64_t val
+ for key in dest_fields:
+ dest = dest_fields[key]
+ source = source_fields[key]
+ for i in range(levels.shape[0]):
+ if levels[i] != level: continue
+ dest[i] = source[file_inds[i], cell_inds[i]]
+
+
cdef int root_node_compare(void *a, void *b) nogil:
cdef OctKey *ao, *bo
ao = <OctKey *>a
@@ -694,130 +743,12 @@
if self.root_nodes != NULL: free(self.root_nodes)
if self.domains != NULL: free(self.domains)
- def file_index_octs(self, SelectorObject selector, int domain_id,
- num_cells = -1):
- # We create oct arrays of the correct size
- cdef np.int64_t i
- cdef np.ndarray[np.uint8_t, ndim=1] levels
- cdef np.ndarray[np.uint8_t, ndim=1] cell_inds
- cdef np.ndarray[np.int64_t, ndim=1] file_inds
- if num_cells < 0:
- num_cells = selector.count_oct_cells(self, domain_id)
- levels = np.zeros(num_cells, dtype="uint8")
- file_inds = np.zeros(num_cells, dtype="int64")
- cell_inds = np.zeros(num_cells, dtype="uint8")
- for i in range(num_cells):
- levels[i] = 100
- file_inds[i] = -1
- cell_inds[i] = 9
- cdef OctVisitorData data
- data.index = 0
- cdef void *p[3]
- p[0] = levels.data
- p[1] = file_inds.data
- p[2] = cell_inds.data
- data.array = p
- data.domain = domain_id
- self.visit_all_octs(selector, oct_visitors.fill_file_indices, &data)
- return levels, cell_inds, file_inds
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
- def fill_level(self, int level,
- np.ndarray[np.uint8_t, ndim=1] levels,
- np.ndarray[np.uint8_t, ndim=1] cell_inds,
- np.ndarray[np.int64_t, ndim=1] file_inds,
- dest_fields, source_fields):
- cdef np.ndarray[np.float64_t, ndim=2] source
- cdef np.ndarray[np.float64_t, ndim=1] dest
- cdef int n
- cdef int i, di
- cdef int local_pos, local_filled
- cdef np.float64_t val
- for key in dest_fields:
- dest = dest_fields[key]
- source = source_fields[key]
- for i in range(levels.shape[0]):
- if levels[i] != level: continue
- dest[i] = source[file_inds[i], cell_inds[i]]
-
cdef class ARTOctreeContainer(OctreeContainer):
def __init__(self, *args, **kwargs):
self.partial_coverage = 1
OctreeContainer.__init__(self, *args, **kwargs)
- @cython.boundscheck(True)
- @cython.wraparound(False)
- @cython.cdivision(True)
- def fill_level(self, int domain, int level, dest_fields, source_fields,
- np.ndarray[np.uint8_t, ndim=2, cast=True] mask, int offset,
- np.int64_t subchunk_offset, np.int64_t subchunk_max):
- #Only minorly different from the RAMSES version
- #The source array is in chunks, just stop when we hit the end
- cdef np.ndarray[np.float64_t, ndim=2] source
- cdef np.ndarray[np.float64_t, ndim=1] dest
- cdef OctAllocationContainer *dom = self.domains[domain - 1]
- cdef Oct *o
- cdef int n
- cdef int i, j, k, ii
- cdef int local_pos, local_filled
- cdef np.float64_t val
- cdef np.int64_t index
- for key in dest_fields:
- local_filled = 0
- dest = dest_fields[key]
- source = source_fields[key]
- for n in range(dom.n):
- o = &dom.my_octs[n]
- index = o.file_ind-subchunk_offset
- # TODO: Uncomment this!
- #if o.level != level: continue
- if index < 0: continue
- if index >= subchunk_max:
- #if we hit the end of the array,
- #immeditely discontinue
- return local_filled
- for i in range(2):
- for j in range(2):
- for k in range(2):
- ii = ((k*2)+j)*2+i
- if mask[o.domain_ind, ii] == 0: continue
- dest[local_filled + offset] = \
- source[index,ii]
- local_filled += 1
- return local_filled
-
- @cython.boundscheck(False)
- @cython.wraparound(False)
- @cython.cdivision(True)
- def fill_level_from_grid(self, SelectorObject selector,
- int domain_id, source_fields):
- #Fill level, but instead of assuming that the source
- #order is that of the oct order, we look up the oct position
- #and fill its children from the the source field
- #As a result, source is 3D grid with 8 times as many
- #elements as the number of octs on this level in this domain
- #and with the shape of an equal-sided cube
- cdef np.ndarray[np.float64_t, ndim=3] source
- cdef np.ndarray[np.float64_t, ndim=1] dest
- cdef OctVisitorData data
- cdef void *p[2]
- num_cells = selector.count_oct_cells(self, domain_id)
- dest_fields = {}
- for key in source_fields:
- dest_fields[key] = dest = \
- np.zeros(num_cells, dtype="float64")
- data.index = 0
- local_filled = 0
- source = source_fields[key]
- p[0] = source.data
- p[1] = dest.data
- data.array = &p
- #self.visit_all_octs(selector, oct_visitors.fill_from_file, &data)
- return dest_fields
-
def allocate_domains(self, domain_counts):
cdef int count, i
cdef OctAllocationContainer *cur = self.cont
https://bitbucket.org/yt_analysis/yt-3.0/commits/90fe041ad494/
Changeset: 90fe041ad494
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-28 23:28:15
Summary: Continuing to convert NMSU-ART
Affected #: 3 files
diff -r e2dbd707505b656f4c9e81ca942d81bb80e99f43 -r 90fe041ad4942ce1e6b82055875d1e5d1f7fe915 yt/frontends/art/data_structures.py
--- a/yt/frontends/art/data_structures.py
+++ b/yt/frontends/art/data_structures.py
@@ -120,6 +120,7 @@
domain = self.domains[0]
domain._read_amr_root(self.oct_handler)
domain._read_amr_level(self.oct_handler)
+ self.oct_handler.finalize()
def _detect_fields(self):
self.particle_field_list = particle_fields
@@ -448,7 +449,7 @@
all_fields = self.domain.pf.h.fluid_field_list
fields = [f for ft, f in ftfields]
field_idxs = [all_fields.index(f) for f in fields]
- tr = {}
+ source, tr = {}, {}
cell_count = selector.count_oct_cells(self.oct_handler, self.domain_id)
levels, cell_inds, file_inds = self.oct_handler.file_index_octs(
selector, self.domain_id, cell_count)
@@ -456,11 +457,14 @@
tr[field] = np.zeros(cell_count, 'float64')
data = _read_root_level(content, self.domain.level_child_offsets,
self.domain.level_count)
+ ns = (8, self.domain.pf.domain_dimensions.prod() / 8)
for field, i in zip(fields, field_idxs):
- temp = np.reshape(data[i, :], self.domain.pf.domain_dimensions,
- order='F')
+ source[field] = data[i, :]
+ source[field].shape = ns
+ source[field] = np.array(source[field], dtype="float64", order='F')
# Need it to be ordered correctly; this is expensive, though ...
- oct_handler.fill_level(0, levels, cell_inds, file_inds, tr, temp)
+ oct_handler.fill_level(0, levels, cell_inds, file_inds, tr, source)
+ del source
# Now we continue with the additional levels.
for level in range(1, self.pf.max_level + 1):
no = self.domain.level_count[level]
@@ -472,9 +476,8 @@
self.domain.pf.domain_dimensions,
self.domain.pf.parameters['ncell0'],
noct_range=noct_range)
- for field, i in zip(fields, field_idxs):
- # Need it to be ordered correctly; this is expensive, though ...
- oct_handler.fill_level(level, levels, cell_inds, file_inds, tr, temp)
+ oct_handler.fill_level(level, levels, cell_inds, file_inds, tr,
+ source)
return tr
class ARTDomainFile(object):
diff -r e2dbd707505b656f4c9e81ca942d81bb80e99f43 -r 90fe041ad4942ce1e6b82055875d1e5d1f7fe915 yt/frontends/art/io.py
--- a/yt/frontends/art/io.py
+++ b/yt/frontends/art/io.py
@@ -59,8 +59,8 @@
rv = subset.fill(f, fields, selector)
for ft, f in fields:
d = rv.pop(f)
- mylog.debug("Filling L%i %s with %s (%0.3e %0.3e) (%s:%s)",
- subset.domain_level, f, d.size, d.min(), d.max(),
+ mylog.debug("Filling %s with %s (%0.3e %0.3e) (%s:%s)",
+ f, d.size, d.min(), d.max(),
cp, cp+d.size)
tr[(ft, f)].append(d)
cp += d.size
diff -r e2dbd707505b656f4c9e81ca942d81bb80e99f43 -r 90fe041ad4942ce1e6b82055875d1e5d1f7fe915 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -161,7 +161,8 @@
for j in range(self.nn[1]):
pos[2] = self.DLE[2] + dds[2]/2.0
for k in range(self.nn[2]):
- if self.root_mesh[i][j][k] == NULL: continue
+ if self.root_mesh[i][j][k] == NULL:
+ raise RuntimeError
data.pos[0] = i
data.pos[1] = j
data.pos[2] = k
@@ -619,6 +620,13 @@
if levels[i] != level: continue
dest[i] = source[file_inds[i], cell_inds[i]]
+ def finalize(self):
+ cdef SelectorObject selector = selection_routines.AlwaysSelector(None)
+ cdef OctVisitorData data
+ data.index = 0
+ data.domain = 1
+ self.visit_all_octs(selector, oct_visitors.assign_domain_ind, &data)
+ assert ((data.global_index+1)*8 == data.index)
cdef int root_node_compare(void *a, void *b) nogil:
cdef OctKey *ao, *bo
@@ -649,14 +657,6 @@
self.DLE[i] = domain_left_edge[i] #0
self.DRE[i] = domain_right_edge[i] #num_grid
- def finalize(self):
- cdef SelectorObject selector = selection_routines.AlwaysSelector(None)
- cdef OctVisitorData data
- data.index = 0
-
- self.visit_all_octs(selector, oct_visitors.assign_domain_ind, &data)
- assert ((data.global_index+1)*8 == data.index)
-
cdef int get_root(self, int ind[3], Oct **o):
o[0] = NULL
cdef int i
https://bitbucket.org/yt_analysis/yt-3.0/commits/ade63d627a80/
Changeset: ade63d627a80
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-28 23:50:31
Summary: Merge
Affected #: 14 files
diff -r 90fe041ad4942ce1e6b82055875d1e5d1f7fe915 -r ade63d627a80d85174235e0bc8af16b71cd74f74 yt/data_objects/construction_data_containers.py
--- a/yt/data_objects/construction_data_containers.py
+++ b/yt/data_objects/construction_data_containers.py
@@ -281,7 +281,7 @@
chunk_fields.append(self.weight_field)
tree = self._get_tree(len(fields))
# We do this once
- for chunk in self.data_source.chunks(None, "io"):
+ for chunk in self.data_source.chunks([], "io"):
self._initialize_chunk(chunk, tree)
# This needs to be parallel_objects-ified
for chunk in parallel_objects(self.data_source.chunks(
@@ -429,6 +429,7 @@
self._data_source.max_level = self.level
def get_data(self, fields = None):
+ if fields is None: return
fields = self._determine_fields(ensure_list(fields))
fields_to_get = [f for f in fields if f not in self.field_data]
fields_to_get = self._identify_dependencies(fields_to_get)
diff -r 90fe041ad4942ce1e6b82055875d1e5d1f7fe915 -r ade63d627a80d85174235e0bc8af16b71cd74f74 yt/data_objects/particle_fields.py
--- a/yt/data_objects/particle_fields.py
+++ b/yt/data_objects/particle_fields.py
@@ -46,6 +46,7 @@
pos = data[ptype, coord_name]
d = data.deposit(pos, method = "count")
return d
+
registry.add_field(("deposit", "%s_count" % ptype),
function = particle_count,
validators = [ValidateSpatial()],
diff -r 90fe041ad4942ce1e6b82055875d1e5d1f7fe915 -r ade63d627a80d85174235e0bc8af16b71cd74f74 yt/data_objects/universal_fields.py
--- a/yt/data_objects/universal_fields.py
+++ b/yt/data_objects/universal_fields.py
@@ -32,7 +32,7 @@
from yt.funcs import *
-from yt.utilities.lib import CICDeposit_3, obtain_rvec, obtain_rv_vec
+from yt.utilities.lib import obtain_rvec, obtain_rv_vec
from yt.utilities.cosmology import Cosmology
from field_info_container import \
add_field, \
@@ -395,7 +395,7 @@
convert_function=_convertCellMassCode)
def _TotalMass(field,data):
- return (data["Density"]+data["Dark_Matter_Density"]) * data["CellVolume"]
+ return (data["Density"]+data[("deposit", "particle_density")]) * data["CellVolume"]
add_field("TotalMass", function=_TotalMass, units=r"\rm{g}")
add_field("TotalMassMsun", units=r"M_{\odot}",
function=_TotalMass,
@@ -940,19 +940,17 @@
add_field("JeansMassMsun",function=_JeansMassMsun,
units=r"\rm{M_{\odot}}")
+# We add these fields so that the field detector can use them
+for field in ["particle_position_%s" % ax for ax in "xyz"]:
+ # This marker should let everyone know not to use the fields, but NullFunc
+ # should do that, too.
+ add_field(field, function=NullFunc, particle_type = True,
+ units=r"UNDEFINED")
+
def _pdensity(field, data):
- blank = np.zeros(data.ActiveDimensions, dtype='float64')
- if data["particle_position_x"].size == 0: return blank
- CICDeposit_3(data["particle_position_x"].astype(np.float64),
- data["particle_position_y"].astype(np.float64),
- data["particle_position_z"].astype(np.float64),
- data["ParticleMass"],
- data["particle_position_x"].size,
- blank, np.array(data.LeftEdge).astype(np.float64),
- np.array(data.ActiveDimensions).astype(np.int32),
- just_one(data['dx']))
- np.divide(blank, data["CellVolume"], blank)
- return blank
+ pmass = data[('deposit','all_mass')]
+ np.divide(pmass, data["CellVolume"], pmass)
+ return pmass
add_field("particle_density", function=_pdensity,
validators=[ValidateGridType()],
display_name=r"\mathrm{Particle}\/\mathrm{Density}")
diff -r 90fe041ad4942ce1e6b82055875d1e5d1f7fe915 -r ade63d627a80d85174235e0bc8af16b71cd74f74 yt/frontends/art/data_structures.py
--- a/yt/frontends/art/data_structures.py
+++ b/yt/frontends/art/data_structures.py
@@ -124,9 +124,9 @@
def _detect_fields(self):
self.particle_field_list = particle_fields
- self.field_list = set(fluid_fields + particle_fields +
- particle_star_fields)
- self.field_list = list(self.field_list)
+ self.field_list = [("gas", f) for f in fluid_fields]
+ self.field_list += set(particle_fields + particle_star_fields \
+ + fluid_fields)
# now generate all of the possible particle fields
if "wspecies" in self.parameter_file.parameters.keys():
wspecies = self.parameter_file.parameters['wspecies']
@@ -136,6 +136,10 @@
self.parameter_file.particle_types.append("specie%i" % specie)
else:
self.parameter_file.particle_types = []
+ for ptype in self.parameter_file.particle_types:
+ for pfield in self.particle_field_list:
+ pfn = (ptype, pfield)
+ self.field_list.append(pfn)
def _setup_classes(self):
dd = self._get_data_reader_dict()
diff -r 90fe041ad4942ce1e6b82055875d1e5d1f7fe915 -r ade63d627a80d85174235e0bc8af16b71cd74f74 yt/frontends/art/fields.py
--- a/yt/frontends/art/fields.py
+++ b/yt/frontends/art/fields.py
@@ -25,6 +25,8 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
import numpy as np
+
+from yt.funcs import *
from yt.data_objects.field_info_container import \
FieldInfoContainer, \
FieldInfo, \
@@ -40,6 +42,10 @@
from yt.utilities.physical_constants import mass_sun_cgs
from yt.frontends.art.definitions import *
+from yt.data_objects.particle_fields import \
+ particle_deposition_functions, \
+ particle_vector_functions
+
KnownARTFields = FieldInfoContainer()
add_art_field = KnownARTFields.add_field
ARTFieldInfo = FieldInfoContainer.create_with_fallback(FieldInfo)
@@ -218,6 +224,7 @@
particle_type=True,
convert_function=lambda x: x.convert("particle_mass"))
+
def _particle_age(field, data):
tr = data["particle_creation_time"]
return data.pf.current_time - tr
@@ -260,3 +267,55 @@
return data["particle_mass"]/mass_sun_cgs
add_field("ParticleMassMsun", function=_ParticleMassMsun, particle_type=True,
take_log=True, units=r"\rm{Msun}")
+
+# Particle Deposition Fields
+_ptypes = ["all", "darkmatter", "stars", "specie0"]
+
+for _ptype in _ptypes:
+ particle_vector_functions(_ptype, ["particle_position_%s" % ax for ax in 'xyz'],
+ ["particle_velocity_%s" % ax for ax in 'xyz'],
+ ARTFieldInfo)
+ particle_deposition_functions(_ptype, "Coordinates", "particle_mass",
+ ARTFieldInfo)
+
+# Mixed Fluid-Particle Fields
+
+def baryon_density(field, data):
+ rho = data["deposit", "stars_density"]
+ rho += data["gas", "Density"]
+ return rho
+
+ARTFieldInfo.add_field(("deposit", "baryon_density"),
+ function = baryon_density,
+ validators = [ValidateSpatial()],
+ display_name = "\\mathrm{Baryon Density}",
+ units = r"\mathrm{g}/\mathrm{cm}^{3}",
+ projected_units = r"\mathrm{g}/\mathrm{cm}^{2}",
+ projection_conversion = 'cm')
+
+def total_density(field, data):
+ rho = data["deposit", "baryon_density"]
+ rho += data["deposit", "specie0_density"]
+ return rho
+
+ARTFieldInfo.add_field(("deposit", "total_density"),
+ function = total_density,
+ validators = [ValidateSpatial()],
+ display_name = "\\mathrm{Total Density}",
+ units = r"\mathrm{g}/\mathrm{cm}^{3}",
+ projected_units = r"\mathrm{g}/\mathrm{cm}^{2}",
+ projection_conversion = 'cm')
+
+def multimass_density(field, data):
+ rho = data["deposit", "baryon_density"]
+ rho += data["deposit", "darkmatter_density"]
+ return rho
+
+ARTFieldInfo.add_field(("deposit", "multimass_density"),
+ function = multimass_density,
+ validators = [ValidateSpatial()],
+ display_name = "\\mathrm{Multimass Density}",
+ units = r"\mathrm{g}/\mathrm{cm}^{3}",
+ projected_units = r"\mathrm{g}/\mathrm{cm}^{2}",
+ projection_conversion = 'cm')
+
diff -r 90fe041ad4942ce1e6b82055875d1e5d1f7fe915 -r ade63d627a80d85174235e0bc8af16b71cd74f74 yt/frontends/art/io.py
--- a/yt/frontends/art/io.py
+++ b/yt/frontends/art/io.py
@@ -43,6 +43,14 @@
class IOHandlerART(BaseIOHandler):
_data_style = "art"
tb, ages = None, None
+ cache = None
+ masks = None
+ caching = True
+
+ def __init__(self):
+ self.cache = {}
+ self.masks = {}
+ super(IOHandlerART, self).__init__()
def _read_fluid_selection(self, chunks, selector, fields, size):
# Chunks in this case will have affiliated domain subset objects
@@ -69,85 +77,103 @@
d[field] = np.concatenate(tr.pop(field))
return d
- def _read_particle_selection(self, chunks, selector, fields):
+ def _get_mask(self, selector, ftype):
+ key = (selector, ftype)
+ if key in self.masks.keys() and self.caching:
+ return self.masks[key]
+ pf = self.pf
+ ptmax = self.ws[-1]
+ pbool, idxa, idxb = _determine_field_size(pf, ftype, self.ls, ptmax)
+ pstr = 'particle_position_%s'
+ x,y,z = [self._get_field((ftype, pstr % ax)) for ax in 'xyz']
+ mask = selector.select_points(x, y, z)
+ if self.caching:
+ self.masks[key] = mask
+ return self.masks[key]
+ else:
+ return mask
+
+ def _get_field(self, field):
+ if field in self.cache.keys() and self.caching:
+ mylog.debug("Cached %s", str(field))
+ return self.cache[field]
+ mylog.debug("Reading %s", str(field))
tr = {}
- fields_read = []
- for chunk in chunks:
- level = chunk.objs[0].domain.domain_level
- pf = chunk.objs[0].domain.pf
- masks = {}
- ws, ls = pf.parameters["wspecies"], pf.parameters["lspecies"]
- sizes = np.diff(np.concatenate(([0], ls)))
- ptmax = ws[-1]
- npt = ls[-1]
- nstars = ls[-1]-ls[-2]
- file_particle = pf._file_particle_data
- file_stars = pf._file_particle_stars
- ftype_old = None
- for field in fields:
- if field in fields_read:
- continue
- ftype, fname = field
- pbool, idxa, idxb = _determine_field_size(pf, ftype, ls, ptmax)
- npa = idxb-idxa
- if not ftype_old == ftype:
- Nrow = pf.parameters["Nrow"]
- rp = lambda ax: read_particles(
- file_particle, Nrow, idxa=idxa,
- idxb=idxb, field=ax)
- x, y, z = (rp(ax) for ax in 'xyz')
- dd = pf.domain_dimensions[0]
- off = 1.0/dd
- x, y, z = (t.astype('f8')/dd - off for t in (x, y, z))
- mask = selector.select_points(x, y, z)
- size = mask.sum()
- for i, ax in enumerate('xyz'):
- if fname.startswith("particle_position_%s" % ax):
- tr[field] = vars()[ax]
- if fname.startswith("particle_velocity_%s" % ax):
- tr[field] = rp('v'+ax)
- if fname == "particle_mass":
- a = 0
- data = np.zeros(npa, dtype='f8')
- for ptb, size, m in zip(pbool, sizes, ws):
- if ptb:
- data[a:a+size] = m
- a += size
- tr[field] = data
- elif fname == "particle_index":
- tr[field] = np.arange(idxa, idxb).astype('int64')
- elif fname == "particle_type":
- a = 0
- data = np.zeros(npa, dtype='int')
- for i, (ptb, size) in enumerate(zip(pbool, sizes)):
- if ptb:
- data[a:a+size] = i
- a += size
- tr[field] = data
- if pbool[-1] and fname in particle_star_fields:
- data = read_star_field(file_stars, field=fname)
- temp = tr.get(field, np.zeros(npa, 'f8'))
- if nstars > 0:
- temp[-nstars:] = data
- tr[field] = temp
- if fname == "particle_creation_time":
- self.tb, self.ages, data = interpolate_ages(
- tr[field][-nstars:],
- file_stars,
- self.tb,
- self.ages,
- pf.current_time)
- temp = tr.get(field, np.zeros(npa, 'f8'))
- temp[-nstars:] = data
- tr[field] = temp
- del data
- tr[field] = tr[field][mask].astype('f8')
- ftype_old = ftype
- fields_read.append(field)
+ ftype, fname = field
+ ptmax = self.ws[-1]
+ pbool, idxa, idxb = _determine_field_size(self.pf, ftype,
+ self.ls, ptmax)
+ npa = idxb - idxa
+ sizes = np.diff(np.concatenate(([0], self.ls)))
+ rp = lambda ax: read_particles(
+ self.file_particle, self.Nrow, idxa=idxa,
+ idxb=idxb, fields=ax)
+ for i, ax in enumerate('xyz'):
+ if fname.startswith("particle_position_%s" % ax):
+ dd = self.pf.domain_dimensions[0]
+ off = 1.0/dd
+ tr[field] = rp([ax])[0]/dd - off
+ if fname.startswith("particle_velocity_%s" % ax):
+ tr[field], = rp(['v'+ax])
+ if fname == "particle_mass":
+ a = 0
+ data = np.zeros(npa, dtype='f8')
+ for ptb, size, m in zip(pbool, sizes, self.ws):
+ if ptb:
+ data[a:a+size] = m
+ a += size
+ tr[field] = data
+ elif fname == "particle_index":
+ tr[field] = np.arange(idxa, idxb)
+ elif fname == "particle_type":
+ a = 0
+ data = np.zeros(npa, dtype='int')
+ for i, (ptb, size) in enumerate(zip(pbool, sizes)):
+ if ptb:
+ data[a: a + size] = i
+ a += size
+ tr[field] = data
+ if pbool[-1] and fname in particle_star_fields:
+ data = read_star_field(self.file_stars, field=fname)
+ temp = tr.get(field, np.zeros(npa, 'f8'))
+ nstars = self.ls[-1]-self.ls[-2]
+ if nstars > 0:
+ temp[-nstars:] = data
+ tr[field] = temp
+ if fname == "particle_creation_time":
+ self.tb, self.ages, data = interpolate_ages(
+ tr[field][-nstars:],
+ self.file_stars,
+ self.tb,
+ self.ages,
+ self.pf.current_time)
+ temp = tr.get(field, np.zeros(npa, 'f8'))
+ temp[-nstars:] = data
+ tr[field] = temp
+ del data
if tr == {}:
tr = dict((f, np.array([])) for f in fields)
- return tr
+ if self.caching:
+ self.cache[field] = tr[field]
+ return self.cache[field]
+ else:
+ return tr[field]
+ def _read_particle_selection(self, chunks, selector, fields):
+ chunk = chunks.next()
+ self.pf = chunk.objs[0].domain.pf
+ self.ws = self.pf.parameters["wspecies"]
+ self.ls = self.pf.parameters["lspecies"]
+ self.file_particle = self.pf._file_particle_data
+ self.file_stars = self.pf._file_particle_stars
+ self.Nrow = self.pf.parameters["Nrow"]
+ data = {f:np.array([]) for f in fields}
+ for f in fields:
+ ftype, fname = f
+ mask = self._get_mask(selector, ftype)
+ arr = self._get_field(f)[mask].astype('f8')
+ data[f] = np.concatenate((arr, data[f]))
+ return data
def _determine_field_size(pf, field, lspecies, ptmax):
pbool = np.zeros(len(lspecies), dtype="bool")
@@ -362,27 +388,29 @@
return ranges
-def read_particles(file, Nrow, idxa, idxb, field):
+def read_particles(file, Nrow, idxa, idxb, fields):
words = 6 # words (reals) per particle: x,y,z,vx,vy,vz
real_size = 4 # for file_particle_data; not always true?
np_per_page = Nrow**2 # defined in ART a_setup.h, # of particles/page
num_pages = os.path.getsize(file)/(real_size*words*np_per_page)
- data = np.array([], 'f4')
fh = open(file, 'r')
skip, count = idxa, idxb - idxa
kwargs = dict(words=words, real_size=real_size,
np_per_page=np_per_page, num_pages=num_pages)
- ranges = get_ranges(skip, count, field, **kwargs)
- data = None
- for seek, this_count in ranges:
- fh.seek(seek)
- temp = np.fromfile(fh, count=this_count, dtype='>f4')
- if data is None:
- data = temp
- else:
- data = np.concatenate((data, temp))
+ arrs = []
+ for field in fields:
+ ranges = get_ranges(skip, count, field, **kwargs)
+ data = None
+ for seek, this_count in ranges:
+ fh.seek(seek)
+ temp = np.fromfile(fh, count=this_count, dtype='>f4')
+ if data is None:
+ data = temp
+ else:
+ data = np.concatenate((data, temp))
+ arrs.append(data.astype('f8'))
fh.close()
- return data
+ return arrs
def read_star_field(file, field=None):
diff -r 90fe041ad4942ce1e6b82055875d1e5d1f7fe915 -r ade63d627a80d85174235e0bc8af16b71cd74f74 yt/frontends/enzo/fields.py
--- a/yt/frontends/enzo/fields.py
+++ b/yt/frontends/enzo/fields.py
@@ -361,41 +361,23 @@
f.take_log = False
def _spdensity(field, data):
- blank = np.zeros(data.ActiveDimensions, dtype='float64')
- if data["particle_position_x"].size == 0: return blank
filter = data['creation_time'] > 0.0
- if not filter.any(): return blank
- amr_utils.CICDeposit_3(data["particle_position_x"][filter].astype(np.float64),
- data["particle_position_y"][filter].astype(np.float64),
- data["particle_position_z"][filter].astype(np.float64),
- data["particle_mass"][filter],
- np.int64(np.where(filter)[0].size),
- blank, np.array(data.LeftEdge).astype(np.float64),
- np.array(data.ActiveDimensions).astype(np.int32),
- just_one(data['dx']))
- return blank
+ pos = data["all", "Coordinates"][filter, :]
+ d = data.deposit(pos, [data['all', 'Mass'][filter]], method='sum')
+ d /= data['CellVolume']
+ return d
add_field("star_density", function=_spdensity,
validators=[ValidateSpatial(0)], convert_function=_convertDensity)
def _dmpdensity(field, data):
- blank = np.zeros(data.ActiveDimensions, dtype='float64')
- if data["particle_position_x"].size == 0: return blank
if 'creation_time' in data.pf.field_info:
filter = data['creation_time'] <= 0.0
- if not filter.any(): return blank
- num = filter.sum()
else:
filter = Ellipsis
- num = data["particle_position_x"].size
- amr_utils.CICDeposit_3(data["particle_position_x"][filter].astype(np.float64),
- data["particle_position_y"][filter].astype(np.float64),
- data["particle_position_z"][filter].astype(np.float64),
- data["particle_mass"][filter],
- num,
- blank, np.array(data.LeftEdge).astype(np.float64),
- np.array(data.ActiveDimensions).astype(np.int32),
- just_one(data['dx']))
- return blank
+ pos = data["all", "Coordinates"][filter, :]
+ d = data.deposit(pos, [data['all', 'Mass'][filter]], method='sum')
+ d /= data['CellVolume']
+ return d
add_field("dm_density", function=_dmpdensity,
validators=[ValidateSpatial(0)], convert_function=_convertDensity)
@@ -405,28 +387,17 @@
using cloud-in-cell deposit.
"""
particle_field = field.name[4:]
- top = np.zeros(data.ActiveDimensions, dtype='float64')
- if data["particle_position_x"].size == 0: return top
- particle_field_data = data[particle_field] * data['particle_mass']
- amr_utils.CICDeposit_3(data["particle_position_x"].astype(np.float64),
- data["particle_position_y"].astype(np.float64),
- data["particle_position_z"].astype(np.float64),
- particle_field_data,
- data["particle_position_x"].size,
- top, np.array(data.LeftEdge).astype(np.float64),
- np.array(data.ActiveDimensions).astype(np.int32),
- just_one(data['dx']))
- del particle_field_data
-
- bottom = np.zeros(data.ActiveDimensions, dtype='float64')
- amr_utils.CICDeposit_3(data["particle_position_x"].astype(np.float64),
- data["particle_position_y"].astype(np.float64),
- data["particle_position_z"].astype(np.float64),
- data["particle_mass"],
- data["particle_position_x"].size,
- bottom, np.array(data.LeftEdge).astype(np.float64),
- np.array(data.ActiveDimensions).astype(np.int32),
- just_one(data['dx']))
+ pos = data[('all', 'Coordinates')]
+ top = data.deposit(
+ pos,
+ [data[('all', particle_field)]*data[('all', 'particle_mass')]],
+ method = 'cic'
+ )
+ bottom = data.deposit(
+ pos,
+ [data[('all', 'particle_mass')]],
+ method = 'cic'
+ )
top[bottom == 0] = 0.0
bnz = bottom.nonzero()
top[bnz] /= bottom[bnz]
@@ -444,30 +415,18 @@
Create a grid field for star quantities, weighted by star mass.
"""
particle_field = field.name[5:]
- top = np.zeros(data.ActiveDimensions, dtype='float64')
- if data["particle_position_x"].size == 0: return top
filter = data['creation_time'] > 0.0
- if not filter.any(): return top
- particle_field_data = data[particle_field][filter] * data['particle_mass'][filter]
- amr_utils.CICDeposit_3(data["particle_position_x"][filter].astype(np.float64),
- data["particle_position_y"][filter].astype(np.float64),
- data["particle_position_z"][filter].astype(np.float64),
- particle_field_data,
- np.int64(np.where(filter)[0].size),
- top, np.array(data.LeftEdge).astype(np.float64),
- np.array(data.ActiveDimensions).astype(np.int32),
- just_one(data['dx']))
- del particle_field_data
-
- bottom = np.zeros(data.ActiveDimensions, dtype='float64')
- amr_utils.CICDeposit_3(data["particle_position_x"][filter].astype(np.float64),
- data["particle_position_y"][filter].astype(np.float64),
- data["particle_position_z"][filter].astype(np.float64),
- data["particle_mass"][filter],
- np.int64(np.where(filter)[0].size),
- bottom, np.array(data.LeftEdge).astype(np.float64),
- np.array(data.ActiveDimensions).astype(np.int32),
- just_one(data['dx']))
+ pos = data['all', 'Coordinates'][filter, :]
+ top = data.deposit(
+ pos,
+ [data['all', particle_field][filter]*data['all', 'Mass'][filter]],
+ method='sum'
+ )
+ bottom = data.deposit(
+ pos,
+ [data['all', 'Mass'][filter]],
+ method='sum'
+ )
top[bottom == 0] = 0.0
bnz = bottom.nonzero()
top[bnz] /= bottom[bnz]
@@ -531,8 +490,8 @@
for pf in ["type", "mass"] + \
["position_%s" % ax for ax in 'xyz']:
- add_enzo_field(("all", "particle_%s" % pf), NullFunc, particle_type=True)
-
+ add_enzo_field(('all',"particle_%s" % pf), NullFunc, particle_type=True)
+
def _convRetainInt(data):
return 1
add_enzo_field(("all", "particle_index"), function=NullFunc,
@@ -549,10 +508,10 @@
particle_type=True)
for pf in ["creation_time", "dynamical_time", "metallicity_fraction"]:
- add_enzo_field(("all", pf), function=NullFunc,
+ add_enzo_field(pf, function=NullFunc,
validators = [ValidateDataField(pf)],
particle_type=True)
-add_field("particle_mass", function=NullFunc, particle_type=True)
+add_field(('all', "particle_mass"), function=NullFunc, particle_type=True)
def _ParticleAge(field, data):
current_time = data.pf.current_time
@@ -564,7 +523,7 @@
particle_type=True, convert_function=_convertParticleAge)
def _ParticleMass(field, data):
- particles = data["particle_mass"].astype('float64') * \
+ particles = data['all', "particle_mass"].astype('float64') * \
just_one(data["CellVolumeCode"].ravel())
# Note that we mandate grid-type here, so this is okay
return particles
diff -r 90fe041ad4942ce1e6b82055875d1e5d1f7fe915 -r ade63d627a80d85174235e0bc8af16b71cd74f74 yt/frontends/stream/data_structures.py
--- a/yt/frontends/stream/data_structures.py
+++ b/yt/frontends/stream/data_structures.py
@@ -285,6 +285,8 @@
self.periodicity = self.stream_handler.periodicity
self.domain_dimensions = self.stream_handler.domain_dimensions
self.current_time = self.stream_handler.simulation_time
+ self.parameters['Gamma'] = 5/3
+ self.parameters['EOSType'] = -1
if self.stream_handler.cosmology_simulation:
self.cosmological_simulation = 1
self.current_redshift = self.stream_handler.current_redshift
diff -r 90fe041ad4942ce1e6b82055875d1e5d1f7fe915 -r ade63d627a80d85174235e0bc8af16b71cd74f74 yt/frontends/stream/io.py
--- a/yt/frontends/stream/io.py
+++ b/yt/frontends/stream/io.py
@@ -54,7 +54,7 @@
def _read_fluid_selection(self, chunks, selector, fields, size):
chunks = list(chunks)
- if any((ftype != "gas" for ftype, fname in fields)):
+ if any((ftype not in ("gas", "deposit") for ftype, fname in fields)):
raise NotImplementedError
rv = {}
for field in fields:
@@ -65,6 +65,8 @@
size, [f2 for f1, f2 in fields], ng)
for field in fields:
ftype, fname = field
+ if ftype == 'deposit':
+ fname = field
ind = 0
for chunk in chunks:
for g in chunk.objs:
diff -r 90fe041ad4942ce1e6b82055875d1e5d1f7fe915 -r ade63d627a80d85174235e0bc8af16b71cd74f74 yt/geometry/geometry_handler.py
--- a/yt/geometry/geometry_handler.py
+++ b/yt/geometry/geometry_handler.py
@@ -42,6 +42,7 @@
from yt.utilities.logger import ytLogger as mylog
from yt.utilities.parallel_tools.parallel_analysis_interface import \
ParallelAnalysisInterface, parallel_splitter
+from yt.utilities.exceptions import YTFieldNotFound
class GeometryHandler(ParallelAnalysisInterface):
_global_mesh = True
@@ -189,6 +190,9 @@
try:
fd = fi[field].get_dependencies(pf = self.parameter_file)
except Exception as e:
+ if type(e) != YTFieldNotFound:
+ mylog.debug("Exception %s raised during field detection" %
+ str(type(e)))
continue
missing = False
# This next bit checks that we can't somehow generate everything.
diff -r 90fe041ad4942ce1e6b82055875d1e5d1f7fe915 -r ade63d627a80d85174235e0bc8af16b71cd74f74 yt/geometry/particle_deposit.pyx
--- a/yt/geometry/particle_deposit.pyx
+++ b/yt/geometry/particle_deposit.pyx
@@ -359,3 +359,39 @@
return self.ofield
deposit_cic = CICDeposit
+
+cdef class WeightedMeanParticleField(ParticleDepositOperation):
+ # Deposit both mass * field and mass into two scalars
+ # then in finalize divide mass * field / mass
+ cdef np.float64_t *wf
+ cdef public object owf
+ cdef np.float64_t *w
+ cdef public object ow
+ def initialize(self):
+ self.owf = np.zeros(self.nvals, dtype='float64')
+ cdef np.ndarray wfarr = self.owf
+ self.wf = <np.float64_t*> wfarr.data
+
+ self.ow = np.zeros(self.nvals, dtype='float64')
+ cdef np.ndarray warr = self.ow
+ self.w = <np.float64_t*> warr.data
+
+ @cython.cdivision(True)
+ cdef void process(self, int dim[3],
+ np.float64_t left_edge[3],
+ np.float64_t dds[3],
+ np.int64_t offset,
+ np.float64_t ppos[3],
+ np.float64_t *fields
+ ):
+ cdef int ii[3], i
+ for i in range(3):
+ ii[i] = <int>((ppos[i] - left_edge[i]) / dds[i])
+ self.w[ gind(ii[0], ii[1], ii[2], dim) + offset] += fields[1]
+ self.wf[gind(ii[0], ii[1], ii[2], dim) + offset] += fields[0] * fields[1]
+
+ def finalize(self):
+ return self.owf / self.ow
+
+deposit_weighted_mean= WeightedMeanParticleField
+
diff -r 90fe041ad4942ce1e6b82055875d1e5d1f7fe915 -r ade63d627a80d85174235e0bc8af16b71cd74f74 yt/visualization/fixed_resolution.py
--- a/yt/visualization/fixed_resolution.py
+++ b/yt/visualization/fixed_resolution.py
@@ -148,7 +148,7 @@
fields = getattr(self.data_source, "fields", [])
fields += getattr(self.data_source, "field_data", {}).keys()
for f in fields:
- if f not in exclude:
+ if f not in exclude and f[0] not in self.data_source.pf.particle_types:
self[f]
def _get_info(self, item):
https://bitbucket.org/yt_analysis/yt-3.0/commits/ebe8fb5fbf5a/
Changeset: ebe8fb5fbf5a
Branch: yt-3.0
User: juxtaposicion
Date: 2013-06-26 03:29:07
Summary: added debug info when guessing the ftype
Affected #: 1 file
diff -r 064dc07441c86efd0a32d616b013fdd77a7a88d1 -r ebe8fb5fbf5a6737bdeb219a68c9ac300f31320f yt/data_objects/static_output.py
--- a/yt/data_objects/static_output.py
+++ b/yt/data_objects/static_output.py
@@ -264,6 +264,8 @@
self._last_finfo = self.field_info[(ftype, fname)]
return self._last_finfo
if fname == self._last_freq[1]:
+ mylog.debug("Guessing field %s is (%s, %s)", fname,
+ self._last_freq[0], self._last_freq[1])
return self._last_finfo
if fname in self.field_info:
self._last_freq = field
@@ -274,6 +276,8 @@
if guessing_type and ("all", fname) in self.field_info:
self._last_freq = ("all", fname)
self._last_finfo = self.field_info["all", fname]
+ mylog.debug("Guessing field %s is (%s, %s)", fname,
+ "all", fname)
return self._last_finfo
raise YTFieldNotFound((ftype, fname), self)
https://bitbucket.org/yt_analysis/yt-3.0/commits/4010d0e4f63a/
Changeset: 4010d0e4f63a
Branch: yt-3.0
User: juxtaposicion
Date: 2013-06-26 03:32:13
Summary: added particle spherical coordinates
Affected #: 1 file
diff -r ebe8fb5fbf5a6737bdeb219a68c9ac300f31320f -r 4010d0e4f63ac8ad15c8655118dc9ac3c8822b72 yt/data_objects/universal_fields.py
--- a/yt/data_objects/universal_fields.py
+++ b/yt/data_objects/universal_fields.py
@@ -5,6 +5,8 @@
Author: Matthew Turk <matthewturk at gmail.com>
Affiliation: KIPAC/SLAC/Stanford
+Author: Chris Moody <chrisemoody at gmail.com>
+Affiliation: UCSC
Homepage: http://yt-project.org/
License:
Copyright (C) 2008-2011 Matthew Turk. All Rights Reserved.
@@ -871,6 +873,66 @@
add_field("RadialVelocityKMSABS", function=_RadialVelocityABS,
convert_function=_ConvertRadialVelocityKMS, units=r"\rm{km}/\rm{s}")
+def _ParticleRadialVelocity(field, data):
+ normal = data.get_field_parameter('normal')
+ center = data.get_field_parameter('center')
+ bv = data.get_field_parameter("bulk_velocity")
+ pos = "particle_position_%s"
+ pos = np.array([data[pos % ax] for ax in "xyz"])
+ vel = "particle_velocity_%s"
+ vel = np.array([data[vel % ax] for ax in "xyz"])
+ theta = get_sph_theta(pos.copy(), center)
+ phi = get_sph_phi(pos.copy(), center)
+ pos = pos - np.reshape(center, (3, 1))
+ vel = vel - np.reshape(bv, (3, 1))
+ sphr = get_sph_r_component(vel, theta, phi, normal)
+ return sphr
+
+add_field("ParticleRadialVelocity", function=_ParticleRadialVelocity,
+ particle_type=True, units=r"\rm{cm}/\rm{s}",
+ validators=[ValidateParameter("normal"),
+ ValidateParameter("center")])
+
+def _ParticleThetaVelocity(field, data):
+ normal = data.get_field_parameter('normal')
+ center = data.get_field_parameter('center')
+ bv = data.get_field_parameter("bulk_velocity")
+ pos = "particle_position_%s"
+ pos = np.array([data[pos % ax] for ax in "xyz"])
+ vel = "particle_velocity_%s"
+ vel = np.array([data[vel % ax] for ax in "xyz"])
+ theta = get_sph_theta(pos.copy(), center)
+ phi = get_sph_phi(pos.copy(), center)
+ pos = pos - np.reshape(center, (3, 1))
+ vel = vel - np.reshape(bv, (3, 1))
+ spht = get_sph_theta_component(vel, theta, phi, normal)
+ return sphrt
+
+add_field("ParticleThetaVelocity", function=_ParticleThetaVelocity,
+ particle_type=True, units=r"\rm{cm}/\rm{s}",
+ validators=[ValidateParameter("normal"),
+ ValidateParameter("center")])
+
+def _ParticlePhiVelocity(field, data):
+ normal = data.get_field_parameter('normal')
+ center = data.get_field_parameter('center')
+ bv = data.get_field_parameter("bulk_velocity")
+ pos = "particle_position_%s"
+ pos = np.array([data[pos % ax] for ax in "xyz"])
+ vel = "particle_velocity_%s"
+ vel = np.array([data[vel % ax] for ax in "xyz"])
+ theta = get_sph_theta(pos.copy(), center)
+ phi = get_sph_phi(pos.copy(), center)
+ pos = pos - np.reshape(center, (3, 1))
+ vel = vel - np.reshape(bv, (3, 1))
+ sphp = get_sph_phi_component(vel, theta, phi, normal)
+ return sphrp
+
+add_field("ParticlePhiVelocity", function=_ParticleThetaVelocity,
+ particle_type=True, units=r"\rm{cm}/\rm{s}",
+ validators=[ValidateParameter("normal"),
+ ValidateParameter("center")])
+
def _TangentialVelocity(field, data):
return np.sqrt(data["VelocityMagnitude"]**2.0
- data["RadialVelocity"]**2.0)
https://bitbucket.org/yt_analysis/yt-3.0/commits/596e05332166/
Changeset: 596e05332166
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-28 23:51:18
Summary: Merging with Chris
Affected #: 2 files
diff -r ade63d627a80d85174235e0bc8af16b71cd74f74 -r 596e053321665b64a69ac7da02a3be845f447fe2 yt/data_objects/static_output.py
--- a/yt/data_objects/static_output.py
+++ b/yt/data_objects/static_output.py
@@ -264,6 +264,8 @@
self._last_finfo = self.field_info[(ftype, fname)]
return self._last_finfo
if fname == self._last_freq[1]:
+ mylog.debug("Guessing field %s is (%s, %s)", fname,
+ self._last_freq[0], self._last_freq[1])
return self._last_finfo
if fname in self.field_info:
self._last_freq = field
@@ -274,6 +276,8 @@
if guessing_type and ("all", fname) in self.field_info:
self._last_freq = ("all", fname)
self._last_finfo = self.field_info["all", fname]
+ mylog.debug("Guessing field %s is (%s, %s)", fname,
+ "all", fname)
return self._last_finfo
raise YTFieldNotFound((ftype, fname), self)
diff -r ade63d627a80d85174235e0bc8af16b71cd74f74 -r 596e053321665b64a69ac7da02a3be845f447fe2 yt/data_objects/universal_fields.py
--- a/yt/data_objects/universal_fields.py
+++ b/yt/data_objects/universal_fields.py
@@ -5,6 +5,8 @@
Author: Matthew Turk <matthewturk at gmail.com>
Affiliation: KIPAC/SLAC/Stanford
+Author: Chris Moody <chrisemoody at gmail.com>
+Affiliation: UCSC
Homepage: http://yt-project.org/
License:
Copyright (C) 2008-2011 Matthew Turk. All Rights Reserved.
@@ -871,6 +873,66 @@
add_field("RadialVelocityKMSABS", function=_RadialVelocityABS,
convert_function=_ConvertRadialVelocityKMS, units=r"\rm{km}/\rm{s}")
+def _ParticleRadialVelocity(field, data):
+ normal = data.get_field_parameter('normal')
+ center = data.get_field_parameter('center')
+ bv = data.get_field_parameter("bulk_velocity")
+ pos = "particle_position_%s"
+ pos = np.array([data[pos % ax] for ax in "xyz"])
+ vel = "particle_velocity_%s"
+ vel = np.array([data[vel % ax] for ax in "xyz"])
+ theta = get_sph_theta(pos.copy(), center)
+ phi = get_sph_phi(pos.copy(), center)
+ pos = pos - np.reshape(center, (3, 1))
+ vel = vel - np.reshape(bv, (3, 1))
+ sphr = get_sph_r_component(vel, theta, phi, normal)
+ return sphr
+
+add_field("ParticleRadialVelocity", function=_ParticleRadialVelocity,
+ particle_type=True, units=r"\rm{cm}/\rm{s}",
+ validators=[ValidateParameter("normal"),
+ ValidateParameter("center")])
+
+def _ParticleThetaVelocity(field, data):
+ normal = data.get_field_parameter('normal')
+ center = data.get_field_parameter('center')
+ bv = data.get_field_parameter("bulk_velocity")
+ pos = "particle_position_%s"
+ pos = np.array([data[pos % ax] for ax in "xyz"])
+ vel = "particle_velocity_%s"
+ vel = np.array([data[vel % ax] for ax in "xyz"])
+ theta = get_sph_theta(pos.copy(), center)
+ phi = get_sph_phi(pos.copy(), center)
+ pos = pos - np.reshape(center, (3, 1))
+ vel = vel - np.reshape(bv, (3, 1))
+ spht = get_sph_theta_component(vel, theta, phi, normal)
+ return sphrt
+
+add_field("ParticleThetaVelocity", function=_ParticleThetaVelocity,
+ particle_type=True, units=r"\rm{cm}/\rm{s}",
+ validators=[ValidateParameter("normal"),
+ ValidateParameter("center")])
+
+def _ParticlePhiVelocity(field, data):
+ normal = data.get_field_parameter('normal')
+ center = data.get_field_parameter('center')
+ bv = data.get_field_parameter("bulk_velocity")
+ pos = "particle_position_%s"
+ pos = np.array([data[pos % ax] for ax in "xyz"])
+ vel = "particle_velocity_%s"
+ vel = np.array([data[vel % ax] for ax in "xyz"])
+ theta = get_sph_theta(pos.copy(), center)
+ phi = get_sph_phi(pos.copy(), center)
+ pos = pos - np.reshape(center, (3, 1))
+ vel = vel - np.reshape(bv, (3, 1))
+ sphp = get_sph_phi_component(vel, theta, phi, normal)
+ return sphrp
+
+add_field("ParticlePhiVelocity", function=_ParticleThetaVelocity,
+ particle_type=True, units=r"\rm{cm}/\rm{s}",
+ validators=[ValidateParameter("normal"),
+ ValidateParameter("center")])
+
def _TangentialVelocity(field, data):
return np.sqrt(data["VelocityMagnitude"]**2.0
- data["RadialVelocity"]**2.0)
https://bitbucket.org/yt_analysis/yt-3.0/commits/9e599fd7cad4/
Changeset: 9e599fd7cad4
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-29 06:34:15
Summary: Fixed NMSU-ART and removed subclass of OctreeContainer.
Affected #: 2 files
diff -r 596e053321665b64a69ac7da02a3be845f447fe2 -r 9e599fd7cad48f5e65df5ef3d441ca2a033acfa6 yt/frontends/art/data_structures.py
--- a/yt/frontends/art/data_structures.py
+++ b/yt/frontends/art/data_structures.py
@@ -43,7 +43,7 @@
from yt.data_objects.octree_subset import \
OctreeSubset
from yt.geometry.oct_container import \
- ARTOctreeContainer
+ OctreeContainer
from yt.data_objects.field_info_container import \
FieldInfoContainer, NullFunc
from .fields import \
@@ -106,10 +106,11 @@
allocate the requisite memory in the oct tree
"""
nv = len(self.fluid_field_list)
- self.oct_handler = ARTOctreeContainer(
+ self.oct_handler = OctreeContainer(
self.parameter_file.domain_dimensions/2, # dd is # of root cells
self.parameter_file.domain_left_edge,
- self.parameter_file.domain_right_edge)
+ self.parameter_file.domain_right_edge,
+ 1)
# The 1 here refers to domain_id == 1 always for ARTIO.
self.domains = [ARTDomainFile(self.parameter_file, nv,
self.oct_handler, 1)]
diff -r 596e053321665b64a69ac7da02a3be845f447fe2 -r 9e599fd7cad48f5e65df5ef3d441ca2a033acfa6 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -95,9 +95,10 @@
cdef class OctreeContainer:
- def __init__(self, oct_domain_dimensions, domain_left_edge, domain_right_edge):
+ def __init__(self, oct_domain_dimensions, domain_left_edge,
+ domain_right_edge, partial_coverage = 0):
# This will just initialize the root mesh octs
- self.partial_coverage = 0
+ self.partial_coverage = partial_coverage
cdef int i, j, k, p
for i in range(3):
self.nn[i] = oct_domain_dimensions[i]
@@ -742,22 +743,3 @@
# called.
if self.root_nodes != NULL: free(self.root_nodes)
if self.domains != NULL: free(self.domains)
-
-cdef class ARTOctreeContainer(OctreeContainer):
-
- def __init__(self, *args, **kwargs):
- self.partial_coverage = 1
- OctreeContainer.__init__(self, *args, **kwargs)
-
- def allocate_domains(self, domain_counts):
- cdef int count, i
- cdef OctAllocationContainer *cur = self.cont
- assert(cur == NULL)
- self.max_domain = len(domain_counts) # 1-indexed
- self.domains = <OctAllocationContainer **> malloc(
- sizeof(OctAllocationContainer *) * len(domain_counts))
- for i, count in enumerate(domain_counts):
- cur = allocate_octs(count, cur)
- if self.cont == NULL: self.cont = cur
- self.domains[i] = cur
-
https://bitbucket.org/yt_analysis/yt-3.0/commits/2b0a401a4705/
Changeset: 2b0a401a4705
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-29 07:12:31
Summary: I believe NMSU-ART is now correct. Slower, though.
Affected #: 3 files
diff -r 9e599fd7cad48f5e65df5ef3d441ca2a033acfa6 -r 2b0a401a4705507b2a798666b9eadfbd5014d9d9 yt/frontends/art/data_structures.py
--- a/yt/frontends/art/data_structures.py
+++ b/yt/frontends/art/data_structures.py
@@ -462,12 +462,17 @@
tr[field] = np.zeros(cell_count, 'float64')
data = _read_root_level(content, self.domain.level_child_offsets,
self.domain.level_count)
- ns = (8, self.domain.pf.domain_dimensions.prod() / 8)
- for field, i in zip(fields, field_idxs):
- source[field] = data[i, :]
- source[field].shape = ns
- source[field] = np.array(source[field], dtype="float64", order='F')
- # Need it to be ordered correctly; this is expensive, though ...
+ ns = (self.domain.pf.domain_dimensions.prod() / 8, 8)
+ for field, fi in zip(fields, field_idxs):
+ source[field] = np.empty(ns, dtype="float64", order="C")
+ dt = data[fi,:].reshape(self.domain.pf.domain_dimensions,
+ order="F")
+ for i in range(2):
+ for j in range(2):
+ for k in range(2):
+ ii = ((k*2)+j)*2+i
+ source[field][:,ii] = \
+ dt[i::2,j::2,k::2].ravel(order="F")
oct_handler.fill_level(0, levels, cell_inds, file_inds, tr, source)
del source
# Now we continue with the additional levels.
diff -r 9e599fd7cad48f5e65df5ef3d441ca2a033acfa6 -r 2b0a401a4705507b2a798666b9eadfbd5014d9d9 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -743,3 +743,46 @@
# called.
if self.root_nodes != NULL: free(self.root_nodes)
if self.domains != NULL: free(self.domains)
+
+cdef class ARTOctreeContainer(OctreeContainer):
+
+ @cython.boundscheck(False)
+ @cython.wraparound(False)
+ @cython.cdivision(True)
+ def fill_level_from_grid(self, int domain, int level, dest_fields,
+ source_fields,
+ np.ndarray[np.uint8_t, ndim=2, cast=True] mask,
+ int offset):
+ #Fill level, but instead of assuming that the source
+ #order is that of the oct order, we look up the oct position
+ #and fill its children from the the source field
+ #As a result, source is 3D grid with 8 times as many
+ #elements as the number of octs on this level in this domain
+ #and with the shape of an equal-sided cube
+ cdef np.ndarray[np.float64_t, ndim=3] source
+ cdef np.ndarray[np.float64_t, ndim=1] dest
+ cdef OctAllocationContainer *dom = self.domains[domain - 1]
+ cdef Oct *o
+ cdef int n
+ cdef int i, j, k, ii
+ cdef int local_pos, local_filled
+ cdef np.float64_t val
+ cdef np.int64_t ox,oy,oz
+ for key in dest_fields:
+ local_filled = 0
+ dest = dest_fields[key]
+ source = source_fields[key]
+ for n in range(dom.n):
+ o = &dom.my_octs[n]
+ #if o.level != level: continue
+ for i in range(2):
+ for j in range(2):
+ for k in range(2):
+ ii = ((k*2)+j)*2+i
+ if mask[o.domain_ind, ii] == 0: continue
+ #ox = (o.pos[0] << 1) + i
+ #oy = (o.pos[1] << 1) + j
+ #oz = (o.pos[2] << 1) + k
+ dest[local_filled + offset] = source[ox,oy,oz]
+ local_filled += 1
+ return local_filled
diff -r 9e599fd7cad48f5e65df5ef3d441ca2a033acfa6 -r 2b0a401a4705507b2a798666b9eadfbd5014d9d9 yt/geometry/oct_visitors.pyx
--- a/yt/geometry/oct_visitors.pyx
+++ b/yt/geometry/oct_visitors.pyx
@@ -162,5 +162,5 @@
cdef np.uint8_t *cell_arr = <np.uint8_t *> p[2]
level_arr[data.index] = data.level
find_arr[data.index] = o.file_ind
- cell_arr[data.index] = rind(data)
+ cell_arr[data.index] = oind(data)
data.index +=1
https://bitbucket.org/yt_analysis/yt-3.0/commits/e0d63962d093/
Changeset: e0d63962d093
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-29 19:32:12
Summary: Merging
Affected #: 1 file
diff -r 2b0a401a4705507b2a798666b9eadfbd5014d9d9 -r e0d63962d0930c0a5207a8af4d9df8506f80a66e yt/utilities/parallel_tools/parallel_analysis_interface.py
--- a/yt/utilities/parallel_tools/parallel_analysis_interface.py
+++ b/yt/utilities/parallel_tools/parallel_analysis_interface.py
@@ -522,13 +522,22 @@
yield obj, generator_func(obj)
return
generate_endpoints = len(objects) != my_size
+ # gback False: send the object backwards
+ # gforw False: receive an object from forwards
+ if len(objects) == my_size:
+ generate_endpoints = False
+ gback = False
+ gforw = False
+ else:
+ # In this case, the first processor (my_rank == 0) will generate.
+ generate_endpoints = True
+ gback = (my_rank == 0)
+ gforw = (my_rank == my_size - 1)
if generate_endpoints and mutable:
raise NotImplementedError
- gforw = generate_endpoints and my_rank == 0
- gback = generate_endpoints and my_rank == my_size - 1
# Now we need to do pairwise sends
- source = (my_rank - 1) % my_size
- dest = (my_rank + 1) % my_size
+ source = (my_rank + 1) % my_size
+ dest = (my_rank - 1) % my_size
oiter = itertools.islice(itertools.cycle(objects),
my_rank, my_rank+len(objects))
idata = None
https://bitbucket.org/yt_analysis/yt-3.0/commits/ebcc89db0a1d/
Changeset: ebcc89db0a1d
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-29 20:39:25
Summary: Adding comment that we didn't invent these constants. :)
Affected #: 1 file
diff -r e0d63962d0930c0a5207a8af4d9df8506f80a66e -r ebcc89db0a1db3f92c042ec30efe40349a3babef yt/utilities/lib/geometry_utils.pyx
--- a/yt/utilities/lib/geometry_utils.pyx
+++ b/yt/utilities/lib/geometry_utils.pyx
@@ -300,6 +300,7 @@
positions[i, j] = p[j]
return positions
+# yt did not invent these! :)
cdef np.uint64_t _const20 = 0x000001FFC00003FF
cdef np.uint64_t _const10 = 0x0007E007C00F801F
cdef np.uint64_t _const04 = 0x00786070C0E181C3
https://bitbucket.org/yt_analysis/yt-3.0/commits/b153b797d39f/
Changeset: b153b797d39f
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-29 20:44:20
Summary: Fixing masks, etc for grid patch data. ``blocks`` is not function for octs yet.
Affected #: 1 file
diff -r ebcc89db0a1db3f92c042ec30efe40349a3babef -r b153b797d39f7557edb64c570fedd8902d008c85 yt/data_objects/data_containers.py
--- a/yt/data_objects/data_containers.py
+++ b/yt/data_objects/data_containers.py
@@ -408,9 +408,10 @@
def blocks(self):
for io_chunk in self.chunks([], "io"):
for i,chunk in enumerate(self.chunks([], "spatial", ngz = 0)):
- mask = self._current_chunk.objs[0].select(self.selector)
+ g = self._current_chunk.objs[0]
+ mask = g._get_selector_mask(self.selector)
if mask is None: continue
- yield self._current_chunk.objs[0], mask
+ yield g, mask
class GenerationInProgress(Exception):
def __init__(self, fields):
https://bitbucket.org/yt_analysis/yt-3.0/commits/c0fe3330521d/
Changeset: c0fe3330521d
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-29 20:48:11
Summary: Fixing IO for Enzo, FLASH, GDF and Stream with new select() args.
Affected #: 5 files
diff -r b153b797d39f7557edb64c570fedd8902d008c85 -r c0fe3330521d10ec0d0219f9668440df0eb75e83 yt/data_objects/grid_patch.py
--- a/yt/data_objects/grid_patch.py
+++ b/yt/data_objects/grid_patch.py
@@ -497,7 +497,7 @@
def select(self, selector, source, dest, offset):
mask = self._get_selector_mask(selector)
count = self.count(selector)
- if count == 0: return
+ if count == 0: return 0
dest[offset:offset+count] = source[mask]
return count
diff -r b153b797d39f7557edb64c570fedd8902d008c85 -r c0fe3330521d10ec0d0219f9668440df0eb75e83 yt/frontends/enzo/io.py
--- a/yt/frontends/enzo/io.py
+++ b/yt/frontends/enzo/io.py
@@ -153,13 +153,10 @@
for chunk in chunks:
data = self._read_chunk_data(chunk, fields)
for g in chunk.objs:
- mask = g.select(selector)
- if mask is None: continue
- nd = mask.sum()
for field in fields:
ftype, fname = field
- gdata = data[g.id].pop(fname).swapaxes(0,2)
- nd = mask_fill(rv[field], ind, mask, gdata)
+ ds = data[g.id].pop(fname).swapaxes(0,2)
+ nd = g.select(selector, ds, rv[field], ind) # caches
ind += nd
data.pop(g.id)
return rv
diff -r b153b797d39f7557edb64c570fedd8902d008c85 -r c0fe3330521d10ec0d0219f9668440df0eb75e83 yt/frontends/flash/io.py
--- a/yt/frontends/flash/io.py
+++ b/yt/frontends/flash/io.py
@@ -92,9 +92,6 @@
ind = 0
for chunk in chunks:
for g in chunk.objs:
- mask = g.select(selector) # caches
- if mask is None: continue
data = ds[g.id - g._id_offset,:,:,:].transpose()[mask]
- rv[field][ind:ind+data.size] = data
- ind += data.size
+ ind += g.select(selector, data, rv[field], ind) # caches
return rv
diff -r b153b797d39f7557edb64c570fedd8902d008c85 -r c0fe3330521d10ec0d0219f9668440df0eb75e83 yt/frontends/gdf/io.py
--- a/yt/frontends/gdf/io.py
+++ b/yt/frontends/gdf/io.py
@@ -84,13 +84,8 @@
ind = 0
for chunk in chunks:
for grid in chunk.objs:
- mask = grid.select(selector) # caches
- if mask is None:
- continue
+ data = fhandle[field_dname(grid.id, fname)][:]
if self.pf.field_ordering == 1:
- data = fhandle[field_dname(grid.id, fname)][:].swapaxes(0, 2)[mask]
- else:
- data = fhandle[field_dname(grid.id, fname)][mask]
- rv[field][ind:ind + data.size] = data
- ind += data.size
+ data = data.swapaxes(0, 2)
+ ind += g.select(selector, data, rv[field], ind) # caches
return rv
diff -r b153b797d39f7557edb64c570fedd8902d008c85 -r c0fe3330521d10ec0d0219f9668440df0eb75e83 yt/frontends/stream/io.py
--- a/yt/frontends/stream/io.py
+++ b/yt/frontends/stream/io.py
@@ -70,12 +70,8 @@
ind = 0
for chunk in chunks:
for g in chunk.objs:
- mask = g.select(selector) # caches
- if mask is None: continue
ds = self.fields[g.id][fname]
- data = ds[mask]
- rv[field][ind:ind+data.size] = data
- ind += data.size
+ ind += g.select(selector, ds, rv[field], ind) # caches
return rv
def _read_particle_selection(self, chunks, selector, fields):
https://bitbucket.org/yt_analysis/yt-3.0/commits/3d9455d76f99/
Changeset: 3d9455d76f99
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-29 21:01:11
Summary: Not 100% happy about this, but updated ParticleOctree tests.
We should add more tests of AlwaysSelector and so on. Out of scope at the
moment.
Affected #: 1 file
diff -r c0fe3330521d10ec0d0219f9668440df0eb75e83 -r 3d9455d76f99935ea734aeb3eb1953ab1ded32d1 yt/geometry/tests/test_particle_octree.py
--- a/yt/geometry/tests/test_particle_octree.py
+++ b/yt/geometry/tests/test_particle_octree.py
@@ -1,38 +1,41 @@
from yt.testing import *
import numpy as np
-from yt.geometry.oct_container import ParticleOctreeContainer
+from yt.geometry.particle_oct_container import ParticleOctreeContainer
+from yt.geometry.oct_container import _ORDER_MAX
+from yt.utilities.lib.geometry_utils import get_morton_indices
import time, os
NPART = 32**3
-NDIM = 64
DLE = np.array([0.0, 0.0, 0.0])
DRE = np.array([10.0, 10.0, 10.0])
+dx = (DRE-DLE)/(2**_ORDER_MAX)
def test_add_particles_random():
np.random.seed(int(0x4d3d3d3))
pos = np.random.normal(0.5, scale=0.05, size=(NPART,3)) * (DRE-DLE) + DLE
+ # Now convert to integers
for i in range(3):
np.clip(pos[:,i], DLE[i], DRE[i], pos[:,i])
+ # Convert to integers
+ pos = np.floor((pos - DLE)/dx).astype("uint64")
+ morton = get_morton_indices(pos)
+ morton.sort()
for ndom in [1, 2, 4, 8]:
- octree = ParticleOctreeContainer((NDIM, NDIM, NDIM), DLE, DRE)
+ octree = ParticleOctreeContainer((1, 1, 1), DLE, DRE)
octree.n_ref = 32
- for dom in range(ndom):
- octree.add(pos[dom::ndom,:], dom)
+ for dom, split in enumerate(np.array_split(morton, ndom)):
+ octree.add(split)
octree.finalize()
# This visits every oct.
- lin_count = octree.linearly_count()
tc = octree.recursively_count()
total_count = np.zeros(len(tc), dtype="int32")
for i in sorted(tc):
total_count[i] = tc[i]
- yield assert_equal, lin_count, total_count.sum()
- mask = np.ones((total_count.sum(), 8), dtype="bool")
+ yield assert_equal, octree.nocts, total_count.sum()
# This visits every cell -- including those covered by octs.
- level_count = octree.count_levels(total_count.size-1, -1, mask)
- for dom in range(ndom):
- level_count += octree.count_levels(total_count.size-1, dom, mask)
- yield assert_equal, level_count[0], NDIM**3 * 8
- yield assert_equal, level_count, total_count * 8
+ #for dom in range(ndom):
+ # level_count += octree.count_levels(total_count.size-1, dom, mask)
+ yield assert_equal, total_count, [1, 8, 64, 104, 184, 480, 1680, 1480]
if __name__=="__main__":
for i in test_add_particles_random():
https://bitbucket.org/yt_analysis/yt-3.0/commits/c3f59cd9dd26/
Changeset: c3f59cd9dd26
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-29 21:10:16
Summary: Fixing size/shape by adding a _reshape_vals operation to grid patches.
Affected #: 3 files
diff -r 3d9455d76f99935ea734aeb3eb1953ab1ded32d1 -r c3f59cd9dd267915b4c4e59b76db493a775804f0 yt/data_objects/construction_data_containers.py
--- a/yt/data_objects/construction_data_containers.py
+++ b/yt/data_objects/construction_data_containers.py
@@ -418,6 +418,10 @@
self.pf.domain_left_edge)/self.dds).astype('int64')
self._setup_data_source()
+ def _reshape_vals(self, arr):
+ if len(arr.shape) == 3: return arr
+ return arr.reshape(self.ActiveDimensions, order="C")
+
@property
def shape(self):
return tuple(self.ActiveDimensions.tolist())
diff -r 3d9455d76f99935ea734aeb3eb1953ab1ded32d1 -r c3f59cd9dd267915b4c4e59b76db493a775804f0 yt/data_objects/grid_patch.py
--- a/yt/data_objects/grid_patch.py
+++ b/yt/data_objects/grid_patch.py
@@ -113,6 +113,10 @@
def shape(self):
return self.ActiveDimensions
+ def _reshape_vals(self, arr):
+ if len(arr.shape) == 3: return arr
+ return arr.reshape(self.ActiveDimensions, order="C")
+
def _generate_container_field(self, field):
if self._current_chunk is None:
self.hierarchy._identify_base_chunk(self)
diff -r 3d9455d76f99935ea734aeb3eb1953ab1ded32d1 -r c3f59cd9dd267915b4c4e59b76db493a775804f0 yt/data_objects/universal_fields.py
--- a/yt/data_objects/universal_fields.py
+++ b/yt/data_objects/universal_fields.py
@@ -98,7 +98,10 @@
display_field = False)
def _Ones(field, data):
- return np.ones(data.ires.shape, dtype='float64')
+ tr = np.ones(data.ires.shape, dtype="float64")
+ if data._spatial:
+ return data._reshape_vals(tr)
+ return tr
add_field("Ones", function=_Ones,
projection_conversion="unitary",
display_field = False)
https://bitbucket.org/yt_analysis/yt-3.0/commits/ffba4792808a/
Changeset: ffba4792808a
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-29 21:22:48
Summary: Adding ires, icoords, fwidth, fcoords to CoveringGrid.
This fixes up remaining tests.
Affected #: 3 files
diff -r c3f59cd9dd267915b4c4e59b76db493a775804f0 -r ffba4792808a17b6d50d0dee75f28a32dc2c3c26 yt/data_objects/construction_data_containers.py
--- a/yt/data_objects/construction_data_containers.py
+++ b/yt/data_objects/construction_data_containers.py
@@ -418,6 +418,34 @@
self.pf.domain_left_edge)/self.dds).astype('int64')
self._setup_data_source()
+ @property
+ def icoords(self):
+ ic = np.indices(self.ActiveDimensions).astype("int64")
+ return np.column_stack([i.ravel() + gi for i, gi in
+ zip(ic, self.get_global_startindex())])
+
+ @property
+ def fwidth(self):
+ fw = np.ones((self.ActiveDimensions.prod(), 3), dtype="float64")
+ fw *= self.dds
+ return fw
+
+ @property
+ def fcoords(self):
+ LE = self.LeftEdge + self.dds/2.0
+ RE = self.RightEdge - self.dds/2.0
+ N = self.ActiveDimensions
+ fc = np.mgrid[LE[0]:RE[0]:N[0]*1j,
+ LE[1]:RE[1]:N[1]*1j,
+ LE[2]:RE[2]:N[2]*1j]
+ return np.column_stack([f.ravel() for f in fc])
+
+ @property
+ def ires(self):
+ tr = np.ones(self.ActiveDimensions.prod(), dtype="int64")
+ tr *= self.level
+ return tr
+
def _reshape_vals(self, arr):
if len(arr.shape) == 3: return arr
return arr.reshape(self.ActiveDimensions, order="C")
diff -r c3f59cd9dd267915b4c4e59b76db493a775804f0 -r ffba4792808a17b6d50d0dee75f28a32dc2c3c26 yt/data_objects/field_info_container.py
--- a/yt/data_objects/field_info_container.py
+++ b/yt/data_objects/field_info_container.py
@@ -264,6 +264,11 @@
lambda: np.ones((nd * nd * nd), dtype='float64')
+ 1e-4*np.random.random((nd * nd * nd)))
+ def _reshape_vals(self, arr):
+ if not self._spatial: return arr
+ if len(arr.shape) == 3: return arr
+ return arr.reshape(self.ActiveDimensions, order="C")
+
def __missing__(self, item):
if hasattr(self.pf, "field_info") and isinstance(item, tuple):
finfo = self.pf._get_field_info(*item)
diff -r c3f59cd9dd267915b4c4e59b76db493a775804f0 -r ffba4792808a17b6d50d0dee75f28a32dc2c3c26 yt/data_objects/universal_fields.py
--- a/yt/data_objects/universal_fields.py
+++ b/yt/data_objects/universal_fields.py
@@ -92,7 +92,7 @@
display_field=False)
def _Zeros(field, data):
- return np.zeros(data.shape, dtype='float64')
+ return np.zeros(data["Ones"].shape, dtype='float64')
add_field("Zeros", function=_Zeros,
projection_conversion="unitary",
display_field = False)
https://bitbucket.org/yt_analysis/yt-3.0/commits/9b36137d26eb/
Changeset: 9b36137d26eb
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-29 23:23:09
Summary: Arbitrary Grids should be C-ordered after deposition.
Affected #: 1 file
diff -r ffba4792808a17b6d50d0dee75f28a32dc2c3c26 -r 9b36137d26eb3bb40bd233fe067586c296131630 yt/data_objects/construction_data_containers.py
--- a/yt/data_objects/construction_data_containers.py
+++ b/yt/data_objects/construction_data_containers.py
@@ -541,7 +541,7 @@
op.initialize()
op.process_grid(self, positions, fields)
vals = op.finalize()
- return vals.reshape(self.ActiveDimensions, order="F")
+ return vals.reshape(self.ActiveDimensions, order="C")
class YTArbitraryGridBase(YTCoveringGridBase):
"""A 3D region with arbitrary bounds and dimensions.
https://bitbucket.org/yt_analysis/yt-3.0/commits/a6ee8f0baf5a/
Changeset: a6ee8f0baf5a
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-30 02:42:07
Summary: Fixing issues with Gadget reading and record sizes.
Affected #: 2 files
diff -r 9b36137d26eb3bb40bd233fe067586c296131630 -r a6ee8f0baf5a8ff72eb55e84a718061a5d58cbe6 yt/frontends/sph/data_structures.py
--- a/yt/frontends/sph/data_structures.py
+++ b/yt/frontends/sph/data_structures.py
@@ -81,13 +81,16 @@
with open(filename, "rb") as f:
self.header = read_record(f, pf._header_spec)
self._position_offset = f.tell()
+ f.seek(0, os.SEEK_END)
+ self._file_size = f.tell()
super(GadgetBinaryFile, self).__init__(pf, io,
filename, file_id)
def _calculate_offsets(self, field_list):
self.field_offsets = self.io._calculate_field_offsets(
- field_list, self.total_particles)
+ field_list, self.total_particles,
+ self._file_size)
class ParticleStaticOutput(StaticOutput):
_unit_base = None
diff -r 9b36137d26eb3bb40bd233fe067586c296131630 -r a6ee8f0baf5a8ff72eb55e84a718061a5d58cbe6 yt/frontends/sph/io.py
--- a/yt/frontends/sph/io.py
+++ b/yt/frontends/sph/io.py
@@ -245,7 +245,7 @@
def _read_field_from_file(self, f, count, name):
if count == 0: return
if name == "ParticleIDs":
- dt = "int32"
+ dt = "uint32"
else:
dt = "float32"
if name in _vector_fields:
@@ -281,24 +281,36 @@
for i, v in enumerate(data_file.header["Npart"]))
return npart
- _header_offset = 256
+ # header is 256, but we have 4 at beginning and end for ints
+ _header_offset = 256 + 8
_field_size = 4
- def _calculate_field_offsets(self, field_list, pcount):
+ def _calculate_field_offsets(self, field_list, pcount,
+ file_size = None):
# field_list is (ftype, fname) but the blocks are ordered
# (fname, ftype) in the file.
- pos = self._header_offset # 256 bytes for the header
+ pos = self._header_offset
fs = self._field_size
offsets = {}
for field in self._fields:
if not isinstance(field, types.StringTypes):
field = field[0]
+ if not any( (ptype, field) in field_list
+ for ptype in self._ptypes):
+ continue
+ pos += 4
for ptype in self._ptypes:
- if (ptype, field) not in field_list: continue
+ if (ptype, field) not in field_list:
+ continue
offsets[(ptype, field)] = pos
if field in _vector_fields:
pos += 3 * pcount[ptype] * fs
else:
pos += pcount[ptype] * fs
+ pos += 4
+ if file_size is not None:
+ if file_size != pos:
+ mylog.warning("Your Gadget-2 file may have extra " +
+ "columns or different precision!")
return offsets
def _identify_fields(self, domain):
https://bitbucket.org/yt_analysis/yt-3.0/commits/81b87d3f57fe/
Changeset: 81b87d3f57fe
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-30 03:29:30
Summary: Making a few warnings and fields more explicit.
Affected #: 3 files
diff -r a6ee8f0baf5a8ff72eb55e84a718061a5d58cbe6 -r 81b87d3f57feff96e934028b7006b84f0906e26d yt/data_objects/particle_fields.py
--- a/yt/data_objects/particle_fields.py
+++ b/yt/data_objects/particle_fields.py
@@ -69,7 +69,7 @@
def particle_density(field, data):
pos = data[ptype, coord_name]
d = data.deposit(pos, [data[ptype, mass_name]], method = "sum")
- d /= data["CellVolume"]
+ d /= data["gas","CellVolume"]
return d
registry.add_field(("deposit", "%s_density" % ptype),
@@ -83,7 +83,7 @@
def particle_cic(field, data):
pos = data[ptype, coord_name]
d = data.deposit(pos, [data[ptype, mass_name]], method = "cic")
- d /= data["CellVolume"]
+ d /= data["gas","CellVolume"]
return d
registry.add_field(("deposit", "%s_cic" % ptype),
diff -r a6ee8f0baf5a8ff72eb55e84a718061a5d58cbe6 -r 81b87d3f57feff96e934028b7006b84f0906e26d yt/frontends/sph/io.py
--- a/yt/frontends/sph/io.py
+++ b/yt/frontends/sph/io.py
@@ -235,8 +235,8 @@
data = self._read_field_from_file(f, tp[ptype], field)
data = data[mask]
my_ind = ind[ptype, field]
- mylog.debug("Filling from %s to %s with %s",
- my_ind, my_ind+data.shape[0], field)
+ mylog.debug("Filling (%s, %s) from %s to %s",
+ ptype, field, my_ind, my_ind+data.shape[0])
rv[ptype, field][my_ind:my_ind + data.shape[0],...] = data
ind[ptype, field] += data.shape[0]
f.close()
diff -r a6ee8f0baf5a8ff72eb55e84a718061a5d58cbe6 -r 81b87d3f57feff96e934028b7006b84f0906e26d yt/geometry/geometry_handler.py
--- a/yt/geometry/geometry_handler.py
+++ b/yt/geometry/geometry_handler.py
@@ -191,8 +191,8 @@
fd = fi[field].get_dependencies(pf = self.parameter_file)
except Exception as e:
if type(e) != YTFieldNotFound:
- mylog.debug("Exception %s raised during field detection" %
- str(type(e)))
+ mylog.debug("Raises %s during field %s detection.",
+ str(type(e)), field)
continue
missing = False
# This next bit checks that we can't somehow generate everything.
https://bitbucket.org/yt_analysis/yt-3.0/commits/68b8b829ab6b/
Changeset: 68b8b829ab6b
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-30 04:30:10
Summary: Gadget coordinate records start with a 4-byte header.
Affected #: 2 files
diff -r 81b87d3f57feff96e934028b7006b84f0906e26d -r 68b8b829ab6bbc23195b3d0660d09588fa7a5924 yt/frontends/sph/data_structures.py
--- a/yt/frontends/sph/data_structures.py
+++ b/yt/frontends/sph/data_structures.py
@@ -90,7 +90,7 @@
def _calculate_offsets(self, field_list):
self.field_offsets = self.io._calculate_field_offsets(
field_list, self.total_particles,
- self._file_size)
+ self._position_offset, self._file_size)
class ParticleStaticOutput(StaticOutput):
_unit_base = None
diff -r 81b87d3f57feff96e934028b7006b84f0906e26d -r 68b8b829ab6bbc23195b3d0660d09588fa7a5924 yt/frontends/sph/io.py
--- a/yt/frontends/sph/io.py
+++ b/yt/frontends/sph/io.py
@@ -261,8 +261,10 @@
DLE = data_file.pf.domain_left_edge
DRE = data_file.pf.domain_right_edge
dx = (DRE - DLE) / 2**_ORDER_MAX
+ pos = np.empty((count, 3), dtype='float64')
with open(data_file.filename, "rb") as f:
- f.seek(self._header_offset)
+ # We add on an additionally 4 for the first record.
+ f.seek(data_file._position_offset + 4)
# The first total_particles * 3 values are positions
pp = np.fromfile(f, dtype = dt, count = count)
pos = np.column_stack([pp['px'], pp['py'], pp['pz']]).astype("float64")
@@ -282,13 +284,12 @@
return npart
# header is 256, but we have 4 at beginning and end for ints
- _header_offset = 256 + 8
_field_size = 4
def _calculate_field_offsets(self, field_list, pcount,
- file_size = None):
+ offset, file_size = None):
# field_list is (ftype, fname) but the blocks are ordered
# (fname, ftype) in the file.
- pos = self._header_offset
+ pos = offset
fs = self._field_size
offsets = {}
for field in self._fields:
https://bitbucket.org/yt_analysis/yt-3.0/commits/ab5d001a8829/
Changeset: ab5d001a8829
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-30 04:43:58
Summary: Explicitly naming particle fields is necessary when we do not have an implicit
particle field defined in a frontend -- for instance, in SPH codes.
Note that this is related to #592, but I believe does not cause a regression.
Affected #: 1 file
diff -r 68b8b829ab6bbc23195b3d0660d09588fa7a5924 -r ab5d001a882986a751b6f6edffeb4fdeb0ee49fc yt/data_objects/universal_fields.py
--- a/yt/data_objects/universal_fields.py
+++ b/yt/data_objects/universal_fields.py
@@ -1009,7 +1009,7 @@
for field in ["particle_position_%s" % ax for ax in "xyz"]:
# This marker should let everyone know not to use the fields, but NullFunc
# should do that, too.
- add_field(field, function=NullFunc, particle_type = True,
+ add_field(("all", field), function=NullFunc, particle_type = True,
units=r"UNDEFINED")
def _pdensity(field, data):
https://bitbucket.org/yt_analysis/yt-3.0/commits/ffdae8519da4/
Changeset: ffdae8519da4
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-30 15:34:19
Summary: Backporting this fix from Nathan's units branch.
Affected #: 1 file
diff -r ab5d001a882986a751b6f6edffeb4fdeb0ee49fc -r ffdae8519da44d382b0e6ca25c90141b461e997a yt/data_objects/field_info_container.py
--- a/yt/data_objects/field_info_container.py
+++ b/yt/data_objects/field_info_container.py
@@ -283,7 +283,7 @@
vv = finfo(self)
except NeedsGridType as exc:
ngz = exc.ghost_zones
- nfd = FieldDetector(self.nd + ngz * 2)
+ nfd = FieldDetector(self.nd + ngz * 2, pf = self.pf)
nfd._num_ghost_zones = ngz
vv = finfo(nfd)
if ngz > 0: vv = vv[ngz:-ngz, ngz:-ngz, ngz:-ngz]
https://bitbucket.org/yt_analysis/yt-3.0/commits/1b16df894cc4/
Changeset: 1b16df894cc4
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-30 17:36:16
Summary: Adding new, simpler Morton order calculation.
Affected #: 3 files
diff -r ffdae8519da44d382b0e6ca25c90141b461e997a -r 1b16df894cc41c28162829b3687e34a12991f951 yt/frontends/sph/io.py
--- a/yt/frontends/sph/io.py
+++ b/yt/frontends/sph/io.py
@@ -33,7 +33,7 @@
from yt.utilities.fortran_utils import read_record
from yt.utilities.lib.geometry_utils import get_morton_indices, \
- get_morton_indices_unravel
+ get_morton_indices_unravel, compute_morton
from yt.geometry.oct_container import _ORDER_MAX
@@ -104,16 +104,18 @@
f = h5py.File(data_file.filename, "r")
pcount = f["/Header"].attrs["NumPart_ThisFile"][:].sum()
morton = np.empty(pcount, dtype='uint64')
- DLE = data_file.pf.domain_left_edge
- DRE = data_file.pf.domain_right_edge
- dx = (DRE - DLE) / 2**_ORDER_MAX
ind = 0
for key in f.keys():
if not key.startswith("PartType"): continue
- pos = f[key]["Coordinates"][:].astype("float64")
+ ds = f[key]["Coordinates"]
+ dt = ds.dtype.newbyteorder("N") # Native
+ pos = np.empty(ds.shape, dtype=dt)
+ pos[:] = ds
regions.add_data_file(pos, data_file.file_id)
- pos = np.floor((pos - DLE)/dx).astype("uint64")
- morton[ind:ind+pos.shape[0]] = get_morton_indices(pos)
+ morton[ind:ind+pos.shape[0]] = compute_morton(
+ pos[:,0], pos[:,1], pos[:,2],
+ data_file.pf.domain_left_edge,
+ data_file.pf.domain_right_edge)
ind += pos.shape[0]
f.close()
return morton
@@ -257,7 +259,6 @@
def _initialize_index(self, data_file, regions):
count = sum(data_file.total_particles.values())
- dt = [("px", "float32"), ("py", "float32"), ("pz", "float32")]
DLE = data_file.pf.domain_left_edge
DRE = data_file.pf.domain_right_edge
dx = (DRE - DLE) / 2**_ORDER_MAX
@@ -266,16 +267,10 @@
# We add on an additionally 4 for the first record.
f.seek(data_file._position_offset + 4)
# The first total_particles * 3 values are positions
- pp = np.fromfile(f, dtype = dt, count = count)
- pos = np.column_stack([pp['px'], pp['py'], pp['pz']]).astype("float64")
- del pp
- regions.add_data_file(pos, data_file.file_id)
- lx = np.floor((pos[:,0] - DLE[0])/dx[0]).astype("uint64")
- ly = np.floor((pos[:,1] - DLE[1])/dx[1]).astype("uint64")
- lz = np.floor((pos[:,2] - DLE[2])/dx[2]).astype("uint64")
- del pos
- morton = get_morton_indices_unravel(lx, ly, lz)
- del lx, ly, lz
+ pp = np.fromfile(f, dtype = 'float32', count = count*3)
+ pp.shape = (count, 3)
+ regions.add_data_file(pp, data_file.file_id)
+ morton = compute_morton(pp[:,0], pp[:,1], pp[:,2], DLE, DRE)
return morton
def _count_particles(self, data_file):
diff -r ffdae8519da44d382b0e6ca25c90141b461e997a -r 1b16df894cc41c28162829b3687e34a12991f951 yt/geometry/particle_oct_container.pyx
--- a/yt/geometry/particle_oct_container.pyx
+++ b/yt/geometry/particle_oct_container.pyx
@@ -261,6 +261,10 @@
self.visit(o.children[cind(i,j,k)], counts, level + 1)
return
+ctypedef fused anyfloat:
+ np.float32_t
+ np.float64_t
+
cdef class ParticleRegions:
cdef np.float64_t left_edge[3]
cdef np.float64_t dds[3]
@@ -282,7 +286,14 @@
for i in range(nfiles/64 + 1):
self.masks.append(np.zeros(dims, dtype="uint64"))
- def add_data_file(self, np.ndarray[np.float64_t, ndim=2] pos, int file_id):
+ def add_data_file(self, np.ndarray pos, int file_id):
+ if pos.dtype == np.float32:
+ self._mask_positions[np.float32_t](pos, file_id)
+ elif pos.dtype == np.float64:
+ self._mask_positions[np.float64_t](pos, file_id)
+
+ cdef void _mask_positions(self, np.ndarray[anyfloat, ndim=2] pos,
+ int file_id):
cdef np.int64_t no = pos.shape[0]
cdef np.int64_t p
cdef int ind[3], i
diff -r ffdae8519da44d382b0e6ca25c90141b461e997a -r 1b16df894cc41c28162829b3687e34a12991f951 yt/utilities/lib/geometry_utils.pyx
--- a/yt/utilities/lib/geometry_utils.pyx
+++ b/yt/utilities/lib/geometry_utils.pyx
@@ -353,6 +353,51 @@
morton_indices[i] = mi
return morton_indices
+ctypedef fused anyfloat:
+ np.float32_t
+ np.float64_t
+
+cdef position_to_morton(np.ndarray[anyfloat, ndim=1] pos_x,
+ np.ndarray[anyfloat, ndim=1] pos_y,
+ np.ndarray[anyfloat, ndim=1] pos_z,
+ np.float64_t dds[3], np.float64_t DLE[3],
+ np.ndarray[np.uint64_t, ndim=1] ind):
+ cdef np.uint64_t mi, ii[3]
+ cdef np.float64_t p[3]
+ cdef np.int64_t i, j
+ for i in range(pos_x.shape[0]):
+ p[0] = <np.float64_t> pos_x[i]
+ p[1] = <np.float64_t> pos_y[i]
+ p[2] = <np.float64_t> pos_z[i]
+ for j in range(3):
+ ii[j] = <np.uint64_t> ((p[j] - DLE[j])/dds[j])
+ mi = 0
+ mi |= spread_bits(ii[2])<<0
+ mi |= spread_bits(ii[1])<<1
+ mi |= spread_bits(ii[0])<<2
+ ind[i] = mi
+
+DEF ORDER_MAX=20
+
+def compute_morton(np.ndarray pos_x, np.ndarray pos_y, np.ndarray pos_z,
+ domain_left_edge, domain_right_edge):
+ cdef int i
+ cdef np.float64_t dds[3], DLE[3], DRE[3]
+ for i in range(3):
+ DLE[i] = domain_left_edge[i]
+ DRE[i] = domain_right_edge[i]
+ dds[i] = (DRE[i] - DLE[i]) / (1 << ORDER_MAX)
+ cdef np.ndarray[np.uint64_t, ndim=1] ind
+ ind = np.zeros(pos_x.shape[0], dtype="uint64")
+ if pos_x.dtype == np.float32:
+ position_to_morton[np.float32_t](pos_x, pos_y, pos_z, dds, DLE, ind)
+ elif pos_x.dtype == np.float64:
+ position_to_morton[np.float64_t](pos_x, pos_y, pos_z, dds, DLE, ind)
+ else:
+ print "Could not identify dtype.", pos_x.dtype
+ raise NotImplementedError
+ return ind
+
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
https://bitbucket.org/yt_analysis/yt-3.0/commits/15e14fe3e829/
Changeset: 15e14fe3e829
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-30 17:48:51
Summary: Converting Tipsy to use new morton computation.
Affected #: 1 file
diff -r 1b16df894cc41c28162829b3687e34a12991f951 -r 15e14fe3e82918f3b840dc23551618e6afaae9a8 yt/frontends/sph/io.py
--- a/yt/frontends/sph/io.py
+++ b/yt/frontends/sph/io.py
@@ -37,6 +37,8 @@
from yt.geometry.oct_container import _ORDER_MAX
+CHUNKSIZE = 10000000
+
_vector_fields = ("Coordinates", "Velocity", "Velocities")
class IOHandlerOWLS(BaseIOHandler):
@@ -426,34 +428,33 @@
# We'll just add the individual types separately
count = data_file.total_particles[ptype]
if count == 0: continue
- pp = np.fromfile(f, dtype = self._pdtypes[ptype],
- count = count)
- mis = np.empty(3, dtype="float64")
- mas = np.empty(3, dtype="float64")
- for axi, ax in enumerate('xyz'):
- mi = pp["Coordinates"][ax].min()
- ma = pp["Coordinates"][ax].max()
- mylog.debug("Spanning: %0.3e .. %0.3e in %s", mi, ma, ax)
- mis[axi] = mi
- mas[axi] = ma
- if np.any(mis < pf.domain_left_edge) or \
- np.any(mas > pf.domain_right_edge):
- raise YTDomainOverflow(mis, mas,
- pf.domain_left_edge,
- pf.domain_right_edge)
- fpos = np.empty((count, 3), dtype="float64")
- fpos[:,0] = pp["Coordinates"]["x"]
- fpos[:,1] = pp["Coordinates"]["y"]
- fpos[:,2] = pp["Coordinates"]["z"]
- regions.add_data_file(fpos, data_file.file_id)
- del fpos
- pos = np.empty((count, 3), dtype="uint64")
- for axi, ax in enumerate("xyz"):
- coords = pp['Coordinates'][ax].astype("float64")
- coords = np.floor((coords - DLE[axi])/dx[axi])
- pos[:,axi] = coords
- morton[ind:ind+count] = get_morton_indices(pos)
- del pp, pos
+ start, stop = ind, ind + count
+ while ind < stop:
+ c = min(CHUNKSIZE, stop - ind)
+ pp = np.fromfile(f, dtype = self._pdtypes[ptype],
+ count = c)
+ mis = np.empty(3, dtype="float64")
+ mas = np.empty(3, dtype="float64")
+ for axi, ax in enumerate('xyz'):
+ mi = pp["Coordinates"][ax].min()
+ ma = pp["Coordinates"][ax].max()
+ mylog.debug("Spanning: %0.3e .. %0.3e in %s", mi, ma, ax)
+ mis[axi] = mi
+ mas[axi] = ma
+ if np.any(mis < pf.domain_left_edge) or \
+ np.any(mas > pf.domain_right_edge):
+ raise YTDomainOverflow(mis, mas,
+ pf.domain_left_edge,
+ pf.domain_right_edge)
+ pos = np.empty((pp.size, 3), dtype="float64")
+ pos[:,0] = pp["Coordinates"]["x"]
+ pos[:,1] = pp["Coordinates"]["y"]
+ pos[:,2] = pp["Coordinates"]["z"]
+ regions.add_data_file(pos, data_file.file_id)
+ morton[ind:ind+c] = compute_morton(
+ pos[:,0], pos[:,1], pos[:,2],
+ DLE, DRE)
+ ind += c
mylog.info("Adding %0.3e particles", morton.size)
return morton
https://bitbucket.org/yt_analysis/yt-3.0/commits/31e07a926600/
Changeset: 31e07a926600
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-30 17:49:06
Summary: Removing unused imports.
Affected #: 1 file
diff -r 15e14fe3e82918f3b840dc23551618e6afaae9a8 -r 31e07a9266001a83c1418d610a0405eeca77f64d yt/frontends/sph/io.py
--- a/yt/frontends/sph/io.py
+++ b/yt/frontends/sph/io.py
@@ -32,8 +32,7 @@
BaseIOHandler
from yt.utilities.fortran_utils import read_record
-from yt.utilities.lib.geometry_utils import get_morton_indices, \
- get_morton_indices_unravel, compute_morton
+from yt.utilities.lib.geometry_utils import compute_morton
from yt.geometry.oct_container import _ORDER_MAX
https://bitbucket.org/yt_analysis/yt-3.0/commits/f3e4516b49c8/
Changeset: f3e4516b49c8
Branch: yt-3.0
User: MatthewTurk
Date: 2013-06-30 23:19:30
Summary: Fixing broken tests.
Affected #: 1 file
diff -r 31e07a9266001a83c1418d610a0405eeca77f64d -r f3e4516b49c81e426fd860c43da02a87d92f5b9e yt/data_objects/tests/test_fields.py
--- a/yt/data_objects/tests/test_fields.py
+++ b/yt/data_objects/tests/test_fields.py
@@ -85,13 +85,17 @@
def test_all_fields():
for field in FieldInfo:
- if field.startswith("CuttingPlane"): continue
- if field.startswith("particle"): continue
- if field.startswith("CIC"): continue
- if field.startswith("WeakLensingConvergence"): continue
- if field.startswith("DensityPerturbation"): continue
- if field.startswith("Matter_Density"): continue
- if field.startswith("Overdensity"): continue
+ if isinstance(field, types.TupleType):
+ fname = field[0]
+ else:
+ fname = field
+ if fname.startswith("CuttingPlane"): continue
+ if fname.startswith("particle"): continue
+ if fname.startswith("CIC"): continue
+ if fname.startswith("WeakLensingConvergence"): continue
+ if fname.startswith("DensityPerturbation"): continue
+ if fname.startswith("Matter_Density"): continue
+ if fname.startswith("Overdensity"): continue
if FieldInfo[field].particle_type: continue
for nproc in [1, 4, 8]:
yield TestFieldAccess(field, nproc)
https://bitbucket.org/yt_analysis/yt-3.0/commits/0ed9759f31d2/
Changeset: 0ed9759f31d2
Branch: yt-3.0
User: MatthewTurk
Date: 2013-07-01 04:44:58
Summary: Superset/Subselect selectors need to respect min/max level.
This fixes the issue Chris reported with the PR.
Affected #: 2 files
diff -r f3e4516b49c81e426fd860c43da02a87d92f5b9e -r 0ed9759f31d2ddd0dd4945e9412c3c15050067c6 yt/frontends/ramses/data_structures.py
--- a/yt/frontends/ramses/data_structures.py
+++ b/yt/frontends/ramses/data_structures.py
@@ -74,6 +74,9 @@
_hydro_offset = None
_level_count = None
+ def __repr__(self):
+ return "RAMSESDomainFile: %i" % self.domain_id
+
@property
def level_count(self):
if self._level_count is not None: return self._level_count
@@ -398,6 +401,8 @@
for unit in mpc_conversion.keys():
self.units[unit] = unit_l * mpc_conversion[unit] / mpc_conversion["cm"]
self.units['%sh' % unit] = self.units[unit] * self.hubble_constant
+ self.units['%scm' % unit] = (self.units[unit] /
+ (1 + self.current_redshift))
self.units['%shcm' % unit] = (self.units['%sh' % unit] /
(1 + self.current_redshift))
for unit in sec_conversion.keys():
diff -r f3e4516b49c81e426fd860c43da02a87d92f5b9e -r 0ed9759f31d2ddd0dd4945e9412c3c15050067c6 yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -1124,6 +1124,8 @@
def __init__(self, dobj):
self.base_selector = dobj.base_selector
+ self.min_level = self.base_selector.min_level
+ self.max_level = self.base_selector.max_level
self.domain_id = dobj.domain_id
self.overlap_cells = 1
@@ -1157,11 +1159,10 @@
Oct *o = NULL) nogil:
# Because visitors now use select_grid, we should be explicitly
# checking this.
- cdef int res
- res = self.base_selector.select_grid(left_edge, right_edge, level, o)
- if res == 1 and o != NULL and o.domain != self.domain_id:
- return -1
- return res
+ return self.base_selector.select_grid(left_edge, right_edge, level, o)
+
+ def get_base(self):
+ return self.base_selector
octree_subset_selector = OctreeSubsetSelector
@@ -1175,6 +1176,8 @@
self.min_ind = dobj.min_ind
self.max_ind = dobj.max_ind
self.base_selector = dobj.base_selector
+ self.min_level = self.base_selector.min_level
+ self.max_level = self.base_selector.max_level
@cython.boundscheck(False)
@cython.wraparound(False)
https://bitbucket.org/yt_analysis/yt-3.0/commits/fd7528b8bafb/
Changeset: fd7528b8bafb
Branch: yt-3.0
User: MatthewTurk
Date: 2013-07-01 06:04:28
Summary: This somewhat simplifies the process of fields for particles by constructing
*new* fields when renaming them, thus enabling their names to be different
during that process.
The process of building a set of fields and particle fields will need to be
improved in the future, but that absolutely has to wait until the field
renaming and units work that Nathan is working on is ready for inclusion.
Affected #: 4 files
diff -r 0ed9759f31d2ddd0dd4945e9412c3c15050067c6 -r fd7528b8bafbc65f724169fddc06a986fb37a5e0 yt/data_objects/particle_fields.py
--- a/yt/data_objects/particle_fields.py
+++ b/yt/data_objects/particle_fields.py
@@ -146,3 +146,4 @@
registry.add_field((ptype, "Velocities"),
function=_get_vec_func(ptype, vel_names),
particle_type=True)
+
diff -r 0ed9759f31d2ddd0dd4945e9412c3c15050067c6 -r fd7528b8bafbc65f724169fddc06a986fb37a5e0 yt/data_objects/universal_fields.py
--- a/yt/data_objects/universal_fields.py
+++ b/yt/data_objects/universal_fields.py
@@ -1005,13 +1005,6 @@
add_field("JeansMassMsun",function=_JeansMassMsun,
units=r"\rm{M_{\odot}}")
-# We add these fields so that the field detector can use them
-for field in ["particle_position_%s" % ax for ax in "xyz"]:
- # This marker should let everyone know not to use the fields, but NullFunc
- # should do that, too.
- add_field(("all", field), function=NullFunc, particle_type = True,
- units=r"UNDEFINED")
-
def _pdensity(field, data):
pmass = data[('deposit','all_mass')]
np.divide(pmass, data["CellVolume"], pmass)
diff -r 0ed9759f31d2ddd0dd4945e9412c3c15050067c6 -r fd7528b8bafbc65f724169fddc06a986fb37a5e0 yt/frontends/enzo/fields.py
--- a/yt/frontends/enzo/fields.py
+++ b/yt/frontends/enzo/fields.py
@@ -507,7 +507,9 @@
add_enzo_field(("all", pf), function=NullFunc, convert_function=cfunc,
particle_type=True)
-for pf in ["creation_time", "dynamical_time", "metallicity_fraction"]:
+for pf in ["creation_time", "dynamical_time", "metallicity_fraction"] \
+ + ["particle_position_%s" % ax for ax in 'xyz'] \
+ + ["particle_velocity_%s" % ax for ax in 'xyz']:
add_enzo_field(pf, function=NullFunc,
validators = [ValidateDataField(pf)],
particle_type=True)
diff -r 0ed9759f31d2ddd0dd4945e9412c3c15050067c6 -r fd7528b8bafbc65f724169fddc06a986fb37a5e0 yt/geometry/geometry_handler.py
--- a/yt/geometry/geometry_handler.py
+++ b/yt/geometry/geometry_handler.py
@@ -31,6 +31,7 @@
from types import ClassType
import numpy as np
import abc
+import copy
from yt.funcs import *
from yt.config import ytcfg
@@ -166,7 +167,6 @@
# First we construct our list of fields to check
fields_to_check = []
fields_to_allcheck = []
- fields_to_add = []
for field in fi:
finfo = fi[field]
# Explicitly defined
@@ -179,13 +179,14 @@
fields_to_check.append(field)
continue
# We do a special case for 'all' later
- new_fields = [(pt, field) for pt in
- self.parameter_file.particle_types]
+ new_fields = []
+ for pt in self.parameter_file.particle_types:
+ new_fi = copy.copy(finfo)
+ new_fi.name = (pt, new_fi.name)
+ fi[new_fi.name] = new_fi
+ new_fields.append(new_fi.name)
fields_to_check += new_fields
- fields_to_add.extend( (new_field, fi[field]) for
- new_field in new_fields )
fields_to_allcheck.append(field)
- fi.update(fields_to_add)
for field in fields_to_check:
try:
fd = fi[field].get_dependencies(pf = self.parameter_file)
https://bitbucket.org/yt_analysis/yt-3.0/commits/5e2716f491e2/
Changeset: 5e2716f491e2
Branch: yt-3.0
User: MatthewTurk
Date: 2013-07-01 06:08:35
Summary: Be more careful about filling fields based on their names, when we are able to
guess them.
Affected #: 1 file
diff -r fd7528b8bafbc65f724169fddc06a986fb37a5e0 -r 5e2716f491e2bc1ff2eeaafbbecde5754451f81d yt/data_objects/field_info_container.py
--- a/yt/data_objects/field_info_container.py
+++ b/yt/data_objects/field_info_container.py
@@ -270,12 +270,17 @@
return arr.reshape(self.ActiveDimensions, order="C")
def __missing__(self, item):
- if hasattr(self.pf, "field_info") and isinstance(item, tuple):
- finfo = self.pf._get_field_info(*item)
+ if hasattr(self.pf, "field_info"):
+ if not isinstance(item, tuple):
+ field = ("unknown", item)
+ else:
+ field = item
+ finfo = self.pf._get_field_info(*field)
else:
FI = getattr(self.pf, "field_info", FieldInfo)
- if item in FI:
- finfo = FI[item]
+ field = item
+ if field in FI:
+ finfo = FI[field]
else:
finfo = None
if finfo is not None and finfo._function.func_name != 'NullFunc':
@@ -293,23 +298,23 @@
if i not in self.requested_parameters:
self.requested_parameters.append(i)
if vv is not None:
- if not self.flat: self[item] = vv
- else: self[item] = vv.ravel()
- return self[item]
+ if not self.flat: self[field] = vv
+ else: self[field] = vv.ravel()
+ return self[field]
elif finfo is not None and finfo.particle_type:
- if item == "Coordinates" or item[1] == "Coordinates" or \
- item == "Velocities" or item[1] == "Velocities":
+ if field == "Coordinates" or field[1] == "Coordinates" or \
+ field == "Velocities" or field[1] == "Velocities":
# A vector
- self[item] = np.ones((self.NumberOfParticles, 3))
+ self[field] = np.ones((self.NumberOfParticles, 3))
else:
# Not a vector
- self[item] = np.ones(self.NumberOfParticles)
- self.requested.append(item)
- return self[item]
- self.requested.append(item)
- if item not in self:
- self[item] = self._read_data(item)
- return self[item]
+ self[field] = np.ones(self.NumberOfParticles)
+ self.requested.append(field)
+ return self[field]
+ self.requested.append(field)
+ if field not in self:
+ self[field] = self._read_data(field)
+ return self[field]
def deposit(self, *args, **kwargs):
return np.random.random((self.nd, self.nd, self.nd))
https://bitbucket.org/yt_analysis/yt-3.0/commits/b9ebc0a0e0e8/
Changeset: b9ebc0a0e0e8
Branch: yt-3.0
User: MatthewTurk
Date: 2013-07-01 06:14:01
Summary: RAMSES, which mixes domains within a given OctHandler, still needs this check.
Affected #: 1 file
diff -r 5e2716f491e2bc1ff2eeaafbbecde5754451f81d -r b9ebc0a0e0e8f296e55c423806085f4ea47a97b2 yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -1159,10 +1159,11 @@
Oct *o = NULL) nogil:
# Because visitors now use select_grid, we should be explicitly
# checking this.
- return self.base_selector.select_grid(left_edge, right_edge, level, o)
-
- def get_base(self):
- return self.base_selector
+ cdef int res
+ res = self.base_selector.select_grid(left_edge, right_edge, level, o)
+ if res == 1 and o != NULL and o.domain != self.domain_id:
+ return -1
+ return res
octree_subset_selector = OctreeSubsetSelector
https://bitbucket.org/yt_analysis/yt-3.0/commits/80b8e52b5f1a/
Changeset: 80b8e52b5f1a
Branch: yt-3.0
User: MatthewTurk
Date: 2013-07-01 06:31:56
Summary: Fix tests and field detection by being explicit about requested items.
Again I feel the need to lament that field detection is difficult when trying
to address multiple fluids/particles, unless we move to closures and
functionally declared fields ...
I've also added a test runner to the test_derived_quantities.py file, which
breaks nicely for some field detection errors, and I've added some known fields
to the Stream frontend.
Affected #: 3 files
diff -r b9ebc0a0e0e8f296e55c423806085f4ea47a97b2 -r 80b8e52b5f1a059033a3cbd231125b979c684207 yt/data_objects/field_info_container.py
--- a/yt/data_objects/field_info_container.py
+++ b/yt/data_objects/field_info_container.py
@@ -278,9 +278,8 @@
finfo = self.pf._get_field_info(*field)
else:
FI = getattr(self.pf, "field_info", FieldInfo)
- field = item
- if field in FI:
- finfo = FI[field]
+ if item in FI:
+ finfo = FI[item]
else:
finfo = None
if finfo is not None and finfo._function.func_name != 'NullFunc':
@@ -298,23 +297,23 @@
if i not in self.requested_parameters:
self.requested_parameters.append(i)
if vv is not None:
- if not self.flat: self[field] = vv
- else: self[field] = vv.ravel()
- return self[field]
+ if not self.flat: self[item] = vv
+ else: self[item] = vv.ravel()
+ return self[item]
elif finfo is not None and finfo.particle_type:
- if field == "Coordinates" or field[1] == "Coordinates" or \
- field == "Velocities" or field[1] == "Velocities":
+ if item == "Coordinates" or item[1] == "Coordinates" or \
+ item == "Velocities" or item[1] == "Velocities":
# A vector
- self[field] = np.ones((self.NumberOfParticles, 3))
+ self[item] = np.ones((self.NumberOfParticles, 3))
else:
# Not a vector
- self[field] = np.ones(self.NumberOfParticles)
- self.requested.append(field)
- return self[field]
- self.requested.append(field)
- if field not in self:
- self[field] = self._read_data(field)
- return self[field]
+ self[item] = np.ones(self.NumberOfParticles)
+ self.requested.append(item)
+ return self[item]
+ self.requested.append(item)
+ if item not in self:
+ self[item] = self._read_data(item)
+ return self[item]
def deposit(self, *args, **kwargs):
return np.random.random((self.nd, self.nd, self.nd))
diff -r b9ebc0a0e0e8f296e55c423806085f4ea47a97b2 -r 80b8e52b5f1a059033a3cbd231125b979c684207 yt/data_objects/tests/test_derived_quantities.py
--- a/yt/data_objects/tests/test_derived_quantities.py
+++ b/yt/data_objects/tests/test_derived_quantities.py
@@ -50,3 +50,7 @@
a_std = np.sqrt((ad["CellMass"] * (ad["Density"] - a_mean)**2).sum() /
ad["CellMass"].sum())
yield assert_rel_equal, my_std, a_std, 12
+
+if __name__ == "__main__":
+ for i in test_extrema():
+ i[0](*i[1:])
diff -r b9ebc0a0e0e8f296e55c423806085f4ea47a97b2 -r 80b8e52b5f1a059033a3cbd231125b979c684207 yt/frontends/stream/fields.py
--- a/yt/frontends/stream/fields.py
+++ b/yt/frontends/stream/fields.py
@@ -42,6 +42,10 @@
add_field = StreamFieldInfo.add_field
add_stream_field("density", function = NullFunc)
+add_stream_field("x-velocity", function = NullFunc)
+add_stream_field("y-velocity", function = NullFunc)
+add_stream_field("z-velocity", function = NullFunc)
+
add_field("Density", function = TranslationFunc("density"))
add_stream_field("particle_position_x", function = NullFunc, particle_type=True)
https://bitbucket.org/yt_analysis/yt-3.0/commits/92a91071ac6b/
Changeset: 92a91071ac6b
Branch: yt-3.0
User: MatthewTurk
Date: 2013-07-01 16:30:59
Summary: We change the dictionary during iteration so we need to use a copy of the key list.
Affected #: 1 file
diff -r 80b8e52b5f1a059033a3cbd231125b979c684207 -r 92a91071ac6b84e1008646de7f2210e6841db9f0 yt/geometry/geometry_handler.py
--- a/yt/geometry/geometry_handler.py
+++ b/yt/geometry/geometry_handler.py
@@ -167,7 +167,7 @@
# First we construct our list of fields to check
fields_to_check = []
fields_to_allcheck = []
- for field in fi:
+ for field in fi.keys():
finfo = fi[field]
# Explicitly defined
if isinstance(field, tuple):
https://bitbucket.org/yt_analysis/yt-3.0/commits/559bca49e90f/
Changeset: 559bca49e90f
Branch: yt-3.0
User: ngoldbaum
Date: 2013-07-01 23:40:29
Summary: Merged in MatthewTurk/yt-3.0 (pull request #54)
Octree diet
Affected #: 45 files
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/analysis_modules/halo_finding/halo_objects.py
--- a/yt/analysis_modules/halo_finding/halo_objects.py
+++ b/yt/analysis_modules/halo_finding/halo_objects.py
@@ -2495,7 +2495,7 @@
if dm_only:
select = self._get_dm_indices()
total_mass = \
- self.comm.mpi_allreduce((self._data_source["ParticleMassMsun"][select]).sum(dtype='float64'), op='sum')
+ self.comm.mpi_allreduce((self._data_source['all', "ParticleMassMsun"][select]).sum(dtype='float64'), op='sum')
else:
total_mass = self.comm.mpi_allreduce(self._data_source.quantities["TotalQuantity"]("ParticleMassMsun")[0], op='sum')
# MJT: Note that instead of this, if we are assuming that the particles
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/analysis_modules/halo_finding/hop/hop_hop.c
--- a/yt/analysis_modules/halo_finding/hop/hop_hop.c
+++ b/yt/analysis_modules/halo_finding/hop/hop_hop.c
@@ -443,7 +443,7 @@
/* Else, this slot was full, go to the next one */
hp++;
if (hp>=smx->hash+smx->nHashLength) hp = smx->hash;
- if (++count>1000) {
+ if (++count>1000000) {
fprintf(stderr,"Hash Table is too full.\n");
exit(1);
}
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/analysis_modules/star_analysis/sfr_spectrum.py
--- a/yt/analysis_modules/star_analysis/sfr_spectrum.py
+++ b/yt/analysis_modules/star_analysis/sfr_spectrum.py
@@ -109,16 +109,19 @@
"""
# Pick out the stars.
if self.mode == 'data_source':
- ct = self._data_source["creation_time"]
+ ct = self._data_source["stars","particle_age"]
+ if ct == None :
+ print 'data source must have particle_age!'
+ sys.exit(1)
ct_stars = ct[ct > 0]
- mass_stars = self._data_source["ParticleMassMsun"][ct > 0]
+ mass_stars = self._data_source["stars", "ParticleMassMsun"][ct > 0]
elif self.mode == 'provided':
ct_stars = self.star_creation_time
mass_stars = self.star_mass
# Find the oldest stars in units of code time.
tmin= min(ct_stars)
# Multiply the end to prevent numerical issues.
- self.time_bins = np.linspace(tmin*0.99, self._pf.current_time,
+ self.time_bins = np.linspace(tmin*1.01, self._pf.current_time,
num = self.bin_count + 1)
# Figure out which bins the stars go into.
inds = np.digitize(ct_stars, self.time_bins) - 1
@@ -131,7 +134,7 @@
for index in xrange(self.bin_count):
self.cum_mass_bins[index+1] += self.cum_mass_bins[index]
# We will want the time taken between bins.
- self.time_bins_dt = self.time_bins[1:] - self.time_bins[:-1]
+ self.time_bins_dt = self.time_bins[:-1] - self.time_bins[1:]
def attach_arrays(self):
"""
@@ -147,7 +150,7 @@
vol = ds.volume('mpc')
elif self.mode == 'provided':
vol = self.volume
- tc = self._pf["Time"]
+ tc = self._pf["Time"] #time to seconds?
self.time = []
self.lookback_time = []
self.redshift = []
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/data_objects/construction_data_containers.py
--- a/yt/data_objects/construction_data_containers.py
+++ b/yt/data_objects/construction_data_containers.py
@@ -286,7 +286,7 @@
# This needs to be parallel_objects-ified
for chunk in parallel_objects(self.data_source.chunks(
chunk_fields, "io")):
- mylog.debug("Adding chunk (%s) to tree", chunk.size)
+ mylog.debug("Adding chunk (%s) to tree", chunk.ires.size)
self._handle_chunk(chunk, fields, tree)
# Note that this will briefly double RAM usage
if self.proj_style == "mip":
@@ -310,7 +310,6 @@
np.multiply(py, self.pf.domain_width[y_dict[self.axis]], py)
np.add(py, oy, py)
np.multiply(pdy, self.pf.domain_width[y_dict[self.axis]], pdy)
-
if self.weight_field is not None:
np.divide(nvals, nwvals[:,None], nvals)
if self.weight_field is None:
@@ -345,7 +344,7 @@
dl = 1.0
else:
dl = chunk.fwidth[:, self.axis]
- v = np.empty((chunk.size, len(fields)), dtype="float64")
+ v = np.empty((chunk.ires.size, len(fields)), dtype="float64")
for i in range(len(fields)):
v[:,i] = chunk[fields[i]] * dl
if self.weight_field is not None:
@@ -353,7 +352,7 @@
np.multiply(v, w[:,None], v)
np.multiply(w, dl, w)
else:
- w = np.ones(chunk.size, dtype="float64")
+ w = np.ones(chunk.ires.size, dtype="float64")
icoords = chunk.icoords
i1 = icoords[:,x_dict[self.axis]]
i2 = icoords[:,y_dict[self.axis]]
@@ -420,6 +419,38 @@
self._setup_data_source()
@property
+ def icoords(self):
+ ic = np.indices(self.ActiveDimensions).astype("int64")
+ return np.column_stack([i.ravel() + gi for i, gi in
+ zip(ic, self.get_global_startindex())])
+
+ @property
+ def fwidth(self):
+ fw = np.ones((self.ActiveDimensions.prod(), 3), dtype="float64")
+ fw *= self.dds
+ return fw
+
+ @property
+ def fcoords(self):
+ LE = self.LeftEdge + self.dds/2.0
+ RE = self.RightEdge - self.dds/2.0
+ N = self.ActiveDimensions
+ fc = np.mgrid[LE[0]:RE[0]:N[0]*1j,
+ LE[1]:RE[1]:N[1]*1j,
+ LE[2]:RE[2]:N[2]*1j]
+ return np.column_stack([f.ravel() for f in fc])
+
+ @property
+ def ires(self):
+ tr = np.ones(self.ActiveDimensions.prod(), dtype="int64")
+ tr *= self.level
+ return tr
+
+ def _reshape_vals(self, arr):
+ if len(arr.shape) == 3: return arr
+ return arr.reshape(self.ActiveDimensions, order="C")
+
+ @property
def shape(self):
return tuple(self.ActiveDimensions.tolist())
@@ -510,7 +541,7 @@
op.initialize()
op.process_grid(self, positions, fields)
vals = op.finalize()
- return vals.reshape(self.ActiveDimensions, order="F")
+ return vals.reshape(self.ActiveDimensions, order="C")
class YTArbitraryGridBase(YTCoveringGridBase):
"""A 3D region with arbitrary bounds and dimensions.
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/data_objects/data_containers.py
--- a/yt/data_objects/data_containers.py
+++ b/yt/data_objects/data_containers.py
@@ -243,33 +243,23 @@
return rv
def _generate_spatial_fluid(self, field, ngz):
- rv = np.empty(self.size, dtype="float64")
+ rv = np.empty(self.ires.size, dtype="float64")
ind = 0
if ngz == 0:
for io_chunk in self.chunks([], "io"):
for i,chunk in enumerate(self.chunks(field, "spatial", ngz = 0)):
- mask = self._current_chunk.objs[0].select(self.selector)
- if mask is None: continue
- data = self[field]
- if len(data.shape) == 4:
- # This is how we keep it consistent between oct ordering
- # and grid ordering.
- data = data.T[mask.T]
- else:
- data = data[mask]
- rv[ind:ind+data.size] = data
- ind += data.size
+ ind += self._current_chunk.objs[0].select(
+ self.selector, self[field], rv, ind)
else:
chunks = self.hierarchy._chunk(self, "spatial", ngz = ngz)
for i, chunk in enumerate(chunks):
with self._chunked_read(chunk):
gz = self._current_chunk.objs[0]
wogz = gz._base_grid
- mask = wogz.select(self.selector)
- if mask is None: continue
- data = gz[field][ngz:-ngz, ngz:-ngz, ngz:-ngz][mask]
- rv[ind:ind+data.size] = data
- ind += data.size
+ ind += wogz.select(
+ self.selector,
+ gz[field][ngz:-ngz, ngz:-ngz, ngz:-ngz],
+ rv, ind)
return rv
def _generate_particle_field(self, field):
@@ -418,9 +408,10 @@
def blocks(self):
for io_chunk in self.chunks([], "io"):
for i,chunk in enumerate(self.chunks([], "spatial", ngz = 0)):
- mask = self._current_chunk.objs[0].select(self.selector)
+ g = self._current_chunk.objs[0]
+ mask = g._get_selector_mask(self.selector)
if mask is None: continue
- yield self._current_chunk.objs[0], mask
+ yield g, mask
class GenerationInProgress(Exception):
def __init__(self, fields):
@@ -432,8 +423,6 @@
_sort_by = None
_selector = None
_current_chunk = None
- size = None
- shape = None
def __init__(self, *args, **kwargs):
super(YTSelectionContainer, self).__init__(*args, **kwargs)
@@ -551,16 +540,10 @@
# There are several items that need to be swapped out
# field_data, size, shape
old_field_data, self.field_data = self.field_data, YTFieldData()
- old_size, self.size = self.size, chunk.data_size
old_chunk, self._current_chunk = self._current_chunk, chunk
old_locked, self._locked = self._locked, False
- if not self._spatial:
- self.shape = (self.size,)
yield
self.field_data = old_field_data
- self.size = old_size
- if not self._spatial:
- self.shape = (old_size,)
self._current_chunk = old_chunk
self._locked = old_locked
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/data_objects/field_info_container.py
--- a/yt/data_objects/field_info_container.py
+++ b/yt/data_objects/field_info_container.py
@@ -264,9 +264,18 @@
lambda: np.ones((nd * nd * nd), dtype='float64')
+ 1e-4*np.random.random((nd * nd * nd)))
+ def _reshape_vals(self, arr):
+ if not self._spatial: return arr
+ if len(arr.shape) == 3: return arr
+ return arr.reshape(self.ActiveDimensions, order="C")
+
def __missing__(self, item):
- if hasattr(self.pf, "field_info") and isinstance(item, tuple):
- finfo = self.pf._get_field_info(*item)
+ if hasattr(self.pf, "field_info"):
+ if not isinstance(item, tuple):
+ field = ("unknown", item)
+ else:
+ field = item
+ finfo = self.pf._get_field_info(*field)
else:
FI = getattr(self.pf, "field_info", FieldInfo)
if item in FI:
@@ -278,7 +287,7 @@
vv = finfo(self)
except NeedsGridType as exc:
ngz = exc.ghost_zones
- nfd = FieldDetector(self.nd + ngz * 2)
+ nfd = FieldDetector(self.nd + ngz * 2, pf = self.pf)
nfd._num_ghost_zones = ngz
vv = finfo(nfd)
if ngz > 0: vv = vv[ngz:-ngz, ngz:-ngz, ngz:-ngz]
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/data_objects/grid_patch.py
--- a/yt/data_objects/grid_patch.py
+++ b/yt/data_objects/grid_patch.py
@@ -113,6 +113,10 @@
def shape(self):
return self.ActiveDimensions
+ def _reshape_vals(self, arr):
+ if len(arr.shape) == 3: return arr
+ return arr.reshape(self.ActiveDimensions, order="C")
+
def _generate_container_field(self, field):
if self._current_chunk is None:
self.hierarchy._identify_base_chunk(self)
@@ -441,14 +445,14 @@
return new_field
def select_icoords(self, dobj):
- mask = self.select(dobj.selector)
+ mask = self._get_selector_mask(dobj.selector)
if mask is None: return np.empty((0,3), dtype='int64')
coords = convert_mask_to_indices(mask, mask.sum())
coords += self.get_global_startindex()[None, :]
return coords
def select_fcoords(self, dobj):
- mask = self.select(dobj.selector)
+ mask = self._get_selector_mask(dobj.selector)
if mask is None: return np.empty((0,3), dtype='float64')
coords = convert_mask_to_indices(mask, mask.sum()).astype("float64")
coords += 0.5
@@ -457,15 +461,15 @@
return coords
def select_fwidth(self, dobj):
- mask = self.select(dobj.selector)
- if mask is None: return np.empty((0,3), dtype='float64')
- coords = np.empty((mask.sum(), 3), dtype='float64')
+ count = self.count(dobj.selector)
+ if count == 0: return np.empty((0,3), dtype='float64')
+ coords = np.empty((count, 3), dtype='float64')
for axis in range(3):
coords[:,axis] = self.dds[axis]
return coords
def select_ires(self, dobj):
- mask = self.select(dobj.selector)
+ mask = self._get_selector_mask(dobj.selector)
if mask is None: return np.empty(0, dtype='int64')
coords = np.empty(mask.sum(), dtype='int64')
coords[:] = self.Level
@@ -484,21 +488,27 @@
op.initialize()
op.process_grid(self, positions, fields)
vals = op.finalize()
- return vals.reshape(self.ActiveDimensions, order="F")
+ return vals.reshape(self.ActiveDimensions, order="C")
- def select(self, selector):
+ def _get_selector_mask(self, selector):
if id(selector) == self._last_selector_id:
- return self._last_mask
- self._last_mask = selector.fill_mask(self)
- self._last_selector_id = id(selector)
- return self._last_mask
+ mask = self._last_mask
+ else:
+ self._last_mask = mask = selector.fill_mask(self)
+ self._last_selector_id = id(selector)
+ return mask
+
+ def select(self, selector, source, dest, offset):
+ mask = self._get_selector_mask(selector)
+ count = self.count(selector)
+ if count == 0: return 0
+ dest[offset:offset+count] = source[mask]
+ return count
def count(self, selector):
- if id(selector) == self._last_selector_id:
- if self._last_mask is None: return 0
- return self._last_mask.sum()
- self.select(selector)
- return self.count(selector)
+ mask = self._get_selector_mask(selector)
+ if mask is None: return 0
+ return mask.sum()
def count_particles(self, selector, x, y, z):
# We don't cache the selector results
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/data_objects/octree_subset.py
--- a/yt/data_objects/octree_subset.py
+++ b/yt/data_objects/octree_subset.py
@@ -36,6 +36,7 @@
NeedsProperty, \
NeedsParameter
import yt.geometry.particle_deposit as particle_deposit
+from yt.funcs import *
class OctreeSubset(YTSelectionContainer):
_spatial = True
@@ -43,28 +44,25 @@
_num_zones = 2
_type_name = 'octree_subset'
_skip_add = True
- _con_args = ('domain', 'mask', 'cell_count')
+ _con_args = ('base_region', 'domain', 'pf')
_container_fields = ("dx", "dy", "dz")
+ _domain_offset = 0
+ _num_octs = -1
- def __init__(self, domain, mask, cell_count):
+ def __init__(self, base_region, domain, pf):
self.field_data = YTFieldData()
self.field_parameters = {}
- self.mask = mask
self.domain = domain
+ self.domain_id = domain.domain_id
self.pf = domain.pf
self.hierarchy = self.pf.hierarchy
- self.oct_handler = domain.pf.h.oct_handler
- self.cell_count = cell_count
- level_counts = self.oct_handler.count_levels(
- self.domain.pf.max_level, self.domain.domain_id, mask)
- assert(level_counts.sum() == cell_count)
- level_counts[1:] = level_counts[:-1]
- level_counts[0] = 0
- self.level_counts = np.add.accumulate(level_counts)
+ self.oct_handler = domain.oct_handler
self._last_mask = None
self._last_selector_id = None
self._current_particle_type = 'all'
self._current_fluid_type = self.pf.default_fluid_type
+ self.base_region = base_region
+ self.base_selector = base_region.selector
def _generate_container_field(self, field):
if self._current_chunk is None:
@@ -75,31 +73,8 @@
return self._current_chunk.fwidth[:,1]
elif field == "dz":
return self._current_chunk.fwidth[:,2]
-
- def select_icoords(self, dobj):
- return self.oct_handler.icoords(self.domain.domain_id, self.mask,
- self.cell_count,
- self.level_counts.copy())
-
- def select_fcoords(self, dobj):
- return self.oct_handler.fcoords(self.domain.domain_id, self.mask,
- self.cell_count,
- self.level_counts.copy())
-
- def select_fwidth(self, dobj):
- # Recall domain_dimensions is the number of cells, not octs
- base_dx = (self.domain.pf.domain_width /
- self.domain.pf.domain_dimensions)
- widths = np.empty((self.cell_count, 3), dtype="float64")
- dds = (2**self.select_ires(dobj))
- for i in range(3):
- widths[:,i] = base_dx[i] / dds
- return widths
-
- def select_ires(self, dobj):
- return self.oct_handler.ires(self.domain.domain_id, self.mask,
- self.cell_count,
- self.level_counts.copy())
+ else:
+ raise RuntimeError
def __getitem__(self, key):
tr = super(OctreeSubset, self).__getitem__(key)
@@ -117,9 +92,11 @@
return tr
def _reshape_vals(self, arr):
+ if len(arr.shape) == 4: return arr
nz = self._num_zones + 2*self._num_ghost_zones
n_oct = arr.shape[0] / (nz**3.0)
arr = arr.reshape((nz, nz, nz, n_oct), order="F")
+ arr = np.asfortranarray(arr)
return arr
_domain_ind = None
@@ -127,7 +104,7 @@
@property
def domain_ind(self):
if self._domain_ind is None:
- di = self.oct_handler.domain_ind(self.mask, self.domain.domain_id)
+ di = self.oct_handler.domain_ind(self.selector)
self._domain_ind = di
return self._domain_ind
@@ -136,22 +113,52 @@
cls = getattr(particle_deposit, "deposit_%s" % method, None)
if cls is None:
raise YTParticleDepositionNotImplemented(method)
- nvals = (self.domain_ind >= 0).sum() * 8
+ nvals = (2, 2, 2, (self.domain_ind >= 0).sum())
op = cls(nvals) # We allocate number of zones, not number of octs
op.initialize()
+ mylog.debug("Depositing %s particles into %s Octs",
+ positions.shape[0], nvals[-1])
op.process_octree(self.oct_handler, self.domain_ind, positions, fields,
- self.domain.domain_id)
+ self.domain_id, self._domain_offset)
vals = op.finalize()
- return self._reshape_vals(vals)
+ return np.asfortranarray(vals)
- def select(self, selector):
- if id(selector) == self._last_selector_id:
- return self._last_mask
- self._last_mask = self.oct_handler.domain_mask(
- self.mask, self.domain.domain_id)
- if self._last_mask.sum() == 0: return None
- self._last_selector_id = id(selector)
- return self._last_mask
+ def select_icoords(self, dobj):
+ d = self.oct_handler.icoords(self.selector, domain_id = self.domain_id,
+ num_octs = self._num_octs)
+ self._num_octs = d.shape[0] / 8
+ tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 3,
+ domain_id = self.domain_id)
+ return tr
+
+ def select_fcoords(self, dobj):
+ d = self.oct_handler.fcoords(self.selector, domain_id = self.domain_id,
+ num_octs = self._num_octs)
+ self._num_octs = d.shape[0] / 8
+ tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 3,
+ domain_id = self.domain_id)
+ return tr
+
+ def select_fwidth(self, dobj):
+ d = self.oct_handler.fwidth(self.selector, domain_id = self.domain_id,
+ num_octs = self._num_octs)
+ self._num_octs = d.shape[0] / 8
+ tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 3,
+ domain_id = self.domain_id)
+ return tr
+
+ def select_ires(self, dobj):
+ d = self.oct_handler.ires(self.selector, domain_id = self.domain_id,
+ num_octs = self._num_octs)
+ self._num_octs = d.shape[0] / 8
+ tr = self.oct_handler.selector_fill(dobj.selector, d, None, 0, 1,
+ domain_id = self.domain_id)
+ return tr
+
+ def select(self, selector, source, dest, offset):
+ n = self.oct_handler.selector_fill(selector, source, dest, offset,
+ domain_id = self.domain_id)
+ return n
def count(self, selector):
if id(selector) == self._last_selector_id:
@@ -168,3 +175,30 @@
def select_particles(self, selector, x, y, z):
mask = selector.select_points(x,y,z)
return mask
+
+class ParticleOctreeSubset(OctreeSubset):
+ # Subclassing OctreeSubset is somewhat dubious.
+ # This is some subset of an octree. Note that the sum of subsets of an
+ # octree may multiply include data files. While we can attempt to mitigate
+ # this, it's unavoidable for many types of data storage on disk.
+ _type_name = 'particle_octree_subset'
+ _con_args = ('data_files', 'pf', 'min_ind', 'max_ind')
+ domain_id = -1
+ def __init__(self, base_region, data_files, pf, min_ind = 0, max_ind = 0):
+ # The first attempt at this will not work in parallel.
+ self.data_files = data_files
+ self.field_data = YTFieldData()
+ self.field_parameters = {}
+ self.pf = pf
+ self.hierarchy = self.pf.hierarchy
+ self.oct_handler = pf.h.oct_handler
+ self.min_ind = min_ind
+ if max_ind == 0: max_ind = (1 << 63)
+ self.max_ind = max_ind
+ self._last_mask = None
+ self._last_selector_id = None
+ self._current_particle_type = 'all'
+ self._current_fluid_type = self.pf.default_fluid_type
+ self.base_region = base_region
+ self.base_selector = base_region.selector
+
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/data_objects/particle_fields.py
--- a/yt/data_objects/particle_fields.py
+++ b/yt/data_objects/particle_fields.py
@@ -69,7 +69,7 @@
def particle_density(field, data):
pos = data[ptype, coord_name]
d = data.deposit(pos, [data[ptype, mass_name]], method = "sum")
- d /= data["CellVolume"]
+ d /= data["gas","CellVolume"]
return d
registry.add_field(("deposit", "%s_density" % ptype),
@@ -83,7 +83,7 @@
def particle_cic(field, data):
pos = data[ptype, coord_name]
d = data.deposit(pos, [data[ptype, mass_name]], method = "cic")
- d /= data["CellVolume"]
+ d /= data["gas","CellVolume"]
return d
registry.add_field(("deposit", "%s_cic" % ptype),
@@ -146,3 +146,4 @@
registry.add_field((ptype, "Velocities"),
function=_get_vec_func(ptype, vel_names),
particle_type=True)
+
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/data_objects/static_output.py
--- a/yt/data_objects/static_output.py
+++ b/yt/data_objects/static_output.py
@@ -265,6 +265,8 @@
self._last_finfo = self.field_info[(ftype, fname)]
return self._last_finfo
if fname == self._last_freq[1]:
+ mylog.debug("Guessing field %s is (%s, %s)", fname,
+ self._last_freq[0], self._last_freq[1])
return self._last_finfo
if fname in self.field_info:
self._last_freq = field
@@ -275,6 +277,8 @@
if guessing_type and ("all", fname) in self.field_info:
self._last_freq = ("all", fname)
self._last_finfo = self.field_info["all", fname]
+ mylog.debug("Guessing field %s is (%s, %s)", fname,
+ "all", fname)
return self._last_finfo
raise YTFieldNotFound((ftype, fname), self)
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/data_objects/tests/test_derived_quantities.py
--- a/yt/data_objects/tests/test_derived_quantities.py
+++ b/yt/data_objects/tests/test_derived_quantities.py
@@ -50,3 +50,7 @@
a_std = np.sqrt((ad["CellMass"] * (ad["Density"] - a_mean)**2).sum() /
ad["CellMass"].sum())
yield assert_rel_equal, my_std, a_std, 12
+
+if __name__ == "__main__":
+ for i in test_extrema():
+ i[0](*i[1:])
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/data_objects/tests/test_fields.py
--- a/yt/data_objects/tests/test_fields.py
+++ b/yt/data_objects/tests/test_fields.py
@@ -85,13 +85,17 @@
def test_all_fields():
for field in FieldInfo:
- if field.startswith("CuttingPlane"): continue
- if field.startswith("particle"): continue
- if field.startswith("CIC"): continue
- if field.startswith("WeakLensingConvergence"): continue
- if field.startswith("DensityPerturbation"): continue
- if field.startswith("Matter_Density"): continue
- if field.startswith("Overdensity"): continue
+ if isinstance(field, types.TupleType):
+ fname = field[0]
+ else:
+ fname = field
+ if fname.startswith("CuttingPlane"): continue
+ if fname.startswith("particle"): continue
+ if fname.startswith("CIC"): continue
+ if fname.startswith("WeakLensingConvergence"): continue
+ if fname.startswith("DensityPerturbation"): continue
+ if fname.startswith("Matter_Density"): continue
+ if fname.startswith("Overdensity"): continue
if FieldInfo[field].particle_type: continue
for nproc in [1, 4, 8]:
yield TestFieldAccess(field, nproc)
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/data_objects/universal_fields.py
--- a/yt/data_objects/universal_fields.py
+++ b/yt/data_objects/universal_fields.py
@@ -5,6 +5,8 @@
Author: Matthew Turk <matthewturk at gmail.com>
Affiliation: KIPAC/SLAC/Stanford
+Author: Chris Moody <chrisemoody at gmail.com>
+Affiliation: UCSC
Homepage: http://yt-project.org/
License:
Copyright (C) 2008-2011 Matthew Turk. All Rights Reserved.
@@ -90,13 +92,16 @@
display_field=False)
def _Zeros(field, data):
- return np.zeros(data.shape, dtype='float64')
+ return np.zeros(data["Ones"].shape, dtype='float64')
add_field("Zeros", function=_Zeros,
projection_conversion="unitary",
display_field = False)
def _Ones(field, data):
- return np.ones(data.shape, dtype='float64')
+ tr = np.ones(data.ires.shape, dtype="float64")
+ if data._spatial:
+ return data._reshape_vals(tr)
+ return tr
add_field("Ones", function=_Ones,
projection_conversion="unitary",
display_field = False)
@@ -871,6 +876,66 @@
add_field("RadialVelocityKMSABS", function=_RadialVelocityABS,
convert_function=_ConvertRadialVelocityKMS, units=r"\rm{km}/\rm{s}")
+def _ParticleRadialVelocity(field, data):
+ normal = data.get_field_parameter('normal')
+ center = data.get_field_parameter('center')
+ bv = data.get_field_parameter("bulk_velocity")
+ pos = "particle_position_%s"
+ pos = np.array([data[pos % ax] for ax in "xyz"])
+ vel = "particle_velocity_%s"
+ vel = np.array([data[vel % ax] for ax in "xyz"])
+ theta = get_sph_theta(pos.copy(), center)
+ phi = get_sph_phi(pos.copy(), center)
+ pos = pos - np.reshape(center, (3, 1))
+ vel = vel - np.reshape(bv, (3, 1))
+ sphr = get_sph_r_component(vel, theta, phi, normal)
+ return sphr
+
+add_field("ParticleRadialVelocity", function=_ParticleRadialVelocity,
+ particle_type=True, units=r"\rm{cm}/\rm{s}",
+ validators=[ValidateParameter("normal"),
+ ValidateParameter("center")])
+
+def _ParticleThetaVelocity(field, data):
+ normal = data.get_field_parameter('normal')
+ center = data.get_field_parameter('center')
+ bv = data.get_field_parameter("bulk_velocity")
+ pos = "particle_position_%s"
+ pos = np.array([data[pos % ax] for ax in "xyz"])
+ vel = "particle_velocity_%s"
+ vel = np.array([data[vel % ax] for ax in "xyz"])
+ theta = get_sph_theta(pos.copy(), center)
+ phi = get_sph_phi(pos.copy(), center)
+ pos = pos - np.reshape(center, (3, 1))
+ vel = vel - np.reshape(bv, (3, 1))
+ spht = get_sph_theta_component(vel, theta, phi, normal)
+ return sphrt
+
+add_field("ParticleThetaVelocity", function=_ParticleThetaVelocity,
+ particle_type=True, units=r"\rm{cm}/\rm{s}",
+ validators=[ValidateParameter("normal"),
+ ValidateParameter("center")])
+
+def _ParticlePhiVelocity(field, data):
+ normal = data.get_field_parameter('normal')
+ center = data.get_field_parameter('center')
+ bv = data.get_field_parameter("bulk_velocity")
+ pos = "particle_position_%s"
+ pos = np.array([data[pos % ax] for ax in "xyz"])
+ vel = "particle_velocity_%s"
+ vel = np.array([data[vel % ax] for ax in "xyz"])
+ theta = get_sph_theta(pos.copy(), center)
+ phi = get_sph_phi(pos.copy(), center)
+ pos = pos - np.reshape(center, (3, 1))
+ vel = vel - np.reshape(bv, (3, 1))
+ sphp = get_sph_phi_component(vel, theta, phi, normal)
+ return sphrp
+
+add_field("ParticlePhiVelocity", function=_ParticleThetaVelocity,
+ particle_type=True, units=r"\rm{cm}/\rm{s}",
+ validators=[ValidateParameter("normal"),
+ ValidateParameter("center")])
+
def _TangentialVelocity(field, data):
return np.sqrt(data["VelocityMagnitude"]**2.0
- data["RadialVelocity"]**2.0)
@@ -940,13 +1005,6 @@
add_field("JeansMassMsun",function=_JeansMassMsun,
units=r"\rm{M_{\odot}}")
-# We add these fields so that the field detector can use them
-for field in ["particle_position_%s" % ax for ax in "xyz"]:
- # This marker should let everyone know not to use the fields, but NullFunc
- # should do that, too.
- add_field(field, function=NullFunc, particle_type = True,
- units=r"UNDEFINED")
-
def _pdensity(field, data):
pmass = data[('deposit','all_mass')]
np.divide(pmass, data["CellVolume"], pmass)
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/frontends/art/data_structures.py
--- a/yt/frontends/art/data_structures.py
+++ b/yt/frontends/art/data_structures.py
@@ -20,7 +20,7 @@
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
-.
+
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
@@ -43,7 +43,7 @@
from yt.data_objects.octree_subset import \
OctreeSubset
from yt.geometry.oct_container import \
- ARTOctreeContainer
+ OctreeContainer
from yt.data_objects.field_info_container import \
FieldInfoContainer, NullFunc
from .fields import \
@@ -106,21 +106,22 @@
allocate the requisite memory in the oct tree
"""
nv = len(self.fluid_field_list)
- self.domains = [ARTDomainFile(self.parameter_file, l+1, nv, l)
- for l in range(self.pf.max_level)]
+ self.oct_handler = OctreeContainer(
+ self.parameter_file.domain_dimensions/2, # dd is # of root cells
+ self.parameter_file.domain_left_edge,
+ self.parameter_file.domain_right_edge,
+ 1)
+ # The 1 here refers to domain_id == 1 always for ARTIO.
+ self.domains = [ARTDomainFile(self.parameter_file, nv,
+ self.oct_handler, 1)]
self.octs_per_domain = [dom.level_count.sum() for dom in self.domains]
self.total_octs = sum(self.octs_per_domain)
- self.oct_handler = ARTOctreeContainer(
- self.parameter_file.domain_dimensions/2, # dd is # of root cells
- self.parameter_file.domain_left_edge,
- self.parameter_file.domain_right_edge)
mylog.debug("Allocating %s octs", self.total_octs)
self.oct_handler.allocate_domains(self.octs_per_domain)
- for domain in self.domains:
- if domain.domain_level == 0:
- domain._read_amr_root(self.oct_handler)
- else:
- domain._read_amr_level(self.oct_handler)
+ domain = self.domains[0]
+ domain._read_amr_root(self.oct_handler)
+ domain._read_amr_level(self.oct_handler)
+ self.oct_handler.finalize()
def _detect_fields(self):
self.particle_field_list = particle_fields
@@ -154,28 +155,21 @@
"""
if getattr(dobj, "_chunk_info", None) is None:
# Get all octs within this oct handler
- mask = dobj.selector.select_octs(self.oct_handler)
- if mask.sum() == 0:
- mylog.debug("Warning: selected zero octs")
- counts = self.oct_handler.count_cells(dobj.selector, mask)
- # For all domains, figure out how many counts we have
- # and build a subset=mask of domains
- subsets = []
- for d, c in zip(self.domains, counts):
- if c < 1:
- continue
- subset = ARTDomainSubset(d, mask, c, d.domain_level)
- subsets.append(subset)
+ domains = [dom for dom in self.domains if
+ dom.included(dobj.selector)]
+ base_region = getattr(dobj, "base_region", dobj)
+ if len(domains) > 1:
+ mylog.debug("Identified %s intersecting domains", len(domains))
+ subsets = [ARTDomainSubset(base_region, domain, self.parameter_file)
+ for domain in domains]
dobj._chunk_info = subsets
- dobj.size = sum(counts)
- dobj.shape = (dobj.size,)
dobj._current_chunk = list(self._chunk_all(dobj))[0]
def _chunk_all(self, dobj):
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
# We pass the chunk both the current chunk and list of chunks,
# as well as the referring data source
- yield YTDataChunk(dobj, "all", oobjs, dobj.size)
+ yield YTDataChunk(dobj, "all", oobjs, None)
def _chunk_spatial(self, dobj, ngz, sort = None):
sobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
@@ -184,9 +178,7 @@
g = og.retrieve_ghost_zones(ngz, [], smoothed=True)
else:
g = og
- size = og.cell_count
- if size == 0: continue
- yield YTDataChunk(dobj, "spatial", [g], size)
+ yield YTDataChunk(dobj, "spatial", [g], None)
def _chunk_io(self, dobj):
"""
@@ -197,7 +189,7 @@
"""
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
for subset in oobjs:
- yield YTDataChunk(dobj, "io", [subset], subset.cell_count)
+ yield YTDataChunk(dobj, "io", [subset], None)
class ARTStaticOutput(StaticOutput):
@@ -449,11 +441,8 @@
return False
class ARTDomainSubset(OctreeSubset):
- def __init__(self, domain, mask, cell_count, domain_level):
- super(ARTDomainSubset, self).__init__(domain, mask, cell_count)
- self.domain_level = domain_level
- def fill_root(self, content, ftfields):
+ def fill(self, content, ftfields, selector):
"""
This is called from IOHandler. It takes content
which is a binary stream, reads the requested field
@@ -464,49 +453,42 @@
oct_handler = self.oct_handler
all_fields = self.domain.pf.h.fluid_field_list
fields = [f for ft, f in ftfields]
- level_offset = 0
field_idxs = [all_fields.index(f) for f in fields]
- dest = {}
+ source, tr = {}, {}
+ cell_count = selector.count_oct_cells(self.oct_handler, self.domain_id)
+ levels, cell_inds, file_inds = self.oct_handler.file_index_octs(
+ selector, self.domain_id, cell_count)
for field in fields:
- dest[field] = np.zeros(self.cell_count, 'float64')-1.
- level = self.domain_level
- source = {}
+ tr[field] = np.zeros(cell_count, 'float64')
data = _read_root_level(content, self.domain.level_child_offsets,
self.domain.level_count)
- for field, i in zip(fields, field_idxs):
- temp = np.reshape(data[i, :], self.domain.pf.domain_dimensions,
- order='F').astype('float64').T
- source[field] = temp
- level_offset += oct_handler.fill_level_from_grid(
- self.domain.domain_id,
- level, dest, source, self.mask, level_offset)
- return dest
-
- def fill_level(self, content, ftfields):
- oct_handler = self.oct_handler
- fields = [f for ft, f in ftfields]
- level_offset = 0
- dest = {}
- for field in fields:
- dest[field] = np.zeros(self.cell_count, 'float64')-1.
- level = self.domain_level
- no = self.domain.level_count[level]
- noct_range = [0, no]
- source = _read_child_level(
- content, self.domain.level_child_offsets,
- self.domain.level_offsets,
- self.domain.level_count, level, fields,
- self.domain.pf.domain_dimensions,
- self.domain.pf.parameters['ncell0'],
- noct_range=noct_range)
- nocts_filling = noct_range[1]-noct_range[0]
- level_offset += oct_handler.fill_level(self.domain.domain_id,
- level, dest, source,
- self.mask, level_offset,
- noct_range[0],
- nocts_filling)
- return dest
-
+ ns = (self.domain.pf.domain_dimensions.prod() / 8, 8)
+ for field, fi in zip(fields, field_idxs):
+ source[field] = np.empty(ns, dtype="float64", order="C")
+ dt = data[fi,:].reshape(self.domain.pf.domain_dimensions,
+ order="F")
+ for i in range(2):
+ for j in range(2):
+ for k in range(2):
+ ii = ((k*2)+j)*2+i
+ source[field][:,ii] = \
+ dt[i::2,j::2,k::2].ravel(order="F")
+ oct_handler.fill_level(0, levels, cell_inds, file_inds, tr, source)
+ del source
+ # Now we continue with the additional levels.
+ for level in range(1, self.pf.max_level + 1):
+ no = self.domain.level_count[level]
+ noct_range = [0, no]
+ source = _read_child_level(
+ content, self.domain.level_child_offsets,
+ self.domain.level_offsets,
+ self.domain.level_count, level, fields,
+ self.domain.pf.domain_dimensions,
+ self.domain.pf.parameters['ncell0'],
+ noct_range=noct_range)
+ oct_handler.fill_level(level, levels, cell_inds, file_inds, tr,
+ source)
+ return tr
class ARTDomainFile(object):
"""
@@ -518,14 +500,14 @@
_last_mask = None
_last_seletor_id = None
- def __init__(self, pf, domain_id, nvar, level):
+ def __init__(self, pf, nvar, oct_handler, domain_id):
self.nvar = nvar
self.pf = pf
self.domain_id = domain_id
- self.domain_level = level
self._level_count = None
self._level_oct_offsets = None
self._level_child_offsets = None
+ self.oct_handler = oct_handler
@property
def level_count(self):
@@ -533,7 +515,7 @@
if self._level_count is not None:
return self._level_count
self.level_offsets
- return self._level_count[self.domain_level]
+ return self._level_count
@property
def level_child_offsets(self):
@@ -573,26 +555,25 @@
"""
self.level_offsets
f = open(self.pf._file_amr, "rb")
- level = self.domain_level
- unitary_center, fl, iocts, nocts, root_level = _read_art_level_info(
- f,
- self._level_oct_offsets, level,
- coarse_grid=self.pf.domain_dimensions[0],
- root_level=self.pf.root_level)
- nocts_check = oct_handler.add(self.domain_id, level, nocts,
- unitary_center, self.domain_id)
- assert(nocts_check == nocts)
- mylog.debug("Added %07i octs on level %02i, cumulative is %07i",
- nocts, level, oct_handler.nocts)
+ for level in range(1, self.pf.max_level + 1):
+ unitary_center, fl, iocts, nocts, root_level = \
+ _read_art_level_info( f,
+ self._level_oct_offsets, level,
+ coarse_grid=self.pf.domain_dimensions[0],
+ root_level=self.pf.root_level)
+ nocts_check = oct_handler.add(self.domain_id, level,
+ unitary_center)
+ assert(nocts_check == nocts)
+ mylog.debug("Added %07i octs on level %02i, cumulative is %07i",
+ nocts, level, oct_handler.nocts)
def _read_amr_root(self, oct_handler):
self.level_offsets
f = open(self.pf._file_amr, "rb")
# add the root *cell* not *oct* mesh
- level = self.domain_level
root_octs_side = self.pf.domain_dimensions[0]/2
NX = np.ones(3)*root_octs_side
- octs_side = NX*2**level
+ octs_side = NX*2 # Level == 0
LE = np.array([0.0, 0.0, 0.0], dtype='float64')
RE = np.array([1.0, 1.0, 1.0], dtype='float64')
root_dx = (RE - LE) / NX
@@ -603,24 +584,14 @@
LL[1]:RL[1]:NX[1]*1j,
LL[2]:RL[2]:NX[2]*1j]
root_fc = np.vstack([p.ravel() for p in root_fc]).T
- nocts_check = oct_handler.add(self.domain_id, level,
- root_octs_side**3,
- root_fc, self.domain_id)
+ nocts_check = oct_handler.add(self.domain_id, 0, root_fc)
assert(oct_handler.nocts == root_fc.shape[0])
mylog.debug("Added %07i octs on level %02i, cumulative is %07i",
root_octs_side**3, 0, oct_handler.nocts)
- def select(self, selector):
- if id(selector) == self._last_selector_id:
- return self._last_mask
- self._last_mask = selector.fill_mask(self)
- self._last_selector_id = id(selector)
- return self._last_mask
-
- def count(self, selector):
- if id(selector) == self._last_selector_id:
- if self._last_mask is None:
- return 0
- return self._last_mask.sum()
- self.select(selector)
- return self.count(selector)
+ def included(self, selector):
+ return True
+ if getattr(selector, "domain_id", None) is not None:
+ return selector.domain_id == self.domain_id
+ domain_ids = self.pf.h.oct_handler.domain_identify(selector)
+ return self.domain_id in domain_ids
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/frontends/art/io.py
--- a/yt/frontends/art/io.py
+++ b/yt/frontends/art/io.py
@@ -30,6 +30,7 @@
import os
import os.path
+from yt.funcs import *
from yt.utilities.io_handler import \
BaseIOHandler
import yt.utilities.lib as au
@@ -55,7 +56,7 @@
# Chunks in this case will have affiliated domain subset objects
# Each domain subset will contain a hydro_offset array, which gives
# pointers to level-by-level hydro information
- tr = dict((f, np.empty(size, dtype='float64')) for f in fields)
+ tr = defaultdict(list)
cp = 0
for chunk in chunks:
for subset in chunk.objs:
@@ -63,18 +64,18 @@
f = open(subset.domain.pf._file_amr, "rb")
# This contains the boundary information, so we skim through
# and pick off the right vectors
- if subset.domain_level == 0:
- rv = subset.fill_root(f, fields)
- else:
- rv = subset.fill_level(f, fields)
+ rv = subset.fill(f, fields, selector)
for ft, f in fields:
- mylog.debug("Filling L%i %s with %s (%0.3e %0.3e) (%s:%s)",
- subset.domain_level,
- f, subset.cell_count, rv[f].min(), rv[f].max(),
- cp, cp+subset.cell_count)
- tr[(ft, f)][cp:cp+subset.cell_count] = rv.pop(f)
- cp += subset.cell_count
- return tr
+ d = rv.pop(f)
+ mylog.debug("Filling %s with %s (%0.3e %0.3e) (%s:%s)",
+ f, d.size, d.min(), d.max(),
+ cp, cp+d.size)
+ tr[(ft, f)].append(d)
+ cp += d.size
+ d = {}
+ for field in fields:
+ d[field] = np.concatenate(tr.pop(field))
+ return d
def _get_mask(self, selector, ftype):
key = (selector, ftype)
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/frontends/artio/_artio_caller.pyx
--- a/yt/frontends/artio/_artio_caller.pyx
+++ b/yt/frontends/artio/_artio_caller.pyx
@@ -341,7 +341,7 @@
if selected_mass[ispec] :
count = len(data[selected_mass[ispec]])
data[selected_mass[ispec]].resize(count+1)
- data[selected_mass[ispec]][count] = self.parameters["particle_species_mass"]
+ data[selected_mass[ispec]][count] = self.parameters["particle_species_mass"][0]
status = artio_particle_read_species_end( self.handle )
check_artio_status(status)
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/frontends/artio/data_structures.py
--- a/yt/frontends/artio/data_structures.py
+++ b/yt/frontends/artio/data_structures.py
@@ -32,7 +32,7 @@
artio_is_valid, artio_fileset
from yt.utilities.definitions import \
mpc_conversion, sec_conversion
-from .fields import ARTIOFieldInfo, KnownARTIOFields
+from .fields import ARTIOFieldInfo, KnownARTIOFields, b2t
from yt.funcs import *
from yt.geometry.geometry_handler import \
@@ -145,7 +145,7 @@
"""
Returns (in code units) the smallest cell size in the simulation.
"""
- return (self.parameter_file.domain_width/(2**self.max_level)).min()
+ return 1.0/(2**self.max_level)
def convert(self, unit):
return self.parameter_file.conversion_factors[unit]
@@ -391,7 +391,7 @@
list(set(art_to_yt[s] for s in
self.artio_parameters["particle_species_labels"])))
- self.current_time = self.artio_parameters["tl"][0]
+ self.current_time = b2t(self.artio_parameters["tl"][0])
# detect cosmology
if "abox" in self.artio_parameters:
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/frontends/artio/fields.py
--- a/yt/frontends/artio/fields.py
+++ b/yt/frontends/artio/fields.py
@@ -295,12 +295,12 @@
#add_artio_field("creation_time", function=NullFunc, particle_type=True)
def _particle_age(field, data):
- pa = b2t(data['creation_time'])
+ pa = b2t(data['stars','creation_time'])
# tr = np.zeros(pa.shape,dtype='float')-1.0
# tr[pa>0] = pa[pa>0]
tr = pa
return tr
-add_field("particle_age", function=_particle_age, units=r"\rm{s}",
+add_field(("stars","particle_age"), function=_particle_age, units=r"\rm{s}",
particle_type=True)
@@ -416,10 +416,10 @@
def b2t(tb, n=1e2, logger=None, **kwargs):
tb = np.array(tb)
- if isinstance(tb, 1.1):
+ if len(np.atleast_1d(tb)) == 1:
return a2t(b2a(tb))
if tb.shape == ():
- return a2t(b2a(tb))
+ return None
if len(tb) < n:
n = len(tb)
age_min = a2t(b2a(tb.max(), **kwargs), **kwargs)
@@ -434,7 +434,7 @@
ages = np.array(ages)
fb2t = np.interp(tb, tbs, ages)
#fb2t = interp1d(tbs,ages)
- return fb2t
+ return fb2t*1e9*31556926
def spread_ages(ages, logger=None, spread=.0e7*365*24*3600):
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/frontends/artio/setup.py
--- a/yt/frontends/artio/setup.py
+++ b/yt/frontends/artio/setup.py
@@ -16,7 +16,10 @@
include_dirs=["yt/frontends/artio/artio_headers/",
"yt/geometry/",
"yt/utilities/lib/"],
- depends=artio_sources)
+ depends=artio_sources +
+ ["yt/utilities/lib/fp_utils.pxd",
+ "yt/geometry/oct_container.pxd",
+ "yt/geometry/selection_routines.pxd"])
config.make_config_py() # installs __config__.py
#config.make_svn_version_py()
return config
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/frontends/enzo/fields.py
--- a/yt/frontends/enzo/fields.py
+++ b/yt/frontends/enzo/fields.py
@@ -507,7 +507,9 @@
add_enzo_field(("all", pf), function=NullFunc, convert_function=cfunc,
particle_type=True)
-for pf in ["creation_time", "dynamical_time", "metallicity_fraction"]:
+for pf in ["creation_time", "dynamical_time", "metallicity_fraction"] \
+ + ["particle_position_%s" % ax for ax in 'xyz'] \
+ + ["particle_velocity_%s" % ax for ax in 'xyz']:
add_enzo_field(pf, function=NullFunc,
validators = [ValidateDataField(pf)],
particle_type=True)
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/frontends/enzo/io.py
--- a/yt/frontends/enzo/io.py
+++ b/yt/frontends/enzo/io.py
@@ -153,13 +153,10 @@
for chunk in chunks:
data = self._read_chunk_data(chunk, fields)
for g in chunk.objs:
- mask = g.select(selector)
- if mask is None: continue
- nd = mask.sum()
for field in fields:
ftype, fname = field
- gdata = data[g.id].pop(fname).swapaxes(0,2)
- nd = mask_fill(rv[field], ind, mask, gdata)
+ ds = data[g.id].pop(fname).swapaxes(0,2)
+ nd = g.select(selector, ds, rv[field], ind) # caches
ind += nd
data.pop(g.id)
return rv
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/frontends/flash/io.py
--- a/yt/frontends/flash/io.py
+++ b/yt/frontends/flash/io.py
@@ -92,9 +92,6 @@
ind = 0
for chunk in chunks:
for g in chunk.objs:
- mask = g.select(selector) # caches
- if mask is None: continue
data = ds[g.id - g._id_offset,:,:,:].transpose()[mask]
- rv[field][ind:ind+data.size] = data
- ind += data.size
+ ind += g.select(selector, data, rv[field], ind) # caches
return rv
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/frontends/gdf/io.py
--- a/yt/frontends/gdf/io.py
+++ b/yt/frontends/gdf/io.py
@@ -84,13 +84,8 @@
ind = 0
for chunk in chunks:
for grid in chunk.objs:
- mask = grid.select(selector) # caches
- if mask is None:
- continue
+ data = fhandle[field_dname(grid.id, fname)][:]
if self.pf.field_ordering == 1:
- data = fhandle[field_dname(grid.id, fname)][:].swapaxes(0, 2)[mask]
- else:
- data = fhandle[field_dname(grid.id, fname)][mask]
- rv[field][ind:ind + data.size] = data
- ind += data.size
+ data = data.swapaxes(0, 2)
+ ind += g.select(selector, data, rv[field], ind) # caches
return rv
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/frontends/ramses/data_structures.py
--- a/yt/frontends/ramses/data_structures.py
+++ b/yt/frontends/ramses/data_structures.py
@@ -69,10 +69,14 @@
setattr(self, "%s_fn" % t, basename % t)
self._read_amr_header()
self._read_particle_header()
+ self._read_amr()
_hydro_offset = None
_level_count = None
+ def __repr__(self):
+ return "RAMSESDomainFile: %i" % self.domain_id
+
@property
def level_count(self):
if self._level_count is not None: return self._level_count
@@ -183,21 +187,26 @@
self.amr_header = hvals
self.amr_offset = f.tell()
self.local_oct_count = hvals['numbl'][self.pf.min_level:, self.domain_id - 1].sum()
+ self.total_oct_count = hvals['numbl'][self.pf.min_level:,:].sum(axis=0)
- def _read_amr(self, oct_handler):
+ def _read_amr(self):
"""Open the oct file, read in octs level-by-level.
For each oct, only the position, index, level and domain
are needed - its position in the octree is found automatically.
The most important is finding all the information to feed
oct_handler.add
"""
+ self.oct_handler = RAMSESOctreeContainer(self.pf.domain_dimensions/2,
+ self.pf.domain_left_edge, self.pf.domain_right_edge)
+ root_nodes = self.amr_header['numbl'][self.pf.min_level,:].sum()
+ self.oct_handler.allocate_domains(self.total_oct_count, root_nodes)
fb = open(self.amr_fn, "rb")
fb.seek(self.amr_offset)
f = cStringIO.StringIO()
f.write(fb.read())
f.seek(0)
mylog.debug("Reading domain AMR % 4i (%0.3e, %0.3e)",
- self.domain_id, self.local_oct_count, self.ngridbound.sum())
+ self.domain_id, self.total_oct_count.sum(), self.ngridbound.sum())
def _ng(c, l):
if c < self.amr_header['ncpu']:
ng = self.amr_header['numbl'][l, c]
@@ -206,7 +215,6 @@
self.amr_header['nboundary']*l]
return ng
min_level = self.pf.min_level
- total = 0
nx, ny, nz = (((i-1.0)/2.0) for i in self.amr_header['nx'])
for level in range(self.amr_header['nlevelmax']):
# Easier if do this 1-indexed
@@ -236,40 +244,34 @@
# rmap[:,i] = fpu.read_vector(f, "I")
# We don't want duplicate grids.
# Note that we're adding *grids*, not individual cells.
- if level >= min_level and cpu + 1 >= self.domain_id:
+ if level >= min_level:
assert(pos.shape[0] == ng)
- if cpu + 1 == self.domain_id:
- total += ng
- oct_handler.add(cpu + 1, level - min_level, ng, pos,
- self.domain_id)
+ n = self.oct_handler.add(cpu + 1, level - min_level, pos)
+ assert(n == ng)
+ self.oct_handler.finalize()
- def select(self, selector):
- if id(selector) == self._last_selector_id:
- return self._last_mask
- self._last_mask = selector.fill_mask(self)
- self._last_selector_id = id(selector)
- return self._last_mask
-
- def count(self, selector):
- if id(selector) == self._last_selector_id:
- if self._last_mask is None: return 0
- return self._last_mask.sum()
- self.select(selector)
- return self.count(selector)
+ def included(self, selector):
+ if getattr(selector, "domain_id", None) is not None:
+ return selector.domain_id == self.domain_id
+ domain_ids = self.oct_handler.domain_identify(selector)
+ return self.domain_id in domain_ids
class RAMSESDomainSubset(OctreeSubset):
- def fill(self, content, fields):
+ _domain_offset = 1
+
+ def fill(self, content, fields, selector):
# Here we get a copy of the file, which we skip through and read the
# bits we want.
oct_handler = self.oct_handler
all_fields = self.domain.pf.h.fluid_field_list
fields = [f for ft, f in fields]
tr = {}
- filled = pos = level_offset = 0
- min_level = self.domain.pf.min_level
+ cell_count = selector.count_oct_cells(self.oct_handler, self.domain_id)
+ levels, cell_inds, file_inds = self.oct_handler.file_index_octs(
+ selector, self.domain_id, cell_count)
for field in fields:
- tr[field] = np.zeros(self.cell_count, 'float64')
+ tr[field] = np.zeros(cell_count, 'float64')
for level, offset in enumerate(self.domain.hydro_offset):
if offset == -1: continue
content.seek(offset)
@@ -280,17 +282,10 @@
for i in range(8):
for field in all_fields:
if field not in fields:
- #print "Skipping %s in %s : %s" % (field, level,
- # self.domain.domain_id)
fpu.skip(content)
else:
- #print "Reading %s in %s : %s" % (field, level,
- # self.domain.domain_id)
temp[field][:,i] = fpu.read_vector(content, 'd') # cell 1
- level_offset += oct_handler.fill_level(self.domain.domain_id, level,
- tr, temp, self.mask, level_offset)
- #print "FILL (%s : %s) %s" % (self.domain.domain_id, level, level_offset)
- #print "DONE (%s) %s of %s" % (self.domain.domain_id, level_offset, self.cell_count)
+ oct_handler.fill_level(level, levels, cell_inds, file_inds, tr, temp)
return tr
class RAMSESGeometryHandler(OctreeGeometryHandler):
@@ -314,21 +309,6 @@
total_octs = sum(dom.local_oct_count #+ dom.ngridbound.sum()
for dom in self.domains)
self.num_grids = total_octs
- #this merely allocates space for the oct tree
- #and nothing else
- self.oct_handler = RAMSESOctreeContainer(
- self.parameter_file.domain_dimensions/2,
- self.parameter_file.domain_left_edge,
- self.parameter_file.domain_right_edge)
- mylog.debug("Allocating %s octs", total_octs)
- self.oct_handler.allocate_domains(
- [dom.local_oct_count #+ dom.ngridbound.sum()
- for dom in self.domains])
- #this actually reads every oct and loads it into the octree
- for dom in self.domains:
- dom._read_amr(self.oct_handler)
- #for dom in self.domains:
- # self.oct_handler.check(dom.domain_id)
def _detect_fields(self):
# TODO: Add additional fields
@@ -345,18 +325,19 @@
def _identify_base_chunk(self, dobj):
if getattr(dobj, "_chunk_info", None) is None:
- mask = dobj.selector.select_octs(self.oct_handler)
- counts = self.oct_handler.count_cells(dobj.selector, mask)
- subsets = [RAMSESDomainSubset(d, mask, c)
- for d, c in zip(self.domains, counts) if c > 0]
+ domains = [dom for dom in self.domains if
+ dom.included(dobj.selector)]
+ base_region = getattr(dobj, "base_region", dobj)
+ if len(domains) > 1:
+ mylog.debug("Identified %s intersecting domains", len(domains))
+ subsets = [RAMSESDomainSubset(base_region, domain, self.parameter_file)
+ for domain in domains]
dobj._chunk_info = subsets
- dobj.size = sum(counts)
- dobj.shape = (dobj.size,)
dobj._current_chunk = list(self._chunk_all(dobj))[0]
def _chunk_all(self, dobj):
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
- yield YTDataChunk(dobj, "all", oobjs, dobj.size)
+ yield YTDataChunk(dobj, "all", oobjs, None)
def _chunk_spatial(self, dobj, ngz, sort = None):
sobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
@@ -365,14 +346,12 @@
g = og.retrieve_ghost_zones(ngz, [], smoothed=True)
else:
g = og
- size = og.cell_count
- if size == 0: continue
- yield YTDataChunk(dobj, "spatial", [g], size)
+ yield YTDataChunk(dobj, "spatial", [g], None)
def _chunk_io(self, dobj):
oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
for subset in oobjs:
- yield YTDataChunk(dobj, "io", [subset], subset.cell_count)
+ yield YTDataChunk(dobj, "io", [subset], None)
class RAMSESStaticOutput(StaticOutput):
_hierarchy_class = RAMSESGeometryHandler
@@ -422,6 +401,8 @@
for unit in mpc_conversion.keys():
self.units[unit] = unit_l * mpc_conversion[unit] / mpc_conversion["cm"]
self.units['%sh' % unit] = self.units[unit] * self.hubble_constant
+ self.units['%scm' % unit] = (self.units[unit] /
+ (1 + self.current_redshift))
self.units['%shcm' % unit] = (self.units['%sh' % unit] /
(1 + self.current_redshift))
for unit in sec_conversion.keys():
@@ -479,7 +460,7 @@
self.omega_lambda = rheader["omega_l"]
self.omega_matter = rheader["omega_m"]
self.hubble_constant = rheader["H0"] / 100.0 # This is H100
- self.max_level = rheader['levelmax'] - rheader['levelmin']
+ self.max_level = rheader['levelmax'] - self.min_level
@classmethod
def _is_valid(self, *args, **kwargs):
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/frontends/ramses/io.py
--- a/yt/frontends/ramses/io.py
+++ b/yt/frontends/ramses/io.py
@@ -39,7 +39,7 @@
# Chunks in this case will have affiliated domain subset objects
# Each domain subset will contain a hydro_offset array, which gives
# pointers to level-by-level hydro information
- tr = dict((f, np.empty(size, dtype='float64')) for f in fields)
+ tr = defaultdict(list)
cp = 0
for chunk in chunks:
for subset in chunk.objs:
@@ -48,14 +48,16 @@
# This contains the boundary information, so we skim through
# and pick off the right vectors
content = cStringIO.StringIO(f.read())
- rv = subset.fill(content, fields)
+ rv = subset.fill(content, fields, selector)
for ft, f in fields:
- mylog.debug("Filling %s with %s (%0.3e %0.3e) (%s:%s)",
- f, subset.cell_count, rv[f].min(), rv[f].max(),
- cp, cp+subset.cell_count)
- tr[(ft, f)][cp:cp+subset.cell_count] = rv.pop(f)
- cp += subset.cell_count
- return tr
+ d = rv.pop(f)
+ mylog.debug("Filling %s with %s (%0.3e %0.3e) (%s zones)",
+ f, d.size, d.min(), d.max(), d.size)
+ tr[(ft, f)].append(d)
+ d = {}
+ for field in fields:
+ d[field] = np.concatenate(tr.pop(field))
+ return d
def _read_particle_selection(self, chunks, selector, fields):
size = 0
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/frontends/sph/data_structures.py
--- a/yt/frontends/sph/data_structures.py
+++ b/yt/frontends/sph/data_structures.py
@@ -32,10 +32,8 @@
from yt.utilities.fortran_utils import read_record
from yt.funcs import *
-from yt.geometry.oct_geometry_handler import \
- OctreeGeometryHandler
-from yt.geometry.oct_container import \
- ParticleOctreeContainer
+from yt.geometry.particle_geometry_handler import \
+ ParticleGeometryHandler
from yt.geometry.geometry_handler import \
GeometryHandler, YTDataChunk
from yt.data_objects.static_output import \
@@ -61,12 +59,12 @@
from yt.data_objects.field_info_container import \
FieldInfoContainer, NullFunc
-class ParticleDomainFile(object):
- def __init__(self, pf, io, domain_filename, domain_id):
+class ParticleFile(object):
+ def __init__(self, pf, io, filename, file_id):
self.pf = pf
self.io = weakref.proxy(io)
- self.domain_filename = domain_filename
- self.domain_id = domain_id
+ self.filename = filename
+ self.file_id = file_id
self.total_particles = self.io._count_particles(self)
def select(self, selector):
@@ -78,103 +76,21 @@
def _calculate_offsets(self, fields):
pass
-class ParticleDomainSubset(OctreeSubset):
- pass
-
-class ParticleGeometryHandler(OctreeGeometryHandler):
-
- def __init__(self, pf, data_style):
- self.data_style = data_style
- self.parameter_file = weakref.proxy(pf)
- # for now, the hierarchy file is the parameter file!
- self.hierarchy_filename = self.parameter_file.parameter_filename
- self.directory = os.path.dirname(self.hierarchy_filename)
- self.float_type = np.float64
- super(ParticleGeometryHandler, self).__init__(pf, data_style)
-
- def _initialize_oct_handler(self):
- self._setup_data_io()
- template = self.parameter_file.domain_template
- ndoms = self.parameter_file.domain_count
- cls = self.parameter_file._domain_class
- self.domains = [cls(self.parameter_file, self.io, template % {'num':i}, i)
- for i in range(ndoms)]
- total_particles = sum(sum(d.total_particles.values())
- for d in self.domains)
- self.oct_handler = ParticleOctreeContainer(
- self.parameter_file.domain_dimensions/2,
- self.parameter_file.domain_left_edge,
- self.parameter_file.domain_right_edge)
- self.oct_handler.n_ref = 64
- mylog.info("Allocating for %0.3e particles", total_particles)
- for dom in self.domains:
- self.io._initialize_octree(dom, self.oct_handler)
- self.oct_handler.finalize()
- self.max_level = self.oct_handler.max_level
- tot = self.oct_handler.linearly_count()
- mylog.info("Identified %0.3e octs", tot)
-
- def _detect_fields(self):
- # TODO: Add additional fields
- pfl = []
- for dom in self.domains:
- fl = self.io._identify_fields(dom)
- dom._calculate_offsets(fl)
- for f in fl:
- if f not in pfl: pfl.append(f)
- self.field_list = pfl
- pf = self.parameter_file
- pf.particle_types = tuple(set(pt for pt, pf in pfl))
- pf.particle_types += ('all',)
-
- def _setup_classes(self):
- dd = self._get_data_reader_dict()
- super(ParticleGeometryHandler, self)._setup_classes(dd)
- self.object_types.sort()
-
- def _identify_base_chunk(self, dobj):
- if getattr(dobj, "_chunk_info", None) is None:
- mask = dobj.selector.select_octs(self.oct_handler)
- counts = self.oct_handler.count_cells(dobj.selector, mask)
- subsets = [ParticleDomainSubset(d, mask, c)
- for d, c in zip(self.domains, counts) if c > 0]
- dobj._chunk_info = subsets
- dobj.size = sum(counts)
- dobj.shape = (dobj.size,)
- dobj._current_chunk = list(self._chunk_all(dobj))[0]
-
- def _chunk_all(self, dobj):
- oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
- yield YTDataChunk(dobj, "all", oobjs, dobj.size)
-
- def _chunk_spatial(self, dobj, ngz, sort = None):
- sobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
- for i,og in enumerate(sobjs):
- if ngz > 0:
- g = og.retrieve_ghost_zones(ngz, [], smoothed=True)
- else:
- g = og
- size = og.cell_count
- if size == 0: continue
- yield YTDataChunk(dobj, "spatial", [g], size)
-
- def _chunk_io(self, dobj):
- oobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
- for subset in oobjs:
- yield YTDataChunk(dobj, "io", [subset], subset.cell_count)
-
-class GadgetBinaryDomainFile(ParticleDomainFile):
- def __init__(self, pf, io, domain_filename, domain_id):
- with open(domain_filename, "rb") as f:
+class GadgetBinaryFile(ParticleFile):
+ def __init__(self, pf, io, filename, file_id):
+ with open(filename, "rb") as f:
self.header = read_record(f, pf._header_spec)
self._position_offset = f.tell()
+ f.seek(0, os.SEEK_END)
+ self._file_size = f.tell()
- super(GadgetBinaryDomainFile, self).__init__(pf, io,
- domain_filename, domain_id)
+ super(GadgetBinaryFile, self).__init__(pf, io,
+ filename, file_id)
def _calculate_offsets(self, field_list):
self.field_offsets = self.io._calculate_field_offsets(
- field_list, self.total_particles)
+ field_list, self.total_particles,
+ self._position_offset, self._file_size)
class ParticleStaticOutput(StaticOutput):
_unit_base = None
@@ -210,7 +126,7 @@
class GadgetStaticOutput(ParticleStaticOutput):
_hierarchy_class = ParticleGeometryHandler
- _domain_class = GadgetBinaryDomainFile
+ _file_class = GadgetBinaryFile
_fieldinfo_fallback = GadgetFieldInfo
_fieldinfo_known = KnownGadgetFields
_header_spec = (('Npart', 6, 'i'),
@@ -232,10 +148,8 @@
('unused', 16, 'i') )
def __init__(self, filename, data_style="gadget_binary",
- additional_fields = (), root_dimensions = 64,
+ additional_fields = (),
unit_base = None):
- self._root_dimensions = root_dimensions
- # Set up the template for domain files
self.storage_filename = None
if unit_base is not None and "UnitLength_in_cm" in unit_base:
# We assume this is comoving, because in the absence of comoving
@@ -268,7 +182,7 @@
self.domain_left_edge = np.zeros(3, "float64")
self.domain_right_edge = np.ones(3, "float64") * hvals["BoxSize"]
- self.domain_dimensions = np.ones(3, "int32") * self._root_dimensions
+ self.domain_dimensions = np.ones(3, "int32") * 2
self.periodicity = (True, True, True)
self.cosmological_simulation = 1
@@ -306,11 +220,11 @@
suffix = self.parameter_filename.rsplit(".", 1)[-1]
if hvals["NumFiles"] > 1:
- self.domain_template = "%s.%%(num)s" % (prefix)
+ self.filename_template = "%s.%%(num)s" % (prefix)
else:
- self.domain_template = self.parameter_filename
+ self.filename_template = self.parameter_filename
- self.domain_count = hvals["NumFiles"]
+ self.file_count = hvals["NumFiles"]
f.close()
@@ -341,17 +255,14 @@
class OWLSStaticOutput(GadgetStaticOutput):
_hierarchy_class = ParticleGeometryHandler
- _domain_class = ParticleDomainFile
+ _file_class = ParticleFile
_fieldinfo_fallback = OWLSFieldInfo # For now we have separate from Gadget
_fieldinfo_known = KnownOWLSFields
_header_spec = None # Override so that there's no confusion
- def __init__(self, filename, data_style="OWLS", root_dimensions = 64):
- self._root_dimensions = root_dimensions
- # Set up the template for domain files
+ def __init__(self, filename, data_style="OWLS"):
self.storage_filename = None
super(OWLSStaticOutput, self).__init__(filename, data_style,
- root_dimensions,
unit_base = None)
def __repr__(self):
@@ -372,7 +283,7 @@
self.current_time = hvals["Time_GYR"] * sec_conversion["Gyr"]
self.domain_left_edge = np.zeros(3, "float64")
self.domain_right_edge = np.ones(3, "float64") * hvals["BoxSize"]
- self.domain_dimensions = np.ones(3, "int32") * self._root_dimensions
+ self.domain_dimensions = np.ones(3, "int32") * 2
self.cosmological_simulation = 1
self.periodicity = (True, True, True)
self.current_redshift = hvals["Redshift"]
@@ -383,8 +294,8 @@
prefix = self.parameter_filename.split(".", 1)[0]
suffix = self.parameter_filename.rsplit(".", 1)[-1]
- self.domain_template = "%s.%%(num)i.%s" % (prefix, suffix)
- self.domain_count = hvals["NumFilesPerSnapshot"]
+ self.filename_template = "%s.%%(num)i.%s" % (prefix, suffix)
+ self.file_count = hvals["NumFilesPerSnapshot"]
# To avoid having to open files twice
self._unit_base = {}
@@ -407,23 +318,23 @@
pass
return False
-class TipsyDomainFile(ParticleDomainFile):
+class TipsyFile(ParticleFile):
def _calculate_offsets(self, field_list):
self.field_offsets = self.io._calculate_particle_offsets(self)
- def __init__(self, pf, io, domain_filename, domain_id):
+ def __init__(self, pf, io, filename, file_id):
# To go above 1 domain, we need to include an indexing step in the
# IOHandler, rather than simply reading from a single file.
- assert domain_id == 0
- super(TipsyDomainFile, self).__init__(pf, io,
- domain_filename, domain_id)
+ assert file_id == 0
+ super(TipsyFile, self).__init__(pf, io,
+ filename, file_id)
io._create_dtypes(self)
class TipsyStaticOutput(ParticleStaticOutput):
_hierarchy_class = ParticleGeometryHandler
- _domain_class = TipsyDomainFile
+ _file_class = TipsyFile
_fieldinfo_fallback = TipsyFieldInfo
_fieldinfo_known = KnownTipsyFields
_header_spec = (('time', 'd'),
@@ -435,15 +346,13 @@
('dummy', 'i'))
def __init__(self, filename, data_style="tipsy",
- root_dimensions = 64, endian = ">",
+ endian = ">",
field_dtypes = None,
domain_left_edge = None,
domain_right_edge = None,
unit_base = None,
cosmology_parameters = None):
self.endian = endian
- self._root_dimensions = root_dimensions
- # Set up the template for domain files
self.storage_filename = None
if domain_left_edge is None:
domain_left_edge = np.zeros(3, "float64") - 0.5
@@ -489,7 +398,7 @@
# NOTE: These are now set in the main initializer.
#self.domain_left_edge = np.zeros(3, "float64") - 0.5
#self.domain_right_edge = np.ones(3, "float64") + 0.5
- self.domain_dimensions = np.ones(3, "int32") * self._root_dimensions
+ self.domain_dimensions = np.ones(3, "int32") * 2
self.periodicity = (True, True, True)
self.cosmological_simulation = 1
@@ -506,8 +415,8 @@
self.parameters = hvals
- self.domain_template = self.parameter_filename
- self.domain_count = 1
+ self.filename_template = self.parameter_filename
+ self.file_count = 1
f.close()
diff -r 24915104ff1604a95f1342f13c10f8fc1c1c0b07 -r 559bca49e90fda8b2e57c941070bc1e681bd80ec yt/frontends/sph/fields.py
--- a/yt/frontends/sph/fields.py
+++ b/yt/frontends/sph/fields.py
@@ -234,3 +234,13 @@
func = _field_concat_slice(iname, axi)
OWLSFieldInfo.add_field(("all", oname + ax), function=func,
particle_type = True)
+
+def SmoothedGas(field, data):
+ pos = data["PartType0", "Coordinates"]
+ sml = data["PartType0", "SmoothingLength"]
+ dens = data["PartType0", "Density"]
+ rv = data.deposit(pos, [sml, dens], method="simple_smooth")
+ return rv
+OWLSFieldInfo.add_field(("deposit", "PartType0_simple_smooth"),
+ function = SmoothedGas, validators = [ValidateSpatial()])
+
This diff is so big that we needed to truncate the remainder.
Repository URL: https://bitbucket.org/yt_analysis/yt-3.0/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the yt-svn
mailing list