[Yt-svn] commit/yt: 3 new changesets
Bitbucket
commits-noreply at bitbucket.org
Mon Oct 10 15:25:29 PDT 2011
3 new changesets in yt:
http://bitbucket.org/yt_analysis/yt/changeset/d36541d955a5/
changeset: d36541d955a5
branch: yt
user: MatthewTurk
date: 2011-10-10 21:17:52
summary: Re-arranging some deckchairs for the quadtree merging and removing unnecessary
barriers from in_order.
affected #: 1 file (-1 bytes)
--- a/yt/utilities/parallel_tools/parallel_analysis_interface.py Fri Oct 07 07:54:47 2011 -0700
+++ b/yt/utilities/parallel_tools/parallel_analysis_interface.py Mon Oct 10 15:17:52 2011 -0400
@@ -211,13 +211,11 @@
"""
@wraps(f1)
def in_order(*args, **kwargs):
- MPI.COMM_WORLD.Barrier()
if MPI.COMM_WORLD.rank == 0:
f1(*args, **kwargs)
MPI.COMM_WORLD.Barrier()
if MPI.COMM_WORLD.rank != 0:
f2(*args, **kwargs)
- MPI.COMM_WORLD.Barrier()
if not parallel_capable: return f1
return in_order
@@ -1255,9 +1253,8 @@
if not obj._distributed: return True
return (obj._owner == MPI.COMM_WORLD.rank)
- def _send_quadtree(self, target, qt, tgd, args):
+ def _send_quadtree(self, target, buf, tgd, args):
sizebuf = na.zeros(1, 'int64')
- buf = qt.tobuffer()
sizebuf[0] = buf[0].size
MPI.COMM_WORLD.Send([sizebuf, MPI.LONG], dest=target)
MPI.COMM_WORLD.Send([buf[0], MPI.INT], dest=target)
@@ -1273,9 +1270,7 @@
MPI.COMM_WORLD.Recv([buf[0], MPI.INT], source=target)
MPI.COMM_WORLD.Recv([buf[1], MPI.DOUBLE], source=target)
MPI.COMM_WORLD.Recv([buf[2], MPI.DOUBLE], source=target)
- qt = QuadTree(tgd, args[2])
- qt.frombuffer(*buf)
- return qt
+ return buf
@parallel_passthrough
def merge_quadtree_buffers(self, qt):
@@ -1294,13 +1289,16 @@
if (mask & rank) != 0:
target = (rank & ~mask) % size
#print "SENDING FROM %02i to %02i" % (rank, target)
- self._send_quadtree(target, qt, tgd, args)
+ buf = qt.tobuffer()
+ self._send_quadtree(target, buf, tgd, args)
#qt = self._recv_quadtree(target, tgd, args)
else:
target = (rank | mask)
if target < size:
#print "RECEIVING FROM %02i on %02i" % (target, rank)
- qto = self._recv_quadtree(target, tgd, args)
+ buf = self._recv_quadtree(target, tgd, args)
+ qto = QuadTree(tgd, args[2])
+ qto.frombuffer(*buf)
merge_quadtrees(qt, qto)
del qto
#self._send_quadtree(target, qt, tgd, args)
http://bitbucket.org/yt_analysis/yt/changeset/2b92374a62df/
changeset: 2b92374a62df
branch: yt
user: MatthewTurk
date: 2011-10-10 23:45:32
summary: Adding new function in Cython to replace _get_choices. Thanks to Sam for
helping me write it.
affected #: 2 files (-1 bytes)
--- a/yt/utilities/_amr_utils/misc_utilities.pyx Mon Oct 10 15:17:52 2011 -0400
+++ b/yt/utilities/_amr_utils/misc_utilities.pyx Mon Oct 10 17:45:32 2011 -0400
@@ -27,6 +27,10 @@
cimport numpy as np
cimport cython
+cdef extern from "stdlib.h":
+ # NOTE that size_t might not be int
+ void *alloca(int)
+
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
@@ -140,3 +144,51 @@
rv[fi] = field[ind[0], ind[1], ind[2]]
return rv
raise KeyError
+
+ at cython.boundscheck(False)
+ at cython.wraparound(False)
+ at cython.cdivision(True)
+def kdtree_get_choices(np.ndarray[np.float64_t, ndim=3] data,
+ np.ndarray[np.float64_t, ndim=1] l_corner,
+ np.ndarray[np.float64_t, ndim=1] r_corner):
+ cdef int i, j, k, dim, n_unique, best_dim, n_best, n_grids, addit, my_split
+ n_grids = data.shape[0]
+ cdef np.float64_t **uniquedims, *uniques
+ uniquedims = <np.float64_t **> alloca(3 * sizeof(np.float64_t*))
+ for i in range(3):
+ uniquedims[i] = <np.float64_t *> \
+ alloca(2*n_grids * sizeof(np.float64_t))
+ my_max = 0
+ for dim in range(3):
+ n_unique = 0
+ uniques = uniquedims[dim]
+ for i in range(n_grids):
+ # Check for disqualification
+ for j in range(2):
+ #print "Checking against", i,j,dim,data[i,j,dim]
+ if not (l_corner[dim] < data[i, j, dim] and
+ data[i, j, dim] < r_corner[dim]):
+ #print "Skipping ", data[i,j,dim]
+ continue
+ skipit = 0
+ # Add our left ...
+ for k in range(n_unique):
+ if uniques[k] == data[i, j, dim]:
+ skipit = 1
+ #print "Identified", uniques[k], data[i,j,dim], n_unique
+ break
+ if skipit == 0:
+ uniques[n_unique] = data[i, j, dim]
+ n_unique += 1
+ if n_unique > my_max:
+ best_dim = dim
+ my_max = n_unique
+ my_split = (n_unique-1)/2
+ # I recognize how lame this is.
+ cdef np.ndarray[np.float64_t, ndim=1] tarr = np.empty(my_max, dtype='float64')
+ for i in range(my_max):
+ #print "Setting tarr: ", i, uniquedims[best_dim][i]
+ tarr[i] = uniquedims[best_dim][i]
+ tarr.sort()
+ # Return out unique values
+ return best_dim, tarr[my_split]
--- a/yt/utilities/amr_kdtree/amr_kdtree.py Mon Oct 10 15:17:52 2011 -0400
+++ b/yt/utilities/amr_kdtree/amr_kdtree.py Mon Oct 10 17:45:32 2011 -0400
@@ -28,7 +28,7 @@
import numpy as na
from yt.funcs import *
from yt.visualization.volume_rendering.grid_partitioner import HomogenizedVolume
-from yt.utilities.amr_utils import PartitionedGrid
+from yt.utilities.amr_utils import PartitionedGrid, kdtree_get_choices
from yt.utilities.performance_counters import yt_counters, time_function
import yt.utilities.parallel_tools.parallel_analysis_interface as PT
from copy import deepcopy
@@ -1060,18 +1060,8 @@
# For some reason doing dim 0 separately is slightly faster.
# This could be rewritten to all be in the loop below.
- best_dim = 0
- best_choices = na.unique(data[:,:,0][(current_node.l_corner[0] < data[:,:,0]) &
- (data[:,:,0] < current_node.r_corner[0])])
-
- for d in range(1,3):
- choices = na.unique(data[:,:,d][(current_node.l_corner[d] < data[:,:,d]) &
- (data[:,:,d] < current_node.r_corner[d])])
-
- if choices.size > best_choices.size:
- best_choices, best_dim = choices, d
-
- split = best_choices[(len(best_choices)-1)/2]
+ best_dim, split = kdtree_get_choices(data, current_node.l_corner,
+ current_node.r_corner)
return data[:,:,best_dim], best_dim, split
def _build_dividing_node(self, current_node):
http://bitbucket.org/yt_analysis/yt/changeset/7bd3ed104c38/
changeset: 7bd3ed104c38
branch: yt
user: MatthewTurk
date: 2011-10-11 00:16:30
summary: Removing some split & id calculations from Python, putting into Cython.
affected #: 2 files (-1 bytes)
--- a/yt/utilities/_amr_utils/misc_utilities.pyx Mon Oct 10 17:45:32 2011 -0400
+++ b/yt/utilities/_amr_utils/misc_utilities.pyx Mon Oct 10 18:16:30 2011 -0400
@@ -153,7 +153,7 @@
np.ndarray[np.float64_t, ndim=1] r_corner):
cdef int i, j, k, dim, n_unique, best_dim, n_best, n_grids, addit, my_split
n_grids = data.shape[0]
- cdef np.float64_t **uniquedims, *uniques
+ cdef np.float64_t **uniquedims, *uniques, split
uniquedims = <np.float64_t **> alloca(3 * sizeof(np.float64_t*))
for i in range(3):
uniquedims[i] = <np.float64_t *> \
@@ -190,5 +190,17 @@
#print "Setting tarr: ", i, uniquedims[best_dim][i]
tarr[i] = uniquedims[best_dim][i]
tarr.sort()
+ split = tarr[my_split]
+ cdef np.ndarray[np.uint8_t, ndim=1] less_ids = np.empty(n_grids, dtype='uint8')
+ cdef np.ndarray[np.uint8_t, ndim=1] greater_ids = np.empty(n_grids, dtype='uint8')
+ for i in range(n_grids):
+ if data[i, 0, best_dim] < split:
+ less_ids[i] = 1
+ else:
+ less_ids[i] = 0
+ if data[i, 1, best_dim] > split:
+ greater_ids[i] = 1
+ else:
+ greater_ids[i] = 0
# Return out unique values
- return best_dim, tarr[my_split]
+ return best_dim, split, less_ids.view("bool"), greater_ids.view("bool")
--- a/yt/utilities/amr_kdtree/amr_kdtree.py Mon Oct 10 17:45:32 2011 -0400
+++ b/yt/utilities/amr_kdtree/amr_kdtree.py Mon Oct 10 18:16:30 2011 -0400
@@ -48,11 +48,11 @@
chosen by specifying the `current_left` or `current_right`.
"""
if(current_left is not None):
- new_left = na.array([current_left[0],current_left[1],current_left[2]])
+ new_left = current_left.copy()
new_left[split_dim] = split
return new_left
elif(current_right is not None):
- new_right = na.array([current_right[0],current_right[1],current_right[2]])
+ new_right = current_right.copy()
new_right[split_dim] = split
return new_right
@@ -736,7 +736,8 @@
thisnode.ri = na.rint((thisnode.r_corner-gle)/dds).astype('int32')
thisnode.dims = (thisnode.ri - thisnode.li).astype('int32')
# Here the cost is actually inversely proportional to 4**Level (empirical)
- thisnode.cost = (na.prod(thisnode.dims)/4.**thisnode.grid.Level).astype('int64')
+ #thisnode.cost = (na.prod(thisnode.dims)/4.**thisnode.grid.Level).astype('int64')
+ thisnode.cost = 1.0
# Here is the old way
# thisnode.cost = na.prod(thisnode.dims).astype('int64')
@@ -1060,9 +1061,9 @@
# For some reason doing dim 0 separately is slightly faster.
# This could be rewritten to all be in the loop below.
- best_dim, split = kdtree_get_choices(data, current_node.l_corner,
- current_node.r_corner)
- return data[:,:,best_dim], best_dim, split
+ best_dim, split, less_ids, greater_ids = \
+ kdtree_get_choices(data, current_node.l_corner, current_node.r_corner)
+ return data[:,:,best_dim], best_dim, split, less_ids, greater_ids
def _build_dividing_node(self, current_node):
'''
@@ -1070,12 +1071,14 @@
left and right children.
'''
- data,best_dim,split = self._get_choices(current_node)
+ data,best_dim,split,less_ids,greater_ids = self._get_choices(current_node)
current_node.split_ax = best_dim
current_node.split_pos = split
- less_ids = na.nonzero(data[:,0] < split)[0]
- greater_ids = na.nonzero(split < data[:,1])[0]
+ #less_ids0 = (data[:,0] < split)
+ #greater_ids0 = (split < data[:,1])
+ #assert(na.all(less_ids0 == less_ids))
+ #assert(na.all(greater_ids0 == greater_ids))
current_node.left_child = MasterNode(my_id=_lchild_id(current_node.id),
parent=current_node,
Repository URL: https://bitbucket.org/yt_analysis/yt/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the yt-svn
mailing list