[yt-svn] commit/yt: xarthisius: Merged in MatthewTurk/yt (pull request #2446)
commits-noreply at bitbucket.org
commits-noreply at bitbucket.org
Tue Dec 6 08:22:49 PST 2016
1 new commit in yt:
https://bitbucket.org/yt_analysis/yt/commits/68cdeca5910d/
Changeset: 68cdeca5910d
Branch: yt
User: xarthisius
Date: 2016-12-06 16:22:23+00:00
Summary: Merged in MatthewTurk/yt (pull request #2446)
Adding Object Pool
Affected #: 13 files
diff -r c5bdb1f946cd9a9233b36a13a758840522b78914 -r 68cdeca5910d377f023631ecf6dbcae1d0794dc3 setup.py
--- a/setup.py
+++ b/setup.py
@@ -180,7 +180,7 @@
"particle_mesh_operations", "depth_first_octree", "fortran_reader",
"interpolators", "misc_utilities", "basic_octree", "image_utilities",
"points_in_volume", "quad_tree", "ray_integrators", "mesh_utilities",
- "amr_kdtools", "lenses", "distance_queue"
+ "amr_kdtools", "lenses", "distance_queue", "allocation_container"
]
for ext_name in lib_exts:
cython_extensions.append(
diff -r c5bdb1f946cd9a9233b36a13a758840522b78914 -r 68cdeca5910d377f023631ecf6dbcae1d0794dc3 yt/frontends/artio/_artio_caller.pyx
--- a/yt/frontends/artio/_artio_caller.pyx
+++ b/yt/frontends/artio/_artio_caller.pyx
@@ -7,7 +7,7 @@
SelectorObject, AlwaysSelector, OctreeSubsetSelector
from yt.utilities.lib.fp_utils cimport imax
from yt.geometry.oct_container cimport \
- SparseOctreeContainer
+ SparseOctreeContainer, OctObjectPool
from yt.geometry.oct_visitors cimport Oct
from yt.geometry.particle_deposit cimport \
ParticleDepositOperation
@@ -923,7 +923,7 @@
super(ARTIOOctreeContainer, self).__init__(dims, DLE, DRE)
self.artio_handle = range_handler.artio_handle
self.level_offset = 1
- self.domains = NULL
+ self.domains = OctObjectPool()
self.root_nodes = NULL
@cython.boundscheck(False)
@@ -949,7 +949,7 @@
# We only allow one root oct.
self.append_domain(oct_count)
- self.domains[self.num_domains - 1].con_id = sfc
+ self.domains.containers[self.num_domains - 1].con_id = sfc
oct_ind = -1
ipos = 0
@@ -1009,7 +1009,7 @@
source_arrays = []
ipos = -1
for i in range(self.num_domains):
- ipos = imax(ipos, self.domains[i].n)
+ ipos = imax(ipos, self.domains.containers[i].n)
for i in range(nf):
field_ind[i] = field_indices[i]
# Note that we subtract one, because we're not using the root mesh.
@@ -1029,13 +1029,13 @@
# double-loop to calculate domain_counts
# The cons should be in order
cdef np.int64_t sfc_start, sfc_end
- sfc_start = self.domains[0].con_id
- sfc_end = self.domains[self.num_domains - 1].con_id
+ sfc_start = self.domains.containers[0].con_id
+ sfc_end = self.domains.containers[self.num_domains - 1].con_id
status = artio_grid_cache_sfc_range(handle, sfc_start, sfc_end)
check_artio_status(status)
cdef np.int64_t offset = 0
for si in range(self.num_domains):
- sfc = self.domains[si].con_id
+ sfc = self.domains.containers[si].con_id
status = artio_grid_read_root_cell_begin( handle, sfc,
dpos, NULL, &num_oct_levels, num_octs_per_level)
check_artio_status(status)
diff -r c5bdb1f946cd9a9233b36a13a758840522b78914 -r 68cdeca5910d377f023631ecf6dbcae1d0794dc3 yt/geometry/oct_container.pxd
--- a/yt/geometry/oct_container.pxd
+++ b/yt/geometry/oct_container.pxd
@@ -22,6 +22,8 @@
from .oct_visitors cimport OctVisitor, Oct, cind
from libc.stdlib cimport bsearch, qsort, realloc, malloc, free
from libc.math cimport floor
+from yt.utilities.lib.allocation_container cimport \
+ ObjectPool, AllocationContainer
cdef int ORDER_MAX
@@ -35,28 +37,31 @@
np.int64_t ipos[3]
np.int32_t level
-cdef struct OctAllocationContainer
-cdef struct OctAllocationContainer:
- np.int64_t n
- np.int64_t n_assigned
- np.int64_t offset
- np.int64_t con_id
- OctAllocationContainer *next
- Oct *my_octs
-
cdef struct OctList
cdef struct OctList:
OctList *next
Oct *o
+# NOTE: This object *has* to be the same size as the AllocationContainer
+# object. There's an assert in the __cinit__ function.
+cdef struct OctAllocationContainer:
+ np.uint64_t n
+ np.uint64_t n_assigned
+ np.uint64_t offset
+ np.int64_t con_id # container id
+ Oct *my_objs
+
+cdef class OctObjectPool(ObjectPool):
+ cdef inline OctAllocationContainer *get_cont(self, int i):
+ return <OctAllocationContainer*> (&self.containers[i])
+
cdef OctList *OctList_append(OctList *list, Oct *o)
cdef int OctList_count(OctList *list)
cdef void OctList_delete(OctList *list)
cdef class OctreeContainer:
- cdef OctAllocationContainer *cont
- cdef OctAllocationContainer **domains
+ cdef public OctObjectPool domains
cdef Oct ****root_mesh
cdef int partial_coverage
cdef int level_offset
@@ -73,7 +78,7 @@
Oct *o, bint periodicity[3])
cdef void oct_bounds(self, Oct *, np.float64_t *, np.float64_t *)
# This function must return the offset from global-to-local domains; i.e.,
- # OctAllocationContainer.offset if such a thing exists.
+ # AllocationContainer.offset if such a thing exists.
cdef np.int64_t get_domain_offset(self, int domain_id)
cdef void visit_all_octs(self,
selection_routines.SelectorObject selector,
diff -r c5bdb1f946cd9a9233b36a13a758840522b78914 -r 68cdeca5910d377f023631ecf6dbcae1d0794dc3 yt/geometry/oct_container.pyx
--- a/yt/geometry/oct_container.pyx
+++ b/yt/geometry/oct_container.pyx
@@ -29,54 +29,16 @@
# NOTE that size_t might not be int
void *alloca(int)
-cdef OctAllocationContainer *allocate_octs(
- int n_octs, OctAllocationContainer *prev):
- cdef OctAllocationContainer *n_cont
- cdef Oct *oct
- cdef int n, i, j, k
- n_cont = <OctAllocationContainer *> malloc(
- sizeof(OctAllocationContainer))
- if prev == NULL:
- n_cont.offset = 0
- else:
- n_cont.offset = prev.offset + prev.n
- n_cont.my_octs = <Oct *> malloc(sizeof(Oct) * n_octs)
- if n_cont.my_octs == NULL:
- raise MemoryError
- n_cont.n = n_octs
- n_cont.n_assigned = 0
- n_cont.con_id = -1
- for n in range(n_octs):
- oct = &n_cont.my_octs[n]
- oct.file_ind = oct.domain = -1
- oct.domain_ind = n + n_cont.offset
- oct.children = NULL
- if prev != NULL:
- prev.next = n_cont
- n_cont.next = NULL
- return n_cont
-
-cdef void free_octs(
- OctAllocationContainer *first):
- cdef OctAllocationContainer *cur
- while first != NULL:
- cur = first
- for i in range(cur.n):
- if cur.my_octs[i].children != NULL:
- free(cur.my_octs[i].children)
- free(first.my_octs)
- first = cur.next
- free(cur)
-
# Here is the strategy for RAMSES containers:
# * Read each domain individually, creating *all* octs found in that domain
# file, even if they reside on other CPUs.
# * Only allocate octs that reside on >= domain
# * For all octs, insert into tree, which may require traversing existing
# octs
-# * Note that this does not allow OctAllocationContainer to exactly be a
-# chunk, but it is close. For IO chunking, we can theoretically examine
-# those octs that live inside a given allocator.
+# * Note that this does not allow one component of an ObjectPool (an
+# AllocationContainer) to exactly be a chunk, but it is close. For IO
+# chunking, we can theoretically examine those octs that live inside a
+# given allocator.
cdef class OctreeContainer:
@@ -86,13 +48,12 @@
# This will just initialize the root mesh octs
self.oref = over_refine
self.partial_coverage = partial_coverage
- self.cont = NULL
cdef int i, j, k, p
for i in range(3):
self.nn[i] = oct_domain_dimensions[i]
self.num_domains = 0
self.level_offset = 0
- self.domains = NULL
+ self.domains = OctObjectPool()
p = 0
self.nocts = 0 # Increment when initialized
for i in range(3):
@@ -112,16 +73,7 @@
@property
def oct_arrays(self):
- cdef OctAllocationContainer *cur = self.cont
- cdef Oct *this
- cdef int i
- cdef OctPadded[:] mm
- rv = []
- while cur != NULL:
- mm = <OctPadded[:cur.n_assigned]> (<OctPadded*> cur.my_octs)
- rv.append(np.asarray(mm))
- cur = cur.next
- return rv
+ return self.domains.to_arrays()
@classmethod
def load_octree(cls, header):
@@ -148,11 +100,11 @@
for i in range(3):
dds[i] = (obj.DRE[i] - obj.DLE[i]) / obj.nn[i]
# Pos is the center of the octs
- cdef OctAllocationContainer *cur = obj.domains[0]
+ cdef OctAllocationContainer *cur = obj.domains.get_cont(0)
cdef Oct *o
- cdef np.int64_t nfinest = 0
+ cdef np.uint64_t nfinest = 0
visitor.ref_mask = ref_mask
- visitor.octs = cur.my_octs
+ visitor.octs = cur.my_objs
visitor.nocts = &cur.n_assigned
visitor.nfinest = &nfinest
pos[0] = obj.DLE[0] + dds[0]/2.0
@@ -163,7 +115,7 @@
for k in range(obj.nn[2]):
if obj.root_mesh[i][j][k] != NULL:
raise RuntimeError
- o = &cur.my_octs[cur.n_assigned]
+ o = &cur.my_objs[cur.n_assigned]
o.domain_ind = o.file_ind = 0
o.domain = 1
obj.root_mesh[i][j][k] = o
@@ -185,7 +137,6 @@
return obj
def __dealloc__(self):
- free_octs(self.cont)
if self.root_mesh == NULL: return
for i in range(self.nn[0]):
if self.root_mesh[i] == NULL: continue
@@ -196,19 +147,6 @@
free(self.root_mesh[i])
free(self.root_mesh)
- def __iter__(self):
- #Get the next oct, will traverse domains
- #Note that oct containers can be sorted
- #so that consecutive octs are on the same domain
- cdef OctAllocationContainer *cur = self.cont
- cdef Oct *this
- cdef int i
- while cur != NULL:
- for i in range(cur.n_assigned):
- this = &cur.my_octs[i]
- yield (this.file_ind, this.domain_ind, this.domain)
- cur = cur.next
-
@cython.cdivision(True)
cdef void visit_all_octs(self, SelectorObject selector,
OctVisitor visitor, int vc = -1):
@@ -649,7 +587,7 @@
cdef np.float64_t dds[3]
no = pos.shape[0] #number of octs
if curdom > self.num_domains: return 0
- cdef OctAllocationContainer *cont = self.domains[curdom - 1]
+ cdef OctAllocationContainer *cont = self.domains.get_cont(curdom - 1)
cdef int initial = cont.n_assigned
cdef int in_boundary = 0
# How do we bootstrap ourselves?
@@ -692,35 +630,20 @@
def allocate_domains(self, domain_counts):
cdef int count, i
- cdef OctAllocationContainer *cur = self.cont
- assert(cur == NULL)
self.num_domains = len(domain_counts) # 1-indexed
- self.domains = <OctAllocationContainer **> malloc(
- sizeof(OctAllocationContainer *) * len(domain_counts))
for i, count in enumerate(domain_counts):
- cur = allocate_octs(count, cur)
- if self.cont == NULL: self.cont = cur
- self.domains[i] = cur
+ self.domains.append(count)
cdef void append_domain(self, np.int64_t domain_count):
self.num_domains += 1
- self.domains = <OctAllocationContainer **> realloc(self.domains,
- sizeof(OctAllocationContainer *) * self.num_domains)
- if self.domains == NULL: raise RuntimeError
- self.domains[self.num_domains - 1] = NULL
- cdef OctAllocationContainer *cur = NULL
- if self.num_domains > 1:
- cur = self.domains[self.num_domains - 2]
- cur = allocate_octs(domain_count, cur)
- if self.cont == NULL: self.cont = cur
- self.domains[self.num_domains - 1] = cur
+ self.domains.append(domain_count)
cdef Oct* next_root(self, int domain_id, int ind[3]):
cdef Oct *next = self.root_mesh[ind[0]][ind[1]][ind[2]]
if next != NULL: return next
- cdef OctAllocationContainer *cont = self.domains[domain_id - 1]
+ cdef OctAllocationContainer *cont = self.domains.get_cont(domain_id - 1)
if cont.n_assigned >= cont.n: raise RuntimeError
- next = &cont.my_octs[cont.n_assigned]
+ next = &cont.my_objs[cont.n_assigned]
cont.n_assigned += 1
self.root_mesh[ind[0]][ind[1]][ind[2]] = next
self.nocts += 1
@@ -737,9 +660,9 @@
for i in range(8):
parent.children[i] = NULL
if next != NULL: return next
- cdef OctAllocationContainer *cont = self.domains[domain_id - 1]
+ cdef OctAllocationContainer *cont = self.domains.get_cont(domain_id - 1)
if cont.n_assigned >= cont.n: raise RuntimeError
- next = &cont.my_octs[cont.n_assigned]
+ next = &cont.my_objs[cont.n_assigned]
cont.n_assigned += 1
parent.children[cind(ind[0],ind[1],ind[2])] = next
self.nocts += 1
@@ -842,6 +765,7 @@
self.oref = over_refine
for i in range(3):
self.nn[i] = domain_dimensions[i]
+ self.domains = OctObjectPool()
self.num_domains = 0
self.level_offset = 0
self.nocts = 0 # Increment when initialized
@@ -931,14 +855,14 @@
cdef Oct *next = NULL
self.get_root(ind, &next)
if next != NULL: return next
- cdef OctAllocationContainer *cont = self.domains[domain_id - 1]
+ cdef OctAllocationContainer *cont = self.domains.get_cont(domain_id - 1)
if cont.n_assigned >= cont.n:
print "Too many assigned."
return NULL
if self.num_root >= self.max_root:
print "Too many roots."
return NULL
- next = &cont.my_octs[cont.n_assigned]
+ next = &cont.my_objs[cont.n_assigned]
cont.n_assigned += 1
cdef np.int64_t key = 0
cdef OctKey *ikey = &self.root_nodes[self.num_root]
@@ -963,7 +887,6 @@
# This gets called BEFORE the superclass deallocation. But, both get
# called.
if self.root_nodes != NULL: free(self.root_nodes)
- if self.domains != NULL: free(self.domains)
cdef class RAMSESOctreeContainer(SparseOctreeContainer):
pass
@@ -1049,3 +972,41 @@
next = this.next
free(this)
this = next
+
+cdef class OctObjectPool(ObjectPool):
+ # This is an inherited version of the ObjectPool that provides setup and
+ # teardown functions for the individually allocated objects. These allow
+ # us to initialize the Octs to default values, and we can also free any
+ # allocated memory in them. Implementing _con_to_array also provides the
+ # opportunity to supply views of the octs in Python code.
+ def __cinit__(self):
+ # Base class will ALSO be called
+ self.itemsize = sizeof(Oct)
+ assert(sizeof(OctAllocationContainer) == sizeof(AllocationContainer))
+
+ cdef void setup_objs(self, void *obj, np.uint64_t n, np.uint64_t offset,
+ np.int64_t con_id):
+ cdef np.uint64_t i
+ cdef Oct* octs = <Oct *> obj
+ for n in range(n):
+ octs[n].file_ind = octs[n].domain = - 1
+ octs[n].domain_ind = n + offset
+ octs[n].children = NULL
+
+ cdef void teardown_objs(self, void *obj, np.uint64_t n, np.uint64_t offset,
+ np.int64_t con_id):
+ cdef np.uint64_t i, j
+ cdef Oct *my_octs = <Oct *> obj
+ for i in range(n):
+ if my_octs[i].children != NULL:
+ free(my_octs[i].children)
+ free(obj)
+
+ def _con_to_array(self, int i):
+ cdef AllocationContainer *obj = &self.containers[i]
+ if obj.n_assigned == 0:
+ return None
+ cdef OctPadded[:] mm = <OctPadded[:obj.n_assigned]> (
+ <OctPadded*> obj.my_objs)
+ rv = np.asarray(mm)
+ return rv
diff -r c5bdb1f946cd9a9233b36a13a758840522b78914 -r 68cdeca5910d377f023631ecf6dbcae1d0794dc3 yt/geometry/oct_visitors.pxd
--- a/yt/geometry/oct_visitors.pxd
+++ b/yt/geometry/oct_visitors.pxd
@@ -117,8 +117,8 @@
cdef class LoadOctree(OctVisitor):
cdef np.uint8_t[:] ref_mask
cdef Oct* octs
- cdef np.int64_t *nocts
- cdef np.int64_t *nfinest
+ cdef np.uint64_t *nocts
+ cdef np.uint64_t *nfinest
cdef inline int cind(int i, int j, int k):
# THIS ONLY WORKS FOR CHILDREN. It is not general for zones.
diff -r c5bdb1f946cd9a9233b36a13a758840522b78914 -r 68cdeca5910d377f023631ecf6dbcae1d0794dc3 yt/geometry/particle_deposit.pxd
--- a/yt/geometry/particle_deposit.pxd
+++ b/yt/geometry/particle_deposit.pxd
@@ -21,7 +21,7 @@
from libc.math cimport sqrt
from yt.utilities.lib.fp_utils cimport *
-from .oct_container cimport Oct, OctAllocationContainer, OctreeContainer
+from .oct_container cimport Oct, OctreeContainer
cdef extern from "platform_dep.h":
void *alloca(int)
diff -r c5bdb1f946cd9a9233b36a13a758840522b78914 -r 68cdeca5910d377f023631ecf6dbcae1d0794dc3 yt/geometry/particle_deposit.pyx
--- a/yt/geometry/particle_deposit.pyx
+++ b/yt/geometry/particle_deposit.pyx
@@ -22,8 +22,8 @@
from cpython cimport PyObject
from yt.utilities.lib.fp_utils cimport *
-from oct_container cimport Oct, OctAllocationContainer, \
- OctreeContainer, OctInfo
+from oct_container cimport \
+ Oct, OctreeContainer, OctInfo
from cpython.array cimport array, clone
from cython.view cimport memoryview as cymemview
from yt.utilities.lib.misc_utilities import OnceIndirect
diff -r c5bdb1f946cd9a9233b36a13a758840522b78914 -r 68cdeca5910d377f023631ecf6dbcae1d0794dc3 yt/geometry/particle_smooth.pxd
--- a/yt/geometry/particle_smooth.pxd
+++ b/yt/geometry/particle_smooth.pxd
@@ -21,7 +21,7 @@
from libc.math cimport sqrt
from yt.utilities.lib.fp_utils cimport *
-from oct_container cimport Oct, OctAllocationContainer, OctreeContainer
+from oct_container cimport Oct, OctreeContainer
from .particle_deposit cimport kernel_func, get_kernel_func, gind
from yt.utilities.lib.distance_queue cimport NeighborList, Neighbor_compare, \
r2dist, DistanceQueue
diff -r c5bdb1f946cd9a9233b36a13a758840522b78914 -r 68cdeca5910d377f023631ecf6dbcae1d0794dc3 yt/geometry/particle_smooth.pyx
--- a/yt/geometry/particle_smooth.pyx
+++ b/yt/geometry/particle_smooth.pyx
@@ -21,8 +21,8 @@
cimport cython
from libc.math cimport sqrt, fabs, sin, cos
-from oct_container cimport Oct, OctAllocationContainer, \
- OctreeContainer, OctInfo
+from oct_container cimport \
+ Oct, OctreeContainer, OctInfo
cdef void spherical_coord_setup(np.float64_t ipos[3], np.float64_t opos[3]):
diff -r c5bdb1f946cd9a9233b36a13a758840522b78914 -r 68cdeca5910d377f023631ecf6dbcae1d0794dc3 yt/geometry/selection_routines.pyx
--- a/yt/geometry/selection_routines.pyx
+++ b/yt/geometry/selection_routines.pyx
@@ -19,7 +19,7 @@
cimport cython
from libc.stdlib cimport malloc, free
from yt.utilities.lib.fp_utils cimport fclip, iclip, fmax, fmin, imin, imax
-from .oct_container cimport OctreeContainer, OctAllocationContainer, Oct
+from .oct_container cimport OctreeContainer, Oct
cimport oct_visitors
from .oct_visitors cimport cind
from yt.utilities.lib.volume_container cimport \
diff -r c5bdb1f946cd9a9233b36a13a758840522b78914 -r 68cdeca5910d377f023631ecf6dbcae1d0794dc3 yt/utilities/lib/allocation_container.pxd
--- /dev/null
+++ b/yt/utilities/lib/allocation_container.pxd
@@ -0,0 +1,35 @@
+"""
+An allocation container and memory pool
+
+
+
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2016, yt Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+cimport numpy as np
+from libc.stdlib cimport malloc, free, realloc
+
+cdef struct AllocationContainer:
+ np.uint64_t n
+ np.uint64_t n_assigned
+ np.uint64_t offset
+ np.int64_t con_id # container id
+ void *my_objs
+
+cdef class ObjectPool:
+ cdef public np.uint64_t itemsize
+ cdef np.uint64_t n_con
+ cdef AllocationContainer* containers
+ cdef void allocate_objs(self, int n_objs, np.int64_t con_id = ?) except *
+ cdef void setup_objs(self, void *obj, np.uint64_t count,
+ np.uint64_t offset, np.int64_t con_id)
+ cdef void teardown_objs(self, void *obj, np.uint64_t n, np.uint64_t offset,
+ np.int64_t con_id)
+
diff -r c5bdb1f946cd9a9233b36a13a758840522b78914 -r 68cdeca5910d377f023631ecf6dbcae1d0794dc3 yt/utilities/lib/allocation_container.pyx
--- /dev/null
+++ b/yt/utilities/lib/allocation_container.pyx
@@ -0,0 +1,123 @@
+"""
+An allocation container and memory pool
+
+
+
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2016, yt Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+cimport numpy as np
+import numpy as np
+
+cdef class ObjectPool:
+ def __cinit__(self):
+ """This class is *not* meant to be initialized directly, but instead
+ through subclasses. Those subclasses need to implement at a minimum
+ the setting of itemsize, but can optionally also implement setup_objs
+ and teardown_objs to either set default values or initialize additional
+ pointers and values, and then free them."""
+ self.containers = NULL
+ self.n_con = 0
+ self.itemsize = -1 # Never use the base class
+
+ def __dealloc__(self):
+ cdef int i
+ cdef AllocationContainer *obj
+ for i in range(self.n_con):
+ obj = &self.containers[i]
+ self.teardown_objs(obj.my_objs, obj.n, obj.offset, obj.con_id)
+ if self.containers != NULL:
+ free(self.containers)
+
+ def __getitem__(self, int i):
+ """This returns an array view (if possible and implemented in a
+ subclass) on the pool of objects specified by i."""
+ return self._con_to_array(i)
+
+ def __len__(self):
+ return self.n_con
+
+ def append(self, int n_objs, np.int64_t con_id = -1):
+ """This allocates a new batch of n_objs objects, with the container id
+ specified as con_id. Return value is a view on them."""
+ self.allocate_objs(n_objs, con_id)
+ return self[self.n_con - 1]
+
+ cdef void allocate_objs(self, int n_objs, np.int64_t con_id = -1) except *:
+ cdef AllocationContainer *n_cont, *prev
+ cdef int n, i, j, k
+ cdef char *obj # char so we can do pointer math
+ self.containers = <AllocationContainer*> realloc(
+ self.containers,
+ sizeof(AllocationContainer) * (self.n_con + 1))
+ n_cont = &self.containers[self.n_con]
+ if self.n_con == 0:
+ n_cont.offset = 0
+ else:
+ prev = &self.containers[self.n_con - 1]
+ n_cont.offset = prev.offset + prev.n
+ self.n_con += 1
+ n_cont.my_objs = malloc(self.itemsize * n_objs)
+ if n_cont.my_objs == NULL:
+ raise MemoryError
+ n_cont.n = n_objs
+ n_cont.n_assigned = 0
+ n_cont.con_id = con_id
+ obj = <char*> n_cont.my_objs
+ self.setup_objs(n_cont.my_objs, n_objs, n_cont.offset, n_cont.con_id)
+
+ cdef void setup_objs(self, void *obj, np.uint64_t count,
+ np.uint64_t offset, np.int64_t con_id):
+ """This can be overridden in the subclass, where it can initialize any
+ of the allocated objects."""
+ pass
+
+ cdef void teardown_objs(self, void *obj, np.uint64_t n, np.uint64_t offset,
+ np.int64_t con_id):
+ # We assume that these are all allocated and have no sub-allocations
+ """If overridden, additional behavior can be provided during teardown
+ of allocations. For instance, if you allocate some memory on each of
+ the allocated objects."""
+ if obj != NULL:
+ free(obj)
+
+ def to_arrays(self):
+ rv = []
+ cdef int i
+ for i in range(self.n_con):
+ rv.append(self._con_to_array(i))
+ return rv
+
+ def _con_to_array(self, int i):
+ """This has to be implemented in a subclass, and should return an
+ appropriate np.asarray() of a memoryview of self.my_objs."""
+ raise NotImplementedError
+
+cdef class BitmaskPool(ObjectPool):
+ def __cinit__(self):
+ """This is an example implementation of object pools for bitmasks
+ (uint8) arrays. It lets you reasonably quickly allocate a set of
+ uint8 arrays which can be accessed and modified, and then virtually
+ append to that."""
+ # Base class will ALSO be called
+ self.itemsize = sizeof(np.uint8_t)
+
+ cdef void setup_objs(self, void *obj, np.uint64_t n, np.uint64_t offset,
+ np.int64_t con_id):
+ cdef np.uint64_t i
+ cdef np.uint8_t *mask = <np.uint8_t *> obj
+ for i in range(n):
+ mask[i] = 0
+
+ def _con_to_array(self, int i):
+ cdef AllocationContainer *obj = &self.containers[i]
+ cdef np.uint8_t[:] o = <np.uint8_t[:obj.n]> (<np.uint8_t*> obj.my_objs)
+ rv = np.asarray(o)
+ return rv
diff -r c5bdb1f946cd9a9233b36a13a758840522b78914 -r 68cdeca5910d377f023631ecf6dbcae1d0794dc3 yt/utilities/lib/tests/test_allocation_container.py
--- /dev/null
+++ b/yt/utilities/lib/tests/test_allocation_container.py
@@ -0,0 +1,22 @@
+from yt.testing import \
+ assert_array_equal, assert_equal
+from yt.utilities.lib.allocation_container import \
+ BitmaskPool
+
+def test_bitmask_pool():
+ bmp = BitmaskPool()
+ assert_equal(len(bmp), 0)
+ bmp.append(100)
+ assert_equal(len(bmp), 1)
+ assert_equal(bmp[0].size, 100)
+ bmp.append(200)
+ assert_equal(len(bmp), 2)
+ assert_equal(bmp[0].size, 100)
+ assert_equal(bmp[1].size, 200)
+ assert_equal(sum(_.size for _ in bmp.to_arrays()), 300)
+ arrs = bmp.to_arrays()
+ assert_equal(arrs[0].size, 100)
+ assert_equal(arrs[1].size, 200)
+ arrs[0][:] = 1
+ arrs = bmp.to_arrays()
+ assert_array_equal(arrs[0], 1)
Repository URL: https://bitbucket.org/yt_analysis/yt/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the yt-svn
mailing list