[yt-svn] commit/yt-3.0: 2 new changesets

[Bitbucket]

2 new commits in yt-3.0:


https://bitbucket.org/yt_analysis/yt-3.0/changeset/9cd456171607/
changeset:   9cd456171607
branch:      yt-3.0
user:        MatthewTurk
date:        2012-08-13 23:11:24
summary:     Attempting to ensmarten the FLASH chunked IO to allow it to run in parallel.
I estimate the number of procs to divide it between, but apply a minimum of 16
subdivisions.  The constant counting, particularly inside a parallel_objects
call, could be a problem.
affected #:  3 files

diff -r 808708d5e6613d68580598bacda78abaf07d06c7 -r 9cd4561716078f778c858afb74aad2fc8095c437 yt/frontends/flash/data_structures.py
--- a/yt/frontends/flash/data_structures.py
+++ b/yt/frontends/flash/data_structures.py
@@ -28,11 +28,14 @@
 import numpy as na
 import weakref
 
+from yt.config import ytcfg
 from yt.funcs import *
 from yt.data_objects.grid_patch import \
     AMRGridPatch
 from yt.geometry.grid_geometry_handler import \
     GridGeometryHandler
+from yt.geometry.geometry_handler import \
+    YTDataChunk
 from yt.data_objects.static_output import \
     StaticOutput
 from yt.utilities.definitions import \
@@ -203,6 +206,13 @@
     def _setup_data_io(self):
         self.io = io_registry[self.data_style](self.parameter_file)
 
+    def _chunk_io(self, dobj):
+        gobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
+        # We'll take the max of 128 and the number of processors
+        nl = max(16, ytcfg.getint("yt", "__topcomm_parallel_size"))
+        for gs in list_chunks(gobjs, nl):
+            yield YTDataChunk(dobj, "io", gs, self._count_selection(dobj, gs))
+
 class FLASHStaticOutput(StaticOutput):
     _hierarchy_class = FLASHHierarchy
     _fieldinfo_fallback = FLASHFieldInfo
@@ -435,5 +445,3 @@
         except:
             pass
         return False
-
-


diff -r 808708d5e6613d68580598bacda78abaf07d06c7 -r 9cd4561716078f778c858afb74aad2fc8095c437 yt/frontends/flash/io.py
--- a/yt/frontends/flash/io.py
+++ b/yt/frontends/flash/io.py
@@ -89,6 +89,7 @@
         return tr.astype("float64")
 
     def _read_fluid_selection(self, chunks, selector, fields, size):
+        chunks = list(chunks)
         if any((ftype != "gas" for ftype, fname in fields)):
             raise NotImplementedError
         f = self._handle


diff -r 808708d5e6613d68580598bacda78abaf07d06c7 -r 9cd4561716078f778c858afb74aad2fc8095c437 yt/funcs.py
--- a/yt/funcs.py
+++ b/yt/funcs.py
@@ -565,3 +565,13 @@
         
 def fix_axis(axis):
     return inv_axis_names.get(axis, axis)
+
+
+# This is a modification of:
+# http://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks-in-python
+def list_chunks(l, n):
+    """ Yield successive n-sized chunks from l.
+    """
+    ty = int(ceil(float(len(l))/n))
+    for i in xrange(0, n):
+        yield l[i*ty:(i+1)*ty]



https://bitbucket.org/yt_analysis/yt-3.0/changeset/57bed7b59fba/
changeset:   57bed7b59fba
branch:      yt-3.0
user:        MatthewTurk
date:        2012-08-13 23:17:54
summary:     Changing data_size to optionally be a callable, eliminating the need to pre-compute it
affected #:  2 files

diff -r 9cd4561716078f778c858afb74aad2fc8095c437 -r 57bed7b59fbadc6ec102fbd5a9ff4bb86268bb1e yt/frontends/flash/data_structures.py
--- a/yt/frontends/flash/data_structures.py
+++ b/yt/frontends/flash/data_structures.py
@@ -211,7 +211,7 @@
         # We'll take the max of 128 and the number of processors
         nl = max(16, ytcfg.getint("yt", "__topcomm_parallel_size"))
         for gs in list_chunks(gobjs, nl):
-            yield YTDataChunk(dobj, "io", gs, self._count_selection(dobj, gs))
+            yield YTDataChunk(dobj, "io", gs, self._count_selection)
 
 class FLASHStaticOutput(StaticOutput):
     _hierarchy_class = FLASHHierarchy


diff -r 9cd4561716078f778c858afb74aad2fc8095c437 -r 57bed7b59fbadc6ec102fbd5a9ff4bb86268bb1e yt/geometry/geometry_handler.py
--- a/yt/geometry/geometry_handler.py
+++ b/yt/geometry/geometry_handler.py
@@ -379,7 +379,13 @@
         self.dobj = dobj
         self.chunk_type = chunk_type
         self.objs = objs
-        self.data_size = data_size
+        self._data_size = data_size
+
+    @property
+    def data_size(self):
+        if callable(self._data_size):
+            self._data_size = self._data_size(self.dobj, self.objs)
+        return self._data_size
 
     _fcoords = None
     @property

Repository URL: https://bitbucket.org/yt_analysis/yt-3.0/

--

This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.