[yt-svn] commit/yt: 3 new changesets
commits-noreply at bitbucket.org
commits-noreply at bitbucket.org
Thu Apr 24 00:30:01 PDT 2014
3 new commits in yt:
https://bitbucket.org/yt_analysis/yt/commits/ac543195aa07/
Changeset: ac543195aa07
Branch: yt-3.0
User: MatthewTurk
Date: 2014-04-23 14:23:38
Summary: Make parallelism mostly run-time checked.
Almost all of the functions are relatively fast without a lot of tight loop
inclusion, so we make parallel_capable a runtime rather than import time check.
Also, remove parallel_splitter, which was only used in "save_data", and which
should be fixed differently elsewhere.
Affected #: 7 files
diff -r 1fb43c387d932803cba2505a859eafe1a063e2c8 -r ac543195aa07f98361565fd701e2d7b85b8ba8e0 doc/source/reference/api/api.rst
--- a/doc/source/reference/api/api.rst
+++ b/doc/source/reference/api/api.rst
@@ -660,7 +660,6 @@
~yt.utilities.parallel_tools.parallel_analysis_interface.parallel_passthrough
~yt.utilities.parallel_tools.parallel_analysis_interface.parallel_root_only
~yt.utilities.parallel_tools.parallel_analysis_interface.parallel_simple_proxy
- ~yt.utilities.parallel_tools.parallel_analysis_interface.parallel_splitter
Math Utilities
--------------
diff -r 1fb43c387d932803cba2505a859eafe1a063e2c8 -r ac543195aa07f98361565fd701e2d7b85b8ba8e0 yt/geometry/coordinate_handler.py
--- a/yt/geometry/coordinate_handler.py
+++ b/yt/geometry/coordinate_handler.py
@@ -24,7 +24,7 @@
from yt.utilities.io_handler import io_registry
from yt.utilities.logger import ytLogger as mylog
from yt.utilities.parallel_tools.parallel_analysis_interface import \
- ParallelAnalysisInterface, parallel_splitter
+ ParallelAnalysisInterface
from yt.utilities.lib.misc_utilities import \
pixelize_cylinder
import yt.visualization._MPL as _MPL
diff -r 1fb43c387d932803cba2505a859eafe1a063e2c8 -r ac543195aa07f98361565fd701e2d7b85b8ba8e0 yt/geometry/geometry_handler.py
--- a/yt/geometry/geometry_handler.py
+++ b/yt/geometry/geometry_handler.py
@@ -36,7 +36,7 @@
from yt.utilities.io_handler import io_registry
from yt.utilities.logger import ytLogger as mylog
from yt.utilities.parallel_tools.parallel_analysis_interface import \
- ParallelAnalysisInterface, parallel_splitter
+ ParallelAnalysisInterface
from yt.utilities.exceptions import YTFieldNotFound
class Index(ParallelAnalysisInterface):
@@ -126,7 +126,7 @@
if getattr(self, "io", None) is not None: return
self.io = io_registry[self.dataset_type](self.parameter_file)
- def _save_data(self, array, node, name, set_attr=None, force=False, passthrough = False):
+ def save_data(self, array, node, name, set_attr=None, force=False, passthrough = False):
"""
Arbitrary numpy data will be saved to the region in the datafile
described by *node* and *name*. If data file does not exist, it throws
@@ -157,14 +157,6 @@
del self._data_file
self._data_file = h5py.File(self.__data_filename, self._data_mode)
- save_data = parallel_splitter(_save_data, _reload_data_file)
-
- def _reset_save_data(self,round_robin=False):
- if round_robin:
- self.save_data = self._save_data
- else:
- self.save_data = parallel_splitter(self._save_data, self._reload_data_file)
-
def save_object(self, obj, name):
"""
Save an object (*obj*) to the data_file using the Pickle protocol,
diff -r 1fb43c387d932803cba2505a859eafe1a063e2c8 -r ac543195aa07f98361565fd701e2d7b85b8ba8e0 yt/geometry/grid_geometry_handler.py
--- a/yt/geometry/grid_geometry_handler.py
+++ b/yt/geometry/grid_geometry_handler.py
@@ -31,7 +31,7 @@
from yt.utilities.physical_constants import sec_per_year
from yt.utilities.io_handler import io_registry
from yt.utilities.parallel_tools.parallel_analysis_interface import \
- ParallelAnalysisInterface, parallel_splitter
+ ParallelAnalysisInterface
from yt.utilities.lib.GridTree import GridTree, MatchPointsToGrids
from yt.data_objects.data_containers import data_object_registry
diff -r 1fb43c387d932803cba2505a859eafe1a063e2c8 -r ac543195aa07f98361565fd701e2d7b85b8ba8e0 yt/geometry/oct_geometry_handler.py
--- a/yt/geometry/oct_geometry_handler.py
+++ b/yt/geometry/oct_geometry_handler.py
@@ -30,7 +30,7 @@
from yt.utilities.definitions import MAXLEVEL
from yt.utilities.io_handler import io_registry
from yt.utilities.parallel_tools.parallel_analysis_interface import \
- ParallelAnalysisInterface, parallel_splitter
+ ParallelAnalysisInterface
from yt.data_objects.data_containers import data_object_registry
diff -r 1fb43c387d932803cba2505a859eafe1a063e2c8 -r ac543195aa07f98361565fd701e2d7b85b8ba8e0 yt/geometry/particle_geometry_handler.py
--- a/yt/geometry/particle_geometry_handler.py
+++ b/yt/geometry/particle_geometry_handler.py
@@ -32,7 +32,7 @@
from yt.utilities.definitions import MAXLEVEL
from yt.utilities.io_handler import io_registry
from yt.utilities.parallel_tools.parallel_analysis_interface import \
- ParallelAnalysisInterface, parallel_splitter
+ ParallelAnalysisInterface
from yt.data_objects.data_containers import data_object_registry
from yt.data_objects.octree_subset import ParticleOctreeSubset
diff -r 1fb43c387d932803cba2505a859eafe1a063e2c8 -r ac543195aa07f98361565fd701e2d7b85b8ba8e0 yt/utilities/parallel_tools/parallel_analysis_interface.py
--- a/yt/utilities/parallel_tools/parallel_analysis_interface.py
+++ b/yt/utilities/parallel_tools/parallel_analysis_interface.py
@@ -179,14 +179,13 @@
used on objects that subclass
:class:`~yt.utilities.parallel_tools.parallel_analysis_interface.ParallelAnalysisInterface`.
"""
- if not parallel_capable: return func
@wraps(func)
def single_proc_results(self, *args, **kwargs):
retval = None
if hasattr(self, "dont_wrap"):
if func.func_name in self.dont_wrap:
return func(self, *args, **kwargs)
- if self._processing or not self._distributed:
+ if not parallel_capable or self._processing or not self._distributed:
return func(self, *args, **kwargs)
comm = _get_comm((self,))
if self._owner == comm.rank:
@@ -243,6 +242,8 @@
"""
@wraps(func)
def barrierize(*args, **kwargs):
+ if not parallel_capable:
+ return func(*args, **kwargs)
mylog.debug("Entering barrier before %s", func.func_name)
comm = _get_comm(args)
comm.barrier()
@@ -250,26 +251,7 @@
mylog.debug("Entering barrier after %s", func.func_name)
comm.barrier()
return retval
- if parallel_capable:
- return barrierize
- else:
- return func
-
-def parallel_splitter(f1, f2):
- """
- This function returns either the function *f1* or *f2* depending on whether
- or not we're the root processor. Mainly used in class definitions.
- """
- @wraps(f1)
- def in_order(*args, **kwargs):
- comm = _get_comm(args)
- if comm.rank == 0:
- f1(*args, **kwargs)
- comm.barrier()
- if comm.rank != 0:
- f2(*args, **kwargs)
- if not parallel_capable: return f1
- return in_order
+ return barrierize
def parallel_root_only(func):
"""
@@ -278,6 +260,8 @@
"""
@wraps(func)
def root_only(*args, **kwargs):
+ if not parallel_capable:
+ return func(*args, **kwargs)
comm = _get_comm(args)
rv = None
if comm.rank == 0:
@@ -292,8 +276,7 @@
all_clear = comm.mpi_bcast(all_clear)
if not all_clear: raise RuntimeError
return rv
- if parallel_capable: return root_only
- return func
+ return root_only
class Workgroup(object):
def __init__(self, size, ranks, comm, name):
https://bitbucket.org/yt_analysis/yt/commits/f1d0c2a75916/
Changeset: f1d0c2a75916
Branch: yt-3.0
User: MatthewTurk
Date: 2014-04-24 02:26:47
Summary: Adding parallel_root_only to save_data.
Affected #: 1 file
diff -r ac543195aa07f98361565fd701e2d7b85b8ba8e0 -r f1d0c2a759169ddb209ae34b42e5dfbb3db863e9 yt/geometry/geometry_handler.py
--- a/yt/geometry/geometry_handler.py
+++ b/yt/geometry/geometry_handler.py
@@ -36,7 +36,7 @@
from yt.utilities.io_handler import io_registry
from yt.utilities.logger import ytLogger as mylog
from yt.utilities.parallel_tools.parallel_analysis_interface import \
- ParallelAnalysisInterface
+ ParallelAnalysisInterface, parallel_root_only
from yt.utilities.exceptions import YTFieldNotFound
class Index(ParallelAnalysisInterface):
@@ -126,6 +126,7 @@
if getattr(self, "io", None) is not None: return
self.io = io_registry[self.dataset_type](self.parameter_file)
+ @parallel_root_only
def save_data(self, array, node, name, set_attr=None, force=False, passthrough = False):
"""
Arbitrary numpy data will be saved to the region in the datafile
https://bitbucket.org/yt_analysis/yt/commits/7bd133d91d39/
Changeset: 7bd133d91d39
Branch: yt-3.0
User: brittonsmith
Date: 2014-04-24 09:29:55
Summary: Merged in MatthewTurk/yt/yt-3.0 (pull request #849)
Make parallelism mostly run-time checked.
Affected #: 7 files
diff -r 0d7d60aa2739292524a230ee3b8cbb28d1d2808a -r 7bd133d91d3934b8f4617f73b01cd102fb23af8b doc/source/reference/api/api.rst
--- a/doc/source/reference/api/api.rst
+++ b/doc/source/reference/api/api.rst
@@ -660,7 +660,6 @@
~yt.utilities.parallel_tools.parallel_analysis_interface.parallel_passthrough
~yt.utilities.parallel_tools.parallel_analysis_interface.parallel_root_only
~yt.utilities.parallel_tools.parallel_analysis_interface.parallel_simple_proxy
- ~yt.utilities.parallel_tools.parallel_analysis_interface.parallel_splitter
Math Utilities
--------------
diff -r 0d7d60aa2739292524a230ee3b8cbb28d1d2808a -r 7bd133d91d3934b8f4617f73b01cd102fb23af8b yt/geometry/coordinate_handler.py
--- a/yt/geometry/coordinate_handler.py
+++ b/yt/geometry/coordinate_handler.py
@@ -24,7 +24,7 @@
from yt.utilities.io_handler import io_registry
from yt.utilities.logger import ytLogger as mylog
from yt.utilities.parallel_tools.parallel_analysis_interface import \
- ParallelAnalysisInterface, parallel_splitter
+ ParallelAnalysisInterface
from yt.utilities.lib.misc_utilities import \
pixelize_cylinder
import yt.visualization._MPL as _MPL
diff -r 0d7d60aa2739292524a230ee3b8cbb28d1d2808a -r 7bd133d91d3934b8f4617f73b01cd102fb23af8b yt/geometry/geometry_handler.py
--- a/yt/geometry/geometry_handler.py
+++ b/yt/geometry/geometry_handler.py
@@ -36,7 +36,7 @@
from yt.utilities.io_handler import io_registry
from yt.utilities.logger import ytLogger as mylog
from yt.utilities.parallel_tools.parallel_analysis_interface import \
- ParallelAnalysisInterface, parallel_splitter
+ ParallelAnalysisInterface, parallel_root_only
from yt.utilities.exceptions import YTFieldNotFound
class Index(ParallelAnalysisInterface):
@@ -126,7 +126,8 @@
if getattr(self, "io", None) is not None: return
self.io = io_registry[self.dataset_type](self.parameter_file)
- def _save_data(self, array, node, name, set_attr=None, force=False, passthrough = False):
+ @parallel_root_only
+ def save_data(self, array, node, name, set_attr=None, force=False, passthrough = False):
"""
Arbitrary numpy data will be saved to the region in the datafile
described by *node* and *name*. If data file does not exist, it throws
@@ -157,14 +158,6 @@
del self._data_file
self._data_file = h5py.File(self.__data_filename, self._data_mode)
- save_data = parallel_splitter(_save_data, _reload_data_file)
-
- def _reset_save_data(self,round_robin=False):
- if round_robin:
- self.save_data = self._save_data
- else:
- self.save_data = parallel_splitter(self._save_data, self._reload_data_file)
-
def save_object(self, obj, name):
"""
Save an object (*obj*) to the data_file using the Pickle protocol,
diff -r 0d7d60aa2739292524a230ee3b8cbb28d1d2808a -r 7bd133d91d3934b8f4617f73b01cd102fb23af8b yt/geometry/grid_geometry_handler.py
--- a/yt/geometry/grid_geometry_handler.py
+++ b/yt/geometry/grid_geometry_handler.py
@@ -31,7 +31,7 @@
from yt.utilities.physical_constants import sec_per_year
from yt.utilities.io_handler import io_registry
from yt.utilities.parallel_tools.parallel_analysis_interface import \
- ParallelAnalysisInterface, parallel_splitter
+ ParallelAnalysisInterface
from yt.utilities.lib.GridTree import GridTree, MatchPointsToGrids
from yt.data_objects.data_containers import data_object_registry
diff -r 0d7d60aa2739292524a230ee3b8cbb28d1d2808a -r 7bd133d91d3934b8f4617f73b01cd102fb23af8b yt/geometry/oct_geometry_handler.py
--- a/yt/geometry/oct_geometry_handler.py
+++ b/yt/geometry/oct_geometry_handler.py
@@ -30,7 +30,7 @@
from yt.utilities.definitions import MAXLEVEL
from yt.utilities.io_handler import io_registry
from yt.utilities.parallel_tools.parallel_analysis_interface import \
- ParallelAnalysisInterface, parallel_splitter
+ ParallelAnalysisInterface
from yt.data_objects.data_containers import data_object_registry
diff -r 0d7d60aa2739292524a230ee3b8cbb28d1d2808a -r 7bd133d91d3934b8f4617f73b01cd102fb23af8b yt/geometry/particle_geometry_handler.py
--- a/yt/geometry/particle_geometry_handler.py
+++ b/yt/geometry/particle_geometry_handler.py
@@ -32,7 +32,7 @@
from yt.utilities.definitions import MAXLEVEL
from yt.utilities.io_handler import io_registry
from yt.utilities.parallel_tools.parallel_analysis_interface import \
- ParallelAnalysisInterface, parallel_splitter
+ ParallelAnalysisInterface
from yt.data_objects.data_containers import data_object_registry
from yt.data_objects.octree_subset import ParticleOctreeSubset
diff -r 0d7d60aa2739292524a230ee3b8cbb28d1d2808a -r 7bd133d91d3934b8f4617f73b01cd102fb23af8b yt/utilities/parallel_tools/parallel_analysis_interface.py
--- a/yt/utilities/parallel_tools/parallel_analysis_interface.py
+++ b/yt/utilities/parallel_tools/parallel_analysis_interface.py
@@ -179,14 +179,13 @@
used on objects that subclass
:class:`~yt.utilities.parallel_tools.parallel_analysis_interface.ParallelAnalysisInterface`.
"""
- if not parallel_capable: return func
@wraps(func)
def single_proc_results(self, *args, **kwargs):
retval = None
if hasattr(self, "dont_wrap"):
if func.func_name in self.dont_wrap:
return func(self, *args, **kwargs)
- if self._processing or not self._distributed:
+ if not parallel_capable or self._processing or not self._distributed:
return func(self, *args, **kwargs)
comm = _get_comm((self,))
if self._owner == comm.rank:
@@ -243,6 +242,8 @@
"""
@wraps(func)
def barrierize(*args, **kwargs):
+ if not parallel_capable:
+ return func(*args, **kwargs)
mylog.debug("Entering barrier before %s", func.func_name)
comm = _get_comm(args)
comm.barrier()
@@ -250,26 +251,7 @@
mylog.debug("Entering barrier after %s", func.func_name)
comm.barrier()
return retval
- if parallel_capable:
- return barrierize
- else:
- return func
-
-def parallel_splitter(f1, f2):
- """
- This function returns either the function *f1* or *f2* depending on whether
- or not we're the root processor. Mainly used in class definitions.
- """
- @wraps(f1)
- def in_order(*args, **kwargs):
- comm = _get_comm(args)
- if comm.rank == 0:
- f1(*args, **kwargs)
- comm.barrier()
- if comm.rank != 0:
- f2(*args, **kwargs)
- if not parallel_capable: return f1
- return in_order
+ return barrierize
def parallel_root_only(func):
"""
@@ -278,6 +260,8 @@
"""
@wraps(func)
def root_only(*args, **kwargs):
+ if not parallel_capable:
+ return func(*args, **kwargs)
comm = _get_comm(args)
rv = None
if comm.rank == 0:
@@ -292,8 +276,7 @@
all_clear = comm.mpi_bcast(all_clear)
if not all_clear: raise RuntimeError
return rv
- if parallel_capable: return root_only
- return func
+ return root_only
class Workgroup(object):
def __init__(self, size, ranks, comm, name):
Repository URL: https://bitbucket.org/yt_analysis/yt/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the yt-svn
mailing list