[Yt-svn] yt-commit r728 - in branches/parallel_profiles/yt: . lagos raven
mturk at wrangler.dreamhost.com
mturk at wrangler.dreamhost.com
Sat Aug 9 14:46:02 PDT 2008
Author: mturk
Date: Sat Aug 9 14:46:01 2008
New Revision: 728
URL: http://yt.spacepope.org/changeset/728
Log:
Almost finished. Now it auto-detects if you are in parallel, and only turns it
on if you are. It should be set up now such that *no* changes need to be made
to a script. This script, for instance, works equally well in parallel and in
serial:
{{{
#!python
from yt.mods import *
pf = EnzoStaticOutput("RedshiftOutput0008")
pc = PlotCollection(pf, center=[0.5]*3)
pc.add_projection("Density",0)
pc.save("hi")
}}}
The plot only gets saved on the root node, as well.
A little bit more testing before it gets merged back to trunk. For instance, I
need to separate out the idea of the lazy_reader from the parallel profiles.
Hmmm.
Modified:
branches/parallel_profiles/yt/config.py
branches/parallel_profiles/yt/funcs.py
branches/parallel_profiles/yt/lagos/ParallelTools.py
branches/parallel_profiles/yt/logger.py
branches/parallel_profiles/yt/raven/PlotTypes.py
Modified: branches/parallel_profiles/yt/config.py
==============================================================================
--- branches/parallel_profiles/yt/config.py (original)
+++ branches/parallel_profiles/yt/config.py Sat Aug 9 14:46:01 2008
@@ -63,7 +63,9 @@
'unifiedlogfile': '1',
'timefunctions':'False',
'inGui':'False',
- 'parallel':'False',
+ '__parallel':'False',
+ '__parallel_rank':'0',
+ '__parallel_size':'1',
},
"raven":{
'ImagePath':".",
Modified: branches/parallel_profiles/yt/funcs.py
==============================================================================
--- branches/parallel_profiles/yt/funcs.py (original)
+++ branches/parallel_profiles/yt/funcs.py Sat Aug 9 14:46:01 2008
@@ -139,3 +139,10 @@
from collections import defaultdict
except ImportError:
defaultdict = __defaultdict
+
+def only_on_root(func, *args, **kwargs):
+ from yt.config import ytcfg
+ if not ytcfg.getboolean("yt","__parallel"):
+ return func(*args,**kwargs)
+ if ytcfg.getint("yt","__parallel_rank") > 0: return
+ return func(*args, **kwargs)
Modified: branches/parallel_profiles/yt/lagos/ParallelTools.py
==============================================================================
--- branches/parallel_profiles/yt/lagos/ParallelTools.py (original)
+++ branches/parallel_profiles/yt/lagos/ParallelTools.py Sat Aug 9 14:46:01 2008
@@ -24,17 +24,28 @@
"""
from yt.lagos import *
+import yt.logger
import itertools
try:
from mpi4py import MPI
- parallel_capable = True
- print "MPI %s / %s" % (MPI.COMM_WORLD.rank, MPI.COMM_WORLD.size)
+ parallel_capable = (MPI.COMM_WORLD.size > 1)
+ mylog.info("Parallel computation enabled: %s / %s",
+ MPI.COMM_WORLD.rank, MPI.COMM_WORLD.size)
+ ytcfg["yt","__parallel_rank"] = str(MPI.COMM_WORLD.rank)
+ ytcfg["yt","__parallel_size"] = str(MPI.COMM_WORLD.size)
+ ytcfg["yt","__parallel"] = "True"
+ # Now let's make sure we have the right options set.
+ if MPI.COMM_WORLD.rank > 0:
+ if ytcfg.getboolean("lagos","serialize"):
+ ytcfg["lagos","onlydeserialize"] = "True"
+ if ytcfg.getboolean("yt","LogFile"):
+ ytcfg["yt","LogFile"] = "False"
+ yt.logger.disable_file_logging()
+
except ImportError:
parallel_capable = False
-print "PARALLEL COMPATIBLE:", parallel_capable
-
class GridIterator(object):
def __init__(self, pobj, just_list = False):
self.pobj = pobj
@@ -88,15 +99,13 @@
_grids = None
def _get_grids(self, *args, **kwargs):
- if parallel_capable and \
- ytcfg.getboolean("yt","parallel"):
+ if parallel_capable:
self._initialize_parallel(*args, **kwargs)
return ParallelGridIterator(self)
return GridIterator(self)
def _get_grid_objs(self):
- if parallel_capable and \
- ytcfg.getboolean("yt","parallel"):
+ if parallel_capable:
return ParallelGridIterator(self, True)
return GridIterator(self, True)
Modified: branches/parallel_profiles/yt/logger.py
==============================================================================
--- branches/parallel_profiles/yt/logger.py (original)
+++ branches/parallel_profiles/yt/logger.py Sat Aug 9 14:46:01 2008
@@ -55,45 +55,60 @@
mb = 10*1024*1024
bc = 10
+loggers = []
+file_handlers = []
+
if ytcfg.getboolean("yt","logfile") and os.access(".", os.W_OK):
if ytcfg.getboolean("yt","unifiedlogfile"):
log_file_name = ytcfg.get("yt","LogFileName")
- ytHandler = handlers.RotatingFileHandler(log_file_name,
+ ytFileHandler = handlers.RotatingFileHandler(log_file_name,
maxBytes=mb, backupCount=bc)
k = logging.Formatter(fstring)
- ytHandler.setFormatter(k)
- ytLogger.addHandler(ytHandler)
+ ytFileHandler.setFormatter(k)
+ ytLogger.addHandler(ytFileHandler)
+ loggers.append(ytLogger)
+ file_handlers.append(ytFileHandler)
else:
# If we *don't* want a unified file handler (which is the default now!)
- fidoHandler = handlers.RotatingFileHandler("fido.log",
+ fidoFileHandler = handlers.RotatingFileHandler("fido.log",
maxBytes=mb, backupCount=bc)
- fidoHandler.setFormatter(f)
- fidoLogger.addHandler(fidoHandler)
+ fidoFileHandler.setFormatter(f)
+ fidoLogger.addHandler(fidoFileHandler)
- ravenHandler = handlers.RotatingFileHandler("raven.log",
+ ravenFileHandler = handlers.RotatingFileHandler("raven.log",
maxBytes=mb, backupCount=bc)
- ravenHandler.setFormatter(f)
- ravenLogger.addHandler(ravenHandler)
+ ravenFileHandler.setFormatter(f)
+ ravenLogger.addHandler(ravenFileHandler)
+ loggers.append(ravenLogger)
+ file_handlers.append(ravenFileHandler)
- lagosHandler = handlers.RotatingFileHandler("lagos.log",
+ lagosFileHandler = handlers.RotatingFileHandler("lagos.log",
maxBytes=mb, backupCount=bc)
- lagosHandler.setFormatter(f)
- lagosLogger.addHandler(lagosHandler)
+ lagosFileHandler.setFormatter(f)
+ lagosLogger.addHandler(lagosFileHandler)
+ loggers.append(lagosLogger)
+ file_handlers.append(lagosFileHandler)
- enkiHandler = handlers.RotatingFileHandler("enki.log",
+ enkiFileHandler = handlers.RotatingFileHandler("enki.log",
maxBytes=mb, backupCount=bc)
- enkiHandler.setFormatter(f)
- enkiLogger.addHandler(enkiHandler)
+ enkiFileHandler.setFormatter(f)
+ enkiLogger.addHandler(enkiFileHandler)
+ loggers.append(enkiLogger)
+ file_handlers.append(enkiFileHandler)
- deliveratorHandler = handlers.RotatingFileHandler("deliverator.log",
+ deliveratorFileHandler = handlers.RotatingFileHandler("deliverator.log",
maxBytes=mb, backupCount=bc)
- deliveratorHandler.setFormatter(f)
- deliveratorLogger.addHandler(deliveratorHandler)
+ deliveratorFileHandler.setFormatter(f)
+ deliveratorLogger.addHandler(deliveratorFileHandler)
+ loggers.append(deliveratorLogger)
+ file_handlers.append(deliveratorFileHandler)
- reasonHandler = handlers.RotatingFileHandler("reason.log",
+ reasonFileHandler = handlers.RotatingFileHandler("reason.log",
maxBytes=mb, backupCount=bc)
- reasonHandler.setFormatter(f)
- reasonLogger.addHandler(reasonHandler)
+ reasonFileHandler.setFormatter(f)
+ reasonLogger.addHandler(reasonFileHandler)
+ loggers.append(reasonLogger)
+ file_handlers.append(reasonFileHandler)
def disable_stream_logging():
# We just remove the root logger's handlers
@@ -101,5 +116,9 @@
if isinstance(handler, logging.StreamHandler):
rootLogger.removeHandler(handler)
+def disable_file_logging():
+ for logger, handler in zip(loggers, file_handlers):
+ logger.removeHandler(handler)
+
if ytcfg.getboolean("yt","suppressStreamLogging"):
disable_stream_logging()
Modified: branches/parallel_profiles/yt/raven/PlotTypes.py
==============================================================================
--- branches/parallel_profiles/yt/raven/PlotTypes.py (original)
+++ branches/parallel_profiles/yt/raven/PlotTypes.py Sat Aug 9 14:46:01 2008
@@ -139,8 +139,7 @@
my_prefix = prefix
fn = ".".join([my_prefix, format])
canvas = engineVals["canvas"](self._figure)
- #self._figure.savefig(fn, format)
- canvas.print_figure(fn)
+ only_on_root(canvas.print_figure, fn)
self["Type"] = self._type_name
self["GeneratedAt"] = self.data.hierarchy["CurrentTimeIdentifier"]
return fn
More information about the yt-svn
mailing list