[yt-svn] commit/yt: 3 new changesets
Bitbucket
commits-noreply at bitbucket.org
Fri Nov 9 20:25:56 PST 2012
3 new commits in yt:
https://bitbucket.org/yt_analysis/yt/changeset/66828be39280/
changeset: 66828be39280
branch: yt
user: chummels
date: 2012-11-10 05:22:19
summary: Fixing stupid mistake with enzo test existence exception.
affected #: 1 file
diff -r 1d22cb72872f253e035a9f4fc9fb4eb63025f073 -r 66828be392800156bc13974a0f32df29f62c0dde yt/utilities/exceptions.py
--- a/yt/utilities/exceptions.py
+++ b/yt/utilities/exceptions.py
@@ -25,6 +25,7 @@
# We don't need to import 'exceptions'
#import exceptions
+import os.path
class YTException(Exception):
def __init__(self, pf = None):
@@ -174,8 +175,9 @@
return "Must have A>=B>=C"
class EnzoTestOutputFileNonExistent(YTException):
- def __init__(self, testname):
- self.testname = testname
+ def __init__(self, filename):
+ self.filename = filename
+ self.testname = os.path.basename(os.path.dirname(filename))
def __str__(self):
return "Enzo test output file (OutputLog) not generated for: " + \
https://bitbucket.org/yt_analysis/yt/changeset/b6fa86f54076/
changeset: b6fa86f54076
branch: yt
user: chummels
date: 2012-11-10 05:23:13
summary: Merging.
affected #: 3 files
diff -r 66828be392800156bc13974a0f32df29f62c0dde -r b6fa86f54076af2dc6a136c695f09c0763244825 .hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -5152,3 +5152,6 @@
0000000000000000000000000000000000000000 svn.993
fff7118f00e25731ccf37cba3082b8fcb73cf90e svn.371
0000000000000000000000000000000000000000 svn.371
+f15825659f5af3ce64aaad30062aff3603cbfb66 hop callback
+0000000000000000000000000000000000000000 hop callback
+0000000000000000000000000000000000000000 hop callback
diff -r 66828be392800156bc13974a0f32df29f62c0dde -r b6fa86f54076af2dc6a136c695f09c0763244825 yt/utilities/answer_testing/framework.py
--- a/yt/utilities/answer_testing/framework.py
+++ b/yt/utilities/answer_testing/framework.py
@@ -36,6 +36,7 @@
from yt.mods import *
from yt.data_objects.static_output import StaticOutput
import cPickle
+import shelve
from yt.utilities.logger import disable_stream_logging
from yt.utilities.command_line import get_yt_version
@@ -61,6 +62,8 @@
help="The name we'll call this set of tests")
parser.add_option("--answer-store", dest="store_results",
default=False, action="store_true")
+ parser.add_option("--local-store", dest="store_local_results",
+ default=False)
def configure(self, options, conf):
super(AnswerTesting, self).configure(options, conf)
@@ -78,40 +81,41 @@
self.result_storage = defaultdict(dict)
if options.compare_name == "SKIP":
options.compare_name = None
- if options.compare_name is not None:
- # Now we grab from our S3 store
- if options.compare_name == "latest":
- options.compare_name = _latest
+ elif options.compare_name == "latest":
+ options.compare_name = _latest
+
+ # We only either store or test.
+ if options.store_local_results == 'True':
AnswerTestingTest.reference_storage = \
- AnswerTestOpener(options.compare_name)
- self.answer_name = options.this_name
+ self.storage = \
+ AnswerTestLocalStorage("%s/%s" % \
+ (os.path.realpath(options.output_dir), options.compare_name), not options.store_results)
+ else:
+ AnswerTestingTest.reference_storage = \
+ self.storage = AnswerTestCloudStorage(options.compare_name, not options.store_results)
+
self.store_results = options.store_results
+ self.store_local_results = options.store_local_results
global run_big_data
run_big_data = options.big_data
- def finalize(self, result):
- # This is where we dump our result storage up to Amazon, if we are able
- # to.
+ def finalize(self, result=None):
if self.store_results is False: return
- import boto
- from boto.s3.key import Key
- c = boto.connect_s3()
- bucket = c.get_bucket("yt-answer-tests")
- for pf_name in self.result_storage:
- rs = cPickle.dumps(self.result_storage[pf_name])
- tk = bucket.get_key("%s_%s" % (self.answer_name, pf_name))
- if tk is not None: tk.delete()
- k = Key(bucket)
- k.key = "%s_%s" % (self.answer_name, pf_name)
- k.set_contents_from_string(rs)
- k.set_acl("public-read")
+ self.storage.dump(self.result_storage)
-class AnswerTestOpener(object):
- def __init__(self, reference_name):
+class AnswerTestStorage(object):
+ def __init__(self, reference_name, read=True):
self.reference_name = reference_name
self.cache = {}
+ self.read = read
+ def dump(self, result_storage, result):
+ pass
+ def get(self, pf_name, default=None):
+ pass
+class AnswerTestCloudStorage(AnswerTestStorage):
def get(self, pf_name, default = None):
+ if not self.read: return default
if pf_name in self.cache: return self.cache[pf_name]
url = _url_path % (self.reference_name, pf_name)
try:
@@ -126,6 +130,47 @@
self.cache[pf_name] = rv
return rv
+ def dump(self, result_storage):
+ if self.read: return
+ # This is where we dump our result storage up to Amazon, if we are able
+ # to.
+ import boto
+ from boto.s3.key import Key
+ c = boto.connect_s3()
+ bucket = c.get_bucket("yt-answer-tests")
+ for pf_name in result_storage:
+ rs = cPickle.dumps(result_storage[pf_name])
+ tk = bucket.get_key("%s_%s" % (self.reference_name, pf_name))
+ if tk is not None: tk.delete()
+ k = Key(bucket)
+ k.key = "%s_%s" % (self.reference_name, pf_name)
+ k.set_contents_from_string(rs)
+ k.set_acl("public-read")
+
+class AnswerTestLocalStorage(AnswerTestStorage):
+ def dump(self, result_storage):
+ if self.read: return
+ # Store data using shelve
+ ds = shelve.open(self.reference_name, protocol=-1)
+ for pf_name in result_storage:
+ answer_name = "%s" % pf_name
+ if name in ds:
+ mylog.info("Overwriting %s", answer_name)
+ ds[answer_name] = result_storage[pf_name]
+ ds.close()
+
+ def get(self, pf_name, default=None):
+ if not self.read: return default
+ # Read data using shelve
+ answer_name = "%s" % pf_name
+ ds = shelve.open(self.reference_name, protocol=-1)
+ try:
+ result = ds[answer_name]
+ except KeyError:
+ result = default
+ ds.close()
+ return result
+
@contextlib.contextmanager
def temp_cwd(cwd):
oldcwd = os.getcwd()
@@ -164,15 +209,16 @@
class AnswerTestingTest(object):
reference_storage = None
+ result_storage = None
prefix = ""
def __init__(self, pf_fn):
self.pf = data_dir_load(pf_fn)
def __call__(self):
nv = self.run()
- if self.reference_storage is not None:
+ if self.reference_storage is not None and self.reference_storage.read:
dd = self.reference_storage.get(self.storage_name)
- if dd is None: raise YTNoOldAnswer()
+ if dd is None: raise YTNoOldAnswer(self.storage_name)
ov = dd[self.description]
self.compare(nv, ov)
else:
diff -r 66828be392800156bc13974a0f32df29f62c0dde -r b6fa86f54076af2dc6a136c695f09c0763244825 yt/utilities/parallel_tools/parallel_analysis_interface.py
--- a/yt/utilities/parallel_tools/parallel_analysis_interface.py
+++ b/yt/utilities/parallel_tools/parallel_analysis_interface.py
@@ -279,12 +279,12 @@
ranks = None
available_ranks = None
tasks = None
- workgroups = []
def __init__(self):
self.comm = communication_system.communicators[-1]
self.size = self.comm.size
self.ranks = range(self.size)
self.available_ranks = range(self.size)
+ self.workgroups = []
def add_workgroup(self, size=None, ranks=None, name=None):
if size is None:
@@ -296,7 +296,7 @@
ranks = [self.available_ranks.pop(0) for i in range(size)]
# Default name to the workgroup number.
if name is None:
- name = string(len(workgroups))
+ name = string(len(self.workgroups))
group = self.comm.comm.Get_group().Incl(ranks)
new_comm = self.comm.comm.Create(group)
if self.comm.rank in ranks:
@@ -304,16 +304,20 @@
self.workgroups.append(Workgroup(len(ranks), ranks, new_comm, name))
def free_workgroup(self, workgroup):
+ # If you want to actually delete the workgroup you will need to
+ # pop it out of the self.workgroups list so you don't have references
+ # that are left dangling, e.g. see free_all() below.
for i in workgroup.ranks:
if self.comm.rank == i:
communication_system.communicators.pop()
self.available_ranks.append(i)
- del workgroup
self.available_ranks.sort()
def free_all(self):
for wg in self.workgroups:
self.free_workgroup(wg)
+ for i in range(len(self.workgroups)):
+ self.workgroups.pop(0)
@classmethod
def from_sizes(cls, sizes):
https://bitbucket.org/yt_analysis/yt/changeset/5aee482e156a/
changeset: 5aee482e156a
branch: yt
user: chummels
date: 2012-11-10 05:25:36
summary: Merging.
affected #: 0 files
Repository URL: https://bitbucket.org/yt_analysis/yt/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the yt-svn
mailing list