[Yt-svn] yt-commit r1072 - in branches/yt-object-serialization: . yt/fido yt/lagos

[mturk at wrangler.dreamhost.com]

Author: mturk
Date: Mon Jan  5 23:00:28 2009
New Revision: 1072
URL: http://yt.spacepope.org/changeset/1072

Log:
Changed all our pickles to protocol -1, which should improve efficiency ...
substantially.  Sigh, I wish I'd realized this before.  Additionally, added a
check to create the ~/.yt directory if it doesn't yet exist; this should be
moved into yt/config.py at some point.


Modified:
   branches/yt-object-serialization/   (props changed)
   branches/yt-object-serialization/yt/fido/ParameterFileStorage.py
   branches/yt-object-serialization/yt/lagos/BaseDataTypes.py
   branches/yt-object-serialization/yt/lagos/HierarchyType.py

Modified: branches/yt-object-serialization/yt/fido/ParameterFileStorage.py
==============================================================================
--- branches/yt-object-serialization/yt/fido/ParameterFileStorage.py	(original)
+++ branches/yt-object-serialization/yt/fido/ParameterFileStorage.py	Mon Jan  5 23:00:28 2009
@@ -89,19 +89,22 @@
         self[pf._hash()] = self._adapt_pf(pf)
 
     def __getitem__(self, key):
-        my_shelf = shelve.open(self._get_db_name(), flag='r')
+        my_shelf = shelve.open(self._get_db_name(), flag='r', protocol=-1)
         return my_shelf[key]
 
     def __store_item(self, key, val):
-        my_shelf = shelve.open(self._get_db_name(), 'c')
+        my_shelf = shelve.open(self._get_db_name(), 'c', protocol=-1)
         my_shelf[key] = val
 
     def __delete_item(self, key):
-        my_shelf = shelve.open(self._get_db_name(), 'c')
+        my_shelf = shelve.open(self._get_db_name(), 'c', protocol=-1)
         del my_shelf[key]
 
     def __init_shelf(self):
-        shelve.open(self._get_db_name(), 'c')
+        dbn = self._get_db_name()
+        dbdir = os.path.dirname(dbn)
+        if not os.path.isdir(dbdir): os.mkdir(dbdir)
+        shelve.open(self._get_db_name(), 'c', protocol=-1)
 
     def __setitem__(self, key, val):
         only_on_root(self.__store_item, key, val)
@@ -110,7 +113,7 @@
         only_on_root(self.__delete_item, key)
 
     def keys(self):
-        my_shelf = shelve.open(self._get_db_name(), flag='r')
+        my_shelf = shelve.open(self._get_db_name(), flag='r', protocol=-1)
         return my_shelf.keys()
 
 class ObjectStorage(object):

Modified: branches/yt-object-serialization/yt/lagos/BaseDataTypes.py
==============================================================================
--- branches/yt-object-serialization/yt/lagos/BaseDataTypes.py	(original)
+++ branches/yt-object-serialization/yt/lagos/BaseDataTypes.py	Mon Jan  5 23:00:28 2009
@@ -231,7 +231,7 @@
 
     def save_object(self, name, filename = None):
         if filename is not None:
-            ds = shelve.open(filename)
+            ds = shelve.open(filename, protocol=-1)
             if name in ds:
                 mylog.info("Overwriting %s in %s", name, filename)
             ds[name] = self

Modified: branches/yt-object-serialization/yt/lagos/HierarchyType.py
==============================================================================
--- branches/yt-object-serialization/yt/lagos/HierarchyType.py	(original)
+++ branches/yt-object-serialization/yt/lagos/HierarchyType.py	Mon Jan  5 23:00:28 2009
@@ -165,7 +165,7 @@
         self._data_file.flush()
 
     def save_object(self, obj, name):
-        s = cPickle.dumps(obj)
+        s = cPickle.dumps(obj, protocol=-1)
         self.save_data(s, "/Objects", name, force = True)
 
     def load_object(self, name):
@@ -815,7 +815,7 @@
                 self.gridLevels[secondGrid] = self.gridLevels[firstGrid]
         pTree = [ [ grid.id - 1 for grid in self.gridTree[i] ] for i in range(self.num_grids) ]
         self.gridReverseTree[0] = -1
-        self.save_data(cPickle.dumps(pTree), "/", "Tree")
+        self.save_data(cPickle.dumps(pTree, protocol=-1), "/", "Tree")
         self.save_data(na.array(self.gridReverseTree), "/", "ReverseTree")
         self.save_data(self.gridLevels, "/", "Levels")