[Yt-svn] yt-commit r1031 - in branches/yt-object-serialization/yt: fido lagos

[mturk at wrangler.dreamhost.com]

Author: mturk
Date: Tue Dec 23 21:10:13 2008
New Revision: 1031
URL: http://yt.spacepope.org/changeset/1031

Log:
Expanding parameter file storage, experimenting with auto-registration (will be
more error tolerant in future) and using WeakValueDictionary instead of a
defaultdict for the caching of parameter files.

Now that parameter files are cached and grabbable, next up is the object
pickling.

Hashes should be unique keys, so we'll use that to snag a PF from an object
store.



Modified:
   branches/yt-object-serialization/yt/fido/ParameterFileStorage.py
   branches/yt-object-serialization/yt/lagos/OutputTypes.py

Modified: branches/yt-object-serialization/yt/fido/ParameterFileStorage.py
==============================================================================
--- branches/yt-object-serialization/yt/fido/ParameterFileStorage.py	(original)
+++ branches/yt-object-serialization/yt/fido/ParameterFileStorage.py	Tue Dec 23 21:10:13 2008
@@ -48,11 +48,20 @@
 
 class ParameterFileStore(object):
 
+    _shared_state = {}
+    _conn = None
+
+    def __new__(cls, *p, **k):
+        self = object.__new__(cls, *p, **k)
+        self.__dict__ = cls._shared_state
+        return self
+
     def __init__(self, in_memory = False):
-        self._conn = sqlite3.connect(self._get_db_name(),
-                detect_types=sqlite3.PARSE_DECLTYPES)
-        self._conn.row_factory = self._convert_pf
-        self._initialize_new()
+        if self._conn is None:
+            self._conn = sqlite3.connect(self._get_db_name(),
+                    detect_types=sqlite3.PARSE_DECLTYPES)
+            self._conn.row_factory = self._convert_pf
+            self._initialize_new()
         
     def _get_db_name(self):
         return os.path.expanduser("~/.yt/pfdb.sql")
@@ -62,9 +71,15 @@
         try:
             c.execute("""create table parameter_files
                             (pf text, path text, time real,
-                             ctid real, hash text)""")
+                             ctid real, hash text primary key unique)""")
+            self._conn.commit()
         except sqlite3.OperationalError:
             pass
+        c.close()
+
+    def wipe_hash(self, hash):
+        c = self._conn.cursor()
+        c.execute("delete from parameter_files where hash=?", (hash,))
         self._conn.commit()
         c.close()
 
@@ -72,19 +87,30 @@
         c = self._conn.cursor()
         c.execute("""select * from parameter_files where hash=?""",
                   (hash,))
+        return c.fetchall()[0] # Unique
+
+    def get_pf_ctid(self, ctid):
+        c = self._conn.cursor()
+        c.execute("""select * from parameter_files where ctid=?""",
+                  (ctid,))
         return c.fetchall()
 
+    def get_count_hash(self, hash):
+        c = self._conn.cursor()
+        c.execute("""select pf, path from parameter_files where hash=?""",
+                  (hash,))
+        res = c.fetchall()
+        return len(res), res
+
     def _adapt_pf(self, pf):
-        print "ADAPTING"
         return (pf.basename, pf.fullpath,
                 pf["InitialTime"], pf["CurrentTimeIdentifier"],
                 pf._hash())
 
     def _convert_pf(self, cursor, row):
-        print "CONVERTING"
+        if len(row) != 5: return row
         bn, fp, t1, ctid, hash = row
         fn = os.path.join(fp, bn)
-        print "Trying", fn
         if os.path.exists(fn):
             pf = yt.lagos.EnzoStaticOutput(
                 os.path.join(fp, bn))
@@ -92,6 +118,19 @@
             raise IOError
         return pf
 
+    def check_pf(self, pf):
+        rc, res = self.get_count_hash(pf._hash())
+        if rc == 0:
+            self.insert_pf(pf)
+            return
+        elif rc > 1:
+            self.wipe_hash(pf._hash())
+            self.insert_pf(pf)
+            return
+        bn, fp = res[0]
+        if bn != pf.basename or fp != pf.fullpath:
+            self.wipe_hash(pf._hash())
+            self.insert_pf(pf)
 
     def insert_pf(self, pf):
         c = self._conn.cursor()

Modified: branches/yt-object-serialization/yt/lagos/OutputTypes.py
==============================================================================
--- branches/yt-object-serialization/yt/lagos/OutputTypes.py	(original)
+++ branches/yt-object-serialization/yt/lagos/OutputTypes.py	Tue Dec 23 21:10:13 2008
@@ -25,6 +25,7 @@
 """
 
 from yt.lagos import *
+from yt.fido import ParameterFileStore
 from yt.funcs import *
 import string, re, gc, time, os, os.path
 
@@ -32,7 +33,8 @@
 # When such a thing comes to pass, I'll move all the stuff that is contant up
 # to here, and then have it instantiate EnzoStaticOutputs as appropriate.
 
-_cached_pfs = defaultdict(lambda: dict())
+_cached_pfs = weakref.WeakValueDictionary()
+_pf_store = ParameterFileStore()
 
 class StaticOutput(object):
     class __metaclass__(type):
@@ -44,9 +46,10 @@
         if not os.path.exists(apath): raise IOError
         if apath not in _cached_pfs:
             obj = object.__new__(cls)
-            _cached_pfs[apath] = obj
             obj.__init__(filename, *args, **kwargs)
-        return _cached_pfs.pop(apath)
+            _cached_pfs[apath] = obj
+            _pf_store.check_pf(obj)
+        return _cached_pfs[apath]
 
     def __init__(self, filename, data_style=None):
         """