[Yt-svn] yt-commit r1201 - in trunk/yt: . fido lagos

[mturk at wrangler.dreamhost.com]

Author: mturk
Date: Fri Mar  6 10:02:47 2009
New Revision: 1201
URL: http://yt.spacepope.org/changeset/1201

Log:
Adding a 'freshness' indicator for the CSV parameter file store.  Defaults to
keeping the last 500 parameter files around, instead of the last infinity
parameter files.  Should seamlessly apply to old entries.



Modified:
   trunk/yt/config.py
   trunk/yt/fido/ParameterFileStorage.py
   trunk/yt/lagos/OutputTypes.py

Modified: trunk/yt/config.py
==============================================================================
--- trunk/yt/config.py	(original)
+++ trunk/yt/config.py	Fri Mar  6 10:02:47 2009
@@ -66,6 +66,7 @@
         '__parallel_size':'1',
         'StoreParameterFiles': 'True',
         'ParameterFileStore': 'parameter_files.csv',
+        'MaximumStoredPFs': '500',
          },
     "raven":{
         'ImagePath':".",

Modified: trunk/yt/fido/ParameterFileStorage.py
==============================================================================
--- trunk/yt/fido/ParameterFileStorage.py	(original)
+++ trunk/yt/fido/ParameterFileStorage.py	Fri Mar  6 10:02:47 2009
@@ -29,9 +29,10 @@
 from yt.lagos.ParallelTools import parallel_simple_proxy
 import csv
 import os.path
+from itertools import islice
 
 output_type_registry = {}
-_field_names = ('hash','bn','fp','tt','ctid','class_name')
+_field_names = ('hash','bn','fp','tt','ctid','class_name','last_seen')
 
 class NoParameterShelf(Exception):
     pass
@@ -94,13 +95,20 @@
                 return self._convert_pf(self._records[h])
 
     def _adapt_pf(self, pf):
+        """
+        This turns a parameter file into a CSV entry
+        """
         return dict(bn=pf.basename,
                     fp=pf.fullpath,
                     tt=pf["InitialTime"],
                     ctid=pf["CurrentTimeIdentifier"],
-                    class_name=pf.__class__.__name__)
+                    class_name=pf.__class__.__name__,
+                    last_seen=pf._instantiated)
 
     def _convert_pf(self, pf_dict):
+        """
+        This turns a CSV entry into a parameter file 
+        """
         bn = pf_dict['bn']
         fp = pf_dict['fp']
         fn = os.path.join(fp, bn)
@@ -112,16 +120,21 @@
             pf = output_type_registry[class_name](os.path.join(fp, bn))
         else:
             raise IOError
+        # This next one is to ensure that we manually update the last_seen
+        # record *now*, for during write_out.
+        self._records[pf._hash()]['last_seen'] = pf._instantiated
         return pf
 
     def check_pf(self, pf):
-        if pf._hash() not in self._records:
+        hash = pf._hash()
+        if hash not in self._records:
             self.insert_pf(pf)
             return
-        pf_dict = self._records[pf._hash()]
+        pf_dict = self._records[hash]
+        self._records[hash]['last_seen'] = pf._instantiated
         if pf_dict['bn'] != pf.basename \
           or pf_dict['fp'] != pf.fullpath:
-            self.wipe_hash(pf._hash())
+            self.wipe_hash(hash)
             self.insert_pf(pf)
 
     def insert_pf(self, pf):
@@ -144,7 +157,9 @@
         fn = self._get_db_name()
         f = open("%s.tmp" % fn, 'wb')
         w = csv.DictWriter(f, _field_names)
-        for h,v in sorted(self._records.items()):
+        maxn = ytcfg.getint("yt","MaximumStoredPFs") # number written
+        for h,v in islice(sorted(self._records.items(),
+                          key=lambda a: -a[1]['last_seen']), 0, maxn):
             v['hash'] = h
             w.writerow(v)
         f.close()
@@ -157,7 +172,9 @@
         db = {}
         for v in vals:
             db[v.pop('hash')] = v
-            
+            if v['last_seen'] is None:
+                v['last_seen'] = 0.0
+            else: v['last_seen'] = float(v['last_seen'])
         return db
 
 class ObjectStorage(object):

Modified: trunk/yt/lagos/OutputTypes.py
==============================================================================
--- trunk/yt/lagos/OutputTypes.py	(original)
+++ trunk/yt/lagos/OutputTypes.py	Fri Mar  6 10:02:47 2009
@@ -67,6 +67,7 @@
         self.basename = os.path.basename(filename)
         self.directory = os.path.expanduser(os.path.dirname(filename))
         self.fullpath = os.path.abspath(self.directory)
+        self._instantiated = time.time()
         if len(self.directory) == 0:
             self.directory = "."
         self.conversion_factors = {}