[Yt-svn] commit/yt: MatthewTurk: Reverting *manually* to all the old stuff. This includes the GDF, static
Bitbucket
commits-noreply at bitbucket.org
Wed Oct 19 12:21:31 PDT 2011
1 new changeset in yt:
http://bitbucket.org/yt_analysis/yt/changeset/5efc2462a321/
changeset: 5efc2462a321
branch: yt
user: MatthewTurk
date: 2011-10-19 21:21:15
summary: Reverting *manually* to all the old stuff. This includes the GDF, static
output changes, and the parameter file storage stuff.
Note that I left in the volume integrator changes, which I think are valid.
affected #: 9 files (-1 bytes)
--- a/yt/convenience.py Wed Oct 19 15:11:20 2011 -0400
+++ b/yt/convenience.py Wed Oct 19 15:21:15 2011 -0400
@@ -32,7 +32,8 @@
from yt.funcs import *
from yt.config import ytcfg
from yt.utilities.parameter_file_storage import \
- output_type_registry
+ output_type_registry, \
+ EnzoRunDatabase
def all_pfs(basedir='.', skip=None, max_depth=1, name_spec="*.hierarchy", **kwargs):
"""
@@ -91,6 +92,15 @@
if len(candidates) == 1:
return output_type_registry[candidates[0]](*args, **kwargs)
if len(candidates) == 0:
+ if ytcfg.get("yt", "enzo_db") != '' \
+ and len(args) == 1 \
+ and isinstance(args[0], types.StringTypes):
+ erdb = EnzoRunDatabase()
+ fn = erdb.find_uuid(args[0])
+ n = "EnzoStaticOutput"
+ if n in output_type_registry \
+ and output_type_registry[n]._is_valid(fn):
+ return output_type_registry[n](fn)
mylog.error("Couldn't figure out output type for %s", args[0])
return None
mylog.error("Multiple output type candidates for %s:", args[0])
--- a/yt/data_objects/static_output.py Wed Oct 19 15:11:20 2011 -0400
+++ b/yt/data_objects/static_output.py Wed Oct 19 15:21:15 2011 -0400
@@ -104,8 +104,6 @@
return self.basename
def _hash(self):
- if "MetaDataDatasetUUID" in self.parameters:
- return self["MetaDataDatasetUUID"]
s = "%s;%s;%s" % (self.basename,
self.current_time, self.unique_identifier)
try:
--- a/yt/frontends/gdf/api.py Wed Oct 19 15:11:20 2011 -0400
+++ b/yt/frontends/gdf/api.py Wed Oct 19 15:21:15 2011 -0400
@@ -29,15 +29,14 @@
"""
from .data_structures import \
- GDFGrid, \
- GDFHierarchy, \
- GDFStaticOutput
+ ChomboGrid, \
+ ChomboHierarchy, \
+ ChomboStaticOutput
from .fields import \
- GDFFieldContainer, \
- GDFFieldInfo, \
- add_gdf_field
+ ChomboFieldContainer, \
+ ChomboFieldInfo, \
+ add_chombo_field
from .io import \
- IOHandlerGDFHDF5
-
+ IOHandlerChomboHDF5
--- a/yt/frontends/gdf/data_structures.py Wed Oct 19 15:11:20 2011 -0400
+++ b/yt/frontends/gdf/data_structures.py Wed Oct 19 15:21:15 2011 -0400
@@ -24,9 +24,6 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
-import h5py
-import numpy as na
-import weakref
from yt.funcs import *
from yt.data_objects.grid_patch import \
AMRGridPatch
@@ -36,7 +33,6 @@
StaticOutput
from .fields import GDFFieldContainer
-import pdb
class GDFGrid(AMRGridPatch):
_id_offset = 0
@@ -62,16 +58,14 @@
self.dds = na.array((RE-LE)/self.ActiveDimensions)
if self.pf.dimensionality < 2: self.dds[1] = 1.0
if self.pf.dimensionality < 3: self.dds[2] = 1.0
- # pdb.set_trace()
self.data['dx'], self.data['dy'], self.data['dz'] = self.dds
class GDFHierarchy(AMRHierarchy):
grid = GDFGrid
-
+
def __init__(self, pf, data_style='grid_data_format'):
self.parameter_file = weakref.proxy(pf)
- self.data_style = data_style
# for now, the hierarchy file is the parameter file!
self.hierarchy_filename = self.parameter_file.parameter_filename
self.directory = os.path.dirname(self.hierarchy_filename)
@@ -84,39 +78,46 @@
pass
def _detect_fields(self):
- self.field_list = self._fhandle['field_types'].keys()
-
+ ncomp = int(self._fhandle['/'].attrs['num_components'])
+ self.field_list = [c[1] for c in self._fhandle['/'].attrs.listitems()[-ncomp:]]
+
def _setup_classes(self):
dd = self._get_data_reader_dict()
AMRHierarchy._setup_classes(self, dd)
self.object_types.sort()
def _count_grids(self):
- self.num_grids = self._fhandle['/grid_parent_id'].shape[0]
-
+ self.num_grids = 0
+ for lev in self._levels:
+ self.num_grids += self._fhandle[lev]['Processors'].len()
+
def _parse_hierarchy(self):
f = self._fhandle # shortcut
-
+
# this relies on the first Group in the H5 file being
# 'Chombo_global'
levels = f.listnames()[1:]
self.grids = []
- for i, grid in enumerate(f['data'].keys()):
- self.grids.append(self.grid(i, self, f['grid_level'][i],
- f['grid_left_index'][i],
- f['grid_dimensions'][i]))
- self.grids[-1]._level_id = f['grid_level'][i]
-
- dx = (self.parameter_file.domain_right_edge-
- self.parameter_file.domain_left_edge)/self.parameter_file.domain_dimensions
- dx = dx/self.parameter_file.refine_by**(f['grid_level'][:])
-
- self.grid_left_edge = self.parameter_file.domain_left_edge + dx*f['grid_left_index'][:]
- self.grid_dimensions = f['grid_dimensions'][:]
- self.grid_right_edge = self.grid_left_edge + dx*self.grid_dimensions
- self.grid_particle_count = f['grid_particle_count'][:]
- self.grids = na.array(self.grids, dtype='object')
- # pdb.set_trace()
+ i = 0
+ for lev in levels:
+ level_number = int(re.match('level_(\d+)',lev).groups()[0])
+ boxes = f[lev]['boxes'].value
+ dx = f[lev].attrs['dx']
+ for level_id, box in enumerate(boxes):
+ si = na.array([box['lo_%s' % ax] for ax in 'ijk'])
+ ei = na.array([box['hi_%s' % ax] for ax in 'ijk'])
+ pg = self.grid(len(self.grids),self,level=level_number,
+ start = si, stop = ei)
+ self.grids.append(pg)
+ self.grids[-1]._level_id = level_id
+ self.grid_left_edge[i] = dx*si.astype(self.float_type)
+ self.grid_right_edge[i] = dx*(ei.astype(self.float_type) + 1)
+ self.grid_particle_count[i] = 0
+ self.grid_dimensions[i] = ei - si + 1
+ i += 1
+ temp_grids = na.empty(len(grids), dtype='object')
+ for gi, g in enumerate(self.grids): temp_grids[gi] = g
+ self.grids = temp_grids
def _populate_grid_objects(self):
for g in self.grids:
@@ -144,14 +145,16 @@
class GDFStaticOutput(StaticOutput):
_hierarchy_class = GDFHierarchy
_fieldinfo_class = GDFFieldContainer
-
+
def __init__(self, filename, data_style='grid_data_format',
storage_filename = None):
StaticOutput.__init__(self, filename, data_style)
+ self._handle = h5py.File(self.filename, "r")
self.storage_filename = storage_filename
- self.filename = filename
self.field_info = self._fieldinfo_class()
-
+ self._handle.close()
+ del self._handle
+
def _set_units(self):
"""
Generates the conversion to various physical _units based on the parameter file
@@ -162,25 +165,21 @@
self._parse_parameter_file()
self.time_units['1'] = 1
self.units['1'] = 1.0
- self.units['unitary'] = 1.0 / (self.domain_right_edge - self.domain_left_edge).max()
+ self.units['unitary'] = 1.0 / (self.domain_right_edge - self.domain_right_edge).max()
seconds = 1
self.time_units['years'] = seconds / (365*3600*24.0)
self.time_units['days'] = seconds / (3600*24.0)
# This should be improved.
- self._handle = h5py.File(self.parameter_filename, "r")
for field_name in self._handle["/field_types"]:
- self.units[field_name] = self._handle["/field_types/%s" % field_name].attrs['field_to_cgs']
- del self._handle
+ self.units[field_name] = self._handle["/%s/field_to_cgs" % field_name]
def _parse_parameter_file(self):
- self._handle = h5py.File(self.parameter_filename, "r")
sp = self._handle["/simulation_parameters"].attrs
self.domain_left_edge = sp["domain_left_edge"][:]
self.domain_right_edge = sp["domain_right_edge"][:]
- self.domain_dimensions = sp["domain_dimensions"][:]
- self.refine_by = sp["refine_by"]
- self.dimensionality = sp["dimensionality"]
- self.current_time = sp["current_time"]
+ self.refine_by = sp["refine_by"][:]
+ self.dimensionality = sp["dimensionality"][:]
+ self.current_time = sp["current_time"][:]
self.unique_identifier = sp["unique_identifier"]
self.cosmological_simulation = sp["cosmological_simulation"]
if sp["num_ghost_zones"] != 0: raise RuntimeError
@@ -194,8 +193,7 @@
else:
self.current_redshift = self.omega_lambda = self.omega_matter = \
self.hubble_constant = self.cosmological_simulation = 0.0
- del self._handle
-
+
@classmethod
def _is_valid(self, *args, **kwargs):
try:
@@ -206,6 +204,4 @@
pass
return False
- def __repr__(self):
- return self.basename.rsplit(".", 1)[0]
--- a/yt/frontends/gdf/fields.py Wed Oct 19 15:11:20 2011 -0400
+++ b/yt/frontends/gdf/fields.py Wed Oct 19 15:21:15 2011 -0400
@@ -1,5 +1,5 @@
"""
-GDF-specific fields
+Chombo-specific fields
Author: J. S. Oishi <jsoishi at gmail.com>
Affiliation: KIPAC/SLAC/Stanford
@@ -32,45 +32,82 @@
ValidateGridType
import yt.data_objects.universal_fields
-class GDFFieldContainer(CodeFieldInfoContainer):
+class ChomboFieldContainer(CodeFieldInfoContainer):
_shared_state = {}
_field_list = {}
-GDFFieldInfo = GDFFieldContainer()
-add_gdf_field = GDFFieldInfo.add_field
+ChomboFieldInfo = ChomboFieldContainer()
+add_chombo_field = ChomboFieldInfo.add_field
-add_field = add_gdf_field
+add_field = add_chombo_field
add_field("density", function=lambda a,b: None, take_log=True,
validators = [ValidateDataField("density")],
units=r"\rm{g}/\rm{cm}^3")
-GDFFieldInfo["density"]._projected_units =r"\rm{g}/\rm{cm}^2"
+ChomboFieldInfo["density"]._projected_units =r"\rm{g}/\rm{cm}^2"
-add_field("specific_energy", function=lambda a,b: None, take_log=True,
- validators = [ValidateDataField("specific_energy")],
- units=r"\rm{erg}/\rm{g}")
+add_field("X-momentum", function=lambda a,b: None, take_log=False,
+ validators = [ValidateDataField("X-Momentum")],
+ units=r"",display_name=r"B_x")
+ChomboFieldInfo["X-momentum"]._projected_units=r""
-add_field("velocity_x", function=lambda a,b: None, take_log=True,
- validators = [ValidateDataField("velocity_x")],
- units=r"\rm{cm}/\rm{s}")
+add_field("Y-momentum", function=lambda a,b: None, take_log=False,
+ validators = [ValidateDataField("Y-Momentum")],
+ units=r"",display_name=r"B_y")
+ChomboFieldInfo["Y-momentum"]._projected_units=r""
-add_field("velocity_y", function=lambda a,b: None, take_log=True,
- validators = [ValidateDataField("velocity_y")],
- units=r"\rm{cm}/\rm{s}")
+add_field("Z-momentum", function=lambda a,b: None, take_log=False,
+ validators = [ValidateDataField("Z-Momentum")],
+ units=r"",display_name=r"B_z")
+ChomboFieldInfo["Z-momentum"]._projected_units=r""
-add_field("velocity_z", function=lambda a,b: None, take_log=True,
- validators = [ValidateDataField("velocity_z")],
- units=r"\rm{cm}/\rm{s}")
+add_field("X-magnfield", function=lambda a,b: None, take_log=False,
+ validators = [ValidateDataField("X-Magnfield")],
+ units=r"",display_name=r"B_x")
+ChomboFieldInfo["X-magnfield"]._projected_units=r""
-add_field("mag_field_x", function=lambda a,b: None, take_log=True,
- validators = [ValidateDataField("mag_field_x")],
- units=r"\rm{cm}/\rm{s}")
+add_field("Y-magnfield", function=lambda a,b: None, take_log=False,
+ validators = [ValidateDataField("Y-Magnfield")],
+ units=r"",display_name=r"B_y")
+ChomboFieldInfo["Y-magnfield"]._projected_units=r""
-add_field("mag_field_y", function=lambda a,b: None, take_log=True,
- validators = [ValidateDataField("mag_field_y")],
- units=r"\rm{cm}/\rm{s}")
+add_field("Z-magnfield", function=lambda a,b: None, take_log=False,
+ validators = [ValidateDataField("Z-Magnfield")],
+ units=r"",display_name=r"B_z")
+ChomboFieldInfo["Z-magnfield"]._projected_units=r""
-add_field("mag_field_z", function=lambda a,b: None, take_log=True,
- validators = [ValidateDataField("mag_field_z")],
- units=r"\rm{cm}/\rm{s}")
+def _MagneticEnergy(field,data):
+ return (data["X-magnfield"]**2 +
+ data["Y-magnfield"]**2 +
+ data["Z-magnfield"]**2)/2.
+add_field("MagneticEnergy", function=_MagneticEnergy, take_log=True,
+ units=r"",display_name=r"B^2/8\pi")
+ChomboFieldInfo["MagneticEnergy"]._projected_units=r""
+
+def _xVelocity(field, data):
+ """generate x-velocity from x-momentum and density
+
+ """
+ return data["X-momentum"]/data["density"]
+add_field("x-velocity",function=_xVelocity, take_log=False,
+ units=r'\rm{cm}/\rm{s}')
+
+def _yVelocity(field,data):
+ """generate y-velocity from y-momentum and density
+
+ """
+ #try:
+ # return data["xvel"]
+ #except KeyError:
+ return data["Y-momentum"]/data["density"]
+add_field("y-velocity",function=_yVelocity, take_log=False,
+ units=r'\rm{cm}/\rm{s}')
+
+def _zVelocity(field,data):
+ """generate z-velocity from z-momentum and density
+
+ """
+ return data["Z-momentum"]/data["density"]
+add_field("z-velocity",function=_zVelocity, take_log=False,
+ units=r'\rm{cm}/\rm{s}')
--- a/yt/frontends/gdf/io.py Wed Oct 19 15:11:20 2011 -0400
+++ b/yt/frontends/gdf/io.py Wed Oct 19 15:21:15 2011 -0400
@@ -25,48 +25,44 @@
"""
from yt.utilities.io_handler import \
BaseIOHandler
-import h5py
-class IOHandlerGDFHDF5(BaseIOHandler):
- _data_style = "grid_data_format"
+class IOHandlerChomboHDF5(BaseIOHandler):
+ _data_style = "chombo_hdf5"
_offset_string = 'data:offsets=0'
_data_string = 'data:datatype=0'
def _field_dict(self,fhandle):
- keys = fhandle['field_types'].keys()
- val = fhandle['field_types'].keys()
- # ncomp = int(fhandle['/'].attrs['num_components'])
- # temp = fhandle['/'].attrs.listitems()[-ncomp:]
- # val, keys = zip(*temp)
- # val = [int(re.match('component_(\d+)',v).groups()[0]) for v in val]
+ ncomp = int(fhandle['/'].attrs['num_components'])
+ temp = fhandle['/'].attrs.listitems()[-ncomp:]
+ val, keys = zip(*temp)
+ val = [int(re.match('component_(\d+)',v).groups()[0]) for v in val]
return dict(zip(keys,val))
def _read_field_names(self,grid):
fhandle = h5py.File(grid.filename,'r')
- return fhandle['field_types'].keys()
+ ncomp = int(fhandle['/'].attrs['num_components'])
+
+ return [c[1] for c in f['/'].attrs.listitems()[-ncomp:]]
def _read_data_set(self,grid,field):
fhandle = h5py.File(grid.hierarchy.hierarchy_filename,'r')
- return fhandle['/data/grid_%010i/'%grid.id+field][:]
- # field_dict = self._field_dict(fhandle)
- # lstring = 'level_%i' % grid.Level
- # lev = fhandle[lstring]
- # dims = grid.ActiveDimensions
- # boxsize = dims.prod()
+
+ field_dict = self._field_dict(fhandle)
+ lstring = 'level_%i' % grid.Level
+ lev = fhandle[lstring]
+ dims = grid.ActiveDimensions
+ boxsize = dims.prod()
- # grid_offset = lev[self._offset_string][grid._level_id]
- # start = grid_offset+field_dict[field]*boxsize
- # stop = start + boxsize
- # data = lev[self._data_string][start:stop]
+ grid_offset = lev[self._offset_string][grid._level_id]
+ start = grid_offset+field_dict[field]*boxsize
+ stop = start + boxsize
+ data = lev[self._data_string][start:stop]
- # return data.reshape(dims, order='F')
+ return data.reshape(dims, order='F')
def _read_data_slice(self, grid, field, axis, coord):
sl = [slice(None), slice(None), slice(None)]
sl[axis] = slice(coord, coord + 1)
- fhandle = h5py.File(grid.hierarchy.hierarchy_filename,'r')
- return fhandle['/data/grid_%010i/'%grid.id+field][:][sl]
+ return self._read_data_set(grid,field)[sl]
- # return self._read_data_set(grid,field)[sl]
-
--- a/yt/frontends/setup.py Wed Oct 19 15:11:20 2011 -0400
+++ b/yt/frontends/setup.py Wed Oct 19 15:21:15 2011 -0400
@@ -6,7 +6,6 @@
config = Configuration('frontends',parent_package,top_path)
config.make_config_py() # installs __config__.py
#config.make_svn_version_py()
- config.add_subpackage("gdf")
config.add_subpackage("chombo")
config.add_subpackage("enzo")
config.add_subpackage("flash")
--- a/yt/mods.py Wed Oct 19 15:11:20 2011 -0400
+++ b/yt/mods.py Wed Oct 19 15:21:15 2011 -0400
@@ -77,9 +77,6 @@
from yt.frontends.chombo.api import \
ChomboStaticOutput, ChomboFieldInfo, add_chombo_field
-from yt.frontends.gdf.api import \
- GDFStaticOutput, GDFFieldInfo, add_gdf_field
-
from yt.frontends.art.api import \
ARTStaticOutput, ARTFieldInfo, add_art_field
--- a/yt/utilities/parameter_file_storage.py Wed Oct 19 15:11:20 2011 -0400
+++ b/yt/utilities/parameter_file_storage.py Wed Oct 19 15:21:15 2011 -0400
@@ -32,8 +32,6 @@
from yt.utilities.parallel_tools.parallel_analysis_interface import \
parallel_simple_proxy
-import yt.utilities.peewee as peewee
-
output_type_registry = {}
_field_names = ('hash', 'bn', 'fp', 'tt', 'ctid', 'class_name', 'last_seen')
@@ -50,20 +48,6 @@
def __repr__(self):
return "%s" % self.name
-_field_spec = dict(
- dset_uuid = peewee.TextField(),
- output_type = peewee.TextField(),
- pf_path = peewee.TextField(),
- creation_time = peewee.IntegerField(),
- last_seen_time = peewee.IntegerField(),
- simulation_uuid = peewee.TextField(),
- redshift = peewee.FloatField(),
- time = peewee.FloatField(),
- topgrid0 = peewee.IntegerField(),
- topgrid1 = peewee.IntegerField(),
- topgrid2 = peewee.IntegerField(),
-)
-
class ParameterFileStore(object):
"""
This class is designed to be a semi-persistent storage for parameter
@@ -78,7 +62,6 @@
_distributed = True
_processing = False
_owner = 0
- conn = None
def __new__(cls, *p, **k):
self = object.__new__(cls, *p, **k)
@@ -94,6 +77,7 @@
if ytcfg.getboolean("yt", "StoreParameterFiles"):
self._read_only = False
self.init_db()
+ self._records = self.read_db()
else:
self._read_only = True
self._records = {}
@@ -109,26 +93,9 @@
if not os.path.isdir(dbdir): os.mkdir(dbdir)
except OSError:
raise NoParameterShelf()
- self.conn = peewee.SqliteDatabase(dbn)
- class SimulationOutputsMeta:
- database = self.conn
- db_table = "simulation_outputs"
- _field_spec["Meta"] = SimulationOutputsMeta
- self.output_model = type(
- "SimulationOutputs",
- (peewee.Model,),
- _field_spec,
- )
- self.output_model._meta.pk_name = "dset_uuid"
- try:
- self.conn.connect()
- except:
- self.conn = None
- try:
- self.output_model.create_table()
- except:
- pass
- self.conn = None
+ open(dbn, 'ab') # make sure it exists, allow to close
+ # Now we read in all our records and return them
+ # these will be broadcast
def _get_db_name(self):
base_file_name = ytcfg.get("yt", "ParameterFileStore")
@@ -137,26 +104,40 @@
return os.path.expanduser("~/.yt/%s" % base_file_name)
def get_pf_hash(self, hash):
- if self.conn is None: return
""" This returns a parameter file based on a hash. """
- output = self.output_model.get(dset_uuid = hash)
- return self._convert_pf(output)
+ return self._convert_pf(self._records[hash])
- def _convert_pf(self, inst):
- """ This turns a model into a parameter file. """
- if self.conn is None: return
- fn = inst.pf_path
- if inst.output_type not in output_type_registry:
- raise UnknownStaticOutputType(inst.output_type)
+ def get_pf_ctid(self, ctid):
+ """ This returns a parameter file based on a CurrentTimeIdentifier. """
+ for h in self._records:
+ if self._records[h]['ctid'] == ctid:
+ return self._convert_pf(self._records[h])
+
+ def _adapt_pf(self, pf):
+ """ This turns a parameter file into a CSV entry. """
+ return dict(bn=pf.basename,
+ fp=pf.fullpath,
+ tt=pf.current_time,
+ ctid=pf.unique_identifier,
+ class_name=pf.__class__.__name__,
+ last_seen=pf._instantiated)
+
+ def _convert_pf(self, pf_dict):
+ """ This turns a CSV entry into a parameter file. """
+ bn = pf_dict['bn']
+ fp = pf_dict['fp']
+ fn = os.path.join(fp, bn)
+ class_name = pf_dict['class_name']
+ if class_name not in output_type_registry:
+ raise UnknownStaticOutputType(class_name)
mylog.info("Checking %s", fn)
if os.path.exists(fn):
- pf = output_type_registry[inst.output_type](fn)
+ pf = output_type_registry[class_name](os.path.join(fp, bn))
else:
raise IOError
# This next one is to ensure that we manually update the last_seen
# record *now*, for during write_out.
- self.output_model.update(last_seen_time = pf._instantiated).where(
- dset_uuid = inst.dset_uuid).execute()
+ self._records[pf._hash()]['last_seen'] = pf._instantiated
return pf
def check_pf(self, pf):
@@ -165,36 +146,86 @@
recorded in the storage unit. In doing so, it will update path
and "last_seen" information.
"""
- if self.conn is None: return
- q = self.output_model.select().where(dset_uuid = pf._hash())
- q.execute()
- if q.count() == 0:
+ hash = pf._hash()
+ if hash not in self._records:
self.insert_pf(pf)
return
- # Otherwise we update
- self.output_model.update(
- last_seen_time = pf._instantiated,
- pf_path = os.path.join(pf.basename, pf.fullpath)
- ).where(
- dset_uuid = pf._hash()).execute(
- )
+ pf_dict = self._records[hash]
+ self._records[hash]['last_seen'] = pf._instantiated
+ if pf_dict['bn'] != pf.basename \
+ or pf_dict['fp'] != pf.fullpath:
+ self.wipe_hash(hash)
+ self.insert_pf(pf)
def insert_pf(self, pf):
""" This will insert a new *pf* and flush the database to disk. """
- if self.conn is None: return
- q = self.output_model.insert(
- dset_uuid = pf._hash(),
- output_type = pf.__class__.__name__,
- pf_path = os.path.join(
- pf.fullpath, pf.basename),
- creation_time = pf.parameters.get(
- "CurrentTimeIdentifier", 0), # Get os.stat
- last_seen_time = pf._instantiated,
- simulation_uuid = pf.parameters.get(
- "SimulationUUID", ""), # NULL
- redshift = pf.current_redshift,
- time = pf.current_time,
- topgrid0 = pf.domain_dimensions[0],
- topgrid1 = pf.domain_dimensions[1],
- topgrid2 = pf.domain_dimensions[2])
- q.execute()
+ self._records[pf._hash()] = self._adapt_pf(pf)
+ self.flush_db()
+
+ def wipe_hash(self, hash):
+ """
+ This removes a *hash* corresponding to a parameter file from the
+ storage.
+ """
+ if hash not in self._records: return
+ del self._records[hash]
+ self.flush_db()
+
+ def flush_db(self):
+ """ This flushes the storage to disk. """
+ if self._read_only: return
+ self._write_out()
+ self.read_db()
+
+ def get_recent(self, n=10):
+ recs = sorted(self._records.values(), key=lambda a: -a['last_seen'])[:n]
+ return recs
+
+ @parallel_simple_proxy
+ def _write_out(self):
+ if self._read_only: return
+ fn = self._get_db_name()
+ f = open("%s.tmp" % fn, 'wb')
+ w = csv.DictWriter(f, _field_names)
+ maxn = ytcfg.getint("yt","MaximumStoredPFs") # number written
+ for h,v in islice(sorted(self._records.items(),
+ key=lambda a: -a[1]['last_seen']), 0, maxn):
+ v['hash'] = h
+ w.writerow(v)
+ f.close()
+ os.rename("%s.tmp" % fn, fn)
+
+ @parallel_simple_proxy
+ def read_db(self):
+ """ This will read the storage device from disk. """
+ f = open(self._get_db_name(), 'rb')
+ vals = csv.DictReader(f, _field_names)
+ db = {}
+ for v in vals:
+ db[v.pop('hash')] = v
+ if v['last_seen'] is None:
+ v['last_seen'] = 0.0
+ else: v['last_seen'] = float(v['last_seen'])
+ return db
+
+class ObjectStorage(object):
+ pass
+
+class EnzoRunDatabase(object):
+ conn = None
+
+ def __init__(self, path = None):
+ if path is None:
+ path = ytcfg.get("yt", "enzo_db")
+ if len(path) == 0: raise Runtime
+ import sqlite3
+ self.conn = sqlite3.connect(path)
+
+ def find_uuid(self, u):
+ cursor = self.conn.execute(
+ "select pf_path from enzo_outputs where dset_uuid = '%s'" % (
+ u))
+ # It's a 'unique key'
+ result = cursor.fetchone()
+ if result is None: return None
+ return result[0]
Repository URL: https://bitbucket.org/yt_analysis/yt/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the yt-svn
mailing list