[Yt-svn] yt-commit r699 - branches/parallel_profiles/yt/lagos

[mturk at wrangler.dreamhost.com]

Author: mturk
Date: Sat Jul 26 16:52:09 2008
New Revision: 699
URL: http://yt.spacepope.org/changeset/699

Log:
Changed the queueing a bit.  Still not well tested, but preloading works.

Changed the reader to take a list of integer ids, which it will process.  This
preserves the id-as-key idea.



Modified:
   branches/parallel_profiles/yt/lagos/DataReadingFuncs.py
   branches/parallel_profiles/yt/lagos/HDF5LightReader.c

Modified: branches/parallel_profiles/yt/lagos/DataReadingFuncs.py
==============================================================================
--- branches/parallel_profiles/yt/lagos/DataReadingFuncs.py	(original)
+++ branches/parallel_profiles/yt/lagos/DataReadingFuncs.py	Sat Jul 26 16:52:09 2008
@@ -142,3 +142,41 @@
 
 def getExceptionHDF5():
     return (exceptions.KeyError, HDF5LightReader.ReadingError)
+
+class BaseDataQueue(object):
+
+    def __init__(self):
+        self.queue = defaultdict(lambda: {})
+
+    # We need a function for reading a list of sets
+    # and a function for *popping* from a queue all the appropriate sets
+
+    def preload(self, grids, sets):
+        pass
+
+    def pop(self, grid, field):
+        if grid.id in self.queue and field in self.queue[grid.id]:
+            return self.queue[grid.id].pop(field)
+        else:
+            # We only read the one set and do not store it if it isn't pre-loaded
+            return self._read_set(grid, field)
+
+    def peek(self, grid, field):
+        return self.queue[grid.id].get(field, None)
+
+    def push(self, grid, field, data):
+        if grid.id in self.queue and field in self.queue[grid.id]:
+            raise ValueError
+        self.queue[grid][field] = data
+
+class DataQueuePackedHDF5(BaseDataQueue):
+    _read_set = readDataHDF5
+
+    def preload(self, grids, sets):
+        # We need to deal with files first
+        files_keys = defaultdict(lambda: [])
+        for g in sorted(grids): files_keys[g.filename].append(g)
+        for file in files_keys:
+            nodes = [g.id for g in grids]
+            data = HDF5LightReader.ReadMultipleGrids(file, nodes, sets)
+            for gid in data: self.queue[gid].update(data[gid])

Modified: branches/parallel_profiles/yt/lagos/HDF5LightReader.c
==============================================================================
--- branches/parallel_profiles/yt/lagos/HDF5LightReader.c	(original)
+++ branches/parallel_profiles/yt/lagos/HDF5LightReader.c	Sat Jul 26 16:52:09 2008
@@ -471,17 +471,18 @@
     // Format arguments
 
     char *filename = NULL;
-    PyObject *grid_names = NULL;
+    char *format_string = NULL;
+    PyObject *grid_ids = NULL;
     PyObject *set_names = NULL;
     Py_ssize_t num_sets = 0;
     Py_ssize_t num_grids = 0;
 
     if (!PyArg_ParseTuple(args, "sOO",
-            &filename, &grid_names, &set_names))
+            &filename, &grid_ids, &set_names))
         return PyErr_Format(_hdf5ReadError,
                "ReadMultipleGrids: Invalid parameters.");
 
-    num_grids = PyList_Size(grid_names);
+    num_grids = PyList_Size(grid_ids);
     num_sets = PyList_Size(set_names);
     PyObject *grids_dict = PyDict_New(); // New reference
     PyObject *grid_key = NULL;
@@ -492,6 +493,8 @@
     hid_t file_id, grid_node;
     file_id = grid_node = 0;
     int i, n;
+    long id;
+    char grid_node_name[13]; // Grid + 8 + \0
 
     file_id = H5Fopen (filename, H5F_ACC_RDONLY, H5P_DEFAULT); 
 
@@ -502,14 +505,16 @@
     }
 
     for(i = 0; i < num_grids; i++) {
-        grid_key = PyList_GetItem(grid_names, i);
+        grid_key = PyList_GetItem(grid_ids, i);
+        id = PyInt_AsLong(grid_key);
+        sprintf(grid_node_name, "Grid%08li", id);
         grid_data = PyDict_New(); // New reference
         PyDict_SetItem(grids_dict, grid_key, grid_data);
-        grid_node = H5Gopen(file_id, PyString_AsString(grid_key));
+        grid_node = H5Gopen(file_id, grid_node_name);
         if (grid_node < 0) {
               PyErr_Format(_hdf5ReadError,
                   "ReadHDF5DataSet: Error opening (%s, %s)",
-                  filename, grid_key);
+                  filename, grid_node_name);
               goto _fail;
         }
         for(n = 0; n < num_sets; n++) {