[Yt-svn] yt-commit r640 - branches/yt-generalization/yt/lagos
joishi at wrangler.dreamhost.com
joishi at wrangler.dreamhost.com
Mon Jun 30 18:20:34 PDT 2008
Author: joishi
Date: Mon Jun 30 18:20:33 2008
New Revision: 640
URL: http://yt.spacepope.org/changeset/640
Log:
Made paranoid reading mode actually paranoid and no longer required. To do so,
the "dtype" string we build up to read data out of Orion Native files is cached
when constructing the Hierarchy. paranoia is now an option when instantiating
an OrionStaticOutput. It should really only be used for testing.
Modified:
branches/yt-generalization/yt/lagos/BaseGridType.py
branches/yt-generalization/yt/lagos/DataReadingFuncs.py
branches/yt-generalization/yt/lagos/HierarchyType.py
branches/yt-generalization/yt/lagos/OrionDefs.py
branches/yt-generalization/yt/lagos/OutputTypes.py
Modified: branches/yt-generalization/yt/lagos/BaseGridType.py
==============================================================================
--- branches/yt-generalization/yt/lagos/BaseGridType.py (original)
+++ branches/yt-generalization/yt/lagos/BaseGridType.py Mon Jun 30 18:20:33 2008
@@ -418,7 +418,7 @@
class OrionGridBase(AMRGridPatch):
_id_offset = 0
- def __init__(self, LeftEdge, RightEdge, index, level, filename, offset, dimensions,start,stop,paranoia=True):
+ def __init__(self, LeftEdge, RightEdge, index, level, filename, offset, dimensions,start,stop,paranoia=False):
AMRGridPatch.__init__(self, index)
self._file_access_pooling = False
self.filename = filename
Modified: branches/yt-generalization/yt/lagos/DataReadingFuncs.py
==============================================================================
--- branches/yt-generalization/yt/lagos/DataReadingFuncs.py (original)
+++ branches/yt-generalization/yt/lagos/DataReadingFuncs.py Mon Jun 30 18:20:33 2008
@@ -155,9 +155,7 @@
if self._paranoid:
mylog.warn("Orion Native reader: Paranoid read mode.")
- pattern = r"^FAB \(\((\d+), \([0-9 ]+\)\),\(\d+, \(([0-9 ]+)\)\)\)\(\((\d+,\d+,\d+)\) \((\d+,\d+,\d+)\) \((\d+,\d+,\d+)\)\) (\d+)\n"
-
- headerRe = re.compile(pattern)
+ headerRe = re.compile(orion_FAB_header_pattern)
bytesPerReal,endian,start,stop,centerType,nComponents = headerRe.search(header).groups()
# we will build up a dtype string, starting with endian
@@ -172,31 +170,52 @@
dtype += ('f%i'% bytesPerReal) #always a floating point
- self._dtype = dtype
# determine size of FAB
- # TODO: we should check consistency of this against the MF header...
start = na.array(map(int,start.split(',')))
stop = na.array(map(int,stop.split(',')))
gridSize = stop - start + 1
+
+ error_count = 0
+ if (start != self.start).any():
+ print "Paranoia Error: Cell_H and %s do not agree on grid start." %self.filename
+ error_count += 1
+ if (stop != self.stop).any():
+ print "Paranoia Error: Cell_H and %s do not agree on grid stop." %self.filename
+ error_count += 1
if (gridSize != self.ActiveDimensions).any():
- pass
- #raise KeyError("Your paranoia was well warrented. Cell_H and %s do not agree on grid size." % self.filename)
+ print "Paranoia Error: Cell_H and %s do not agree on grid dimensions." %self.filename
+ error_count += 1
+ if bytesPerReal != self.hierarchy._bytesPerReal:
+ print "Paranoia Error: Cell_H and %s do not agree on bytes per real number." %self.filename
+ error_count += 1
+ if (bytesPerReal == self.hierarchy._bytesPerReal and dtype != self.hierarchy._dtype):
+ print "Paranoia Error: Cell_H and %s do not agree on endianness." %self.filename
+ error_count += 1
+
+ if error_count > 0:
+ raise RunTimeError("Paranoia unveiled %i differences between Cell_H and %s." % (error_count, self.filename))
+
+ else:
+ start = self.start
+ stop = self.stop
+ dtype = self.hierarchy._dtype
+ bytesPerReal = self.hierarchy._bytesPerReal
+
nElements = self.ActiveDimensions.prod()
# one field has nElements*bytesPerReal bytes and is located
# nElements*bytesPerReal*field_index from the offset location
field_index = self.field_indexes[yt2orionFieldsDict[field]]
inFile.seek(int(nElements*bytesPerReal*field_index),1)
- field = na.fromfile(inFile,count=nElements,dtype=self._dtype)
+ field = na.fromfile(inFile,count=nElements,dtype=dtype)
field = field.reshape(self.ActiveDimensions[::-1]).swapaxes(0,2)
# we can/should also check against the max and min in the header file
inFile.close()
return field
- return na.ones(self.ActiveDimensions, dtype='float64')#field
-
+
def readAllDataNative():
pass
Modified: branches/yt-generalization/yt/lagos/HierarchyType.py
==============================================================================
--- branches/yt-generalization/yt/lagos/HierarchyType.py (original)
+++ branches/yt-generalization/yt/lagos/HierarchyType.py Mon Jun 30 18:20:33 2008
@@ -808,11 +808,12 @@
self.directory = pf.fullpath
self.data_style = data_style
self._setup_classes()
- self.readGlobalHeader(header_filename)
+ self.readGlobalHeader(header_filename,self.parameter_file.paranoid_read) # also sets up the grid objects
+ self.__cache_endianness(self.levels[-1].grids[-1])
AMRHierarchy.__init__(self,pf)
self._setup_field_list()
- def readGlobalHeader(self,filename):
+ def readGlobalHeader(self,filename,paranoid_read):
"""
read the global header file for an Orion plotfile output.
@@ -891,8 +892,8 @@
grid_file_offset = re_file_finder.findall(level_header_file)
start_stop_index = re_dim_finder.findall(level_header_file)
for grid in range(0,ngrids):
- gfn = os.path.join(fn,grid_file_offset[grid][0])
- gfo = int(grid_file_offset[grid][1])
+ gfn = os.path.join(fn,grid_file_offset[grid][0]) # filename of file containing this grid
+ gfo = int(grid_file_offset[grid][1]) # offset within that file
xlo,xhi = map(float,self.__global_header_lines[counter].split())
counter+=1
ylo,yhi = map(float,self.__global_header_lines[counter].split())
@@ -902,7 +903,7 @@
lo = na.array([xlo,ylo,zlo])
hi = na.array([xhi,yhi,zhi])
dims,start,stop = self.__calculate_grid_dimensions(start_stop_index[grid])
- self.levels[-1].grids.append(self.grid(lo,hi,grid_counter,level,gfn, gfo, dims,start,stop))
+ self.levels[-1].grids.append(self.grid(lo,hi,grid_counter,level,gfn, gfo, dims,start,stop,paranoia=paranoid_read))
grid_counter += 1 # this is global, and shouldn't be reset
# for each level
self.levels[-1]._fileprefix = self.__global_header_lines[counter]
@@ -914,6 +915,34 @@
self.max_level = self.n_levels - 1
header_file.close()
+ def __cache_endianness(self,test_grid):
+ """
+ Cache the endianness and bytes perreal of the grids by using a
+ test grid and assuming that all grids have the same
+ endianness. This is a pretty safe assumption since Orion uses
+ one file per processor, and if you're running on a cluster
+ with different endian processors, then you're on your own!
+ """
+ # open the test file & grab the header
+ inFile = open(os.path.expanduser(test_grid.filename),'rb')
+ header = inFile.readline()
+ inFile.close()
+ header.strip()
+
+ # parse it. the patter is in OrionDefs.py
+ headerRe = re.compile(orion_FAB_header_pattern)
+ bytesPerReal,endian,start,stop,centerType,nComponents = headerRe.search(header).groups()
+ self._bytesPerReal = int(bytesPerReal)
+ if self._bytesPerReal == int(endian[0]):
+ dtype = '<'
+ elif self._bytesPerReal == int(endian[-1]):
+ dtype = '>'
+ else:
+ raise ValueError("FAB header is neither big nor little endian. Perhaps the file is corrupt?")
+
+ dtype += ('f%i' % self._bytesPerReal) # always a floating point
+ self._dtype = dtype
+
def __calculate_grid_dimensions(self,start_stop):
start = na.array(map(int,start_stop[0].split(',')))
stop = na.array(map(int,start_stop[1].split(',')))
Modified: branches/yt-generalization/yt/lagos/OrionDefs.py
==============================================================================
--- branches/yt-generalization/yt/lagos/OrionDefs.py (original)
+++ branches/yt-generalization/yt/lagos/OrionDefs.py Mon Jun 30 18:20:33 2008
@@ -43,5 +43,8 @@
"Temperature": "temperature",
}
orion2ytFieldsDict = {}
+
for f,v in yt2orionFieldsDict.items():
orion2ytFieldsDict[v] = f
+
+orion_FAB_header_pattern = r"^FAB \(\((\d+), \([0-9 ]+\)\),\(\d+, \(([0-9 ]+)\)\)\)\(\((\d+,\d+,\d+)\) \((\d+,\d+,\d+)\) \((\d+,\d+,\d+)\)\) (\d+)\n"
Modified: branches/yt-generalization/yt/lagos/OutputTypes.py
==============================================================================
--- branches/yt-generalization/yt/lagos/OutputTypes.py (original)
+++ branches/yt-generalization/yt/lagos/OutputTypes.py Mon Jun 30 18:20:33 2008
@@ -286,7 +286,7 @@
"""
_hierarchy_class = OrionHierarchy
- def __init__(self, plotname, paramFilename='inputs',fparamFilename='probin',data_style=7):
+ def __init__(self, plotname, paramFilename='inputs',fparamFilename='probin',data_style=7,paranoia=False):
"""need to override for Orion file structure.
the paramfile is usually called "inputs"
@@ -299,6 +299,7 @@
"""
self.data_style = data_style
+ self.paranoid_read = paranoia
plotname = plotname.rstrip('/')
self.basename = os.path.basename(plotname)
# this will be the directory ENCLOSING the pltNNNN directory
More information about the yt-svn
mailing list