[yt-svn] commit/yt: 2 new changesets

Bitbucket commits-noreply at bitbucket.org
Fri Feb 15 05:56:28 PST 2013


2 new commits in yt:

https://bitbucket.org/yt_analysis/yt/commits/3407bf928aed/
changeset:   3407bf928aed
branch:      yt
user:        MatthewTurk
date:        2013-02-15 14:17:31
summary:     Adding tests for Chombo and Orion

This also manually sets Chombo periodicity to True and uses the full path for
Chombo data files.
affected #:  3 files

diff -r a9fd65a915e7a1c072f799d479e0d61eebbc574f -r 3407bf928aed1ad624c2e3fb880e01d2ada7874f yt/frontends/chombo/data_structures.py
--- a/yt/frontends/chombo/data_structures.py
+++ b/yt/frontends/chombo/data_structures.py
@@ -104,8 +104,8 @@
         self.field_indexes = {}
         self.parameter_file = weakref.proxy(pf)
         # for now, the hierarchy file is the parameter file!
-        self.hierarchy_filename = self.parameter_file.parameter_filename
-        self.hierarchy = os.path.abspath(self.hierarchy_filename)
+        self.hierarchy_filename = os.path.abspath(
+            self.parameter_file.parameter_filename)
         self.directory = pf.fullpath
         self._fhandle = h5py.File(self.hierarchy_filename, 'r')
 
@@ -223,6 +223,15 @@
         self.cosmological_simulation = False
         fileh.close()
 
+        # These are parameters that I very much wish to get rid of.
+        self.parameters["HydroMethod"] = 'chombo' # always PPM DE
+        self.parameters["DualEnergyFormalism"] = 0 
+        self.parameters["EOSType"] = -1 # default
+
+        if self.fparameters.has_key("mu"):
+            self.parameters["mu"] = self.fparameters["mu"]
+
+
     def _set_units(self):
         """
         Generates the conversion to various physical _units based on the parameter file
@@ -277,6 +286,7 @@
             fileh = h5py.File(self.parameter_filename,'r')
             self.refine_by = fileh['/level_0'].attrs['ref_ratio']
             fileh.close()
+        self.periodicity = (True, True, True)
 
     def _parse_pluto_file(self, ini_filename):
         """

diff -r a9fd65a915e7a1c072f799d479e0d61eebbc574f -r 3407bf928aed1ad624c2e3fb880e01d2ada7874f yt/frontends/chombo/tests/test_outputs.py
--- /dev/null
+++ b/yt/frontends/chombo/tests/test_outputs.py
@@ -0,0 +1,51 @@
+"""
+Chombo frontend tests
+
+Author: Matthew Turk <matthewturk at gmail.com>
+Affiliation: Columbia University
+Homepage: http://yt-project.org/
+License:
+  Copyright (C) 2013 Matthew Turk.  All Rights Reserved.
+
+  This file is part of yt.
+
+  yt is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+from yt.testing import *
+from yt.utilities.answer_testing.framework import \
+    requires_pf, \
+    small_patch_amr, \
+    big_patch_amr, \
+    data_dir_load
+from yt.frontends.chombo.api import ChomboStaticOutput
+
+_fields = ("Density", "VelocityMagnitude", "DivV",
+           "X-magnfield", "Y-magnfield", "Z-magnfield")
+
+gc = "GaussianCloud/data.0077.3d.hdf5"
+ at requires_pf(gc)
+def test_gc():
+    pf = data_dir_load(gc)
+    yield assert_equal, str(pf), "data.0077.3d.hdf5"
+    for test in small_patch_amr(gc, _fields):
+        yield test
+
+tb = "TurbBoxLowRes/data.0005.3d.hdf5"
+ at requires_pf(tb)
+def test_tb():
+    pf = data_dir_load(tb)
+    yield assert_equal, str(pf), "data.0005.3d.hdf5"
+    for test in small_patch_amr(tb, _fields):
+        yield test

diff -r a9fd65a915e7a1c072f799d479e0d61eebbc574f -r 3407bf928aed1ad624c2e3fb880e01d2ada7874f yt/frontends/orion/tests/test_outputs.py
--- /dev/null
+++ b/yt/frontends/orion/tests/test_outputs.py
@@ -0,0 +1,50 @@
+"""
+Orion frontend tests
+
+Author: Matthew Turk <matthewturk at gmail.com>
+Affiliation: Columbia University
+Homepage: http://yt-project.org/
+License:
+  Copyright (C) 2013 Matthew Turk.  All Rights Reserved.
+
+  This file is part of yt.
+
+  yt is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+from yt.testing import *
+from yt.utilities.answer_testing.framework import \
+    requires_pf, \
+    small_patch_amr, \
+    big_patch_amr, \
+    data_dir_load
+from yt.frontends.orion.api import OrionStaticOutput
+
+_fields = ("Temperature", "Density", "VelocityMagnitude", "DivV")
+
+radadvect = "RadAdvect/plt00000"
+ at requires_pf(radadvect)
+def test_radadvect():
+    pf = data_dir_load(radadvect)
+    yield assert_equal, str(pf), "plt00000"
+    for test in small_patch_amr(radadvect, _fields):
+        yield test
+
+rt = "RadTube/plt00500"
+ at requires_pf(rt)
+def test_radtube():
+    pf = data_dir_load(rt)
+    yield assert_equal, str(pf), "plt00500"
+    for test in small_patch_amr(rt, _fields):
+        yield test


https://bitbucket.org/yt_analysis/yt/commits/6ad7685b3771/
changeset:   6ad7685b3771
branch:      yt
user:        MatthewTurk
date:        2013-02-15 14:44:00
summary:     Retain the file handle inside Chombo datasets, the way we do for FLASH.

This speed up IO for chombo considerably.
affected #:  3 files

diff -r 3407bf928aed1ad624c2e3fb880e01d2ada7874f -r 6ad7685b37719a895551010381a52b51b57e53c7 yt/frontends/chombo/data_structures.py
--- a/yt/frontends/chombo/data_structures.py
+++ b/yt/frontends/chombo/data_structures.py
@@ -54,6 +54,8 @@
      mpc_conversion, sec_conversion
 from yt.utilities.parallel_tools.parallel_analysis_interface import \
      parallel_root_only
+from yt.utilities.io_handler import \
+    io_registry
 
 from yt.data_objects.field_info_container import \
     FieldInfoContainer, NullFunc
@@ -107,13 +109,12 @@
         self.hierarchy_filename = os.path.abspath(
             self.parameter_file.parameter_filename)
         self.directory = pf.fullpath
-        self._fhandle = h5py.File(self.hierarchy_filename, 'r')
+        self._handle = pf._handle
 
-        self.float_type = self._fhandle['/level_0']['data:datatype=0'].dtype.name
-        self._levels = self._fhandle.keys()[1:]
+        self.float_type = self._handle['/level_0']['data:datatype=0'].dtype.name
+        self._levels = self._handle.keys()[1:]
         AMRHierarchy.__init__(self,pf,data_style)
         self._read_particles()
-        self._fhandle.close()
 
     def _read_particles(self):
         self.particle_filename = self.hierarchy_filename[:-4] + 'sink'
@@ -144,8 +145,8 @@
                     self.grids[ind].NumberOfParticles += 1
 
     def _detect_fields(self):
-        ncomp = int(self._fhandle['/'].attrs['num_components'])
-        self.field_list = [c[1] for c in self._fhandle['/'].attrs.items()[-ncomp:]]
+        ncomp = int(self._handle['/'].attrs['num_components'])
+        self.field_list = [c[1] for c in self._handle['/'].attrs.items()[-ncomp:]]
           
     def _setup_classes(self):
         dd = self._get_data_reader_dict()
@@ -155,10 +156,10 @@
     def _count_grids(self):
         self.num_grids = 0
         for lev in self._levels:
-            self.num_grids += self._fhandle[lev]['Processors'].len()
+            self.num_grids += self._handle[lev]['Processors'].len()
 
     def _parse_hierarchy(self):
-        f = self._fhandle # shortcut
+        f = self._handle # shortcut
 
         # this relies on the first Group in the H5 file being
         # 'Chombo_global'
@@ -207,6 +208,9 @@
         mask[grid_ind] = True
         return [g for g in self.grids[mask] if g.Level == grid.Level + 1]
 
+    def _setup_data_io(self):
+        self.io = io_registry[self.data_style](self.parameter_file)
+
 class ChomboStaticOutput(StaticOutput):
     _hierarchy_class = ChomboHierarchy
     _fieldinfo_fallback = ChomboFieldInfo
@@ -214,23 +218,21 @@
 
     def __init__(self, filename, data_style='chombo_hdf5',
                  storage_filename = None, ini_filename = None):
-        fileh = h5py.File(filename,'r')
-        self.current_time = fileh.attrs['time']
+        self._handle = h5py.File(filename,'r')
+        self.current_time = self._handle.attrs['time']
         self.ini_filename = ini_filename
         self.fullplotdir = os.path.abspath(filename)
         StaticOutput.__init__(self,filename,data_style)
         self.storage_filename = storage_filename
         self.cosmological_simulation = False
-        fileh.close()
 
         # These are parameters that I very much wish to get rid of.
         self.parameters["HydroMethod"] = 'chombo' # always PPM DE
         self.parameters["DualEnergyFormalism"] = 0 
         self.parameters["EOSType"] = -1 # default
 
-        if self.fparameters.has_key("mu"):
-            self.parameters["mu"] = self.fparameters["mu"]
-
+    def __del__(self):
+        self._handle.close()
 
     def _set_units(self):
         """
@@ -283,9 +285,7 @@
             self.domain_right_edge = self.__calc_right_edge()
             self.domain_dimensions = self.__calc_domain_dimensions()
             self.dimensionality = 3
-            fileh = h5py.File(self.parameter_filename,'r')
-            self.refine_by = fileh['/level_0'].attrs['ref_ratio']
-            fileh.close()
+            self.refine_by = self._handle['/level_0'].attrs['ref_ratio']
         self.periodicity = (True, True, True)
 
     def _parse_pluto_file(self, ini_filename):
@@ -317,24 +317,21 @@
                         self.parameters[paramName] = t
 
     def __calc_left_edge(self):
-        fileh = h5py.File(self.parameter_filename,'r')
+        fileh = self._handle
         dx0 = fileh['/level_0'].attrs['dx']
         LE = dx0*((np.array(list(fileh['/level_0'].attrs['prob_domain'])))[0:3])
-        fileh.close()
         return LE
 
     def __calc_right_edge(self):
-        fileh = h5py.File(self.parameter_filename,'r')
+        fileh = self._handle
         dx0 = fileh['/level_0'].attrs['dx']
         RE = dx0*((np.array(list(fileh['/level_0'].attrs['prob_domain'])))[3:] + 1)
-        fileh.close()
         return RE
 
     def __calc_domain_dimensions(self):
-        fileh = h5py.File(self.parameter_filename,'r')
+        fileh = self._handle
         L_index = ((np.array(list(fileh['/level_0'].attrs['prob_domain'])))[0:3])
         R_index = ((np.array(list(fileh['/level_0'].attrs['prob_domain'])))[3:] + 1)
-        fileh.close()
         return R_index - L_index
 
     @classmethod

diff -r 3407bf928aed1ad624c2e3fb880e01d2ada7874f -r 6ad7685b37719a895551010381a52b51b57e53c7 yt/frontends/chombo/io.py
--- a/yt/frontends/chombo/io.py
+++ b/yt/frontends/chombo/io.py
@@ -36,36 +36,40 @@
     _offset_string = 'data:offsets=0'
     _data_string = 'data:datatype=0'
 
-    def _field_dict(self,fhandle):
-        ncomp = int(fhandle['/'].attrs['num_components'])
-        temp =  fhandle['/'].attrs.items()[-ncomp:]
+    def __init__(self, pf, *args, **kwargs):
+        BaseIOHandler.__init__(self, *args, **kwargs)
+        self.pf = pf
+        self._handle = pf._handle
+
+    _field_dict = None
+    @property
+    def field_dict(self):
+        if self._field_dict is not None:
+            return self._field_dict
+        ncomp = int(self._handle['/'].attrs['num_components'])
+        temp =  self._handle['/'].attrs.items()[-ncomp:]
         val, keys = zip(*temp)
         val = [int(re.match('component_(\d+)',v).groups()[0]) for v in val]
-        return dict(zip(keys,val))
+        self._field_dict = dict(zip(keys,val))
+        return self._field_dict
         
     def _read_field_names(self,grid):
-        fhandle = h5py.File(grid.filename,'r')
-        ncomp = int(fhandle['/'].attrs['num_components'])
+        ncomp = int(self._handle['/'].attrs['num_components'])
 
         fns = [c[1] for c in f['/'].attrs.items()[-ncomp-1:-1]]
-        fhandle.close()
     
     def _read_data(self,grid,field):
 
-        fhandle = h5py.File(grid.hierarchy.hierarchy_filename,'r')
-
-        field_dict = self._field_dict(fhandle)
         lstring = 'level_%i' % grid.Level
-        lev = fhandle[lstring]
+        lev = self._handle[lstring]
         dims = grid.ActiveDimensions
         boxsize = dims.prod()
         
         grid_offset = lev[self._offset_string][grid._level_id]
-        start = grid_offset+field_dict[field]*boxsize
+        start = grid_offset+self.field_dict[field]*boxsize
         stop = start + boxsize
         data = lev[self._data_string][start:stop]
         
-        fhandle.close()
         return data.reshape(dims, order='F')
 
     def _read_particles(self, grid, field):

diff -r 3407bf928aed1ad624c2e3fb880e01d2ada7874f -r 6ad7685b37719a895551010381a52b51b57e53c7 yt/frontends/chombo/tests/test_outputs.py
--- a/yt/frontends/chombo/tests/test_outputs.py
+++ b/yt/frontends/chombo/tests/test_outputs.py
@@ -31,8 +31,7 @@
     data_dir_load
 from yt.frontends.chombo.api import ChomboStaticOutput
 
-_fields = ("Density", "VelocityMagnitude", "DivV",
-           "X-magnfield", "Y-magnfield", "Z-magnfield")
+_fields = ("Density", "VelocityMagnitude", "DivV", "X-magnfield")
 
 gc = "GaussianCloud/data.0077.3d.hdf5"
 @requires_pf(gc)

Repository URL: https://bitbucket.org/yt_analysis/yt/

--

This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.



More information about the yt-svn mailing list