[yt-svn] commit/yt: 13 new changesets

commits-noreply at bitbucket.org commits-noreply at bitbucket.org
Tue Oct 21 04:41:46 PDT 2014


13 new commits in yt:

https://bitbucket.org/yt_analysis/yt/commits/c0a911fe5cdf/
Changeset:   c0a911fe5cdf
Branch:      yt
User:        ChrisMalone
Date:        2014-10-16 22:08:54+00:00
Summary:     start modernizing skeleton frontend
Affected #:  2 files

diff -r 1089b61212d7bfd105a109728fb804b6ef84aadf -r c0a911fe5cdffc0b947fceae13a575f768c02da6 yt/frontends/_skeleton/api.py
--- a/yt/frontends/_skeleton/api.py
+++ b/yt/frontends/_skeleton/api.py
@@ -19,8 +19,7 @@
       SkeletonDataset
 
 from .fields import \
-      SkeletonFieldInfo, \
-      add_skeleton_field
+      SkeletonFieldInfo
 
 from .io import \
       IOHandlerSkeleton

diff -r 1089b61212d7bfd105a109728fb804b6ef84aadf -r c0a911fe5cdffc0b947fceae13a575f768c02da6 yt/frontends/_skeleton/data_structures.py
--- a/yt/frontends/_skeleton/data_structures.py
+++ b/yt/frontends/_skeleton/data_structures.py
@@ -13,18 +13,13 @@
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
-import numpy as np
-
-from yt.data_objects.grid_patch import \
-    AMRGridPatch
 from yt.data_objects.grid_patch import \
     AMRGridPatch
 from yt.geometry.grid_geometry_handler import \
     GridIndex
 from yt.data_objects.static_output import \
     Dataset
-from yt.utilities.lib.misc_utilities import \
-    get_box_grids_level
+from .fields import SkeletonFieldInfo
 
 class SkeletonGrid(AMRGridPatch):
     _id_offset = 0
@@ -41,17 +36,15 @@
     def __repr__(self):
         return "SkeletonGrid_%04i (%s)" % (self.id, self.ActiveDimensions)
 
-class SkeletonHierarchy(AMRHierarchy):
-
+class SkeletonHierarchy(GridIndex):
     grid = SkeletonGrid
     
     def __init__(self, ds, dataset_type='skeleton'):
         self.dataset_type = dataset_type
-        self.dataset = weakref.proxy(ds)
         # for now, the index file is the dataset!
         self.index_filename = self.dataset.parameter_filename
         self.directory = os.path.dirname(self.index_filename)
-        AMRHierarchy.__init__(self, ds, dataset_type)
+        GridIndex.__init__(self, ds, dataset_type)
 
     def _initialize_data_storage(self):
         pass


https://bitbucket.org/yt_analysis/yt/commits/521c502bbf4b/
Changeset:   521c502bbf4b
Branch:      yt
User:        ChrisMalone
Date:        2014-10-17 18:31:55+00:00
Summary:     update data-meaning section
Affected #:  1 file

diff -r c0a911fe5cdffc0b947fceae13a575f768c02da6 -r 521c502bbf4b5ca2ad5bcb80438e41d6b8f80cfd doc/source/developing/creating_frontend.rst
--- a/doc/source/developing/creating_frontend.rst
+++ b/doc/source/developing/creating_frontend.rst
@@ -33,31 +33,50 @@
 If you are interested in adding a new code, be sure to drop us a line on
 `yt-dev <http://lists.spacepope.org/listinfo.cgi/yt-dev-spacepope.org>`_!
 
-To get started, make a new directory in ``yt/frontends`` with the name of your
-code -- you can start by copying into it the contents of the ``stream``
-directory, which is a pretty empty format. You'll then have to create a subclass
-of ``Dataset``. This subclass will need to handle conversion between the
-different physical units and the code units; for the most part, the examples of
-``OrionDataset`` and ``EnzoDataset`` should be followed, but
-``ChomboDataset``, as a slightly newer addition, can also be used as an
-instructive example -- be sure to add an ``_is_valid`` classmethod that will
-verify if a filename is valid for that output type, as that is how "load" works.
+To get started, make a new directory in ``yt/frontends`` with the name
+of your code.  Copying the contents of the ``yt/frontends/_skeleton``
+directory will add a lot of boilerplate for the required classes and
+methods that are needed.  In particular, you'll have to create a
+subclass of ``Dataset``. This subclass will need to handle conversion
+between the different physical units and the code units (typically in
+the ``_set_code_unit_attributes`` method), read in metadata describing
+the overall data on disk (via the ``_parse_parameter_file`` method),
+and provide a ``classmethod`` called ``_is_valid`` that lets the
+``yt.load`` method help identify an input file as belonging to *this*
+particular ``Dataset`` subclass.  For the most part, the examples of
+``yt.frontends.boxlib.OrionDataset`` and
+``yt.frontends.enzo.EnzoDataset`` should be followed, but
+``yt.frontends.chombo.ChomboDataset``, as a slightly newer addition,
+can also be used as an instructive example.
 
-A new set of fields must be added in the file ``fields.py`` in that directory.
-For the most part this means subclassing ``CodeFieldInfoContainer`` and adding
-the necessary fields specific to that code. Here is the Chombo field container:
+A new set of fields must be added in the file ``fields.py`` in your
+new directory.  For the most part this means subclassing 
+``FieldInfoContainer`` and adding the necessary fields specific to
+your code. Here is the base Chombo field container:
 
 .. code-block:: python
 
-    from UniversalFields import *
-    class ChomboFieldContainer(CodeFieldInfoContainer):
-        _shared_state = {}
-        _field_list = {}
-    ChomboFieldInfo = ChomboFieldContainer()
-    add_chombo_field = ChomboFieldInfo.add_field
+    from yt.fields.field_info_container import FieldInfoContainer
+    class ChomboFieldInfo(FieldInfoContainer):
+        known_other_fields = ()
+	known_particle_fields = ()
 
-The field container is a shared state object, which is why we explicitly set
-``_shared_state`` equal to a mutable.
+This is a very stripped down class.  There are several codes
+(e.g. Orion2 and Pluto) that use a Chombo-based datastructure, and
+those particular codes subclass the ``ChomboFieldInfo`` class.  The
+tuples, ``known_other_fields`` and ``known_particle_fields`` contain
+entries, which are tuples of the form ``("name", ("units", ["fields",
+"to", "alias"], "display_name"))``.  ``"name"`` is the name of a field
+stored on-disk in the dataset. ``"units"`` corresponds to the units of
+that field.  The list ``["fields", "to", "alias"]`` allows you to
+specify additional aliases to this particular field; for example, if
+your on-disk field for the x-direction velocity were
+``"x-direction-velocity"``, maybe you'd prefer to alias to the more
+terse name of ``"xvel"``.  ``"display_name"`` is an optional parameter
+that can be used to specify how you want the field to be displayed on
+a plot; this can be LaTeX-code, for example the density field could
+have a display name of ``r"\rho"``.  Omitting the ``"display_name"``
+will result in using a capitalized version of the ``"name"``.
 
 Data Localization Structures
 ----------------------------


https://bitbucket.org/yt_analysis/yt/commits/2d05db9ccc99/
Changeset:   2d05db9ccc99
Branch:      yt
User:        ChrisMalone
Date:        2014-10-17 18:33:50+00:00
Summary:     typo
Affected #:  1 file

diff -r 521c502bbf4b5ca2ad5bcb80438e41d6b8f80cfd -r 2d05db9ccc991170cff3a1fd6b3515f6fca7bc0e doc/source/developing/creating_frontend.rst
--- a/doc/source/developing/creating_frontend.rst
+++ b/doc/source/developing/creating_frontend.rst
@@ -74,7 +74,7 @@
 ``"x-direction-velocity"``, maybe you'd prefer to alias to the more
 terse name of ``"xvel"``.  ``"display_name"`` is an optional parameter
 that can be used to specify how you want the field to be displayed on
-a plot; this can be LaTeX-code, for example the density field could
+a plot; this can be LaTeX code, for example the density field could
 have a display name of ``r"\rho"``.  Omitting the ``"display_name"``
 will result in using a capitalized version of the ``"name"``.
 


https://bitbucket.org/yt_analysis/yt/commits/7bc9cf7c0d9b/
Changeset:   7bc9cf7c0d9b
Branch:      yt
User:        ChrisMalone
Date:        2014-10-17 19:49:31+00:00
Summary:     more modernizing the docs
Affected #:  1 file

diff -r 2d05db9ccc991170cff3a1fd6b3515f6fca7bc0e -r 7bc9cf7c0d9b382a9f31ec65f2b465271384d579 doc/source/developing/creating_frontend.rst
--- a/doc/source/developing/creating_frontend.rst
+++ b/doc/source/developing/creating_frontend.rst
@@ -37,17 +37,18 @@
 of your code.  Copying the contents of the ``yt/frontends/_skeleton``
 directory will add a lot of boilerplate for the required classes and
 methods that are needed.  In particular, you'll have to create a
-subclass of ``Dataset``. This subclass will need to handle conversion
-between the different physical units and the code units (typically in
-the ``_set_code_unit_attributes`` method), read in metadata describing
-the overall data on disk (via the ``_parse_parameter_file`` method),
-and provide a ``classmethod`` called ``_is_valid`` that lets the
-``yt.load`` method help identify an input file as belonging to *this*
-particular ``Dataset`` subclass.  For the most part, the examples of
-``yt.frontends.boxlib.OrionDataset`` and
-``yt.frontends.enzo.EnzoDataset`` should be followed, but
-``yt.frontends.chombo.ChomboDataset``, as a slightly newer addition,
-can also be used as an instructive example.
+subclass of ``Dataset`` in the data_structures.py file. This subclass
+will need to handle conversion between the different physical units
+and the code units (typically in the ``_set_code_unit_attributes()``
+method), read in metadata describing the overall data on disk (via the
+``_parse_parameter_file()`` method), and provide a ``classmethod``
+called ``_is_valid()`` that lets the ``yt.load`` method help identify an
+input file as belonging to *this* particular ``Dataset`` subclass.
+For the most part, the examples of
+``yt.frontends.boxlib.data_structures.OrionDataset`` and
+``yt.frontends.enzo.data_structures.EnzoDataset`` should be followed,
+but ``yt.frontends.chombo.data_structures.ChomboDataset``, as a
+slightly newer addition, can also be used as an instructive example.
 
 A new set of fields must be added in the file ``fields.py`` in your
 new directory.  For the most part this means subclassing 
@@ -61,7 +62,7 @@
         known_other_fields = ()
 	known_particle_fields = ()
 
-This is a very stripped down class.  There are several codes
+This is a very stripped-down class.  There are several codes
 (e.g. Orion2 and Pluto) that use a Chombo-based datastructure, and
 those particular codes subclass the ``ChomboFieldInfo`` class.  The
 tuples, ``known_other_fields`` and ``known_particle_fields`` contain
@@ -81,44 +82,51 @@
 Data Localization Structures
 ----------------------------
 
-As of right now, the "grid patch" mechanism is going to remain in yt, however in
-the future that may change. As such, some other output formats -- like Gadget --
-may be shoe-horned in, slightly.
+These functions and classes let yt know about how the arrangement of
+data on disk corresponds to the physical arrangement of data within
+the simulation.  There are some subtle differences between
+AMR/patch-based codes and Octree-based codes, however, both approaches
+have a concept of a *Hierarchy* or *Index* (used somewhat
+interchangeably in the code) of datastructures and something that
+describes the elements that make up the Hierarchy or Index.  For
+AMR-based codes, the Index is a collection of ``AMRGridPatch`` objects
+that describe a block of zones.  For Octree-based codes, the Index
+contains datastructures that hold information about the individual
+octs, namely an ``OctreeContainer``.
 
-Hierarchy
-^^^^^^^^^
+Hierarchy or Index
+^^^^^^^^^^^^^^^^^^
 
-To set up data localization, an ``AMRHierarchy`` subclass must be added in the
-file ``data_structures.py``. The index object must override the following
-methods:
+To set up data localization, a ``GridIndex`` subclass for AMR-based
+codes or an ``OctreeIndex`` subclass for Octree-based codes must be
+added in the file ``data_structures.py``. Examples of these different
+types of ``Index`` can be found in, for example, the
+``yt.frontends.chombo.data_structures.ChomboHierarchy`` for AMR-based
+codes and ``yt.frontends.ramses.data_structures.RAMSESIndex`` for
+Octree-based codes.  
 
- * ``_detect_fields``: ``self.field_list`` must be populated as a list of
-   strings corresponding to "native" fields in the data files.
- * ``_setup_classes``: it's probably safe to crib this from one of the other
-   ``AMRHierarchy`` subclasses.
- * ``_count_grids``: this must set self.num_grids to be the total number of
-   grids in the simulation.
- * ``_parse_index``: this must fill in ``grid_left_edge``,
+For the most part, the ``GridIndex`` subclass must override (at a
+minimum) the following methods:
+
+ * ``_detect_output_fields()``: ``self.field_list`` must be populated as a list
+   of strings corresponding to "native" fields in the data files.
+ * ``_count_grids()``: this must set ``self.num_grids`` to be the total number
+   of grids (equivalently ``AMRGridPatch``'es) in the simulation.
+ * ``_parse_index()``: this must fill in ``grid_left_edge``,
    ``grid_right_edge``, ``grid_particle_count``, ``grid_dimensions`` and
-   ``grid_levels`` with the appropriate information. Additionally, ``grids``
-   must be an array of grid objects that already know their IDs.
- * ``_populate_grid_objects``: this initializes the grids by calling
-   ``_prepare_grid`` and ``_setup_dx`` on all of them.  Additionally, it should
-   set up ``Children`` and ``Parent`` lists on each grid object.
- * ``_setup_unknown_fields``: If a field is in the data file that yt doesn't
-   already know, this is where you make a guess at it.
- * ``_setup_derived_fields``: ``self.derived_field_list`` needs to be made a
-   list of strings that correspond to all derived fields valid for this
-   index.
-
-For the most part, the ``ChomboHierarchy`` should be the first place to look for
-hints on how to do this; ``EnzoHierarchy`` is also instructive.
+   ``grid_levels`` with the appropriate information.  Each of these variables 
+   is an array, with an entry for each of the ``self.num_grids`` grids.  
+   Additionally, ``grids``  must be an array of ``AMRGridPatch`` objects that 
+   already know their IDs.
+ * ``_populate_grid_objects()``: this initializes the grids by calling
+   ``_prepare_grid()`` and ``_setup_dx()`` on all of them.  Additionally, it 
+   should set up ``Children`` and ``Parent`` lists on each grid object.
 
 Grids
 ^^^^^
 
-A new grid object, subclassing ``AMRGridPatch``, will also have to be added.
-This should go in ``data_structures.py``. For the most part, this may be all
+A new grid object, subclassing ``AMRGridPatch``, will also have to be added in
+``data_structures.py``. For the most part, this may be all
 that is needed:
 
 .. code-block:: python
@@ -134,32 +142,35 @@
             self.Level = level
 
 
-Even the most complex grid object, ``OrionGrid``, is still relatively simple.
+Even one of the more complex grid object,
+``yt.frontends.boxlib.BoxlibGrid``, is still relatively simple.
 
 Data Reading Functions
 ----------------------
 
 In ``io.py``, there are a number of IO handlers that handle the mechanisms by
 which data is read off disk.  To implement a new data reader, you must subclass
-``BaseIOHandler`` and override the following methods:
+``BaseIOHandler``.  The various frontend IO handlers are stored in an IO registry - essentially a dictionary that uses the name of the frontend as a key, and the specific IO handler as a value.  It is important, therefore, to set the ``dataset_type`` attribute of your subclass, which is what is used as the key in the IO registry.  For example:
 
- * ``_read_field_names``: this routine accepts a grid object and must return all
-   the fields in the data file affiliated with that grid. It is used at the
-   initialization of the ``AMRHierarchy`` but likely not later.
- * ``modify``: This accepts a field from a data file and returns it ready to be
-   used by yt. This is used in Enzo data for preloading.
- * ``_read_data_set``: This accepts a grid object and a field name and must
-   return that field, ready to be used by yt as a NumPy array. Note that this
-   presupposes that any actions done in ``modify`` (above) have been executed.
- * ``_read_data_slice``: This accepts a grid object, a field name, an axis and
-   an (integer) coordinate, and it must return a slice through the array at that
-   value.
- * ``preload``: (optional) This accepts a list of grids and a list of datasets
-   and it populates ``self.queue`` (a dict keyed by grid id) with dicts of
-   datasets.
- * ``_read_exception``: (property) This is a tuple of exceptions that can be
-   raised by the data reading to indicate a field does not exist in the file.
+.. code-block:: python
 
+    class IOHandlerBoxlib(BaseIOHandler):
+        _dataset_type = "boxlib_native"
+	...
+
+At a minimum, one should also override the following methods
+
+* ``_read_fluid_selection()``: this receives a collection of data "chunks", a 
+  selector describing which "chunks" you are concerned with, a list of fields,
+  and the size of the data to read.  It should create and return a dictionary 
+  whose keys are the fields, and whose values are numpy arrays containing the 
+  data.  The data should actually be read via the ``_read_chunk_data()`` 
+  method.
+* ``_read_chunk_data()``: this method receives a "chunk" of data along with a 
+  list of fields we want to read.  It loops over all the grid objects within 
+  the "chunk" of data and reads from disk the specific fields, returning a 
+  dictionary whose keys are the fields and whose values are numpy arrays of
+  the data.
 
 And that just about covers it. Please feel free to email
 `yt-users <http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org>`_ or


https://bitbucket.org/yt_analysis/yt/commits/9716d88c14a2/
Changeset:   9716d88c14a2
Branch:      yt
User:        ChrisMalone
Date:        2014-10-17 19:50:07+00:00
Summary:     remove unused modules
Affected #:  1 file

diff -r 7bc9cf7c0d9b382a9f31ec65f2b465271384d579 -r 9716d88c14a2f40af3854b56bf2d409e23828614 yt/frontends/_skeleton/io.py
--- a/yt/frontends/_skeleton/io.py
+++ b/yt/frontends/_skeleton/io.py
@@ -13,9 +13,6 @@
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
-import numpy as np
-import h5py
-
 from yt.utilities.io_handler import \
     BaseIOHandler
 


https://bitbucket.org/yt_analysis/yt/commits/e16ad5b3fd23/
Changeset:   e16ad5b3fd23
Branch:      yt
User:        ChrisMalone
Date:        2014-10-17 19:50:23+00:00
Summary:     remove unused method
Affected #:  1 file

diff -r 9716d88c14a2f40af3854b56bf2d409e23828614 -r e16ad5b3fd232ecba54637c5dfc61914a2f77546 yt/frontends/_skeleton/data_structures.py
--- a/yt/frontends/_skeleton/data_structures.py
+++ b/yt/frontends/_skeleton/data_structures.py
@@ -46,9 +46,6 @@
         self.directory = os.path.dirname(self.index_filename)
         GridIndex.__init__(self, ds, dataset_type)
 
-    def _initialize_data_storage(self):
-        pass
-
     def _detect_output_fields(self):
         # This needs to set a self.field_list that contains all the available,
         # on-disk fields.


https://bitbucket.org/yt_analysis/yt/commits/3309b76428a7/
Changeset:   3309b76428a7
Branch:      yt
User:        ChrisMalone
Date:        2014-10-17 19:54:04+00:00
Summary:     merge
Affected #:  4 files

diff -r 558d9f064464adf497d777b760b9e7ac12f9baea -r 3309b76428a71a81dbfd67e2184d2587807dd860 doc/source/developing/creating_frontend.rst
--- a/doc/source/developing/creating_frontend.rst
+++ b/doc/source/developing/creating_frontend.rst
@@ -33,73 +33,100 @@
 If you are interested in adding a new code, be sure to drop us a line on
 `yt-dev <http://lists.spacepope.org/listinfo.cgi/yt-dev-spacepope.org>`_!
 
-To get started, make a new directory in ``yt/frontends`` with the name of your
-code -- you can start by copying into it the contents of the ``stream``
-directory, which is a pretty empty format. You'll then have to create a subclass
-of ``Dataset``. This subclass will need to handle conversion between the
-different physical units and the code units; for the most part, the examples of
-``OrionDataset`` and ``EnzoDataset`` should be followed, but
-``ChomboDataset``, as a slightly newer addition, can also be used as an
-instructive example -- be sure to add an ``_is_valid`` classmethod that will
-verify if a filename is valid for that output type, as that is how "load" works.
+To get started, make a new directory in ``yt/frontends`` with the name
+of your code.  Copying the contents of the ``yt/frontends/_skeleton``
+directory will add a lot of boilerplate for the required classes and
+methods that are needed.  In particular, you'll have to create a
+subclass of ``Dataset`` in the data_structures.py file. This subclass
+will need to handle conversion between the different physical units
+and the code units (typically in the ``_set_code_unit_attributes()``
+method), read in metadata describing the overall data on disk (via the
+``_parse_parameter_file()`` method), and provide a ``classmethod``
+called ``_is_valid()`` that lets the ``yt.load`` method help identify an
+input file as belonging to *this* particular ``Dataset`` subclass.
+For the most part, the examples of
+``yt.frontends.boxlib.data_structures.OrionDataset`` and
+``yt.frontends.enzo.data_structures.EnzoDataset`` should be followed,
+but ``yt.frontends.chombo.data_structures.ChomboDataset``, as a
+slightly newer addition, can also be used as an instructive example.
 
-A new set of fields must be added in the file ``fields.py`` in that directory.
-For the most part this means subclassing ``CodeFieldInfoContainer`` and adding
-the necessary fields specific to that code. Here is the Chombo field container:
+A new set of fields must be added in the file ``fields.py`` in your
+new directory.  For the most part this means subclassing 
+``FieldInfoContainer`` and adding the necessary fields specific to
+your code. Here is the base Chombo field container:
 
 .. code-block:: python
 
-    from UniversalFields import *
-    class ChomboFieldContainer(CodeFieldInfoContainer):
-        _shared_state = {}
-        _field_list = {}
-    ChomboFieldInfo = ChomboFieldContainer()
-    add_chombo_field = ChomboFieldInfo.add_field
+    from yt.fields.field_info_container import FieldInfoContainer
+    class ChomboFieldInfo(FieldInfoContainer):
+        known_other_fields = ()
+	known_particle_fields = ()
 
-The field container is a shared state object, which is why we explicitly set
-``_shared_state`` equal to a mutable.
+This is a very stripped-down class.  There are several codes
+(e.g. Orion2 and Pluto) that use a Chombo-based datastructure, and
+those particular codes subclass the ``ChomboFieldInfo`` class.  The
+tuples, ``known_other_fields`` and ``known_particle_fields`` contain
+entries, which are tuples of the form ``("name", ("units", ["fields",
+"to", "alias"], "display_name"))``.  ``"name"`` is the name of a field
+stored on-disk in the dataset. ``"units"`` corresponds to the units of
+that field.  The list ``["fields", "to", "alias"]`` allows you to
+specify additional aliases to this particular field; for example, if
+your on-disk field for the x-direction velocity were
+``"x-direction-velocity"``, maybe you'd prefer to alias to the more
+terse name of ``"xvel"``.  ``"display_name"`` is an optional parameter
+that can be used to specify how you want the field to be displayed on
+a plot; this can be LaTeX code, for example the density field could
+have a display name of ``r"\rho"``.  Omitting the ``"display_name"``
+will result in using a capitalized version of the ``"name"``.
 
 Data Localization Structures
 ----------------------------
 
-As of right now, the "grid patch" mechanism is going to remain in yt, however in
-the future that may change. As such, some other output formats -- like Gadget --
-may be shoe-horned in, slightly.
+These functions and classes let yt know about how the arrangement of
+data on disk corresponds to the physical arrangement of data within
+the simulation.  There are some subtle differences between
+AMR/patch-based codes and Octree-based codes, however, both approaches
+have a concept of a *Hierarchy* or *Index* (used somewhat
+interchangeably in the code) of datastructures and something that
+describes the elements that make up the Hierarchy or Index.  For
+AMR-based codes, the Index is a collection of ``AMRGridPatch`` objects
+that describe a block of zones.  For Octree-based codes, the Index
+contains datastructures that hold information about the individual
+octs, namely an ``OctreeContainer``.
 
-Hierarchy
-^^^^^^^^^
+Hierarchy or Index
+^^^^^^^^^^^^^^^^^^
 
-To set up data localization, an ``AMRHierarchy`` subclass must be added in the
-file ``data_structures.py``. The index object must override the following
-methods:
+To set up data localization, a ``GridIndex`` subclass for AMR-based
+codes or an ``OctreeIndex`` subclass for Octree-based codes must be
+added in the file ``data_structures.py``. Examples of these different
+types of ``Index`` can be found in, for example, the
+``yt.frontends.chombo.data_structures.ChomboHierarchy`` for AMR-based
+codes and ``yt.frontends.ramses.data_structures.RAMSESIndex`` for
+Octree-based codes.  
 
- * ``_detect_fields``: ``self.field_list`` must be populated as a list of
-   strings corresponding to "native" fields in the data files.
- * ``_setup_classes``: it's probably safe to crib this from one of the other
-   ``AMRHierarchy`` subclasses.
- * ``_count_grids``: this must set self.num_grids to be the total number of
-   grids in the simulation.
- * ``_parse_index``: this must fill in ``grid_left_edge``,
+For the most part, the ``GridIndex`` subclass must override (at a
+minimum) the following methods:
+
+ * ``_detect_output_fields()``: ``self.field_list`` must be populated as a list
+   of strings corresponding to "native" fields in the data files.
+ * ``_count_grids()``: this must set ``self.num_grids`` to be the total number
+   of grids (equivalently ``AMRGridPatch``'es) in the simulation.
+ * ``_parse_index()``: this must fill in ``grid_left_edge``,
    ``grid_right_edge``, ``grid_particle_count``, ``grid_dimensions`` and
-   ``grid_levels`` with the appropriate information. Additionally, ``grids``
-   must be an array of grid objects that already know their IDs.
- * ``_populate_grid_objects``: this initializes the grids by calling
-   ``_prepare_grid`` and ``_setup_dx`` on all of them.  Additionally, it should
-   set up ``Children`` and ``Parent`` lists on each grid object.
- * ``_setup_unknown_fields``: If a field is in the data file that yt doesn't
-   already know, this is where you make a guess at it.
- * ``_setup_derived_fields``: ``self.derived_field_list`` needs to be made a
-   list of strings that correspond to all derived fields valid for this
-   index.
-
-For the most part, the ``ChomboHierarchy`` should be the first place to look for
-hints on how to do this; ``EnzoHierarchy`` is also instructive.
+   ``grid_levels`` with the appropriate information.  Each of these variables 
+   is an array, with an entry for each of the ``self.num_grids`` grids.  
+   Additionally, ``grids``  must be an array of ``AMRGridPatch`` objects that 
+   already know their IDs.
+ * ``_populate_grid_objects()``: this initializes the grids by calling
+   ``_prepare_grid()`` and ``_setup_dx()`` on all of them.  Additionally, it 
+   should set up ``Children`` and ``Parent`` lists on each grid object.
 
 Grids
 ^^^^^
 
-A new grid object, subclassing ``AMRGridPatch``, will also have to be added.
-This should go in ``data_structures.py``. For the most part, this may be all
+A new grid object, subclassing ``AMRGridPatch``, will also have to be added in
+``data_structures.py``. For the most part, this may be all
 that is needed:
 
 .. code-block:: python
@@ -115,32 +142,35 @@
             self.Level = level
 
 
-Even the most complex grid object, ``OrionGrid``, is still relatively simple.
+Even one of the more complex grid object,
+``yt.frontends.boxlib.BoxlibGrid``, is still relatively simple.
 
 Data Reading Functions
 ----------------------
 
 In ``io.py``, there are a number of IO handlers that handle the mechanisms by
 which data is read off disk.  To implement a new data reader, you must subclass
-``BaseIOHandler`` and override the following methods:
+``BaseIOHandler``.  The various frontend IO handlers are stored in an IO registry - essentially a dictionary that uses the name of the frontend as a key, and the specific IO handler as a value.  It is important, therefore, to set the ``dataset_type`` attribute of your subclass, which is what is used as the key in the IO registry.  For example:
 
- * ``_read_field_names``: this routine accepts a grid object and must return all
-   the fields in the data file affiliated with that grid. It is used at the
-   initialization of the ``AMRHierarchy`` but likely not later.
- * ``modify``: This accepts a field from a data file and returns it ready to be
-   used by yt. This is used in Enzo data for preloading.
- * ``_read_data_set``: This accepts a grid object and a field name and must
-   return that field, ready to be used by yt as a NumPy array. Note that this
-   presupposes that any actions done in ``modify`` (above) have been executed.
- * ``_read_data_slice``: This accepts a grid object, a field name, an axis and
-   an (integer) coordinate, and it must return a slice through the array at that
-   value.
- * ``preload``: (optional) This accepts a list of grids and a list of datasets
-   and it populates ``self.queue`` (a dict keyed by grid id) with dicts of
-   datasets.
- * ``_read_exception``: (property) This is a tuple of exceptions that can be
-   raised by the data reading to indicate a field does not exist in the file.
+.. code-block:: python
 
+    class IOHandlerBoxlib(BaseIOHandler):
+        _dataset_type = "boxlib_native"
+	...
+
+At a minimum, one should also override the following methods
+
+* ``_read_fluid_selection()``: this receives a collection of data "chunks", a 
+  selector describing which "chunks" you are concerned with, a list of fields,
+  and the size of the data to read.  It should create and return a dictionary 
+  whose keys are the fields, and whose values are numpy arrays containing the 
+  data.  The data should actually be read via the ``_read_chunk_data()`` 
+  method.
+* ``_read_chunk_data()``: this method receives a "chunk" of data along with a 
+  list of fields we want to read.  It loops over all the grid objects within 
+  the "chunk" of data and reads from disk the specific fields, returning a 
+  dictionary whose keys are the fields and whose values are numpy arrays of
+  the data.
 
 And that just about covers it. Please feel free to email
 `yt-users <http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org>`_ or

diff -r 558d9f064464adf497d777b760b9e7ac12f9baea -r 3309b76428a71a81dbfd67e2184d2587807dd860 yt/frontends/_skeleton/api.py
--- a/yt/frontends/_skeleton/api.py
+++ b/yt/frontends/_skeleton/api.py
@@ -19,8 +19,7 @@
       SkeletonDataset
 
 from .fields import \
-      SkeletonFieldInfo, \
-      add_skeleton_field
+      SkeletonFieldInfo
 
 from .io import \
       IOHandlerSkeleton

diff -r 558d9f064464adf497d777b760b9e7ac12f9baea -r 3309b76428a71a81dbfd67e2184d2587807dd860 yt/frontends/_skeleton/data_structures.py
--- a/yt/frontends/_skeleton/data_structures.py
+++ b/yt/frontends/_skeleton/data_structures.py
@@ -13,18 +13,13 @@
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
-import numpy as np
-
-from yt.data_objects.grid_patch import \
-    AMRGridPatch
 from yt.data_objects.grid_patch import \
     AMRGridPatch
 from yt.geometry.grid_geometry_handler import \
     GridIndex
 from yt.data_objects.static_output import \
     Dataset
-from yt.utilities.lib.misc_utilities import \
-    get_box_grids_level
+from .fields import SkeletonFieldInfo
 
 class SkeletonGrid(AMRGridPatch):
     _id_offset = 0
@@ -41,20 +36,15 @@
     def __repr__(self):
         return "SkeletonGrid_%04i (%s)" % (self.id, self.ActiveDimensions)
 
-class SkeletonHierarchy(AMRHierarchy):
-
+class SkeletonHierarchy(GridIndex):
     grid = SkeletonGrid
     
     def __init__(self, ds, dataset_type='skeleton'):
         self.dataset_type = dataset_type
-        self.dataset = weakref.proxy(ds)
         # for now, the index file is the dataset!
         self.index_filename = self.dataset.parameter_filename
         self.directory = os.path.dirname(self.index_filename)
-        AMRHierarchy.__init__(self, ds, dataset_type)
-
-    def _initialize_data_storage(self):
-        pass
+        GridIndex.__init__(self, ds, dataset_type)
 
     def _detect_output_fields(self):
         # This needs to set a self.field_list that contains all the available,

diff -r 558d9f064464adf497d777b760b9e7ac12f9baea -r 3309b76428a71a81dbfd67e2184d2587807dd860 yt/frontends/_skeleton/io.py
--- a/yt/frontends/_skeleton/io.py
+++ b/yt/frontends/_skeleton/io.py
@@ -13,9 +13,6 @@
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
-import numpy as np
-import h5py
-
 from yt.utilities.io_handler import \
     BaseIOHandler
 


https://bitbucket.org/yt_analysis/yt/commits/5af9beae91f7/
Changeset:   5af9beae91f7
Branch:      yt
User:        ChrisMalone
Date:        2014-10-17 20:11:28+00:00
Summary:     updates
Affected #:  1 file

diff -r 3309b76428a71a81dbfd67e2184d2587807dd860 -r 5af9beae91f7571294d52bde51a688cff42bf19c doc/source/developing/creating_frontend.rst
--- a/doc/source/developing/creating_frontend.rst
+++ b/doc/source/developing/creating_frontend.rst
@@ -172,6 +172,12 @@
   dictionary whose keys are the fields and whose values are numpy arrays of
   the data.
 
+If your dataset has particle information, you'll want to override the
+``_read_particle_coords()`` and ``read_particle_fields()`` methods as
+well.  Each code is going to read data from disk in a different
+fashion, but the ``yt.frontends.boxlib.io.IOHandlerBoxlib`` is a
+decent place to start.
+
 And that just about covers it. Please feel free to email
 `yt-users <http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org>`_ or
 `yt-dev <http://lists.spacepope.org/listinfo.cgi/yt-dev-spacepope.org>`_ with


https://bitbucket.org/yt_analysis/yt/commits/726ee92e6df5/
Changeset:   726ee92e6df5
Branch:      yt
User:        ChrisMalone
Date:        2014-10-20 15:34:49+00:00
Summary:     merge with yt tip
Affected #:  2 files

diff -r 5af9beae91f7571294d52bde51a688cff42bf19c -r 726ee92e6df5248896ea25c4d6e5d8114dbae4c7 yt/frontends/chombo/data_structures.py
--- a/yt/frontends/chombo/data_structures.py
+++ b/yt/frontends/chombo/data_structures.py
@@ -113,7 +113,8 @@
         self.directory = ds.fullpath
         self._handle = ds._handle
 
-        self.float_type = self._handle['Chombo_global'].attrs['testReal'].dtype.name
+        tr = self._handle['Chombo_global'].attrs.get("testReal", "float32")
+            
         self._levels = [key for key in self._handle.keys() if key.startswith('level')]
         GridIndex.__init__(self, ds, dataset_type)
 
@@ -161,7 +162,13 @@
     def _count_grids(self):
         self.num_grids = 0
         for lev in self._levels:
-            self.num_grids += self._handle[lev]['Processors'].len()
+            d = self._handle[lev]
+            if 'Processors' in d:
+                self.num_grids += d['Processors'].len()
+            elif 'boxes' in d:
+                self.num_grids += d['boxes'].len()
+            else:
+                raise RuntimeError("Uknown file specification")
 
     def _parse_index(self):
         f = self._handle # shortcut

diff -r 5af9beae91f7571294d52bde51a688cff42bf19c -r 726ee92e6df5248896ea25c4d6e5d8114dbae4c7 yt/frontends/chombo/io.py
--- a/yt/frontends/chombo/io.py
+++ b/yt/frontends/chombo/io.py
@@ -25,6 +25,7 @@
     _dataset_type = "chombo_hdf5"
     _offset_string = 'data:offsets=0'
     _data_string = 'data:datatype=0'
+    _offsets = None
 
     def __init__(self, ds, *args, **kwargs):
         BaseIOHandler.__init__(self, ds, *args, **kwargs)
@@ -32,6 +33,29 @@
         self._handle = ds._handle
         self.dim = self._handle['Chombo_global/'].attrs['SpaceDim']
         self._read_ghost_info()
+        if self._offset_string not in self._handle['level_0']:
+            self._calculate_offsets()
+
+    def _calculate_offsets(self):
+        def box_size(corners):
+            size = 1
+            for idim in range(self.dim):
+                size *= (corners[idim+self.dim] - corners[idim] + 1)
+            return size
+
+        self._offsets = {}
+        num_comp = self._handle.attrs['num_components']
+        level = 0
+        while 1:
+            lname = 'level_%i' % level
+            if lname not in self._handle: break
+            boxes = self._handle['level_0']['boxes'].value
+            box_sizes = np.array([box_size(box) for box in boxes])
+
+            offsets = np.cumsum(box_sizes*num_comp, dtype='int64') 
+            offsets -= offsets[0]
+            self._offsets[level] = offsets
+            level += 1
 
     def _read_ghost_info(self):
         try:
@@ -41,7 +65,7 @@
             self.ghost = np.array(self.ghost)
         except KeyError:
             # assume zero ghosts if outputGhosts not present
-            self.ghost = np.zeros(self.dim)
+            self.ghost = np.zeros(self.dim, 'int64')
 
     _field_dict = None
     @property
@@ -80,7 +104,10 @@
         shape = grid.ActiveDimensions + 2*self.ghost
         boxsize = shape.prod()
 
-        grid_offset = lev[self._offset_string][grid._level_id]
+        if self._offsets is not None:
+            grid_offset = self._offsets[grid.Level][grid._level_id]
+        else:
+            grid_offset = lev[self._offset_string][grid._level_id]
         start = grid_offset+self.field_dict[field]*boxsize
         stop = start + boxsize
         data = lev[self._data_string][start:stop]


https://bitbucket.org/yt_analysis/yt/commits/dc847107cce0/
Changeset:   dc847107cce0
Branch:      yt
User:        ChrisMalone
Date:        2014-10-20 15:46:09+00:00
Summary:     address Andrew's suggestions
Affected #:  1 file

diff -r 726ee92e6df5248896ea25c4d6e5d8114dbae4c7 -r dc847107cce0ece3114f254cc0a134b604648d89 doc/source/developing/creating_frontend.rst
--- a/doc/source/developing/creating_frontend.rst
+++ b/doc/source/developing/creating_frontend.rst
@@ -7,14 +7,14 @@
             have a question about making a custom derived quantity, please
             contact the mailing list.
 
-yt is designed to support analysis and visualization of data from multiple
-different simulation codes, although it has so far been most successfully
-applied to Adaptive Mesh Refinement (AMR) data. For a list of codes and the
-level of support they enjoy, see :ref:`code-support`.
+yt is designed to support analysis and visualization of data from
+multiple different simulation codes. For a list of codes and the level
+of support they enjoy, see :ref:`code-support`.
 
-We'd like to support a broad range of codes, both AMR-based and otherwise. To
-add support for a new code, a few things need to be put into place. These
-necessary structures can be classified into a couple categories:
+We'd like to support a broad range of codes, both Adaptive Mesh
+Refinement (AMR)-based and otherwise. To add support for a new code, a
+few things need to be put into place. These necessary structures can
+be classified into a couple categories:
 
  * Data meaning: This is the set of parameters that convert the data into
    physically relevant units; things like spatial and mass conversions, time
@@ -53,25 +53,50 @@
 A new set of fields must be added in the file ``fields.py`` in your
 new directory.  For the most part this means subclassing 
 ``FieldInfoContainer`` and adding the necessary fields specific to
-your code. Here is the base Chombo field container:
+your code. Here is a snippet from the base BoxLib field container:
 
 .. code-block:: python
 
     from yt.fields.field_info_container import FieldInfoContainer
-    class ChomboFieldInfo(FieldInfoContainer):
-        known_other_fields = ()
-	known_particle_fields = ()
+    class BoxlibFieldInfo(FieldInfoContainer):
+        known_other_fields = (
+            ("density", (rho_units, ["density"], None)),
+	    ("eden", (eden_units, ["energy_density"], None)),
+	    ("xmom", (mom_units, ["momentum_x"], None)),
+	    ("ymom", (mom_units, ["momentum_y"], None)),
+	    ("zmom", (mom_units, ["momentum_z"], None)),
+	    ("temperature", ("K", ["temperature"], None)),
+	    ("Temp", ("K", ["temperature"], None)),
+	    ("x_velocity", ("cm/s", ["velocity_x"], None)),
+	    ("y_velocity", ("cm/s", ["velocity_y"], None)),
+	    ("z_velocity", ("cm/s", ["velocity_z"], None)),
+	    ("xvel", ("cm/s", ["velocity_x"], None)),
+	    ("yvel", ("cm/s", ["velocity_y"], None)),
+	    ("zvel", ("cm/s", ["velocity_z"], None)),
+	)
 
-This is a very stripped-down class.  There are several codes
-(e.g. Orion2 and Pluto) that use a Chombo-based datastructure, and
-those particular codes subclass the ``ChomboFieldInfo`` class.  The
-tuples, ``known_other_fields`` and ``known_particle_fields`` contain
-entries, which are tuples of the form ``("name", ("units", ["fields",
-"to", "alias"], "display_name"))``.  ``"name"`` is the name of a field
-stored on-disk in the dataset. ``"units"`` corresponds to the units of
-that field.  The list ``["fields", "to", "alias"]`` allows you to
-specify additional aliases to this particular field; for example, if
-your on-disk field for the x-direction velocity were
+	known_particle_fields = (
+	    ("particle_mass", ("code_mass", [], None)),
+	    ("particle_position_x", ("code_length", [], None)),
+	    ("particle_position_y", ("code_length", [], None)),
+	    ("particle_position_z", ("code_length", [], None)),
+	    ("particle_momentum_x", (mom_units, [], None)),
+	    ("particle_momentum_y", (mom_units, [], None)),
+	    ("particle_momentum_z", (mom_units, [], None)),
+	    ("particle_angmomen_x", ("code_length**2/code_time", [], None)),
+	    ("particle_angmomen_y", ("code_length**2/code_time", [], None)),
+	    ("particle_angmomen_z", ("code_length**2/code_time", [], None)),
+	    ("particle_id", ("", ["particle_index"], None)),
+	    ("particle_mdot", ("code_mass/code_time", [], None)),
+	)
+
+The tuples, ``known_other_fields`` and ``known_particle_fields``
+contain entries, which are tuples of the form ``("name", ("units",
+["fields", "to", "alias"], "display_name"))``.  ``"name"`` is the name
+of a field stored on-disk in the dataset. ``"units"`` corresponds to
+the units of that field.  The list ``["fields", "to", "alias"]``
+allows you to specify additional aliases to this particular field; for
+example, if your on-disk field for the x-direction velocity were
 ``"x-direction-velocity"``, maybe you'd prefer to alias to the more
 terse name of ``"xvel"``.  ``"display_name"`` is an optional parameter
 that can be used to specify how you want the field to be displayed on
@@ -142,7 +167,7 @@
             self.Level = level
 
 
-Even one of the more complex grid object,
+Even one of the more complex grid objects,
 ``yt.frontends.boxlib.BoxlibGrid``, is still relatively simple.
 
 Data Reading Functions


https://bitbucket.org/yt_analysis/yt/commits/d169ebcd429a/
Changeset:   d169ebcd429a
Branch:      yt
User:        ChrisMalone
Date:        2014-10-20 16:00:58+00:00
Summary:     attempt to address 'octree' issue
Affected #:  1 file

diff -r dc847107cce0ece3114f254cc0a134b604648d89 -r d169ebcd429ae622b3e88bbbf49f7175493f0a9c doc/source/developing/creating_frontend.rst
--- a/doc/source/developing/creating_frontend.rst
+++ b/doc/source/developing/creating_frontend.rst
@@ -109,26 +109,33 @@
 
 These functions and classes let yt know about how the arrangement of
 data on disk corresponds to the physical arrangement of data within
-the simulation.  There are some subtle differences between
-AMR/patch-based codes and Octree-based codes, however, both approaches
-have a concept of a *Hierarchy* or *Index* (used somewhat
-interchangeably in the code) of datastructures and something that
-describes the elements that make up the Hierarchy or Index.  For
-AMR-based codes, the Index is a collection of ``AMRGridPatch`` objects
-that describe a block of zones.  For Octree-based codes, the Index
-contains datastructures that hold information about the individual
-octs, namely an ``OctreeContainer``.
+the simulation.  yt has grid datastructures for handling both
+patch-based and octree-based AMR codes.  The terms 'patch-based'
+and 'octree-based' are used somewhat loosely here.  For example,
+traditionally, the FLASH code used the paramesh AMR library, which is
+based on a tree structure, but the FLASH frontend in yt utilizes yt's
+patch-based datastructures.  It is up to the frontend developer to
+determine which yt datastructures best match the datastructures of
+their simulation code.
+
+Both approaches -- patch-based and octree-based -- have a concept of a
+*Hierarchy* or *Index* (used somewhat interchangeably in the code) of
+datastructures and something that describes the elements that make up
+the Hierarchy or Index.  For patch-based codes, the Index is a
+collection of ``AMRGridPatch`` objects that describe a block of zones.
+For octree-based codes, the Index contains datastructures that hold
+information about the individual octs, namely an ``OctreeContainer``.
 
 Hierarchy or Index
 ^^^^^^^^^^^^^^^^^^
 
-To set up data localization, a ``GridIndex`` subclass for AMR-based
-codes or an ``OctreeIndex`` subclass for Octree-based codes must be
+To set up data localization, a ``GridIndex`` subclass for patch-based
+codes or an ``OctreeIndex`` subclass for octree-based codes must be
 added in the file ``data_structures.py``. Examples of these different
 types of ``Index`` can be found in, for example, the
-``yt.frontends.chombo.data_structures.ChomboHierarchy`` for AMR-based
+``yt.frontends.chombo.data_structures.ChomboHierarchy`` for patch-based
 codes and ``yt.frontends.ramses.data_structures.RAMSESIndex`` for
-Octree-based codes.  
+octree-based codes.  
 
 For the most part, the ``GridIndex`` subclass must override (at a
 minimum) the following methods:


https://bitbucket.org/yt_analysis/yt/commits/d11d8f936079/
Changeset:   d11d8f936079
Branch:      yt
User:        ChrisMalone
Date:        2014-10-20 17:21:48+00:00
Summary:     address Nathan's comment and add more about octrees
Affected #:  1 file

diff -r d169ebcd429ae622b3e88bbbf49f7175493f0a9c -r d11d8f9360792b946ec48082b0e0ffd76a307c24 doc/source/developing/creating_frontend.rst
--- a/doc/source/developing/creating_frontend.rst
+++ b/doc/source/developing/creating_frontend.rst
@@ -154,9 +154,24 @@
    ``_prepare_grid()`` and ``_setup_dx()`` on all of them.  Additionally, it 
    should set up ``Children`` and ``Parent`` lists on each grid object.
 
+The ``OctreeIndex`` has somewhat analogous methods, but often with
+different names; both ``OctreeIndex`` and ``GridIndex`` are subclasses
+of the ``Index`` class.  In particular, for the ``OctreeIndex``, the
+method ``_initialize_oct_handler()`` setups up much of the oct
+metadata that is analogous to the grid metadata created in the
+``GridIndex`` methods ``_count_grids()``, ``_parse_index()``, and
+``_populate_grid_objects()``.
+
 Grids
 ^^^^^
 
+.. note:: This section only applies to the approach using yt's patch-based
+	  datastructures.  For the octree-based approach, one does not create
+	  a grid object, but rather an ``OctreeSubset``, which has methods
+	  for filling out portions of the octree structure.  Again, see the
+	  code in ``yt.frontends.ramses.data_structures`` for an example of
+	  the octree approach.
+
 A new grid object, subclassing ``AMRGridPatch``, will also have to be added in
 ``data_structures.py``. For the most part, this may be all
 that is needed:
@@ -180,9 +195,14 @@
 Data Reading Functions
 ----------------------
 
-In ``io.py``, there are a number of IO handlers that handle the mechanisms by
-which data is read off disk.  To implement a new data reader, you must subclass
-``BaseIOHandler``.  The various frontend IO handlers are stored in an IO registry - essentially a dictionary that uses the name of the frontend as a key, and the specific IO handler as a value.  It is important, therefore, to set the ``dataset_type`` attribute of your subclass, which is what is used as the key in the IO registry.  For example:
+In ``io.py``, there are a number of IO handlers that handle the
+mechanisms by which data is read off disk.  To implement a new data
+reader, you must subclass ``BaseIOHandler``.  The various frontend IO
+handlers are stored in an IO registry - essentially a dictionary that
+uses the name of the frontend as a key, and the specific IO handler as
+a value.  It is important, therefore, to set the ``dataset_type``
+attribute of your subclass, which is what is used as the key in the IO
+registry.  For example:
 
 .. code-block:: python
 


https://bitbucket.org/yt_analysis/yt/commits/6f5a2e3f405f/
Changeset:   6f5a2e3f405f
Branch:      yt
User:        MatthewTurk
Date:        2014-10-21 11:41:38+00:00
Summary:     Merged in ChrisMalone/yt (pull request #1270)

On updating the frontend docs
Affected #:  4 files

diff -r de7802382dc0cc71b3e1eb15b37bbdcfec5b218a -r 6f5a2e3f405f9ec96211c59085cc6101891ad655 doc/source/developing/creating_frontend.rst
--- a/doc/source/developing/creating_frontend.rst
+++ b/doc/source/developing/creating_frontend.rst
@@ -7,14 +7,14 @@
             have a question about making a custom derived quantity, please
             contact the mailing list.
 
-yt is designed to support analysis and visualization of data from multiple
-different simulation codes, although it has so far been most successfully
-applied to Adaptive Mesh Refinement (AMR) data. For a list of codes and the
-level of support they enjoy, see :ref:`code-support`.
+yt is designed to support analysis and visualization of data from
+multiple different simulation codes. For a list of codes and the level
+of support they enjoy, see :ref:`code-support`.
 
-We'd like to support a broad range of codes, both AMR-based and otherwise. To
-add support for a new code, a few things need to be put into place. These
-necessary structures can be classified into a couple categories:
+We'd like to support a broad range of codes, both Adaptive Mesh
+Refinement (AMR)-based and otherwise. To add support for a new code, a
+few things need to be put into place. These necessary structures can
+be classified into a couple categories:
 
  * Data meaning: This is the set of parameters that convert the data into
    physically relevant units; things like spatial and mass conversions, time
@@ -33,73 +33,147 @@
 If you are interested in adding a new code, be sure to drop us a line on
 `yt-dev <http://lists.spacepope.org/listinfo.cgi/yt-dev-spacepope.org>`_!
 
-To get started, make a new directory in ``yt/frontends`` with the name of your
-code -- you can start by copying into it the contents of the ``stream``
-directory, which is a pretty empty format. You'll then have to create a subclass
-of ``Dataset``. This subclass will need to handle conversion between the
-different physical units and the code units; for the most part, the examples of
-``OrionDataset`` and ``EnzoDataset`` should be followed, but
-``ChomboDataset``, as a slightly newer addition, can also be used as an
-instructive example -- be sure to add an ``_is_valid`` classmethod that will
-verify if a filename is valid for that output type, as that is how "load" works.
+To get started, make a new directory in ``yt/frontends`` with the name
+of your code.  Copying the contents of the ``yt/frontends/_skeleton``
+directory will add a lot of boilerplate for the required classes and
+methods that are needed.  In particular, you'll have to create a
+subclass of ``Dataset`` in the data_structures.py file. This subclass
+will need to handle conversion between the different physical units
+and the code units (typically in the ``_set_code_unit_attributes()``
+method), read in metadata describing the overall data on disk (via the
+``_parse_parameter_file()`` method), and provide a ``classmethod``
+called ``_is_valid()`` that lets the ``yt.load`` method help identify an
+input file as belonging to *this* particular ``Dataset`` subclass.
+For the most part, the examples of
+``yt.frontends.boxlib.data_structures.OrionDataset`` and
+``yt.frontends.enzo.data_structures.EnzoDataset`` should be followed,
+but ``yt.frontends.chombo.data_structures.ChomboDataset``, as a
+slightly newer addition, can also be used as an instructive example.
 
-A new set of fields must be added in the file ``fields.py`` in that directory.
-For the most part this means subclassing ``CodeFieldInfoContainer`` and adding
-the necessary fields specific to that code. Here is the Chombo field container:
+A new set of fields must be added in the file ``fields.py`` in your
+new directory.  For the most part this means subclassing 
+``FieldInfoContainer`` and adding the necessary fields specific to
+your code. Here is a snippet from the base BoxLib field container:
 
 .. code-block:: python
 
-    from UniversalFields import *
-    class ChomboFieldContainer(CodeFieldInfoContainer):
-        _shared_state = {}
-        _field_list = {}
-    ChomboFieldInfo = ChomboFieldContainer()
-    add_chombo_field = ChomboFieldInfo.add_field
+    from yt.fields.field_info_container import FieldInfoContainer
+    class BoxlibFieldInfo(FieldInfoContainer):
+        known_other_fields = (
+            ("density", (rho_units, ["density"], None)),
+	    ("eden", (eden_units, ["energy_density"], None)),
+	    ("xmom", (mom_units, ["momentum_x"], None)),
+	    ("ymom", (mom_units, ["momentum_y"], None)),
+	    ("zmom", (mom_units, ["momentum_z"], None)),
+	    ("temperature", ("K", ["temperature"], None)),
+	    ("Temp", ("K", ["temperature"], None)),
+	    ("x_velocity", ("cm/s", ["velocity_x"], None)),
+	    ("y_velocity", ("cm/s", ["velocity_y"], None)),
+	    ("z_velocity", ("cm/s", ["velocity_z"], None)),
+	    ("xvel", ("cm/s", ["velocity_x"], None)),
+	    ("yvel", ("cm/s", ["velocity_y"], None)),
+	    ("zvel", ("cm/s", ["velocity_z"], None)),
+	)
 
-The field container is a shared state object, which is why we explicitly set
-``_shared_state`` equal to a mutable.
+	known_particle_fields = (
+	    ("particle_mass", ("code_mass", [], None)),
+	    ("particle_position_x", ("code_length", [], None)),
+	    ("particle_position_y", ("code_length", [], None)),
+	    ("particle_position_z", ("code_length", [], None)),
+	    ("particle_momentum_x", (mom_units, [], None)),
+	    ("particle_momentum_y", (mom_units, [], None)),
+	    ("particle_momentum_z", (mom_units, [], None)),
+	    ("particle_angmomen_x", ("code_length**2/code_time", [], None)),
+	    ("particle_angmomen_y", ("code_length**2/code_time", [], None)),
+	    ("particle_angmomen_z", ("code_length**2/code_time", [], None)),
+	    ("particle_id", ("", ["particle_index"], None)),
+	    ("particle_mdot", ("code_mass/code_time", [], None)),
+	)
+
+The tuples, ``known_other_fields`` and ``known_particle_fields``
+contain entries, which are tuples of the form ``("name", ("units",
+["fields", "to", "alias"], "display_name"))``.  ``"name"`` is the name
+of a field stored on-disk in the dataset. ``"units"`` corresponds to
+the units of that field.  The list ``["fields", "to", "alias"]``
+allows you to specify additional aliases to this particular field; for
+example, if your on-disk field for the x-direction velocity were
+``"x-direction-velocity"``, maybe you'd prefer to alias to the more
+terse name of ``"xvel"``.  ``"display_name"`` is an optional parameter
+that can be used to specify how you want the field to be displayed on
+a plot; this can be LaTeX code, for example the density field could
+have a display name of ``r"\rho"``.  Omitting the ``"display_name"``
+will result in using a capitalized version of the ``"name"``.
 
 Data Localization Structures
 ----------------------------
 
-As of right now, the "grid patch" mechanism is going to remain in yt, however in
-the future that may change. As such, some other output formats -- like Gadget --
-may be shoe-horned in, slightly.
+These functions and classes let yt know about how the arrangement of
+data on disk corresponds to the physical arrangement of data within
+the simulation.  yt has grid datastructures for handling both
+patch-based and octree-based AMR codes.  The terms 'patch-based'
+and 'octree-based' are used somewhat loosely here.  For example,
+traditionally, the FLASH code used the paramesh AMR library, which is
+based on a tree structure, but the FLASH frontend in yt utilizes yt's
+patch-based datastructures.  It is up to the frontend developer to
+determine which yt datastructures best match the datastructures of
+their simulation code.
 
-Hierarchy
-^^^^^^^^^
+Both approaches -- patch-based and octree-based -- have a concept of a
+*Hierarchy* or *Index* (used somewhat interchangeably in the code) of
+datastructures and something that describes the elements that make up
+the Hierarchy or Index.  For patch-based codes, the Index is a
+collection of ``AMRGridPatch`` objects that describe a block of zones.
+For octree-based codes, the Index contains datastructures that hold
+information about the individual octs, namely an ``OctreeContainer``.
 
-To set up data localization, an ``AMRHierarchy`` subclass must be added in the
-file ``data_structures.py``. The index object must override the following
-methods:
+Hierarchy or Index
+^^^^^^^^^^^^^^^^^^
 
- * ``_detect_fields``: ``self.field_list`` must be populated as a list of
-   strings corresponding to "native" fields in the data files.
- * ``_setup_classes``: it's probably safe to crib this from one of the other
-   ``AMRHierarchy`` subclasses.
- * ``_count_grids``: this must set self.num_grids to be the total number of
-   grids in the simulation.
- * ``_parse_index``: this must fill in ``grid_left_edge``,
+To set up data localization, a ``GridIndex`` subclass for patch-based
+codes or an ``OctreeIndex`` subclass for octree-based codes must be
+added in the file ``data_structures.py``. Examples of these different
+types of ``Index`` can be found in, for example, the
+``yt.frontends.chombo.data_structures.ChomboHierarchy`` for patch-based
+codes and ``yt.frontends.ramses.data_structures.RAMSESIndex`` for
+octree-based codes.  
+
+For the most part, the ``GridIndex`` subclass must override (at a
+minimum) the following methods:
+
+ * ``_detect_output_fields()``: ``self.field_list`` must be populated as a list
+   of strings corresponding to "native" fields in the data files.
+ * ``_count_grids()``: this must set ``self.num_grids`` to be the total number
+   of grids (equivalently ``AMRGridPatch``'es) in the simulation.
+ * ``_parse_index()``: this must fill in ``grid_left_edge``,
    ``grid_right_edge``, ``grid_particle_count``, ``grid_dimensions`` and
-   ``grid_levels`` with the appropriate information. Additionally, ``grids``
-   must be an array of grid objects that already know their IDs.
- * ``_populate_grid_objects``: this initializes the grids by calling
-   ``_prepare_grid`` and ``_setup_dx`` on all of them.  Additionally, it should
-   set up ``Children`` and ``Parent`` lists on each grid object.
- * ``_setup_unknown_fields``: If a field is in the data file that yt doesn't
-   already know, this is where you make a guess at it.
- * ``_setup_derived_fields``: ``self.derived_field_list`` needs to be made a
-   list of strings that correspond to all derived fields valid for this
-   index.
+   ``grid_levels`` with the appropriate information.  Each of these variables 
+   is an array, with an entry for each of the ``self.num_grids`` grids.  
+   Additionally, ``grids``  must be an array of ``AMRGridPatch`` objects that 
+   already know their IDs.
+ * ``_populate_grid_objects()``: this initializes the grids by calling
+   ``_prepare_grid()`` and ``_setup_dx()`` on all of them.  Additionally, it 
+   should set up ``Children`` and ``Parent`` lists on each grid object.
 
-For the most part, the ``ChomboHierarchy`` should be the first place to look for
-hints on how to do this; ``EnzoHierarchy`` is also instructive.
+The ``OctreeIndex`` has somewhat analogous methods, but often with
+different names; both ``OctreeIndex`` and ``GridIndex`` are subclasses
+of the ``Index`` class.  In particular, for the ``OctreeIndex``, the
+method ``_initialize_oct_handler()`` setups up much of the oct
+metadata that is analogous to the grid metadata created in the
+``GridIndex`` methods ``_count_grids()``, ``_parse_index()``, and
+``_populate_grid_objects()``.
 
 Grids
 ^^^^^
 
-A new grid object, subclassing ``AMRGridPatch``, will also have to be added.
-This should go in ``data_structures.py``. For the most part, this may be all
+.. note:: This section only applies to the approach using yt's patch-based
+	  datastructures.  For the octree-based approach, one does not create
+	  a grid object, but rather an ``OctreeSubset``, which has methods
+	  for filling out portions of the octree structure.  Again, see the
+	  code in ``yt.frontends.ramses.data_structures`` for an example of
+	  the octree approach.
+
+A new grid object, subclassing ``AMRGridPatch``, will also have to be added in
+``data_structures.py``. For the most part, this may be all
 that is needed:
 
 .. code-block:: python
@@ -115,32 +189,46 @@
             self.Level = level
 
 
-Even the most complex grid object, ``OrionGrid``, is still relatively simple.
+Even one of the more complex grid objects,
+``yt.frontends.boxlib.BoxlibGrid``, is still relatively simple.
 
 Data Reading Functions
 ----------------------
 
-In ``io.py``, there are a number of IO handlers that handle the mechanisms by
-which data is read off disk.  To implement a new data reader, you must subclass
-``BaseIOHandler`` and override the following methods:
+In ``io.py``, there are a number of IO handlers that handle the
+mechanisms by which data is read off disk.  To implement a new data
+reader, you must subclass ``BaseIOHandler``.  The various frontend IO
+handlers are stored in an IO registry - essentially a dictionary that
+uses the name of the frontend as a key, and the specific IO handler as
+a value.  It is important, therefore, to set the ``dataset_type``
+attribute of your subclass, which is what is used as the key in the IO
+registry.  For example:
 
- * ``_read_field_names``: this routine accepts a grid object and must return all
-   the fields in the data file affiliated with that grid. It is used at the
-   initialization of the ``AMRHierarchy`` but likely not later.
- * ``modify``: This accepts a field from a data file and returns it ready to be
-   used by yt. This is used in Enzo data for preloading.
- * ``_read_data_set``: This accepts a grid object and a field name and must
-   return that field, ready to be used by yt as a NumPy array. Note that this
-   presupposes that any actions done in ``modify`` (above) have been executed.
- * ``_read_data_slice``: This accepts a grid object, a field name, an axis and
-   an (integer) coordinate, and it must return a slice through the array at that
-   value.
- * ``preload``: (optional) This accepts a list of grids and a list of datasets
-   and it populates ``self.queue`` (a dict keyed by grid id) with dicts of
-   datasets.
- * ``_read_exception``: (property) This is a tuple of exceptions that can be
-   raised by the data reading to indicate a field does not exist in the file.
+.. code-block:: python
 
+    class IOHandlerBoxlib(BaseIOHandler):
+        _dataset_type = "boxlib_native"
+	...
+
+At a minimum, one should also override the following methods
+
+* ``_read_fluid_selection()``: this receives a collection of data "chunks", a 
+  selector describing which "chunks" you are concerned with, a list of fields,
+  and the size of the data to read.  It should create and return a dictionary 
+  whose keys are the fields, and whose values are numpy arrays containing the 
+  data.  The data should actually be read via the ``_read_chunk_data()`` 
+  method.
+* ``_read_chunk_data()``: this method receives a "chunk" of data along with a 
+  list of fields we want to read.  It loops over all the grid objects within 
+  the "chunk" of data and reads from disk the specific fields, returning a 
+  dictionary whose keys are the fields and whose values are numpy arrays of
+  the data.
+
+If your dataset has particle information, you'll want to override the
+``_read_particle_coords()`` and ``read_particle_fields()`` methods as
+well.  Each code is going to read data from disk in a different
+fashion, but the ``yt.frontends.boxlib.io.IOHandlerBoxlib`` is a
+decent place to start.
 
 And that just about covers it. Please feel free to email
 `yt-users <http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org>`_ or

diff -r de7802382dc0cc71b3e1eb15b37bbdcfec5b218a -r 6f5a2e3f405f9ec96211c59085cc6101891ad655 yt/frontends/_skeleton/api.py
--- a/yt/frontends/_skeleton/api.py
+++ b/yt/frontends/_skeleton/api.py
@@ -19,8 +19,7 @@
       SkeletonDataset
 
 from .fields import \
-      SkeletonFieldInfo, \
-      add_skeleton_field
+      SkeletonFieldInfo
 
 from .io import \
       IOHandlerSkeleton

diff -r de7802382dc0cc71b3e1eb15b37bbdcfec5b218a -r 6f5a2e3f405f9ec96211c59085cc6101891ad655 yt/frontends/_skeleton/data_structures.py
--- a/yt/frontends/_skeleton/data_structures.py
+++ b/yt/frontends/_skeleton/data_structures.py
@@ -13,18 +13,13 @@
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
-import numpy as np
-
-from yt.data_objects.grid_patch import \
-    AMRGridPatch
 from yt.data_objects.grid_patch import \
     AMRGridPatch
 from yt.geometry.grid_geometry_handler import \
     GridIndex
 from yt.data_objects.static_output import \
     Dataset
-from yt.utilities.lib.misc_utilities import \
-    get_box_grids_level
+from .fields import SkeletonFieldInfo
 
 class SkeletonGrid(AMRGridPatch):
     _id_offset = 0
@@ -41,20 +36,15 @@
     def __repr__(self):
         return "SkeletonGrid_%04i (%s)" % (self.id, self.ActiveDimensions)
 
-class SkeletonHierarchy(AMRHierarchy):
-
+class SkeletonHierarchy(GridIndex):
     grid = SkeletonGrid
     
     def __init__(self, ds, dataset_type='skeleton'):
         self.dataset_type = dataset_type
-        self.dataset = weakref.proxy(ds)
         # for now, the index file is the dataset!
         self.index_filename = self.dataset.parameter_filename
         self.directory = os.path.dirname(self.index_filename)
-        AMRHierarchy.__init__(self, ds, dataset_type)
-
-    def _initialize_data_storage(self):
-        pass
+        GridIndex.__init__(self, ds, dataset_type)
 
     def _detect_output_fields(self):
         # This needs to set a self.field_list that contains all the available,

diff -r de7802382dc0cc71b3e1eb15b37bbdcfec5b218a -r 6f5a2e3f405f9ec96211c59085cc6101891ad655 yt/frontends/_skeleton/io.py
--- a/yt/frontends/_skeleton/io.py
+++ b/yt/frontends/_skeleton/io.py
@@ -13,9 +13,6 @@
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
-import numpy as np
-import h5py
-
 from yt.utilities.io_handler import \
     BaseIOHandler

Repository URL: https://bitbucket.org/yt_analysis/yt/

--

This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.



More information about the yt-svn mailing list