[yt-svn] commit/yt: 13 new changesets
commits-noreply at bitbucket.org
commits-noreply at bitbucket.org
Tue Oct 21 04:41:46 PDT 2014
13 new commits in yt:
https://bitbucket.org/yt_analysis/yt/commits/c0a911fe5cdf/
Changeset: c0a911fe5cdf
Branch: yt
User: ChrisMalone
Date: 2014-10-16 22:08:54+00:00
Summary: start modernizing skeleton frontend
Affected #: 2 files
diff -r 1089b61212d7bfd105a109728fb804b6ef84aadf -r c0a911fe5cdffc0b947fceae13a575f768c02da6 yt/frontends/_skeleton/api.py
--- a/yt/frontends/_skeleton/api.py
+++ b/yt/frontends/_skeleton/api.py
@@ -19,8 +19,7 @@
SkeletonDataset
from .fields import \
- SkeletonFieldInfo, \
- add_skeleton_field
+ SkeletonFieldInfo
from .io import \
IOHandlerSkeleton
diff -r 1089b61212d7bfd105a109728fb804b6ef84aadf -r c0a911fe5cdffc0b947fceae13a575f768c02da6 yt/frontends/_skeleton/data_structures.py
--- a/yt/frontends/_skeleton/data_structures.py
+++ b/yt/frontends/_skeleton/data_structures.py
@@ -13,18 +13,13 @@
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
-import numpy as np
-
-from yt.data_objects.grid_patch import \
- AMRGridPatch
from yt.data_objects.grid_patch import \
AMRGridPatch
from yt.geometry.grid_geometry_handler import \
GridIndex
from yt.data_objects.static_output import \
Dataset
-from yt.utilities.lib.misc_utilities import \
- get_box_grids_level
+from .fields import SkeletonFieldInfo
class SkeletonGrid(AMRGridPatch):
_id_offset = 0
@@ -41,17 +36,15 @@
def __repr__(self):
return "SkeletonGrid_%04i (%s)" % (self.id, self.ActiveDimensions)
-class SkeletonHierarchy(AMRHierarchy):
-
+class SkeletonHierarchy(GridIndex):
grid = SkeletonGrid
def __init__(self, ds, dataset_type='skeleton'):
self.dataset_type = dataset_type
- self.dataset = weakref.proxy(ds)
# for now, the index file is the dataset!
self.index_filename = self.dataset.parameter_filename
self.directory = os.path.dirname(self.index_filename)
- AMRHierarchy.__init__(self, ds, dataset_type)
+ GridIndex.__init__(self, ds, dataset_type)
def _initialize_data_storage(self):
pass
https://bitbucket.org/yt_analysis/yt/commits/521c502bbf4b/
Changeset: 521c502bbf4b
Branch: yt
User: ChrisMalone
Date: 2014-10-17 18:31:55+00:00
Summary: update data-meaning section
Affected #: 1 file
diff -r c0a911fe5cdffc0b947fceae13a575f768c02da6 -r 521c502bbf4b5ca2ad5bcb80438e41d6b8f80cfd doc/source/developing/creating_frontend.rst
--- a/doc/source/developing/creating_frontend.rst
+++ b/doc/source/developing/creating_frontend.rst
@@ -33,31 +33,50 @@
If you are interested in adding a new code, be sure to drop us a line on
`yt-dev <http://lists.spacepope.org/listinfo.cgi/yt-dev-spacepope.org>`_!
-To get started, make a new directory in ``yt/frontends`` with the name of your
-code -- you can start by copying into it the contents of the ``stream``
-directory, which is a pretty empty format. You'll then have to create a subclass
-of ``Dataset``. This subclass will need to handle conversion between the
-different physical units and the code units; for the most part, the examples of
-``OrionDataset`` and ``EnzoDataset`` should be followed, but
-``ChomboDataset``, as a slightly newer addition, can also be used as an
-instructive example -- be sure to add an ``_is_valid`` classmethod that will
-verify if a filename is valid for that output type, as that is how "load" works.
+To get started, make a new directory in ``yt/frontends`` with the name
+of your code. Copying the contents of the ``yt/frontends/_skeleton``
+directory will add a lot of boilerplate for the required classes and
+methods that are needed. In particular, you'll have to create a
+subclass of ``Dataset``. This subclass will need to handle conversion
+between the different physical units and the code units (typically in
+the ``_set_code_unit_attributes`` method), read in metadata describing
+the overall data on disk (via the ``_parse_parameter_file`` method),
+and provide a ``classmethod`` called ``_is_valid`` that lets the
+``yt.load`` method help identify an input file as belonging to *this*
+particular ``Dataset`` subclass. For the most part, the examples of
+``yt.frontends.boxlib.OrionDataset`` and
+``yt.frontends.enzo.EnzoDataset`` should be followed, but
+``yt.frontends.chombo.ChomboDataset``, as a slightly newer addition,
+can also be used as an instructive example.
-A new set of fields must be added in the file ``fields.py`` in that directory.
-For the most part this means subclassing ``CodeFieldInfoContainer`` and adding
-the necessary fields specific to that code. Here is the Chombo field container:
+A new set of fields must be added in the file ``fields.py`` in your
+new directory. For the most part this means subclassing
+``FieldInfoContainer`` and adding the necessary fields specific to
+your code. Here is the base Chombo field container:
.. code-block:: python
- from UniversalFields import *
- class ChomboFieldContainer(CodeFieldInfoContainer):
- _shared_state = {}
- _field_list = {}
- ChomboFieldInfo = ChomboFieldContainer()
- add_chombo_field = ChomboFieldInfo.add_field
+ from yt.fields.field_info_container import FieldInfoContainer
+ class ChomboFieldInfo(FieldInfoContainer):
+ known_other_fields = ()
+ known_particle_fields = ()
-The field container is a shared state object, which is why we explicitly set
-``_shared_state`` equal to a mutable.
+This is a very stripped down class. There are several codes
+(e.g. Orion2 and Pluto) that use a Chombo-based datastructure, and
+those particular codes subclass the ``ChomboFieldInfo`` class. The
+tuples, ``known_other_fields`` and ``known_particle_fields`` contain
+entries, which are tuples of the form ``("name", ("units", ["fields",
+"to", "alias"], "display_name"))``. ``"name"`` is the name of a field
+stored on-disk in the dataset. ``"units"`` corresponds to the units of
+that field. The list ``["fields", "to", "alias"]`` allows you to
+specify additional aliases to this particular field; for example, if
+your on-disk field for the x-direction velocity were
+``"x-direction-velocity"``, maybe you'd prefer to alias to the more
+terse name of ``"xvel"``. ``"display_name"`` is an optional parameter
+that can be used to specify how you want the field to be displayed on
+a plot; this can be LaTeX-code, for example the density field could
+have a display name of ``r"\rho"``. Omitting the ``"display_name"``
+will result in using a capitalized version of the ``"name"``.
Data Localization Structures
----------------------------
https://bitbucket.org/yt_analysis/yt/commits/2d05db9ccc99/
Changeset: 2d05db9ccc99
Branch: yt
User: ChrisMalone
Date: 2014-10-17 18:33:50+00:00
Summary: typo
Affected #: 1 file
diff -r 521c502bbf4b5ca2ad5bcb80438e41d6b8f80cfd -r 2d05db9ccc991170cff3a1fd6b3515f6fca7bc0e doc/source/developing/creating_frontend.rst
--- a/doc/source/developing/creating_frontend.rst
+++ b/doc/source/developing/creating_frontend.rst
@@ -74,7 +74,7 @@
``"x-direction-velocity"``, maybe you'd prefer to alias to the more
terse name of ``"xvel"``. ``"display_name"`` is an optional parameter
that can be used to specify how you want the field to be displayed on
-a plot; this can be LaTeX-code, for example the density field could
+a plot; this can be LaTeX code, for example the density field could
have a display name of ``r"\rho"``. Omitting the ``"display_name"``
will result in using a capitalized version of the ``"name"``.
https://bitbucket.org/yt_analysis/yt/commits/7bc9cf7c0d9b/
Changeset: 7bc9cf7c0d9b
Branch: yt
User: ChrisMalone
Date: 2014-10-17 19:49:31+00:00
Summary: more modernizing the docs
Affected #: 1 file
diff -r 2d05db9ccc991170cff3a1fd6b3515f6fca7bc0e -r 7bc9cf7c0d9b382a9f31ec65f2b465271384d579 doc/source/developing/creating_frontend.rst
--- a/doc/source/developing/creating_frontend.rst
+++ b/doc/source/developing/creating_frontend.rst
@@ -37,17 +37,18 @@
of your code. Copying the contents of the ``yt/frontends/_skeleton``
directory will add a lot of boilerplate for the required classes and
methods that are needed. In particular, you'll have to create a
-subclass of ``Dataset``. This subclass will need to handle conversion
-between the different physical units and the code units (typically in
-the ``_set_code_unit_attributes`` method), read in metadata describing
-the overall data on disk (via the ``_parse_parameter_file`` method),
-and provide a ``classmethod`` called ``_is_valid`` that lets the
-``yt.load`` method help identify an input file as belonging to *this*
-particular ``Dataset`` subclass. For the most part, the examples of
-``yt.frontends.boxlib.OrionDataset`` and
-``yt.frontends.enzo.EnzoDataset`` should be followed, but
-``yt.frontends.chombo.ChomboDataset``, as a slightly newer addition,
-can also be used as an instructive example.
+subclass of ``Dataset`` in the data_structures.py file. This subclass
+will need to handle conversion between the different physical units
+and the code units (typically in the ``_set_code_unit_attributes()``
+method), read in metadata describing the overall data on disk (via the
+``_parse_parameter_file()`` method), and provide a ``classmethod``
+called ``_is_valid()`` that lets the ``yt.load`` method help identify an
+input file as belonging to *this* particular ``Dataset`` subclass.
+For the most part, the examples of
+``yt.frontends.boxlib.data_structures.OrionDataset`` and
+``yt.frontends.enzo.data_structures.EnzoDataset`` should be followed,
+but ``yt.frontends.chombo.data_structures.ChomboDataset``, as a
+slightly newer addition, can also be used as an instructive example.
A new set of fields must be added in the file ``fields.py`` in your
new directory. For the most part this means subclassing
@@ -61,7 +62,7 @@
known_other_fields = ()
known_particle_fields = ()
-This is a very stripped down class. There are several codes
+This is a very stripped-down class. There are several codes
(e.g. Orion2 and Pluto) that use a Chombo-based datastructure, and
those particular codes subclass the ``ChomboFieldInfo`` class. The
tuples, ``known_other_fields`` and ``known_particle_fields`` contain
@@ -81,44 +82,51 @@
Data Localization Structures
----------------------------
-As of right now, the "grid patch" mechanism is going to remain in yt, however in
-the future that may change. As such, some other output formats -- like Gadget --
-may be shoe-horned in, slightly.
+These functions and classes let yt know about how the arrangement of
+data on disk corresponds to the physical arrangement of data within
+the simulation. There are some subtle differences between
+AMR/patch-based codes and Octree-based codes, however, both approaches
+have a concept of a *Hierarchy* or *Index* (used somewhat
+interchangeably in the code) of datastructures and something that
+describes the elements that make up the Hierarchy or Index. For
+AMR-based codes, the Index is a collection of ``AMRGridPatch`` objects
+that describe a block of zones. For Octree-based codes, the Index
+contains datastructures that hold information about the individual
+octs, namely an ``OctreeContainer``.
-Hierarchy
-^^^^^^^^^
+Hierarchy or Index
+^^^^^^^^^^^^^^^^^^
-To set up data localization, an ``AMRHierarchy`` subclass must be added in the
-file ``data_structures.py``. The index object must override the following
-methods:
+To set up data localization, a ``GridIndex`` subclass for AMR-based
+codes or an ``OctreeIndex`` subclass for Octree-based codes must be
+added in the file ``data_structures.py``. Examples of these different
+types of ``Index`` can be found in, for example, the
+``yt.frontends.chombo.data_structures.ChomboHierarchy`` for AMR-based
+codes and ``yt.frontends.ramses.data_structures.RAMSESIndex`` for
+Octree-based codes.
- * ``_detect_fields``: ``self.field_list`` must be populated as a list of
- strings corresponding to "native" fields in the data files.
- * ``_setup_classes``: it's probably safe to crib this from one of the other
- ``AMRHierarchy`` subclasses.
- * ``_count_grids``: this must set self.num_grids to be the total number of
- grids in the simulation.
- * ``_parse_index``: this must fill in ``grid_left_edge``,
+For the most part, the ``GridIndex`` subclass must override (at a
+minimum) the following methods:
+
+ * ``_detect_output_fields()``: ``self.field_list`` must be populated as a list
+ of strings corresponding to "native" fields in the data files.
+ * ``_count_grids()``: this must set ``self.num_grids`` to be the total number
+ of grids (equivalently ``AMRGridPatch``'es) in the simulation.
+ * ``_parse_index()``: this must fill in ``grid_left_edge``,
``grid_right_edge``, ``grid_particle_count``, ``grid_dimensions`` and
- ``grid_levels`` with the appropriate information. Additionally, ``grids``
- must be an array of grid objects that already know their IDs.
- * ``_populate_grid_objects``: this initializes the grids by calling
- ``_prepare_grid`` and ``_setup_dx`` on all of them. Additionally, it should
- set up ``Children`` and ``Parent`` lists on each grid object.
- * ``_setup_unknown_fields``: If a field is in the data file that yt doesn't
- already know, this is where you make a guess at it.
- * ``_setup_derived_fields``: ``self.derived_field_list`` needs to be made a
- list of strings that correspond to all derived fields valid for this
- index.
-
-For the most part, the ``ChomboHierarchy`` should be the first place to look for
-hints on how to do this; ``EnzoHierarchy`` is also instructive.
+ ``grid_levels`` with the appropriate information. Each of these variables
+ is an array, with an entry for each of the ``self.num_grids`` grids.
+ Additionally, ``grids`` must be an array of ``AMRGridPatch`` objects that
+ already know their IDs.
+ * ``_populate_grid_objects()``: this initializes the grids by calling
+ ``_prepare_grid()`` and ``_setup_dx()`` on all of them. Additionally, it
+ should set up ``Children`` and ``Parent`` lists on each grid object.
Grids
^^^^^
-A new grid object, subclassing ``AMRGridPatch``, will also have to be added.
-This should go in ``data_structures.py``. For the most part, this may be all
+A new grid object, subclassing ``AMRGridPatch``, will also have to be added in
+``data_structures.py``. For the most part, this may be all
that is needed:
.. code-block:: python
@@ -134,32 +142,35 @@
self.Level = level
-Even the most complex grid object, ``OrionGrid``, is still relatively simple.
+Even one of the more complex grid object,
+``yt.frontends.boxlib.BoxlibGrid``, is still relatively simple.
Data Reading Functions
----------------------
In ``io.py``, there are a number of IO handlers that handle the mechanisms by
which data is read off disk. To implement a new data reader, you must subclass
-``BaseIOHandler`` and override the following methods:
+``BaseIOHandler``. The various frontend IO handlers are stored in an IO registry - essentially a dictionary that uses the name of the frontend as a key, and the specific IO handler as a value. It is important, therefore, to set the ``dataset_type`` attribute of your subclass, which is what is used as the key in the IO registry. For example:
- * ``_read_field_names``: this routine accepts a grid object and must return all
- the fields in the data file affiliated with that grid. It is used at the
- initialization of the ``AMRHierarchy`` but likely not later.
- * ``modify``: This accepts a field from a data file and returns it ready to be
- used by yt. This is used in Enzo data for preloading.
- * ``_read_data_set``: This accepts a grid object and a field name and must
- return that field, ready to be used by yt as a NumPy array. Note that this
- presupposes that any actions done in ``modify`` (above) have been executed.
- * ``_read_data_slice``: This accepts a grid object, a field name, an axis and
- an (integer) coordinate, and it must return a slice through the array at that
- value.
- * ``preload``: (optional) This accepts a list of grids and a list of datasets
- and it populates ``self.queue`` (a dict keyed by grid id) with dicts of
- datasets.
- * ``_read_exception``: (property) This is a tuple of exceptions that can be
- raised by the data reading to indicate a field does not exist in the file.
+.. code-block:: python
+ class IOHandlerBoxlib(BaseIOHandler):
+ _dataset_type = "boxlib_native"
+ ...
+
+At a minimum, one should also override the following methods
+
+* ``_read_fluid_selection()``: this receives a collection of data "chunks", a
+ selector describing which "chunks" you are concerned with, a list of fields,
+ and the size of the data to read. It should create and return a dictionary
+ whose keys are the fields, and whose values are numpy arrays containing the
+ data. The data should actually be read via the ``_read_chunk_data()``
+ method.
+* ``_read_chunk_data()``: this method receives a "chunk" of data along with a
+ list of fields we want to read. It loops over all the grid objects within
+ the "chunk" of data and reads from disk the specific fields, returning a
+ dictionary whose keys are the fields and whose values are numpy arrays of
+ the data.
And that just about covers it. Please feel free to email
`yt-users <http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org>`_ or
https://bitbucket.org/yt_analysis/yt/commits/9716d88c14a2/
Changeset: 9716d88c14a2
Branch: yt
User: ChrisMalone
Date: 2014-10-17 19:50:07+00:00
Summary: remove unused modules
Affected #: 1 file
diff -r 7bc9cf7c0d9b382a9f31ec65f2b465271384d579 -r 9716d88c14a2f40af3854b56bf2d409e23828614 yt/frontends/_skeleton/io.py
--- a/yt/frontends/_skeleton/io.py
+++ b/yt/frontends/_skeleton/io.py
@@ -13,9 +13,6 @@
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
-import numpy as np
-import h5py
-
from yt.utilities.io_handler import \
BaseIOHandler
https://bitbucket.org/yt_analysis/yt/commits/e16ad5b3fd23/
Changeset: e16ad5b3fd23
Branch: yt
User: ChrisMalone
Date: 2014-10-17 19:50:23+00:00
Summary: remove unused method
Affected #: 1 file
diff -r 9716d88c14a2f40af3854b56bf2d409e23828614 -r e16ad5b3fd232ecba54637c5dfc61914a2f77546 yt/frontends/_skeleton/data_structures.py
--- a/yt/frontends/_skeleton/data_structures.py
+++ b/yt/frontends/_skeleton/data_structures.py
@@ -46,9 +46,6 @@
self.directory = os.path.dirname(self.index_filename)
GridIndex.__init__(self, ds, dataset_type)
- def _initialize_data_storage(self):
- pass
-
def _detect_output_fields(self):
# This needs to set a self.field_list that contains all the available,
# on-disk fields.
https://bitbucket.org/yt_analysis/yt/commits/3309b76428a7/
Changeset: 3309b76428a7
Branch: yt
User: ChrisMalone
Date: 2014-10-17 19:54:04+00:00
Summary: merge
Affected #: 4 files
diff -r 558d9f064464adf497d777b760b9e7ac12f9baea -r 3309b76428a71a81dbfd67e2184d2587807dd860 doc/source/developing/creating_frontend.rst
--- a/doc/source/developing/creating_frontend.rst
+++ b/doc/source/developing/creating_frontend.rst
@@ -33,73 +33,100 @@
If you are interested in adding a new code, be sure to drop us a line on
`yt-dev <http://lists.spacepope.org/listinfo.cgi/yt-dev-spacepope.org>`_!
-To get started, make a new directory in ``yt/frontends`` with the name of your
-code -- you can start by copying into it the contents of the ``stream``
-directory, which is a pretty empty format. You'll then have to create a subclass
-of ``Dataset``. This subclass will need to handle conversion between the
-different physical units and the code units; for the most part, the examples of
-``OrionDataset`` and ``EnzoDataset`` should be followed, but
-``ChomboDataset``, as a slightly newer addition, can also be used as an
-instructive example -- be sure to add an ``_is_valid`` classmethod that will
-verify if a filename is valid for that output type, as that is how "load" works.
+To get started, make a new directory in ``yt/frontends`` with the name
+of your code. Copying the contents of the ``yt/frontends/_skeleton``
+directory will add a lot of boilerplate for the required classes and
+methods that are needed. In particular, you'll have to create a
+subclass of ``Dataset`` in the data_structures.py file. This subclass
+will need to handle conversion between the different physical units
+and the code units (typically in the ``_set_code_unit_attributes()``
+method), read in metadata describing the overall data on disk (via the
+``_parse_parameter_file()`` method), and provide a ``classmethod``
+called ``_is_valid()`` that lets the ``yt.load`` method help identify an
+input file as belonging to *this* particular ``Dataset`` subclass.
+For the most part, the examples of
+``yt.frontends.boxlib.data_structures.OrionDataset`` and
+``yt.frontends.enzo.data_structures.EnzoDataset`` should be followed,
+but ``yt.frontends.chombo.data_structures.ChomboDataset``, as a
+slightly newer addition, can also be used as an instructive example.
-A new set of fields must be added in the file ``fields.py`` in that directory.
-For the most part this means subclassing ``CodeFieldInfoContainer`` and adding
-the necessary fields specific to that code. Here is the Chombo field container:
+A new set of fields must be added in the file ``fields.py`` in your
+new directory. For the most part this means subclassing
+``FieldInfoContainer`` and adding the necessary fields specific to
+your code. Here is the base Chombo field container:
.. code-block:: python
- from UniversalFields import *
- class ChomboFieldContainer(CodeFieldInfoContainer):
- _shared_state = {}
- _field_list = {}
- ChomboFieldInfo = ChomboFieldContainer()
- add_chombo_field = ChomboFieldInfo.add_field
+ from yt.fields.field_info_container import FieldInfoContainer
+ class ChomboFieldInfo(FieldInfoContainer):
+ known_other_fields = ()
+ known_particle_fields = ()
-The field container is a shared state object, which is why we explicitly set
-``_shared_state`` equal to a mutable.
+This is a very stripped-down class. There are several codes
+(e.g. Orion2 and Pluto) that use a Chombo-based datastructure, and
+those particular codes subclass the ``ChomboFieldInfo`` class. The
+tuples, ``known_other_fields`` and ``known_particle_fields`` contain
+entries, which are tuples of the form ``("name", ("units", ["fields",
+"to", "alias"], "display_name"))``. ``"name"`` is the name of a field
+stored on-disk in the dataset. ``"units"`` corresponds to the units of
+that field. The list ``["fields", "to", "alias"]`` allows you to
+specify additional aliases to this particular field; for example, if
+your on-disk field for the x-direction velocity were
+``"x-direction-velocity"``, maybe you'd prefer to alias to the more
+terse name of ``"xvel"``. ``"display_name"`` is an optional parameter
+that can be used to specify how you want the field to be displayed on
+a plot; this can be LaTeX code, for example the density field could
+have a display name of ``r"\rho"``. Omitting the ``"display_name"``
+will result in using a capitalized version of the ``"name"``.
Data Localization Structures
----------------------------
-As of right now, the "grid patch" mechanism is going to remain in yt, however in
-the future that may change. As such, some other output formats -- like Gadget --
-may be shoe-horned in, slightly.
+These functions and classes let yt know about how the arrangement of
+data on disk corresponds to the physical arrangement of data within
+the simulation. There are some subtle differences between
+AMR/patch-based codes and Octree-based codes, however, both approaches
+have a concept of a *Hierarchy* or *Index* (used somewhat
+interchangeably in the code) of datastructures and something that
+describes the elements that make up the Hierarchy or Index. For
+AMR-based codes, the Index is a collection of ``AMRGridPatch`` objects
+that describe a block of zones. For Octree-based codes, the Index
+contains datastructures that hold information about the individual
+octs, namely an ``OctreeContainer``.
-Hierarchy
-^^^^^^^^^
+Hierarchy or Index
+^^^^^^^^^^^^^^^^^^
-To set up data localization, an ``AMRHierarchy`` subclass must be added in the
-file ``data_structures.py``. The index object must override the following
-methods:
+To set up data localization, a ``GridIndex`` subclass for AMR-based
+codes or an ``OctreeIndex`` subclass for Octree-based codes must be
+added in the file ``data_structures.py``. Examples of these different
+types of ``Index`` can be found in, for example, the
+``yt.frontends.chombo.data_structures.ChomboHierarchy`` for AMR-based
+codes and ``yt.frontends.ramses.data_structures.RAMSESIndex`` for
+Octree-based codes.
- * ``_detect_fields``: ``self.field_list`` must be populated as a list of
- strings corresponding to "native" fields in the data files.
- * ``_setup_classes``: it's probably safe to crib this from one of the other
- ``AMRHierarchy`` subclasses.
- * ``_count_grids``: this must set self.num_grids to be the total number of
- grids in the simulation.
- * ``_parse_index``: this must fill in ``grid_left_edge``,
+For the most part, the ``GridIndex`` subclass must override (at a
+minimum) the following methods:
+
+ * ``_detect_output_fields()``: ``self.field_list`` must be populated as a list
+ of strings corresponding to "native" fields in the data files.
+ * ``_count_grids()``: this must set ``self.num_grids`` to be the total number
+ of grids (equivalently ``AMRGridPatch``'es) in the simulation.
+ * ``_parse_index()``: this must fill in ``grid_left_edge``,
``grid_right_edge``, ``grid_particle_count``, ``grid_dimensions`` and
- ``grid_levels`` with the appropriate information. Additionally, ``grids``
- must be an array of grid objects that already know their IDs.
- * ``_populate_grid_objects``: this initializes the grids by calling
- ``_prepare_grid`` and ``_setup_dx`` on all of them. Additionally, it should
- set up ``Children`` and ``Parent`` lists on each grid object.
- * ``_setup_unknown_fields``: If a field is in the data file that yt doesn't
- already know, this is where you make a guess at it.
- * ``_setup_derived_fields``: ``self.derived_field_list`` needs to be made a
- list of strings that correspond to all derived fields valid for this
- index.
-
-For the most part, the ``ChomboHierarchy`` should be the first place to look for
-hints on how to do this; ``EnzoHierarchy`` is also instructive.
+ ``grid_levels`` with the appropriate information. Each of these variables
+ is an array, with an entry for each of the ``self.num_grids`` grids.
+ Additionally, ``grids`` must be an array of ``AMRGridPatch`` objects that
+ already know their IDs.
+ * ``_populate_grid_objects()``: this initializes the grids by calling
+ ``_prepare_grid()`` and ``_setup_dx()`` on all of them. Additionally, it
+ should set up ``Children`` and ``Parent`` lists on each grid object.
Grids
^^^^^
-A new grid object, subclassing ``AMRGridPatch``, will also have to be added.
-This should go in ``data_structures.py``. For the most part, this may be all
+A new grid object, subclassing ``AMRGridPatch``, will also have to be added in
+``data_structures.py``. For the most part, this may be all
that is needed:
.. code-block:: python
@@ -115,32 +142,35 @@
self.Level = level
-Even the most complex grid object, ``OrionGrid``, is still relatively simple.
+Even one of the more complex grid object,
+``yt.frontends.boxlib.BoxlibGrid``, is still relatively simple.
Data Reading Functions
----------------------
In ``io.py``, there are a number of IO handlers that handle the mechanisms by
which data is read off disk. To implement a new data reader, you must subclass
-``BaseIOHandler`` and override the following methods:
+``BaseIOHandler``. The various frontend IO handlers are stored in an IO registry - essentially a dictionary that uses the name of the frontend as a key, and the specific IO handler as a value. It is important, therefore, to set the ``dataset_type`` attribute of your subclass, which is what is used as the key in the IO registry. For example:
- * ``_read_field_names``: this routine accepts a grid object and must return all
- the fields in the data file affiliated with that grid. It is used at the
- initialization of the ``AMRHierarchy`` but likely not later.
- * ``modify``: This accepts a field from a data file and returns it ready to be
- used by yt. This is used in Enzo data for preloading.
- * ``_read_data_set``: This accepts a grid object and a field name and must
- return that field, ready to be used by yt as a NumPy array. Note that this
- presupposes that any actions done in ``modify`` (above) have been executed.
- * ``_read_data_slice``: This accepts a grid object, a field name, an axis and
- an (integer) coordinate, and it must return a slice through the array at that
- value.
- * ``preload``: (optional) This accepts a list of grids and a list of datasets
- and it populates ``self.queue`` (a dict keyed by grid id) with dicts of
- datasets.
- * ``_read_exception``: (property) This is a tuple of exceptions that can be
- raised by the data reading to indicate a field does not exist in the file.
+.. code-block:: python
+ class IOHandlerBoxlib(BaseIOHandler):
+ _dataset_type = "boxlib_native"
+ ...
+
+At a minimum, one should also override the following methods
+
+* ``_read_fluid_selection()``: this receives a collection of data "chunks", a
+ selector describing which "chunks" you are concerned with, a list of fields,
+ and the size of the data to read. It should create and return a dictionary
+ whose keys are the fields, and whose values are numpy arrays containing the
+ data. The data should actually be read via the ``_read_chunk_data()``
+ method.
+* ``_read_chunk_data()``: this method receives a "chunk" of data along with a
+ list of fields we want to read. It loops over all the grid objects within
+ the "chunk" of data and reads from disk the specific fields, returning a
+ dictionary whose keys are the fields and whose values are numpy arrays of
+ the data.
And that just about covers it. Please feel free to email
`yt-users <http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org>`_ or
diff -r 558d9f064464adf497d777b760b9e7ac12f9baea -r 3309b76428a71a81dbfd67e2184d2587807dd860 yt/frontends/_skeleton/api.py
--- a/yt/frontends/_skeleton/api.py
+++ b/yt/frontends/_skeleton/api.py
@@ -19,8 +19,7 @@
SkeletonDataset
from .fields import \
- SkeletonFieldInfo, \
- add_skeleton_field
+ SkeletonFieldInfo
from .io import \
IOHandlerSkeleton
diff -r 558d9f064464adf497d777b760b9e7ac12f9baea -r 3309b76428a71a81dbfd67e2184d2587807dd860 yt/frontends/_skeleton/data_structures.py
--- a/yt/frontends/_skeleton/data_structures.py
+++ b/yt/frontends/_skeleton/data_structures.py
@@ -13,18 +13,13 @@
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
-import numpy as np
-
-from yt.data_objects.grid_patch import \
- AMRGridPatch
from yt.data_objects.grid_patch import \
AMRGridPatch
from yt.geometry.grid_geometry_handler import \
GridIndex
from yt.data_objects.static_output import \
Dataset
-from yt.utilities.lib.misc_utilities import \
- get_box_grids_level
+from .fields import SkeletonFieldInfo
class SkeletonGrid(AMRGridPatch):
_id_offset = 0
@@ -41,20 +36,15 @@
def __repr__(self):
return "SkeletonGrid_%04i (%s)" % (self.id, self.ActiveDimensions)
-class SkeletonHierarchy(AMRHierarchy):
-
+class SkeletonHierarchy(GridIndex):
grid = SkeletonGrid
def __init__(self, ds, dataset_type='skeleton'):
self.dataset_type = dataset_type
- self.dataset = weakref.proxy(ds)
# for now, the index file is the dataset!
self.index_filename = self.dataset.parameter_filename
self.directory = os.path.dirname(self.index_filename)
- AMRHierarchy.__init__(self, ds, dataset_type)
-
- def _initialize_data_storage(self):
- pass
+ GridIndex.__init__(self, ds, dataset_type)
def _detect_output_fields(self):
# This needs to set a self.field_list that contains all the available,
diff -r 558d9f064464adf497d777b760b9e7ac12f9baea -r 3309b76428a71a81dbfd67e2184d2587807dd860 yt/frontends/_skeleton/io.py
--- a/yt/frontends/_skeleton/io.py
+++ b/yt/frontends/_skeleton/io.py
@@ -13,9 +13,6 @@
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
-import numpy as np
-import h5py
-
from yt.utilities.io_handler import \
BaseIOHandler
https://bitbucket.org/yt_analysis/yt/commits/5af9beae91f7/
Changeset: 5af9beae91f7
Branch: yt
User: ChrisMalone
Date: 2014-10-17 20:11:28+00:00
Summary: updates
Affected #: 1 file
diff -r 3309b76428a71a81dbfd67e2184d2587807dd860 -r 5af9beae91f7571294d52bde51a688cff42bf19c doc/source/developing/creating_frontend.rst
--- a/doc/source/developing/creating_frontend.rst
+++ b/doc/source/developing/creating_frontend.rst
@@ -172,6 +172,12 @@
dictionary whose keys are the fields and whose values are numpy arrays of
the data.
+If your dataset has particle information, you'll want to override the
+``_read_particle_coords()`` and ``read_particle_fields()`` methods as
+well. Each code is going to read data from disk in a different
+fashion, but the ``yt.frontends.boxlib.io.IOHandlerBoxlib`` is a
+decent place to start.
+
And that just about covers it. Please feel free to email
`yt-users <http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org>`_ or
`yt-dev <http://lists.spacepope.org/listinfo.cgi/yt-dev-spacepope.org>`_ with
https://bitbucket.org/yt_analysis/yt/commits/726ee92e6df5/
Changeset: 726ee92e6df5
Branch: yt
User: ChrisMalone
Date: 2014-10-20 15:34:49+00:00
Summary: merge with yt tip
Affected #: 2 files
diff -r 5af9beae91f7571294d52bde51a688cff42bf19c -r 726ee92e6df5248896ea25c4d6e5d8114dbae4c7 yt/frontends/chombo/data_structures.py
--- a/yt/frontends/chombo/data_structures.py
+++ b/yt/frontends/chombo/data_structures.py
@@ -113,7 +113,8 @@
self.directory = ds.fullpath
self._handle = ds._handle
- self.float_type = self._handle['Chombo_global'].attrs['testReal'].dtype.name
+ tr = self._handle['Chombo_global'].attrs.get("testReal", "float32")
+
self._levels = [key for key in self._handle.keys() if key.startswith('level')]
GridIndex.__init__(self, ds, dataset_type)
@@ -161,7 +162,13 @@
def _count_grids(self):
self.num_grids = 0
for lev in self._levels:
- self.num_grids += self._handle[lev]['Processors'].len()
+ d = self._handle[lev]
+ if 'Processors' in d:
+ self.num_grids += d['Processors'].len()
+ elif 'boxes' in d:
+ self.num_grids += d['boxes'].len()
+ else:
+ raise RuntimeError("Uknown file specification")
def _parse_index(self):
f = self._handle # shortcut
diff -r 5af9beae91f7571294d52bde51a688cff42bf19c -r 726ee92e6df5248896ea25c4d6e5d8114dbae4c7 yt/frontends/chombo/io.py
--- a/yt/frontends/chombo/io.py
+++ b/yt/frontends/chombo/io.py
@@ -25,6 +25,7 @@
_dataset_type = "chombo_hdf5"
_offset_string = 'data:offsets=0'
_data_string = 'data:datatype=0'
+ _offsets = None
def __init__(self, ds, *args, **kwargs):
BaseIOHandler.__init__(self, ds, *args, **kwargs)
@@ -32,6 +33,29 @@
self._handle = ds._handle
self.dim = self._handle['Chombo_global/'].attrs['SpaceDim']
self._read_ghost_info()
+ if self._offset_string not in self._handle['level_0']:
+ self._calculate_offsets()
+
+ def _calculate_offsets(self):
+ def box_size(corners):
+ size = 1
+ for idim in range(self.dim):
+ size *= (corners[idim+self.dim] - corners[idim] + 1)
+ return size
+
+ self._offsets = {}
+ num_comp = self._handle.attrs['num_components']
+ level = 0
+ while 1:
+ lname = 'level_%i' % level
+ if lname not in self._handle: break
+ boxes = self._handle['level_0']['boxes'].value
+ box_sizes = np.array([box_size(box) for box in boxes])
+
+ offsets = np.cumsum(box_sizes*num_comp, dtype='int64')
+ offsets -= offsets[0]
+ self._offsets[level] = offsets
+ level += 1
def _read_ghost_info(self):
try:
@@ -41,7 +65,7 @@
self.ghost = np.array(self.ghost)
except KeyError:
# assume zero ghosts if outputGhosts not present
- self.ghost = np.zeros(self.dim)
+ self.ghost = np.zeros(self.dim, 'int64')
_field_dict = None
@property
@@ -80,7 +104,10 @@
shape = grid.ActiveDimensions + 2*self.ghost
boxsize = shape.prod()
- grid_offset = lev[self._offset_string][grid._level_id]
+ if self._offsets is not None:
+ grid_offset = self._offsets[grid.Level][grid._level_id]
+ else:
+ grid_offset = lev[self._offset_string][grid._level_id]
start = grid_offset+self.field_dict[field]*boxsize
stop = start + boxsize
data = lev[self._data_string][start:stop]
https://bitbucket.org/yt_analysis/yt/commits/dc847107cce0/
Changeset: dc847107cce0
Branch: yt
User: ChrisMalone
Date: 2014-10-20 15:46:09+00:00
Summary: address Andrew's suggestions
Affected #: 1 file
diff -r 726ee92e6df5248896ea25c4d6e5d8114dbae4c7 -r dc847107cce0ece3114f254cc0a134b604648d89 doc/source/developing/creating_frontend.rst
--- a/doc/source/developing/creating_frontend.rst
+++ b/doc/source/developing/creating_frontend.rst
@@ -7,14 +7,14 @@
have a question about making a custom derived quantity, please
contact the mailing list.
-yt is designed to support analysis and visualization of data from multiple
-different simulation codes, although it has so far been most successfully
-applied to Adaptive Mesh Refinement (AMR) data. For a list of codes and the
-level of support they enjoy, see :ref:`code-support`.
+yt is designed to support analysis and visualization of data from
+multiple different simulation codes. For a list of codes and the level
+of support they enjoy, see :ref:`code-support`.
-We'd like to support a broad range of codes, both AMR-based and otherwise. To
-add support for a new code, a few things need to be put into place. These
-necessary structures can be classified into a couple categories:
+We'd like to support a broad range of codes, both Adaptive Mesh
+Refinement (AMR)-based and otherwise. To add support for a new code, a
+few things need to be put into place. These necessary structures can
+be classified into a couple categories:
* Data meaning: This is the set of parameters that convert the data into
physically relevant units; things like spatial and mass conversions, time
@@ -53,25 +53,50 @@
A new set of fields must be added in the file ``fields.py`` in your
new directory. For the most part this means subclassing
``FieldInfoContainer`` and adding the necessary fields specific to
-your code. Here is the base Chombo field container:
+your code. Here is a snippet from the base BoxLib field container:
.. code-block:: python
from yt.fields.field_info_container import FieldInfoContainer
- class ChomboFieldInfo(FieldInfoContainer):
- known_other_fields = ()
- known_particle_fields = ()
+ class BoxlibFieldInfo(FieldInfoContainer):
+ known_other_fields = (
+ ("density", (rho_units, ["density"], None)),
+ ("eden", (eden_units, ["energy_density"], None)),
+ ("xmom", (mom_units, ["momentum_x"], None)),
+ ("ymom", (mom_units, ["momentum_y"], None)),
+ ("zmom", (mom_units, ["momentum_z"], None)),
+ ("temperature", ("K", ["temperature"], None)),
+ ("Temp", ("K", ["temperature"], None)),
+ ("x_velocity", ("cm/s", ["velocity_x"], None)),
+ ("y_velocity", ("cm/s", ["velocity_y"], None)),
+ ("z_velocity", ("cm/s", ["velocity_z"], None)),
+ ("xvel", ("cm/s", ["velocity_x"], None)),
+ ("yvel", ("cm/s", ["velocity_y"], None)),
+ ("zvel", ("cm/s", ["velocity_z"], None)),
+ )
-This is a very stripped-down class. There are several codes
-(e.g. Orion2 and Pluto) that use a Chombo-based datastructure, and
-those particular codes subclass the ``ChomboFieldInfo`` class. The
-tuples, ``known_other_fields`` and ``known_particle_fields`` contain
-entries, which are tuples of the form ``("name", ("units", ["fields",
-"to", "alias"], "display_name"))``. ``"name"`` is the name of a field
-stored on-disk in the dataset. ``"units"`` corresponds to the units of
-that field. The list ``["fields", "to", "alias"]`` allows you to
-specify additional aliases to this particular field; for example, if
-your on-disk field for the x-direction velocity were
+ known_particle_fields = (
+ ("particle_mass", ("code_mass", [], None)),
+ ("particle_position_x", ("code_length", [], None)),
+ ("particle_position_y", ("code_length", [], None)),
+ ("particle_position_z", ("code_length", [], None)),
+ ("particle_momentum_x", (mom_units, [], None)),
+ ("particle_momentum_y", (mom_units, [], None)),
+ ("particle_momentum_z", (mom_units, [], None)),
+ ("particle_angmomen_x", ("code_length**2/code_time", [], None)),
+ ("particle_angmomen_y", ("code_length**2/code_time", [], None)),
+ ("particle_angmomen_z", ("code_length**2/code_time", [], None)),
+ ("particle_id", ("", ["particle_index"], None)),
+ ("particle_mdot", ("code_mass/code_time", [], None)),
+ )
+
+The tuples, ``known_other_fields`` and ``known_particle_fields``
+contain entries, which are tuples of the form ``("name", ("units",
+["fields", "to", "alias"], "display_name"))``. ``"name"`` is the name
+of a field stored on-disk in the dataset. ``"units"`` corresponds to
+the units of that field. The list ``["fields", "to", "alias"]``
+allows you to specify additional aliases to this particular field; for
+example, if your on-disk field for the x-direction velocity were
``"x-direction-velocity"``, maybe you'd prefer to alias to the more
terse name of ``"xvel"``. ``"display_name"`` is an optional parameter
that can be used to specify how you want the field to be displayed on
@@ -142,7 +167,7 @@
self.Level = level
-Even one of the more complex grid object,
+Even one of the more complex grid objects,
``yt.frontends.boxlib.BoxlibGrid``, is still relatively simple.
Data Reading Functions
https://bitbucket.org/yt_analysis/yt/commits/d169ebcd429a/
Changeset: d169ebcd429a
Branch: yt
User: ChrisMalone
Date: 2014-10-20 16:00:58+00:00
Summary: attempt to address 'octree' issue
Affected #: 1 file
diff -r dc847107cce0ece3114f254cc0a134b604648d89 -r d169ebcd429ae622b3e88bbbf49f7175493f0a9c doc/source/developing/creating_frontend.rst
--- a/doc/source/developing/creating_frontend.rst
+++ b/doc/source/developing/creating_frontend.rst
@@ -109,26 +109,33 @@
These functions and classes let yt know about how the arrangement of
data on disk corresponds to the physical arrangement of data within
-the simulation. There are some subtle differences between
-AMR/patch-based codes and Octree-based codes, however, both approaches
-have a concept of a *Hierarchy* or *Index* (used somewhat
-interchangeably in the code) of datastructures and something that
-describes the elements that make up the Hierarchy or Index. For
-AMR-based codes, the Index is a collection of ``AMRGridPatch`` objects
-that describe a block of zones. For Octree-based codes, the Index
-contains datastructures that hold information about the individual
-octs, namely an ``OctreeContainer``.
+the simulation. yt has grid datastructures for handling both
+patch-based and octree-based AMR codes. The terms 'patch-based'
+and 'octree-based' are used somewhat loosely here. For example,
+traditionally, the FLASH code used the paramesh AMR library, which is
+based on a tree structure, but the FLASH frontend in yt utilizes yt's
+patch-based datastructures. It is up to the frontend developer to
+determine which yt datastructures best match the datastructures of
+their simulation code.
+
+Both approaches -- patch-based and octree-based -- have a concept of a
+*Hierarchy* or *Index* (used somewhat interchangeably in the code) of
+datastructures and something that describes the elements that make up
+the Hierarchy or Index. For patch-based codes, the Index is a
+collection of ``AMRGridPatch`` objects that describe a block of zones.
+For octree-based codes, the Index contains datastructures that hold
+information about the individual octs, namely an ``OctreeContainer``.
Hierarchy or Index
^^^^^^^^^^^^^^^^^^
-To set up data localization, a ``GridIndex`` subclass for AMR-based
-codes or an ``OctreeIndex`` subclass for Octree-based codes must be
+To set up data localization, a ``GridIndex`` subclass for patch-based
+codes or an ``OctreeIndex`` subclass for octree-based codes must be
added in the file ``data_structures.py``. Examples of these different
types of ``Index`` can be found in, for example, the
-``yt.frontends.chombo.data_structures.ChomboHierarchy`` for AMR-based
+``yt.frontends.chombo.data_structures.ChomboHierarchy`` for patch-based
codes and ``yt.frontends.ramses.data_structures.RAMSESIndex`` for
-Octree-based codes.
+octree-based codes.
For the most part, the ``GridIndex`` subclass must override (at a
minimum) the following methods:
https://bitbucket.org/yt_analysis/yt/commits/d11d8f936079/
Changeset: d11d8f936079
Branch: yt
User: ChrisMalone
Date: 2014-10-20 17:21:48+00:00
Summary: address Nathan's comment and add more about octrees
Affected #: 1 file
diff -r d169ebcd429ae622b3e88bbbf49f7175493f0a9c -r d11d8f9360792b946ec48082b0e0ffd76a307c24 doc/source/developing/creating_frontend.rst
--- a/doc/source/developing/creating_frontend.rst
+++ b/doc/source/developing/creating_frontend.rst
@@ -154,9 +154,24 @@
``_prepare_grid()`` and ``_setup_dx()`` on all of them. Additionally, it
should set up ``Children`` and ``Parent`` lists on each grid object.
+The ``OctreeIndex`` has somewhat analogous methods, but often with
+different names; both ``OctreeIndex`` and ``GridIndex`` are subclasses
+of the ``Index`` class. In particular, for the ``OctreeIndex``, the
+method ``_initialize_oct_handler()`` setups up much of the oct
+metadata that is analogous to the grid metadata created in the
+``GridIndex`` methods ``_count_grids()``, ``_parse_index()``, and
+``_populate_grid_objects()``.
+
Grids
^^^^^
+.. note:: This section only applies to the approach using yt's patch-based
+ datastructures. For the octree-based approach, one does not create
+ a grid object, but rather an ``OctreeSubset``, which has methods
+ for filling out portions of the octree structure. Again, see the
+ code in ``yt.frontends.ramses.data_structures`` for an example of
+ the octree approach.
+
A new grid object, subclassing ``AMRGridPatch``, will also have to be added in
``data_structures.py``. For the most part, this may be all
that is needed:
@@ -180,9 +195,14 @@
Data Reading Functions
----------------------
-In ``io.py``, there are a number of IO handlers that handle the mechanisms by
-which data is read off disk. To implement a new data reader, you must subclass
-``BaseIOHandler``. The various frontend IO handlers are stored in an IO registry - essentially a dictionary that uses the name of the frontend as a key, and the specific IO handler as a value. It is important, therefore, to set the ``dataset_type`` attribute of your subclass, which is what is used as the key in the IO registry. For example:
+In ``io.py``, there are a number of IO handlers that handle the
+mechanisms by which data is read off disk. To implement a new data
+reader, you must subclass ``BaseIOHandler``. The various frontend IO
+handlers are stored in an IO registry - essentially a dictionary that
+uses the name of the frontend as a key, and the specific IO handler as
+a value. It is important, therefore, to set the ``dataset_type``
+attribute of your subclass, which is what is used as the key in the IO
+registry. For example:
.. code-block:: python
https://bitbucket.org/yt_analysis/yt/commits/6f5a2e3f405f/
Changeset: 6f5a2e3f405f
Branch: yt
User: MatthewTurk
Date: 2014-10-21 11:41:38+00:00
Summary: Merged in ChrisMalone/yt (pull request #1270)
On updating the frontend docs
Affected #: 4 files
diff -r de7802382dc0cc71b3e1eb15b37bbdcfec5b218a -r 6f5a2e3f405f9ec96211c59085cc6101891ad655 doc/source/developing/creating_frontend.rst
--- a/doc/source/developing/creating_frontend.rst
+++ b/doc/source/developing/creating_frontend.rst
@@ -7,14 +7,14 @@
have a question about making a custom derived quantity, please
contact the mailing list.
-yt is designed to support analysis and visualization of data from multiple
-different simulation codes, although it has so far been most successfully
-applied to Adaptive Mesh Refinement (AMR) data. For a list of codes and the
-level of support they enjoy, see :ref:`code-support`.
+yt is designed to support analysis and visualization of data from
+multiple different simulation codes. For a list of codes and the level
+of support they enjoy, see :ref:`code-support`.
-We'd like to support a broad range of codes, both AMR-based and otherwise. To
-add support for a new code, a few things need to be put into place. These
-necessary structures can be classified into a couple categories:
+We'd like to support a broad range of codes, both Adaptive Mesh
+Refinement (AMR)-based and otherwise. To add support for a new code, a
+few things need to be put into place. These necessary structures can
+be classified into a couple categories:
* Data meaning: This is the set of parameters that convert the data into
physically relevant units; things like spatial and mass conversions, time
@@ -33,73 +33,147 @@
If you are interested in adding a new code, be sure to drop us a line on
`yt-dev <http://lists.spacepope.org/listinfo.cgi/yt-dev-spacepope.org>`_!
-To get started, make a new directory in ``yt/frontends`` with the name of your
-code -- you can start by copying into it the contents of the ``stream``
-directory, which is a pretty empty format. You'll then have to create a subclass
-of ``Dataset``. This subclass will need to handle conversion between the
-different physical units and the code units; for the most part, the examples of
-``OrionDataset`` and ``EnzoDataset`` should be followed, but
-``ChomboDataset``, as a slightly newer addition, can also be used as an
-instructive example -- be sure to add an ``_is_valid`` classmethod that will
-verify if a filename is valid for that output type, as that is how "load" works.
+To get started, make a new directory in ``yt/frontends`` with the name
+of your code. Copying the contents of the ``yt/frontends/_skeleton``
+directory will add a lot of boilerplate for the required classes and
+methods that are needed. In particular, you'll have to create a
+subclass of ``Dataset`` in the data_structures.py file. This subclass
+will need to handle conversion between the different physical units
+and the code units (typically in the ``_set_code_unit_attributes()``
+method), read in metadata describing the overall data on disk (via the
+``_parse_parameter_file()`` method), and provide a ``classmethod``
+called ``_is_valid()`` that lets the ``yt.load`` method help identify an
+input file as belonging to *this* particular ``Dataset`` subclass.
+For the most part, the examples of
+``yt.frontends.boxlib.data_structures.OrionDataset`` and
+``yt.frontends.enzo.data_structures.EnzoDataset`` should be followed,
+but ``yt.frontends.chombo.data_structures.ChomboDataset``, as a
+slightly newer addition, can also be used as an instructive example.
-A new set of fields must be added in the file ``fields.py`` in that directory.
-For the most part this means subclassing ``CodeFieldInfoContainer`` and adding
-the necessary fields specific to that code. Here is the Chombo field container:
+A new set of fields must be added in the file ``fields.py`` in your
+new directory. For the most part this means subclassing
+``FieldInfoContainer`` and adding the necessary fields specific to
+your code. Here is a snippet from the base BoxLib field container:
.. code-block:: python
- from UniversalFields import *
- class ChomboFieldContainer(CodeFieldInfoContainer):
- _shared_state = {}
- _field_list = {}
- ChomboFieldInfo = ChomboFieldContainer()
- add_chombo_field = ChomboFieldInfo.add_field
+ from yt.fields.field_info_container import FieldInfoContainer
+ class BoxlibFieldInfo(FieldInfoContainer):
+ known_other_fields = (
+ ("density", (rho_units, ["density"], None)),
+ ("eden", (eden_units, ["energy_density"], None)),
+ ("xmom", (mom_units, ["momentum_x"], None)),
+ ("ymom", (mom_units, ["momentum_y"], None)),
+ ("zmom", (mom_units, ["momentum_z"], None)),
+ ("temperature", ("K", ["temperature"], None)),
+ ("Temp", ("K", ["temperature"], None)),
+ ("x_velocity", ("cm/s", ["velocity_x"], None)),
+ ("y_velocity", ("cm/s", ["velocity_y"], None)),
+ ("z_velocity", ("cm/s", ["velocity_z"], None)),
+ ("xvel", ("cm/s", ["velocity_x"], None)),
+ ("yvel", ("cm/s", ["velocity_y"], None)),
+ ("zvel", ("cm/s", ["velocity_z"], None)),
+ )
-The field container is a shared state object, which is why we explicitly set
-``_shared_state`` equal to a mutable.
+ known_particle_fields = (
+ ("particle_mass", ("code_mass", [], None)),
+ ("particle_position_x", ("code_length", [], None)),
+ ("particle_position_y", ("code_length", [], None)),
+ ("particle_position_z", ("code_length", [], None)),
+ ("particle_momentum_x", (mom_units, [], None)),
+ ("particle_momentum_y", (mom_units, [], None)),
+ ("particle_momentum_z", (mom_units, [], None)),
+ ("particle_angmomen_x", ("code_length**2/code_time", [], None)),
+ ("particle_angmomen_y", ("code_length**2/code_time", [], None)),
+ ("particle_angmomen_z", ("code_length**2/code_time", [], None)),
+ ("particle_id", ("", ["particle_index"], None)),
+ ("particle_mdot", ("code_mass/code_time", [], None)),
+ )
+
+The tuples, ``known_other_fields`` and ``known_particle_fields``
+contain entries, which are tuples of the form ``("name", ("units",
+["fields", "to", "alias"], "display_name"))``. ``"name"`` is the name
+of a field stored on-disk in the dataset. ``"units"`` corresponds to
+the units of that field. The list ``["fields", "to", "alias"]``
+allows you to specify additional aliases to this particular field; for
+example, if your on-disk field for the x-direction velocity were
+``"x-direction-velocity"``, maybe you'd prefer to alias to the more
+terse name of ``"xvel"``. ``"display_name"`` is an optional parameter
+that can be used to specify how you want the field to be displayed on
+a plot; this can be LaTeX code, for example the density field could
+have a display name of ``r"\rho"``. Omitting the ``"display_name"``
+will result in using a capitalized version of the ``"name"``.
Data Localization Structures
----------------------------
-As of right now, the "grid patch" mechanism is going to remain in yt, however in
-the future that may change. As such, some other output formats -- like Gadget --
-may be shoe-horned in, slightly.
+These functions and classes let yt know about how the arrangement of
+data on disk corresponds to the physical arrangement of data within
+the simulation. yt has grid datastructures for handling both
+patch-based and octree-based AMR codes. The terms 'patch-based'
+and 'octree-based' are used somewhat loosely here. For example,
+traditionally, the FLASH code used the paramesh AMR library, which is
+based on a tree structure, but the FLASH frontend in yt utilizes yt's
+patch-based datastructures. It is up to the frontend developer to
+determine which yt datastructures best match the datastructures of
+their simulation code.
-Hierarchy
-^^^^^^^^^
+Both approaches -- patch-based and octree-based -- have a concept of a
+*Hierarchy* or *Index* (used somewhat interchangeably in the code) of
+datastructures and something that describes the elements that make up
+the Hierarchy or Index. For patch-based codes, the Index is a
+collection of ``AMRGridPatch`` objects that describe a block of zones.
+For octree-based codes, the Index contains datastructures that hold
+information about the individual octs, namely an ``OctreeContainer``.
-To set up data localization, an ``AMRHierarchy`` subclass must be added in the
-file ``data_structures.py``. The index object must override the following
-methods:
+Hierarchy or Index
+^^^^^^^^^^^^^^^^^^
- * ``_detect_fields``: ``self.field_list`` must be populated as a list of
- strings corresponding to "native" fields in the data files.
- * ``_setup_classes``: it's probably safe to crib this from one of the other
- ``AMRHierarchy`` subclasses.
- * ``_count_grids``: this must set self.num_grids to be the total number of
- grids in the simulation.
- * ``_parse_index``: this must fill in ``grid_left_edge``,
+To set up data localization, a ``GridIndex`` subclass for patch-based
+codes or an ``OctreeIndex`` subclass for octree-based codes must be
+added in the file ``data_structures.py``. Examples of these different
+types of ``Index`` can be found in, for example, the
+``yt.frontends.chombo.data_structures.ChomboHierarchy`` for patch-based
+codes and ``yt.frontends.ramses.data_structures.RAMSESIndex`` for
+octree-based codes.
+
+For the most part, the ``GridIndex`` subclass must override (at a
+minimum) the following methods:
+
+ * ``_detect_output_fields()``: ``self.field_list`` must be populated as a list
+ of strings corresponding to "native" fields in the data files.
+ * ``_count_grids()``: this must set ``self.num_grids`` to be the total number
+ of grids (equivalently ``AMRGridPatch``'es) in the simulation.
+ * ``_parse_index()``: this must fill in ``grid_left_edge``,
``grid_right_edge``, ``grid_particle_count``, ``grid_dimensions`` and
- ``grid_levels`` with the appropriate information. Additionally, ``grids``
- must be an array of grid objects that already know their IDs.
- * ``_populate_grid_objects``: this initializes the grids by calling
- ``_prepare_grid`` and ``_setup_dx`` on all of them. Additionally, it should
- set up ``Children`` and ``Parent`` lists on each grid object.
- * ``_setup_unknown_fields``: If a field is in the data file that yt doesn't
- already know, this is where you make a guess at it.
- * ``_setup_derived_fields``: ``self.derived_field_list`` needs to be made a
- list of strings that correspond to all derived fields valid for this
- index.
+ ``grid_levels`` with the appropriate information. Each of these variables
+ is an array, with an entry for each of the ``self.num_grids`` grids.
+ Additionally, ``grids`` must be an array of ``AMRGridPatch`` objects that
+ already know their IDs.
+ * ``_populate_grid_objects()``: this initializes the grids by calling
+ ``_prepare_grid()`` and ``_setup_dx()`` on all of them. Additionally, it
+ should set up ``Children`` and ``Parent`` lists on each grid object.
-For the most part, the ``ChomboHierarchy`` should be the first place to look for
-hints on how to do this; ``EnzoHierarchy`` is also instructive.
+The ``OctreeIndex`` has somewhat analogous methods, but often with
+different names; both ``OctreeIndex`` and ``GridIndex`` are subclasses
+of the ``Index`` class. In particular, for the ``OctreeIndex``, the
+method ``_initialize_oct_handler()`` setups up much of the oct
+metadata that is analogous to the grid metadata created in the
+``GridIndex`` methods ``_count_grids()``, ``_parse_index()``, and
+``_populate_grid_objects()``.
Grids
^^^^^
-A new grid object, subclassing ``AMRGridPatch``, will also have to be added.
-This should go in ``data_structures.py``. For the most part, this may be all
+.. note:: This section only applies to the approach using yt's patch-based
+ datastructures. For the octree-based approach, one does not create
+ a grid object, but rather an ``OctreeSubset``, which has methods
+ for filling out portions of the octree structure. Again, see the
+ code in ``yt.frontends.ramses.data_structures`` for an example of
+ the octree approach.
+
+A new grid object, subclassing ``AMRGridPatch``, will also have to be added in
+``data_structures.py``. For the most part, this may be all
that is needed:
.. code-block:: python
@@ -115,32 +189,46 @@
self.Level = level
-Even the most complex grid object, ``OrionGrid``, is still relatively simple.
+Even one of the more complex grid objects,
+``yt.frontends.boxlib.BoxlibGrid``, is still relatively simple.
Data Reading Functions
----------------------
-In ``io.py``, there are a number of IO handlers that handle the mechanisms by
-which data is read off disk. To implement a new data reader, you must subclass
-``BaseIOHandler`` and override the following methods:
+In ``io.py``, there are a number of IO handlers that handle the
+mechanisms by which data is read off disk. To implement a new data
+reader, you must subclass ``BaseIOHandler``. The various frontend IO
+handlers are stored in an IO registry - essentially a dictionary that
+uses the name of the frontend as a key, and the specific IO handler as
+a value. It is important, therefore, to set the ``dataset_type``
+attribute of your subclass, which is what is used as the key in the IO
+registry. For example:
- * ``_read_field_names``: this routine accepts a grid object and must return all
- the fields in the data file affiliated with that grid. It is used at the
- initialization of the ``AMRHierarchy`` but likely not later.
- * ``modify``: This accepts a field from a data file and returns it ready to be
- used by yt. This is used in Enzo data for preloading.
- * ``_read_data_set``: This accepts a grid object and a field name and must
- return that field, ready to be used by yt as a NumPy array. Note that this
- presupposes that any actions done in ``modify`` (above) have been executed.
- * ``_read_data_slice``: This accepts a grid object, a field name, an axis and
- an (integer) coordinate, and it must return a slice through the array at that
- value.
- * ``preload``: (optional) This accepts a list of grids and a list of datasets
- and it populates ``self.queue`` (a dict keyed by grid id) with dicts of
- datasets.
- * ``_read_exception``: (property) This is a tuple of exceptions that can be
- raised by the data reading to indicate a field does not exist in the file.
+.. code-block:: python
+ class IOHandlerBoxlib(BaseIOHandler):
+ _dataset_type = "boxlib_native"
+ ...
+
+At a minimum, one should also override the following methods
+
+* ``_read_fluid_selection()``: this receives a collection of data "chunks", a
+ selector describing which "chunks" you are concerned with, a list of fields,
+ and the size of the data to read. It should create and return a dictionary
+ whose keys are the fields, and whose values are numpy arrays containing the
+ data. The data should actually be read via the ``_read_chunk_data()``
+ method.
+* ``_read_chunk_data()``: this method receives a "chunk" of data along with a
+ list of fields we want to read. It loops over all the grid objects within
+ the "chunk" of data and reads from disk the specific fields, returning a
+ dictionary whose keys are the fields and whose values are numpy arrays of
+ the data.
+
+If your dataset has particle information, you'll want to override the
+``_read_particle_coords()`` and ``read_particle_fields()`` methods as
+well. Each code is going to read data from disk in a different
+fashion, but the ``yt.frontends.boxlib.io.IOHandlerBoxlib`` is a
+decent place to start.
And that just about covers it. Please feel free to email
`yt-users <http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org>`_ or
diff -r de7802382dc0cc71b3e1eb15b37bbdcfec5b218a -r 6f5a2e3f405f9ec96211c59085cc6101891ad655 yt/frontends/_skeleton/api.py
--- a/yt/frontends/_skeleton/api.py
+++ b/yt/frontends/_skeleton/api.py
@@ -19,8 +19,7 @@
SkeletonDataset
from .fields import \
- SkeletonFieldInfo, \
- add_skeleton_field
+ SkeletonFieldInfo
from .io import \
IOHandlerSkeleton
diff -r de7802382dc0cc71b3e1eb15b37bbdcfec5b218a -r 6f5a2e3f405f9ec96211c59085cc6101891ad655 yt/frontends/_skeleton/data_structures.py
--- a/yt/frontends/_skeleton/data_structures.py
+++ b/yt/frontends/_skeleton/data_structures.py
@@ -13,18 +13,13 @@
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
-import numpy as np
-
-from yt.data_objects.grid_patch import \
- AMRGridPatch
from yt.data_objects.grid_patch import \
AMRGridPatch
from yt.geometry.grid_geometry_handler import \
GridIndex
from yt.data_objects.static_output import \
Dataset
-from yt.utilities.lib.misc_utilities import \
- get_box_grids_level
+from .fields import SkeletonFieldInfo
class SkeletonGrid(AMRGridPatch):
_id_offset = 0
@@ -41,20 +36,15 @@
def __repr__(self):
return "SkeletonGrid_%04i (%s)" % (self.id, self.ActiveDimensions)
-class SkeletonHierarchy(AMRHierarchy):
-
+class SkeletonHierarchy(GridIndex):
grid = SkeletonGrid
def __init__(self, ds, dataset_type='skeleton'):
self.dataset_type = dataset_type
- self.dataset = weakref.proxy(ds)
# for now, the index file is the dataset!
self.index_filename = self.dataset.parameter_filename
self.directory = os.path.dirname(self.index_filename)
- AMRHierarchy.__init__(self, ds, dataset_type)
-
- def _initialize_data_storage(self):
- pass
+ GridIndex.__init__(self, ds, dataset_type)
def _detect_output_fields(self):
# This needs to set a self.field_list that contains all the available,
diff -r de7802382dc0cc71b3e1eb15b37bbdcfec5b218a -r 6f5a2e3f405f9ec96211c59085cc6101891ad655 yt/frontends/_skeleton/io.py
--- a/yt/frontends/_skeleton/io.py
+++ b/yt/frontends/_skeleton/io.py
@@ -13,9 +13,6 @@
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
-import numpy as np
-import h5py
-
from yt.utilities.io_handler import \
BaseIOHandler
Repository URL: https://bitbucket.org/yt_analysis/yt/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the yt-svn
mailing list