[yt-svn] commit/yt: 76 new changesets
commits-noreply at bitbucket.org
commits-noreply at bitbucket.org
Fri Sep 23 10:56:26 PDT 2016
76 new commits in yt:
https://bitbucket.org/yt_analysis/yt/commits/b186ca295595/
Changeset: b186ca295595
Branch: yt
User: NTAu... at honeypot.fritz.box
Date: 2016-05-22 12:39:16+00:00
Summary: Import existing openPMD code into yt
Affected #: 8 files
diff -r c07d12bf5903b40c0c6e65a8ea8f28a82f142e1d -r b186ca2955959cb04b0a37602855c28b3d89189f yt/frontends/openPMD/__init__.py
--- /dev/null
+++ b/yt/frontends/openPMD/__init__.py
@@ -0,0 +1,15 @@
+"""
+API for yt.frontends.skeleton
+
+
+
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2013, yt Development Team.
+# Copyright (c) 2015, Daniel Grassinger (HZDR)
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
diff -r c07d12bf5903b40c0c6e65a8ea8f28a82f142e1d -r b186ca2955959cb04b0a37602855c28b3d89189f yt/frontends/openPMD/api.py
--- /dev/null
+++ b/yt/frontends/openPMD/api.py
@@ -0,0 +1,25 @@
+"""
+API for yt.frontends._skeleton
+
+
+
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2013, yt Development Team.
+# Copyright (c) 2015, Daniel Grassinger (HZDR)
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+from .data_structures import \
+ openPMDDataset, \
+ openPMDGrid, \
+ openPMDHierarchy
+
+from .fields import \
+ openPMDFieldInfo
+
+from .io import \
+ IOHandlerOpenPMD
diff -r c07d12bf5903b40c0c6e65a8ea8f28a82f142e1d -r b186ca2955959cb04b0a37602855c28b3d89189f yt/frontends/openPMD/data_structures.py
--- /dev/null
+++ b/yt/frontends/openPMD/data_structures.py
@@ -0,0 +1,401 @@
+"""
+openPMD data structures
+
+
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2013, yt Development Team.
+# Copyright (c) 2015, Daniel Grassinger (HZDR)
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+from yt.data_objects.grid_patch import \
+ AMRGridPatch
+from yt.geometry.grid_geometry_handler import \
+ GridIndex
+from yt.data_objects.static_output import \
+ Dataset
+from .fields import openPMDFieldInfo
+
+from yt.utilities.file_handler import \
+ HDF5FileHandler
+
+import yt.frontends.openPMD.misc as validator
+
+import h5py
+import numpy as np
+import os
+from yt.utilities.logger import ytLogger as mylog
+
+import sys
+
+class openPMDBasePathException(Exception) :
+ pass
+
+class openPMDBasePath :
+ def _setBasePath(self, handle) :
+ """
+ Set the base path for the first iteration found in the file.
+ TODO implement into distinct methods:
+ - __init__(self, handle)
+ - getIterations(self)
+ - getBasePath(self, iteration)
+ """
+ # basePath is fixed in openPMD 1.X to `/data/%T/`
+ dataPath = u"/data"
+
+ # if the file messed up the base path we avoid throwing a cluttered
+ # exception below while looking for iterations:
+ if handle.attrs["basePath"].decode("utf-8") != u"/data/%T/" :
+ raise openPMDBasePathException("openPMD: basePath is non-standard!")
+
+ # does `/data/` exist?
+ if not u"/data" in handle :
+ raise openPMDBasePathException("openPMD: group for basePath does not exist!")
+
+ # find iterations in basePath
+ list_iterations = []
+ for i in list(handle[dataPath].keys()) :
+ list_iterations.append(i)
+ mylog.warning("openPMD: found {} iterations in file".format(len(list_iterations)))
+
+ # We found no iterations in basePath
+ # TODO in the future (see above) this can be a mylog.warning instead of an error
+ if len(list_iterations) == 0 :
+ raise openPMDBasePathException("openPMD: no iterations found in basePath!")
+
+ # just handle the first iteration found
+ mylog.warning("openPMD: only choose to load first iteration in file")
+ self.basePath = "{}/{}/".format(dataPath, list_iterations[0])
+
+
+
+
+class openPMDGrid(AMRGridPatch):
+ """
+ This class defines the characteristics of the grids
+ Actually there is only one grid for the whole simolation box
+ """
+ _id_offset = 0
+ __slots__ = ["_level_id"]
+
+ def __init__(self, id, index, level=-1):
+ AMRGridPatch.__init__(self, id, filename=index.index_filename,
+ index=index)
+ # There is only one grid. So there are no parent or child grids
+ self.Parent = None
+ self.Children = []
+ self.Level = level
+
+ def __repr__(self):
+ return "openPMDGrid_%04i (%s)" % (self.id, self.ActiveDimensions)
+
+
+class openPMDHierarchy(GridIndex, openPMDBasePath):
+ """
+ Defines which fields and particles are created and read from the hard disk
+ Furthermore it defines the characteristics of the grids
+ """
+ grid = openPMDGrid
+
+ def __init__(self, ds, dataset_type='openPMD'):
+ self.dataset_type = dataset_type
+ # for now, the index file is the dataset!
+ self.dataset = ds
+ self.index_filename = ds.parameter_filename
+ self.directory = os.path.dirname(self.index_filename)
+ self._setBasePath(self.dataset._handle)
+ GridIndex.__init__(self, ds, dataset_type)
+
+ def _detect_output_fields(self):
+ """
+ Parses the dataset to define field names for yt.
+
+ NOTE: Each should be a tuple, where the first element is the on-disk
+ fluid type or particle type. Convention suggests that the on-disk
+ fluid type is usually the dataset_type and the on-disk particle type
+ (for a single population of particles) is "io".
+ look for fluid fields
+
+ From yt doc:
+ self.field_list must be populated as a list of strings corresponding to "native" fields in the data files.
+
+ Parameters
+ ----------
+ self:
+ A reference to self
+ """
+ # TODO This only parses one file
+ f = self.dataset._handle
+ meshesPath = f.attrs["meshesPath"]
+ particlesPath = f.attrs["particlesPath"]
+ output_fields = []
+
+ for group in f[self.basePath + meshesPath].keys():
+ try:
+ for direction in f[self.basePath + meshesPath + group].keys():
+ output_fields.append(group + "_" + direction)
+ except:
+ # This is for dataSets, they do not have keys
+ output_fields.append(group.replace("_","-"))
+ self.field_list = [("openPMD", str(c)) for c in output_fields]
+
+ def is_const_component(record_component):
+ return ("value" in record_component.attrs.keys())
+
+ # TODO Pay attention to scalars
+ particle_fields = []
+ # WHY would this equal false?
+ if self.basePath + particlesPath in f:
+ for particleName in f[self.basePath + particlesPath].keys():
+ for record in f[self.basePath + particlesPath + particleName].keys():
+ if is_const_component(f[self.basePath + particlesPath + particleName + "/" + record]):
+ # Record itself (eg particle_mass) is constant
+ mylog.info("%s", record)
+ particle_fields.append(particleName + "_" + record)
+ elif 'particlePatches' not in record:
+ try:
+ keys = f[self.basePath + particlesPath + particleName + "/" + record].keys()
+ for axis in keys:
+ mylog.info("%s_%s", record, axis)
+ particle_fields.append(particleName + "_" + record + "_" + axis)
+ pass
+ except:
+ mylog.info("%s", record)
+ particle_fields.append(particleName + "_" + record)
+ pass
+ else:
+ # We probably do not want particlePatches as accessible field lists
+ pass
+ self.field_list.extend([(str(c).split("_")[0], 'particle' + c.lstrip(c.split("_")[0])) for c in particle_fields])
+
+ def _count_grids(self):
+ """
+ Counts the number of grids in the dataSet. (Only one in the current standard)
+
+ From yt doc:
+ this must set self.num_grids to be the total number of grids (equiv AMRGridPatch'es) in the simulation
+
+ Parameters
+ ----------
+ self:
+ A reference to self
+ """
+ self.num_grids = 1
+
+ def _parse_index(self):
+ """
+ Parses dimensions from self._handle into self.
+
+ From yt doc:
+ this must fill in
+ grid_left_edge,
+ grid_right_edge,
+ grid_particle_count,
+ grid_dimensions and
+ grid_levels
+ with the appropriate information.
+ Each of these variables is an array, with an entry for each of the self.num_grids grids.
+ Additionally, grids must be an array of AMRGridPatch objects that already know their IDs.
+
+ Parameters
+ ----------
+ self:
+ A reference to self
+ """
+ # This needs to fill the following arrays, where N is self.num_grids:
+ meshesPath = self.dataset._handle.attrs["meshesPath"]
+ particlesPath = self.dataset._handle.attrs["particlesPath"]
+
+ dims = self.dataset.domain_left_edge.shape[0]#self.dimensionality
+
+ # These objects are expecting 3 values so we pad with zeros in 1D adims 2D
+ self.grid_left_edge[
+ 0,:dims] = self.dataset.domain_left_edge.copy() # (N, 3) <= float64
+ self.grid_right_edge[
+ 0,:dims] = self.dataset.domain_right_edge.copy() # (N, 3) <= float64
+ self.grid_dimensions[
+ 0][:dims] = self.dataset.domain_dimensions[:dims] # (N, 3) <= int
+
+# TODO this disables particle reads for now
+# Should might be set in _read_particles for each species,
+# also each species might need its own grid (?)
+# self.grid_particle_count[0] = self.dataset._handle[
+# self.basePath + particlesPath + "/electrons/position/x"].shape[0] # (N, 1) <= int
+ # self.grid_levels = 1 #(N, 1) <= int
+ # self.grids = np.empty(1, dtype='object') #(N, 1) <= grid objects
+
+ self.grid_levels.flat[:] = 0
+ self.grids = np.empty(self.num_grids, dtype='object')
+ # You have to inalize the grids
+ for i in range(self.num_grids):
+ self.grids[i] = self.grid(i, self, self.grid_levels[i, 0])
+
+ def _populate_grid_objects(self):
+ """
+ This function initializes the grids
+
+ From yt doc:
+ this initializes the grids by calling _prepare_grid() and _setup_dx() on all of them.
+ Additionally, it should set up Children and Parent lists on each grid object.
+
+ Parameters
+ ----------
+ self:
+ A reference to self
+ """
+
+ # self._reconstruct_parent_child()
+
+ for i in range(self.num_grids):
+ self.grids[i]._prepare_grid()
+ self.grids[i]._setup_dx()
+ self.max_level = 0
+
+
+class openPMDDataset(Dataset, openPMDBasePath):
+ """
+ A dataset object contains all the information of the simulation and
+ is intialized with yt.load()
+
+ TODO Ideally, a data set object should only contain a single data set.
+ afaik, yt.load() can load multiple data sets and also supports
+ multiple iteration-loading if done that way, e.g., from a prefix
+ of files.
+ """
+ _index_class = openPMDHierarchy
+ _field_info_class = openPMDFieldInfo
+
+ def __init__(self, filename, dataset_type='openPMD',
+ storage_filename=None,
+ units_override=None):
+ # Opens a HDF5 file and stores its file handle in _handle
+ # All _handle objects refers to the file
+ self._handle = HDF5FileHandler(filename)
+ self._setBasePath(self._handle)
+ Dataset.__init__(self, filename, dataset_type,
+ units_override=units_override)
+ self.storage_filename = storage_filename
+ self.fluid_types += ('openPMD',)
+ self.particle_types = self._handle[self.basePath + self._handle.attrs["particlesPath"]].keys()
+ self.particle_types = tuple(self.particle_types)
+ self.particle_types += ('all',)
+ self.particle_types_raw = self.particle_types
+
+
+ def _set_code_unit_attributes(self):
+ """
+ From yt doc:
+ handle conversion between the different physical units and the code units
+
+ Parameters
+ ----------
+ self:
+ A reference to self
+ """
+ # This is where quantities are created that represent the various
+ # on-disk units. These are the currently available quantities which
+ # should be set, along with examples of how to set them to standard
+ # values.
+ #
+ self.length_unit = self.quan(1.0, "m")
+ self.mass_unit = self.quan(1.0, "kg")
+ self.time_unit = self.quan(1.0, "s")
+ self.velocity_unit = self.quan(1.0, "m/s")
+ self.magnetic_unit = self.quan(1.0, "T")
+
+ def _parse_parameter_file(self):
+ """
+ From yt doc:
+ read in metadata describing the overall data on disk
+
+ Parameters
+ ----------
+ self:
+ A reference to self
+ """
+ # This needs to set up the following items. Note that these are all
+ # assumed to be in code units; domain_left_edge and domain_right_edge
+ # will be updated to be in code units at a later time. This includes
+ # the cosmological parameters.
+
+ f = self._handle
+ meshesPath = f.attrs["meshesPath"].decode()
+ particlesPath = f.attrs["particlesPath"].decode()
+ positionPath = self.basePath + particlesPath + "/electrons/position/"
+
+ # This defines the size of the simulaion box
+ self.unique_identifier = 0 # no identifier
+ self.parameters = 0 # no additional parameters <= full of code-specific items of use
+
+ # TODO At this point one assumes the whole file/simulation
+ # contains for all mesh records the same dimensionality and shapes
+ # TODO Support particle-only files
+ # pick first field
+ try :
+ firstIteration = list(f["/data/"].keys())[0]
+ meshes = f["/data/" + str(firstIteration) + "/" + meshesPath]
+ firstMeshName = list(meshes.keys())[0]
+ firstMesh = meshes[firstMeshName]
+ if type(firstMesh) == h5py.Dataset :
+ fshape = firstMesh.shape
+ else :
+ fshape = firstMesh[list(firstMesh.keys())[0]].shape
+ except :
+ print("ERROR: Can only read files that have at least one mesh entry!")
+
+ # Usually 2D/3D for picongpu
+ self.dimensionality = len(fshape)
+
+ # TODO fill me with actual start and end positions in reasonable units
+ self.domain_left_edge = np.zeros(3, dtype=np.float64)
+ self.domain_right_edge = np.ones(3, dtype=np.float64)
+
+ # gridding of the meshes (assumed all mesh entries are on the same mesh)
+ self.domain_dimensions = np.ones(3, dtype=np.int64)
+ self.domain_dimensions[:len(fshape)] = fshape
+
+ # TODO assumes non-peridic boundary conditions
+ self.periodicity = np.zeros(3, dtype=np.bool)
+
+ self.current_time = f[self.basePath].attrs[
+ "time"] # <= simulation time in code units
+
+ # Used for AMR, not applicable for us
+ # TODO Probably 1
+ self.refine_by = 2
+
+ # Not a cosmological simulation
+ self.cosmological_simulation = 0 # <= int, 0 or 1
+ # Not necessary to set these according to Axel
+ # self.current_redshift = 0 # <= float
+ # self.omega_lambda = 0 # <= float
+ # self.omega_matter = 0 # <= float
+ # self.hubble_constant = 0 # <= float
+
+ @classmethod
+ def _is_valid(self, *args, **kwargs):
+ """
+ This function test if the (with yt.load()) a file could be opened with
+ this frontend
+ """
+ try:
+ f = validator.open_file(args[0])
+ except:
+ return False
+ verbose = False
+ extension_pic = False
+ # root attributes at "/"
+ result_array = np.array([0, 0])
+ result_array += validator.check_root_attr(f, verbose, extension_pic)
+
+ # Go through all the iterations, checking both the particles
+ # and the meshes
+ result_array += validator.check_iterations(f, verbose, extension_pic)
+ if result_array[0] != 0:
+ return False
+ return True
diff -r c07d12bf5903b40c0c6e65a8ea8f28a82f142e1d -r b186ca2955959cb04b0a37602855c28b3d89189f yt/frontends/openPMD/definitions.py
--- /dev/null
+++ b/yt/frontends/openPMD/definitions.py
@@ -0,0 +1,1 @@
+# This file is often empty. It can hold definitions related to a frontend.
diff -r c07d12bf5903b40c0c6e65a8ea8f28a82f142e1d -r b186ca2955959cb04b0a37602855c28b3d89189f yt/frontends/openPMD/fields.py
--- /dev/null
+++ b/yt/frontends/openPMD/fields.py
@@ -0,0 +1,170 @@
+"""
+openPMD-specific fields
+
+
+
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2013, yt Development Team.
+# Copyright (c) 2015, Daniel Grassinger (HZDR)
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+import numpy as np
+from yt.funcs import mylog
+import yt.utilities.physical_constants
+from yt.fields.field_info_container import \
+ FieldInfoContainer, \
+ particle_deposition_functions, \
+ particle_vector_functions, \
+ standard_particle_fields
+
+def _kinetic_energy(field, data):
+ """
+ Function to calculate new fields out of other fields
+ if a error occurs the field is not calculated
+ """
+ mylog.info("oPMD - fields - _kinetic_energy")
+ # calculate kinetic energy out of momentum
+ c = 2.997e8 # velocity of light
+ ke = (data["particle_momentum_x"] ** 2
+ + data["particle_momentum_y"] ** 2
+ + data["particle_momentum_z"] ** 2) * c
+ return ke
+
+
+def setup_momentum_to_velocity(self, ptype):
+ """
+ TODO This function does no work !!
+ """
+ mylog.info("oPMD - fields - setup_momentum_to_velocity")
+ def _get_vel(axis):
+ def velocity(field, data):
+ c = 2.997e8
+ moment = data[ptype, "particle_momentum_%" % axis]
+ return moment / ((data[ptype, "particle_mass"] * data[ptype, "particle_weighting"]) ** 2 + (moment ** 2) / (c ** 2)) ** 0.5
+ return velocity
+
+ for ax in 'xyz':
+ self.add_field((ptype, "particle_velocity_%s" % ax),
+ function=_get_vel(ax),
+ units="m/s")
+
+
+def setup_poynting_vector(self):
+ mylog.info("oPMD - fields - setup_poynting_vector")
+ def _get_poyn(axis):
+ def poynting(field, data):
+ Efieldx = data["E_x"]
+ Efieldy = data["E_y"]
+ Efieldz = data["E_z"]
+ Bfieldx = data["B_x"]
+ Bfieldy = data["B_y"]
+ Bfieldz = data["B_z"]
+
+ u = 79577.4715459 # = 1/magnetic permeability
+
+ if(axis == 'x'):
+ return u * (Efieldy * Bfieldz - Efieldz * Bfieldy)
+ elif(axis == 'y'):
+ return u * (Efieldz * Bfieldx - Efieldx * Bfieldz)
+ elif(axis == 'z'):
+ return u * (Efieldx * Bfieldy - Efieldy * Bfieldx)
+
+ return poynting
+ for ax in 'xyz':
+ self.add_field(("openPMD", "poynting_vector_%s" % ax),
+ function=_get_poyn(ax),
+ units="T*V/m") # N/(m*s)
+
+
+class openPMDFieldInfo(FieldInfoContainer):
+ """
+ We need to specify which fields we might have in our dataset. The field info
+ container subclass here will define which fields it knows about. There are
+ optionally methods on it that get called which can be subclassed.
+
+ !! TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+ This class defines, which fields and particle fields could be in the HDF5-file
+ The field names have to match the names in "openPMDHierarchy" in data_structures.py
+ This also defines the units of the fields
+ """
+ # TODO Generate this when parsing a file
+ known_other_fields = (
+ # Each entry here is of the form
+ # ( "name", ("units", ["fields", "to", "alias"], # "display_name")),
+ ("B_x", ("T", [], None)),
+ ("B_y", ("T", [], None)),
+ ("B_z", ("T", [], None)),
+ ("E_x", ("V/m", [], None)),
+ ("E_y", ("V/m", [], None)),
+ ("E_z", ("V/m", [], None)),
+ ("J_x", ("A/m**2", [], None)),
+ ("J_y", ("A/m**2", [], None)),
+ ("J_z", ("A/m**2", [], None)),
+ ("rho", ("kg*s/m**3.0", [], None)),
+
+ )
+
+ # TODO Generate this when parsing a file
+ known_particle_fields = (
+ # Identical form to above
+ # ( "name", ("units", ["fields", "to", "alias"], # "display_name")),
+ ("particle_charge", ("A*s", [], None)),
+ ("particle_mass", ("kg", [], None)),
+ ("particle_momentum_x", ("kg*m/s", [], None)),
+ ("particle_momentum_y", ("kg*m/s", [], None)),
+ ("particle_momentum_z", ("kg*m/s", [], None)),
+ ("particle_position_x", ("m", [], None)),
+ ("particle_position_y", ("m", [], None)),
+ ("particle_position_z", ("m", [], None)),
+ ("particle_positionOffset_x", ("m", [], None)),
+ ("particle_positionOffset_y", ("m", [], None)),
+ ("particle_positionOffset_z", ("m", [], None)),
+ ("particle_weighting", ("", [], None)),
+ ("particle_kinetic_energy", ("dimensionless", [], None)),
+
+ )
+
+ def __init__(self, ds, field_list):
+ super(openPMDFieldInfo, self).__init__(ds, field_list)
+ # If you want, you can check field_list
+
+ def setup_fluid_fields(self):
+ """
+ Here you can create functions to calculate out of existing fields the
+ values of new fields e.g. calculate out of E-field and B-field the
+ Poynting vector
+ """
+ mylog.info("oPMD - fields - setup_fluid_fields")
+ # Here we do anything that might need info about the dataset.
+ # You can use self.alias, self.add_output_field and self.add_field .
+
+ setup_poynting_vector(self)
+
+ def setup_particle_fields(self, ptype):
+ """
+ This will get called for every particle type.
+
+ TODO You have to call the function of the parent class to load particles
+ """
+
+ mylog.info("oPMD - fields - setup_particle_fields(%s)",ptype)
+ self.add_field(
+ (ptype,
+ "particle_kinetic_energy"),
+ function=_kinetic_energy,
+ units="dimensionless")
+ setup_momentum_to_velocity(self, ptype)
+
+ particle_deposition_functions(ptype, "particle_position",
+ "particle_weighting", self)
+ standard_particle_fields(self, ptype)
+
+ # TODO Has to be called to load particles
+ super(openPMDFieldInfo, self).setup_particle_fields(ptype)
diff -r c07d12bf5903b40c0c6e65a8ea8f28a82f142e1d -r b186ca2955959cb04b0a37602855c28b3d89189f yt/frontends/openPMD/io.py
--- /dev/null
+++ b/yt/frontends/openPMD/io.py
@@ -0,0 +1,400 @@
+"""
+openPMD-specific IO functions
+
+
+
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2013, yt Development Team.
+# Copyright (c) 2015, Daniel Grassinger (HZDR)
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+from yt.utilities.io_handler import \
+ BaseIOHandler
+from yt.extern.six import u, b, iteritems
+from yt.geometry.selection_routines import mask_fill, AlwaysSelector
+from yt.utilities.logger import ytLogger as mylog
+import h5py
+import numpy as np
+from collections import defaultdict
+from .data_structures import openPMDBasePath
+
+# Unused copy-paste?
+_convert_mass = ("particle_mass", "mass")
+
+
+class IOHandlerOpenPMD(BaseIOHandler, openPMDBasePath):
+ # TODO Data should be loaded chunk-wise to support parallelism. Fields can be chunked arbitrarily in space, particles should be read via particlePatches.
+ # TODO Maybe reuse hdf5 file handles in loops?
+ # TODO Change strategies when dealing with different iterationEncoding?
+ # TODO since the standard allows \w+ regexes, replace _'s in oPMD for yt
+
+ # TODO Unify numpy data types?
+ # THERE WAS EVEN A GLOBAL VARIABLE FOR THIS
+ _field_dtype = "float32"
+ _dataset_type = "openPMD"
+
+ def __init__(self, ds, *args, **kwargs):
+
+ self.ds = ds
+ self._handle = ds._handle
+ self._setBasePath(self._handle)
+ self.meshPath = self._handle["/"].attrs["meshesPath"]
+ self.particlesPath = self._handle["/"].attrs["particlesPath"]
+
+ def _read_particle_coords(self, chunks, ptf):
+ """
+ Reads coordinates for all specified particle-types from file.
+
+ Parameters
+ ----------
+ chunks:
+ A list of chunks
+ A chunk is a list of grids
+
+ ptf:
+ A dictionary
+ - keys are ptype
+ - values are lists of (paricle?) fields
+
+ Yields
+ ------
+ A series of tuples of (ptype_name, (x_coords, y_coords, z_coords)).
+ ptype_name is a type pf particle,
+ x_coords, y_coords and z_coords are arrays of positions/coordinates of all particles of that type
+ """
+ mylog.info("_read_particle_coords")
+ chunks = list(chunks)
+ self._array_fields = {}
+ for chunk in chunks:
+ f = None
+ for g in chunk.objs:
+ if g.filename is None:
+ continue
+ if f is None:
+ f = h5py.File(g.filename, "r")
+
+ dds = f[self.basePath]
+ # Sorted by what key? What for?
+ for ptype, field_list in sorted(ptf.items()):
+ # TODO Too generic, pay attention to scalars
+ # Checking a list with a string literal? This should NEVER equal true
+ if field_list == "particle_mass":
+ continue
+ # Get a particle group of a particular particle type (e.g. /data/3500/particles/e/)
+ pds = dds.get("%s/%s" % (f.attrs["particlesPath"], ptype))
+
+ # Get single 1D-arrays of coordinates from all particular particles
+ # TODO Make easier to read (put the loop outside?)
+ # TODO Doesn't include offset
+ # TODO Do not naively assume 3D. Check which axes actually exist
+ x, y, z = (np.asarray(pds.get("position/" + ax).value, dtype="float32")
+ for ax in 'xyz')
+ for field in field_list:
+ # Trim field path (yt scheme) to match openPMD scheme
+ nfield = field.replace("particle_", "")
+ nfield = nfield.replace("_", "/")
+
+ # Save the size of the particle fields
+ # TODO This CAN be used for speedup, not sure how
+ if np.asarray(pds[nfield]).ndim > 1:
+ self._array_fields[field] = pds[nfield].shape
+ yield (ptype, (x, y, z))
+ if f:
+ f.close()
+
+ def _read_particle_fields(self, chunks, ptf, selector):
+ """
+ Reads particle fields masked by particle type and field out of a list of chunks from file.
+
+ Parameters
+ ----------
+ chunks:
+ A list of chunks
+ A chunk is a list of grids
+
+ ptf:
+ A dictionary
+ - keys are ptype
+ - values are lists of (particle?) fields
+
+
+ selector:
+ yt-project.org/docs/dev/quickstart/data_inspection.html?highlight=selector#Examining-Data-in-Regions
+ yt-project.org/doc/developing/creating_datatypes.html
+ A region (in and/or outside your domain) specifying the field you want to read
+ Selector objects have a .select_points(x,y,z) that returns a mask, so you need to do your masking here.
+
+ Yields
+ -------
+ Tuples of structure ((ptype, field), data)
+ where data is a numpy array of data masked to
+ - a particle type p
+ - in a specified field
+ """
+ mylog.info("_read_particle_fields")
+ chunks = list(chunks)
+ for chunk in chunks:
+ f = None
+ for g in chunk.objs:
+ if g.filename is None:
+ continue
+ if f is None:
+ f = h5py.File(u(g.filename), 'r')
+
+ ds = f[self.basePath]
+ # Sorted by what key? What for?
+ for ptype, field_list in sorted(ptf.items()):
+ # Get a particle group of a particular particle type (e.g. /data/3500/particles/e/)
+ pds = ds.get("%s/%s/" % (f.attrs["particlesPath"], ptype))
+
+ # Get single arrays of coordinates from all particular particles
+ # TODO Make easier to read (put the loop outside?)
+ # TODO Doesn't include offset
+ # TODO Do not naively assume 3D. Check which axes actually exist
+ # (axis_labels, geometry)
+ x, y, z = (np.asarray(pds.get("position/" + ax).value, dtype="float32")
+ for ax in 'xyz')
+
+ mask = selector.select_points(x, y, z, 0.0)
+ if mask is None:
+ continue
+ for field in field_list:
+ nfield = field.replace("particle_", "")
+ nfield = nfield.replace("_", "/")
+ # TODO Incomplete because mass and charge are a attributes and the rest are arrays
+ # "the rest"?
+ # TODO Too generic, not only those 2 can be const (attrs 'value' for this)
+ if field == "particle_mass" or field == "particle_charge":
+ """
+ /data/3500/particles/e (u'charge', -0.00014378012565430254)
+ /data/3500/particles/e (u'mass', 0.00014378012565430254)
+ """
+ # Creates an array of dimension x.shape[0],
+ # Filled only with the specified value
+ # What for?
+ data = np.full(
+ x.shape[0],
+ pds.attrs.get(nfield), #*unitSI
+ # TODO use pds.attrs.get(nfield).dtype for data arrays
+ "=f8"
+ )
+ else:
+ data = np.asarray(pds.get(nfield), "=f8")
+ # Here you could multiply mass with weighting
+ # if field in _convert_mass:
+ # data *= g.dds.prod(dtype="f8")
+
+ yield ((ptype, field), data[mask])
+ if f:
+ f.close()
+
+ # def _read_particle_selection(self, chunks, selector, fields):
+ # """
+ # Reads masked selection of particles from specified fields.
+ #
+ # Receives a collection of data "chunks", a selector describing which "chunks" you are concerned with and
+ # a list of fields.
+ # It should create and return a dictionary whose keys are the fields,
+ # and whose values are numpy arrays containing the data.
+ # The data should actually be read via the _read_chunk_data() method.
+ #
+ # Parameters
+ # ----------
+ # chunks:
+ # A list of chunks
+ # A chunk is a list of grids
+ #
+ # selector:
+ # yt-project.org/docs/dev/quickstart/data_inspection.html?highlight=selector#Examining-Data-in-Regions
+ # yt-project.org/doc/developing/creating_datatypes.html
+ # A region (in and/or outside your domain) specifying the field you want to read
+ #
+ # fields:
+ # A list of (fname, ftype) tuples representing a field
+ #
+ # size:
+ # Size of the data arrays you want to read
+ #
+ # Returns
+ # -------
+ # A dictionary:
+ # - keys are (ftype, fname) tuples representing a field
+ # - values are numpy arrays with data form that field
+ # """
+ # mylog.debug("Read particle selection")
+ # rv = {}
+ # chunks = list(chunks)
+ #
+ # if selector.__class__.__name__ == "GridSelector":
+ # if not (len(chunks) == len(chunks[0].objs) == 1):
+ # raise RuntimeError
+ # grid = chunks[0].objs[0]
+ # for ftype, fname in fields:
+ # mylog.info("read particles ftype %s, fname %s, grid %s", ftype, fname, grid)
+ # # rv[ftype, fname] = self._read_particles(grid, fname)
+ # return rv
+ #
+ # rv = {f: np.array([]) for f in fields}
+ # for chunk in chunks:
+ # for grid in chunk.objs:
+ # for ftype, fname in fields:
+ # mylog.info("read particles ftype %s, fname %s, grid %s", ftype, fname, grid)
+ # # data = self._read_particles(grid, fname)
+ # # rv[ftype, fname] = np.concatenate((data, rv[ftype, fname]))
+ # return rv
+
+ def _read_fluid_selection(self, chunks, selector, fields, size):
+ """
+ Reads selected fields of given size from file.
+
+ From yt doc:
+ Receives a collection of data "chunks", a selector describing which "chunks" you are concerned with,
+ a list of fields, and the size of the data to read.
+ It should create and return a dictionary whose keys are the fields,
+ and whose values are numpy arrays containing the data.
+ The data should actually be read via the _read_chunk_data() method.
+
+ Parameters
+ ----------
+ chunks:
+ A list of chunks
+ A chunk is a list of grids
+
+ selector:
+ yt-project.org/docs/dev/quickstart/data_inspection.html?highlight=selector#Examining-Data-in-Regions
+ yt-project.org/doc/developing/creating_datatypes.html
+ A region (in and/or outside your domain) specifying the field you want to read
+
+ fields:
+ A list of (fname, ftype) tuples representing a field
+
+ size:
+ Size of the data arrays you want to read
+
+ Returns
+ -------
+ A dictionary:
+ - keys are (ftype, fname) tuples representing a field
+ - values are numpy arrays with data form that field
+ """
+ mylog.info("_read_fluid_selection")
+ rv = {}
+ chunks = list(chunks)
+ if selector.__class__.__name__ == "GridSelector":
+ if not (len(chunks) == len(chunks[0].objs) == 1):
+ raise RuntimeError
+ # TODO We can probably skip this since the following loops do this, anyway
+ # g = chunks[0].objs[0]
+ # rv.update(self._read_chunk_data(chunks[0],fields))
+ #f = h5py.File(u(g.filename), 'r')
+ #for ftype, fname in fields:
+ # rv[ftype, fname] = self._read_data(g, fname)
+ #f.close()
+ # return rv
+
+ if size is None:
+ size = sum((g.count(selector) for chunk in chunks
+ for g in chunk.objs))
+ for field in fields:
+ rv[field] = np.empty(size, dtype="float64")
+
+ for chunk in chunks:
+ rv.update(self._read_chunk_data(chunk, fields))
+ # for g in chunk.objs:
+ # # WHY do we create a filehandle but not use it?
+ # f = h5py.File(u(g.filename), 'r')
+ # ###self._handle = f
+ # for ftype, fname in fields:
+ # # WHY call with g (a grid) as self?
+ # rv[ftype, fname] = self._read_data(g, fname)
+ # f.close()
+ return rv
+
+ # TODO We *probably* do not need this function
+ # def _read_data(self, field):
+ # """
+ # Reads data from file belonging to a (mesh or particle) field
+ #
+ # Parameters
+ # ----------
+ # field:
+ # Field to get the data for
+ #
+ # Returns
+ # -------
+ # A flat numpy array
+ # """
+ # mylog.debug("Read data")
+ #
+ # if field.startswith("particle"):
+ # # particles are not loaded here
+ # # data =
+ # # self._handle[self.basePath+self.particlesPath+field.replace("_","/")]
+ # pass
+ # else:
+ # data = self._handle[
+ # self.basePath +
+ # self.meshPath +
+ # field.replace("_", "/")]
+ # return np.array(data).flatten()
+
+ def _read_chunk_data(self, chunk, fields):
+ """
+ Reads fields specified in the chunk from disk.
+
+ From yt doc:
+ Receives a "chunk" of data along with a list of fields we want to read. It loops over all the grid objects
+ within the "chunk" of data and reads from disk the specific fields,
+ returning a dictionary whose keys are the fields and whose values are numpy arrays of the data.
+
+ Parameters
+ ----------
+ chunk:
+ A single chunk is a list of grids
+
+ fields:
+ A list of fields to be read
+
+ Returns
+ -------
+ A dictionary:
+ - keys are (ftype, fname) field tuples
+ - values are flat numpy arrays with data form that field
+ """
+ mylog.info("_read_chunk_data")
+ rv = {}
+ fluid_fields, particle_fields = [], []
+ for ftype, fname in fields:
+ # TODO generate particle_types from file itself
+ if ftype in self.ds.particle_types:
+ particle_fields.append((ftype, fname))
+ else:
+ fluid_fields.append((ftype, fname))
+ if len(particle_fields) > 0:
+ selector = AlwaysSelector(self.ds)
+ rv.update(self._read_particle_selection(
+ [chunk], selector, particle_fields))
+ if len(fluid_fields) == 0: return rv
+ grids_by_file = defaultdict(list)
+ # Read fluid fields
+ for g in chunk.objs:
+ if g.filename is None:
+ continue
+ grids_by_file[g.filename].append(g)
+ for filename in grids_by_file:
+ grids = grids_by_file[filename]
+ grids.sort()
+ f = h5py.File(filename, 'r')
+ for g in grids:
+ for ftype, fname in fluid_fields:
+ # TODO update basePath for different files
+ data = f[self.basePath + self.meshPath + fname.replace("_", "/").replace("-","_")]
+ rv[(ftype,fname)] = np.array(data).flatten()
+ f.close()
+ return rv
diff -r c07d12bf5903b40c0c6e65a8ea8f28a82f142e1d -r b186ca2955959cb04b0a37602855c28b3d89189f yt/frontends/openPMD/misc.py
--- /dev/null
+++ b/yt/frontends/openPMD/misc.py
@@ -0,0 +1,768 @@
+# Copyright (c) 2015, Axel Huebl, Remi Lehe
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+
+import h5py as h5
+import numpy as np
+import re
+import string
+import collections # for isinstance
+import sys, getopt, os.path
+
+openPMD = "1.0.0"
+
+ext_list = [["ED-PIC", np.uint32(1)]]
+
+def open_file(file_name):
+ try:
+ f = h5.File(file_name, "r")
+ return(f)
+ except:
+ raise
+
+def get_attr(f, name):
+ """
+ Try to access the path `name` in the file `f`
+ Return the corresponding attribute if it is present
+ """
+ if name in list(f.attrs.keys()):
+ return(True, f.attrs[name])
+ else:
+ return(False, None)
+
+def test_record(g, r):
+ """
+ Checks if a record is valid
+
+ Parameters
+ ----------
+ g : h5py.Group
+ The group the record resides in
+
+ r : string
+ The name of the record.
+
+ Returns
+ -------
+ An array with 2 elements :
+ - The first element is 1 if an error occured, and 0 otherwise
+ - The second element is 0 if a warning arised, and 0 otherwise
+ """
+ regEx = re.compile("^\w+$") # Python3 only: re.ASCII
+ if regEx.match(r):
+ # test component names
+ result_array = np.array([0,0])
+ if not is_scalar_record(g[r]) :
+ for component_name in g[r]:
+ if not regEx.match(component_name):
+ print("Error: Component %s of record %s is NOT" \
+ " named properly (a-Z0-9_)!" %(component_name, g[r].name) )
+ result_array += np.array([1,0])
+ else:
+ print("Error: Record %s is NOT named properly (a-Z0-9_)!" \
+ %(r.name) )
+ result_array = np.array([1,0])
+
+ return(result_array)
+
+def test_key(f, v, request, name):
+ """
+ Checks whether a key is present. A key can either be
+ a h5py.Group or a h5py.Dataset.
+ Returns an error if the key if absent and requested
+ Returns a warning if the key if absent and recommended
+
+ Parameters
+ ----------
+ f : an h5py.File or h5py.Group object
+ The object in which to find the key
+
+ v : bool
+ Verbose option
+
+ request : string
+ Either "required", "recommended" or "optional"
+
+ name : string
+ The name of the key within this File, Group or DataSet
+
+ Returns
+ -------
+ An array with 2 elements :
+ - The first element is 1 if an error occured, and 0 otherwise
+ - The second element is 0 if a warning arised, and 0 otherwise
+ """
+ valid = (name in list(f.keys()))
+ if valid:
+ if v:
+ print("Key %s (%s) exists in `%s`!" %(name, request, str(f.name) ) )
+ result_array = np.array([0,0])
+ else:
+ if request == "required":
+ print("Error: Key %s (%s) does NOT exist in `%s`!" \
+ %(name, request, str(f.name)) )
+ result_array = np.array([1, 0])
+ elif request == "recommended":
+ print("Warning: Key %s (%s) does NOT exist in `%s`!" \
+ %(name, request, str(f.name)) )
+ result_array = np.array([0, 1])
+ elif request == "optional":
+ if v:
+ print("Info: Key %s (%s) does NOT exist in `%s`!" \
+ %(name, request, str(f.name)) )
+ result_array = np.array([0, 0])
+ else :
+ raise ValueError("Unrecognized string for `request` : %s" %request)
+
+ return(result_array)
+
+def test_attr(f, v, request, name, is_type=None, type_format=None):
+ """
+ Checks whether an attribute is present.
+ Returns an error if the attribute if absent and requested
+ Returns a warning if the attribute if absent and recommanded
+
+ Parameters
+ ----------
+ f : an h5py.File, h5py.Group or h5py.DataSet object
+ The object in which to find the key
+
+ v : bool
+ Verbose option
+
+ request : string
+ Either "required", "recommended" or "optional
+
+ name : string
+ The name of the attribute within this File, Group or DataSet
+
+ is_type : (numpy or python) data type
+ The type of the attribute. Default is "arbitrary" for None.
+ Can be a list of data types where at least one data type must match
+ but this list can not be combined with type_format.
+
+ type_format: (numpy or python) data type
+ Used with is_type to specify numpy ndarray dtypes or a
+ base np.string_ format regex. Can be a list of data types
+ for ndarrays where at least one data type must match.
+
+ Returns
+ -------
+ An array with 2 elements :
+ - The first element is 1 if an error occured, and 0 otherwise
+ - The second element is 0 if a warning arised, and 0 otherwise
+ """
+ valid, value = get_attr(f, name)
+ if valid:
+ if v:
+ print("Attribute %s (%s) exists in `%s`! Type = %s, Value = %s" \
+ %(name, request, str(f.name), type(value), str(value)) )
+
+ # test type
+ if is_type is not None:
+ if not type_format is None and not is_type is np.string_ and \
+ not isinstance(type_format, collections.Iterable):
+ type_format = [type_format]
+ type_format_names = map(lambda x: x.__name__, type_format)
+ if not is_type is None and not isinstance(is_type, collections.Iterable):
+ is_type = [is_type]
+ is_type_names = map(lambda x: x.__name__, is_type)
+ # add for each type in is_type -> wrong, need to add this at the comparison level!
+ if type(value) in is_type:
+ # np.string_ format or general ndarray dtype text
+ if type(value) is np.string_ and type_format is not None:
+ regEx = re.compile(type_format) # Python3 only: re.ASCII
+ if regEx.match(value.decode()) :
+ result_array = np.array([0,0])
+ else:
+ print("Error: Attribute %s in `%s` does not satisfy " \
+ "format ('%s' should be in format '%s')!" \
+ %(name, str(f.name), value.decode(), type_format ) )
+ result_array = np.array([1,0])
+ # ndarray dtypes
+ elif type(value) is np.ndarray:
+ if value.dtype.type in type_format:
+ result_array = np.array([0,0])
+ elif type_format is None:
+ result_array = np.array([0,0])
+ else:
+ print("Error: Attribute %s in `%s` is not of type " \
+ "ndarray of '%s' (is ndarray of '%s')!" \
+ %(name, str(f.name), type_format_names, \
+ value.dtype.type.__name__) )
+ result_array = np.array([1,0])
+ else:
+ result_array = np.array([0,0])
+ else:
+ print(
+ "Error: Attribute %s in `%s` is not of type '%s' (is '%s')!" \
+ %(name, str(f.name), str(is_type_names), \
+ type(value).__name__) )
+ result_array = np.array([1,0])
+ else: # is_type is None (== arbitrary)
+ result_array = np.array([0,0])
+ else:
+ if request == "required":
+ print("Error: Attribute %s (%s) does NOT exist in `%s`!" \
+ %(name, request, str(f.name)) )
+ result_array = np.array([1, 0])
+ elif request == "recommended":
+ print("Warning: Attribute %s (%s) does NOT exist in `%s`!" \
+ %(name, request, str(f.name)) )
+ result_array = np.array([0, 1])
+ elif request == "optional":
+ if v:
+ print("Info: Attribute %s (%s) does NOT exist in `%s`!" \
+ %(name, request, str(f.name)) )
+ result_array = np.array([0, 0])
+ else :
+ raise ValueError("Unrecognized string for `request` : %s" %request)
+
+ return(result_array)
+
+def is_scalar_record(r):
+ """
+ Checks if a record is a scalar record or not.
+
+ Parameters
+ ----------
+ r : an h5py.Group or h5py.Dataset object
+ the record that shall be tested
+
+ Returns
+ -------
+ bool : true if the record is a scalar record, false if the record
+ is either a vector or an other type of tensor record
+ """
+ if type(r) is h5.Group :
+ # now it could be either a vector/tensor record
+ # or a scalar record with a constant component
+
+ valid, value = get_attr(r, "value")
+ # constant components require a "value" and a "shape" attribute
+ if valid :
+ return True
+ else:
+ return False
+ else :
+ return True
+
+def test_component(c, v) :
+ """
+ Checks if a record component defines all required attributes.
+
+ Parameters
+ ----------
+ c : an h5py.Group or h5py.Dataset object
+ the record component that shall be tested
+
+ v : bool
+ Verbose option
+
+ Returns
+ -------
+ An array with 2 elements :
+ - The first element is the number of errors encountered
+ - The second element is the number of warnings encountered
+ """
+ # Initialize the result array
+ # First element : number of errors
+ # Second element : number of warnings
+ result_array = np.array([0,0])
+
+ if type(c) is h5.Group :
+ # since this check tests components, this must be a constant
+ # component: requires "value" and "shape" attributes
+ result_array += test_attr(c, v, "required", "value") # type can be arbitrary
+ result_array += test_attr(c, v, "required", "shape", np.ndarray, np.uint64)
+
+ # default attributes for all components
+ result_array += test_attr(c, v, "required", "unitSI", np.float64)
+
+ return(result_array)
+
+
+def check_root_attr(f, v, pic):
+ """
+ Scan the root of the file and make sure that all the attributes are present
+
+ Parameters
+ ----------
+ f : an h5py.File object
+ The HDF5 file in which to find the attribute
+
+ v : bool
+ Verbose option
+
+ pic : bool
+ Whether to check for the ED-PIC extension attributes
+
+ Returns
+ -------
+ An array with 2 elements :
+ - The first element is the number of errors encountered
+ - The second element is the number of warnings encountered
+ """
+ # Initialize the result array
+ # First element : number of errors
+ # Second element : number of warnings
+ result_array = np.array([0,0])
+
+ # STANDARD.md
+ # required
+ result_array += test_attr(f, v, "required", "openPMD", np.string_, "^[0-9]+\.[0-9]+\.[0-9]+$")
+ result_array += test_attr(f, v, "required", "openPMDextension", np.uint32)
+ result_array += test_attr(f, v, "required", "basePath", np.string_, "^\/data\/\%T\/$")
+ result_array += test_attr(f, v, "required", "meshesPath", np.string_)
+ result_array += test_attr(f, v, "required", "particlesPath", np.string_)
+ result_array += test_attr(f, v, "required", "iterationEncoding", np.string_, "^groupBased|fileBased$")
+ result_array += test_attr(f, v, "required", "iterationFormat", np.string_)
+
+ # groupBased iteration encoding needs to match basePath
+ if result_array[0] == 0 :
+ if f.attrs["iterationEncoding"].decode() == "groupBased" :
+ if f.attrs["iterationFormat"].decode() != f.attrs["basePath"].decode() :
+ print("Error: for groupBased iterationEncoding the basePath "
+ "and iterationFormat must match!")
+ result_array += np.array([1,0])
+
+ # recommended
+ result_array += test_attr(f, v, "recommended", "author", np.string_)
+ result_array += test_attr(f, v, "recommended", "software", np.string_)
+ result_array += test_attr(f, v, "recommended",
+ "softwareVersion", np.string_)
+ result_array += test_attr(f, v, "recommended", "date", np.string_,
+ "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [\+|-][0-9]{4}$")
+
+ # optional
+ result_array += test_attr(f, v, "optional", "comment", np.string_)
+
+ # Extension: ED-PIC
+ if pic:
+ valid, extensionIDs = get_attr(f, "openPMDextension")
+ if valid:
+ if (ext_list[0][1] & extensionIDs) != extensionIDs:
+ print("Error: ID=%s for extension `%s` not found in " \
+ "`openPMDextension` (is %s)!" \
+ %(ext_list[0][1], ext_list[0][0], extensionIDs) )
+ result_array += np.array([1,0])
+
+ return(result_array)
+
+
+def check_iterations(f, v, pic) :
+ """
+ Scan all the iterations present in the file, checking both
+ the meshes and the particles
+
+ Parameters
+ ----------
+ f : an h5py.File object
+ The HDF5 file in which to find the attribute
+
+ v : bool
+ Verbose option
+
+ pic : bool
+ Whether to check for the ED-PIC extension attributes
+
+ Returns
+ -------
+ An array with 2 elements :
+ - The first element is the number of errors encountered
+ - The second element is the number of warnings encountered
+ """
+
+ # Find all the iterations
+ format_error = False
+ try :
+ list_iterations = list(f['/data/'].keys())
+ except KeyError :
+ format_error = True
+ else :
+ # Check that these iterations are indeed encoded as integers
+ for iteration in list_iterations :
+ for character in iteration : # go through the string
+ if not (character in string.digits) :
+ format_error = True
+ # Detect any error and interrupt execution if one is found
+ if format_error == True :
+ print("Error: it seems that the path of the data within the HDF5 file "
+ "is not of the form '/data/%T/', where %T corresponds to an "
+ "actual integer.")
+ return(np.array([1, 0]))
+ else :
+ print("Found %d iteration(s)" % len(list_iterations) )
+
+ # Initialize the result array
+ # First element : number of errors
+ # Second element : number of warnings
+ result_array = np.array([ 0, 0])
+
+ # Loop over the iterations and check the meshes and the particles
+ for iteration in list_iterations :
+ result_array += check_base_path(f, iteration, v, pic)
+ # Go deeper only if there is no error at this point
+ if result_array[0] == 0 :
+ result_array += check_meshes(f, iteration, v, pic)
+ result_array += check_particles(f, iteration, v, pic)
+
+ return(result_array)
+
+def check_base_path(f, iteration, v, pic):
+ """
+ Scan the base_path that corresponds to this iteration
+
+ Parameters
+ ----------
+ f : an h5py.File object
+ The HDF5 file in which to find the attribute
+
+ iteration : string representing an integer
+ The iteration at which to scan the meshes
+
+ v : bool
+ Verbose option
+
+ pic : bool
+ Whether to check for the ED-PIC extension attributes
+
+ Returns
+ -------
+ An array with 2 elements :
+ - The first element is the number of errors encountered
+ - The second element is the number of warnings encountered
+ """
+ # Initialize the result array
+ # First element : number of errors
+ # Second element : number of warnings
+ result_array = np.array([ 0, 0])
+
+ # Find the path to the data
+ base_path = ("/data/%s/" % iteration).encode('ascii')
+ bp = f[base_path]
+
+ # Check for the attributes of the STANDARD.md
+ result_array += test_attr(bp, v, "required", "time", [np.float32, np.float64])
+ result_array += test_attr(bp, v, "required", "dt", [np.float32, np.float64])
+ result_array += test_attr(bp, v, "required", "timeUnitSI", np.float64)
+
+ return(result_array)
+
+def check_meshes(f, iteration, v, pic):
+ """
+ Scan all the meshes corresponding to one iteration
+
+ Parameters
+ ----------
+ f : an h5py.File object
+ The HDF5 file in which to find the attribute
+
+ iteration : string representing an integer
+ The iteration at which to scan the meshes
+
+ v : bool
+ Verbose option
+
+ pic : bool
+ Whether to check for the ED-PIC extension attributes
+
+ Returns
+ -------
+ An array with 2 elements :
+ - The first element is the number of errors encountered
+ - The second element is the number of warnings encountered
+ """
+ # Initialize the result array
+ # First element : number of errors
+ # Second element : number of warnings
+ result_array = np.array([ 0, 0])
+
+ # Find the path to the data
+ base_path = "/data/%s/" % iteration
+ valid, meshes_path = get_attr(f, "meshesPath")
+ if not valid :
+ print("Error: `meshesPath` is missing or malformed in '/'")
+ return( np.array([1, 0]) )
+ meshes_path = meshes_path.decode()
+
+ if os.path.join( base_path, meshes_path) != ( base_path + meshes_path ):
+ print("Error: `basePath`+`meshesPath` seems to be malformed "
+ "(is `basePath` absolute and ends on a `/` ?)")
+ return( np.array([1, 0]) )
+ else:
+ full_meshes_path = (base_path + meshes_path).encode('ascii')
+ # Find all the meshes
+ try:
+ list_meshes = list(f[full_meshes_path].keys())
+ except KeyError:
+ list_meshes = []
+ print( "Iteration %s : found %d meshes"
+ %( iteration, len(list_meshes) ) )
+
+ # Check for the attributes of the STANDARD.md
+ for field_name in list_meshes :
+ field = f[full_meshes_path + field_name.encode('ascii')]
+
+ result_array += test_record(f[full_meshes_path], field_name)
+
+ # General attributes of the record
+ result_array += test_attr(field, v, "required",
+ "unitDimension", np.ndarray, np.float64)
+ result_array += test_attr(field, v, "required",
+ "timeOffset", [np.float32, np.float64])
+ result_array += test_attr(field, v, "required",
+ "gridSpacing", np.ndarray, [np.float32, np.float64])
+ result_array += test_attr(field, v, "required",
+ "gridGlobalOffset", np.ndarray, [np.float32, np.float64])
+ result_array += test_attr(field, v, "required",
+ "gridUnitSI", np.float64)
+ result_array += test_attr(field, v, "required",
+ "dataOrder", np.string_)
+ result_array += test_attr(field, v, "required",
+ "axisLabels", np.ndarray, np.string_)
+ # Specific check for geometry
+ geometry_test = test_attr(field, v, "required", "geometry", np.string_)
+ result_array += geometry_test
+ # geometryParameters is required when using thetaMode
+ if geometry_test[0] == 0 and field.attrs["geometry"] == b"thetaMode" :
+ result_array += test_attr(field, v, "required",
+ "geometryParameters", np.string_)
+ # otherwise it is optional
+ else :
+ result_array += test_attr(field, v, "optional",
+ "geometryParameters", np.string_)
+
+ # Attributes of the record's components
+ if is_scalar_record(field) : # If the record is a scalar field
+ result_array += test_component(field, v)
+ result_array += test_attr(field, v,
+ "required", "position", np.ndarray, [np.float32, np.float64])
+ else: # If the record is a vector field
+ # Loop over the components
+ for component_name in list(field.keys()) :
+ component = field[component_name]
+ result_array += test_component(component, v)
+ result_array += test_attr(component, v,
+ "required", "position", np.ndarray, [np.float32, np.float64])
+
+ # Check for the attributes of the PIC extension,
+ # if asked to do so by the user
+ if pic:
+
+ # Check the attributes associated with the field solver
+ result_array += test_attr(f[full_meshes_path], v, "required",
+ "fieldSolver", np.string_)
+ valid, field_solver = get_attr(f[full_meshes_path], "fieldSolver")
+ if (valid == True) and (field_solver in ["other", "GPSTD"]) :
+ result_array += test_attr(f[full_meshes_path], v, "required",
+ "fieldSolverParameters", np.string_)
+
+ # Check for the attributes associated with the field boundaries
+ result_array += test_attr(f[full_meshes_path], v, "required",
+ "fieldBoundary", np.ndarray, np.string_)
+ valid, field_boundary = get_attr(f[full_meshes_path], "fieldBoundary")
+ if (valid == True) and (np.any(field_boundary == b"other")) :
+ result_array += test_attr(f[full_meshes_path], v, "required",
+ "fieldBoundaryParameters", np.ndarray, np.string_)
+
+ # Check for the attributes associated with the field boundaries
+ result_array += test_attr(f[full_meshes_path], v, "required",
+ "particleBoundary", np.ndarray, np.string_)
+ valid, particle_boundary = get_attr(f[full_meshes_path], "particleBoundary")
+ if (valid == True) and (np.any(particle_boundary == b"other")) :
+ result_array += test_attr(f[full_meshes_path], v, "required",
+ "particleBoundaryParameters", np.ndarray, np.string_)
+
+ # Check the attributes associated with the current smoothing
+ result_array += test_attr(f[full_meshes_path], v, "required",
+ "currentSmoothing", np.string_)
+ valid, current_smoothing = get_attr(f[full_meshes_path], "currentSmoothing")
+ if (valid == True) and (current_smoothing != b"none") :
+ result_array += test_attr(f[full_meshes_path], v, "required",
+ "currentSmoothingParameters", np.string_)
+
+ # Check the attributes associated with the charge conservation
+ result_array += test_attr(f[full_meshes_path], v, "required",
+ "chargeCorrection", np.string_)
+ valid, charge_correction = get_attr(f[full_meshes_path], "chargeCorrection")
+ if valid == True and charge_correction != b"none":
+ result_array += test_attr(f[full_meshes_path], v, "required",
+ "chargeCorrectionParameters", np.string_)
+
+ # Check for the attributes of each record
+ for field_name in list_meshes :
+ field = f[full_meshes_path + field_name.encode('ascii')]
+ result_array + test_attr(field, v, "required",
+ "fieldSmoothing", np.string_)
+ valid, field_smoothing = get_attr(field, "fieldSmoothing")
+ if (valid == True) and (field_smoothing != b"none") :
+ result_array += test_attr(field,v, "required",
+ "fieldSmoothingParameters", np.string_)
+ return(result_array)
+
+
+def check_particles(f, iteration, v, pic) :
+ """
+ Scan all the particle data corresponding to one iteration
+
+ Parameters
+ ----------
+ f : an h5py.File object
+ The HDF5 file in which to find the attribute
+
+ iteration : string representing an integer
+ The iteration at which to scan the particle data
+
+ v : bool
+ Verbose option
+
+ pic : bool
+ Whether to check for the ED-PIC extension attributes
+
+ Returns
+ -------
+ An array with 2 elements :
+ - The first element is the number of errors encountered
+ - The second element is the number of warnings encountered
+ """
+ # Initialize the result array
+ # First element : number of errors
+ # Second element : number of warnings
+ result_array = np.array([ 0, 0])
+
+ # Find the path to the data
+ base_path = ("/data/%s/" % iteration).encode('ascii')
+ valid, particles_path = get_attr(f, "particlesPath")
+ if os.path.join( base_path, particles_path) != \
+ ( base_path + particles_path ) :
+ print("Error: `basePath`+`meshesPath` seems to be malformed "
+ "(is `basePath` absolute and ends on a `/` ?)")
+ return( np.array([1, 0]) )
+ else:
+ full_particle_path = base_path + particles_path
+ # Find all the particle species
+ try:
+ list_species = list(f[full_particle_path].keys())
+ except KeyError:
+ list_species = []
+ print( "Iteration %s : found %d particle species"
+ %( iteration, len(list_species) ) )
+
+ # Go through all the particle species
+ for species_name in list_species :
+ species = f[full_particle_path + species_name.encode('ascii')]
+
+ # Check all records for this species
+ for species_record_name in species :
+ result_array += test_record(species, species_record_name)
+
+ # Check the position record of the particles
+ result_array += test_key(species, v, "required", "position")
+
+ # Check the position offset record of the particles
+ result_array += test_key(species, v, "required", "positionOffset")
+ if result_array[0] == 0 :
+ position_dimensions = len(species["position"].keys())
+ positionOffset_dimensions = len(species["positionOffset"].keys())
+ if position_dimensions != positionOffset_dimensions :
+ print("Error: `position` (ndim=%s) and `positionOffset` " \
+ "(ndim=%s) do not have the same dimensions in " \
+ "species `%s`!" \
+ %(str(position_dimensions), \
+ str(positionOffset_dimensions),
+ species.name) )
+ result_array += np.array([ 1, 0])
+
+ # Check the particlePatches record of the particles
+ patch_test = test_key(species, v, "recommended", "particlePatches")
+ result_array += patch_test
+ if result_array[0] == 0 and patch_test[1] == 0 :
+ result_array += test_key(species["particlePatches"], v, "required",
+ "numParticles")
+ result_array += test_key(species["particlePatches"], v, "required",
+ "numParticlesOffset")
+ result_array += test_key(species["particlePatches"], v, "required",
+ "offset")
+ result_array += test_key(species["particlePatches"], v, "required",
+ "extent")
+ if result_array[0] == 0 :
+ offset = species["particlePatches"]["offset"]
+ extent = species["particlePatches"]["extent"]
+ # Attributes of the components
+ for component_name in list(species["position"].keys()) :
+ result_array += test_key( offset, v, "required",
+ component_name)
+ result_array += test_key( extent, v, "required",
+ component_name)
+ if result_array[0] == 0 :
+ dset_offset = offset[component_name]
+ result_array += test_component(dset_offset, v)
+ dset_extent = extent[component_name]
+ result_array += test_component(dset_extent, v)
+
+ # Check the records required by the PIC extension
+ if pic :
+ result_array += test_key(species, v, "required", "momentum")
+ result_array += test_key(species, v, "required", "charge")
+ result_array += test_key(species, v, "required", "mass")
+ result_array += test_key(species, v, "required", "weighting")
+ result_array += test_key(species, v, "optional", "boundElectrons")
+ result_array += test_key(species, v, "optional", "protonNumber")
+ result_array += test_key(species, v, "optional", "neutronNumber")
+
+ # Check the attributes associated with the PIC extension
+ if pic :
+ result_array += test_attr(species, v, "required",
+ "particleShape", [np.float32, np.float64])
+ result_array += test_attr(species, v, "required",
+ "currentDeposition", np.string_)
+ result_array += test_attr(species, v, "required",
+ "particlePush", np.string_)
+ result_array += test_attr(species, v, "required",
+ "particleInterpolation", np.string_)
+
+ # Check for the attributes associated with the particle smoothing
+ result_array += test_attr(species, v, "required",
+ "particleSmoothing", np.string_)
+ valid, particle_smoothing = get_attr(species, "particleSmoothing")
+ if valid == True and particle_smoothing != b"none":
+ result_array += test_attr(species, v, "required",
+ "particleSmoothingParameters", np.string_)
+
+ # Check attributes of each record of the particle
+ for record in list(species.keys()) :
+ # all records (but particlePatches) require units
+ if record != "particlePatches":
+ result_array += test_attr(species[record], v,
+ "required", "unitDimension", np.ndarray, np.float64)
+ time_type = f[base_path].attrs["time"].dtype.type
+ result_array += test_attr(species[record], v, "required",
+ "timeOffset", time_type)
+ if pic :
+ result_array += test_attr(species[record], v, "required",
+ "weightingPower", np.float64)
+ result_array += test_attr(species[record], v, "required",
+ "macroWeighted", np.uint32)
+ # Attributes of the components
+ if is_scalar_record( species[record] ) : # Scalar record
+ dset = species[record]
+ result_array += test_component(dset, v)
+ else : # Vector record
+ # Loop over the components
+ for component_name in list(species[record].keys()):
+ dset = species[ os.path.join(record, component_name) ]
+ result_array += test_component(dset, v)
+
+ return(result_array)
diff -r c07d12bf5903b40c0c6e65a8ea8f28a82f142e1d -r b186ca2955959cb04b0a37602855c28b3d89189f yt/frontends/openPMD/setup.py
--- /dev/null
+++ b/yt/frontends/openPMD/setup.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+"""
+
+Copyright (c) 2015, Daniel Grassinger (HZDR)
+
+Distributed under the terms of the Modified BSD License.
+
+The full license is in the file COPYING.txt, distributed with this software.
+"""
+
+import setuptools
+import os
+import sys
+import os.path
+
+
+def configuration(parent_package='', top_path=None):
+ from numpy.distutils.misc_util import Configuration
+ config = Configuration('openPMD', parent_package, top_path)
+ config.make_config_py() # installs __config__.py
+ # config.make_svn_version_py()
+ return config
https://bitbucket.org/yt_analysis/yt/commits/2c6aae3e2989/
Changeset: 2c6aae3e2989
Branch: yt
User: NTAu... at honeypot.fritz.box
Date: 2016-05-22 14:23:47+00:00
Summary: Distinguish between filebased and groupbased Iterations
Affected #: 1 file
diff -r b186ca2955959cb04b0a37602855c28b3d89189f -r 2c6aae3e29894c97acf684b9bdadc0afa122493c yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -29,6 +29,7 @@
import h5py
import numpy as np
import os
+import re
from yt.utilities.logger import ytLogger as mylog
import sys
@@ -37,7 +38,7 @@
pass
class openPMDBasePath :
- def _setBasePath(self, handle) :
+ def _setBasePath(self, handle, filepath):
"""
Set the base path for the first iteration found in the file.
TODO implement into distinct methods:
@@ -50,27 +51,39 @@
# if the file messed up the base path we avoid throwing a cluttered
# exception below while looking for iterations:
- if handle.attrs["basePath"].decode("utf-8") != u"/data/%T/" :
+ if handle.attrs["basePath"].decode("utf-8") != u"/data/%T/":
raise openPMDBasePathException("openPMD: basePath is non-standard!")
# does `/data/` exist?
- if not u"/data" in handle :
+ if not u"/data" in handle:
raise openPMDBasePathException("openPMD: group for basePath does not exist!")
# find iterations in basePath
list_iterations = []
- for i in list(handle[dataPath].keys()) :
- list_iterations.append(i)
- mylog.warning("openPMD: found {} iterations in file".format(len(list_iterations)))
+ if u"groupBased" in handle.attrs["iterationEncoding"]:
+ for i in list(handle[dataPath].keys()):
+ list_iterations.append(i)
+ mylog.info("openPMD: found {} iterations in file".format(len(list_iterations)))
+ elif u"fileBased" in handle.attrs["iterationEncoding"]:
+ regex = u"^" + handle.attrs["iterationFormat"].replace('%T', '[0-9]+') + u"$"
+ if filepath is '':
+ mylog.warning("openPMD: For file based iterations, please use absolute file paths!")
+ pass
+ for file in os.listdir(filepath):
+ if re.match(regex, file):
+ list_iterations.append(file)
+ mylog.info("openPMD: found {} iterations in directory".format(len(list_iterations)))
+ else:
+ mylog.warning("openOMD: File does not have valid iteration encoding:")
+ mylog.warning(handle.attrs["iterationEncoding"])
- # We found no iterations in basePath
# TODO in the future (see above) this can be a mylog.warning instead of an error
if len(list_iterations) == 0 :
- raise openPMDBasePathException("openPMD: no iterations found in basePath!")
+ raise openPMDBasePathException("openPMD: no iterations found!")
# just handle the first iteration found
- mylog.warning("openPMD: only choose to load first iteration in file")
- self.basePath = "{}/{}/".format(dataPath, list_iterations[0])
+ mylog.warning("openPMD: only choose to load the first iteration")
+ self.basePath = "{}/{}/".format(dataPath, handle[dataPath].keys()[0])
@@ -108,7 +121,7 @@
self.dataset = ds
self.index_filename = ds.parameter_filename
self.directory = os.path.dirname(self.index_filename)
- self._setBasePath(self.dataset._handle)
+ self._setBasePath(self.dataset._handle, self.directory)
GridIndex.__init__(self, ds, dataset_type)
def _detect_output_fields(self):
@@ -276,7 +289,8 @@
# Opens a HDF5 file and stores its file handle in _handle
# All _handle objects refers to the file
self._handle = HDF5FileHandler(filename)
- self._setBasePath(self._handle)
+ mylog.info(os.path.dirname(filename))
+ self._setBasePath(self._handle, os.path.dirname(filename))
Dataset.__init__(self, filename, dataset_type,
units_override=units_override)
self.storage_filename = storage_filename
https://bitbucket.org/yt_analysis/yt/commits/9445548b4238/
Changeset: 9445548b4238
Branch: yt
User: NTAu... at guest053.fz-rossendorf.de
Date: 2016-05-24 08:44:31+00:00
Summary: Include openPMD as a frontend
Affected #: 2 files
diff -r 2c6aae3e29894c97acf684b9bdadc0afa122493c -r 9445548b4238304567247ba2294252754e77bf1f yt/frontends/api.py
--- a/yt/frontends/api.py
+++ b/yt/frontends/api.py
@@ -35,6 +35,7 @@
'halo_catalog',
'http_stream',
'moab',
+ 'openPMD',
'owls',
'owls_subfind',
'ramses',
diff -r 2c6aae3e29894c97acf684b9bdadc0afa122493c -r 9445548b4238304567247ba2294252754e77bf1f yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -289,8 +289,8 @@
# Opens a HDF5 file and stores its file handle in _handle
# All _handle objects refers to the file
self._handle = HDF5FileHandler(filename)
- mylog.info(os.path.dirname(filename))
- self._setBasePath(self._handle, os.path.dirname(filename))
+ self._filepath = os.path.dirname(filename)
+ self._setBasePath(self._handle, self._filepath)
Dataset.__init__(self, filename, dataset_type,
units_override=units_override)
self.storage_filename = storage_filename
@@ -321,6 +321,7 @@
self.time_unit = self.quan(1.0, "s")
self.velocity_unit = self.quan(1.0, "m/s")
self.magnetic_unit = self.quan(1.0, "T")
+ self.unit_registry.modify("code_magnetic", self.magnetic_unit)
def _parse_parameter_file(self):
"""
https://bitbucket.org/yt_analysis/yt/commits/2c5c9d0a7420/
Changeset: 2c5c9d0a7420
Branch: yt
User: NTAu... at guest053.fz-rossendorf.de
Date: 2016-05-24 14:13:50+00:00
Summary: Read full and correct records, no matter whether const/scalar/vector
Affected #: 3 files
diff -r 9445548b4238304567247ba2294252754e77bf1f -r 2c5c9d0a7420e6e2e1b74bbf50aa3ff9408a208c yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -69,9 +69,9 @@
if filepath is '':
mylog.warning("openPMD: For file based iterations, please use absolute file paths!")
pass
- for file in os.listdir(filepath):
- if re.match(regex, file):
- list_iterations.append(file)
+ for filename in os.listdir(filepath):
+ if re.match(regex, filename):
+ list_iterations.append(filename)
mylog.info("openPMD: found {} iterations in directory".format(len(list_iterations)))
else:
mylog.warning("openOMD: File does not have valid iteration encoding:")
@@ -82,7 +82,7 @@
raise openPMDBasePathException("openPMD: no iterations found!")
# just handle the first iteration found
- mylog.warning("openPMD: only choose to load the first iteration")
+ mylog.warning("openPMD: only choose to load the first iteration ({})".format(handle[dataPath].keys()[0]))
self.basePath = "{}/{}/".format(dataPath, handle[dataPath].keys()[0])
@@ -184,7 +184,8 @@
else:
# We probably do not want particlePatches as accessible field lists
pass
- self.field_list.extend([(str(c).split("_")[0], 'particle' + c.lstrip(c.split("_")[0])) for c in particle_fields])
+ self.field_list.extend([("io", c) for c in particle_fields])
+ #self.field_list.extend([(str(c).split("_")[0], 'particle' + c.lstrip(c.split("_")[0])) for c in particle_fields])
def _count_grids(self):
"""
@@ -295,9 +296,11 @@
units_override=units_override)
self.storage_filename = storage_filename
self.fluid_types += ('openPMD',)
- self.particle_types = self._handle[self.basePath + self._handle.attrs["particlesPath"]].keys()
+ #self.particle_types = self._handle[self.basePath + self._handle.attrs["particlesPath"]].keys()
+ #self.particle_types = tuple(self.particle_types)
+ #self.particle_types += ('all',)
+ self.particle_types = ["io", "all"]
self.particle_types = tuple(self.particle_types)
- self.particle_types += ('all',)
self.particle_types_raw = self.particle_types
@@ -320,8 +323,8 @@
self.mass_unit = self.quan(1.0, "kg")
self.time_unit = self.quan(1.0, "s")
self.velocity_unit = self.quan(1.0, "m/s")
- self.magnetic_unit = self.quan(1.0, "T")
- self.unit_registry.modify("code_magnetic", self.magnetic_unit)
+ self.magnetic_unit = self.quan(1.0, "gauss")
+ #self.magnetic_unit = self.quan(1.0, "T")
def _parse_parameter_file(self):
"""
@@ -349,6 +352,7 @@
# TODO At this point one assumes the whole file/simulation
# contains for all mesh records the same dimensionality and shapes
+ # TODO This probably won't work for const records
# TODO Support particle-only files
# pick first field
try :
@@ -382,7 +386,7 @@
# Used for AMR, not applicable for us
# TODO Probably 1
- self.refine_by = 2
+ self.refine_by = 1
# Not a cosmological simulation
self.cosmological_simulation = 0 # <= int, 0 or 1
diff -r 9445548b4238304567247ba2294252754e77bf1f -r 2c5c9d0a7420e6e2e1b74bbf50aa3ff9408a208c yt/frontends/openPMD/fields.py
--- a/yt/frontends/openPMD/fields.py
+++ b/yt/frontends/openPMD/fields.py
@@ -81,7 +81,6 @@
function=_get_poyn(ax),
units="T*V/m") # N/(m*s)
-
class openPMDFieldInfo(FieldInfoContainer):
"""
We need to specify which fields we might have in our dataset. The field info
@@ -155,12 +154,12 @@
"""
mylog.info("oPMD - fields - setup_particle_fields(%s)",ptype)
- self.add_field(
- (ptype,
- "particle_kinetic_energy"),
- function=_kinetic_energy,
- units="dimensionless")
- setup_momentum_to_velocity(self, ptype)
+ # self.add_field(
+ # (ptype,
+ # "particle_kinetic_energy"),
+ # function=_kinetic_energy,
+ # units="dimensionless")
+ # setup_momentum_to_velocity(self, ptype)
particle_deposition_functions(ptype, "particle_position",
"particle_weighting", self)
diff -r 9445548b4238304567247ba2294252754e77bf1f -r 2c5c9d0a7420e6e2e1b74bbf50aa3ff9408a208c yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -24,9 +24,15 @@
from collections import defaultdict
from .data_structures import openPMDBasePath
-# Unused copy-paste?
-_convert_mass = ("particle_mass", "mass")
+def is_const_component(record_component):
+ return ("value" in record_component.attrs.keys())
+def get_component(group, component_name):
+ record_component = group[component_name]
+ if is_const_component(record_component):
+ return record_component.attrs["value"]
+ else:
+ return record_component.value
class IOHandlerOpenPMD(BaseIOHandler, openPMDBasePath):
# TODO Data should be loaded chunk-wise to support parallelism. Fields can be chunked arbitrarily in space, particles should be read via particlePatches.
@@ -43,7 +49,7 @@
self.ds = ds
self._handle = ds._handle
- self._setBasePath(self._handle)
+ self._setBasePath(self._handle, self.ds._filepath)
self.meshPath = self._handle["/"].attrs["meshesPath"]
self.particlesPath = self._handle["/"].attrs["particlesPath"]
@@ -82,28 +88,32 @@
dds = f[self.basePath]
# Sorted by what key? What for?
for ptype, field_list in sorted(ptf.items()):
- # TODO Too generic, pay attention to scalars
- # Checking a list with a string literal? This should NEVER equal true
- if field_list == "particle_mass":
- continue
- # Get a particle group of a particular particle type (e.g. /data/3500/particles/e/)
- pds = dds.get("%s/%s" % (f.attrs["particlesPath"], ptype))
+ # Inefficient for const records
+
+ # Get a particle species (e.g. /data/3500/particles/e/)
+ pds = dds.get("%s/%s" % (f.attrs["particlesPath"], field_list[0].split("_")[0]))
# Get single 1D-arrays of coordinates from all particular particles
- # TODO Make easier to read (put the loop outside?)
- # TODO Doesn't include offset
# TODO Do not naively assume 3D. Check which axes actually exist
- x, y, z = (np.asarray(pds.get("position/" + ax).value, dtype="float32")
- for ax in 'xyz')
+ xpos, ypos, zpos = (get_component(pds, "position/" + ax)
+ for ax in 'xyz')
+ xoff, yoff, zoff = (get_component(pds, "positionOffset/" + ax)
+ for ax in 'xyz')
+ x = xpos * pds["position/x"].attrs["unitSI"] + xoff * pds["positionOffset/x"].attrs["unitSI"]
+ y = ypos * pds["position/y"].attrs["unitSI"] + yoff * pds["positionOffset/y"].attrs["unitSI"]
+ z = zpos * pds["position/z"].attrs["unitSI"] + zoff * pds["positionOffset/z"].attrs["unitSI"]
for field in field_list:
# Trim field path (yt scheme) to match openPMD scheme
- nfield = field.replace("particle_", "")
- nfield = nfield.replace("_", "/")
+ nfield = "/".join(field.split("_")[1:])
# Save the size of the particle fields
# TODO This CAN be used for speedup, not sure how
- if np.asarray(pds[nfield]).ndim > 1:
- self._array_fields[field] = pds[nfield].shape
+ if is_const_component(pds[nfield]):
+ shape = np.asarray(pds[nfield].attrs["shape"])
+ else:
+ shape = np.asarray(pds[nfield].shape)
+ if shape.ndim > 1:
+ self._array_fields[field] = shape
yield (ptype, (x, y, z))
if f:
f.close()
@@ -150,46 +160,31 @@
ds = f[self.basePath]
# Sorted by what key? What for?
for ptype, field_list in sorted(ptf.items()):
- # Get a particle group of a particular particle type (e.g. /data/3500/particles/e/)
- pds = ds.get("%s/%s/" % (f.attrs["particlesPath"], ptype))
+ # Inefficient for const records
+
+ # Get a particle species (e.g. /data/3500/particles/e/)
+ # TODO Do this for all fields in fields list
+ pds = ds.get("%s/%s" % (f.attrs["particlesPath"], field_list[0].split("_")[0]))
# Get single arrays of coordinates from all particular particles
- # TODO Make easier to read (put the loop outside?)
- # TODO Doesn't include offset
# TODO Do not naively assume 3D. Check which axes actually exist
# (axis_labels, geometry)
- x, y, z = (np.asarray(pds.get("position/" + ax).value, dtype="float32")
- for ax in 'xyz')
-
+ xpos, ypos, zpos = (get_component(pds, "position/" + ax)
+ for ax in 'xyz')
+ xoff, yoff, zoff = (get_component(pds, "positionOffset/" + ax)
+ for ax in 'xyz')
+ x = xpos * pds["position/x"].attrs["unitSI"] + xoff * pds["positionOffset/x"].attrs["unitSI"]
+ y = ypos * pds["position/y"].attrs["unitSI"] + yoff * pds["positionOffset/y"].attrs["unitSI"]
+ z = zpos * pds["position/z"].attrs["unitSI"] + zoff * pds["positionOffset/z"].attrs["unitSI"]
mask = selector.select_points(x, y, z, 0.0)
if mask is None:
continue
for field in field_list:
- nfield = field.replace("particle_", "")
- nfield = nfield.replace("_", "/")
- # TODO Incomplete because mass and charge are a attributes and the rest are arrays
- # "the rest"?
- # TODO Too generic, not only those 2 can be const (attrs 'value' for this)
- if field == "particle_mass" or field == "particle_charge":
- """
- /data/3500/particles/e (u'charge', -0.00014378012565430254)
- /data/3500/particles/e (u'mass', 0.00014378012565430254)
- """
- # Creates an array of dimension x.shape[0],
- # Filled only with the specified value
- # What for?
- data = np.full(
- x.shape[0],
- pds.attrs.get(nfield), #*unitSI
- # TODO use pds.attrs.get(nfield).dtype for data arrays
- "=f8"
- )
+ nfield = "/".join(field.split("_")[1:])
+ if is_const_component(pds[nfield]):
+ data = np.full(pds[nfield].attrs["shape"], pds[nfield].attrs["value"])
else:
- data = np.asarray(pds.get(nfield), "=f8")
- # Here you could multiply mass with weighting
- # if field in _convert_mass:
- # data *= g.dds.prod(dtype="f8")
-
+ data = pds[nfield].value
yield ((ptype, field), data[mask])
if f:
f.close()
https://bitbucket.org/yt_analysis/yt/commits/44aba12da271/
Changeset: 44aba12da271
Branch: yt
User: NTAu... at guest053.fz-rossendorf.de
Date: 2016-05-27 12:16:33+00:00
Summary: Use yt's log instead of print
Affected #: 3 files
diff -r 2c5c9d0a7420e6e2e1b74bbf50aa3ff9408a208c -r 44aba12da271ecffa9bc1075fe59c842c67e7165 yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -32,8 +32,6 @@
import re
from yt.utilities.logger import ytLogger as mylog
-import sys
-
class openPMDBasePathException(Exception) :
pass
@@ -344,7 +342,6 @@
f = self._handle
meshesPath = f.attrs["meshesPath"].decode()
particlesPath = f.attrs["particlesPath"].decode()
- positionPath = self.basePath + particlesPath + "/electrons/position/"
# This defines the size of the simulaion box
self.unique_identifier = 0 # no identifier
diff -r 2c5c9d0a7420e6e2e1b74bbf50aa3ff9408a208c -r 44aba12da271ecffa9bc1075fe59c842c67e7165 yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -86,7 +86,6 @@
f = h5py.File(g.filename, "r")
dds = f[self.basePath]
- # Sorted by what key? What for?
for ptype, field_list in sorted(ptf.items()):
# Inefficient for const records
@@ -158,7 +157,6 @@
f = h5py.File(u(g.filename), 'r')
ds = f[self.basePath]
- # Sorted by what key? What for?
for ptype, field_list in sorted(ptf.items()):
# Inefficient for const records
diff -r 2c5c9d0a7420e6e2e1b74bbf50aa3ff9408a208c -r 44aba12da271ecffa9bc1075fe59c842c67e7165 yt/frontends/openPMD/misc.py
--- a/yt/frontends/openPMD/misc.py
+++ b/yt/frontends/openPMD/misc.py
@@ -19,6 +19,7 @@
import string
import collections # for isinstance
import sys, getopt, os.path
+from yt.utilities.logger import ytLogger as mylog
openPMD = "1.0.0"
@@ -66,11 +67,11 @@
if not is_scalar_record(g[r]) :
for component_name in g[r]:
if not regEx.match(component_name):
- print("Error: Component %s of record %s is NOT" \
+ mylog.warning("openPMD: Component %s of record %s is NOT" \
" named properly (a-Z0-9_)!" %(component_name, g[r].name) )
result_array += np.array([1,0])
else:
- print("Error: Record %s is NOT named properly (a-Z0-9_)!" \
+ mylog.warning("openPMD: Record %s is NOT named properly (a-Z0-9_)!" \
%(r.name) )
result_array = np.array([1,0])
@@ -106,20 +107,20 @@
valid = (name in list(f.keys()))
if valid:
if v:
- print("Key %s (%s) exists in `%s`!" %(name, request, str(f.name) ) )
+ mylog.info("Key %s (%s) exists in `%s`!" %(name, request, str(f.name) ) )
result_array = np.array([0,0])
else:
if request == "required":
- print("Error: Key %s (%s) does NOT exist in `%s`!" \
+ mylog.warning("openPMD: Key %s (%s) does NOT exist in `%s`!" \
%(name, request, str(f.name)) )
result_array = np.array([1, 0])
elif request == "recommended":
- print("Warning: Key %s (%s) does NOT exist in `%s`!" \
+ mylog.info("openPMD: Key %s (%s) does NOT exist in `%s`!" \
%(name, request, str(f.name)) )
result_array = np.array([0, 1])
elif request == "optional":
if v:
- print("Info: Key %s (%s) does NOT exist in `%s`!" \
+ mylog.info("openPMD: Key %s (%s) does NOT exist in `%s`!" \
%(name, request, str(f.name)) )
result_array = np.array([0, 0])
else :
@@ -166,7 +167,7 @@
valid, value = get_attr(f, name)
if valid:
if v:
- print("Attribute %s (%s) exists in `%s`! Type = %s, Value = %s" \
+ mylog.info("openPMD: Attribute %s (%s) exists in `%s`! Type = %s, Value = %s" \
%(name, request, str(f.name), type(value), str(value)) )
# test type
@@ -186,7 +187,7 @@
if regEx.match(value.decode()) :
result_array = np.array([0,0])
else:
- print("Error: Attribute %s in `%s` does not satisfy " \
+ mylog.warning("openPMD: Attribute %s in `%s` does not satisfy " \
"format ('%s' should be in format '%s')!" \
%(name, str(f.name), value.decode(), type_format ) )
result_array = np.array([1,0])
@@ -197,7 +198,7 @@
elif type_format is None:
result_array = np.array([0,0])
else:
- print("Error: Attribute %s in `%s` is not of type " \
+ mylog.warning("openPMD: Attribute %s in `%s` is not of type " \
"ndarray of '%s' (is ndarray of '%s')!" \
%(name, str(f.name), type_format_names, \
value.dtype.type.__name__) )
@@ -214,16 +215,16 @@
result_array = np.array([0,0])
else:
if request == "required":
- print("Error: Attribute %s (%s) does NOT exist in `%s`!" \
+ mylog.warning("openPMD: Attribute %s (%s) does NOT exist in `%s`!" \
%(name, request, str(f.name)) )
result_array = np.array([1, 0])
elif request == "recommended":
- print("Warning: Attribute %s (%s) does NOT exist in `%s`!" \
+ mylog.info("openPMD: Attribute %s (%s) does NOT exist in `%s`!" \
%(name, request, str(f.name)) )
result_array = np.array([0, 1])
elif request == "optional":
if v:
- print("Info: Attribute %s (%s) does NOT exist in `%s`!" \
+ mylog.info("openPMD: Attribute %s (%s) does NOT exist in `%s`!" \
%(name, request, str(f.name)) )
result_array = np.array([0, 0])
else :
@@ -333,7 +334,7 @@
if result_array[0] == 0 :
if f.attrs["iterationEncoding"].decode() == "groupBased" :
if f.attrs["iterationFormat"].decode() != f.attrs["basePath"].decode() :
- print("Error: for groupBased iterationEncoding the basePath "
+ mylog.warning("openPMD: for groupBased iterationEncoding the basePath "
"and iterationFormat must match!")
result_array += np.array([1,0])
@@ -353,7 +354,7 @@
valid, extensionIDs = get_attr(f, "openPMDextension")
if valid:
if (ext_list[0][1] & extensionIDs) != extensionIDs:
- print("Error: ID=%s for extension `%s` not found in " \
+ mylog.warning("openPMD: ID=%s for extension `%s` not found in " \
"`openPMDextension` (is %s)!" \
%(ext_list[0][1], ext_list[0][0], extensionIDs) )
result_array += np.array([1,0])
@@ -398,12 +399,12 @@
format_error = True
# Detect any error and interrupt execution if one is found
if format_error == True :
- print("Error: it seems that the path of the data within the HDF5 file "
+ mylog.warning("openPMD: it seems that the path of the data within the HDF5 file "
"is not of the form '/data/%T/', where %T corresponds to an "
"actual integer.")
return(np.array([1, 0]))
else :
- print("Found %d iteration(s)" % len(list_iterations) )
+ mylog.info("openPMD: Found %d iteration(s)" % len(list_iterations) )
# Initialize the result array
# First element : number of errors
@@ -493,12 +494,12 @@
base_path = "/data/%s/" % iteration
valid, meshes_path = get_attr(f, "meshesPath")
if not valid :
- print("Error: `meshesPath` is missing or malformed in '/'")
+ mylog.warning("openPMD: `meshesPath` is missing or malformed in '/'")
return( np.array([1, 0]) )
meshes_path = meshes_path.decode()
if os.path.join( base_path, meshes_path) != ( base_path + meshes_path ):
- print("Error: `basePath`+`meshesPath` seems to be malformed "
+ mylog.warning("openPMD: `basePath`+`meshesPath` seems to be malformed "
"(is `basePath` absolute and ends on a `/` ?)")
return( np.array([1, 0]) )
else:
@@ -647,7 +648,7 @@
valid, particles_path = get_attr(f, "particlesPath")
if os.path.join( base_path, particles_path) != \
( base_path + particles_path ) :
- print("Error: `basePath`+`meshesPath` seems to be malformed "
+ mylog.warning("openPMD: `basePath`+`meshesPath` seems to be malformed "
"(is `basePath` absolute and ends on a `/` ?)")
return( np.array([1, 0]) )
else:
@@ -677,7 +678,7 @@
position_dimensions = len(species["position"].keys())
positionOffset_dimensions = len(species["positionOffset"].keys())
if position_dimensions != positionOffset_dimensions :
- print("Error: `position` (ndim=%s) and `positionOffset` " \
+ mylog.warning("openPMD: `position` (ndim=%s) and `positionOffset` " \
"(ndim=%s) do not have the same dimensions in " \
"species `%s`!" \
%(str(position_dimensions), \
https://bitbucket.org/yt_analysis/yt/commits/402b24b26827/
Changeset: 402b24b26827
Branch: yt
User: NTAu... at guest692.fz-rossendorf.de
Date: 2016-05-31 13:10:55+00:00
Summary: Change naming conventions. This allows yt to generate a lot of derived fields by itself. Particle plots now work (scaling is broken, reading data is inefficient)
Affected #: 4 files
diff -r 44aba12da271ecffa9bc1075fe59c842c67e7165 -r 402b24b268273ef744dff573406431b96e685b4c yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -182,7 +182,13 @@
else:
# We probably do not want particlePatches as accessible field lists
pass
- self.field_list.extend([("io", c) for c in particle_fields])
+ if len(f[self.basePath + particlesPath].keys()) > 1:
+ self.field_list.extend(
+ [(str(c).split("_")[0], ("particle_" + "_".join(str(c).split("_")[1:]))) for c in particle_fields])
+ else:
+ self.field_list.extend(
+ [("io", ("particle_" + "_".join(str(c).split("_")[1:]))) for c in particle_fields])
+ #self.field_list.extend([("io", c) for c in particle_fields])
#self.field_list.extend([(str(c).split("_")[0], 'particle' + c.lstrip(c.split("_")[0])) for c in particle_fields])
def _count_grids(self):
@@ -284,22 +290,27 @@
def __init__(self, filename, dataset_type='openPMD',
storage_filename=None,
- units_override=None):
+ units_override=None,
+ unit_system="mks"):
# Opens a HDF5 file and stores its file handle in _handle
# All _handle objects refers to the file
self._handle = HDF5FileHandler(filename)
self._filepath = os.path.dirname(filename)
self._setBasePath(self._handle, self._filepath)
Dataset.__init__(self, filename, dataset_type,
- units_override=units_override)
+ units_override=units_override,
+ unit_system=unit_system)
self.storage_filename = storage_filename
self.fluid_types += ('openPMD',)
- #self.particle_types = self._handle[self.basePath + self._handle.attrs["particlesPath"]].keys()
- #self.particle_types = tuple(self.particle_types)
+ parts = tuple(str(c) for c in self._handle[self.basePath + self._handle.attrs["particlesPath"]].keys())
+ if len(parts) > 1:
+ self.particle_types = parts
+ mylog.info("openPMD - self.particle_types: {}".format(self.particle_types))
+ self.particle_types_raw = self.particle_types
+ self.particle_types = tuple(self.particle_types)
#self.particle_types += ('all',)
- self.particle_types = ["io", "all"]
+ #self.particle_types = ["io", "all"]
self.particle_types = tuple(self.particle_types)
- self.particle_types_raw = self.particle_types
def _set_code_unit_attributes(self):
diff -r 44aba12da271ecffa9bc1075fe59c842c67e7165 -r 402b24b268273ef744dff573406431b96e685b4c yt/frontends/openPMD/fields.py
--- a/yt/frontends/openPMD/fields.py
+++ b/yt/frontends/openPMD/fields.py
@@ -22,6 +22,7 @@
particle_deposition_functions, \
particle_vector_functions, \
standard_particle_fields
+from misc import parse_unitDimension
def _kinetic_energy(field, data):
"""
@@ -133,6 +134,31 @@
def __init__(self, ds, field_list):
super(openPMDFieldInfo, self).__init__(ds, field_list)
# If you want, you can check field_list
+ mylog.info("oPMD - fields - __init__")
+ other_fields = (
+ # Each entry here is of the form
+ # ( "name", ("units", ["fields", "to", "alias"], # "display_name")),
+ # ("B_x", ("T", [], None)),
+
+ #oPMD:
+ # length L,
+ # mass M,
+ # time T,
+ # electric current I,
+ # thermodynamic temperature theta,
+ # amount of substance N,
+ # luminous intensity J
+
+
+ )
+ f = ds._handle
+ meshesPath = f.attrs["meshesPath"]
+ fields = f[ds.basePath + meshesPath]
+ for i in fields.keys():
+ field = fields.get(i)
+ for i in field.attrs["axisLabels"]:
+ print field, i, parse_unitDimension(np.asarray(field.attrs["unitDimension"], dtype='int'))
+
def setup_fluid_fields(self):
"""
diff -r 44aba12da271ecffa9bc1075fe59c842c67e7165 -r 402b24b268273ef744dff573406431b96e685b4c yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -23,16 +23,8 @@
import numpy as np
from collections import defaultdict
from .data_structures import openPMDBasePath
+from .misc import *
-def is_const_component(record_component):
- return ("value" in record_component.attrs.keys())
-
-def get_component(group, component_name):
- record_component = group[component_name]
- if is_const_component(record_component):
- return record_component.attrs["value"]
- else:
- return record_component.value
class IOHandlerOpenPMD(BaseIOHandler, openPMDBasePath):
# TODO Data should be loaded chunk-wise to support parallelism. Fields can be chunked arbitrarily in space, particles should be read via particlePatches.
@@ -88,9 +80,16 @@
dds = f[self.basePath]
for ptype, field_list in sorted(ptf.items()):
# Inefficient for const records
+ mylog.info("openPMD - _read_particle_coords: {}, {}".format(ptype, field_list))
# Get a particle species (e.g. /data/3500/particles/e/)
- pds = dds.get("%s/%s" % (f.attrs["particlesPath"], field_list[0].split("_")[0]))
+ if "io" in ptype:
+ spec = dds[f.attrs["particlesPath"]].keys()[0]
+ else:
+ spec = field_list[0].split("_")[0]
+ pds = dds.get("%s/%s" % (f.attrs["particlesPath"], spec))
+ if pds is None:
+ mylog.info("openPMD - _read_particle_coords: {}/{} yields None".format(f.attrs["particlesPath"], spec))
# Get single 1D-arrays of coordinates from all particular particles
# TODO Do not naively assume 3D. Check which axes actually exist
@@ -162,11 +161,18 @@
# Get a particle species (e.g. /data/3500/particles/e/)
# TODO Do this for all fields in fields list
- pds = ds.get("%s/%s" % (f.attrs["particlesPath"], field_list[0].split("_")[0]))
+ if "io" in ptype:
+ spec = ds[f.attrs["particlesPath"]].keys()[0]
+ else:
+ spec = field_list[0].split("_")[0]
+ pds = ds.get("%s/%s" % (f.attrs["particlesPath"], spec))
+ if pds is None:
+ mylog.info("openPMD - _read_particle_coords: {}/{} yields None".format(f.attrs["particlesPath"], spec))
# Get single arrays of coordinates from all particular particles
# TODO Do not naively assume 3D. Check which axes actually exist
# (axis_labels, geometry)
+ # TODO Pay attention to const records
xpos, ypos, zpos = (get_component(pds, "position/" + ax)
for ax in 'xyz')
xoff, yoff, zoff = (get_component(pds, "positionOffset/" + ax)
@@ -240,7 +246,13 @@
# mylog.info("read particles ftype %s, fname %s, grid %s", ftype, fname, grid)
# # data = self._read_particles(grid, fname)
# # rv[ftype, fname] = np.concatenate((data, rv[ftype, fname]))
+ #
# return rv
+ # """
+ # From Exo 2
+ # for g in chunk.objs:
+ # ind += g.select(selector, data, rv[field], ind)
+ # """
def _read_fluid_selection(self, chunks, selector, fields, size):
"""
@@ -291,6 +303,8 @@
#f.close()
# return rv
+
+
if size is None:
size = sum((g.count(selector) for chunk in chunks
for g in chunk.objs))
diff -r 44aba12da271ecffa9bc1075fe59c842c67e7165 -r 402b24b268273ef744dff573406431b96e685b4c yt/frontends/openPMD/misc.py
--- a/yt/frontends/openPMD/misc.py
+++ b/yt/frontends/openPMD/misc.py
@@ -767,3 +767,43 @@
result_array += test_component(dset, v)
return(result_array)
+
+def is_const_component(record_component):
+ return ("value" in record_component.attrs.keys())
+
+def get_component(group, component_name):
+ mylog.info("openPMD - misc - get_component: {},{}".format(group, component_name))
+ record_component = group[component_name]
+ if is_const_component(record_component):
+ return record_component.attrs["value"]
+ else:
+ return record_component.value
+
+
+def parse_unitDimension(unitDimension):
+ if len(unitDimension) is not 7:
+ mylog.error("SI must have 7 base dimensions! {} is off by {}".format(unitDimension, len(unitDimension) - 7))
+ str = []
+ # oPMD:
+ # length L,
+ # mass M,
+ # time T,
+ # electric current I,
+ # thermodynamic temperature theta,
+ # amount of substance N,
+ # luminous intensity J
+ if unitDimension[0] < 0.0 or unitDimension[0] > 0.0:
+ str.append("m**{}".format(unitDimension[0]))
+ if unitDimension[1] < 0.0 or unitDimension[1] > 0.0:
+ str.append("kg**{}".format(unitDimension[1]))
+ if unitDimension[2] < 0.0 or unitDimension[2] > 0.0:
+ str.append("s**{}".format(unitDimension[2]))
+ if unitDimension[3] < 0.0 or unitDimension[3] > 0.0:
+ str.append("A**{}".format(unitDimension[3]))
+ if unitDimension[4] < 0.0 or unitDimension[4] > 0.0:
+ str.append("C**{}".format(unitDimension[4]))
+ if unitDimension[5] < 0.0 or unitDimension[5] > 0.0:
+ str.append("mol**{}".format(unitDimension[5]))
+ if unitDimension[6] < 0.0 or unitDimension[6] > 0.0:
+ str.append("cd**{}".format(unitDimension[6]))
+ return "*".join(str)
https://bitbucket.org/yt_analysis/yt/commits/6b8ff8531830/
Changeset: 6b8ff8531830
Branch: yt
User: NTAu... at guest692.fz-rossendorf.de
Date: 2016-05-31 15:04:11+00:00
Summary: Generate known fields from file
Affected #: 2 files
diff -r 402b24b268273ef744dff573406431b96e685b4c -r 6b8ff8531830c70a8f53e4f2e10aa30474b9d1b5 yt/frontends/openPMD/fields.py
--- a/yt/frontends/openPMD/fields.py
+++ b/yt/frontends/openPMD/fields.py
@@ -18,12 +18,10 @@
from yt.funcs import mylog
import yt.utilities.physical_constants
from yt.fields.field_info_container import \
- FieldInfoContainer, \
- particle_deposition_functions, \
- particle_vector_functions, \
- standard_particle_fields
+ FieldInfoContainer
from misc import parse_unitDimension
+
def _kinetic_energy(field, data):
"""
Function to calculate new fields out of other fields
@@ -32,10 +30,7 @@
mylog.info("oPMD - fields - _kinetic_energy")
# calculate kinetic energy out of momentum
c = 2.997e8 # velocity of light
- ke = (data["particle_momentum_x"] ** 2
- + data["particle_momentum_y"] ** 2
- + data["particle_momentum_z"] ** 2) * c
- return ke
+ return (data["particle_momentum_x"]**2 + data["particle_momentum_y"]**2 + data["particle_momentum_z"]**2) * c
def setup_momentum_to_velocity(self, ptype):
@@ -82,6 +77,20 @@
function=_get_poyn(ax),
units="T*V/m") # N/(m*s)
+def setup_relative_position(self, ptype):
+ mylog.info("oPMD - fields setup_relative_position")
+ def _rel_pos(axis):
+ def rp(field, data):
+ return
+
+ return rp
+
+ for ax in 'xyz':
+ self.add_field((ptype, "particle_position_relative_%s" % ax),
+ function=_rel_pos(ax),
+ units="m")
+
+
class openPMDFieldInfo(FieldInfoContainer):
"""
We need to specify which fields we might have in our dataset. The field info
@@ -94,6 +103,7 @@
The field names have to match the names in "openPMDHierarchy" in data_structures.py
This also defines the units of the fields
"""
+ """
# TODO Generate this when parsing a file
known_other_fields = (
# Each entry here is of the form
@@ -130,34 +140,47 @@
("particle_kinetic_energy", ("dimensionless", [], None)),
)
+ """
def __init__(self, ds, field_list):
super(openPMDFieldInfo, self).__init__(ds, field_list)
# If you want, you can check field_list
mylog.info("oPMD - fields - __init__")
- other_fields = (
- # Each entry here is of the form
- # ( "name", ("units", ["fields", "to", "alias"], # "display_name")),
- # ("B_x", ("T", [], None)),
- #oPMD:
- # length L,
- # mass M,
- # time T,
- # electric current I,
- # thermodynamic temperature theta,
- # amount of substance N,
- # luminous intensity J
-
-
- )
+ other_fields = ()
f = ds._handle
- meshesPath = f.attrs["meshesPath"]
- fields = f[ds.basePath + meshesPath]
+ fields = f[ds.basePath + f.attrs["meshesPath"]]
for i in fields.keys():
field = fields.get(i)
- for i in field.attrs["axisLabels"]:
- print field, i, parse_unitDimension(np.asarray(field.attrs["unitDimension"], dtype='int'))
+ for j in field.attrs["axisLabels"]:
+ parsed = parse_unitDimension(np.asarray(field.attrs["unitDimension"], dtype='int'))
+ other_fields += ((str("_".join([i,j])), (yt.YTQuantity(1, parsed).units, [], None)),)
+ self.known_other_fields = other_fields
+ for i in self.known_other_fields:
+ mylog.info("oPMD - fields - known_other_fields - {}".format(i))
+
+ particle_fields = ()
+ particles = f[ds.basePath + f.attrs["particlesPath"]]
+ for species in particles.keys():
+ for attrib in particles.get(species).keys():
+ try:
+ if "weighting" in attrib:
+ particle_fields += (("particle_weighting", ("", [], None)),)
+ continue
+ udim = particles.get(species).get(attrib).attrs["unitDimension"]
+ parsed = parse_unitDimension(np.asarray(udim, dtype='int'))
+ for axis in particles.get(species).get(attrib).keys():
+ if axis in "rxyz":
+ particle_fields += (
+ (str("_".join(["particle", attrib, axis])), (yt.YTQuantity(1, parsed).units, [], None)),)
+ else:
+ particle_fields += (
+ (str("_".join(["particle", attrib])), (yt.YTQuantity(1, parsed).units, [], None)),)
+ except:
+ mylog.info("{}_{} does not seem to have unitDimension".format(species, attrib))
+ self.known_particle_fields = particle_fields
+ for i in self.known_particle_fields:
+ mylog.info("oPMD - fields - known_particle_fields - {}".format(i))
def setup_fluid_fields(self):
@@ -187,9 +210,11 @@
# units="dimensionless")
# setup_momentum_to_velocity(self, ptype)
+ """
particle_deposition_functions(ptype, "particle_position",
"particle_weighting", self)
standard_particle_fields(self, ptype)
+ """
# TODO Has to be called to load particles
super(openPMDFieldInfo, self).setup_particle_fields(ptype)
diff -r 402b24b268273ef744dff573406431b96e685b4c -r 6b8ff8531830c70a8f53e4f2e10aa30474b9d1b5 yt/frontends/openPMD/misc.py
--- a/yt/frontends/openPMD/misc.py
+++ b/yt/frontends/openPMD/misc.py
@@ -766,10 +766,12 @@
dset = species[ os.path.join(record, component_name) ]
result_array += test_component(dset, v)
- return(result_array)
+ return result_array
+
def is_const_component(record_component):
- return ("value" in record_component.attrs.keys())
+ return "value" in record_component.attrs.keys()
+
def get_component(group, component_name):
mylog.info("openPMD - misc - get_component: {},{}".format(group, component_name))
@@ -783,27 +785,26 @@
def parse_unitDimension(unitDimension):
if len(unitDimension) is not 7:
mylog.error("SI must have 7 base dimensions! {} is off by {}".format(unitDimension, len(unitDimension) - 7))
- str = []
- # oPMD:
- # length L,
- # mass M,
- # time T,
- # electric current I,
- # thermodynamic temperature theta,
- # amount of substance N,
- # luminous intensity J
+ dim = []
if unitDimension[0] < 0.0 or unitDimension[0] > 0.0:
- str.append("m**{}".format(unitDimension[0]))
+ # length L,
+ dim.append("m**{}".format(unitDimension[0]))
if unitDimension[1] < 0.0 or unitDimension[1] > 0.0:
- str.append("kg**{}".format(unitDimension[1]))
+ # mass M,
+ dim.append("kg**{}".format(unitDimension[1]))
if unitDimension[2] < 0.0 or unitDimension[2] > 0.0:
- str.append("s**{}".format(unitDimension[2]))
+ # time T,
+ dim.append("s**{}".format(unitDimension[2]))
if unitDimension[3] < 0.0 or unitDimension[3] > 0.0:
- str.append("A**{}".format(unitDimension[3]))
+ # electric current I,
+ dim.append("A**{}".format(unitDimension[3]))
if unitDimension[4] < 0.0 or unitDimension[4] > 0.0:
- str.append("C**{}".format(unitDimension[4]))
+ # thermodynamic temperature theta,
+ dim.append("C**{}".format(unitDimension[4]))
if unitDimension[5] < 0.0 or unitDimension[5] > 0.0:
- str.append("mol**{}".format(unitDimension[5]))
+ # amount of substance N,
+ dim.append("mol**{}".format(unitDimension[5]))
if unitDimension[6] < 0.0 or unitDimension[6] > 0.0:
- str.append("cd**{}".format(unitDimension[6]))
- return "*".join(str)
+ # luminous intensity J
+ dim.append("cd**{}".format(unitDimension[6]))
+ return "*".join(dim)
https://bitbucket.org/yt_analysis/yt/commits/0390fc45dc7e/
Changeset: 0390fc45dc7e
Branch: yt
User: NTAu... at guest692.fz-rossendorf.de
Date: 2016-06-03 08:26:48+00:00
Summary: Use VERY naive caching algorithm (won't work on general openPMD datasets yet), achieve a speedup of ~5.
get_component() now always returns shaped data
Affected #: 4 files
diff -r 6b8ff8531830c70a8f53e4f2e10aa30474b9d1b5 -r 0390fc45dc7eecd191fcaf90059908c57f70eb6d yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -373,7 +373,7 @@
else :
fshape = firstMesh[list(firstMesh.keys())[0]].shape
except :
- print("ERROR: Can only read files that have at least one mesh entry!")
+ print("ERROR: Can only read files that have at least one mesh entry!")
# Usually 2D/3D for picongpu
self.dimensionality = len(fshape)
diff -r 6b8ff8531830c70a8f53e4f2e10aa30474b9d1b5 -r 0390fc45dc7eecd191fcaf90059908c57f70eb6d yt/frontends/openPMD/fields.py
--- a/yt/frontends/openPMD/fields.py
+++ b/yt/frontends/openPMD/fields.py
@@ -19,7 +19,7 @@
import yt.utilities.physical_constants
from yt.fields.field_info_container import \
FieldInfoContainer
-from misc import parse_unitDimension
+from .misc import parse_unitDimension
def _kinetic_energy(field, data):
diff -r 6b8ff8531830c70a8f53e4f2e10aa30474b9d1b5 -r 0390fc45dc7eecd191fcaf90059908c57f70eb6d yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -44,6 +44,7 @@
self._setBasePath(self._handle, self.ds._filepath)
self.meshPath = self._handle["/"].attrs["meshesPath"]
self.particlesPath = self._handle["/"].attrs["particlesPath"]
+ self._cached_it = None
def _read_particle_coords(self, chunks, ptf):
"""
@@ -69,6 +70,14 @@
mylog.info("_read_particle_coords")
chunks = list(chunks)
self._array_fields = {}
+ # TODO consider individual fields, not species
+ if self._cached_it is not None:
+ if self._cached_it is self.basePath:
+ self._cache = True
+ # use the cached x,y,z
+ else:
+ self._cached_it = self.basePath
+ self._cache = False
for chunk in chunks:
f = None
for g in chunk.objs:
@@ -81,38 +90,47 @@
for ptype, field_list in sorted(ptf.items()):
# Inefficient for const records
mylog.info("openPMD - _read_particle_coords: {}, {}".format(ptype, field_list))
+ if self._cache:
+ mylog.info("openPMD - _read_particle_coords: using cache")
+ yield (ptype, (self._cachex, self._cachey, self._cachez))
+ else:
+ # Get a particle species (e.g. /data/3500/particles/e/)
+ if "io" in ptype:
+ spec = dds[f.attrs["particlesPath"]].keys()[0]
+ else:
+ # TODO Probably need to use ptype here
+ spec = field_list[0].split("_")[0]
+ pds = dds.get("%s/%s" % (f.attrs["particlesPath"], spec))
+ if pds is None:
+ mylog.info("openPMD - _read_particle_coords: {}/{} yields None".format(f.attrs["particlesPath"], spec))
- # Get a particle species (e.g. /data/3500/particles/e/)
- if "io" in ptype:
- spec = dds[f.attrs["particlesPath"]].keys()[0]
- else:
- spec = field_list[0].split("_")[0]
- pds = dds.get("%s/%s" % (f.attrs["particlesPath"], spec))
- if pds is None:
- mylog.info("openPMD - _read_particle_coords: {}/{} yields None".format(f.attrs["particlesPath"], spec))
+ # Get single 1D-arrays of coordinates from all particular particles
+ # TODO Do not naively assume 3D. Check which axes actually exist
+ xpos, ypos, zpos = (get_component(pds, "position/" + ax)
+ for ax in 'xyz')
+ xoff, yoff, zoff = (get_component(pds, "positionOffset/" + ax)
+ for ax in 'xyz')
+ self._cachex = xpos * pds["position/x"].attrs["unitSI"] \
+ + xoff * pds["positionOffset/x"].attrs["unitSI"]
+ self._cachey = ypos * pds["position/y"].attrs["unitSI"] \
+ + yoff * pds["positionOffset/y"].attrs["unitSI"]
+ self._cachez = zpos * pds["position/z"].attrs["unitSI"] \
+ + zoff * pds["positionOffset/z"].attrs["unitSI"]
+ self._cache = True
- # Get single 1D-arrays of coordinates from all particular particles
- # TODO Do not naively assume 3D. Check which axes actually exist
- xpos, ypos, zpos = (get_component(pds, "position/" + ax)
- for ax in 'xyz')
- xoff, yoff, zoff = (get_component(pds, "positionOffset/" + ax)
- for ax in 'xyz')
- x = xpos * pds["position/x"].attrs["unitSI"] + xoff * pds["positionOffset/x"].attrs["unitSI"]
- y = ypos * pds["position/y"].attrs["unitSI"] + yoff * pds["positionOffset/y"].attrs["unitSI"]
- z = zpos * pds["position/z"].attrs["unitSI"] + zoff * pds["positionOffset/z"].attrs["unitSI"]
- for field in field_list:
- # Trim field path (yt scheme) to match openPMD scheme
- nfield = "/".join(field.split("_")[1:])
+ for field in field_list:
+ # Trim field path (yt scheme) to match openPMD scheme
+ nfield = "/".join(field.split("_")[1:])
- # Save the size of the particle fields
- # TODO This CAN be used for speedup, not sure how
- if is_const_component(pds[nfield]):
- shape = np.asarray(pds[nfield].attrs["shape"])
- else:
- shape = np.asarray(pds[nfield].shape)
- if shape.ndim > 1:
- self._array_fields[field] = shape
- yield (ptype, (x, y, z))
+ # Save the size of the particle fields
+ # TODO This CAN be used for speedup, not sure how
+ if is_const_component(pds[nfield]):
+ shape = np.asarray(pds[nfield].attrs["shape"])
+ else:
+ shape = np.asarray(pds[nfield].shape)
+ if shape.ndim > 1:
+ self._array_fields[field] = shape
+ yield (ptype, (self._cachex, self._cachey, self._cachez))
if f:
f.close()
@@ -147,6 +165,14 @@
"""
mylog.info("_read_particle_fields")
chunks = list(chunks)
+ # TODO consider individual fields, not species
+ if self._cached_it is not None:
+ if self._cached_it is self.basePath:
+ self._cache = True
+ # use the cached x,y,z
+ else:
+ self._cached_it = self.basePath
+ self._cache = False
for chunk in chunks:
f = None
for g in chunk.objs:
@@ -164,31 +190,34 @@
if "io" in ptype:
spec = ds[f.attrs["particlesPath"]].keys()[0]
else:
+ # TODO probably need to use ptype here
spec = field_list[0].split("_")[0]
pds = ds.get("%s/%s" % (f.attrs["particlesPath"], spec))
if pds is None:
- mylog.info("openPMD - _read_particle_coords: {}/{} yields None".format(f.attrs["particlesPath"], spec))
+ mylog.info("openPMD - _read_particle_fields: {}/{} yields None".format(f.attrs["particlesPath"], spec))
- # Get single arrays of coordinates from all particular particles
- # TODO Do not naively assume 3D. Check which axes actually exist
- # (axis_labels, geometry)
- # TODO Pay attention to const records
- xpos, ypos, zpos = (get_component(pds, "position/" + ax)
- for ax in 'xyz')
- xoff, yoff, zoff = (get_component(pds, "positionOffset/" + ax)
- for ax in 'xyz')
- x = xpos * pds["position/x"].attrs["unitSI"] + xoff * pds["positionOffset/x"].attrs["unitSI"]
- y = ypos * pds["position/y"].attrs["unitSI"] + yoff * pds["positionOffset/y"].attrs["unitSI"]
- z = zpos * pds["position/z"].attrs["unitSI"] + zoff * pds["positionOffset/z"].attrs["unitSI"]
- mask = selector.select_points(x, y, z, 0.0)
+ if self._cache:
+ mylog.info("openPMD - _read_particle_fields: using cache")
+ #yield (ptype, (self._cachex, self._cachey, self._cachez))
+ else:
+ # Get single arrays of coordinates from all particular particles
+ # TODO Do not naively assume 3D. Check which axes actually exist
+ # (axis_labels, geometry)
+ # TODO Pay attention to const records
+ xpos, ypos, zpos = (get_component(pds, "position/" + ax)
+ for ax in 'xyz')
+ xoff, yoff, zoff = (get_component(pds, "positionOffset/" + ax)
+ for ax in 'xyz')
+ self._cachex = xpos * pds["position/x"].attrs["unitSI"] + xoff * pds["positionOffset/x"].attrs["unitSI"]
+ self._cachey = ypos * pds["position/y"].attrs["unitSI"] + yoff * pds["positionOffset/y"].attrs["unitSI"]
+ self._cachez = zpos * pds["position/z"].attrs["unitSI"] + zoff * pds["positionOffset/z"].attrs["unitSI"]
+ self.cache = True
+ mask = selector.select_points(self._cachex, self._cachey, self._cachez, 0.0)
if mask is None:
continue
for field in field_list:
nfield = "/".join(field.split("_")[1:])
- if is_const_component(pds[nfield]):
- data = np.full(pds[nfield].attrs["shape"], pds[nfield].attrs["value"])
- else:
- data = pds[nfield].value
+ data = get_component(pds, nfield)
yield ((ptype, field), data[mask])
if f:
f.close()
diff -r 6b8ff8531830c70a8f53e4f2e10aa30474b9d1b5 -r 0390fc45dc7eecd191fcaf90059908c57f70eb6d yt/frontends/openPMD/misc.py
--- a/yt/frontends/openPMD/misc.py
+++ b/yt/frontends/openPMD/misc.py
@@ -777,7 +777,7 @@
mylog.info("openPMD - misc - get_component: {},{}".format(group, component_name))
record_component = group[component_name]
if is_const_component(record_component):
- return record_component.attrs["value"]
+ return np.full(record_component.attrs["shape"], record_component.attrs["value"])
else:
return record_component.value
https://bitbucket.org/yt_analysis/yt/commits/039b6277269d/
Changeset: 039b6277269d
Branch: yt
User: NTAu... at guest692.fz-rossendorf.de
Date: 2016-06-03 09:20:17+00:00
Summary: get_component() return all data scaled with unitSI
Affected #: 2 files
diff -r 0390fc45dc7eecd191fcaf90059908c57f70eb6d -r 039b6277269da4f28e77ca831ef1b6c02e7c9b86 yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -110,12 +110,9 @@
for ax in 'xyz')
xoff, yoff, zoff = (get_component(pds, "positionOffset/" + ax)
for ax in 'xyz')
- self._cachex = xpos * pds["position/x"].attrs["unitSI"] \
- + xoff * pds["positionOffset/x"].attrs["unitSI"]
- self._cachey = ypos * pds["position/y"].attrs["unitSI"] \
- + yoff * pds["positionOffset/y"].attrs["unitSI"]
- self._cachez = zpos * pds["position/z"].attrs["unitSI"] \
- + zoff * pds["positionOffset/z"].attrs["unitSI"]
+ self._cachex = xpos + xoff
+ self._cachey = ypos + yoff
+ self._cachez = zpos + zoff
self._cache = True
for field in field_list:
@@ -208,9 +205,9 @@
for ax in 'xyz')
xoff, yoff, zoff = (get_component(pds, "positionOffset/" + ax)
for ax in 'xyz')
- self._cachex = xpos * pds["position/x"].attrs["unitSI"] + xoff * pds["positionOffset/x"].attrs["unitSI"]
- self._cachey = ypos * pds["position/y"].attrs["unitSI"] + yoff * pds["positionOffset/y"].attrs["unitSI"]
- self._cachez = zpos * pds["position/z"].attrs["unitSI"] + zoff * pds["positionOffset/z"].attrs["unitSI"]
+ self._cachex = xpos + xoff
+ self._cachey = ypos + yoff
+ self._cachez = zpos + zoff
self.cache = True
mask = selector.select_points(self._cachex, self._cachey, self._cachez, 0.0)
if mask is None:
diff -r 0390fc45dc7eecd191fcaf90059908c57f70eb6d -r 039b6277269da4f28e77ca831ef1b6c02e7c9b86 yt/frontends/openPMD/misc.py
--- a/yt/frontends/openPMD/misc.py
+++ b/yt/frontends/openPMD/misc.py
@@ -777,9 +777,9 @@
mylog.info("openPMD - misc - get_component: {},{}".format(group, component_name))
record_component = group[component_name]
if is_const_component(record_component):
- return np.full(record_component.attrs["shape"], record_component.attrs["value"])
+ return np.full(record_component.attrs["shape"], record_component.attrs["value"] * record_component.attrs["unitSI"])
else:
- return record_component.value
+ return record_component.value * record_component.attrs["unitSI"]
def parse_unitDimension(unitDimension):
https://bitbucket.org/yt_analysis/yt/commits/234327bd7ea5/
Changeset: 234327bd7ea5
Branch: yt
User: NTAu... at guest692.fz-rossendorf.de
Date: 2016-06-03 13:48:14+00:00
Summary: get_component() return all data scaled with unitSI,
apply horrendously stupid caching per file,
replace as much print with yt's log as possible,
create sensible relative positions,
Affected #: 4 files
diff -r 039b6277269da4f28e77ca831ef1b6c02e7c9b86 -r 234327bd7ea5f14e5d828274b746428f82fac212 yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -166,17 +166,14 @@
for record in f[self.basePath + particlesPath + particleName].keys():
if is_const_component(f[self.basePath + particlesPath + particleName + "/" + record]):
# Record itself (eg particle_mass) is constant
- mylog.info("%s", record)
particle_fields.append(particleName + "_" + record)
elif 'particlePatches' not in record:
try:
keys = f[self.basePath + particlesPath + particleName + "/" + record].keys()
for axis in keys:
- mylog.info("%s_%s", record, axis)
particle_fields.append(particleName + "_" + record + "_" + axis)
pass
except:
- mylog.info("%s", record)
particle_fields.append(particleName + "_" + record)
pass
else:
@@ -188,8 +185,6 @@
else:
self.field_list.extend(
[("io", ("particle_" + "_".join(str(c).split("_")[1:]))) for c in particle_fields])
- #self.field_list.extend([("io", c) for c in particle_fields])
- #self.field_list.extend([(str(c).split("_")[0], 'particle' + c.lstrip(c.split("_")[0])) for c in particle_fields])
def _count_grids(self):
"""
@@ -373,7 +368,7 @@
else :
fshape = firstMesh[list(firstMesh.keys())[0]].shape
except :
- print("ERROR: Can only read files that have at least one mesh entry!")
+ mylog.error("ERROR: Can only read files that have at least one mesh entry!")
# Usually 2D/3D for picongpu
self.dimensionality = len(fshape)
diff -r 039b6277269da4f28e77ca831ef1b6c02e7c9b86 -r 234327bd7ea5f14e5d828274b746428f82fac212 yt/frontends/openPMD/fields.py
--- a/yt/frontends/openPMD/fields.py
+++ b/yt/frontends/openPMD/fields.py
@@ -77,18 +77,21 @@
function=_get_poyn(ax),
units="T*V/m") # N/(m*s)
-def setup_relative_position(self, ptype):
- mylog.info("oPMD - fields setup_relative_position")
+
+def setup_relative_positions(self, ptype):
def _rel_pos(axis):
def rp(field, data):
- return
+ pos = data[ptype, "particle_position_{}".format(axis)]
+ off = data[ptype, "particle_positionOffset_{}".format(axis)]
+ return pos + off
return rp
for ax in 'xyz':
self.add_field((ptype, "particle_position_relative_%s" % ax),
function=_rel_pos(ax),
- units="m")
+ units="m",
+ particle_type=True)
class openPMDFieldInfo(FieldInfoContainer):
@@ -97,55 +100,13 @@
container subclass here will define which fields it knows about. There are
optionally methods on it that get called which can be subclassed.
- !! TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-
This class defines, which fields and particle fields could be in the HDF5-file
The field names have to match the names in "openPMDHierarchy" in data_structures.py
This also defines the units of the fields
"""
- """
- # TODO Generate this when parsing a file
- known_other_fields = (
- # Each entry here is of the form
- # ( "name", ("units", ["fields", "to", "alias"], # "display_name")),
- ("B_x", ("T", [], None)),
- ("B_y", ("T", [], None)),
- ("B_z", ("T", [], None)),
- ("E_x", ("V/m", [], None)),
- ("E_y", ("V/m", [], None)),
- ("E_z", ("V/m", [], None)),
- ("J_x", ("A/m**2", [], None)),
- ("J_y", ("A/m**2", [], None)),
- ("J_z", ("A/m**2", [], None)),
- ("rho", ("kg*s/m**3.0", [], None)),
-
- )
-
- # TODO Generate this when parsing a file
- known_particle_fields = (
- # Identical form to above
- # ( "name", ("units", ["fields", "to", "alias"], # "display_name")),
- ("particle_charge", ("A*s", [], None)),
- ("particle_mass", ("kg", [], None)),
- ("particle_momentum_x", ("kg*m/s", [], None)),
- ("particle_momentum_y", ("kg*m/s", [], None)),
- ("particle_momentum_z", ("kg*m/s", [], None)),
- ("particle_position_x", ("m", [], None)),
- ("particle_position_y", ("m", [], None)),
- ("particle_position_z", ("m", [], None)),
- ("particle_positionOffset_x", ("m", [], None)),
- ("particle_positionOffset_y", ("m", [], None)),
- ("particle_positionOffset_z", ("m", [], None)),
- ("particle_weighting", ("", [], None)),
- ("particle_kinetic_energy", ("dimensionless", [], None)),
-
- )
- """
def __init__(self, ds, field_list):
super(openPMDFieldInfo, self).__init__(ds, field_list)
- # If you want, you can check field_list
- mylog.info("oPMD - fields - __init__")
other_fields = ()
f = ds._handle
@@ -157,7 +118,7 @@
other_fields += ((str("_".join([i,j])), (yt.YTQuantity(1, parsed).units, [], None)),)
self.known_other_fields = other_fields
for i in self.known_other_fields:
- mylog.info("oPMD - fields - known_other_fields - {}".format(i))
+ mylog.debug("oPMD - fields - known_other_fields - {}".format(i))
particle_fields = ()
particles = f[ds.basePath + f.attrs["particlesPath"]]
@@ -180,7 +141,7 @@
mylog.info("{}_{} does not seem to have unitDimension".format(species, attrib))
self.known_particle_fields = particle_fields
for i in self.known_particle_fields:
- mylog.info("oPMD - fields - known_particle_fields - {}".format(i))
+ mylog.debug("oPMD - fields - known_particle_fields - {}".format(i))
def setup_fluid_fields(self):
@@ -189,7 +150,6 @@
values of new fields e.g. calculate out of E-field and B-field the
Poynting vector
"""
- mylog.info("oPMD - fields - setup_fluid_fields")
# Here we do anything that might need info about the dataset.
# You can use self.alias, self.add_output_field and self.add_field .
@@ -201,8 +161,6 @@
TODO You have to call the function of the parent class to load particles
"""
-
- mylog.info("oPMD - fields - setup_particle_fields(%s)",ptype)
# self.add_field(
# (ptype,
# "particle_kinetic_energy"),
@@ -210,11 +168,5 @@
# units="dimensionless")
# setup_momentum_to_velocity(self, ptype)
- """
- particle_deposition_functions(ptype, "particle_position",
- "particle_weighting", self)
- standard_particle_fields(self, ptype)
- """
-
- # TODO Has to be called to load particles
+ setup_relative_positions(self, ptype)
super(openPMDFieldInfo, self).setup_particle_fields(ptype)
diff -r 039b6277269da4f28e77ca831ef1b6c02e7c9b86 -r 234327bd7ea5f14e5d828274b746428f82fac212 yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -45,6 +45,7 @@
self.meshPath = self._handle["/"].attrs["meshesPath"]
self.particlesPath = self._handle["/"].attrs["particlesPath"]
self._cached_it = None
+ self._cache = Cache()
def _read_particle_coords(self, chunks, ptf):
"""
@@ -67,17 +68,16 @@
ptype_name is a type pf particle,
x_coords, y_coords and z_coords are arrays of positions/coordinates of all particles of that type
"""
- mylog.info("_read_particle_coords")
chunks = list(chunks)
self._array_fields = {}
# TODO consider individual fields, not species
if self._cached_it is not None:
if self._cached_it is self.basePath:
- self._cache = True
+ self._use_cache = True
# use the cached x,y,z
else:
self._cached_it = self.basePath
- self._cache = False
+ self._use_cache = False
for chunk in chunks:
f = None
for g in chunk.objs:
@@ -89,9 +89,8 @@
dds = f[self.basePath]
for ptype, field_list in sorted(ptf.items()):
# Inefficient for const records
- mylog.info("openPMD - _read_particle_coords: {}, {}".format(ptype, field_list))
- if self._cache:
- mylog.info("openPMD - _read_particle_coords: using cache")
+ mylog.debug("openPMD - _read_particle_coords: {}, {}".format(ptype, field_list))
+ if self._use_cache:
yield (ptype, (self._cachex, self._cachey, self._cachez))
else:
# Get a particle species (e.g. /data/3500/particles/e/)
@@ -106,14 +105,14 @@
# Get single 1D-arrays of coordinates from all particular particles
# TODO Do not naively assume 3D. Check which axes actually exist
- xpos, ypos, zpos = (get_component(pds, "position/" + ax)
+ xpos, ypos, zpos = (self._cache.get_component(pds, "position/" + ax)
for ax in 'xyz')
- xoff, yoff, zoff = (get_component(pds, "positionOffset/" + ax)
+ xoff, yoff, zoff = (self._cache.get_component(pds, "positionOffset/" + ax)
for ax in 'xyz')
self._cachex = xpos + xoff
self._cachey = ypos + yoff
self._cachez = zpos + zoff
- self._cache = True
+ self._use_cache = True
for field in field_list:
# Trim field path (yt scheme) to match openPMD scheme
@@ -165,11 +164,11 @@
# TODO consider individual fields, not species
if self._cached_it is not None:
if self._cached_it is self.basePath:
- self._cache = True
+ self._use_cache = True
# use the cached x,y,z
else:
self._cached_it = self.basePath
- self._cache = False
+ self._use_cache = False
for chunk in chunks:
f = None
for g in chunk.objs:
@@ -181,6 +180,7 @@
ds = f[self.basePath]
for ptype, field_list in sorted(ptf.items()):
# Inefficient for const records
+ mylog.debug("openPMD - _read_particle_fields: {}, {}".format(ptype, field_list))
# Get a particle species (e.g. /data/3500/particles/e/)
# TODO Do this for all fields in fields list
@@ -193,28 +193,25 @@
if pds is None:
mylog.info("openPMD - _read_particle_fields: {}/{} yields None".format(f.attrs["particlesPath"], spec))
- if self._cache:
- mylog.info("openPMD - _read_particle_fields: using cache")
- #yield (ptype, (self._cachex, self._cachey, self._cachez))
- else:
+ if not self._use_cache:
# Get single arrays of coordinates from all particular particles
# TODO Do not naively assume 3D. Check which axes actually exist
# (axis_labels, geometry)
# TODO Pay attention to const records
- xpos, ypos, zpos = (get_component(pds, "position/" + ax)
+ xpos, ypos, zpos = (self._cache.get_component(pds, "position/" + ax)
for ax in 'xyz')
- xoff, yoff, zoff = (get_component(pds, "positionOffset/" + ax)
+ xoff, yoff, zoff = (self._cache.get_component(pds, "positionOffset/" + ax)
for ax in 'xyz')
self._cachex = xpos + xoff
self._cachey = ypos + yoff
self._cachez = zpos + zoff
- self.cache = True
+ self._use_cache = True
mask = selector.select_points(self._cachex, self._cachey, self._cachez, 0.0)
if mask is None:
continue
for field in field_list:
nfield = "/".join(field.split("_")[1:])
- data = get_component(pds, nfield)
+ data = self._cache.get_component(pds, nfield)
yield ((ptype, field), data[mask])
if f:
f.close()
diff -r 039b6277269da4f28e77ca831ef1b6c02e7c9b86 -r 234327bd7ea5f14e5d828274b746428f82fac212 yt/frontends/openPMD/misc.py
--- a/yt/frontends/openPMD/misc.py
+++ b/yt/frontends/openPMD/misc.py
@@ -206,7 +206,7 @@
else:
result_array = np.array([0,0])
else:
- print(
+ mylog.error(
"Error: Attribute %s in `%s` is not of type '%s' (is '%s')!" \
%(name, str(f.name), str(is_type_names), \
type(value).__name__) )
@@ -509,7 +509,7 @@
list_meshes = list(f[full_meshes_path].keys())
except KeyError:
list_meshes = []
- print( "Iteration %s : found %d meshes"
+ mylog.debug( "Iteration %s : found %d meshes"
%( iteration, len(list_meshes) ) )
# Check for the attributes of the STANDARD.md
@@ -658,7 +658,7 @@
list_species = list(f[full_particle_path].keys())
except KeyError:
list_species = []
- print( "Iteration %s : found %d particle species"
+ mylog.debug( "Iteration %s : found %d particle species"
%( iteration, len(list_species) ) )
# Go through all the particle species
@@ -773,13 +773,21 @@
return "value" in record_component.attrs.keys()
-def get_component(group, component_name):
- mylog.info("openPMD - misc - get_component: {},{}".format(group, component_name))
- record_component = group[component_name]
- if is_const_component(record_component):
- return np.full(record_component.attrs["shape"], record_component.attrs["value"] * record_component.attrs["unitSI"])
- else:
- return record_component.value * record_component.attrs["unitSI"]
+class Cache():
+
+ _c = {}
+
+ def get_component(self, group, component_name):
+ record_component = group[component_name]
+ mylog.debug("openPMD - misc - get_component: {}".format(record_component.name))
+ if record_component.name not in self._c:
+ if is_const_component(record_component):
+ self._c[record_component.name] = \
+ np.full(record_component.attrs["shape"],
+ record_component.attrs["value"] * record_component.attrs["unitSI"])
+ else:
+ self._c[record_component.name] = record_component.value * record_component.attrs["unitSI"]
+ return self._c[record_component.name]
def parse_unitDimension(unitDimension):
https://bitbucket.org/yt_analysis/yt/commits/5be12201f538/
Changeset: 5be12201f538
Branch: yt
User: NTAu... at guest692.fz-rossendorf.de
Date: 2016-06-03 13:50:30+00:00
Summary: OS X specific ignore
Affected #: 1 file
diff -r 234327bd7ea5f14e5d828274b746428f82fac212 -r 5be12201f5389a093cb47b9a965bda012d003534 .hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -85,3 +85,4 @@
doc/_temp/*
doc/source/bootcamp/.ipynb_checkpoints/
dist
+*.DS_Store
https://bitbucket.org/yt_analysis/yt/commits/bae2b89d3747/
Changeset: bae2b89d3747
Branch: yt
User: NTAu... at guest692.fz-rossendorf.de
Date: 2016-06-10 08:35:30+00:00
Summary: Integrate relevant part of https://bitbucket.org/yt_analysis/yt/pull-requests/2213/mks-code-unit-system-fixes-closes-1229/diff to check compatibility with openPMD frontend
Affected #: 5 files
diff -r 5be12201f5389a093cb47b9a965bda012d003534 -r bae2b89d374745692a833698bf612ac3c30c0587 yt/fields/field_info_container.py
--- a/yt/fields/field_info_container.py
+++ b/yt/fields/field_info_container.py
@@ -357,15 +357,15 @@
for field in fields_to_check:
mylog.debug("Checking %s", field)
if field not in self: raise RuntimeError
- fi = self[field]
+ fi = self[field]f
try:
fd = fi.get_dependencies(ds = self.ds)
except Exception as e:
if field in self._show_field_errors:
raise
if type(e) != YTFieldNotFound:
- mylog.debug("Raises %s during field %s detection.",
- str(type(e)), field)
+ mylog.debug("Raises %s (%) during field %s detection.",
+ str(type(e)), e, field)
self.pop(field)
continue
# This next bit checks that we can't somehow generate everything.
diff -r 5be12201f5389a093cb47b9a965bda012d003534 -r bae2b89d374745692a833698bf612ac3c30c0587 yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -80,20 +80,18 @@
raise openPMDBasePathException("openPMD: no iterations found!")
# just handle the first iteration found
- mylog.warning("openPMD: only choose to load the first iteration ({})".format(handle[dataPath].keys()[0]))
+ mylog.warning("openPMD: only choose to load one iteration ({})".format(handle[dataPath].keys()[0]))
self.basePath = "{}/{}/".format(dataPath, handle[dataPath].keys()[0])
-
-
class openPMDGrid(AMRGridPatch):
+ # TODO THIS IS NOT TRUE FOR THE YT SENSE OF "GRID"
"""
This class defines the characteristics of the grids
- Actually there is only one grid for the whole simolation box
+ Actually there is only one grid for the whole simulation box
"""
_id_offset = 0
__slots__ = ["_level_id"]
-
def __init__(self, id, index, level=-1):
AMRGridPatch.__init__(self, id, filename=index.index_filename,
index=index)
@@ -158,31 +156,33 @@
def is_const_component(record_component):
return ("value" in record_component.attrs.keys())
- # TODO Pay attention to scalars
particle_fields = []
- # WHY would this equal false?
if self.basePath + particlesPath in f:
for particleName in f[self.basePath + particlesPath].keys():
for record in f[self.basePath + particlesPath + particleName].keys():
if is_const_component(f[self.basePath + particlesPath + particleName + "/" + record]):
- # Record itself (eg particle_mass) is constant
+ # Record itself (e.g. particle_mass) is constant
particle_fields.append(particleName + "_" + record)
elif 'particlePatches' not in record:
try:
+ # Create a field for every axis (x,y,z) of every property (position)
+ # of every species (electrons)
keys = f[self.basePath + particlesPath + particleName + "/" + record].keys()
for axis in keys:
particle_fields.append(particleName + "_" + record + "_" + axis)
- pass
except:
+ # Record is a dataset, does not have axes (e.g. weighting)
particle_fields.append(particleName + "_" + record)
pass
else:
# We probably do not want particlePatches as accessible field lists
pass
if len(f[self.basePath + particlesPath].keys()) > 1:
+ # There is more than one particle species, use the specific names as field types
self.field_list.extend(
[(str(c).split("_")[0], ("particle_" + "_".join(str(c).split("_")[1:]))) for c in particle_fields])
else:
+ # Only one particle species, fall back to "io"
self.field_list.extend(
[("io", ("particle_" + "_".join(str(c).split("_")[1:]))) for c in particle_fields])
@@ -287,8 +287,6 @@
storage_filename=None,
units_override=None,
unit_system="mks"):
- # Opens a HDF5 file and stores its file handle in _handle
- # All _handle objects refers to the file
self._handle = HDF5FileHandler(filename)
self._filepath = os.path.dirname(filename)
self._setBasePath(self._handle, self._filepath)
@@ -299,12 +297,11 @@
self.fluid_types += ('openPMD',)
parts = tuple(str(c) for c in self._handle[self.basePath + self._handle.attrs["particlesPath"]].keys())
if len(parts) > 1:
+ # Only use infile particle names if there is more than one species
self.particle_types = parts
- mylog.info("openPMD - self.particle_types: {}".format(self.particle_types))
+ mylog.debug("openPMD - self.particle_types: {}".format(self.particle_types))
self.particle_types_raw = self.particle_types
self.particle_types = tuple(self.particle_types)
- #self.particle_types += ('all',)
- #self.particle_types = ["io", "all"]
self.particle_types = tuple(self.particle_types)
@@ -318,11 +315,6 @@
self:
A reference to self
"""
- # This is where quantities are created that represent the various
- # on-disk units. These are the currently available quantities which
- # should be set, along with examples of how to set them to standard
- # values.
- #
self.length_unit = self.quan(1.0, "m")
self.mass_unit = self.quan(1.0, "kg")
self.time_unit = self.quan(1.0, "s")
@@ -356,22 +348,30 @@
# TODO At this point one assumes the whole file/simulation
# contains for all mesh records the same dimensionality and shapes
# TODO This probably won't work for const records
- # TODO Support particle-only files
- # pick first field
- try :
- firstIteration = list(f["/data/"].keys())[0]
- meshes = f["/data/" + str(firstIteration) + "/" + meshesPath]
- firstMeshName = list(meshes.keys())[0]
- firstMesh = meshes[firstMeshName]
- if type(firstMesh) == h5py.Dataset :
- fshape = firstMesh.shape
- else :
- fshape = firstMesh[list(firstMesh.keys())[0]].shape
- except :
- mylog.error("ERROR: Can only read files that have at least one mesh entry!")
+ if len(f[self.basePath + meshesPath].keys()) > 0:
+ # There is at least one field, check its dimensionality
+ dim_mesh = max(
+ len(f[self.basePath + meshesPath + "/" + mesh].attrs["axisLabel"])
+ for mesh in f[self.basePath + meshesPath].keys())
+ if len(f[self.basePath + particlesPath].keys()) > 0:
+ # There is at least one particle species, check the dimensionality
+ # TODO
+ dim_part = 0
+ if dim_mesh < 1 and dim_part < 1:
+ mylog.error("Your data does not seem to have dimensionality!")
+ self.dimensionality = max(dim_mesh, dim_part)
+ # try :
+ # firstIteration = list(f["/data/"].keys())[0]
+ # meshes = f["/data/" + str(firstIteration) + "/" + meshesPath]
+ # firstMeshName = list(meshes.keys())[0]
+ # firstMesh = meshes[firstMeshName]
+ # if type(firstMesh) == h5py.Dataset :
+ # fshape = firstMesh.shape
+ # else :
+ # fshape = firstMesh[list(firstMesh.keys())[0]].shape
# Usually 2D/3D for picongpu
- self.dimensionality = len(fshape)
+ # self.dimensionality = len(fshape)
# TODO fill me with actual start and end positions in reasonable units
self.domain_left_edge = np.zeros(3, dtype=np.float64)
@@ -392,18 +392,13 @@
self.refine_by = 1
# Not a cosmological simulation
- self.cosmological_simulation = 0 # <= int, 0 or 1
- # Not necessary to set these according to Axel
- # self.current_redshift = 0 # <= float
- # self.omega_lambda = 0 # <= float
- # self.omega_matter = 0 # <= float
- # self.hubble_constant = 0 # <= float
+ self.cosmological_simulation = 0
@classmethod
def _is_valid(self, *args, **kwargs):
"""
- This function test if the (with yt.load()) a file could be opened with
- this frontend
+ Checks whether the supplied file adheres to the required openPMD standards
+ and thus can be read by this frontend
"""
try:
f = validator.open_file(args[0])
diff -r 5be12201f5389a093cb47b9a965bda012d003534 -r bae2b89d374745692a833698bf612ac3c30c0587 yt/frontends/openPMD/fields.py
--- a/yt/frontends/openPMD/fields.py
+++ b/yt/frontends/openPMD/fields.py
@@ -16,10 +16,11 @@
import numpy as np
from yt.funcs import mylog
-import yt.utilities.physical_constants
+from yt.utilities.physical_constants import speed_of_light
from yt.fields.field_info_container import \
FieldInfoContainer
from .misc import parse_unitDimension
+import yt
def _kinetic_energy(field, data):
@@ -27,32 +28,36 @@
Function to calculate new fields out of other fields
if a error occurs the field is not calculated
"""
- mylog.info("oPMD - fields - _kinetic_energy")
+ #mylog.info("oPMD - fields - _kinetic_energy")
# calculate kinetic energy out of momentum
c = 2.997e8 # velocity of light
return (data["particle_momentum_x"]**2 + data["particle_momentum_y"]**2 + data["particle_momentum_z"]**2) * c
-def setup_momentum_to_velocity(self, ptype):
- """
- TODO This function does no work !!
- """
+def setup_velocity(self, ptype):
mylog.info("oPMD - fields - setup_momentum_to_velocity")
def _get_vel(axis):
def velocity(field, data):
- c = 2.997e8
- moment = data[ptype, "particle_momentum_%" % axis]
- return moment / ((data[ptype, "particle_mass"] * data[ptype, "particle_weighting"]) ** 2 + (moment ** 2) / (c ** 2)) ** 0.5
+ c = speed_of_light
+ momentum = data[ptype, "particle_momentum_{}".format(axis)]
+ mass = data[ptype, "particle_mass"]
+ weighting = data[ptype, "particle_weighting"]
+ return momentum / (
+ (mass * weighting)**2 +
+ (momentum**2) / (c**2)
+ ) ** 0.5
+
return velocity
for ax in 'xyz':
+ mylog.info("oPMD - fields - setup_momentum_to_velocity - particle_velocity_{}".format(ax))
self.add_field((ptype, "particle_velocity_%s" % ax),
function=_get_vel(ax),
- units="m/s")
+ units="m/s",
+ particle_type=True)
def setup_poynting_vector(self):
- mylog.info("oPMD - fields - setup_poynting_vector")
def _get_poyn(axis):
def poynting(field, data):
Efieldx = data["E_x"]
@@ -72,6 +77,7 @@
return u * (Efieldx * Bfieldy - Efieldy * Bfieldx)
return poynting
+
for ax in 'xyz':
self.add_field(("openPMD", "poynting_vector_%s" % ax),
function=_get_poyn(ax),
@@ -115,7 +121,7 @@
field = fields.get(i)
for j in field.attrs["axisLabels"]:
parsed = parse_unitDimension(np.asarray(field.attrs["unitDimension"], dtype='int'))
- other_fields += ((str("_".join([i,j])), (yt.YTQuantity(1, parsed).units, [], None)),)
+ other_fields += ((str("_".join([i.replace("_","-"),j])), (yt.YTQuantity(1, parsed).units, [], None)),)
self.known_other_fields = other_fields
for i in self.known_other_fields:
mylog.debug("oPMD - fields - known_other_fields - {}".format(i))
@@ -154,6 +160,9 @@
# You can use self.alias, self.add_output_field and self.add_field .
setup_poynting_vector(self)
+ from yt.fields.magnetic_field import \
+ setup_magnetic_field_aliases
+ setup_magnetic_field_aliases(self, "openPMD", ["B_%s" % ax for ax in "xyz"])
def setup_particle_fields(self, ptype):
"""
@@ -166,7 +175,6 @@
# "particle_kinetic_energy"),
# function=_kinetic_energy,
# units="dimensionless")
- # setup_momentum_to_velocity(self, ptype)
-
+ setup_velocity(self, ptype)
setup_relative_positions(self, ptype)
super(openPMDFieldInfo, self).setup_particle_fields(ptype)
diff -r 5be12201f5389a093cb47b9a965bda012d003534 -r bae2b89d374745692a833698bf612ac3c30c0587 yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -42,10 +42,11 @@
self.ds = ds
self._handle = ds._handle
self._setBasePath(self._handle, self.ds._filepath)
- self.meshPath = self._handle["/"].attrs["meshesPath"]
+ self.meshesPath = self._handle["/"].attrs["meshesPath"]
self.particlesPath = self._handle["/"].attrs["particlesPath"]
- self._cached_it = None
- self._cache = Cache()
+ self._cached_ptype = ""
+ self._cache = {}
+ self._component_getter = ComponentGetter()
def _read_particle_coords(self, chunks, ptf):
"""
@@ -59,8 +60,8 @@
ptf:
A dictionary
- - keys are ptype
- - values are lists of (paricle?) fields
+ - keys are ptypes
+ - values are lists of particle fields
Yields
------
@@ -70,49 +71,40 @@
"""
chunks = list(chunks)
self._array_fields = {}
- # TODO consider individual fields, not species
- if self._cached_it is not None:
- if self._cached_it is self.basePath:
- self._use_cache = True
- # use the cached x,y,z
- else:
- self._cached_it = self.basePath
- self._use_cache = False
for chunk in chunks:
- f = None
for g in chunk.objs:
if g.filename is None:
continue
- if f is None:
- f = h5py.File(g.filename, "r")
+ f = self._handle
dds = f[self.basePath]
for ptype, field_list in sorted(ptf.items()):
- # Inefficient for const records
mylog.debug("openPMD - _read_particle_coords: {}, {}".format(ptype, field_list))
- if self._use_cache:
- yield (ptype, (self._cachex, self._cachey, self._cachez))
+ if ptype in self._cached_ptype:
+ # use the cached x,y,z
+ yield (ptype, (self._cache['x'], self._cache['y'], self._cache['z']))
else:
# Get a particle species (e.g. /data/3500/particles/e/)
if "io" in ptype:
- spec = dds[f.attrs["particlesPath"]].keys()[0]
+ spec = dds[self.particlesPath].keys()[0]
else:
- # TODO Probably need to use ptype here
- spec = field_list[0].split("_")[0]
- pds = dds.get("%s/%s" % (f.attrs["particlesPath"], spec))
- if pds is None:
- mylog.info("openPMD - _read_particle_coords: {}/{} yields None".format(f.attrs["particlesPath"], spec))
+ spec = ptype
+ pds = dds[self.particlesPath + "/" + spec]
# Get single 1D-arrays of coordinates from all particular particles
- # TODO Do not naively assume 3D. Check which axes actually exist
- xpos, ypos, zpos = (self._cache.get_component(pds, "position/" + ax)
- for ax in 'xyz')
- xoff, yoff, zoff = (self._cache.get_component(pds, "positionOffset/" + ax)
- for ax in 'xyz')
- self._cachex = xpos + xoff
- self._cachey = ypos + yoff
- self._cachez = zpos + zoff
- self._use_cache = True
+ axes = [str(ax) for ax in pds["position"].keys()]
+ for i in 'xyz':
+ if i not in axes:
+ # TODO Get 2D particle data working
+ mylog.error("openPMD - Can not handle 2D particle data for the moment!")
+ continue
+ pos = {}
+ off = {}
+ for ax in axes:
+ pos[ax] = self._component_getter.get_component(pds, "position/" + ax)
+ off[ax] = self._component_getter.get_component(pds, "positionOffset/" + ax)
+ self._cache[ax] = pos[ax] + off[ax]
+ self._cached_ptype = ptype
for field in field_list:
# Trim field path (yt scheme) to match openPMD scheme
@@ -126,9 +118,7 @@
shape = np.asarray(pds[nfield].shape)
if shape.ndim > 1:
self._array_fields[field] = shape
- yield (ptype, (self._cachex, self._cachey, self._cachez))
- if f:
- f.close()
+ yield (ptype, (self._cache['x'], self._cache['y'], self._cache['z']))
def _read_particle_fields(self, chunks, ptf, selector):
"""
@@ -159,62 +149,48 @@
- a particle type p
- in a specified field
"""
- mylog.info("_read_particle_fields")
chunks = list(chunks)
- # TODO consider individual fields, not species
- if self._cached_it is not None:
- if self._cached_it is self.basePath:
- self._use_cache = True
- # use the cached x,y,z
- else:
- self._cached_it = self.basePath
- self._use_cache = False
for chunk in chunks:
- f = None
for g in chunk.objs:
if g.filename is None:
continue
- if f is None:
- f = h5py.File(u(g.filename), 'r')
+ f = self._handle
ds = f[self.basePath]
for ptype, field_list in sorted(ptf.items()):
- # Inefficient for const records
mylog.debug("openPMD - _read_particle_fields: {}, {}".format(ptype, field_list))
# Get a particle species (e.g. /data/3500/particles/e/)
- # TODO Do this for all fields in fields list
if "io" in ptype:
- spec = ds[f.attrs["particlesPath"]].keys()[0]
+ spec = ds[self.particlesPath].keys()[0]
else:
- # TODO probably need to use ptype here
- spec = field_list[0].split("_")[0]
- pds = ds.get("%s/%s" % (f.attrs["particlesPath"], spec))
- if pds is None:
- mylog.info("openPMD - _read_particle_fields: {}/{} yields None".format(f.attrs["particlesPath"], spec))
+ spec = ptype
+ pds = ds[self.particlesPath + "/" + spec]
- if not self._use_cache:
- # Get single arrays of coordinates from all particular particles
- # TODO Do not naively assume 3D. Check which axes actually exist
- # (axis_labels, geometry)
- # TODO Pay attention to const records
- xpos, ypos, zpos = (self._cache.get_component(pds, "position/" + ax)
- for ax in 'xyz')
- xoff, yoff, zoff = (self._cache.get_component(pds, "positionOffset/" + ax)
- for ax in 'xyz')
- self._cachex = xpos + xoff
- self._cachey = ypos + yoff
- self._cachez = zpos + zoff
- self._use_cache = True
- mask = selector.select_points(self._cachex, self._cachey, self._cachez, 0.0)
+ if ptype not in self._cached_ptype:
+ # Get single 1D-arrays of coordinates from all particular particles
+ axes = [str(ax) for ax in pds["position"].keys()]
+ for i in 'xyz':
+ if i not in axes:
+ # TODO Get 2D particle data working
+ mylog.error("openPMD - Can not handle 2D particle data for the moment!")
+ continue
+ pos = {}
+ off = {}
+ for ax in axes:
+ pos[ax] = self._component_getter.get_component(pds, "position/" + ax)
+ off[ax] = self._component_getter.get_component(pds, "positionOffset/" + ax)
+ self._cache[ax] = pos[ax] + off[ax]
+ self._cached_ptype = ptype
+
+ mask = selector.select_points(self._cache['x'], self._cache['y'], self._cache['z'], 0.0)
if mask is None:
continue
for field in field_list:
nfield = "/".join(field.split("_")[1:])
- data = self._cache.get_component(pds, nfield)
+ data = self._component_getter.get_component(pds, nfield)
yield ((ptype, field), data[mask])
- if f:
- f.close()
+
# def _read_particle_selection(self, chunks, selector, fields):
# """
@@ -256,25 +232,29 @@
# if selector.__class__.__name__ == "GridSelector":
# if not (len(chunks) == len(chunks[0].objs) == 1):
# raise RuntimeError
- # grid = chunks[0].objs[0]
- # for ftype, fname in fields:
- # mylog.info("read particles ftype %s, fname %s, grid %s", ftype, fname, grid)
- # # rv[ftype, fname] = self._read_particles(grid, fname)
- # return rv
#
# rv = {f: np.array([]) for f in fields}
- # for chunk in chunks:
- # for grid in chunk.objs:
- # for ftype, fname in fields:
- # mylog.info("read particles ftype %s, fname %s, grid %s", ftype, fname, grid)
- # # data = self._read_particles(grid, fname)
- # # rv[ftype, fname] = np.concatenate((data, rv[ftype, fname]))
+ # for field in fields:
+ # ftype, fname = field
+ # for chunk in chunks:
+ # for g in chunk.objs:
+ # mylog.debug("g.select({}, {}, rv[{}], {})".format(selector, data, field,ind))
+ # rv.update(self._read_chunk_data(chunk, fields))
+ # return rv
+ # # for chunk in chunks:
+ # # for i in self._read_chunk_data(chunk,fields):
+ # # mylog.info("self._read_chunk_data({},{}):{}".format(chunk,fields,i))
#
- # return rv
+ # # for grid in chunk.objs:
+ # # for ftype, fname in fields:
+ # # data = self._read_data((ftype, fname))
+ # # rv[ftype, fname] = np.concatenate((data, rv[ftype, fname]))
+ #
# """
# From Exo 2
- # for g in chunk.objs:
- # ind += g.select(selector, data, rv[field], ind)
+ # for field in fields:
+ # for g in chunk.objs:
+ # ind += g.select(selector, data, rv[field], ind)
# """
def _read_fluid_selection(self, chunks, selector, fields, size):
@@ -346,33 +326,36 @@
# f.close()
return rv
- # TODO We *probably* do not need this function
- # def _read_data(self, field):
- # """
- # Reads data from file belonging to a (mesh or particle) field
- #
- # Parameters
- # ----------
- # field:
- # Field to get the data for
- #
- # Returns
- # -------
- # A flat numpy array
- # """
- # mylog.debug("Read data")
- #
- # if field.startswith("particle"):
- # # particles are not loaded here
- # # data =
- # # self._handle[self.basePath+self.particlesPath+field.replace("_","/")]
- # pass
- # else:
- # data = self._handle[
- # self.basePath +
- # self.meshPath +
- # field.replace("_", "/")]
- # return np.array(data).flatten()
+ def _read_data(self, field):
+ """
+ Reads data from file belonging to a (mesh or particle) field
+
+ Parameters
+ ----------
+ fieldname:
+ Field to get the data for
+
+ Returns
+ -------
+ A flat numpy array
+ """
+ ftype, fname = field
+ mylog.debug("openPMD - _read_data - reading field: {}.{}".format(ftype, fname))
+
+ if ftype in self.ds.particle_types:
+ if "io" in ftype:
+ # There is only one particle species, use that one ("io" does not suffice)
+ spec = self._handle[self.basePath + self.particlesPath].keys()[0]
+ else:
+ # Use the provided ftype that corresponds to the species name in the hdf5 file
+ spec = str(ftype)
+ ds = self._handle[self.basePath + self.particlesPath].get(spec)
+ nfield = "/".join(fname.split("_")[1:]).replace("-","_")
+ else:
+ ds = self._handle[self.basePath + self.meshesPath]
+ nfield = fname.replace("_", "/").replace("-","_")
+ data = self._component_getter.get_component(ds, nfield)
+ return np.array(data).flatten()
def _read_chunk_data(self, chunk, fields):
"""
@@ -397,16 +380,16 @@
- keys are (ftype, fname) field tuples
- values are flat numpy arrays with data form that field
"""
- mylog.info("_read_chunk_data")
+ mylog.debug("_read_chunk_data")
rv = {}
fluid_fields, particle_fields = [], []
for ftype, fname in fields:
- # TODO generate particle_types from file itself
if ftype in self.ds.particle_types:
particle_fields.append((ftype, fname))
else:
fluid_fields.append((ftype, fname))
if len(particle_fields) > 0:
+ mylog.debug("reading particle fields: {}".format(particle_fields))
selector = AlwaysSelector(self.ds)
rv.update(self._read_particle_selection(
[chunk], selector, particle_fields))
@@ -424,7 +407,8 @@
for g in grids:
for ftype, fname in fluid_fields:
# TODO update basePath for different files
- data = f[self.basePath + self.meshPath + fname.replace("_", "/").replace("-","_")]
+ data = self._read_data((ftype, fname))
+ # data = f[self.basePath + self.meshPath + fname.replace("_", "/").replace("-","_")]
rv[(ftype,fname)] = np.array(data).flatten()
f.close()
return rv
diff -r 5be12201f5389a093cb47b9a965bda012d003534 -r bae2b89d374745692a833698bf612ac3c30c0587 yt/frontends/openPMD/misc.py
--- a/yt/frontends/openPMD/misc.py
+++ b/yt/frontends/openPMD/misc.py
@@ -773,7 +773,7 @@
return "value" in record_component.attrs.keys()
-class Cache():
+class ComponentGetter():
_c = {}
https://bitbucket.org/yt_analysis/yt/commits/78cfb1ce714b/
Changeset: 78cfb1ce714b
Branch: yt
User: NTAu... at guest692.fz-rossendorf.de
Date: 2016-06-10 12:21:52+00:00
Summary: Trying to get the fields in known_other_fields to not be dimensionless after deriving a field list
Affected #: 4 files
diff -r bae2b89d374745692a833698bf612ac3c30c0587 -r 78cfb1ce714bc1e9118b881ce01d090b1cd8db2c yt/fields/field_info_container.py
--- a/yt/fields/field_info_container.py
+++ b/yt/fields/field_info_container.py
@@ -357,7 +357,7 @@
for field in fields_to_check:
mylog.debug("Checking %s", field)
if field not in self: raise RuntimeError
- fi = self[field]f
+ fi = self[field]
try:
fd = fi.get_dependencies(ds = self.ds)
except Exception as e:
diff -r bae2b89d374745692a833698bf612ac3c30c0587 -r 78cfb1ce714bc1e9118b881ce01d090b1cd8db2c yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -315,6 +315,8 @@
self:
A reference to self
"""
+ # We hardcode these to 1.0 since every dataset can have different code <-> physical scalings
+ # We get the actual unit by multiplying with "unitSI" when actually getting our data
self.length_unit = self.quan(1.0, "m")
self.mass_unit = self.quan(1.0, "kg")
self.time_unit = self.quan(1.0, "s")
diff -r bae2b89d374745692a833698bf612ac3c30c0587 -r 78cfb1ce714bc1e9118b881ce01d090b1cd8db2c yt/frontends/openPMD/fields.py
--- a/yt/frontends/openPMD/fields.py
+++ b/yt/frontends/openPMD/fields.py
@@ -84,6 +84,32 @@
units="T*V/m") # N/(m*s)
+def setup_magnetic(self):
+ def _get_mag(axis):
+ def magnetic(field, data):
+ return data["B_{}".format(axis)]
+
+ return magnetic
+
+ for ax in 'xyz':
+ self.add_field(("openPMD", "magnetic_field_%s" % ax),
+ function=_get_mag(ax),
+ units="T")
+
+
+def setup_electrical(self):
+ def _get_el(axis):
+ def electrical(field, data):
+ return data["E_{}".format(axis)]
+
+ return electrical
+
+ for ax in 'xyz':
+ self.add_field(("openPMD", "electrical_field_%s" % ax),
+ function=_get_el(ax),
+ units="kg*m/(A*s**3)")
+
+
def setup_relative_positions(self, ptype):
def _rel_pos(axis):
def rp(field, data):
@@ -160,9 +186,13 @@
# You can use self.alias, self.add_output_field and self.add_field .
setup_poynting_vector(self)
+ setup_magnetic(self)
+ setup_electrical(self)
from yt.fields.magnetic_field import \
setup_magnetic_field_aliases
- setup_magnetic_field_aliases(self, "openPMD", ["B_%s" % ax for ax in "xyz"])
+ for ax in "xyz":
+ mylog.info("setup_magnetic_field_aliases(self, openPMD, B_{}".format(ax))
+ setup_magnetic_field_aliases(self, "openPMD", ["magnetic_field_{}".format(ax)])
def setup_particle_fields(self, ptype):
"""
diff -r bae2b89d374745692a833698bf612ac3c30c0587 -r 78cfb1ce714bc1e9118b881ce01d090b1cd8db2c yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -389,7 +389,6 @@
else:
fluid_fields.append((ftype, fname))
if len(particle_fields) > 0:
- mylog.debug("reading particle fields: {}".format(particle_fields))
selector = AlwaysSelector(self.ds)
rv.update(self._read_particle_selection(
[chunk], selector, particle_fields))
https://bitbucket.org/yt_analysis/yt/commits/58e596e7aaf0/
Changeset: 58e596e7aaf0
Branch: yt
User: NTAu... at guest692.fz-rossendorf.de
Date: 2016-06-14 09:20:42+00:00
Summary: Correctly alias magnetic fields
Affected #: 3 files
diff -r 78cfb1ce714bc1e9118b881ce01d090b1cd8db2c -r 58e596e7aaf0d9cbc7ba58d830b6286bc0205bd6 yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -321,8 +321,8 @@
self.mass_unit = self.quan(1.0, "kg")
self.time_unit = self.quan(1.0, "s")
self.velocity_unit = self.quan(1.0, "m/s")
- self.magnetic_unit = self.quan(1.0, "gauss")
- #self.magnetic_unit = self.quan(1.0, "T")
+ #self.magnetic_unit = self.quan(1.0, "gauss")
+ self.magnetic_unit = self.quan(1.0, "T")
def _parse_parameter_file(self):
"""
@@ -350,30 +350,32 @@
# TODO At this point one assumes the whole file/simulation
# contains for all mesh records the same dimensionality and shapes
# TODO This probably won't work for const records
- if len(f[self.basePath + meshesPath].keys()) > 0:
- # There is at least one field, check its dimensionality
- dim_mesh = max(
- len(f[self.basePath + meshesPath + "/" + mesh].attrs["axisLabel"])
- for mesh in f[self.basePath + meshesPath].keys())
- if len(f[self.basePath + particlesPath].keys()) > 0:
- # There is at least one particle species, check the dimensionality
- # TODO
- dim_part = 0
- if dim_mesh < 1 and dim_part < 1:
+ # if len(f[self.basePath + meshesPath].keys()) > 0:
+ # # There is at least one field, check its dimensionality
+ # dim_mesh = max(
+ # len(f[self.basePath + meshesPath + "/" + mesh].attrs["axisLabel"])
+ # for mesh in f[self.basePath + meshesPath].keys())
+ # if len(f[self.basePath + particlesPath].keys()) > 0:
+ # # There is at least one particle species, check the dimensionality
+ # # TODO
+ # dim_part = 0
+ # if dim_mesh < 1 and dim_part < 1:
+ # mylog.error("Your data does not seem to have dimensionality!")
+ # self.dimensionality = max(dim_mesh, dim_part)
+ try :
+ firstIteration = list(f["/data/"].keys())[0]
+ meshes = f["/data/" + str(firstIteration) + "/" + meshesPath]
+ firstMeshName = list(meshes.keys())[0]
+ firstMesh = meshes[firstMeshName]
+ if type(firstMesh) == h5py.Dataset :
+ fshape = firstMesh.shape
+ else :
+ fshape = firstMesh[list(firstMesh.keys())[0]].shape
+ except:
mylog.error("Your data does not seem to have dimensionality!")
- self.dimensionality = max(dim_mesh, dim_part)
- # try :
- # firstIteration = list(f["/data/"].keys())[0]
- # meshes = f["/data/" + str(firstIteration) + "/" + meshesPath]
- # firstMeshName = list(meshes.keys())[0]
- # firstMesh = meshes[firstMeshName]
- # if type(firstMesh) == h5py.Dataset :
- # fshape = firstMesh.shape
- # else :
- # fshape = firstMesh[list(firstMesh.keys())[0]].shape
# Usually 2D/3D for picongpu
- # self.dimensionality = len(fshape)
+ self.dimensionality = len(fshape)
# TODO fill me with actual start and end positions in reasonable units
self.domain_left_edge = np.zeros(3, dtype=np.float64)
diff -r 78cfb1ce714bc1e9118b881ce01d090b1cd8db2c -r 58e596e7aaf0d9cbc7ba58d830b6286bc0205bd6 yt/frontends/openPMD/fields.py
--- a/yt/frontends/openPMD/fields.py
+++ b/yt/frontends/openPMD/fields.py
@@ -23,19 +23,25 @@
import yt
-def _kinetic_energy(field, data):
- """
- Function to calculate new fields out of other fields
- if a error occurs the field is not calculated
- """
- #mylog.info("oPMD - fields - _kinetic_energy")
- # calculate kinetic energy out of momentum
- c = 2.997e8 # velocity of light
- return (data["particle_momentum_x"]**2 + data["particle_momentum_y"]**2 + data["particle_momentum_z"]**2) * c
+def setup_kinetic_energy(self, ptype):
+ def _kin_en(field, data):
+ # Calculation seems to be wrong:
+ # YTFieldUnitError
+ # The field function associated with the field '('io', 'particle_kinetic_energy')'
+ # returned data with units 'cm*kg**2*m**2/s**3' but was defined with units 'kg*m**2/s**2'.
+ return (
+ data[ptype, "particle_momentum_x"]**2 +
+ data[ptype, "particle_momentum_y"]**2 +
+ data[ptype, "particle_momentum_z"]**2
+ ) * speed_of_light
+
+ self.add_field((ptype, "particle_kinetic_energy"),
+ function=_kin_en,
+ units="kg*m**2/s**2",
+ particle_type=True)
def setup_velocity(self, ptype):
- mylog.info("oPMD - fields - setup_momentum_to_velocity")
def _get_vel(axis):
def velocity(field, data):
c = speed_of_light
@@ -50,7 +56,6 @@
return velocity
for ax in 'xyz':
- mylog.info("oPMD - fields - setup_momentum_to_velocity - particle_velocity_{}".format(ax))
self.add_field((ptype, "particle_velocity_%s" % ax),
function=_get_vel(ax),
units="m/s",
@@ -63,9 +68,9 @@
Efieldx = data["E_x"]
Efieldy = data["E_y"]
Efieldz = data["E_z"]
- Bfieldx = data["B_x"]
- Bfieldy = data["B_y"]
- Bfieldz = data["B_z"]
+ Bfieldx = data["magnetic_field_x"]
+ Bfieldy = data["magnetic_field_y"]
+ Bfieldz = data["magnetic_field_z"]
u = 79577.4715459 # = 1/magnetic permeability
@@ -81,33 +86,7 @@
for ax in 'xyz':
self.add_field(("openPMD", "poynting_vector_%s" % ax),
function=_get_poyn(ax),
- units="T*V/m") # N/(m*s)
-
-
-def setup_magnetic(self):
- def _get_mag(axis):
- def magnetic(field, data):
- return data["B_{}".format(axis)]
-
- return magnetic
-
- for ax in 'xyz':
- self.add_field(("openPMD", "magnetic_field_%s" % ax),
- function=_get_mag(ax),
- units="T")
-
-
-def setup_electrical(self):
- def _get_el(axis):
- def electrical(field, data):
- return data["E_{}".format(axis)]
-
- return electrical
-
- for ax in 'xyz':
- self.add_field(("openPMD", "electrical_field_%s" % ax),
- function=_get_el(ax),
- units="kg*m/(A*s**3)")
+ units="T*V/m")
def setup_relative_positions(self, ptype):
@@ -137,18 +116,23 @@
This also defines the units of the fields
"""
+ _mag_fields = []
+
def __init__(self, ds, field_list):
- super(openPMDFieldInfo, self).__init__(ds, field_list)
-
- other_fields = ()
f = ds._handle
fields = f[ds.basePath + f.attrs["meshesPath"]]
- for i in fields.keys():
- field = fields.get(i)
- for j in field.attrs["axisLabels"]:
+ for fname in fields.keys():
+ field = fields.get(fname)
+ for axis in field.attrs["axisLabels"]:
+ ytname = str("_".join([fname.replace("_", "-"), axis]))
parsed = parse_unitDimension(np.asarray(field.attrs["unitDimension"], dtype='int'))
- other_fields += ((str("_".join([i.replace("_","-"),j])), (yt.YTQuantity(1, parsed).units, [], None)),)
- self.known_other_fields = other_fields
+ unit = str(yt.YTQuantity(1, parsed).units)
+ aliases = []
+ # Save a list of magnetic fields for aliasing later on
+ # We can not reasonably infer field type by name in openPMD
+ if "T" in unit or "kg/(A*s**2)" in unit:
+ self._mag_fields.append(ytname)
+ self.known_other_fields += ((ytname, (unit, aliases, None)), )
for i in self.known_other_fields:
mylog.debug("oPMD - fields - known_other_fields - {}".format(i))
@@ -174,6 +158,7 @@
self.known_particle_fields = particle_fields
for i in self.known_particle_fields:
mylog.debug("oPMD - fields - known_particle_fields - {}".format(i))
+ super(openPMDFieldInfo, self).__init__(ds, field_list)
def setup_fluid_fields(self):
@@ -182,17 +167,11 @@
values of new fields e.g. calculate out of E-field and B-field the
Poynting vector
"""
- # Here we do anything that might need info about the dataset.
- # You can use self.alias, self.add_output_field and self.add_field .
-
- setup_poynting_vector(self)
- setup_magnetic(self)
- setup_electrical(self)
from yt.fields.magnetic_field import \
setup_magnetic_field_aliases
- for ax in "xyz":
- mylog.info("setup_magnetic_field_aliases(self, openPMD, B_{}".format(ax))
- setup_magnetic_field_aliases(self, "openPMD", ["magnetic_field_{}".format(ax)])
+ setup_magnetic_field_aliases(self, "openPMD", self._mag_fields)
+ # Set up aliases first so the setup for poynting can use them
+ setup_poynting_vector(self)
def setup_particle_fields(self, ptype):
"""
@@ -200,11 +179,7 @@
TODO You have to call the function of the parent class to load particles
"""
- # self.add_field(
- # (ptype,
- # "particle_kinetic_energy"),
- # function=_kinetic_energy,
- # units="dimensionless")
+ setup_relative_positions(self, ptype)
+ #setup_kinetic_energy(self, ptype)
setup_velocity(self, ptype)
- setup_relative_positions(self, ptype)
super(openPMDFieldInfo, self).setup_particle_fields(ptype)
diff -r 78cfb1ce714bc1e9118b881ce01d090b1cd8db2c -r 58e596e7aaf0d9cbc7ba58d830b6286bc0205bd6 yt/frontends/openPMD/misc.py
--- a/yt/frontends/openPMD/misc.py
+++ b/yt/frontends/openPMD/misc.py
@@ -778,6 +778,7 @@
_c = {}
def get_component(self, group, component_name):
+ self._c = {}
record_component = group[component_name]
mylog.debug("openPMD - misc - get_component: {}".format(record_component.name))
if record_component.name not in self._c:
https://bitbucket.org/yt_analysis/yt/commits/ca640bfc5411/
Changeset: ca640bfc5411
Branch: yt
User: NTAu... at guest692.fz-rossendorf.de
Date: 2016-06-14 13:34:52+00:00
Summary: Further differentiate on-disk fields (datasets do not need axis-wise handling, it's done by yt); clean up code
Affected #: 2 files
diff -r 58e596e7aaf0d9cbc7ba58d830b6286bc0205bd6 -r ca640bfc541161b5e4454a40f58c8d19d20b3b87 yt/frontends/openPMD/fields.py
--- a/yt/frontends/openPMD/fields.py
+++ b/yt/frontends/openPMD/fields.py
@@ -123,8 +123,9 @@
fields = f[ds.basePath + f.attrs["meshesPath"]]
for fname in fields.keys():
field = fields.get(fname)
- for axis in field.attrs["axisLabels"]:
- ytname = str("_".join([fname.replace("_", "-"), axis]))
+ if "dataset" in str(field).split(" ")[1]:
+ # We have a dataset, don't consider axes since yt does all the work for us
+ ytname = str("_".join([fname.replace("_", "-")]))
parsed = parse_unitDimension(np.asarray(field.attrs["unitDimension"], dtype='int'))
unit = str(yt.YTQuantity(1, parsed).units)
aliases = []
@@ -133,6 +134,17 @@
if "T" in unit or "kg/(A*s**2)" in unit:
self._mag_fields.append(ytname)
self.known_other_fields += ((ytname, (unit, aliases, None)), )
+ else:
+ for axis in field.attrs["axisLabels"]:
+ ytname = str("_".join([fname.replace("_", "-"), axis]))
+ parsed = parse_unitDimension(np.asarray(field.attrs["unitDimension"], dtype='int'))
+ unit = str(yt.YTQuantity(1, parsed).units)
+ aliases = []
+ # Save a list of magnetic fields for aliasing later on
+ # We can not reasonably infer field type by name in openPMD
+ if "T" in unit or "kg/(A*s**2)" in unit:
+ self._mag_fields.append(ytname)
+ self.known_other_fields += ((ytname, (unit, aliases, None)), )
for i in self.known_other_fields:
mylog.debug("oPMD - fields - known_other_fields - {}".format(i))
@@ -140,19 +152,19 @@
particles = f[ds.basePath + f.attrs["particlesPath"]]
for species in particles.keys():
for attrib in particles.get(species).keys():
+ if "weighting" in attrib:
+ particle_fields += (("particle_weighting", ("", [], None)),)
+ continue
try:
- if "weighting" in attrib:
- particle_fields += (("particle_weighting", ("", [], None)),)
- continue
- udim = particles.get(species).get(attrib).attrs["unitDimension"]
- parsed = parse_unitDimension(np.asarray(udim, dtype='int'))
+ parsed = parse_unitDimension(
+ np.asarray(particles.get(species).get(attrib).attrs["unitDimension"], dtype='int'))
+ unit = str(yt.YTQuantity(1, parsed).units)
+ name = ["particle", attrib]
for axis in particles.get(species).get(attrib).keys():
if axis in "rxyz":
- particle_fields += (
- (str("_".join(["particle", attrib, axis])), (yt.YTQuantity(1, parsed).units, [], None)),)
- else:
- particle_fields += (
- (str("_".join(["particle", attrib])), (yt.YTQuantity(1, parsed).units, [], None)),)
+ name = ["particle", attrib, axis]
+ ytname = str("_".join(name))
+ particle_fields += ((ytname, (unit, [], None)), )
except:
mylog.info("{}_{} does not seem to have unitDimension".format(species, attrib))
self.known_particle_fields = particle_fields
@@ -181,5 +193,5 @@
"""
setup_relative_positions(self, ptype)
#setup_kinetic_energy(self, ptype)
- setup_velocity(self, ptype)
+ #setup_velocity(self, ptype)
super(openPMDFieldInfo, self).setup_particle_fields(ptype)
diff -r 58e596e7aaf0d9cbc7ba58d830b6286bc0205bd6 -r ca640bfc541161b5e4454a40f58c8d19d20b3b87 yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -26,6 +26,10 @@
from .misc import *
+from yt.utilities.parallel_tools.parallel_analysis_interface import \
+ parallel_objects
+
+
class IOHandlerOpenPMD(BaseIOHandler, openPMDBasePath):
# TODO Data should be loaded chunk-wise to support parallelism. Fields can be chunked arbitrarily in space, particles should be read via particlePatches.
# TODO Maybe reuse hdf5 file handles in loops?
@@ -48,6 +52,29 @@
self._cache = {}
self._component_getter = ComponentGetter()
+
+ def _fill_cache(self, ptype):
+ # Get a particle species (e.g. /data/3500/particles/e/)
+ if "io" in ptype:
+ spec = self._handle[self.basePath + self.particlesPath].keys()[0]
+ else:
+ spec = ptype
+ pds = self._handle[self.basePath + self.particlesPath + "/" + spec]
+ # Get 1D-Arrays for individual particle positions along axes
+ axes = [str(ax) for ax in pds["position"].keys()]
+ pos = {}
+ off = {}
+ for ax in axes:
+ pos[ax] = self._component_getter.get_component(pds, "position/" + ax)
+ off[ax] = self._component_getter.get_component(pds, "positionOffset/" + ax)
+ self._cache[ax] = pos[ax] + off[ax]
+ # Pad accordingly with zeros to make 1D/2D datasets compatible
+ for req in "xyz":
+ if req not in axes:
+ self._cache[req] = np.zeros(self._cache['x'].shape)
+ self._cached_ptype = ptype
+
+
def _read_particle_coords(self, chunks, ptf):
"""
Reads coordinates for all specified particle-types from file.
@@ -70,55 +97,16 @@
x_coords, y_coords and z_coords are arrays of positions/coordinates of all particles of that type
"""
chunks = list(chunks)
- self._array_fields = {}
for chunk in chunks:
for g in chunk.objs:
if g.filename is None:
continue
- f = self._handle
-
- dds = f[self.basePath]
for ptype, field_list in sorted(ptf.items()):
mylog.debug("openPMD - _read_particle_coords: {}, {}".format(ptype, field_list))
- if ptype in self._cached_ptype:
- # use the cached x,y,z
- yield (ptype, (self._cache['x'], self._cache['y'], self._cache['z']))
- else:
- # Get a particle species (e.g. /data/3500/particles/e/)
- if "io" in ptype:
- spec = dds[self.particlesPath].keys()[0]
- else:
- spec = ptype
- pds = dds[self.particlesPath + "/" + spec]
+ if ptype not in self._cached_ptype:
+ self._fill_cache(ptype)
+ yield (ptype, (self._cache['x'], self._cache['y'], self._cache['z']))
- # Get single 1D-arrays of coordinates from all particular particles
- axes = [str(ax) for ax in pds["position"].keys()]
- for i in 'xyz':
- if i not in axes:
- # TODO Get 2D particle data working
- mylog.error("openPMD - Can not handle 2D particle data for the moment!")
- continue
- pos = {}
- off = {}
- for ax in axes:
- pos[ax] = self._component_getter.get_component(pds, "position/" + ax)
- off[ax] = self._component_getter.get_component(pds, "positionOffset/" + ax)
- self._cache[ax] = pos[ax] + off[ax]
- self._cached_ptype = ptype
-
- for field in field_list:
- # Trim field path (yt scheme) to match openPMD scheme
- nfield = "/".join(field.split("_")[1:])
-
- # Save the size of the particle fields
- # TODO This CAN be used for speedup, not sure how
- if is_const_component(pds[nfield]):
- shape = np.asarray(pds[nfield].attrs["shape"])
- else:
- shape = np.asarray(pds[nfield].shape)
- if shape.ndim > 1:
- self._array_fields[field] = shape
- yield (ptype, (self._cache['x'], self._cache['y'], self._cache['z']))
def _read_particle_fields(self, chunks, ptf, selector):
"""
@@ -159,34 +147,18 @@
ds = f[self.basePath]
for ptype, field_list in sorted(ptf.items()):
mylog.debug("openPMD - _read_particle_fields: {}, {}".format(ptype, field_list))
-
+ if ptype not in self._cached_ptype:
+ self._fill_cache(ptype)
+ mask = selector.select_points(self._cache['x'], self._cache['y'], self._cache['z'], 0.0)
+ if mask is None:
+ continue
# Get a particle species (e.g. /data/3500/particles/e/)
if "io" in ptype:
spec = ds[self.particlesPath].keys()[0]
else:
spec = ptype
pds = ds[self.particlesPath + "/" + spec]
-
- if ptype not in self._cached_ptype:
- # Get single 1D-arrays of coordinates from all particular particles
- axes = [str(ax) for ax in pds["position"].keys()]
- for i in 'xyz':
- if i not in axes:
- # TODO Get 2D particle data working
- mylog.error("openPMD - Can not handle 2D particle data for the moment!")
- continue
- pos = {}
- off = {}
- for ax in axes:
- pos[ax] = self._component_getter.get_component(pds, "position/" + ax)
- off[ax] = self._component_getter.get_component(pds, "positionOffset/" + ax)
- self._cache[ax] = pos[ax] + off[ax]
- self._cached_ptype = ptype
-
- mask = selector.select_points(self._cache['x'], self._cache['y'], self._cache['z'], 0.0)
- if mask is None:
- continue
- for field in field_list:
+ for field in parallel_objects(field_list):
nfield = "/".join(field.split("_")[1:])
data = self._component_getter.get_component(pds, nfield)
yield ((ptype, field), data[mask])
https://bitbucket.org/yt_analysis/yt/commits/a5db78e9a2e7/
Changeset: a5db78e9a2e7
Branch: yt
User: Fabian Koller
Date: 2016-06-17 13:48:26+00:00
Summary: Only alias magnetic fields when we detect any, force particle coords to be calculated in float64, integrate preliminary h5py-level slicing
Affected #: 3 files
diff -r ca640bfc541161b5e4454a40f58c8d19d20b3b87 -r a5db78e9a2e7478ff377664f833e6f77e0f28b66 yt/frontends/openPMD/fields.py
--- a/yt/frontends/openPMD/fields.py
+++ b/yt/frontends/openPMD/fields.py
@@ -131,6 +131,7 @@
aliases = []
# Save a list of magnetic fields for aliasing later on
# We can not reasonably infer field type by name in openPMD
+ mylog.info(fname, field, unit)
if "T" in unit or "kg/(A*s**2)" in unit:
self._mag_fields.append(ytname)
self.known_other_fields += ((ytname, (unit, aliases, None)), )
@@ -142,6 +143,7 @@
aliases = []
# Save a list of magnetic fields for aliasing later on
# We can not reasonably infer field type by name in openPMD
+ mylog.info(fname, field, unit, axis)
if "T" in unit or "kg/(A*s**2)" in unit:
self._mag_fields.append(ytname)
self.known_other_fields += ((ytname, (unit, aliases, None)), )
@@ -179,9 +181,10 @@
values of new fields e.g. calculate out of E-field and B-field the
Poynting vector
"""
- from yt.fields.magnetic_field import \
- setup_magnetic_field_aliases
- setup_magnetic_field_aliases(self, "openPMD", self._mag_fields)
+ if len(self._mag_fields) > 0:
+ from yt.fields.magnetic_field import \
+ setup_magnetic_field_aliases
+ setup_magnetic_field_aliases(self, "openPMD", self._mag_fields)
# Set up aliases first so the setup for poynting can use them
setup_poynting_vector(self)
diff -r ca640bfc541161b5e4454a40f58c8d19d20b3b87 -r a5db78e9a2e7478ff377664f833e6f77e0f28b66 yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -23,7 +23,7 @@
import numpy as np
from collections import defaultdict
from .data_structures import openPMDBasePath
-from .misc import *
+from .misc import get_component
from yt.utilities.parallel_tools.parallel_analysis_interface import \
@@ -48,10 +48,9 @@
self._setBasePath(self._handle, self.ds._filepath)
self.meshesPath = self._handle["/"].attrs["meshesPath"]
self.particlesPath = self._handle["/"].attrs["particlesPath"]
+ self._array_fields = {}
self._cached_ptype = ""
self._cache = {}
- self._component_getter = ComponentGetter()
-
def _fill_cache(self, ptype):
# Get a particle species (e.g. /data/3500/particles/e/)
@@ -65,16 +64,16 @@
pos = {}
off = {}
for ax in axes:
- pos[ax] = self._component_getter.get_component(pds, "position/" + ax)
- off[ax] = self._component_getter.get_component(pds, "positionOffset/" + ax)
+ pos[ax] = np.array(get_component(pds, "position/" + ax), dtype="float64")
+ off[ax] = np.array(get_component(pds, "positionOffset/" + ax), dtype="float64")
self._cache[ax] = pos[ax] + off[ax]
# Pad accordingly with zeros to make 1D/2D datasets compatible
+ # These have to be the same shape as the existing axes dataset since that equals the number of particles
for req in "xyz":
if req not in axes:
- self._cache[req] = np.zeros(self._cache['x'].shape)
+ self._cache[req] = np.zeros(self._cache[self._cache.keys()[0]].shape)
self._cached_ptype = ptype
-
def _read_particle_coords(self, chunks, ptf):
"""
Reads coordinates for all specified particle-types from file.
@@ -107,7 +106,6 @@
self._fill_cache(ptype)
yield (ptype, (self._cache['x'], self._cache['y'], self._cache['z']))
-
def _read_particle_fields(self, chunks, ptf, selector):
"""
Reads particle fields masked by particle type and field out of a list of chunks from file.
@@ -160,7 +158,9 @@
pds = ds[self.particlesPath + "/" + spec]
for field in parallel_objects(field_list):
nfield = "/".join(field.split("_")[1:])
- data = self._component_getter.get_component(pds, nfield)
+ data = get_component(pds, nfield)
+ mylog.debug("data {}".format(data))
+ mylog.debug("mask {}".format(mask))
yield ((ptype, field), data[mask])
@@ -326,7 +326,7 @@
else:
ds = self._handle[self.basePath + self.meshesPath]
nfield = fname.replace("_", "/").replace("-","_")
- data = self._component_getter.get_component(ds, nfield)
+ data = get_component(ds, nfield)
return np.array(data).flatten()
def _read_chunk_data(self, chunk, fields):
@@ -378,8 +378,11 @@
for g in grids:
for ftype, fname in fluid_fields:
# TODO update basePath for different files
- data = self._read_data((ftype, fname))
- # data = f[self.basePath + self.meshPath + fname.replace("_", "/").replace("-","_")]
+ #data = self._read_data((ftype, fname))
+ ds = self._handle[self.basePath + self.meshesPath]
+ nfield = fname.replace("_", "/").replace("-","_")
+ data = get_component(ds, nfield)
+ mylog.debug("data.shape = {}".format(data.shape))
rv[(ftype,fname)] = np.array(data).flatten()
f.close()
return rv
diff -r ca640bfc541161b5e4454a40f58c8d19d20b3b87 -r a5db78e9a2e7478ff377664f833e6f77e0f28b66 yt/frontends/openPMD/misc.py
--- a/yt/frontends/openPMD/misc.py
+++ b/yt/frontends/openPMD/misc.py
@@ -773,22 +773,39 @@
return "value" in record_component.attrs.keys()
-class ComponentGetter():
+def get_component(group, component_name, index=0, offset=None):
+ # TODO - Make slicing possible for higher dimensional data
+ """Grab a component from a group
- _c = {}
+ Parameters
+ ----------
+ group: hdf5 group to get whole/masked component from
+ component_name: string of a component (relative path) inside the group
+ index: (optional) start index of data to return
+ offset: (optional) size of the data to return
- def get_component(self, group, component_name):
- self._c = {}
- record_component = group[component_name]
- mylog.debug("openPMD - misc - get_component: {}".format(record_component.name))
- if record_component.name not in self._c:
- if is_const_component(record_component):
- self._c[record_component.name] = \
- np.full(record_component.attrs["shape"],
- record_component.attrs["value"] * record_component.attrs["unitSI"])
- else:
- self._c[record_component.name] = record_component.value * record_component.attrs["unitSI"]
- return self._c[record_component.name]
+ Returns
+ -------
+ n-dimensional numpy array filled with values of component
+ """
+
+ record_component = group[component_name]
+ mylog.debug("openPMD - misc - get_component: {}[{}:{}]".format(record_component.name, index, offset))
+ if is_const_component(record_component):
+ # This allows for masking of the component
+ # Slicing data directly at the h5py layer improves performance
+ if offset != None:
+ shape = offset - index
+ else:
+ shape = record_component.attrs["shape"] - index
+ # Our component is constant, craft an array by hand
+ return np.full(shape, record_component.attrs["value"] * record_component.attrs["unitSI"])
+ else:
+ if offset != None:
+ offset += index
+ # Component is a dataset, return it (possibly masked)
+ # Slicing is not smart, supplying lower dimensional indices only slices those dimensions
+ return record_component[index:offset] * record_component.attrs["unitSI"]
def parse_unitDimension(unitDimension):
https://bitbucket.org/yt_analysis/yt/commits/8f311029f1bf/
Changeset: 8f311029f1bf
Branch: yt
User: Fabian Koller
Date: 2016-06-21 08:59:45+00:00
Summary: Use defined grid-size instead of particle patches
Affected #: 3 files
diff -r a5db78e9a2e7478ff377664f833e6f77e0f28b66 -r 8f311029f1bf74fea3cef30e78e5bd815c864b5d yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -31,11 +31,14 @@
import os
import re
from yt.utilities.logger import ytLogger as mylog
+from .misc import get_component, is_const_component
-class openPMDBasePathException(Exception) :
+
+class openPMDBasePathException(Exception):
pass
-class openPMDBasePath :
+
+class openPMDBasePath:
def _setBasePath(self, handle, filepath):
"""
Set the base path for the first iteration found in the file.
@@ -56,6 +59,7 @@
if not u"/data" in handle:
raise openPMDBasePathException("openPMD: group for basePath does not exist!")
+ # TODO Everything prior to this should (in theory) already be done by the validator
# find iterations in basePath
list_iterations = []
if u"groupBased" in handle.attrs["iterationEncoding"]:
@@ -72,15 +76,15 @@
list_iterations.append(filename)
mylog.info("openPMD: found {} iterations in directory".format(len(list_iterations)))
else:
- mylog.warning("openOMD: File does not have valid iteration encoding:")
- mylog.warning(handle.attrs["iterationEncoding"])
+ mylog.warning(
+ "openOMD: File does not have valid iteration encoding: {}".format(handle.attrs["iterationEncoding"]))
- # TODO in the future (see above) this can be a mylog.warning instead of an error
- if len(list_iterations) == 0 :
- raise openPMDBasePathException("openPMD: no iterations found!")
+ if len(list_iterations) == 0:
+ mylog.warning("openOMD: No iterations found!")
# just handle the first iteration found
- mylog.warning("openPMD: only choose to load one iteration ({})".format(handle[dataPath].keys()[0]))
+ if u"groupBased" in handle.attrs["iterationEncoding"] and len(list_iterations) > 1:
+ mylog.warning("openPMD: only choose to load one iteration ({})".format(handle[dataPath].keys()[0]))
self.basePath = "{}/{}/".format(dataPath, handle[dataPath].keys()[0])
@@ -95,7 +99,6 @@
def __init__(self, id, index, level=-1):
AMRGridPatch.__init__(self, id, filename=index.index_filename,
index=index)
- # There is only one grid. So there are no parent or child grids
self.Parent = None
self.Children = []
self.Level = level
@@ -198,7 +201,26 @@
self:
A reference to self
"""
- self.num_grids = 1
+ try:
+ f = self.dataset._handle
+ bp = self.basePath
+ pp = f.attrs["particlesPath"]
+ spec = f[bp + pp].keys()[0]
+ # Using particlePatches as grids
+ # We assume particlePatches to be present
+ # self.num_grids = f[bp + pp + spec + "/particlePatches/numParticles"].size
+ # Alternatively, we could do this by setting an upper bound for the size of each grid
+ # Applying a 100MB restriction to particle data
+ pos = f[bp + pp + spec + "/position"].keys()[0]
+ if is_const_component(f[bp + pp + spec + "/position/" + pos]):
+ numparts = f[bp + pp + spec + "/position/" + pos].attrs["shape"]
+ else:
+ numparts = f[bp + pp + spec + "/position/" + pos].len()
+ mylog.debug("numparts {}".format(numparts))
+ self.num_grids = 1
+ except:
+ mylog.info("Could not detect particlePatch, falling back to single grid!")
+ self.num_grids = 1
def _parse_index(self):
"""
@@ -212,7 +234,7 @@
grid_dimensions and
grid_levels
with the appropriate information.
- Each of these variables is an array, with an entry for each of the self.num_grids grids.
+ Each of these variables is an array with an entry for each of the self.num_grids grids.
Additionally, grids must be an array of AMRGridPatch objects that already know their IDs.
Parameters
@@ -220,31 +242,60 @@
self:
A reference to self
"""
- # This needs to fill the following arrays, where N is self.num_grids:
- meshesPath = self.dataset._handle.attrs["meshesPath"]
- particlesPath = self.dataset._handle.attrs["particlesPath"]
+ f = self.dataset._handle
+ bp = self.basePath
+ pp = f.attrs["particlesPath"]
+ spec = f[bp + pp].keys()[0]
+
+ offset = {}
+ extent = {}
+ pds = f[bp + pp + "/" + spec + "/particlePatches"]
+ numpart = pds["numParticles"]
+ for axis in f[bp + pp + "/" + spec + "/particlePatches/offset"].keys():
+ offset[axis] = get_component(pds, "offset/{}".format(axis))
+ extent[axis] = get_component(pds, "extent/{}".format(axis))
+ # try:
+ # # TODO Multiply by unitSI
+ # pds = f[bp + pp + "/" + spec + "/particlePatches"]
+ # numpart = get_component(pds, "numParticles")
+ # for axis in f[bp + pp + "/" + spec + "/particlePatches/offset"].keys():
+ # mylog.debug("getting ")
+ # offset[axis] = get_component(pds, "offset/{}".format(axis))
+ # #offset[axis] = f[bp + pp + "/" + spec + "/particlePatches/offset/" + axis][:]
+ # extent[axis] = get_component(pds, "extent/{}".format(axis))
+ # #extent[axis] = f[bp + pp + "/" + spec + "/particlePatches/extent/" + axis][:]
+ # except:
+ # numpart = 0 # Size of ANY of the attributes of the species
+ # offset["x"] = [0]
+ # extent["x"] = [1] # Extent of the whole simulation, read from mesh?
+
+ for req in "xyz"[:self.ds.dimensionality]:
+ if req not in offset:
+ offset[req] = np.zeros(len(offset[offset.keys()[0]]))
+ extent[req] = np.zeros(len(offset[offset.keys()[0]]))
dims = self.dataset.domain_left_edge.shape[0]#self.dimensionality
- # These objects are expecting 3 values so we pad with zeros in 1D adims 2D
- self.grid_left_edge[
- 0,:dims] = self.dataset.domain_left_edge.copy() # (N, 3) <= float64
- self.grid_right_edge[
- 0,:dims] = self.dataset.domain_right_edge.copy() # (N, 3) <= float64
- self.grid_dimensions[
- 0][:dims] = self.dataset.domain_dimensions[:dims] # (N, 3) <= int
-
-# TODO this disables particle reads for now
-# Should might be set in _read_particles for each species,
-# also each species might need its own grid (?)
-# self.grid_particle_count[0] = self.dataset._handle[
-# self.basePath + particlesPath + "/electrons/position/x"].shape[0] # (N, 1) <= int
- # self.grid_levels = 1 #(N, 1) <= int
- # self.grids = np.empty(1, dtype='object') #(N, 1) <= grid objects
+ # TODO Does NOT work for 1D/2D yet
+ for i in range(self.num_grids):
+ self.grid_left_edge[i] = [offset["x"][i],
+ offset["y"][i],
+ offset["z"][i]]
+ self.grid_right_edge[i] = [offset["x"][i]+extent["x"][i],
+ offset["y"][i]+extent["y"][i],
+ offset["z"][i]+extent["z"][i]]
+ self.grid_dimensions[i] = [extent["x"][i],
+ extent["y"][i],
+ extent["z"][i]]
+ self.grid_particle_count[i] = numpart[i]
+ mylog.debug("self.grid_left_edge[{}] - {}".format(i, self.grid_left_edge[i]))
+ mylog.debug("self.grid_right_edge[{}] - {}".format(i, self.grid_right_edge[i]))
+ mylog.debug("self.grid_dimensions[{}] - {}".format(i, self.grid_dimensions[i]))
+ mylog.debug("self.grid_particle_count[{}] - {}".format(i, self.grid_particle_count[i]))
self.grid_levels.flat[:] = 0
self.grids = np.empty(self.num_grids, dtype='object')
- # You have to inalize the grids
+ # You have to initialize the grids
for i in range(self.num_grids):
self.grids[i] = self.grid(i, self, self.grid_levels[i, 0])
@@ -315,13 +366,12 @@
self:
A reference to self
"""
- # We hardcode these to 1.0 since every dataset can have different code <-> physical scalings
- # We get the actual unit by multiplying with "unitSI" when actually getting our data
+ # We hardcode these to 1.0 since every dataset can have different code <-> physical scaling
+ # We get the actual unit by multiplying with "unitSI" when getting our data from disk
self.length_unit = self.quan(1.0, "m")
self.mass_unit = self.quan(1.0, "kg")
self.time_unit = self.quan(1.0, "s")
self.velocity_unit = self.quan(1.0, "m/s")
- #self.magnetic_unit = self.quan(1.0, "gauss")
self.magnetic_unit = self.quan(1.0, "T")
def _parse_parameter_file(self):
@@ -334,65 +384,46 @@
self:
A reference to self
"""
- # This needs to set up the following items. Note that these are all
- # assumed to be in code units; domain_left_edge and domain_right_edge
- # will be updated to be in code units at a later time. This includes
- # the cosmological parameters.
-
f = self._handle
meshesPath = f.attrs["meshesPath"].decode()
particlesPath = f.attrs["particlesPath"].decode()
- # This defines the size of the simulaion box
self.unique_identifier = 0 # no identifier
self.parameters = 0 # no additional parameters <= full of code-specific items of use
- # TODO At this point one assumes the whole file/simulation
- # contains for all mesh records the same dimensionality and shapes
- # TODO This probably won't work for const records
- # if len(f[self.basePath + meshesPath].keys()) > 0:
- # # There is at least one field, check its dimensionality
- # dim_mesh = max(
- # len(f[self.basePath + meshesPath + "/" + mesh].attrs["axisLabel"])
- # for mesh in f[self.basePath + meshesPath].keys())
- # if len(f[self.basePath + particlesPath].keys()) > 0:
- # # There is at least one particle species, check the dimensionality
- # # TODO
- # dim_part = 0
- # if dim_mesh < 1 and dim_part < 1:
- # mylog.error("Your data does not seem to have dimensionality!")
- # self.dimensionality = max(dim_mesh, dim_part)
- try :
- firstIteration = list(f["/data/"].keys())[0]
- meshes = f["/data/" + str(firstIteration) + "/" + meshesPath]
- firstMeshName = list(meshes.keys())[0]
- firstMesh = meshes[firstMeshName]
- if type(firstMesh) == h5py.Dataset :
- fshape = firstMesh.shape
- else :
- fshape = firstMesh[list(firstMesh.keys())[0]].shape
+ # We set the highest dimensionality in the simulation as the global one
+ fshape = None
+ try:
+ for mesh in f[self.basePath + meshesPath].keys():
+ for axis in f[self.basePath + meshesPath + "/" + mesh].keys():
+ fshape = np.maximum(fshape, f[self.basePath + meshesPath + "/" + mesh + "/" + axis].shape)
except:
- mylog.error("Your data does not seem to have dimensionality!")
-
- # Usually 2D/3D for picongpu
- self.dimensionality = len(fshape)
+ pass
+ if len(fshape) < 1:
+ self.dimensionality = 0
+ for species in f[self.basePath + particlesPath].keys():
+ self.dimensionality = max(
+ len(f[self.basePath + particlesPath + "/" + species].keys()),
+ self.dimensionality)
+ else:
+ self.dimensionality = len(fshape)
# TODO fill me with actual start and end positions in reasonable units
- self.domain_left_edge = np.zeros(3, dtype=np.float64)
- self.domain_right_edge = np.ones(3, dtype=np.float64)
+ # This COULD be done with minimum/maximum particle positions in the simulation
+ # or through extent of the field meshes
+ self.domain_left_edge = np.zeros(self.dimensionality, dtype=np.float64)
+ self.domain_right_edge = np.ones(self.dimensionality, dtype=np.float64)
# gridding of the meshes (assumed all mesh entries are on the same mesh)
- self.domain_dimensions = np.ones(3, dtype=np.int64)
- self.domain_dimensions[:len(fshape)] = fshape
+ self.domain_dimensions = np.ones(self.dimensionality, dtype=np.int64)
+ self.domain_dimensions[:self.dimensionality] = fshape
- # TODO assumes non-peridic boundary conditions
- self.periodicity = np.zeros(3, dtype=np.bool)
+ # TODO assumes non-periodic boundary conditions
+ self.periodicity = np.zeros(self.dimensionality, dtype=np.bool)
- self.current_time = f[self.basePath].attrs[
- "time"] # <= simulation time in code units
+ self.current_time = f[self.basePath].attrs["time"] # <= simulation time in code units
# Used for AMR, not applicable for us
- # TODO Probably 1
self.refine_by = 1
# Not a cosmological simulation
diff -r a5db78e9a2e7478ff377664f833e6f77e0f28b66 -r 8f311029f1bf74fea3cef30e78e5bd815c864b5d yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -101,7 +101,7 @@
if g.filename is None:
continue
for ptype, field_list in sorted(ptf.items()):
- mylog.debug("openPMD - _read_particle_coords: {}, {}".format(ptype, field_list))
+ mylog.debug("openPMD - _read_particle_coords: (grid {}) {}, {}".format(g, ptype, field_list))
if ptype not in self._cached_ptype:
self._fill_cache(ptype)
yield (ptype, (self._cache['x'], self._cache['y'], self._cache['z']))
@@ -144,7 +144,7 @@
ds = f[self.basePath]
for ptype, field_list in sorted(ptf.items()):
- mylog.debug("openPMD - _read_particle_fields: {}, {}".format(ptype, field_list))
+ mylog.debug("openPMD - _read_particle_fields: (grid {}) {}, {}".format(g, ptype, field_list))
if ptype not in self._cached_ptype:
self._fill_cache(ptype)
mask = selector.select_points(self._cache['x'], self._cache['y'], self._cache['z'], 0.0)
@@ -159,8 +159,8 @@
for field in parallel_objects(field_list):
nfield = "/".join(field.split("_")[1:])
data = get_component(pds, nfield)
- mylog.debug("data {}".format(data))
- mylog.debug("mask {}".format(mask))
+ #mylog.debug("data {}".format(data))
+ #mylog.debug("mask {}".format(mask))
yield ((ptype, field), data[mask])
diff -r a5db78e9a2e7478ff377664f833e6f77e0f28b66 -r 8f311029f1bf74fea3cef30e78e5bd815c864b5d yt/frontends/openPMD/misc.py
--- a/yt/frontends/openPMD/misc.py
+++ b/yt/frontends/openPMD/misc.py
@@ -789,8 +789,8 @@
n-dimensional numpy array filled with values of component
"""
+ mylog.debug("openPMD - misc - get_component: {}/{}[{}:{}]".format(group.name, component_name, index, offset))
record_component = group[component_name]
- mylog.debug("openPMD - misc - get_component: {}[{}:{}]".format(record_component.name, index, offset))
if is_const_component(record_component):
# This allows for masking of the component
# Slicing data directly at the h5py layer improves performance
https://bitbucket.org/yt_analysis/yt/commits/0550dc208391/
Changeset: 0550dc208391
Branch: yt
User: Fabian Koller
Date: 2016-06-21 09:39:48+00:00
Summary: Create grid edges according to size-limits
Affected #: 1 file
diff -r 8f311029f1bf74fea3cef30e78e5bd815c864b5d -r 0550dc2083914c811560177946ec2ccdf46023e3 yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -210,14 +210,20 @@
# We assume particlePatches to be present
# self.num_grids = f[bp + pp + spec + "/particlePatches/numParticles"].size
# Alternatively, we could do this by setting an upper bound for the size of each grid
- # Applying a 100MB restriction to particle data
+ # Applying a (rough) 100MB restriction to particle data
pos = f[bp + pp + spec + "/position"].keys()[0]
if is_const_component(f[bp + pp + spec + "/position/" + pos]):
numparts = f[bp + pp + spec + "/position/" + pos].attrs["shape"]
else:
numparts = f[bp + pp + spec + "/position/" + pos].len()
- mylog.debug("numparts {}".format(numparts))
- self.num_grids = 1
+ self.np = numparts
+ # For 3D: about 8 Mio. particles * 3 dimensions
+ self.ppg = 8e6
+ # For 2D: about 12,5 Mio particles
+ # For 1D: about 25 Mio. particles
+ from math import ceil
+ # Use an upper bound of equally sized grids, last one might be smaller
+ self.num_grids = int(ceil(self.np/self.ppg))
except:
mylog.info("Could not detect particlePatch, falling back to single grid!")
self.num_grids = 1
@@ -247,47 +253,16 @@
pp = f.attrs["particlesPath"]
spec = f[bp + pp].keys()[0]
- offset = {}
- extent = {}
- pds = f[bp + pp + "/" + spec + "/particlePatches"]
- numpart = pds["numParticles"]
- for axis in f[bp + pp + "/" + spec + "/particlePatches/offset"].keys():
- offset[axis] = get_component(pds, "offset/{}".format(axis))
- extent[axis] = get_component(pds, "extent/{}".format(axis))
- # try:
- # # TODO Multiply by unitSI
- # pds = f[bp + pp + "/" + spec + "/particlePatches"]
- # numpart = get_component(pds, "numParticles")
- # for axis in f[bp + pp + "/" + spec + "/particlePatches/offset"].keys():
- # mylog.debug("getting ")
- # offset[axis] = get_component(pds, "offset/{}".format(axis))
- # #offset[axis] = f[bp + pp + "/" + spec + "/particlePatches/offset/" + axis][:]
- # extent[axis] = get_component(pds, "extent/{}".format(axis))
- # #extent[axis] = f[bp + pp + "/" + spec + "/particlePatches/extent/" + axis][:]
- # except:
- # numpart = 0 # Size of ANY of the attributes of the species
- # offset["x"] = [0]
- # extent["x"] = [1] # Extent of the whole simulation, read from mesh?
-
- for req in "xyz"[:self.ds.dimensionality]:
- if req not in offset:
- offset[req] = np.zeros(len(offset[offset.keys()[0]]))
- extent[req] = np.zeros(len(offset[offset.keys()[0]]))
-
- dims = self.dataset.domain_left_edge.shape[0]#self.dimensionality
-
# TODO Does NOT work for 1D/2D yet
for i in range(self.num_grids):
- self.grid_left_edge[i] = [offset["x"][i],
- offset["y"][i],
- offset["z"][i]]
- self.grid_right_edge[i] = [offset["x"][i]+extent["x"][i],
- offset["y"][i]+extent["y"][i],
- offset["z"][i]+extent["z"][i]]
- self.grid_dimensions[i] = [extent["x"][i],
- extent["y"][i],
- extent["z"][i]]
- self.grid_particle_count[i] = numpart[i]
+ if i != self.num_grids-1:
+ self.grid_particle_count[i] = self.ppg
+ else:
+ # The last grid need not be the same size as the previous ones
+ self.grid_particle_count[i] = self.np%self.ppg
+ self.grid_left_edge[i] = [i/self.num_grids, i/self.num_grids, i/self.num_grids]
+ self.grid_right_edge[i] = [i+1/self.num_grids, i+1/self.num_grids, i+1/self.num_grids]
+ self.grid_dimensions[i] = self.grid_right_edge[i] - self.grid_left_edge[i]
mylog.debug("self.grid_left_edge[{}] - {}".format(i, self.grid_left_edge[i]))
mylog.debug("self.grid_right_edge[{}] - {}".format(i, self.grid_right_edge[i]))
mylog.debug("self.grid_dimensions[{}] - {}".format(i, self.grid_dimensions[i]))
@@ -393,12 +368,12 @@
# We set the highest dimensionality in the simulation as the global one
fshape = None
- try:
- for mesh in f[self.basePath + meshesPath].keys():
- for axis in f[self.basePath + meshesPath + "/" + mesh].keys():
+ for mesh in f[self.basePath + meshesPath].keys():
+ for axis in f[self.basePath + meshesPath + "/" + mesh].keys():
+ try:
fshape = np.maximum(fshape, f[self.basePath + meshesPath + "/" + mesh + "/" + axis].shape)
- except:
- pass
+ except:
+ pass
if len(fshape) < 1:
self.dimensionality = 0
for species in f[self.basePath + particlesPath].keys():
https://bitbucket.org/yt_analysis/yt/commits/b59f4b704539/
Changeset: b59f4b704539
Branch: yt
User: Fabian Koller
Date: 2016-06-21 09:55:22+00:00
Summary: Read particle data according to grids
Affected #: 2 files
diff -r 0550dc2083914c811560177946ec2ccdf46023e3 -r b59f4b7045390f22c33c152d904b8bcc519fe43b yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -368,12 +368,12 @@
# We set the highest dimensionality in the simulation as the global one
fshape = None
- for mesh in f[self.basePath + meshesPath].keys():
- for axis in f[self.basePath + meshesPath + "/" + mesh].keys():
- try:
+ try:
+ for mesh in f[self.basePath + meshesPath].keys():
+ for axis in f[self.basePath + meshesPath + "/" + mesh].keys():
fshape = np.maximum(fshape, f[self.basePath + meshesPath + "/" + mesh + "/" + axis].shape)
- except:
- pass
+ except:
+ pass
if len(fshape) < 1:
self.dimensionality = 0
for species in f[self.basePath + particlesPath].keys():
diff -r 0550dc2083914c811560177946ec2ccdf46023e3 -r b59f4b7045390f22c33c152d904b8bcc519fe43b yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -52,7 +52,7 @@
self._cached_ptype = ""
self._cache = {}
- def _fill_cache(self, ptype):
+ def _fill_cache(self, ptype, index=0, offset=None):
# Get a particle species (e.g. /data/3500/particles/e/)
if "io" in ptype:
spec = self._handle[self.basePath + self.particlesPath].keys()[0]
@@ -64,15 +64,15 @@
pos = {}
off = {}
for ax in axes:
- pos[ax] = np.array(get_component(pds, "position/" + ax), dtype="float64")
- off[ax] = np.array(get_component(pds, "positionOffset/" + ax), dtype="float64")
+ pos[ax] = np.array(get_component(pds, "position/" + ax, index, offset), dtype="float64")
+ off[ax] = np.array(get_component(pds, "positionOffset/" + ax, index, offset), dtype="float64")
self._cache[ax] = pos[ax] + off[ax]
# Pad accordingly with zeros to make 1D/2D datasets compatible
# These have to be the same shape as the existing axes dataset since that equals the number of particles
for req in "xyz":
if req not in axes:
self._cache[req] = np.zeros(self._cache[self._cache.keys()[0]].shape)
- self._cached_ptype = ptype
+ # TODO self._cached_ptype = ptype
def _read_particle_coords(self, chunks, ptf):
"""
@@ -103,7 +103,9 @@
for ptype, field_list in sorted(ptf.items()):
mylog.debug("openPMD - _read_particle_coords: (grid {}) {}, {}".format(g, ptype, field_list))
if ptype not in self._cached_ptype:
- self._fill_cache(ptype)
+ index = g.ActiveDimensions[0] * 8e6
+ offset = g.NumberOfParticles
+ self._fill_cache(ptype, index, offset)
yield (ptype, (self._cache['x'], self._cache['y'], self._cache['z']))
def _read_particle_fields(self, chunks, ptf, selector):
@@ -146,7 +148,9 @@
for ptype, field_list in sorted(ptf.items()):
mylog.debug("openPMD - _read_particle_fields: (grid {}) {}, {}".format(g, ptype, field_list))
if ptype not in self._cached_ptype:
- self._fill_cache(ptype)
+ index = g.ActiveDimensions[0] * 8e6
+ offset = g.NumberOfParticles
+ self._fill_cache(ptype, index, offset)
mask = selector.select_points(self._cache['x'], self._cache['y'], self._cache['z'], 0.0)
if mask is None:
continue
@@ -158,7 +162,7 @@
pds = ds[self.particlesPath + "/" + spec]
for field in parallel_objects(field_list):
nfield = "/".join(field.split("_")[1:])
- data = get_component(pds, nfield)
+ data = get_component(pds, nfield, index, offset)
#mylog.debug("data {}".format(data))
#mylog.debug("mask {}".format(mask))
yield ((ptype, field), data[mask])
https://bitbucket.org/yt_analysis/yt/commits/d99a46b311cf/
Changeset: d99a46b311cf
Branch: yt
User: Fabian Koller
Date: 2016-06-21 11:31:06+00:00
Summary: Cast slicing indices from float to int
Affected #: 3 files
diff -r b59f4b7045390f22c33c152d904b8bcc519fe43b -r d99a46b311cf0809fe2ad18b75dfa6a7699fe507 yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -201,15 +201,13 @@
self:
A reference to self
"""
+ # TODO For the moment we only create grids if there are particles present
try:
f = self.dataset._handle
bp = self.basePath
pp = f.attrs["particlesPath"]
spec = f[bp + pp].keys()[0]
- # Using particlePatches as grids
- # We assume particlePatches to be present
- # self.num_grids = f[bp + pp + spec + "/particlePatches/numParticles"].size
- # Alternatively, we could do this by setting an upper bound for the size of each grid
+ # We could do this by setting an upper bound for the size of each grid
# Applying a (rough) 100MB restriction to particle data
pos = f[bp + pp + spec + "/position"].keys()[0]
if is_const_component(f[bp + pp + spec + "/position/" + pos]):
@@ -217,10 +215,11 @@
else:
numparts = f[bp + pp + spec + "/position/" + pos].len()
self.np = numparts
- # For 3D: about 8 Mio. particles * 3 dimensions
- self.ppg = 8e6
- # For 2D: about 12,5 Mio particles
- # For 1D: about 25 Mio. particles
+ gridsize = 100 * 10**6 # Bytes
+ # For 3D: about 8 Mio. particles per grid
+ # For 2D: about 12,5 Mio. particles per grid
+ # For 1D: about 25 Mio. particles per grid
+ self.ppg = int(gridsize/(self.dataset.dimensionality*4)) # 4 Byte per value per dimension (f32)
from math import ceil
# Use an upper bound of equally sized grids, last one might be smaller
self.num_grids = int(ceil(self.np/self.ppg))
diff -r b59f4b7045390f22c33c152d904b8bcc519fe43b -r d99a46b311cf0809fe2ad18b75dfa6a7699fe507 yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -103,7 +103,7 @@
for ptype, field_list in sorted(ptf.items()):
mylog.debug("openPMD - _read_particle_coords: (grid {}) {}, {}".format(g, ptype, field_list))
if ptype not in self._cached_ptype:
- index = g.ActiveDimensions[0] * 8e6
+ index = int(g.ActiveDimensions[0] * 8e6)
offset = g.NumberOfParticles
self._fill_cache(ptype, index, offset)
yield (ptype, (self._cache['x'], self._cache['y'], self._cache['z']))
@@ -148,7 +148,7 @@
for ptype, field_list in sorted(ptf.items()):
mylog.debug("openPMD - _read_particle_fields: (grid {}) {}, {}".format(g, ptype, field_list))
if ptype not in self._cached_ptype:
- index = g.ActiveDimensions[0] * 8e6
+ index = int(g.ActiveDimensions[0] * 8e6)
offset = g.NumberOfParticles
self._fill_cache(ptype, index, offset)
mask = selector.select_points(self._cache['x'], self._cache['y'], self._cache['z'], 0.0)
diff -r b59f4b7045390f22c33c152d904b8bcc519fe43b -r d99a46b311cf0809fe2ad18b75dfa6a7699fe507 yt/frontends/openPMD/misc.py
--- a/yt/frontends/openPMD/misc.py
+++ b/yt/frontends/openPMD/misc.py
@@ -789,19 +789,19 @@
n-dimensional numpy array filled with values of component
"""
- mylog.debug("openPMD - misc - get_component: {}/{}[{}:{}]".format(group.name, component_name, index, offset))
+ mylog.debug("openPMD - misc - get_component: {}/{}[{} + {}]".format(group.name, component_name, index, offset))
record_component = group[component_name]
if is_const_component(record_component):
# This allows for masking of the component
# Slicing data directly at the h5py layer improves performance
- if offset != None:
+ if offset is not None:
shape = offset - index
else:
shape = record_component.attrs["shape"] - index
# Our component is constant, craft an array by hand
return np.full(shape, record_component.attrs["value"] * record_component.attrs["unitSI"])
else:
- if offset != None:
+ if offset is not None:
offset += index
# Component is a dataset, return it (possibly masked)
# Slicing is not smart, supplying lower dimensional indices only slices those dimensions
https://bitbucket.org/yt_analysis/yt/commits/107541d61645/
Changeset: 107541d61645
Branch: yt
User: Fabian Koller
Date: 2016-06-21 13:02:50+00:00
Summary: Correctly calculate relative grid edge (does not consider last grid yet)
Affected #: 2 files
diff -r d99a46b311cf0809fe2ad18b75dfa6a7699fe507 -r 107541d61645187038282609e1cb52271b31b84c yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -247,11 +247,6 @@
self:
A reference to self
"""
- f = self.dataset._handle
- bp = self.basePath
- pp = f.attrs["particlesPath"]
- spec = f[bp + pp].keys()[0]
-
# TODO Does NOT work for 1D/2D yet
for i in range(self.num_grids):
if i != self.num_grids-1:
@@ -259,8 +254,8 @@
else:
# The last grid need not be the same size as the previous ones
self.grid_particle_count[i] = self.np%self.ppg
- self.grid_left_edge[i] = [i/self.num_grids, i/self.num_grids, i/self.num_grids]
- self.grid_right_edge[i] = [i+1/self.num_grids, i+1/self.num_grids, i+1/self.num_grids]
+ self.grid_left_edge[i] = [i*self.num_grids**-1, i*self.num_grids**-1, i*self.num_grids**-1]
+ self.grid_right_edge[i] = [(i+1)*self.num_grids**-1, (i+1)*self.num_grids**-1, (i+1)*self.num_grids**-1]
self.grid_dimensions[i] = self.grid_right_edge[i] - self.grid_left_edge[i]
mylog.debug("self.grid_left_edge[{}] - {}".format(i, self.grid_left_edge[i]))
mylog.debug("self.grid_right_edge[{}] - {}".format(i, self.grid_right_edge[i]))
diff -r d99a46b311cf0809fe2ad18b75dfa6a7699fe507 -r 107541d61645187038282609e1cb52271b31b84c yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -103,7 +103,7 @@
for ptype, field_list in sorted(ptf.items()):
mylog.debug("openPMD - _read_particle_coords: (grid {}) {}, {}".format(g, ptype, field_list))
if ptype not in self._cached_ptype:
- index = int(g.ActiveDimensions[0] * 8e6)
+ index = int(g.ActiveDimensions[0] * self.ds.index.ppg)
offset = g.NumberOfParticles
self._fill_cache(ptype, index, offset)
yield (ptype, (self._cache['x'], self._cache['y'], self._cache['z']))
@@ -148,7 +148,7 @@
for ptype, field_list in sorted(ptf.items()):
mylog.debug("openPMD - _read_particle_fields: (grid {}) {}, {}".format(g, ptype, field_list))
if ptype not in self._cached_ptype:
- index = int(g.ActiveDimensions[0] * 8e6)
+ index = int(g.ActiveDimensions[0] * self.ds.index.ppg)
offset = g.NumberOfParticles
self._fill_cache(ptype, index, offset)
mask = selector.select_points(self._cache['x'], self._cache['y'], self._cache['z'], 0.0)
@@ -210,28 +210,12 @@
# raise RuntimeError
#
# rv = {f: np.array([]) for f in fields}
- # for field in fields:
- # ftype, fname = field
- # for chunk in chunks:
- # for g in chunk.objs:
- # mylog.debug("g.select({}, {}, rv[{}], {})".format(selector, data, field,ind))
- # rv.update(self._read_chunk_data(chunk, fields))
+ # for chunk in chunks:
+ # for grid in chunk.objs:
+ # for ftype, fname in fields:
+ # data = self._read_particle_fields(chunks, ptf, selector)
+ # rv[ftype, fname] = np.concatenate((data, rv[ftype, fname]))
# return rv
- # # for chunk in chunks:
- # # for i in self._read_chunk_data(chunk,fields):
- # # mylog.info("self._read_chunk_data({},{}):{}".format(chunk,fields,i))
- #
- # # for grid in chunk.objs:
- # # for ftype, fname in fields:
- # # data = self._read_data((ftype, fname))
- # # rv[ftype, fname] = np.concatenate((data, rv[ftype, fname]))
- #
- # """
- # From Exo 2
- # for field in fields:
- # for g in chunk.objs:
- # ind += g.select(selector, data, rv[field], ind)
- # """
def _read_fluid_selection(self, chunks, selector, fields, size):
"""
https://bitbucket.org/yt_analysis/yt/commits/208f760a4792/
Changeset: 208f760a4792
Branch: yt
User: Fabian Koller
Date: 2016-06-21 13:48:30+00:00
Summary: User floats for ceiling (python 2.7 interprets int/int as int)
Affected #: 1 file
diff -r 107541d61645187038282609e1cb52271b31b84c -r 208f760a4792beb1ad09a7b586f323275ed15cc5 yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -135,11 +135,6 @@
From yt doc:
self.field_list must be populated as a list of strings corresponding to "native" fields in the data files.
-
- Parameters
- ----------
- self:
- A reference to self
"""
# TODO This only parses one file
f = self.dataset._handle
@@ -195,11 +190,6 @@
From yt doc:
this must set self.num_grids to be the total number of grids (equiv AMRGridPatch'es) in the simulation
-
- Parameters
- ----------
- self:
- A reference to self
"""
# TODO For the moment we only create grids if there are particles present
try:
@@ -222,7 +212,7 @@
self.ppg = int(gridsize/(self.dataset.dimensionality*4)) # 4 Byte per value per dimension (f32)
from math import ceil
# Use an upper bound of equally sized grids, last one might be smaller
- self.num_grids = int(ceil(self.np/self.ppg))
+ self.num_grids = int(ceil(self.np/(self.ppg*1.0)))
except:
mylog.info("Could not detect particlePatch, falling back to single grid!")
self.num_grids = 1
@@ -241,21 +231,18 @@
with the appropriate information.
Each of these variables is an array with an entry for each of the self.num_grids grids.
Additionally, grids must be an array of AMRGridPatch objects that already know their IDs.
-
- Parameters
- ----------
- self:
- A reference to self
"""
# TODO Does NOT work for 1D/2D yet
+ dim = self.dataset.dimensionality
for i in range(self.num_grids):
if i != self.num_grids-1:
self.grid_particle_count[i] = self.ppg
else:
# The last grid need not be the same size as the previous ones
self.grid_particle_count[i] = self.np%self.ppg
- self.grid_left_edge[i] = [i*self.num_grids**-1, i*self.num_grids**-1, i*self.num_grids**-1]
- self.grid_right_edge[i] = [(i+1)*self.num_grids**-1, (i+1)*self.num_grids**-1, (i+1)*self.num_grids**-1]
+ # The edges have in fact nothing to do with physical boundaries, they are sections of the raw data
+ self.grid_left_edge[i] = [i*self.num_grids**-1, i*self.num_grids**-1, i*self.num_grids**-1][:dim]
+ self.grid_right_edge[i] = [(i+1)*self.num_grids**-1, (i+1)*self.num_grids**-1, (i+1)*self.num_grids**-1][:dim]
self.grid_dimensions[i] = self.grid_right_edge[i] - self.grid_left_edge[i]
mylog.debug("self.grid_left_edge[{}] - {}".format(i, self.grid_left_edge[i]))
mylog.debug("self.grid_right_edge[{}] - {}".format(i, self.grid_right_edge[i]))
@@ -275,11 +262,6 @@
From yt doc:
this initializes the grids by calling _prepare_grid() and _setup_dx() on all of them.
Additionally, it should set up Children and Parent lists on each grid object.
-
- Parameters
- ----------
- self:
- A reference to self
"""
# self._reconstruct_parent_child()
@@ -329,11 +311,6 @@
"""
From yt doc:
handle conversion between the different physical units and the code units
-
- Parameters
- ----------
- self:
- A reference to self
"""
# We hardcode these to 1.0 since every dataset can have different code <-> physical scaling
# We get the actual unit by multiplying with "unitSI" when getting our data from disk
@@ -347,11 +324,6 @@
"""
From yt doc:
read in metadata describing the overall data on disk
-
- Parameters
- ----------
- self:
- A reference to self
"""
f = self._handle
meshesPath = f.attrs["meshesPath"].decode()
https://bitbucket.org/yt_analysis/yt/commits/26e2aeee07b0/
Changeset: 26e2aeee07b0
Branch: yt
User: Fabian Koller
Date: 2016-06-21 14:50:21+00:00
Summary: Revert to calculating grid edges by mesh dimensionality
Affected #: 1 file
diff -r 208f760a4792beb1ad09a7b586f323275ed15cc5 -r 26e2aeee07b09fda27a7874f7845e3962d23ad4e yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -30,6 +30,7 @@
import numpy as np
import os
import re
+from math import ceil
from yt.utilities.logger import ytLogger as mylog
from .misc import get_component, is_const_component
@@ -210,9 +211,8 @@
# For 2D: about 12,5 Mio. particles per grid
# For 1D: about 25 Mio. particles per grid
self.ppg = int(gridsize/(self.dataset.dimensionality*4)) # 4 Byte per value per dimension (f32)
- from math import ceil
# Use an upper bound of equally sized grids, last one might be smaller
- self.num_grids = int(ceil(self.np/(self.ppg*1.0)))
+ self.num_grids = int(ceil(self.np*self.ppg**-1))
except:
mylog.info("Could not detect particlePatch, falling back to single grid!")
self.num_grids = 1
@@ -232,22 +232,33 @@
Each of these variables is an array with an entry for each of the self.num_grids grids.
Additionally, grids must be an array of AMRGridPatch objects that already know their IDs.
"""
- # TODO Does NOT work for 1D/2D yet
+ # TODO This (particle grid-data) needs to be included in every grid
+ # TODO Furthermore, we have to use grids from meshes to represent a physical layout
+ # dim = self.dataset.dimensionality
+ # for i in range(self.num_grids):
+ # if i != self.num_grids-1:
+ # self.grid_particle_count[i] = self.ppg
+ # else:
+ # # The last grid need not be the same size as the previous ones
+ # self.grid_particle_count[i] = self.np%self.ppg
+ # # The edges have in fact nothing to do with physical boundaries, they are sections of the raw data
+ # self.grid_left_edge[i] = [i*self.num_grids**-1] * dim
+ # self.grid_right_edge[i] = [(i+1)*self.num_grids**-1] * dim
+ # self.grid_dimensions[i] = self.grid_right_edge[i] - self.grid_left_edge[i]
+ # mylog.debug("self.grid_left_edge[{}] - {}".format(i, self.grid_left_edge[i]))
+ # mylog.debug("self.grid_right_edge[{}] - {}".format(i, self.grid_right_edge[i]))
+ # mylog.debug("self.grid_dimensions[{}] - {}".format(i, self.grid_dimensions[i]))
+ # mylog.debug("self.grid_particle_count[{}] - {}".format(i, self.grid_particle_count[i]))
+
dim = self.dataset.dimensionality
+
for i in range(self.num_grids):
- if i != self.num_grids-1:
- self.grid_particle_count[i] = self.ppg
- else:
- # The last grid need not be the same size as the previous ones
- self.grid_particle_count[i] = self.np%self.ppg
- # The edges have in fact nothing to do with physical boundaries, they are sections of the raw data
- self.grid_left_edge[i] = [i*self.num_grids**-1, i*self.num_grids**-1, i*self.num_grids**-1][:dim]
- self.grid_right_edge[i] = [(i+1)*self.num_grids**-1, (i+1)*self.num_grids**-1, (i+1)*self.num_grids**-1][:dim]
- self.grid_dimensions[i] = self.grid_right_edge[i] - self.grid_left_edge[i]
- mylog.debug("self.grid_left_edge[{}] - {}".format(i, self.grid_left_edge[i]))
- mylog.debug("self.grid_right_edge[{}] - {}".format(i, self.grid_right_edge[i]))
- mylog.debug("self.grid_dimensions[{}] - {}".format(i, self.grid_dimensions[i]))
- mylog.debug("self.grid_particle_count[{}] - {}".format(i, self.grid_particle_count[i]))
+ self.grid_left_edge[i][:dim] = self.dataset.domain_left_edge.copy() # (N, 3) <= float64
+ self.grid_left_edge[i][0] = i * self.num_grids**-1
+ self.grid_right_edge[i][:dim] = self.dataset.domain_right_edge.copy() # (N, 3) <= float64
+ self.grid_right_edge[i][0] = (i + 1) * self.num_grids**-1
+ self.grid_dimensions[i][:dim] = self.dataset.domain_dimensions[:dim] # (N, 3) <= int
+ self.grid_dimensions[i][0] *= self.num_grids**-1
self.grid_levels.flat[:] = 0
self.grids = np.empty(self.num_grids, dtype='object')
https://bitbucket.org/yt_analysis/yt/commits/32265a055ce1/
Changeset: 32265a055ce1
Branch: yt
User: Fabian Koller
Date: 2016-06-24 13:15:09+00:00
Summary: Gridding done in hybrid form between meshes, particles. Reading is done grid-wise
Affected #: 2 files
diff -r 26e2aeee07b09fda27a7874f7845e3962d23ad4e -r 32265a055ce101287baa2c20a46f3c7e7b114eed yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -97,9 +97,18 @@
"""
_id_offset = 0
__slots__ = ["_level_id"]
- def __init__(self, id, index, level=-1):
+ part_ind = 0
+ off_part = 0
+ mesh_ind = 0
+ off_mesh = 0
+
+ def __init__(self, id, index, level=-1, pi=0, op=0, mi=0, om=0):
AMRGridPatch.__init__(self, id, filename=index.index_filename,
index=index)
+ self.part_ind = pi
+ self.off_part = op
+ self.mesh_ind = mi
+ self.off_mesh = om
self.Parent = None
self.Children = []
self.Level = level
@@ -152,9 +161,6 @@
output_fields.append(group.replace("_","-"))
self.field_list = [("openPMD", str(c)) for c in output_fields]
- def is_const_component(record_component):
- return ("value" in record_component.attrs.keys())
-
particle_fields = []
if self.basePath + particlesPath in f:
for particleName in f[self.basePath + particlesPath].keys():
@@ -193,6 +199,7 @@
this must set self.num_grids to be the total number of grids (equiv AMRGridPatch'es) in the simulation
"""
# TODO For the moment we only create grids if there are particles present
+ # TODO This breaks datasets with multiple types of particles and different amounts of particles
try:
f = self.dataset._handle
bp = self.basePath
@@ -202,10 +209,9 @@
# Applying a (rough) 100MB restriction to particle data
pos = f[bp + pp + spec + "/position"].keys()[0]
if is_const_component(f[bp + pp + spec + "/position/" + pos]):
- numparts = f[bp + pp + spec + "/position/" + pos].attrs["shape"]
+ self.np = f[bp + pp + spec + "/position/" + pos].attrs["shape"]
else:
- numparts = f[bp + pp + spec + "/position/" + pos].len()
- self.np = numparts
+ self.np = f[bp + pp + spec + "/position/" + pos].len()
gridsize = 100 * 10**6 # Bytes
# For 3D: about 8 Mio. particles per grid
# For 2D: about 12,5 Mio. particles per grid
@@ -250,21 +256,37 @@
# mylog.debug("self.grid_dimensions[{}] - {}".format(i, self.grid_dimensions[i]))
# mylog.debug("self.grid_particle_count[{}] - {}".format(i, self.grid_particle_count[i]))
- dim = self.dataset.dimensionality
+ # There is only one refinement level in openPMD
+ self.grid_levels.flat[:] = 0
+ self.grids = np.empty(self.num_grids, dtype='object')
+
+ remaining = self.dataset.domain_dimensions[0]
for i in range(self.num_grids):
- self.grid_left_edge[i][:dim] = self.dataset.domain_left_edge.copy() # (N, 3) <= float64
+ self.grid_left_edge[i] = self.dataset.domain_left_edge.copy() # (N, 3) <= float64
self.grid_left_edge[i][0] = i * self.num_grids**-1
- self.grid_right_edge[i][:dim] = self.dataset.domain_right_edge.copy() # (N, 3) <= float64
+ self.grid_right_edge[i] = self.dataset.domain_right_edge.copy() # (N, 3) <= float64
self.grid_right_edge[i][0] = (i + 1) * self.num_grids**-1
- self.grid_dimensions[i][:dim] = self.dataset.domain_dimensions[:dim] # (N, 3) <= int
- self.grid_dimensions[i][0] *= self.num_grids**-1
-
- self.grid_levels.flat[:] = 0
- self.grids = np.empty(self.num_grids, dtype='object')
- # You have to initialize the grids
- for i in range(self.num_grids):
- self.grids[i] = self.grid(i, self, self.grid_levels[i, 0])
+ self.grid_dimensions[i] = self.dataset.domain_dimensions # (N, 3) <= int
+ if i != self.num_grids-1:
+ self.grid_dimensions[i][0] = int(ceil(self.grid_dimensions[i][0] * self.num_grids**-1))
+ remaining -= self.grid_dimensions[i][0]
+ nap = self.ppg
+ else:
+ # The last grid need not be the same size as the previous ones
+ self.grid_dimensions[i][0] = remaining
+ nap = self.np % self.ppg
+ self.grid_particle_count[i] = nap
+ # You have to initialize the grids
+ self.grids[i] = self.grid(
+ i, self, self.grid_levels[i, 0],
+ pi=i*self.ppg, op=nap,
+ mi=int(ceil(self.dataset.domain_dimensions[0] * self.num_grids**-1)) * i, om=self.grid_dimensions[i][0])
+ mylog.debug("remaining {}".format(remaining))
+ mylog.debug("self.grid_left_edge[{}] - {}".format(i, self.grid_left_edge[i]))
+ mylog.debug("self.grid_right_edge[{}] - {}".format(i, self.grid_right_edge[i]))
+ mylog.debug("self.grid_dimensions[{}] - {}".format(i, self.grid_dimensions[i]))
+ mylog.debug("self.grid_particle_count[{}] - {}".format(i, self.grid_particle_count[i]))
def _populate_grid_objects(self):
"""
@@ -363,15 +385,15 @@
# TODO fill me with actual start and end positions in reasonable units
# This COULD be done with minimum/maximum particle positions in the simulation
# or through extent of the field meshes
- self.domain_left_edge = np.zeros(self.dimensionality, dtype=np.float64)
- self.domain_right_edge = np.ones(self.dimensionality, dtype=np.float64)
+ self.domain_left_edge = np.zeros(3, dtype=np.float64)
+ self.domain_right_edge = np.ones(3, dtype=np.float64)
# gridding of the meshes (assumed all mesh entries are on the same mesh)
- self.domain_dimensions = np.ones(self.dimensionality, dtype=np.int64)
+ self.domain_dimensions = np.ones(3, dtype=np.int64)
self.domain_dimensions[:self.dimensionality] = fshape
# TODO assumes non-periodic boundary conditions
- self.periodicity = np.zeros(self.dimensionality, dtype=np.bool)
+ self.periodicity = np.zeros(3, dtype=np.bool)
self.current_time = f[self.basePath].attrs["time"] # <= simulation time in code units
diff -r 26e2aeee07b09fda27a7874f7845e3962d23ad4e -r 32265a055ce101287baa2c20a46f3c7e7b114eed yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -72,7 +72,7 @@
for req in "xyz":
if req not in axes:
self._cache[req] = np.zeros(self._cache[self._cache.keys()[0]].shape)
- # TODO self._cached_ptype = ptype
+ self._cached_ptype = str((ptype, index, offset))
def _read_particle_coords(self, chunks, ptf):
"""
@@ -102,9 +102,9 @@
continue
for ptype, field_list in sorted(ptf.items()):
mylog.debug("openPMD - _read_particle_coords: (grid {}) {}, {}".format(g, ptype, field_list))
- if ptype not in self._cached_ptype:
- index = int(g.ActiveDimensions[0] * self.ds.index.ppg)
- offset = g.NumberOfParticles
+ index = g.part_ind
+ offset = g.off_part
+ if str((ptype, index, offset)) not in self._cached_ptype:
self._fill_cache(ptype, index, offset)
yield (ptype, (self._cache['x'], self._cache['y'], self._cache['z']))
@@ -143,13 +143,12 @@
if g.filename is None:
continue
f = self._handle
-
ds = f[self.basePath]
- for ptype, field_list in sorted(ptf.items()):
+ for ptype, field_list in parallel_objects(sorted(ptf.items())):
mylog.debug("openPMD - _read_particle_fields: (grid {}) {}, {}".format(g, ptype, field_list))
- if ptype not in self._cached_ptype:
- index = int(g.ActiveDimensions[0] * self.ds.index.ppg)
- offset = g.NumberOfParticles
+ index = g.part_ind
+ offset = g.off_part
+ if str((ptype, index, offset)) not in self._cached_ptype:
self._fill_cache(ptype, index, offset)
mask = selector.select_points(self._cache['x'], self._cache['y'], self._cache['z'], 0.0)
if mask is None:
@@ -160,7 +159,7 @@
else:
spec = ptype
pds = ds[self.particlesPath + "/" + spec]
- for field in parallel_objects(field_list):
+ for field in parallel_objects(field_list, 8):
nfield = "/".join(field.split("_")[1:])
data = get_component(pds, nfield, index, offset)
#mylog.debug("data {}".format(data))
@@ -251,39 +250,34 @@
- keys are (ftype, fname) tuples representing a field
- values are numpy arrays with data form that field
"""
- mylog.info("_read_fluid_selection")
+ mylog.info("_read_fluid_selection {} {} {} {}".format(chunks, selector, fields, size))
rv = {}
chunks = list(chunks)
if selector.__class__.__name__ == "GridSelector":
if not (len(chunks) == len(chunks[0].objs) == 1):
raise RuntimeError
- # TODO We can probably skip this since the following loops do this, anyway
- # g = chunks[0].objs[0]
- # rv.update(self._read_chunk_data(chunks[0],fields))
- #f = h5py.File(u(g.filename), 'r')
- #for ftype, fname in fields:
- # rv[ftype, fname] = self._read_data(g, fname)
- #f.close()
- # return rv
-
-
if size is None:
size = sum((g.count(selector) for chunk in chunks
for g in chunk.objs))
+ ind = {}
for field in fields:
rv[field] = np.empty(size, dtype="float64")
+ ind[field] = 0
- for chunk in chunks:
- rv.update(self._read_chunk_data(chunk, fields))
- # for g in chunk.objs:
- # # WHY do we create a filehandle but not use it?
- # f = h5py.File(u(g.filename), 'r')
- # ###self._handle = f
- # for ftype, fname in fields:
- # # WHY call with g (a grid) as self?
- # rv[ftype, fname] = self._read_data(g, fname)
- # f.close()
+ for ftype, fname in parallel_objects(fields, 8):
+ field = (ftype, fname)
+ for chunk in chunks:
+ for g in chunk.objs:
+ ds = self._handle[self.basePath + self.meshesPath]
+ nfield = fname.replace("_", "/").replace("-","_")
+ index = g.mesh_ind
+ offset = g.off_mesh
+ data = np.array(get_component(ds, nfield, index, offset)).flatten()
+ off = ind[field] + data.size
+ mylog.debug("({}, {})[{}:{}]".format(ftype, fname, ind[field], off))
+ rv[ftype, fname][ind[field]:off] = data
+ ind[field] = off
return rv
def _read_data(self, field):
@@ -340,7 +334,7 @@
- keys are (ftype, fname) field tuples
- values are flat numpy arrays with data form that field
"""
- mylog.debug("_read_chunk_data")
+ mylog.debug("_read_chunk_data {} {}".format(chunk, fields))
rv = {}
fluid_fields, particle_fields = [], []
for ftype, fname in fields:
@@ -359,18 +353,19 @@
if g.filename is None:
continue
grids_by_file[g.filename].append(g)
+ rv = {f: np.array([]) for f in fields}
for filename in grids_by_file:
grids = grids_by_file[filename]
grids.sort()
- f = h5py.File(filename, 'r')
- for g in grids:
- for ftype, fname in fluid_fields:
+ #f = h5py.File(filename, 'r')
+ for ftype, fname in fluid_fields:
+ for g in grids:
# TODO update basePath for different files
#data = self._read_data((ftype, fname))
ds = self._handle[self.basePath + self.meshesPath]
nfield = fname.replace("_", "/").replace("-","_")
- data = get_component(ds, nfield)
- mylog.debug("data.shape = {}".format(data.shape))
- rv[(ftype,fname)] = np.array(data).flatten()
- f.close()
+ data = get_component(ds, nfield, g.get_global_startindex()[0], g.ActiveDimensions[0])
+ rv[ftype, fname] = np.concatenate((rv[ftype, fname], np.array(data).flatten()))
+ mylog.debug("rv = {}".format(rv))
+ #f.close()
return rv
https://bitbucket.org/yt_analysis/yt/commits/f56efe1bc220/
Changeset: f56efe1bc220
Branch: yt
User: Fabian Koller
Date: 2016-06-28 09:17:19+00:00
Summary: Fix wrongly calculated offset when reading raw hdf5 data
Affected #: 3 files
diff -r 32265a055ce101287baa2c20a46f3c7e7b114eed -r f56efe1bc220253640b925cb4f626389baf50203 yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -148,13 +148,14 @@
"""
# TODO This only parses one file
f = self.dataset._handle
- meshesPath = f.attrs["meshesPath"]
- particlesPath = f.attrs["particlesPath"]
+ bp = self.basePath
+ mp = f.attrs["meshesPath"]
+ pp = f.attrs["particlesPath"]
output_fields = []
- for group in f[self.basePath + meshesPath].keys():
+ for group in f[bp + mp].keys():
try:
- for direction in f[self.basePath + meshesPath + group].keys():
+ for direction in f[bp + mp + group].keys():
output_fields.append(group + "_" + direction)
except:
# This is for dataSets, they do not have keys
@@ -162,17 +163,17 @@
self.field_list = [("openPMD", str(c)) for c in output_fields]
particle_fields = []
- if self.basePath + particlesPath in f:
- for particleName in f[self.basePath + particlesPath].keys():
- for record in f[self.basePath + particlesPath + particleName].keys():
- if is_const_component(f[self.basePath + particlesPath + particleName + "/" + record]):
+ if bp + pp in f:
+ for particleName in f[bp + pp].keys():
+ for record in f[bp + pp + particleName].keys():
+ if is_const_component(f[bp + pp + particleName + "/" + record]):
# Record itself (e.g. particle_mass) is constant
particle_fields.append(particleName + "_" + record)
elif 'particlePatches' not in record:
try:
# Create a field for every axis (x,y,z) of every property (position)
# of every species (electrons)
- keys = f[self.basePath + particlesPath + particleName + "/" + record].keys()
+ keys = f[bp + pp + particleName + "/" + record].keys()
for axis in keys:
particle_fields.append(particleName + "_" + record + "_" + axis)
except:
@@ -182,7 +183,7 @@
else:
# We probably do not want particlePatches as accessible field lists
pass
- if len(f[self.basePath + particlesPath].keys()) > 1:
+ if len(f[bp + pp].keys()) > 1:
# There is more than one particle species, use the specific names as field types
self.field_list.extend(
[(str(c).split("_")[0], ("particle_" + "_".join(str(c).split("_")[1:]))) for c in particle_fields])
@@ -199,7 +200,7 @@
this must set self.num_grids to be the total number of grids (equiv AMRGridPatch'es) in the simulation
"""
# TODO For the moment we only create grids if there are particles present
- # TODO This breaks datasets with multiple types of particles and different amounts of particles
+ # TODO every species might (read: does) have a different number of active particles
try:
f = self.dataset._handle
bp = self.basePath
@@ -238,30 +239,11 @@
Each of these variables is an array with an entry for each of the self.num_grids grids.
Additionally, grids must be an array of AMRGridPatch objects that already know their IDs.
"""
- # TODO This (particle grid-data) needs to be included in every grid
- # TODO Furthermore, we have to use grids from meshes to represent a physical layout
- # dim = self.dataset.dimensionality
- # for i in range(self.num_grids):
- # if i != self.num_grids-1:
- # self.grid_particle_count[i] = self.ppg
- # else:
- # # The last grid need not be the same size as the previous ones
- # self.grid_particle_count[i] = self.np%self.ppg
- # # The edges have in fact nothing to do with physical boundaries, they are sections of the raw data
- # self.grid_left_edge[i] = [i*self.num_grids**-1] * dim
- # self.grid_right_edge[i] = [(i+1)*self.num_grids**-1] * dim
- # self.grid_dimensions[i] = self.grid_right_edge[i] - self.grid_left_edge[i]
- # mylog.debug("self.grid_left_edge[{}] - {}".format(i, self.grid_left_edge[i]))
- # mylog.debug("self.grid_right_edge[{}] - {}".format(i, self.grid_right_edge[i]))
- # mylog.debug("self.grid_dimensions[{}] - {}".format(i, self.grid_dimensions[i]))
- # mylog.debug("self.grid_particle_count[{}] - {}".format(i, self.grid_particle_count[i]))
-
# There is only one refinement level in openPMD
self.grid_levels.flat[:] = 0
self.grids = np.empty(self.num_grids, dtype='object')
remaining = self.dataset.domain_dimensions[0]
-
for i in range(self.num_grids):
self.grid_left_edge[i] = self.dataset.domain_left_edge.copy() # (N, 3) <= float64
self.grid_left_edge[i][0] = i * self.num_grids**-1
@@ -277,16 +259,10 @@
self.grid_dimensions[i][0] = remaining
nap = self.np % self.ppg
self.grid_particle_count[i] = nap
- # You have to initialize the grids
self.grids[i] = self.grid(
i, self, self.grid_levels[i, 0],
pi=i*self.ppg, op=nap,
mi=int(ceil(self.dataset.domain_dimensions[0] * self.num_grids**-1)) * i, om=self.grid_dimensions[i][0])
- mylog.debug("remaining {}".format(remaining))
- mylog.debug("self.grid_left_edge[{}] - {}".format(i, self.grid_left_edge[i]))
- mylog.debug("self.grid_right_edge[{}] - {}".format(i, self.grid_right_edge[i]))
- mylog.debug("self.grid_dimensions[{}] - {}".format(i, self.grid_dimensions[i]))
- mylog.debug("self.grid_particle_count[{}] - {}".format(i, self.grid_particle_count[i]))
def _populate_grid_objects(self):
"""
@@ -296,9 +272,6 @@
this initializes the grids by calling _prepare_grid() and _setup_dx() on all of them.
Additionally, it should set up Children and Parent lists on each grid object.
"""
-
- # self._reconstruct_parent_child()
-
for i in range(self.num_grids):
self.grids[i]._prepare_grid()
self.grids[i]._setup_dx()
@@ -359,25 +332,26 @@
read in metadata describing the overall data on disk
"""
f = self._handle
- meshesPath = f.attrs["meshesPath"].decode()
- particlesPath = f.attrs["particlesPath"].decode()
+ bp = self.basePath
+ mp = f.attrs["meshesPath"]
+ pp = f.attrs["particlesPath"]
- self.unique_identifier = 0 # no identifier
+ self.unique_identifier = 0
self.parameters = 0 # no additional parameters <= full of code-specific items of use
# We set the highest dimensionality in the simulation as the global one
fshape = None
try:
- for mesh in f[self.basePath + meshesPath].keys():
- for axis in f[self.basePath + meshesPath + "/" + mesh].keys():
- fshape = np.maximum(fshape, f[self.basePath + meshesPath + "/" + mesh + "/" + axis].shape)
+ for mesh in f[bp + mp].keys():
+ for axis in f[bp + mp + "/" + mesh].keys():
+ fshape = np.maximum(fshape, f[bp + mp + "/" + mesh + "/" + axis].shape)
except:
pass
if len(fshape) < 1:
self.dimensionality = 0
- for species in f[self.basePath + particlesPath].keys():
+ for species in f[bp + pp].keys():
self.dimensionality = max(
- len(f[self.basePath + particlesPath + "/" + species].keys()),
+ len(f[bp + pp + "/" + species].keys()),
self.dimensionality)
else:
self.dimensionality = len(fshape)
@@ -392,14 +366,11 @@
self.domain_dimensions = np.ones(3, dtype=np.int64)
self.domain_dimensions[:self.dimensionality] = fshape
- # TODO assumes non-periodic boundary conditions
+ self.current_time = f[bp].attrs["time"]
+ # We here assume non-periodic boundary conditions
self.periodicity = np.zeros(3, dtype=np.bool)
-
- self.current_time = f[self.basePath].attrs["time"] # <= simulation time in code units
-
# Used for AMR, not applicable for us
self.refine_by = 1
-
# Not a cosmological simulation
self.cosmological_simulation = 0
diff -r 32265a055ce101287baa2c20a46f3c7e7b114eed -r f56efe1bc220253640b925cb4f626389baf50203 yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -159,7 +159,7 @@
else:
spec = ptype
pds = ds[self.particlesPath + "/" + spec]
- for field in parallel_objects(field_list, 8):
+ for field in parallel_objects(field_list, -1):
nfield = "/".join(field.split("_")[1:])
data = get_component(pds, nfield, index, offset)
#mylog.debug("data {}".format(data))
@@ -265,7 +265,7 @@
rv[field] = np.empty(size, dtype="float64")
ind[field] = 0
- for ftype, fname in parallel_objects(fields, 8):
+ for ftype, fname in parallel_objects(fields, -1):
field = (ftype, fname)
for chunk in chunks:
for g in chunk.objs:
diff -r 32265a055ce101287baa2c20a46f3c7e7b114eed -r f56efe1bc220253640b925cb4f626389baf50203 yt/frontends/openPMD/misc.py
--- a/yt/frontends/openPMD/misc.py
+++ b/yt/frontends/openPMD/misc.py
@@ -795,7 +795,7 @@
# This allows for masking of the component
# Slicing data directly at the h5py layer improves performance
if offset is not None:
- shape = offset - index
+ shape = offset
else:
shape = record_component.attrs["shape"] - index
# Our component is constant, craft an array by hand
https://bitbucket.org/yt_analysis/yt/commits/f0cf04fefcaf/
Changeset: f0cf04fefcaf
Branch: yt
User: Fabian Koller
Date: 2016-06-28 14:30:37+00:00
Summary: Do not cast position/positionOffset to float64 since we add int32 to float32 which likely will not result in loss of precision,
cache particle positions grid-wise,
preliminary work for Richard's files
Affected #: 2 files
diff -r f56efe1bc220253640b925cb4f626389baf50203 -r f0cf04fefcaf0a25ea715214be5c2370b8ba8f05 yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -40,7 +40,11 @@
class openPMDBasePath:
- def _setBasePath(self, handle, filepath):
+ def _setNonStandardBasePath(self, handle):
+ iteration = handle["/data"].keys()[0]
+ self.basePath = "/data/{}/".format(iteration)
+
+ def _setBasePath(self, handle, filepath):
"""
Set the base path for the first iteration found in the file.
TODO implement into distinct methods:
@@ -90,15 +94,15 @@
class openPMDGrid(AMRGridPatch):
- # TODO THIS IS NOT TRUE FOR THE YT SENSE OF "GRID"
"""
- This class defines the characteristics of the grids
- Actually there is only one grid for the whole simulation box
+ This class defines the characteristics of the grids
"""
_id_offset = 0
__slots__ = ["_level_id"]
+ # TODO consider these for every ptype
part_ind = 0
off_part = 0
+ # TODO consider these for every ftype
mesh_ind = 0
off_mesh = 0
@@ -126,11 +130,13 @@
def __init__(self, ds, dataset_type='openPMD'):
self.dataset_type = dataset_type
- # for now, the index file is the dataset!
self.dataset = ds
self.index_filename = ds.parameter_filename
self.directory = os.path.dirname(self.index_filename)
- self._setBasePath(self.dataset._handle, self.directory)
+ if self.dataset._nonstandard:
+ self._setNonStandardBasePath(self, self.dataset._handle)
+ else:
+ self._setBasePath(self.dataset._handle, self.directory)
GridIndex.__init__(self, ds, dataset_type)
def _detect_output_fields(self):
@@ -194,7 +200,7 @@
def _count_grids(self):
"""
- Counts the number of grids in the dataSet. (Only one in the current standard)
+ Counts the number of grids in the dataSet.
From yt doc:
this must set self.num_grids to be the total number of grids (equiv AMRGridPatch'es) in the simulation
@@ -290,6 +296,7 @@
"""
_index_class = openPMDHierarchy
_field_info_class = openPMDFieldInfo
+ _nonstandard = True
def __init__(self, filename, dataset_type='openPMD',
storage_filename=None,
@@ -297,7 +304,10 @@
unit_system="mks"):
self._handle = HDF5FileHandler(filename)
self._filepath = os.path.dirname(filename)
- self._setBasePath(self._handle, self._filepath)
+ if self._nonstandard:
+ self._setNonStandardBasePath(self._handle)
+ else:
+ self._setBasePath(self._handle, self._filepath)
Dataset.__init__(self, filename, dataset_type,
units_override=units_override,
unit_system=unit_system)
@@ -333,8 +343,12 @@
"""
f = self._handle
bp = self.basePath
- mp = f.attrs["meshesPath"]
- pp = f.attrs["particlesPath"]
+ if self._nonstandard:
+ mp = "fields"
+ pp = "particles"
+ else:
+ mp = f.attrs["meshesPath"]
+ pp = f.attrs["particlesPath"]
self.unique_identifier = 0
self.parameters = 0 # no additional parameters <= full of code-specific items of use
@@ -380,6 +394,9 @@
Checks whether the supplied file adheres to the required openPMD standards
and thus can be read by this frontend
"""
+ if "iexplicitlyacceptthatthismightfail" in args[1]:
+ self._nonstandard = True
+ return True
try:
f = validator.open_file(args[0])
except:
diff -r f56efe1bc220253640b925cb4f626389baf50203 -r f0cf04fefcaf0a25ea715214be5c2370b8ba8f05 yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -51,27 +51,32 @@
self._array_fields = {}
self._cached_ptype = ""
self._cache = {}
+ self._cachecntr = {}
def _fill_cache(self, ptype, index=0, offset=None):
- # Get a particle species (e.g. /data/3500/particles/e/)
- if "io" in ptype:
- spec = self._handle[self.basePath + self.particlesPath].keys()[0]
+ if str((ptype, index, offset)) not in self._cachecntr:
+ # Get a particle species (e.g. /data/3500/particles/e/)
+ if "io" in ptype:
+ spec = self._handle[self.basePath + self.particlesPath].keys()[0]
+ else:
+ spec = ptype
+ pds = self._handle[self.basePath + self.particlesPath + "/" + spec]
+ # Get 1D-Arrays for individual particle positions along axes
+ axes = [str(ax) for ax in pds["position"].keys()]
+ pos = {}
+ off = {}
+ for ax in axes:
+ pos[ax] = get_component(pds, "position/" + ax, index, offset)
+ off[ax] = get_component(pds, "positionOffset/" + ax, index, offset)
+ self._cache[ax] = pos[ax] + off[ax]
+ # Pad accordingly with zeros to make 1D/2D datasets compatible
+ # These have to be the same shape as the existing axes dataset since that equals the number of particles
+ for req in "xyz":
+ if req not in axes:
+ self._cache[req] = np.zeros(self._cache[self._cache.keys()[0]].shape)
+ self._cachecntr[str((ptype, index, offset))] = (self._cache['x'], self._cache['y'], self._cache['z'])
else:
- spec = ptype
- pds = self._handle[self.basePath + self.particlesPath + "/" + spec]
- # Get 1D-Arrays for individual particle positions along axes
- axes = [str(ax) for ax in pds["position"].keys()]
- pos = {}
- off = {}
- for ax in axes:
- pos[ax] = np.array(get_component(pds, "position/" + ax, index, offset), dtype="float64")
- off[ax] = np.array(get_component(pds, "positionOffset/" + ax, index, offset), dtype="float64")
- self._cache[ax] = pos[ax] + off[ax]
- # Pad accordingly with zeros to make 1D/2D datasets compatible
- # These have to be the same shape as the existing axes dataset since that equals the number of particles
- for req in "xyz":
- if req not in axes:
- self._cache[req] = np.zeros(self._cache[self._cache.keys()[0]].shape)
+ self._cache['x'], self._cache['y'], self._cache['z'] = self._cachecntr[str((ptype, index, offset))]
self._cached_ptype = str((ptype, index, offset))
def _read_particle_coords(self, chunks, ptf):
https://bitbucket.org/yt_analysis/yt/commits/ed9544a1f108/
Changeset: ed9544a1f108
Branch: yt
User: Fabian Koller
Date: 2016-07-12 11:38:45+00:00
Summary: Calculate grid-position correctly (fixes wrongly aligned meshplots),
limit cache size,
nonstandard PIC output is now plottable
Affected #: 4 files
diff -r f0cf04fefcaf0a25ea715214be5c2370b8ba8f05 -r ed9544a1f1086c05e32e68a7c715f5cddfa50f23 yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -30,7 +30,7 @@
import numpy as np
import os
import re
-from math import ceil
+from math import ceil, floor
from yt.utilities.logger import ytLogger as mylog
from .misc import get_component, is_const_component
@@ -134,7 +134,7 @@
self.index_filename = ds.parameter_filename
self.directory = os.path.dirname(self.index_filename)
if self.dataset._nonstandard:
- self._setNonStandardBasePath(self, self.dataset._handle)
+ self._setNonStandardBasePath(self.dataset._handle)
else:
self._setBasePath(self.dataset._handle, self.directory)
GridIndex.__init__(self, ds, dataset_type)
@@ -155,8 +155,12 @@
# TODO This only parses one file
f = self.dataset._handle
bp = self.basePath
- mp = f.attrs["meshesPath"]
- pp = f.attrs["particlesPath"]
+ if self.dataset._nonstandard:
+ mp = "fields/"
+ pp = "particles/"
+ else:
+ mp = f.attrs["meshesPath"]
+ pp = f.attrs["particlesPath"]
output_fields = []
for group in f[bp + mp].keys():
@@ -207,28 +211,30 @@
"""
# TODO For the moment we only create grids if there are particles present
# TODO every species might (read: does) have a different number of active particles
- try:
- f = self.dataset._handle
- bp = self.basePath
+ # TODO Calculate the ppg not solely on particle count, also on meshsize
+ f = self.dataset._handle
+ bp = self.basePath
+ if self.dataset._nonstandard:
+ mp = "fields/"
+ pp = "particles/"
+ else:
+ mp = f.attrs["meshesPath"]
pp = f.attrs["particlesPath"]
- spec = f[bp + pp].keys()[0]
- # We could do this by setting an upper bound for the size of each grid
- # Applying a (rough) 100MB restriction to particle data
- pos = f[bp + pp + spec + "/position"].keys()[0]
- if is_const_component(f[bp + pp + spec + "/position/" + pos]):
- self.np = f[bp + pp + spec + "/position/" + pos].attrs["shape"]
- else:
- self.np = f[bp + pp + spec + "/position/" + pos].len()
- gridsize = 100 * 10**6 # Bytes
- # For 3D: about 8 Mio. particles per grid
- # For 2D: about 12,5 Mio. particles per grid
- # For 1D: about 25 Mio. particles per grid
- self.ppg = int(gridsize/(self.dataset.dimensionality*4)) # 4 Byte per value per dimension (f32)
- # Use an upper bound of equally sized grids, last one might be smaller
- self.num_grids = int(ceil(self.np*self.ppg**-1))
- except:
- mylog.info("Could not detect particlePatch, falling back to single grid!")
- self.num_grids = 1
+ spec = f[bp + pp].keys()[0]
+ pos = f[bp + pp + spec + "/position"].keys()[0]
+ # We could do this by setting an upper bound for the size of each grid
+ # Applying a (rough) 100MB restriction to particle data
+ if is_const_component(f[bp + pp + spec + "/position/" + pos]):
+ self.np = f[bp + pp + spec + "/position/" + pos].attrs["shape"]
+ else:
+ self.np = f[bp + pp + spec + "/position/" + pos].len()
+ gridsize = 100 * 10**6 # Bytes
+ # For 3D: about 8 Mio. particles per grid
+ # For 2D: about 12,5 Mio. particles per grid
+ # For 1D: about 25 Mio. particles per grid
+ self.ppg = int(gridsize/(self.dataset.dimensionality*4)) # 4 Byte per value per dimension (f32)
+ # Use an upper bound of equally sized grids, last one might be smaller
+ self.num_grids = int(ceil(self.np*self.ppg**-1))
def _parse_index(self):
"""
@@ -250,25 +256,37 @@
self.grids = np.empty(self.num_grids, dtype='object')
remaining = self.dataset.domain_dimensions[0]
+ meshindex = 0
+ meshedge = self.dataset.domain_left_edge.copy()[0]
for i in range(self.num_grids):
- self.grid_left_edge[i] = self.dataset.domain_left_edge.copy() # (N, 3) <= float64
- self.grid_left_edge[i][0] = i * self.num_grids**-1
- self.grid_right_edge[i] = self.dataset.domain_right_edge.copy() # (N, 3) <= float64
- self.grid_right_edge[i][0] = (i + 1) * self.num_grids**-1
self.grid_dimensions[i] = self.dataset.domain_dimensions # (N, 3) <= int
- if i != self.num_grids-1:
- self.grid_dimensions[i][0] = int(ceil(self.grid_dimensions[i][0] * self.num_grids**-1))
- remaining -= self.grid_dimensions[i][0]
+ prev = remaining
+ remaining -= self.grid_dimensions[i][0] * self.num_grids**-1
+ self.grid_dimensions[i][0] = int(round(prev, 0) - round(remaining, 0))
+ #self.grid_dimensions[i][0] = int(floor(self.dataset.domain_dimensions[0]*self.num_grids**-1))
+ if i != self.num_grids - 1:
nap = self.ppg
else:
# The last grid need not be the same size as the previous ones
- self.grid_dimensions[i][0] = remaining
+ #self.grid_dimensions[i][0] = remaining
nap = self.np % self.ppg
+ self.grid_left_edge[i] = self.dataset.domain_left_edge.copy() # (N, 3) <= float64
+ self.grid_left_edge[i][0] = meshedge
+ self.grid_right_edge[i] = self.dataset.domain_right_edge.copy() # (N, 3) <= float64
+ self.grid_right_edge[i][0] = self.grid_left_edge[i][0] \
+ + self.grid_dimensions[i][0]\
+ * self.dataset.domain_dimensions[0]**-1\
+ * self.dataset.domain_right_edge[0]
+ meshedge = self.grid_right_edge[i][0]
self.grid_particle_count[i] = nap
self.grids[i] = self.grid(
i, self, self.grid_levels[i, 0],
- pi=i*self.ppg, op=nap,
- mi=int(ceil(self.dataset.domain_dimensions[0] * self.num_grids**-1)) * i, om=self.grid_dimensions[i][0])
+ pi=i*self.ppg,
+ op=nap,
+ mi=meshindex,
+ om=self.grid_dimensions[i][0])
+ meshindex += self.grid_dimensions[i][0]
+ remaining -= self.grid_dimensions[i][0]
def _populate_grid_objects(self):
"""
@@ -296,7 +314,7 @@
"""
_index_class = openPMDHierarchy
_field_info_class = openPMDFieldInfo
- _nonstandard = True
+ _nonstandard = False
def __init__(self, filename, dataset_type='openPMD',
storage_filename=None,
@@ -313,7 +331,11 @@
unit_system=unit_system)
self.storage_filename = storage_filename
self.fluid_types += ('openPMD',)
- parts = tuple(str(c) for c in self._handle[self.basePath + self._handle.attrs["particlesPath"]].keys())
+ if self._nonstandard:
+ pp = "particles"
+ else:
+ pp = self._handle.attrs["particlesPath"]
+ parts = tuple(str(c) for c in self._handle[self.basePath + pp].keys())
if len(parts) > 1:
# Only use infile particle names if there is more than one species
self.particle_types = parts
@@ -380,7 +402,10 @@
self.domain_dimensions = np.ones(3, dtype=np.int64)
self.domain_dimensions[:self.dimensionality] = fshape
- self.current_time = f[bp].attrs["time"]
+ if self._nonstandard:
+ self.current_time = 0
+ else:
+ self.current_time = f[bp].attrs["time"]
# We here assume non-periodic boundary conditions
self.periodicity = np.zeros(3, dtype=np.bool)
# Used for AMR, not applicable for us
@@ -394,9 +419,6 @@
Checks whether the supplied file adheres to the required openPMD standards
and thus can be read by this frontend
"""
- if "iexplicitlyacceptthatthismightfail" in args[1]:
- self._nonstandard = True
- return True
try:
f = validator.open_file(args[0])
except:
@@ -411,5 +433,12 @@
# and the meshes
result_array += validator.check_iterations(f, verbose, extension_pic)
if result_array[0] != 0:
- return False
+ try:
+ if "/data" in f and f["/data"].keys()[0].isdigit():
+ self._nonstandard = True
+ mylog.info("Reading a file with the openPMD frontend that does not respect standards? "
+ "Just understand that you're on your own for this!")
+ return True
+ except:
+ return False
return True
diff -r f0cf04fefcaf0a25ea715214be5c2370b8ba8f05 -r ed9544a1f1086c05e32e68a7c715f5cddfa50f23 yt/frontends/openPMD/fields.py
--- a/yt/frontends/openPMD/fields.py
+++ b/yt/frontends/openPMD/fields.py
@@ -120,13 +120,23 @@
def __init__(self, ds, field_list):
f = ds._handle
- fields = f[ds.basePath + f.attrs["meshesPath"]]
+ bp = ds.basePath
+ if ds._nonstandard:
+ mp = "fields/"
+ pp = "particles/"
+ else:
+ mp = f.attrs["meshesPath"]
+ pp = f.attrs["particlesPath"]
+ fields = f[bp + mp]
for fname in fields.keys():
field = fields.get(fname)
if "dataset" in str(field).split(" ")[1]:
- # We have a dataset, don't consider axes since yt does all the work for us
+ # We have a dataset, don't consider axes. This appears to be a vector field of single dimensionality
ytname = str("_".join([fname.replace("_", "-")]))
- parsed = parse_unitDimension(np.asarray(field.attrs["unitDimension"], dtype='int'))
+ if ds._nonstandard:
+ parsed = ""
+ else:
+ parsed = parse_unitDimension(np.asarray(field.attrs["unitDimension"], dtype='int'))
unit = str(yt.YTQuantity(1, parsed).units)
aliases = []
# Save a list of magnetic fields for aliasing later on
@@ -136,9 +146,16 @@
self._mag_fields.append(ytname)
self.known_other_fields += ((ytname, (unit, aliases, None)), )
else:
- for axis in field.attrs["axisLabels"]:
+ if ds._nonstandard:
+ axes = "xyz" # I naively assume all fields in pre-oPMD files are 3D
+ else:
+ axes = field.attrs["axisLabels"]
+ for axis in axes:
ytname = str("_".join([fname.replace("_", "-"), axis]))
- parsed = parse_unitDimension(np.asarray(field.attrs["unitDimension"], dtype='int'))
+ if ds._nonstandard:
+ parsed = ""
+ else:
+ parsed = parse_unitDimension(np.asarray(field.attrs["unitDimension"], dtype='int'))
unit = str(yt.YTQuantity(1, parsed).units)
aliases = []
# Save a list of magnetic fields for aliasing later on
@@ -151,22 +168,31 @@
mylog.debug("oPMD - fields - known_other_fields - {}".format(i))
particle_fields = ()
- particles = f[ds.basePath + f.attrs["particlesPath"]]
+ particles = f[bp + pp]
for species in particles.keys():
for attrib in particles.get(species).keys():
if "weighting" in attrib:
particle_fields += (("particle_weighting", ("", [], None)),)
continue
try:
- parsed = parse_unitDimension(
- np.asarray(particles.get(species).get(attrib).attrs["unitDimension"], dtype='int'))
+ if ds._nonstandard:
+ if "globalCellIdx" in attrib or "position" in attrib:
+ parsed = "m" # Required for spatial selection of particles
+ else:
+ parsed = ""
+ else:
+ parsed = parse_unitDimension(
+ np.asarray(particles.get(species).get(attrib).attrs["unitDimension"], dtype='int'))
unit = str(yt.YTQuantity(1, parsed).units)
name = ["particle", attrib]
for axis in particles.get(species).get(attrib).keys():
+ aliases = []
if axis in "rxyz":
name = ["particle", attrib, axis]
ytname = str("_".join(name))
- particle_fields += ((ytname, (unit, [], None)), )
+ if ds._nonstandard and "globalCellIdx" in ytname:
+ aliases.append(ytname.replace("globalCellIdx", "positionOffset"))
+ particle_fields += ((ytname, (unit, aliases, None)), )
except:
mylog.info("{}_{} does not seem to have unitDimension".format(species, attrib))
self.known_particle_fields = particle_fields
diff -r f0cf04fefcaf0a25ea715214be5c2370b8ba8f05 -r ed9544a1f1086c05e32e68a7c715f5cddfa50f23 yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -25,6 +25,8 @@
from .data_structures import openPMDBasePath
from .misc import get_component
+import random
+
from yt.utilities.parallel_tools.parallel_analysis_interface import \
parallel_objects
@@ -45,9 +47,14 @@
self.ds = ds
self._handle = ds._handle
- self._setBasePath(self._handle, self.ds._filepath)
- self.meshesPath = self._handle["/"].attrs["meshesPath"]
- self.particlesPath = self._handle["/"].attrs["particlesPath"]
+ if self.ds._nonstandard:
+ self._setNonStandardBasePath(self._handle)
+ self.meshesPath = "fields/"
+ self.particlesPath = "particles/"
+ else:
+ self._setBasePath(self._handle, self.ds._filepath)
+ self.meshesPath = self._handle["/"].attrs["meshesPath"]
+ self.particlesPath = self._handle["/"].attrs["particlesPath"]
self._array_fields = {}
self._cached_ptype = ""
self._cache = {}
@@ -55,6 +62,10 @@
def _fill_cache(self, ptype, index=0, offset=None):
if str((ptype, index, offset)) not in self._cachecntr:
+ # Do not let the cache size get out of hand
+ if len(self._cachecntr) > 20:
+ self._cachecntr.pop(random.choice(self._cachecntr.keys()))
+
# Get a particle species (e.g. /data/3500/particles/e/)
if "io" in ptype:
spec = self._handle[self.basePath + self.particlesPath].keys()[0]
@@ -66,8 +77,11 @@
pos = {}
off = {}
for ax in axes:
- pos[ax] = get_component(pds, "position/" + ax, index, offset)
- off[ax] = get_component(pds, "positionOffset/" + ax, index, offset)
+ pos[ax] = get_component(pds, "position/" + ax, index, offset, self.ds._nonstandard)
+ if self.ds._nonstandard:
+ off[ax] = get_component(pds, "globalCellIdx/" + ax, index, offset, self.ds._nonstandard)
+ else:
+ off[ax] = get_component(pds, "positionOffset/" + ax, index, offset, self.ds._nonstandard)
self._cache[ax] = pos[ax] + off[ax]
# Pad accordingly with zeros to make 1D/2D datasets compatible
# These have to be the same shape as the existing axes dataset since that equals the number of particles
@@ -157,6 +171,7 @@
self._fill_cache(ptype, index, offset)
mask = selector.select_points(self._cache['x'], self._cache['y'], self._cache['z'], 0.0)
if mask is None:
+ mylog.info("Mask is none!")
continue
# Get a particle species (e.g. /data/3500/particles/e/)
if "io" in ptype:
@@ -166,7 +181,7 @@
pds = ds[self.particlesPath + "/" + spec]
for field in parallel_objects(field_list, -1):
nfield = "/".join(field.split("_")[1:])
- data = get_component(pds, nfield, index, offset)
+ data = get_component(pds, nfield, index, offset, self.ds._nonstandard)
#mylog.debug("data {}".format(data))
#mylog.debug("mask {}".format(mask))
yield ((ptype, field), data[mask])
@@ -256,8 +271,15 @@
- values are numpy arrays with data form that field
"""
mylog.info("_read_fluid_selection {} {} {} {}".format(chunks, selector, fields, size))
+ f = self._handle
+ bp = self.basePath
+ if self.ds._nonstandard:
+ mp = "fields/"
+ else:
+ mp = f.attrs["meshesPath"]
rv = {}
chunks = list(chunks)
+
if selector.__class__.__name__ == "GridSelector":
if not (len(chunks) == len(chunks[0].objs) == 1):
raise RuntimeError
@@ -274,13 +296,12 @@
field = (ftype, fname)
for chunk in chunks:
for g in chunk.objs:
- ds = self._handle[self.basePath + self.meshesPath]
+ ds = f[bp + mp]
nfield = fname.replace("_", "/").replace("-","_")
index = g.mesh_ind
offset = g.off_mesh
- data = np.array(get_component(ds, nfield, index, offset)).flatten()
+ data = np.array(get_component(ds, nfield, index, offset, self.ds._nonstandard)).flatten()
off = ind[field] + data.size
- mylog.debug("({}, {})[{}:{}]".format(ftype, fname, ind[field], off))
rv[ftype, fname][ind[field]:off] = data
ind[field] = off
return rv
@@ -369,7 +390,7 @@
#data = self._read_data((ftype, fname))
ds = self._handle[self.basePath + self.meshesPath]
nfield = fname.replace("_", "/").replace("-","_")
- data = get_component(ds, nfield, g.get_global_startindex()[0], g.ActiveDimensions[0])
+ data = get_component(ds, nfield, g.get_global_startindex()[0], g.ActiveDimensions[0], self.ds._nonstandard)
rv[ftype, fname] = np.concatenate((rv[ftype, fname], np.array(data).flatten()))
mylog.debug("rv = {}".format(rv))
#f.close()
diff -r f0cf04fefcaf0a25ea715214be5c2370b8ba8f05 -r ed9544a1f1086c05e32e68a7c715f5cddfa50f23 yt/frontends/openPMD/misc.py
--- a/yt/frontends/openPMD/misc.py
+++ b/yt/frontends/openPMD/misc.py
@@ -773,7 +773,7 @@
return "value" in record_component.attrs.keys()
-def get_component(group, component_name, index=0, offset=None):
+def get_component(group, component_name, index=0, offset=None, nonstandard=False):
# TODO - Make slicing possible for higher dimensional data
"""Grab a component from a group
@@ -789,8 +789,23 @@
n-dimensional numpy array filled with values of component
"""
- mylog.debug("openPMD - misc - get_component: {}/{}[{} + {}]".format(group.name, component_name, index, offset))
record_component = group[component_name]
+ if nonstandard:
+ try:
+ unitSI = record_component.attrs["sim_unit"]
+ if "globalCellIdx" in component_name:
+ it = group["/data"].keys()[0]
+ len = group["/data/" + it].attrs['unit_length']
+ # TODO which of these is x,y,z?
+ width = group["/data/" + it].attrs['cell_width']
+ height = group["/data/" + it].attrs['cell_width']
+ depth = group["/data/" + it].attrs['cell_width']
+ unitSI *= len
+ except:
+ unitSI = 1.0
+ else:
+ unitSI = record_component.attrs["unitSI"]
+ #mylog.debug("unitSI is {}".format(unitSI))
if is_const_component(record_component):
# This allows for masking of the component
# Slicing data directly at the h5py layer improves performance
@@ -799,13 +814,15 @@
else:
shape = record_component.attrs["shape"] - index
# Our component is constant, craft an array by hand
- return np.full(shape, record_component.attrs["value"] * record_component.attrs["unitSI"])
+ mylog.debug("openPMD - misc - get_component (const): {}/{}({})".format(group.name, component_name, shape))
+ return np.full(shape, record_component.attrs["value"] * unitSI)
else:
if offset is not None:
offset += index
# Component is a dataset, return it (possibly masked)
# Slicing is not smart, supplying lower dimensional indices only slices those dimensions
- return record_component[index:offset] * record_component.attrs["unitSI"]
+ mylog.debug("openPMD - misc - get_component: {}/{}[{}:{}]".format(group.name, component_name, index, offset))
+ return record_component[index:offset] * unitSI
def parse_unitDimension(unitDimension):
https://bitbucket.org/yt_analysis/yt/commits/699c87dec757/
Changeset: 699c87dec757
Branch: yt
User: Fabian Koller
Date: 2016-07-15 14:16:28+00:00
Summary: Set domain edges according to mesh extent and offset,
actually read fluids selected (no mask was applied before)
Affected #: 3 files
diff -r ed9544a1f1086c05e32e68a7c715f5cddfa50f23 -r 699c87dec7576fc6333f5a9b609058c21806378b yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -373,16 +373,17 @@
pp = f.attrs["particlesPath"]
self.unique_identifier = 0
- self.parameters = 0 # no additional parameters <= full of code-specific items of use
+ self.parameters = 0
- # We set the highest dimensionality in the simulation as the global one
- fshape = None
+ # We assume all fields to have the same shape
try:
- for mesh in f[bp + mp].keys():
- for axis in f[bp + mp + "/" + mesh].keys():
- fshape = np.maximum(fshape, f[bp + mp + "/" + mesh + "/" + axis].shape)
+ mesh = f[bp + mp].keys()[0]
+ axis = f[bp + mp + "/" + mesh].keys()[0]
+ fshape = f[bp + mp + "/" + mesh + "/" + axis].shape
except:
- pass
+ mylog.warning("Could not detect shape of simulated field! "
+ "Assuming a single cell and thus setting fshape to [1, 1, 1]!")
+ fshape = np.array([1, 1, 1])
if len(fshape) < 1:
self.dimensionality = 0
for species in f[bp + pp].keys():
@@ -392,25 +393,40 @@
else:
self.dimensionality = len(fshape)
- # TODO fill me with actual start and end positions in reasonable units
- # This COULD be done with minimum/maximum particle positions in the simulation
- # or through extent of the field meshes
- self.domain_left_edge = np.zeros(3, dtype=np.float64)
- self.domain_right_edge = np.ones(3, dtype=np.float64)
-
# gridding of the meshes (assumed all mesh entries are on the same mesh)
self.domain_dimensions = np.ones(3, dtype=np.int64)
self.domain_dimensions[:self.dimensionality] = fshape
+ self.domain_left_edge = np.empty(3, dtype=np.float64)
+ self.domain_right_edge = np.empty(3, dtype=np.float64)
+ try:
+ mesh = f[bp + mp].keys()[0]
+ if self._nonstandard:
+ offset = np.zeros(3, dtype=np.float64)
+ width = f[bp].attrs['cell_width']
+ height = f[bp].attrs['cell_height']
+ depth = f[bp].attrs['cell_depth']
+ spacing = [width, height, depth]
+ unitSI = f[bp].attrs['unit_length']
+ else:
+ offset = f[bp + mp + "/" + mesh].attrs["gridGlobalOffset"]
+ spacing = f[bp + mp + "/" + mesh].attrs["gridSpacing"]
+ unitSI = f[bp + mp + "/" + mesh].attrs["gridUnitSI"]
+ self.domain_left_edge = offset * unitSI
+ self.domain_right_edge = self.domain_left_edge + self.domain_dimensions * spacing * unitSI
+ except:
+ mylog.warning("The domain extent could not be calculated! Setting the field extent to 1m**dimensionality! "
+ "This WILL break particle-overplotting!")
+ self.domain_left_edge = np.zeros(3, dtype=np.float64)
+ self.domain_right_edge = np.ones(3, dtype=np.float64)
+
if self._nonstandard:
self.current_time = 0
else:
self.current_time = f[bp].attrs["time"]
- # We here assume non-periodic boundary conditions
+
self.periodicity = np.zeros(3, dtype=np.bool)
- # Used for AMR, not applicable for us
self.refine_by = 1
- # Not a cosmological simulation
self.cosmological_simulation = 0
@classmethod
diff -r ed9544a1f1086c05e32e68a7c715f5cddfa50f23 -r 699c87dec7576fc6333f5a9b609058c21806378b yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -33,13 +33,8 @@
class IOHandlerOpenPMD(BaseIOHandler, openPMDBasePath):
- # TODO Data should be loaded chunk-wise to support parallelism. Fields can be chunked arbitrarily in space, particles should be read via particlePatches.
- # TODO Maybe reuse hdf5 file handles in loops?
# TODO Change strategies when dealing with different iterationEncoding?
- # TODO since the standard allows \w+ regexes, replace _'s in oPMD for yt
- # TODO Unify numpy data types?
- # THERE WAS EVEN A GLOBAL VARIABLE FOR THIS
_field_dtype = "float32"
_dataset_type = "openPMD"
@@ -61,10 +56,7 @@
self._cachecntr = {}
def _fill_cache(self, ptype, index=0, offset=None):
- if str((ptype, index, offset)) not in self._cachecntr:
- # Do not let the cache size get out of hand
- if len(self._cachecntr) > 20:
- self._cachecntr.pop(random.choice(self._cachecntr.keys()))
+ if str((ptype, index, offset)) not in self._cached_ptype: #self._cachecntr:
# Get a particle species (e.g. /data/3500/particles/e/)
if "io" in ptype:
@@ -88,9 +80,6 @@
for req in "xyz":
if req not in axes:
self._cache[req] = np.zeros(self._cache[self._cache.keys()[0]].shape)
- self._cachecntr[str((ptype, index, offset))] = (self._cache['x'], self._cache['y'], self._cache['z'])
- else:
- self._cache['x'], self._cache['y'], self._cache['z'] = self._cachecntr[str((ptype, index, offset))]
self._cached_ptype = str((ptype, index, offset))
def _read_particle_coords(self, chunks, ptf):
@@ -179,63 +168,11 @@
else:
spec = ptype
pds = ds[self.particlesPath + "/" + spec]
- for field in parallel_objects(field_list, -1):
+ for field in field_list:
nfield = "/".join(field.split("_")[1:])
data = get_component(pds, nfield, index, offset, self.ds._nonstandard)
- #mylog.debug("data {}".format(data))
- #mylog.debug("mask {}".format(mask))
yield ((ptype, field), data[mask])
-
- # def _read_particle_selection(self, chunks, selector, fields):
- # """
- # Reads masked selection of particles from specified fields.
- #
- # Receives a collection of data "chunks", a selector describing which "chunks" you are concerned with and
- # a list of fields.
- # It should create and return a dictionary whose keys are the fields,
- # and whose values are numpy arrays containing the data.
- # The data should actually be read via the _read_chunk_data() method.
- #
- # Parameters
- # ----------
- # chunks:
- # A list of chunks
- # A chunk is a list of grids
- #
- # selector:
- # yt-project.org/docs/dev/quickstart/data_inspection.html?highlight=selector#Examining-Data-in-Regions
- # yt-project.org/doc/developing/creating_datatypes.html
- # A region (in and/or outside your domain) specifying the field you want to read
- #
- # fields:
- # A list of (fname, ftype) tuples representing a field
- #
- # size:
- # Size of the data arrays you want to read
- #
- # Returns
- # -------
- # A dictionary:
- # - keys are (ftype, fname) tuples representing a field
- # - values are numpy arrays with data form that field
- # """
- # mylog.debug("Read particle selection")
- # rv = {}
- # chunks = list(chunks)
- #
- # if selector.__class__.__name__ == "GridSelector":
- # if not (len(chunks) == len(chunks[0].objs) == 1):
- # raise RuntimeError
- #
- # rv = {f: np.array([]) for f in fields}
- # for chunk in chunks:
- # for grid in chunk.objs:
- # for ftype, fname in fields:
- # data = self._read_particle_fields(chunks, ptf, selector)
- # rv[ftype, fname] = np.concatenate((data, rv[ftype, fname]))
- # return rv
-
def _read_fluid_selection(self, chunks, selector, fields, size):
"""
Reads selected fields of given size from file.
@@ -292,7 +229,7 @@
rv[field] = np.empty(size, dtype="float64")
ind[field] = 0
- for ftype, fname in parallel_objects(fields, -1):
+ for ftype, fname in fields:
field = (ftype, fname)
for chunk in chunks:
for g in chunk.objs:
@@ -300,98 +237,9 @@
nfield = fname.replace("_", "/").replace("-","_")
index = g.mesh_ind
offset = g.off_mesh
- data = np.array(get_component(ds, nfield, index, offset, self.ds._nonstandard)).flatten()
- off = ind[field] + data.size
- rv[ftype, fname][ind[field]:off] = data
- ind[field] = off
+ data = np.array(get_component(ds, nfield, index, offset, self.ds._nonstandard))
+ nd = g.select(selector, data, rv[field], ind[field])
+ ind[field] += nd
+ for i in rv:
+ rv[i].flatten()
return rv
-
- def _read_data(self, field):
- """
- Reads data from file belonging to a (mesh or particle) field
-
- Parameters
- ----------
- fieldname:
- Field to get the data for
-
- Returns
- -------
- A flat numpy array
- """
- ftype, fname = field
- mylog.debug("openPMD - _read_data - reading field: {}.{}".format(ftype, fname))
-
- if ftype in self.ds.particle_types:
- if "io" in ftype:
- # There is only one particle species, use that one ("io" does not suffice)
- spec = self._handle[self.basePath + self.particlesPath].keys()[0]
- else:
- # Use the provided ftype that corresponds to the species name in the hdf5 file
- spec = str(ftype)
- ds = self._handle[self.basePath + self.particlesPath].get(spec)
- nfield = "/".join(fname.split("_")[1:]).replace("-","_")
- else:
- ds = self._handle[self.basePath + self.meshesPath]
- nfield = fname.replace("_", "/").replace("-","_")
- data = get_component(ds, nfield)
- return np.array(data).flatten()
-
- def _read_chunk_data(self, chunk, fields):
- """
- Reads fields specified in the chunk from disk.
-
- From yt doc:
- Receives a "chunk" of data along with a list of fields we want to read. It loops over all the grid objects
- within the "chunk" of data and reads from disk the specific fields,
- returning a dictionary whose keys are the fields and whose values are numpy arrays of the data.
-
- Parameters
- ----------
- chunk:
- A single chunk is a list of grids
-
- fields:
- A list of fields to be read
-
- Returns
- -------
- A dictionary:
- - keys are (ftype, fname) field tuples
- - values are flat numpy arrays with data form that field
- """
- mylog.debug("_read_chunk_data {} {}".format(chunk, fields))
- rv = {}
- fluid_fields, particle_fields = [], []
- for ftype, fname in fields:
- if ftype in self.ds.particle_types:
- particle_fields.append((ftype, fname))
- else:
- fluid_fields.append((ftype, fname))
- if len(particle_fields) > 0:
- selector = AlwaysSelector(self.ds)
- rv.update(self._read_particle_selection(
- [chunk], selector, particle_fields))
- if len(fluid_fields) == 0: return rv
- grids_by_file = defaultdict(list)
- # Read fluid fields
- for g in chunk.objs:
- if g.filename is None:
- continue
- grids_by_file[g.filename].append(g)
- rv = {f: np.array([]) for f in fields}
- for filename in grids_by_file:
- grids = grids_by_file[filename]
- grids.sort()
- #f = h5py.File(filename, 'r')
- for ftype, fname in fluid_fields:
- for g in grids:
- # TODO update basePath for different files
- #data = self._read_data((ftype, fname))
- ds = self._handle[self.basePath + self.meshesPath]
- nfield = fname.replace("_", "/").replace("-","_")
- data = get_component(ds, nfield, g.get_global_startindex()[0], g.ActiveDimensions[0], self.ds._nonstandard)
- rv[ftype, fname] = np.concatenate((rv[ftype, fname], np.array(data).flatten()))
- mylog.debug("rv = {}".format(rv))
- #f.close()
- return rv
diff -r ed9544a1f1086c05e32e68a7c715f5cddfa50f23 -r 699c87dec7576fc6333f5a9b609058c21806378b yt/frontends/openPMD/misc.py
--- a/yt/frontends/openPMD/misc.py
+++ b/yt/frontends/openPMD/misc.py
@@ -783,6 +783,7 @@
component_name: string of a component (relative path) inside the group
index: (optional) start index of data to return
offset: (optional) size of the data to return
+ nonstandard: (optional) bool to signal whether to assume (non)standard PMD
Returns
-------
@@ -796,31 +797,23 @@
if "globalCellIdx" in component_name:
it = group["/data"].keys()[0]
len = group["/data/" + it].attrs['unit_length']
- # TODO which of these is x,y,z?
- width = group["/data/" + it].attrs['cell_width']
- height = group["/data/" + it].attrs['cell_width']
- depth = group["/data/" + it].attrs['cell_width']
unitSI *= len
except:
unitSI = 1.0
else:
unitSI = record_component.attrs["unitSI"]
- #mylog.debug("unitSI is {}".format(unitSI))
if is_const_component(record_component):
- # This allows for masking of the component
- # Slicing data directly at the h5py layer improves performance
if offset is not None:
shape = offset
else:
shape = record_component.attrs["shape"] - index
- # Our component is constant, craft an array by hand
+ # component is constant, craft an array by hand
mylog.debug("openPMD - misc - get_component (const): {}/{}({})".format(group.name, component_name, shape))
return np.full(shape, record_component.attrs["value"] * unitSI)
else:
if offset is not None:
offset += index
- # Component is a dataset, return it (possibly masked)
- # Slicing is not smart, supplying lower dimensional indices only slices those dimensions
+ # component is a dataset, return it (possibly masked)
mylog.debug("openPMD - misc - get_component: {}/{}[{}:{}]".format(group.name, component_name, index, offset))
return record_component[index:offset] * unitSI
https://bitbucket.org/yt_analysis/yt/commits/6badcbfacceb/
Changeset: 6badcbfacceb
Branch: yt
User: Fabian Koller
Date: 2016-07-19 12:03:22+00:00
Summary: Consider individual chunk sizes for different ptypes,
Workaround for 1D/2D data
Affected #: 2 files
diff -r 699c87dec7576fc6333f5a9b609058c21806378b -r 6badcbfacceb8ab728e60491a62ab4aedb756bf0 yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -99,18 +99,19 @@
"""
_id_offset = 0
__slots__ = ["_level_id"]
- # TODO consider these for every ptype
- part_ind = 0
- off_part = 0
- # TODO consider these for every ftype
+ part_ind = {}
+ off_part = {}
+ # TODO (maybe) consider these for every ftype
mesh_ind = 0
off_mesh = 0
- def __init__(self, id, index, level=-1, pi=0, op=0, mi=0, om=0):
+ def __init__(self, id, index, level=-1, pi=None, op=None, mi=0, om=0):
AMRGridPatch.__init__(self, id, filename=index.index_filename,
index=index)
- self.part_ind = pi
- self.off_part = op
+ if pi:
+ self.part_ind = pi
+ if op:
+ self.off_part = op
self.mesh_ind = mi
self.off_mesh = om
self.Parent = None
@@ -210,7 +211,6 @@
this must set self.num_grids to be the total number of grids (equiv AMRGridPatch'es) in the simulation
"""
# TODO For the moment we only create grids if there are particles present
- # TODO every species might (read: does) have a different number of active particles
# TODO Calculate the ppg not solely on particle count, also on meshsize
f = self.dataset._handle
bp = self.basePath
@@ -220,21 +220,24 @@
else:
mp = f.attrs["meshesPath"]
pp = f.attrs["particlesPath"]
- spec = f[bp + pp].keys()[0]
- pos = f[bp + pp + spec + "/position"].keys()[0]
- # We could do this by setting an upper bound for the size of each grid
- # Applying a (rough) 100MB restriction to particle data
- if is_const_component(f[bp + pp + spec + "/position/" + pos]):
- self.np = f[bp + pp + spec + "/position/" + pos].attrs["shape"]
- else:
- self.np = f[bp + pp + spec + "/position/" + pos].len()
gridsize = 100 * 10**6 # Bytes
+ species = f[bp + pp].keys()
+ self.np = {}
+ maxnp = 0
+ for spec in species:
+ pos = f[bp + pp + spec + "/position"].keys()[0]
+ if is_const_component(f[bp + pp + spec + "/position/" + pos]):
+ self.np[spec] = f[bp + pp + spec + "/position/" + pos].attrs["shape"]
+ else:
+ self.np[spec] = f[bp + pp + spec + "/position/" + pos].len()
+ if self.np[spec] > maxnp:
+ maxnp = self.np[spec]
# For 3D: about 8 Mio. particles per grid
# For 2D: about 12,5 Mio. particles per grid
# For 1D: about 25 Mio. particles per grid
- self.ppg = int(gridsize/(self.dataset.dimensionality*4)) # 4 Byte per value per dimension (f32)
+ ppg = int(gridsize/(self.dataset.dimensionality*4)) # 4 Byte per value per dimension (f32)
# Use an upper bound of equally sized grids, last one might be smaller
- self.num_grids = int(ceil(self.np*self.ppg**-1))
+ self.num_grids = int(ceil(maxnp * ppg**-1))
def _parse_index(self):
"""
@@ -255,6 +258,10 @@
self.grid_levels.flat[:] = 0
self.grids = np.empty(self.num_grids, dtype='object')
+ nrp = self.np.copy() # Number of remaining particles from the dataset
+ pci = {} # Index for particle chunk
+ for spec in nrp:
+ pci[spec] = 0
remaining = self.dataset.domain_dimensions[0]
meshindex = 0
meshedge = self.dataset.domain_left_edge.copy()[0]
@@ -263,13 +270,6 @@
prev = remaining
remaining -= self.grid_dimensions[i][0] * self.num_grids**-1
self.grid_dimensions[i][0] = int(round(prev, 0) - round(remaining, 0))
- #self.grid_dimensions[i][0] = int(floor(self.dataset.domain_dimensions[0]*self.num_grids**-1))
- if i != self.num_grids - 1:
- nap = self.ppg
- else:
- # The last grid need not be the same size as the previous ones
- #self.grid_dimensions[i][0] = remaining
- nap = self.np % self.ppg
self.grid_left_edge[i] = self.dataset.domain_left_edge.copy() # (N, 3) <= float64
self.grid_left_edge[i][0] = meshedge
self.grid_right_edge[i] = self.dataset.domain_right_edge.copy() # (N, 3) <= float64
@@ -278,13 +278,26 @@
* self.dataset.domain_dimensions[0]**-1\
* self.dataset.domain_right_edge[0]
meshedge = self.grid_right_edge[i][0]
- self.grid_particle_count[i] = nap
+ particlecount = []
+ particleindex = []
+ for spec in self.np:
+ particleindex += [(spec, pci[spec])]
+ if i is (self.num_grids - 1):
+ # The last grid need not be the same size as the previous ones
+ num = nrp[spec]
+ else:
+ num = int(floor(self.np[spec] * self.num_grids**-1))
+ particlecount += [(spec, num)]
+ nrp[spec] -= num
+ self.grid_particle_count[i] += num
self.grids[i] = self.grid(
i, self, self.grid_levels[i, 0],
- pi=i*self.ppg,
- op=nap,
+ pi=particleindex,
+ op=particlecount,
mi=meshindex,
om=self.grid_dimensions[i][0])
+ for spec, val in particlecount:
+ pci[spec] += val
meshindex += self.grid_dimensions[i][0]
remaining -= self.grid_dimensions[i][0]
@@ -397,8 +410,8 @@
self.domain_dimensions = np.ones(3, dtype=np.int64)
self.domain_dimensions[:self.dimensionality] = fshape
- self.domain_left_edge = np.empty(3, dtype=np.float64)
- self.domain_right_edge = np.empty(3, dtype=np.float64)
+ self.domain_left_edge = np.zeros(3, dtype=np.float64)
+ self.domain_right_edge = np.ones(3, dtype=np.float64)
try:
mesh = f[bp + mp].keys()[0]
if self._nonstandard:
@@ -412,10 +425,13 @@
offset = f[bp + mp + "/" + mesh].attrs["gridGlobalOffset"]
spacing = f[bp + mp + "/" + mesh].attrs["gridSpacing"]
unitSI = f[bp + mp + "/" + mesh].attrs["gridUnitSI"]
- self.domain_left_edge = offset * unitSI
- self.domain_right_edge = self.domain_left_edge + self.domain_dimensions * spacing * unitSI
+ dim = len(spacing)
+ self.domain_left_edge[:dim] += offset * unitSI
+ self.domain_right_edge *= self.domain_dimensions * unitSI
+ self.domain_right_edge[:dim] *= spacing
+ self.domain_right_edge += self.domain_left_edge
except:
- mylog.warning("The domain extent could not be calculated! Setting the field extent to 1m**dimensionality! "
+ mylog.warning("The domain extent could not be calculated! Setting the field extent to 1m**3! "
"This WILL break particle-overplotting!")
self.domain_left_edge = np.zeros(3, dtype=np.float64)
self.domain_right_edge = np.ones(3, dtype=np.float64)
diff -r 699c87dec7576fc6333f5a9b609058c21806378b -r 6badcbfacceb8ab728e60491a62ab4aedb756bf0 yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -17,20 +17,12 @@
from yt.utilities.io_handler import \
BaseIOHandler
from yt.extern.six import u, b, iteritems
-from yt.geometry.selection_routines import mask_fill, AlwaysSelector
from yt.utilities.logger import ytLogger as mylog
import h5py
import numpy as np
-from collections import defaultdict
from .data_structures import openPMDBasePath
from .misc import get_component
-import random
-
-
-from yt.utilities.parallel_tools.parallel_analysis_interface import \
- parallel_objects
-
class IOHandlerOpenPMD(BaseIOHandler, openPMDBasePath):
# TODO Change strategies when dealing with different iterationEncoding?
@@ -56,8 +48,7 @@
self._cachecntr = {}
def _fill_cache(self, ptype, index=0, offset=None):
- if str((ptype, index, offset)) not in self._cached_ptype: #self._cachecntr:
-
+ if str((ptype, index, offset)) not in self._cached_ptype:
# Get a particle species (e.g. /data/3500/particles/e/)
if "io" in ptype:
spec = self._handle[self.basePath + self.particlesPath].keys()[0]
@@ -104,14 +95,24 @@
x_coords, y_coords and z_coords are arrays of positions/coordinates of all particles of that type
"""
chunks = list(chunks)
+ f = self._handle
+ ds = f[self.basePath]
for chunk in chunks:
for g in chunk.objs:
if g.filename is None:
continue
for ptype, field_list in sorted(ptf.items()):
mylog.debug("openPMD - _read_particle_coords: (grid {}) {}, {}".format(g, ptype, field_list))
- index = g.part_ind
- offset = g.off_part
+ if "io" in ptype:
+ spec = ds[self.particlesPath].keys()[0]
+ else:
+ spec = ptype
+ for gridptype, idx in g.part_ind:
+ if str(gridptype) == str(spec):
+ index = idx
+ for gridptype, ofs in g.off_part:
+ if str(gridptype) == str(spec):
+ offset = ofs
if str((ptype, index, offset)) not in self._cached_ptype:
self._fill_cache(ptype, index, offset)
yield (ptype, (self._cache['x'], self._cache['y'], self._cache['z']))
@@ -146,27 +147,31 @@
- in a specified field
"""
chunks = list(chunks)
+ f = self._handle
+ ds = f[self.basePath]
for chunk in chunks:
for g in chunk.objs:
if g.filename is None:
continue
- f = self._handle
- ds = f[self.basePath]
- for ptype, field_list in parallel_objects(sorted(ptf.items())):
+ for ptype, field_list in sorted(ptf.items()):
mylog.debug("openPMD - _read_particle_fields: (grid {}) {}, {}".format(g, ptype, field_list))
- index = g.part_ind
- offset = g.off_part
- if str((ptype, index, offset)) not in self._cached_ptype:
- self._fill_cache(ptype, index, offset)
- mask = selector.select_points(self._cache['x'], self._cache['y'], self._cache['z'], 0.0)
- if mask is None:
- mylog.info("Mask is none!")
- continue
# Get a particle species (e.g. /data/3500/particles/e/)
if "io" in ptype:
spec = ds[self.particlesPath].keys()[0]
else:
spec = ptype
+ for gridptype, idx in g.part_ind:
+ if str(gridptype) == str(spec):
+ index = idx
+ for gridptype, ofs in g.off_part:
+ if str(gridptype) == str(spec):
+ offset = ofs
+ if str((ptype, index, offset)) not in self._cached_ptype:
+ self._fill_cache(ptype, index, offset)
+ mask = selector.select_points(self._cache['x'], self._cache['y'], self._cache['z'], 0.0)
+ if mask is None:
+ mylog.debug("Particle selection mask is None!")
+ continue
pds = ds[self.particlesPath + "/" + spec]
for field in field_list:
nfield = "/".join(field.split("_")[1:])
@@ -238,8 +243,13 @@
index = g.mesh_ind
offset = g.off_mesh
data = np.array(get_component(ds, nfield, index, offset, self.ds._nonstandard))
- nd = g.select(selector, data, rv[field], ind[field])
- ind[field] += nd
+ # The following is a modified AMRGridPatch.select(...)
+ mask = g._get_selector_mask(selector)
+ if data.shape is not mask.shape:
+ data = data.reshape(mask.shape) # Workaround - casts a 2D (x,y) array to 3D (x,y,1)
+ count = g.count(selector)
+ rv[field][ind[field]:ind[field] + count] = data[mask]
+ ind[field] += count
for i in rv:
rv[i].flatten()
return rv
https://bitbucket.org/yt_analysis/yt/commits/ead6c60e0c38/
Changeset: ead6c60e0c38
Branch: yt
User: Fabian Koller
Date: 2016-08-01 12:47:13+00:00
Summary: Do not consider the global offset of fields (moving window),
change naming scheme of "position" to fit yt's assumptions,
clean up codebase
Affected #: 5 files
diff -r 6badcbfacceb8ab728e60491a62ab4aedb756bf0 -r ead6c60e0c388127a655153d06ed9d09e30f77fe yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -26,18 +26,14 @@
import yt.frontends.openPMD.misc as validator
-import h5py
import numpy as np
import os
import re
from math import ceil, floor
from yt.utilities.logger import ytLogger as mylog
-from .misc import get_component, is_const_component
-
-class openPMDBasePathException(Exception):
- pass
-
+def is_const_component(record_component):
+ return "value" in record_component.attrs.keys()
class openPMDBasePath:
def _setNonStandardBasePath(self, handle):
@@ -52,23 +48,10 @@
- getIterations(self)
- getBasePath(self, iteration)
"""
- # basePath is fixed in openPMD 1.X to `/data/%T/`
- dataPath = u"/data"
-
- # if the file messed up the base path we avoid throwing a cluttered
- # exception below while looking for iterations:
- if handle.attrs["basePath"].decode("utf-8") != u"/data/%T/":
- raise openPMDBasePathException("openPMD: basePath is non-standard!")
-
- # does `/data/` exist?
- if not u"/data" in handle:
- raise openPMDBasePathException("openPMD: group for basePath does not exist!")
-
- # TODO Everything prior to this should (in theory) already be done by the validator
# find iterations in basePath
list_iterations = []
if u"groupBased" in handle.attrs["iterationEncoding"]:
- for i in list(handle[dataPath].keys()):
+ for i in list(handle["/data"].keys()):
list_iterations.append(i)
mylog.info("openPMD: found {} iterations in file".format(len(list_iterations)))
elif u"fileBased" in handle.attrs["iterationEncoding"]:
@@ -89,8 +72,8 @@
# just handle the first iteration found
if u"groupBased" in handle.attrs["iterationEncoding"] and len(list_iterations) > 1:
- mylog.warning("openPMD: only choose to load one iteration ({})".format(handle[dataPath].keys()[0]))
- self.basePath = "{}/{}/".format(dataPath, handle[dataPath].keys()[0])
+ mylog.warning("openPMD: only choose to load one iteration ({})".format(handle["/data"].keys()[0]))
+ self.basePath = "{}/{}/".format("/data", handle["/data"].keys()[0])
class openPMDGrid(AMRGridPatch):
@@ -185,6 +168,8 @@
# Create a field for every axis (x,y,z) of every property (position)
# of every species (electrons)
keys = f[bp + pp + particleName + "/" + record].keys()
+ if record in "position":
+ record = "positionCoarse"
for axis in keys:
particle_fields.append(particleName + "_" + record + "_" + axis)
except:
@@ -232,9 +217,7 @@
self.np[spec] = f[bp + pp + spec + "/position/" + pos].len()
if self.np[spec] > maxnp:
maxnp = self.np[spec]
- # For 3D: about 8 Mio. particles per grid
- # For 2D: about 12,5 Mio. particles per grid
- # For 1D: about 25 Mio. particles per grid
+ # Limit particles per grid by resulting memory footprint
ppg = int(gridsize/(self.dataset.dimensionality*4)) # 4 Byte per value per dimension (f32)
# Use an upper bound of equally sized grids, last one might be smaller
self.num_grids = int(ceil(maxnp * ppg**-1))
@@ -392,11 +375,11 @@
try:
mesh = f[bp + mp].keys()[0]
axis = f[bp + mp + "/" + mesh].keys()[0]
- fshape = f[bp + mp + "/" + mesh + "/" + axis].shape
+ fshape = np.asarray(f[bp + mp + "/" + mesh + "/" + axis].shape, dtype=np.int64)
except:
+ fshape = np.array([1, 1, 1], dtype=np.int64)
mylog.warning("Could not detect shape of simulated field! "
"Assuming a single cell and thus setting fshape to [1, 1, 1]!")
- fshape = np.array([1, 1, 1])
if len(fshape) < 1:
self.dimensionality = 0
for species in f[bp + pp].keys():
@@ -406,34 +389,26 @@
else:
self.dimensionality = len(fshape)
- # gridding of the meshes (assumed all mesh entries are on the same mesh)
- self.domain_dimensions = np.ones(3, dtype=np.int64)
- self.domain_dimensions[:self.dimensionality] = fshape
+ fshape = np.append(fshape, np.ones(3 - self.dimensionality))
+ self.domain_dimensions = fshape
self.domain_left_edge = np.zeros(3, dtype=np.float64)
- self.domain_right_edge = np.ones(3, dtype=np.float64)
try:
mesh = f[bp + mp].keys()[0]
if self._nonstandard:
- offset = np.zeros(3, dtype=np.float64)
width = f[bp].attrs['cell_width']
height = f[bp].attrs['cell_height']
depth = f[bp].attrs['cell_depth']
- spacing = [width, height, depth]
+ spacing = np.asarray([width, height, depth])
unitSI = f[bp].attrs['unit_length']
else:
- offset = f[bp + mp + "/" + mesh].attrs["gridGlobalOffset"]
- spacing = f[bp + mp + "/" + mesh].attrs["gridSpacing"]
+ spacing = np.asarray(f[bp + mp + "/" + mesh].attrs["gridSpacing"])
+ np.append(spacing, np.ones(3 - len(spacing)))
unitSI = f[bp + mp + "/" + mesh].attrs["gridUnitSI"]
- dim = len(spacing)
- self.domain_left_edge[:dim] += offset * unitSI
- self.domain_right_edge *= self.domain_dimensions * unitSI
- self.domain_right_edge[:dim] *= spacing
- self.domain_right_edge += self.domain_left_edge
+ self.domain_right_edge = self.domain_dimensions * unitSI * spacing
except:
mylog.warning("The domain extent could not be calculated! Setting the field extent to 1m**3! "
"This WILL break particle-overplotting!")
- self.domain_left_edge = np.zeros(3, dtype=np.float64)
self.domain_right_edge = np.ones(3, dtype=np.float64)
if self._nonstandard:
@@ -464,6 +439,8 @@
# Go through all the iterations, checking both the particles
# and the meshes
result_array += validator.check_iterations(f, verbose, extension_pic)
+
+ # this might still be a compatible file not fully respecting openPMD standards
if result_array[0] != 0:
try:
if "/data" in f and f["/data"].keys()[0].isdigit():
diff -r 6badcbfacceb8ab728e60491a62ab4aedb756bf0 -r ead6c60e0c388127a655153d06ed9d09e30f77fe yt/frontends/openPMD/fields.py
--- a/yt/frontends/openPMD/fields.py
+++ b/yt/frontends/openPMD/fields.py
@@ -89,18 +89,18 @@
units="T*V/m")
-def setup_relative_positions(self, ptype):
- def _rel_pos(axis):
- def rp(field, data):
- pos = data[ptype, "particle_position_{}".format(axis)]
+def setup_absolute_positions(self, ptype):
+ def _abs_pos(axis):
+ def ap(field, data):
+ pos = data[ptype, "particle_positionCoarse_{}".format(axis)]
off = data[ptype, "particle_positionOffset_{}".format(axis)]
return pos + off
- return rp
+ return ap
for ax in 'xyz':
- self.add_field((ptype, "particle_position_relative_%s" % ax),
- function=_rel_pos(ax),
+ self.add_field((ptype, "particle_position_%s" % ax),
+ function=_abs_pos(ax),
units="m",
particle_type=True)
@@ -185,10 +185,13 @@
np.asarray(particles.get(species).get(attrib).attrs["unitDimension"], dtype='int'))
unit = str(yt.YTQuantity(1, parsed).units)
name = ["particle", attrib]
+ ytattrib = attrib
+ if ytattrib in "position":
+ ytattrib = "positionCoarse"
for axis in particles.get(species).get(attrib).keys():
aliases = []
if axis in "rxyz":
- name = ["particle", attrib, axis]
+ name = ["particle", ytattrib, axis]
ytname = str("_".join(name))
if ds._nonstandard and "globalCellIdx" in ytname:
aliases.append(ytname.replace("globalCellIdx", "positionOffset"))
@@ -220,7 +223,7 @@
TODO You have to call the function of the parent class to load particles
"""
- setup_relative_positions(self, ptype)
+ setup_absolute_positions(self, ptype)
#setup_kinetic_energy(self, ptype)
#setup_velocity(self, ptype)
super(openPMDFieldInfo, self).setup_particle_fields(ptype)
diff -r 6badcbfacceb8ab728e60491a62ab4aedb756bf0 -r ead6c60e0c388127a655153d06ed9d09e30f77fe yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -16,17 +16,12 @@
from yt.utilities.io_handler import \
BaseIOHandler
-from yt.extern.six import u, b, iteritems
from yt.utilities.logger import ytLogger as mylog
-import h5py
import numpy as np
from .data_structures import openPMDBasePath
-from .misc import get_component
class IOHandlerOpenPMD(BaseIOHandler, openPMDBasePath):
- # TODO Change strategies when dealing with different iterationEncoding?
-
_field_dtype = "float32"
_dataset_type = "openPMD"
@@ -60,11 +55,11 @@
pos = {}
off = {}
for ax in axes:
- pos[ax] = get_component(pds, "position/" + ax, index, offset, self.ds._nonstandard)
+ pos[ax] = self.get_component(pds, "position/" + ax, index, offset, self.ds._nonstandard)
if self.ds._nonstandard:
- off[ax] = get_component(pds, "globalCellIdx/" + ax, index, offset, self.ds._nonstandard)
+ off[ax] = self.get_component(pds, "globalCellIdx/" + ax, index, offset, self.ds._nonstandard)
else:
- off[ax] = get_component(pds, "positionOffset/" + ax, index, offset, self.ds._nonstandard)
+ off[ax] = self.get_component(pds, "positionOffset/" + ax, index, offset, self.ds._nonstandard)
self._cache[ax] = pos[ax] + off[ax]
# Pad accordingly with zeros to make 1D/2D datasets compatible
# These have to be the same shape as the existing axes dataset since that equals the number of particles
@@ -102,7 +97,6 @@
if g.filename is None:
continue
for ptype, field_list in sorted(ptf.items()):
- mylog.debug("openPMD - _read_particle_coords: (grid {}) {}, {}".format(g, ptype, field_list))
if "io" in ptype:
spec = ds[self.particlesPath].keys()[0]
else:
@@ -113,6 +107,7 @@
for gridptype, ofs in g.off_part:
if str(gridptype) == str(spec):
offset = ofs
+ mylog.debug("openPMD - _read_particle_coords: (grid {}) {}, {}".format(g, ptype, field_list))
if str((ptype, index, offset)) not in self._cached_ptype:
self._fill_cache(ptype, index, offset)
yield (ptype, (self._cache['x'], self._cache['y'], self._cache['z']))
@@ -154,7 +149,6 @@
if g.filename is None:
continue
for ptype, field_list in sorted(ptf.items()):
- mylog.debug("openPMD - _read_particle_fields: (grid {}) {}, {}".format(g, ptype, field_list))
# Get a particle species (e.g. /data/3500/particles/e/)
if "io" in ptype:
spec = ds[self.particlesPath].keys()[0]
@@ -168,14 +162,15 @@
offset = ofs
if str((ptype, index, offset)) not in self._cached_ptype:
self._fill_cache(ptype, index, offset)
+ mylog.debug("openPMD - _read_particle_fields: (grid {}) {}, {}".format(g, ptype, field_list))
mask = selector.select_points(self._cache['x'], self._cache['y'], self._cache['z'], 0.0)
if mask is None:
mylog.debug("Particle selection mask is None!")
continue
pds = ds[self.particlesPath + "/" + spec]
for field in field_list:
- nfield = "/".join(field.split("_")[1:])
- data = get_component(pds, nfield, index, offset, self.ds._nonstandard)
+ nfield = "/".join(field.split("_")[1:]).replace("positionCoarse", "position")
+ data = self.get_component(pds, nfield, index, offset, self.ds._nonstandard)
yield ((ptype, field), data[mask])
def _read_fluid_selection(self, chunks, selector, fields, size):
@@ -242,14 +237,58 @@
nfield = fname.replace("_", "/").replace("-","_")
index = g.mesh_ind
offset = g.off_mesh
- data = np.array(get_component(ds, nfield, index, offset, self.ds._nonstandard))
+ data = np.array(self.get_component(ds, nfield, index, offset, self.ds._nonstandard))
# The following is a modified AMRGridPatch.select(...)
mask = g._get_selector_mask(selector)
- if data.shape is not mask.shape:
- data = data.reshape(mask.shape) # Workaround - casts a 2D (x,y) array to 3D (x,y,1)
+ data = data.reshape(mask.shape) # Workaround - casts a 2D (x,y) array to 3D (x,y,1)
count = g.count(selector)
rv[field][ind[field]:ind[field] + count] = data[mask]
ind[field] += count
for i in rv:
rv[i].flatten()
return rv
+
+ def get_component(self, group, component_name, index=0, offset=None, nonstandard=False):
+ """Grab a component from a group
+
+ Parameters
+ ----------
+ group: hdf5 group to get whole/masked component from
+ component_name: string of a component (relative path) inside the group
+ index: (optional) start index of data to return
+ offset: (optional) size of the data to return
+ nonstandard: (optional) bool to signal whether to assume (non)standard PMD
+
+ Returns
+ -------
+ n-dimensional numpy array filled with values of component
+ """
+
+ record_component = group[component_name]
+ if nonstandard:
+ try:
+ unitSI = record_component.attrs["sim_unit"]
+ if "globalCellIdx" in component_name:
+ it = group["/data"].keys()[0]
+ len = group["/data/" + it].attrs['unit_length']
+ unitSI *= len
+ except:
+ unitSI = 1.0
+ else:
+ unitSI = record_component.attrs["unitSI"]
+ # check whether component is constant
+ if "value" in record_component.attrs.keys():
+ if offset is not None:
+ shape = offset
+ else:
+ shape = record_component.attrs["shape"] - index
+ # component is constant, craft an array by hand
+ mylog.debug("openPMD - misc - get_component (const): {}/{}({})".format(group.name, component_name, shape))
+ return np.full(shape, record_component.attrs["value"] * unitSI)
+ else:
+ if offset is not None:
+ offset += index
+ # component is a dataset, return it (possibly masked)
+ mylog.debug(
+ "openPMD - misc - get_component: {}/{}[{}:{}]".format(group.name, component_name, index, offset))
+ return record_component[index:offset] * unitSI
diff -r 6badcbfacceb8ab728e60491a62ab4aedb756bf0 -r ead6c60e0c388127a655153d06ed9d09e30f77fe yt/frontends/openPMD/misc.py
--- a/yt/frontends/openPMD/misc.py
+++ b/yt/frontends/openPMD/misc.py
@@ -769,55 +769,6 @@
return result_array
-def is_const_component(record_component):
- return "value" in record_component.attrs.keys()
-
-
-def get_component(group, component_name, index=0, offset=None, nonstandard=False):
- # TODO - Make slicing possible for higher dimensional data
- """Grab a component from a group
-
- Parameters
- ----------
- group: hdf5 group to get whole/masked component from
- component_name: string of a component (relative path) inside the group
- index: (optional) start index of data to return
- offset: (optional) size of the data to return
- nonstandard: (optional) bool to signal whether to assume (non)standard PMD
-
- Returns
- -------
- n-dimensional numpy array filled with values of component
- """
-
- record_component = group[component_name]
- if nonstandard:
- try:
- unitSI = record_component.attrs["sim_unit"]
- if "globalCellIdx" in component_name:
- it = group["/data"].keys()[0]
- len = group["/data/" + it].attrs['unit_length']
- unitSI *= len
- except:
- unitSI = 1.0
- else:
- unitSI = record_component.attrs["unitSI"]
- if is_const_component(record_component):
- if offset is not None:
- shape = offset
- else:
- shape = record_component.attrs["shape"] - index
- # component is constant, craft an array by hand
- mylog.debug("openPMD - misc - get_component (const): {}/{}({})".format(group.name, component_name, shape))
- return np.full(shape, record_component.attrs["value"] * unitSI)
- else:
- if offset is not None:
- offset += index
- # component is a dataset, return it (possibly masked)
- mylog.debug("openPMD - misc - get_component: {}/{}[{}:{}]".format(group.name, component_name, index, offset))
- return record_component[index:offset] * unitSI
-
-
def parse_unitDimension(unitDimension):
if len(unitDimension) is not 7:
mylog.error("SI must have 7 base dimensions! {} is off by {}".format(unitDimension, len(unitDimension) - 7))
diff -r 6badcbfacceb8ab728e60491a62ab4aedb756bf0 -r ead6c60e0c388127a655153d06ed9d09e30f77fe yt/frontends/openPMD/setup.py
--- a/yt/frontends/openPMD/setup.py
+++ b/yt/frontends/openPMD/setup.py
@@ -8,11 +8,6 @@
The full license is in the file COPYING.txt, distributed with this software.
"""
-import setuptools
-import os
-import sys
-import os.path
-
def configuration(parent_package='', top_path=None):
from numpy.distutils.misc_util import Configuration
https://bitbucket.org/yt_analysis/yt/commits/1096d01db5b6/
Changeset: 1096d01db5b6
Branch: yt
User: Fabian Koller
Date: 2016-08-02 13:38:25+00:00
Summary: Field extent for 1D/2D data should not cause errors any more,
adhering to yt coding standards a little more
Affected #: 4 files
diff -r ead6c60e0c388127a655153d06ed9d09e30f77fe -r 1096d01db5b6bbbe6dbadfd2b9995e614687793b yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -7,6 +7,7 @@
#-----------------------------------------------------------------------------
# Copyright (c) 2013, yt Development Team.
# Copyright (c) 2015, Daniel Grassinger (HZDR)
+# Copyright (c) 2016, Fabian Koller (HZDR)
#
# Distributed under the terms of the Modified BSD License.
#
@@ -136,7 +137,6 @@
From yt doc:
self.field_list must be populated as a list of strings corresponding to "native" fields in the data files.
"""
- # TODO This only parses one file
f = self.dataset._handle
bp = self.basePath
if self.dataset._nonstandard:
@@ -403,12 +403,12 @@
unitSI = f[bp].attrs['unit_length']
else:
spacing = np.asarray(f[bp + mp + "/" + mesh].attrs["gridSpacing"])
- np.append(spacing, np.ones(3 - len(spacing)))
unitSI = f[bp + mp + "/" + mesh].attrs["gridUnitSI"]
- self.domain_right_edge = self.domain_dimensions * unitSI * spacing
- except:
- mylog.warning("The domain extent could not be calculated! Setting the field extent to 1m**3! "
- "This WILL break particle-overplotting!")
+ self.domain_right_edge = self.domain_dimensions[:spacing.size] * unitSI * spacing
+ self.domain_right_edge = np.append(self.domain_right_edge, np.ones(3 - self.domain_right_edge.size))
+ except Exception as e:
+ mylog.warning("The domain extent could not be calculated! ({}) Setting the field extent to 1m**3! "
+ "This WILL break particle-overplotting!".format(e))
self.domain_right_edge = np.ones(3, dtype=np.float64)
if self._nonstandard:
diff -r ead6c60e0c388127a655153d06ed9d09e30f77fe -r 1096d01db5b6bbbe6dbadfd2b9995e614687793b yt/frontends/openPMD/fields.py
--- a/yt/frontends/openPMD/fields.py
+++ b/yt/frontends/openPMD/fields.py
@@ -8,6 +8,7 @@
#-----------------------------------------------------------------------------
# Copyright (c) 2013, yt Development Team.
# Copyright (c) 2015, Daniel Grassinger (HZDR)
+# Copyright (c) 2016, Fabian Koller (HZDR)
#
# Distributed under the terms of the Modified BSD License.
#
@@ -19,7 +20,9 @@
from yt.utilities.physical_constants import speed_of_light
from yt.fields.field_info_container import \
FieldInfoContainer
-from .misc import parse_unitDimension
+from yt.fields.magnetic_field import \
+ setup_magnetic_field_aliases
+from .misc import parse_unit_dimension
import yt
@@ -55,7 +58,7 @@
return velocity
- for ax in 'xyz':
+ for ax in "xyz":
self.add_field((ptype, "particle_velocity_%s" % ax),
function=_get_vel(ax),
units="m/s",
@@ -65,25 +68,18 @@
def setup_poynting_vector(self):
def _get_poyn(axis):
def poynting(field, data):
- Efieldx = data["E_x"]
- Efieldy = data["E_y"]
- Efieldz = data["E_z"]
- Bfieldx = data["magnetic_field_x"]
- Bfieldy = data["magnetic_field_y"]
- Bfieldz = data["magnetic_field_z"]
-
u = 79577.4715459 # = 1/magnetic permeability
- if(axis == 'x'):
- return u * (Efieldy * Bfieldz - Efieldz * Bfieldy)
- elif(axis == 'y'):
- return u * (Efieldz * Bfieldx - Efieldx * Bfieldz)
- elif(axis == 'z'):
- return u * (Efieldx * Bfieldy - Efieldy * Bfieldx)
+ if axis in "x":
+ return u * (data["E_y"] * data["magnetic_field_z"] - data["E_z"] * data["magnetic_field_y"])
+ elif axis in "y":
+ return u * (data["E_z"] * data["magnetic_field_x"] - data["E_x"] * data["magnetic_field_z"])
+ elif axis in "z":
+ return u * (data["E_x"] * data["magnetic_field_y"] - data["E_y"] * data["magnetic_field_x"])
return poynting
- for ax in 'xyz':
+ for ax in "xyz":
self.add_field(("openPMD", "poynting_vector_%s" % ax),
function=_get_poyn(ax),
units="T*V/m")
@@ -98,7 +94,7 @@
return ap
- for ax in 'xyz':
+ for ax in "xyz":
self.add_field((ptype, "particle_position_%s" % ax),
function=_abs_pos(ax),
units="m",
@@ -136,7 +132,7 @@
if ds._nonstandard:
parsed = ""
else:
- parsed = parse_unitDimension(np.asarray(field.attrs["unitDimension"], dtype='int'))
+ parsed = parse_unit_dimension(np.asarray(field.attrs["unitDimension"], dtype="int"))
unit = str(yt.YTQuantity(1, parsed).units)
aliases = []
# Save a list of magnetic fields for aliasing later on
@@ -147,7 +143,7 @@
self.known_other_fields += ((ytname, (unit, aliases, None)), )
else:
if ds._nonstandard:
- axes = "xyz" # I naively assume all fields in pre-oPMD files are 3D
+ axes = "xyz" # naively assume all fields in non-standard files are 3D
else:
axes = field.attrs["axisLabels"]
for axis in axes:
@@ -155,7 +151,7 @@
if ds._nonstandard:
parsed = ""
else:
- parsed = parse_unitDimension(np.asarray(field.attrs["unitDimension"], dtype='int'))
+ parsed = parse_unit_dimension(np.asarray(field.attrs["unitDimension"], dtype="int"))
unit = str(yt.YTQuantity(1, parsed).units)
aliases = []
# Save a list of magnetic fields for aliasing later on
@@ -181,8 +177,8 @@
else:
parsed = ""
else:
- parsed = parse_unitDimension(
- np.asarray(particles.get(species).get(attrib).attrs["unitDimension"], dtype='int'))
+ parsed = parse_unit_dimension(
+ np.asarray(particles.get(species).get(attrib).attrs["unitDimension"], dtype="int"))
unit = str(yt.YTQuantity(1, parsed).units)
name = ["particle", attrib]
ytattrib = attrib
@@ -203,27 +199,22 @@
mylog.debug("oPMD - fields - known_particle_fields - {}".format(i))
super(openPMDFieldInfo, self).__init__(ds, field_list)
-
def setup_fluid_fields(self):
"""
Here you can create functions to calculate out of existing fields the
values of new fields e.g. calculate out of E-field and B-field the
Poynting vector
"""
+ # Set up aliases first so the setup for poynting can use them
if len(self._mag_fields) > 0:
- from yt.fields.magnetic_field import \
- setup_magnetic_field_aliases
setup_magnetic_field_aliases(self, "openPMD", self._mag_fields)
- # Set up aliases first so the setup for poynting can use them
- setup_poynting_vector(self)
+ setup_poynting_vector(self)
def setup_particle_fields(self, ptype):
"""
This will get called for every particle type.
-
- TODO You have to call the function of the parent class to load particles
"""
setup_absolute_positions(self, ptype)
- #setup_kinetic_energy(self, ptype)
- #setup_velocity(self, ptype)
+ setup_kinetic_energy(self, ptype)
+ setup_velocity(self, ptype)
super(openPMDFieldInfo, self).setup_particle_fields(ptype)
diff -r ead6c60e0c388127a655153d06ed9d09e30f77fe -r 1096d01db5b6bbbe6dbadfd2b9995e614687793b yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -8,6 +8,7 @@
#-----------------------------------------------------------------------------
# Copyright (c) 2013, yt Development Team.
# Copyright (c) 2015, Daniel Grassinger (HZDR)
+# Copyright (c) 2016, Fabian Koller (HZDR)
#
# Distributed under the terms of the Modified BSD License.
#
@@ -107,7 +108,7 @@
for gridptype, ofs in g.off_part:
if str(gridptype) == str(spec):
offset = ofs
- mylog.debug("openPMD - _read_particle_coords: (grid {}) {}, {}".format(g, ptype, field_list))
+ mylog.debug("openPMD - _read_particle_coords: (grid {}) {}, {} [{}:{}]".format(g, ptype, field_list, index,offset))
if str((ptype, index, offset)) not in self._cached_ptype:
self._fill_cache(ptype, index, offset)
yield (ptype, (self._cache['x'], self._cache['y'], self._cache['z']))
@@ -162,10 +163,9 @@
offset = ofs
if str((ptype, index, offset)) not in self._cached_ptype:
self._fill_cache(ptype, index, offset)
- mylog.debug("openPMD - _read_particle_fields: (grid {}) {}, {}".format(g, ptype, field_list))
+ mylog.debug("openPMD - _read_particle_fields: (grid {}) {}, {} [{}:{}]".format(g, ptype, field_list, index, offset))
mask = selector.select_points(self._cache['x'], self._cache['y'], self._cache['z'], 0.0)
if mask is None:
- mylog.debug("Particle selection mask is None!")
continue
pds = ds[self.particlesPath + "/" + spec]
for field in field_list:
@@ -240,7 +240,7 @@
data = np.array(self.get_component(ds, nfield, index, offset, self.ds._nonstandard))
# The following is a modified AMRGridPatch.select(...)
mask = g._get_selector_mask(selector)
- data = data.reshape(mask.shape) # Workaround - casts a 2D (x,y) array to 3D (x,y,1)
+ data.shape = mask.shape # Workaround - casts a 2D (x,y) array to 3D (x,y,1)
count = g.count(selector)
rv[field][ind[field]:ind[field] + count] = data[mask]
ind[field] += count
@@ -270,8 +270,8 @@
unitSI = record_component.attrs["sim_unit"]
if "globalCellIdx" in component_name:
it = group["/data"].keys()[0]
- len = group["/data/" + it].attrs['unit_length']
- unitSI *= len
+ length = group["/data/" + it].attrs['unit_length']
+ unitSI *= length
except:
unitSI = 1.0
else:
diff -r ead6c60e0c388127a655153d06ed9d09e30f77fe -r 1096d01db5b6bbbe6dbadfd2b9995e614687793b yt/frontends/openPMD/misc.py
--- a/yt/frontends/openPMD/misc.py
+++ b/yt/frontends/openPMD/misc.py
@@ -1,4 +1,6 @@
+#-----------------------------------------------------------------------------
# Copyright (c) 2015, Axel Huebl, Remi Lehe
+# Copyright (c) 2016, Fabian Koller (HZDR)
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
@@ -12,6 +14,7 @@
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
+#-----------------------------------------------------------------------------
import h5py as h5
import numpy as np
@@ -25,6 +28,7 @@
ext_list = [["ED-PIC", np.uint32(1)]]
+
def open_file(file_name):
try:
f = h5.File(file_name, "r")
@@ -32,6 +36,7 @@
except:
raise
+
def get_attr(f, name):
"""
Try to access the path `name` in the file `f`
@@ -42,6 +47,7 @@
else:
return(False, None)
+
def test_record(g, r):
"""
Checks if a record is valid
@@ -77,6 +83,7 @@
return(result_array)
+
def test_key(f, v, request, name):
"""
Checks whether a key is present. A key can either be
@@ -128,6 +135,7 @@
return(result_array)
+
def test_attr(f, v, request, name, is_type=None, type_format=None):
"""
Checks whether an attribute is present.
@@ -232,6 +240,7 @@
return(result_array)
+
def is_scalar_record(r):
"""
Checks if a record is a scalar record or not.
@@ -259,6 +268,7 @@
else :
return True
+
def test_component(c, v) :
"""
Checks if a record component defines all required attributes.
@@ -421,6 +431,7 @@
return(result_array)
+
def check_base_path(f, iteration, v, pic):
"""
Scan the base_path that corresponds to this iteration
@@ -461,6 +472,7 @@
return(result_array)
+
def check_meshes(f, iteration, v, pic):
"""
Scan all the meshes corresponding to one iteration
@@ -743,7 +755,7 @@
"particleSmoothingParameters", np.string_)
# Check attributes of each record of the particle
- for record in list(species.keys()) :
+ for record in list(species.keys()):
# all records (but particlePatches) require units
if record != "particlePatches":
result_array += test_attr(species[record], v,
@@ -751,25 +763,25 @@
time_type = f[base_path].attrs["time"].dtype.type
result_array += test_attr(species[record], v, "required",
"timeOffset", time_type)
- if pic :
+ if pic:
result_array += test_attr(species[record], v, "required",
"weightingPower", np.float64)
result_array += test_attr(species[record], v, "required",
"macroWeighted", np.uint32)
# Attributes of the components
- if is_scalar_record( species[record] ) : # Scalar record
+ if is_scalar_record(species[record]): # Scalar record
dset = species[record]
result_array += test_component(dset, v)
- else : # Vector record
+ else: # Vector record
# Loop over the components
for component_name in list(species[record].keys()):
- dset = species[ os.path.join(record, component_name) ]
+ dset = species[os.path.join(record, component_name)]
result_array += test_component(dset, v)
return result_array
-def parse_unitDimension(unitDimension):
+def parse_unit_dimension(unitDimension):
if len(unitDimension) is not 7:
mylog.error("SI must have 7 base dimensions! {} is off by {}".format(unitDimension, len(unitDimension) - 7))
dim = []
https://bitbucket.org/yt_analysis/yt/commits/6593592067f5/
Changeset: 6593592067f5
Branch: yt
User: Fabian Koller
Date: 2016-08-03 09:38:37+00:00
Summary: Refactor classes (got rid of OpenPMDBasePath),
reimplement the derived field for kinetic energy
Affected #: 4 files
diff -r 1096d01db5b6bbbe6dbadfd2b9995e614687793b -r 6593592067f5f5afb843f7455e6abdbbb5cd64a5 yt/frontends/openPMD/api.py
--- a/yt/frontends/openPMD/api.py
+++ b/yt/frontends/openPMD/api.py
@@ -14,12 +14,12 @@
#-----------------------------------------------------------------------------
from .data_structures import \
- openPMDDataset, \
- openPMDGrid, \
- openPMDHierarchy
+ OpenPMDDataset, \
+ OpenPMDGrid, \
+ OpenPMDHierarchy
from .fields import \
- openPMDFieldInfo
+ OpenPMDFieldInfo
from .io import \
IOHandlerOpenPMD
diff -r 1096d01db5b6bbbe6dbadfd2b9995e614687793b -r 6593592067f5f5afb843f7455e6abdbbb5cd64a5 yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -20,7 +20,7 @@
GridIndex
from yt.data_objects.static_output import \
Dataset
-from .fields import openPMDFieldInfo
+from .fields import OpenPMDFieldInfo
from yt.utilities.file_handler import \
HDF5FileHandler
@@ -33,95 +33,52 @@
from math import ceil, floor
from yt.utilities.logger import ytLogger as mylog
+
def is_const_component(record_component):
return "value" in record_component.attrs.keys()
-class openPMDBasePath:
- def _setNonStandardBasePath(self, handle):
- iteration = handle["/data"].keys()[0]
- self.basePath = "/data/{}/".format(iteration)
- def _setBasePath(self, handle, filepath):
- """
- Set the base path for the first iteration found in the file.
- TODO implement into distinct methods:
- - __init__(self, handle)
- - getIterations(self)
- - getBasePath(self, iteration)
- """
- # find iterations in basePath
- list_iterations = []
- if u"groupBased" in handle.attrs["iterationEncoding"]:
- for i in list(handle["/data"].keys()):
- list_iterations.append(i)
- mylog.info("openPMD: found {} iterations in file".format(len(list_iterations)))
- elif u"fileBased" in handle.attrs["iterationEncoding"]:
- regex = u"^" + handle.attrs["iterationFormat"].replace('%T', '[0-9]+') + u"$"
- if filepath is '':
- mylog.warning("openPMD: For file based iterations, please use absolute file paths!")
- pass
- for filename in os.listdir(filepath):
- if re.match(regex, filename):
- list_iterations.append(filename)
- mylog.info("openPMD: found {} iterations in directory".format(len(list_iterations)))
- else:
- mylog.warning(
- "openOMD: File does not have valid iteration encoding: {}".format(handle.attrs["iterationEncoding"]))
-
- if len(list_iterations) == 0:
- mylog.warning("openOMD: No iterations found!")
-
- # just handle the first iteration found
- if u"groupBased" in handle.attrs["iterationEncoding"] and len(list_iterations) > 1:
- mylog.warning("openPMD: only choose to load one iteration ({})".format(handle["/data"].keys()[0]))
- self.basePath = "{}/{}/".format("/data", handle["/data"].keys()[0])
-
-
-class openPMDGrid(AMRGridPatch):
+class OpenPMDGrid(AMRGridPatch):
"""
This class defines the characteristics of the grids
"""
_id_offset = 0
__slots__ = ["_level_id"]
- part_ind = {}
- off_part = {}
+ particle_index = {}
+ particle_offset = {}
# TODO (maybe) consider these for every ftype
- mesh_ind = 0
- off_mesh = 0
+ mesh_index = 0
+ mesh_offset = 0
- def __init__(self, id, index, level=-1, pi=None, op=None, mi=0, om=0):
+ def __init__(self, id, index, level=-1, pi=None, po=None, mi=0, mo=0):
AMRGridPatch.__init__(self, id, filename=index.index_filename,
index=index)
if pi:
- self.part_ind = pi
- if op:
- self.off_part = op
- self.mesh_ind = mi
- self.off_mesh = om
+ self.particle_index = pi
+ if po:
+ self.particle_offset = po
+ self.mesh_index = mi
+ self.mesh_offset = mo
self.Parent = None
self.Children = []
self.Level = level
def __repr__(self):
- return "openPMDGrid_%04i (%s)" % (self.id, self.ActiveDimensions)
+ return "OpenPMDGrid_%04i (%s)" % (self.id, self.ActiveDimensions)
-class openPMDHierarchy(GridIndex, openPMDBasePath):
+class OpenPMDHierarchy(GridIndex):
"""
Defines which fields and particles are created and read from the hard disk
Furthermore it defines the characteristics of the grids
"""
- grid = openPMDGrid
+ grid = OpenPMDGrid
def __init__(self, ds, dataset_type='openPMD'):
self.dataset_type = dataset_type
self.dataset = ds
self.index_filename = ds.parameter_filename
self.directory = os.path.dirname(self.index_filename)
- if self.dataset._nonstandard:
- self._setNonStandardBasePath(self.dataset._handle)
- else:
- self._setBasePath(self.dataset._handle, self.directory)
GridIndex.__init__(self, ds, dataset_type)
def _detect_output_fields(self):
@@ -138,43 +95,39 @@
self.field_list must be populated as a list of strings corresponding to "native" fields in the data files.
"""
f = self.dataset._handle
- bp = self.basePath
- if self.dataset._nonstandard:
- mp = "fields/"
- pp = "particles/"
- else:
- mp = f.attrs["meshesPath"]
- pp = f.attrs["particlesPath"]
+ bp = self.dataset.base_path
+ mp = self.dataset.meshes_path
+ pp = self.dataset.particles_path
output_fields = []
- for group in f[bp + mp].keys():
+ for field in f[bp + mp].keys():
try:
- for direction in f[bp + mp + group].keys():
- output_fields.append(group + "_" + direction)
+ for axis in f[bp + mp + field].keys():
+ output_fields.append(field + "_" + axis)
except:
# This is for dataSets, they do not have keys
- output_fields.append(group.replace("_","-"))
+ output_fields.append(field.replace("_","-"))
self.field_list = [("openPMD", str(c)) for c in output_fields]
particle_fields = []
if bp + pp in f:
- for particleName in f[bp + pp].keys():
- for record in f[bp + pp + particleName].keys():
- if is_const_component(f[bp + pp + particleName + "/" + record]):
+ for species in f[bp + pp].keys():
+ for record in f[bp + pp + species].keys():
+ if is_const_component(f[bp + pp + species + "/" + record]):
# Record itself (e.g. particle_mass) is constant
- particle_fields.append(particleName + "_" + record)
+ particle_fields.append(species + "_" + record)
elif 'particlePatches' not in record:
try:
# Create a field for every axis (x,y,z) of every property (position)
# of every species (electrons)
- keys = f[bp + pp + particleName + "/" + record].keys()
+ keys = f[bp + pp + species + "/" + record].keys()
if record in "position":
record = "positionCoarse"
for axis in keys:
- particle_fields.append(particleName + "_" + record + "_" + axis)
+ particle_fields.append(species + "_" + record + "_" + axis)
except:
# Record is a dataset, does not have axes (e.g. weighting)
- particle_fields.append(particleName + "_" + record)
+ particle_fields.append(species + "_" + record)
pass
else:
# We probably do not want particlePatches as accessible field lists
@@ -195,28 +148,24 @@
From yt doc:
this must set self.num_grids to be the total number of grids (equiv AMRGridPatch'es) in the simulation
"""
- # TODO For the moment we only create grids if there are particles present
# TODO Calculate the ppg not solely on particle count, also on meshsize
f = self.dataset._handle
- bp = self.basePath
- if self.dataset._nonstandard:
- mp = "fields/"
- pp = "particles/"
- else:
- mp = f.attrs["meshesPath"]
- pp = f.attrs["particlesPath"]
+ bp = self.dataset.base_path
+ mp = self.dataset.meshes_path
+ pp = self.dataset.particles_path
+
gridsize = 100 * 10**6 # Bytes
- species = f[bp + pp].keys()
+ maxnp = 1
self.np = {}
- maxnp = 0
- for spec in species:
- pos = f[bp + pp + spec + "/position"].keys()[0]
- if is_const_component(f[bp + pp + spec + "/position/" + pos]):
- self.np[spec] = f[bp + pp + spec + "/position/" + pos].attrs["shape"]
+
+ for species in f[bp + pp].keys():
+ axis = f[bp + pp + species + "/position"].keys()[0]
+ if is_const_component(f[bp + pp + species + "/position/" + axis]):
+ self.np[species] = f[bp + pp + species + "/position/" + axis].attrs["shape"]
else:
- self.np[spec] = f[bp + pp + spec + "/position/" + pos].len()
- if self.np[spec] > maxnp:
- maxnp = self.np[spec]
+ self.np[species] = f[bp + pp + species + "/position/" + axis].len()
+ if self.np[species] > maxnp:
+ maxnp = self.np[species]
# Limit particles per grid by resulting memory footprint
ppg = int(gridsize/(self.dataset.dimensionality*4)) # 4 Byte per value per dimension (f32)
# Use an upper bound of equally sized grids, last one might be smaller
@@ -276,9 +225,9 @@
self.grids[i] = self.grid(
i, self, self.grid_levels[i, 0],
pi=particleindex,
- op=particlecount,
+ po=particlecount,
mi=meshindex,
- om=self.grid_dimensions[i][0])
+ mo=self.grid_dimensions[i][0])
for spec, val in particlecount:
pci[spec] += val
meshindex += self.grid_dimensions[i][0]
@@ -298,7 +247,7 @@
self.max_level = 0
-class openPMDDataset(Dataset, openPMDBasePath):
+class OpenPMDDataset(Dataset):
"""
A dataset object contains all the information of the simulation and
is intialized with yt.load()
@@ -308,8 +257,8 @@
multiple iteration-loading if done that way, e.g., from a prefix
of files.
"""
- _index_class = openPMDHierarchy
- _field_info_class = openPMDFieldInfo
+ _index_class = OpenPMDHierarchy
+ _field_info_class = OpenPMDFieldInfo
_nonstandard = False
def __init__(self, filename, dataset_type='openPMD',
@@ -319,19 +268,15 @@
self._handle = HDF5FileHandler(filename)
self._filepath = os.path.dirname(filename)
if self._nonstandard:
- self._setNonStandardBasePath(self._handle)
+ self._set_nonstandard_paths(self._handle)
else:
- self._setBasePath(self._handle, self._filepath)
+ self._set_paths(self._handle, self._filepath)
Dataset.__init__(self, filename, dataset_type,
units_override=units_override,
unit_system=unit_system)
self.storage_filename = storage_filename
self.fluid_types += ('openPMD',)
- if self._nonstandard:
- pp = "particles"
- else:
- pp = self._handle.attrs["particlesPath"]
- parts = tuple(str(c) for c in self._handle[self.basePath + pp].keys())
+ parts = tuple(str(c) for c in self._handle[self.base_path + self.particles_path].keys())
if len(parts) > 1:
# Only use infile particle names if there is more than one species
self.particle_types = parts
@@ -340,6 +285,39 @@
self.particle_types = tuple(self.particle_types)
self.particle_types = tuple(self.particle_types)
+ def _set_nonstandard_paths(self, handle):
+ iteration = handle["/data"].keys()[0]
+ self.base_path = "/data/{}/".format(iteration)
+ self.meshes_path = "fields/"
+ self.particles_path = "particles/"
+
+ def _set_paths(self, handle, filepath):
+ list_iterations = []
+ if u"groupBased" in handle.attrs["iterationEncoding"]:
+ for i in list(handle["/data"].keys()):
+ list_iterations.append(i)
+ mylog.info("openPMD: found {} iterations in file".format(len(list_iterations)))
+ elif u"fileBased" in handle.attrs["iterationEncoding"]:
+ regex = u"^" + handle.attrs["iterationFormat"].replace('%T', '[0-9]+') + u"$"
+ if filepath is '':
+ mylog.warning("openPMD: For file based iterations, please use absolute file paths!")
+ pass
+ for filename in os.listdir(filepath):
+ if re.match(regex, filename):
+ list_iterations.append(filename)
+ mylog.info("openPMD: found {} iterations in directory".format(len(list_iterations)))
+ else:
+ mylog.warning(
+ "openOMD: File does not have valid iteration encoding: {}".format(handle.attrs["iterationEncoding"]))
+
+ if len(list_iterations) == 0:
+ mylog.warning("openOMD: No iterations found!")
+ if u"groupBased" in handle.attrs["iterationEncoding"] and len(list_iterations) > 1:
+ mylog.warning("openPMD: only choose to load one iteration ({})".format(handle["/data"].keys()[0]))
+
+ self.base_path = "{}/{}/".format("/data", handle["/data"].keys()[0])
+ self.meshes_path = self._handle["/"].attrs["meshesPath"]
+ self.particles_path = self._handle["/"].attrs["particlesPath"]
def _set_code_unit_attributes(self):
"""
@@ -360,13 +338,8 @@
read in metadata describing the overall data on disk
"""
f = self._handle
- bp = self.basePath
- if self._nonstandard:
- mp = "fields"
- pp = "particles"
- else:
- mp = f.attrs["meshesPath"]
- pp = f.attrs["particlesPath"]
+ bp = self.base_path
+ mp = self.meshes_path
self.unique_identifier = 0
self.parameters = 0
@@ -380,14 +353,7 @@
fshape = np.array([1, 1, 1], dtype=np.int64)
mylog.warning("Could not detect shape of simulated field! "
"Assuming a single cell and thus setting fshape to [1, 1, 1]!")
- if len(fshape) < 1:
- self.dimensionality = 0
- for species in f[bp + pp].keys():
- self.dimensionality = max(
- len(f[bp + pp + "/" + species].keys()),
- self.dimensionality)
- else:
- self.dimensionality = len(fshape)
+ self.dimensionality = len(fshape)
fshape = np.append(fshape, np.ones(3 - self.dimensionality))
self.domain_dimensions = fshape
diff -r 1096d01db5b6bbbe6dbadfd2b9995e614687793b -r 6593592067f5f5afb843f7455e6abdbbb5cd64a5 yt/frontends/openPMD/fields.py
--- a/yt/frontends/openPMD/fields.py
+++ b/yt/frontends/openPMD/fields.py
@@ -28,15 +28,12 @@
def setup_kinetic_energy(self, ptype):
def _kin_en(field, data):
- # Calculation seems to be wrong:
- # YTFieldUnitError
- # The field function associated with the field '('io', 'particle_kinetic_energy')'
- # returned data with units 'cm*kg**2*m**2/s**3' but was defined with units 'kg*m**2/s**2'.
- return (
- data[ptype, "particle_momentum_x"]**2 +
- data[ptype, "particle_momentum_y"]**2 +
- data[ptype, "particle_momentum_z"]**2
- ) * speed_of_light
+ c = yt.YTQuantity(speed_of_light, "m/s")
+ p2 = (data[ptype, "particle_momentum_x"]**2 +
+ data[ptype, "particle_momentum_y"]**2 +
+ data[ptype, "particle_momentum_z"]**2)
+ mass = data[ptype, "particle_mass"] * data[ptype, "particle_weighting"]
+ return c * np.sqrt(p2 + mass**2 * c**2) - mass * c**2
self.add_field((ptype, "particle_kinetic_energy"),
function=_kin_en,
@@ -69,7 +66,6 @@
def _get_poyn(axis):
def poynting(field, data):
u = 79577.4715459 # = 1/magnetic permeability
-
if axis in "x":
return u * (data["E_y"] * data["magnetic_field_z"] - data["E_z"] * data["magnetic_field_y"])
elif axis in "y":
@@ -101,7 +97,7 @@
particle_type=True)
-class openPMDFieldInfo(FieldInfoContainer):
+class OpenPMDFieldInfo(FieldInfoContainer):
"""
We need to specify which fields we might have in our dataset. The field info
container subclass here will define which fields it knows about. There are
@@ -116,14 +112,11 @@
def __init__(self, ds, field_list):
f = ds._handle
- bp = ds.basePath
- if ds._nonstandard:
- mp = "fields/"
- pp = "particles/"
- else:
- mp = f.attrs["meshesPath"]
- pp = f.attrs["particlesPath"]
+ bp = ds.base_path
+ mp = ds.meshes_path
+ pp = ds.particles_path
fields = f[bp + mp]
+
for fname in fields.keys():
field = fields.get(fname)
if "dataset" in str(field).split(" ")[1]:
@@ -197,7 +190,7 @@
self.known_particle_fields = particle_fields
for i in self.known_particle_fields:
mylog.debug("oPMD - fields - known_particle_fields - {}".format(i))
- super(openPMDFieldInfo, self).__init__(ds, field_list)
+ super(OpenPMDFieldInfo, self).__init__(ds, field_list)
def setup_fluid_fields(self):
"""
@@ -217,4 +210,4 @@
setup_absolute_positions(self, ptype)
setup_kinetic_energy(self, ptype)
setup_velocity(self, ptype)
- super(openPMDFieldInfo, self).setup_particle_fields(ptype)
+ super(OpenPMDFieldInfo, self).setup_particle_fields(ptype)
diff -r 1096d01db5b6bbbe6dbadfd2b9995e614687793b -r 6593592067f5f5afb843f7455e6abdbbb5cd64a5 yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -19,38 +19,30 @@
BaseIOHandler
from yt.utilities.logger import ytLogger as mylog
import numpy as np
-from .data_structures import openPMDBasePath
-class IOHandlerOpenPMD(BaseIOHandler, openPMDBasePath):
+class IOHandlerOpenPMD(BaseIOHandler):
_field_dtype = "float32"
_dataset_type = "openPMD"
def __init__(self, ds, *args, **kwargs):
-
self.ds = ds
self._handle = ds._handle
- if self.ds._nonstandard:
- self._setNonStandardBasePath(self._handle)
- self.meshesPath = "fields/"
- self.particlesPath = "particles/"
- else:
- self._setBasePath(self._handle, self.ds._filepath)
- self.meshesPath = self._handle["/"].attrs["meshesPath"]
- self.particlesPath = self._handle["/"].attrs["particlesPath"]
+ self.base_path = ds.base_path
+ self.meshes_path = ds.meshes_path
+ self.particles_path = ds.particles_path
self._array_fields = {}
self._cached_ptype = ""
self._cache = {}
- self._cachecntr = {}
def _fill_cache(self, ptype, index=0, offset=None):
if str((ptype, index, offset)) not in self._cached_ptype:
# Get a particle species (e.g. /data/3500/particles/e/)
if "io" in ptype:
- spec = self._handle[self.basePath + self.particlesPath].keys()[0]
+ spec = self._handle[self.base_path + self.particles_path].keys()[0]
else:
spec = ptype
- pds = self._handle[self.basePath + self.particlesPath + "/" + spec]
+ pds = self._handle[self.base_path + self.particles_path + "/" + spec]
# Get 1D-Arrays for individual particle positions along axes
axes = [str(ax) for ax in pds["position"].keys()]
pos = {}
@@ -92,23 +84,23 @@
"""
chunks = list(chunks)
f = self._handle
- ds = f[self.basePath]
+ ds = f[self.base_path]
for chunk in chunks:
for g in chunk.objs:
if g.filename is None:
continue
for ptype, field_list in sorted(ptf.items()):
if "io" in ptype:
- spec = ds[self.particlesPath].keys()[0]
+ spec = ds[self.particles_path].keys()[0]
else:
spec = ptype
- for gridptype, idx in g.part_ind:
+ for gridptype, idx in g.particle_index:
if str(gridptype) == str(spec):
index = idx
- for gridptype, ofs in g.off_part:
+ for gridptype, ofs in g.particle_offset:
if str(gridptype) == str(spec):
offset = ofs
- mylog.debug("openPMD - _read_particle_coords: (grid {}) {}, {} [{}:{}]".format(g, ptype, field_list, index,offset))
+ mylog.debug("openPMD - _read_particle_coords: (grid {}) {}, {} [{}:{}]".format(g, ptype, field_list, index, offset))
if str((ptype, index, offset)) not in self._cached_ptype:
self._fill_cache(ptype, index, offset)
yield (ptype, (self._cache['x'], self._cache['y'], self._cache['z']))
@@ -144,7 +136,7 @@
"""
chunks = list(chunks)
f = self._handle
- ds = f[self.basePath]
+ ds = f[self.base_path]
for chunk in chunks:
for g in chunk.objs:
if g.filename is None:
@@ -152,13 +144,13 @@
for ptype, field_list in sorted(ptf.items()):
# Get a particle species (e.g. /data/3500/particles/e/)
if "io" in ptype:
- spec = ds[self.particlesPath].keys()[0]
+ spec = ds[self.particles_path].keys()[0]
else:
spec = ptype
- for gridptype, idx in g.part_ind:
+ for gridptype, idx in g.particle_index:
if str(gridptype) == str(spec):
index = idx
- for gridptype, ofs in g.off_part:
+ for gridptype, ofs in g.particle_offset:
if str(gridptype) == str(spec):
offset = ofs
if str((ptype, index, offset)) not in self._cached_ptype:
@@ -167,7 +159,7 @@
mask = selector.select_points(self._cache['x'], self._cache['y'], self._cache['z'], 0.0)
if mask is None:
continue
- pds = ds[self.particlesPath + "/" + spec]
+ pds = ds[self.particles_path + "/" + spec]
for field in field_list:
nfield = "/".join(field.split("_")[1:]).replace("positionCoarse", "position")
data = self.get_component(pds, nfield, index, offset, self.ds._nonstandard)
@@ -209,11 +201,8 @@
"""
mylog.info("_read_fluid_selection {} {} {} {}".format(chunks, selector, fields, size))
f = self._handle
- bp = self.basePath
- if self.ds._nonstandard:
- mp = "fields/"
- else:
- mp = f.attrs["meshesPath"]
+ bp = self.base_path
+ mp = self.meshes_path
rv = {}
chunks = list(chunks)
@@ -235,8 +224,8 @@
for g in chunk.objs:
ds = f[bp + mp]
nfield = fname.replace("_", "/").replace("-","_")
- index = g.mesh_ind
- offset = g.off_mesh
+ index = g.mesh_index
+ offset = g.mesh_offset
data = np.array(self.get_component(ds, nfield, index, offset, self.ds._nonstandard))
# The following is a modified AMRGridPatch.select(...)
mask = g._get_selector_mask(selector)
https://bitbucket.org/yt_analysis/yt/commits/cb452bf12825/
Changeset: cb452bf12825
Branch: yt
User: Fabian Koller
Date: 2016-08-03 13:40:54+00:00
Summary: Reduce default grid size to ~40MB,
use numpy add and multiply where reasonable
Affected #: 4 files
diff -r 6593592067f5f5afb843f7455e6abdbbb5cd64a5 -r cb452bf1282587483afb193829c3cad67b1ee557 yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ b/yt/frontends/openPMD/data_structures.py
@@ -154,7 +154,7 @@
mp = self.dataset.meshes_path
pp = self.dataset.particles_path
- gridsize = 100 * 10**6 # Bytes
+ gridsize = 40 * 10**6 # Bytes
maxnp = 1
self.np = {}
@@ -198,13 +198,13 @@
meshindex = 0
meshedge = self.dataset.domain_left_edge.copy()[0]
for i in range(self.num_grids):
- self.grid_dimensions[i] = self.dataset.domain_dimensions # (N, 3) <= int
+ self.grid_dimensions[i] = self.dataset.domain_dimensions
prev = remaining
remaining -= self.grid_dimensions[i][0] * self.num_grids**-1
self.grid_dimensions[i][0] = int(round(prev, 0) - round(remaining, 0))
- self.grid_left_edge[i] = self.dataset.domain_left_edge.copy() # (N, 3) <= float64
+ self.grid_left_edge[i] = self.dataset.domain_left_edge.copy()
self.grid_left_edge[i][0] = meshedge
- self.grid_right_edge[i] = self.dataset.domain_right_edge.copy() # (N, 3) <= float64
+ self.grid_right_edge[i] = self.dataset.domain_right_edge.copy()
self.grid_right_edge[i][0] = self.grid_left_edge[i][0] \
+ self.grid_dimensions[i][0]\
* self.dataset.domain_dimensions[0]**-1\
@@ -283,7 +283,6 @@
mylog.debug("openPMD - self.particle_types: {}".format(self.particle_types))
self.particle_types_raw = self.particle_types
self.particle_types = tuple(self.particle_types)
- self.particle_types = tuple(self.particle_types)
def _set_nonstandard_paths(self, handle):
iteration = handle["/data"].keys()[0]
diff -r 6593592067f5f5afb843f7455e6abdbbb5cd64a5 -r cb452bf1282587483afb193829c3cad67b1ee557 yt/frontends/openPMD/fields.py
--- a/yt/frontends/openPMD/fields.py
+++ b/yt/frontends/openPMD/fields.py
@@ -28,12 +28,11 @@
def setup_kinetic_energy(self, ptype):
def _kin_en(field, data):
- c = yt.YTQuantity(speed_of_light, "m/s")
p2 = (data[ptype, "particle_momentum_x"]**2 +
data[ptype, "particle_momentum_y"]**2 +
data[ptype, "particle_momentum_z"]**2)
mass = data[ptype, "particle_mass"] * data[ptype, "particle_weighting"]
- return c * np.sqrt(p2 + mass**2 * c**2) - mass * c**2
+ return speed_of_light * np.sqrt(p2 + mass**2 * speed_of_light**2) - mass * speed_of_light**2
self.add_field((ptype, "particle_kinetic_energy"),
function=_kin_en,
@@ -84,9 +83,8 @@
def setup_absolute_positions(self, ptype):
def _abs_pos(axis):
def ap(field, data):
- pos = data[ptype, "particle_positionCoarse_{}".format(axis)]
- off = data[ptype, "particle_positionOffset_{}".format(axis)]
- return pos + off
+ return np.add(data[ptype, "particle_positionCoarse_{}".format(axis)],
+ data[ptype, "particle_positionOffset_{}".format(axis)])
return ap
@@ -130,7 +128,6 @@
aliases = []
# Save a list of magnetic fields for aliasing later on
# We can not reasonably infer field type by name in openPMD
- mylog.info(fname, field, unit)
if "T" in unit or "kg/(A*s**2)" in unit:
self._mag_fields.append(ytname)
self.known_other_fields += ((ytname, (unit, aliases, None)), )
@@ -149,7 +146,6 @@
aliases = []
# Save a list of magnetic fields for aliasing later on
# We can not reasonably infer field type by name in openPMD
- mylog.info(fname, field, unit, axis)
if "T" in unit or "kg/(A*s**2)" in unit:
self._mag_fields.append(ytname)
self.known_other_fields += ((ytname, (unit, aliases, None)), )
@@ -175,6 +171,8 @@
unit = str(yt.YTQuantity(1, parsed).units)
name = ["particle", attrib]
ytattrib = attrib
+ # Symbolically rename position to preserve yt's interpretation of the pfield
+ # particle_position is later derived in setup_absolute_positions
if ytattrib in "position":
ytattrib = "positionCoarse"
for axis in particles.get(species).get(attrib).keys():
diff -r 6593592067f5f5afb843f7455e6abdbbb5cd64a5 -r cb452bf1282587483afb193829c3cad67b1ee557 yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ b/yt/frontends/openPMD/io.py
@@ -33,32 +33,32 @@
self.particles_path = ds.particles_path
self._array_fields = {}
self._cached_ptype = ""
- self._cache = {}
def _fill_cache(self, ptype, index=0, offset=None):
if str((ptype, index, offset)) not in self._cached_ptype:
# Get a particle species (e.g. /data/3500/particles/e/)
- if "io" in ptype:
+ if ptype in "io":
spec = self._handle[self.base_path + self.particles_path].keys()[0]
else:
spec = ptype
pds = self._handle[self.base_path + self.particles_path + "/" + spec]
# Get 1D-Arrays for individual particle positions along axes
axes = [str(ax) for ax in pds["position"].keys()]
- pos = {}
- off = {}
- for ax in axes:
- pos[ax] = self.get_component(pds, "position/" + ax, index, offset, self.ds._nonstandard)
- if self.ds._nonstandard:
- off[ax] = self.get_component(pds, "globalCellIdx/" + ax, index, offset, self.ds._nonstandard)
+ if self.ds._nonstandard:
+ position_offset = "globalCellIdx/"
+ else:
+ position_offset = "positionOffset/"
+ self.cache = np.empty((3, offset), dtype="float64")
+ for i in range(3):
+ ax = "xyz"[i]
+ if ax in axes:
+ np.add(self.get_component(pds, "position/" + ax, index, offset, self.ds._nonstandard),
+ self.get_component(pds, position_offset + ax, index, offset, self.ds._nonstandard),
+ self.cache[i])
else:
- off[ax] = self.get_component(pds, "positionOffset/" + ax, index, offset, self.ds._nonstandard)
- self._cache[ax] = pos[ax] + off[ax]
- # Pad accordingly with zeros to make 1D/2D datasets compatible
- # These have to be the same shape as the existing axes dataset since that equals the number of particles
- for req in "xyz":
- if req not in axes:
- self._cache[req] = np.zeros(self._cache[self._cache.keys()[0]].shape)
+ # Pad accordingly with zeros to make 1D/2D datasets compatible
+ # These have to be the same shape as the existing axes since that equals the number of particles
+ self.cache[i] = np.zeros(offset)
self._cached_ptype = str((ptype, index, offset))
def _read_particle_coords(self, chunks, ptf):
@@ -90,7 +90,7 @@
if g.filename is None:
continue
for ptype, field_list in sorted(ptf.items()):
- if "io" in ptype:
+ if ptype in "io":
spec = ds[self.particles_path].keys()[0]
else:
spec = ptype
@@ -103,7 +103,7 @@
mylog.debug("openPMD - _read_particle_coords: (grid {}) {}, {} [{}:{}]".format(g, ptype, field_list, index, offset))
if str((ptype, index, offset)) not in self._cached_ptype:
self._fill_cache(ptype, index, offset)
- yield (ptype, (self._cache['x'], self._cache['y'], self._cache['z']))
+ yield (ptype, (self.cache[0], self.cache[1], self.cache[2]))
def _read_particle_fields(self, chunks, ptf, selector):
"""
@@ -143,20 +143,16 @@
continue
for ptype, field_list in sorted(ptf.items()):
# Get a particle species (e.g. /data/3500/particles/e/)
- if "io" in ptype:
+ if ptype in "io":
spec = ds[self.particles_path].keys()[0]
else:
spec = ptype
- for gridptype, idx in g.particle_index:
- if str(gridptype) == str(spec):
- index = idx
- for gridptype, ofs in g.particle_offset:
- if str(gridptype) == str(spec):
- offset = ofs
+ index = dict(g.particle_index).get(spec)
+ offset = dict(g.particle_offset).get(spec)
+ mylog.debug("openPMD - _read_particle_fields: (grid {}) {}, {} [{}:{}]".format(g, ptype, field_list, index, offset))
if str((ptype, index, offset)) not in self._cached_ptype:
self._fill_cache(ptype, index, offset)
- mylog.debug("openPMD - _read_particle_fields: (grid {}) {}, {} [{}:{}]".format(g, ptype, field_list, index, offset))
- mask = selector.select_points(self._cache['x'], self._cache['y'], self._cache['z'], 0.0)
+ mask = selector.select_points(self.cache[0], self.cache[1], self.cache[2], 0.0)
if mask is None:
continue
pds = ds[self.particles_path + "/" + spec]
@@ -280,4 +276,4 @@
# component is a dataset, return it (possibly masked)
mylog.debug(
"openPMD - misc - get_component: {}/{}[{}:{}]".format(group.name, component_name, index, offset))
- return record_component[index:offset] * unitSI
+ return np.multiply(record_component[index:offset], unitSI)
\ No newline at end of file
diff -r 6593592067f5f5afb843f7455e6abdbbb5cd64a5 -r cb452bf1282587483afb193829c3cad67b1ee557 yt/frontends/openPMD/misc.py
--- a/yt/frontends/openPMD/misc.py
+++ b/yt/frontends/openPMD/misc.py
@@ -781,29 +781,19 @@
return result_array
-def parse_unit_dimension(unitDimension):
- if len(unitDimension) is not 7:
- mylog.error("SI must have 7 base dimensions! {} is off by {}".format(unitDimension, len(unitDimension) - 7))
+def parse_unit_dimension(unit_dimension):
+ if len(unit_dimension) is not 7:
+ mylog.error("SI must have 7 base dimensions!")
+ unit_dimension = np.asarray(unit_dimension, dtype='int')
dim = []
- if unitDimension[0] < 0.0 or unitDimension[0] > 0.0:
- # length L,
- dim.append("m**{}".format(unitDimension[0]))
- if unitDimension[1] < 0.0 or unitDimension[1] > 0.0:
- # mass M,
- dim.append("kg**{}".format(unitDimension[1]))
- if unitDimension[2] < 0.0 or unitDimension[2] > 0.0:
- # time T,
- dim.append("s**{}".format(unitDimension[2]))
- if unitDimension[3] < 0.0 or unitDimension[3] > 0.0:
- # electric current I,
- dim.append("A**{}".format(unitDimension[3]))
- if unitDimension[4] < 0.0 or unitDimension[4] > 0.0:
- # thermodynamic temperature theta,
- dim.append("C**{}".format(unitDimension[4]))
- if unitDimension[5] < 0.0 or unitDimension[5] > 0.0:
- # amount of substance N,
- dim.append("mol**{}".format(unitDimension[5]))
- if unitDimension[6] < 0.0 or unitDimension[6] > 0.0:
- # luminous intensity J
- dim.append("cd**{}".format(unitDimension[6]))
+ si = ["m",
+ "kg",
+ "s",
+ "A",
+ "C",
+ "mol",
+ "cd"]
+ for i in range(7):
+ if unit_dimension[i] != 0:
+ dim.append("{}**{}".format(si[i], unit_dimension[i]))
return "*".join(dim)
https://bitbucket.org/yt_analysis/yt/commits/c6cf6d23e854/
Changeset: c6cf6d23e854
Branch: yt
User: Fabian Koller
Date: 2016-08-04 16:40:01+00:00
Summary: Refactor frontend folder,
docstrings
Affected #: 17 files
diff -r cb452bf1282587483afb193829c3cad67b1ee557 -r c6cf6d23e8544b19bb570d804a99d93bd3465978 yt/frontends/api.py
--- a/yt/frontends/api.py
+++ b/yt/frontends/api.py
@@ -35,7 +35,7 @@
'halo_catalog',
'http_stream',
'moab',
- 'openPMD',
+ 'open_pmd',
'owls',
'owls_subfind',
'ramses',
diff -r cb452bf1282587483afb193829c3cad67b1ee557 -r c6cf6d23e8544b19bb570d804a99d93bd3465978 yt/frontends/openPMD/__init__.py
--- a/yt/frontends/openPMD/__init__.py
+++ /dev/null
@@ -1,15 +0,0 @@
-"""
-API for yt.frontends.skeleton
-
-
-
-"""
-
-#-----------------------------------------------------------------------------
-# Copyright (c) 2013, yt Development Team.
-# Copyright (c) 2015, Daniel Grassinger (HZDR)
-#
-# Distributed under the terms of the Modified BSD License.
-#
-# The full license is in the file COPYING.txt, distributed with this software.
-#-----------------------------------------------------------------------------
diff -r cb452bf1282587483afb193829c3cad67b1ee557 -r c6cf6d23e8544b19bb570d804a99d93bd3465978 yt/frontends/openPMD/api.py
--- a/yt/frontends/openPMD/api.py
+++ /dev/null
@@ -1,25 +0,0 @@
-"""
-API for yt.frontends._skeleton
-
-
-
-"""
-
-#-----------------------------------------------------------------------------
-# Copyright (c) 2013, yt Development Team.
-# Copyright (c) 2015, Daniel Grassinger (HZDR)
-# Distributed under the terms of the Modified BSD License.
-#
-# The full license is in the file COPYING.txt, distributed with this software.
-#-----------------------------------------------------------------------------
-
-from .data_structures import \
- OpenPMDDataset, \
- OpenPMDGrid, \
- OpenPMDHierarchy
-
-from .fields import \
- OpenPMDFieldInfo
-
-from .io import \
- IOHandlerOpenPMD
diff -r cb452bf1282587483afb193829c3cad67b1ee557 -r c6cf6d23e8544b19bb570d804a99d93bd3465978 yt/frontends/openPMD/data_structures.py
--- a/yt/frontends/openPMD/data_structures.py
+++ /dev/null
@@ -1,418 +0,0 @@
-"""
-openPMD data structures
-
-
-"""
-
-#-----------------------------------------------------------------------------
-# Copyright (c) 2013, yt Development Team.
-# Copyright (c) 2015, Daniel Grassinger (HZDR)
-# Copyright (c) 2016, Fabian Koller (HZDR)
-#
-# Distributed under the terms of the Modified BSD License.
-#
-# The full license is in the file COPYING.txt, distributed with this software.
-#-----------------------------------------------------------------------------
-
-from yt.data_objects.grid_patch import \
- AMRGridPatch
-from yt.geometry.grid_geometry_handler import \
- GridIndex
-from yt.data_objects.static_output import \
- Dataset
-from .fields import OpenPMDFieldInfo
-
-from yt.utilities.file_handler import \
- HDF5FileHandler
-
-import yt.frontends.openPMD.misc as validator
-
-import numpy as np
-import os
-import re
-from math import ceil, floor
-from yt.utilities.logger import ytLogger as mylog
-
-
-def is_const_component(record_component):
- return "value" in record_component.attrs.keys()
-
-
-class OpenPMDGrid(AMRGridPatch):
- """
- This class defines the characteristics of the grids
- """
- _id_offset = 0
- __slots__ = ["_level_id"]
- particle_index = {}
- particle_offset = {}
- # TODO (maybe) consider these for every ftype
- mesh_index = 0
- mesh_offset = 0
-
- def __init__(self, id, index, level=-1, pi=None, po=None, mi=0, mo=0):
- AMRGridPatch.__init__(self, id, filename=index.index_filename,
- index=index)
- if pi:
- self.particle_index = pi
- if po:
- self.particle_offset = po
- self.mesh_index = mi
- self.mesh_offset = mo
- self.Parent = None
- self.Children = []
- self.Level = level
-
- def __repr__(self):
- return "OpenPMDGrid_%04i (%s)" % (self.id, self.ActiveDimensions)
-
-
-class OpenPMDHierarchy(GridIndex):
- """
- Defines which fields and particles are created and read from the hard disk
- Furthermore it defines the characteristics of the grids
- """
- grid = OpenPMDGrid
-
- def __init__(self, ds, dataset_type='openPMD'):
- self.dataset_type = dataset_type
- self.dataset = ds
- self.index_filename = ds.parameter_filename
- self.directory = os.path.dirname(self.index_filename)
- GridIndex.__init__(self, ds, dataset_type)
-
- def _detect_output_fields(self):
- """
- Parses the dataset to define field names for yt.
-
- NOTE: Each should be a tuple, where the first element is the on-disk
- fluid type or particle type. Convention suggests that the on-disk
- fluid type is usually the dataset_type and the on-disk particle type
- (for a single population of particles) is "io".
- look for fluid fields
-
- From yt doc:
- self.field_list must be populated as a list of strings corresponding to "native" fields in the data files.
- """
- f = self.dataset._handle
- bp = self.dataset.base_path
- mp = self.dataset.meshes_path
- pp = self.dataset.particles_path
- output_fields = []
-
- for field in f[bp + mp].keys():
- try:
- for axis in f[bp + mp + field].keys():
- output_fields.append(field + "_" + axis)
- except:
- # This is for dataSets, they do not have keys
- output_fields.append(field.replace("_","-"))
- self.field_list = [("openPMD", str(c)) for c in output_fields]
-
- particle_fields = []
- if bp + pp in f:
- for species in f[bp + pp].keys():
- for record in f[bp + pp + species].keys():
- if is_const_component(f[bp + pp + species + "/" + record]):
- # Record itself (e.g. particle_mass) is constant
- particle_fields.append(species + "_" + record)
- elif 'particlePatches' not in record:
- try:
- # Create a field for every axis (x,y,z) of every property (position)
- # of every species (electrons)
- keys = f[bp + pp + species + "/" + record].keys()
- if record in "position":
- record = "positionCoarse"
- for axis in keys:
- particle_fields.append(species + "_" + record + "_" + axis)
- except:
- # Record is a dataset, does not have axes (e.g. weighting)
- particle_fields.append(species + "_" + record)
- pass
- else:
- # We probably do not want particlePatches as accessible field lists
- pass
- if len(f[bp + pp].keys()) > 1:
- # There is more than one particle species, use the specific names as field types
- self.field_list.extend(
- [(str(c).split("_")[0], ("particle_" + "_".join(str(c).split("_")[1:]))) for c in particle_fields])
- else:
- # Only one particle species, fall back to "io"
- self.field_list.extend(
- [("io", ("particle_" + "_".join(str(c).split("_")[1:]))) for c in particle_fields])
-
- def _count_grids(self):
- """
- Counts the number of grids in the dataSet.
-
- From yt doc:
- this must set self.num_grids to be the total number of grids (equiv AMRGridPatch'es) in the simulation
- """
- # TODO Calculate the ppg not solely on particle count, also on meshsize
- f = self.dataset._handle
- bp = self.dataset.base_path
- mp = self.dataset.meshes_path
- pp = self.dataset.particles_path
-
- gridsize = 40 * 10**6 # Bytes
- maxnp = 1
- self.np = {}
-
- for species in f[bp + pp].keys():
- axis = f[bp + pp + species + "/position"].keys()[0]
- if is_const_component(f[bp + pp + species + "/position/" + axis]):
- self.np[species] = f[bp + pp + species + "/position/" + axis].attrs["shape"]
- else:
- self.np[species] = f[bp + pp + species + "/position/" + axis].len()
- if self.np[species] > maxnp:
- maxnp = self.np[species]
- # Limit particles per grid by resulting memory footprint
- ppg = int(gridsize/(self.dataset.dimensionality*4)) # 4 Byte per value per dimension (f32)
- # Use an upper bound of equally sized grids, last one might be smaller
- self.num_grids = int(ceil(maxnp * ppg**-1))
-
- def _parse_index(self):
- """
- Parses dimensions from self._handle into self.
-
- From yt doc:
- this must fill in
- grid_left_edge,
- grid_right_edge,
- grid_particle_count,
- grid_dimensions and
- grid_levels
- with the appropriate information.
- Each of these variables is an array with an entry for each of the self.num_grids grids.
- Additionally, grids must be an array of AMRGridPatch objects that already know their IDs.
- """
- # There is only one refinement level in openPMD
- self.grid_levels.flat[:] = 0
- self.grids = np.empty(self.num_grids, dtype='object')
-
- nrp = self.np.copy() # Number of remaining particles from the dataset
- pci = {} # Index for particle chunk
- for spec in nrp:
- pci[spec] = 0
- remaining = self.dataset.domain_dimensions[0]
- meshindex = 0
- meshedge = self.dataset.domain_left_edge.copy()[0]
- for i in range(self.num_grids):
- self.grid_dimensions[i] = self.dataset.domain_dimensions
- prev = remaining
- remaining -= self.grid_dimensions[i][0] * self.num_grids**-1
- self.grid_dimensions[i][0] = int(round(prev, 0) - round(remaining, 0))
- self.grid_left_edge[i] = self.dataset.domain_left_edge.copy()
- self.grid_left_edge[i][0] = meshedge
- self.grid_right_edge[i] = self.dataset.domain_right_edge.copy()
- self.grid_right_edge[i][0] = self.grid_left_edge[i][0] \
- + self.grid_dimensions[i][0]\
- * self.dataset.domain_dimensions[0]**-1\
- * self.dataset.domain_right_edge[0]
- meshedge = self.grid_right_edge[i][0]
- particlecount = []
- particleindex = []
- for spec in self.np:
- particleindex += [(spec, pci[spec])]
- if i is (self.num_grids - 1):
- # The last grid need not be the same size as the previous ones
- num = nrp[spec]
- else:
- num = int(floor(self.np[spec] * self.num_grids**-1))
- particlecount += [(spec, num)]
- nrp[spec] -= num
- self.grid_particle_count[i] += num
- self.grids[i] = self.grid(
- i, self, self.grid_levels[i, 0],
- pi=particleindex,
- po=particlecount,
- mi=meshindex,
- mo=self.grid_dimensions[i][0])
- for spec, val in particlecount:
- pci[spec] += val
- meshindex += self.grid_dimensions[i][0]
- remaining -= self.grid_dimensions[i][0]
-
- def _populate_grid_objects(self):
- """
- This function initializes the grids
-
- From yt doc:
- this initializes the grids by calling _prepare_grid() and _setup_dx() on all of them.
- Additionally, it should set up Children and Parent lists on each grid object.
- """
- for i in range(self.num_grids):
- self.grids[i]._prepare_grid()
- self.grids[i]._setup_dx()
- self.max_level = 0
-
-
-class OpenPMDDataset(Dataset):
- """
- A dataset object contains all the information of the simulation and
- is intialized with yt.load()
-
- TODO Ideally, a data set object should only contain a single data set.
- afaik, yt.load() can load multiple data sets and also supports
- multiple iteration-loading if done that way, e.g., from a prefix
- of files.
- """
- _index_class = OpenPMDHierarchy
- _field_info_class = OpenPMDFieldInfo
- _nonstandard = False
-
- def __init__(self, filename, dataset_type='openPMD',
- storage_filename=None,
- units_override=None,
- unit_system="mks"):
- self._handle = HDF5FileHandler(filename)
- self._filepath = os.path.dirname(filename)
- if self._nonstandard:
- self._set_nonstandard_paths(self._handle)
- else:
- self._set_paths(self._handle, self._filepath)
- Dataset.__init__(self, filename, dataset_type,
- units_override=units_override,
- unit_system=unit_system)
- self.storage_filename = storage_filename
- self.fluid_types += ('openPMD',)
- parts = tuple(str(c) for c in self._handle[self.base_path + self.particles_path].keys())
- if len(parts) > 1:
- # Only use infile particle names if there is more than one species
- self.particle_types = parts
- mylog.debug("openPMD - self.particle_types: {}".format(self.particle_types))
- self.particle_types_raw = self.particle_types
- self.particle_types = tuple(self.particle_types)
-
- def _set_nonstandard_paths(self, handle):
- iteration = handle["/data"].keys()[0]
- self.base_path = "/data/{}/".format(iteration)
- self.meshes_path = "fields/"
- self.particles_path = "particles/"
-
- def _set_paths(self, handle, filepath):
- list_iterations = []
- if u"groupBased" in handle.attrs["iterationEncoding"]:
- for i in list(handle["/data"].keys()):
- list_iterations.append(i)
- mylog.info("openPMD: found {} iterations in file".format(len(list_iterations)))
- elif u"fileBased" in handle.attrs["iterationEncoding"]:
- regex = u"^" + handle.attrs["iterationFormat"].replace('%T', '[0-9]+') + u"$"
- if filepath is '':
- mylog.warning("openPMD: For file based iterations, please use absolute file paths!")
- pass
- for filename in os.listdir(filepath):
- if re.match(regex, filename):
- list_iterations.append(filename)
- mylog.info("openPMD: found {} iterations in directory".format(len(list_iterations)))
- else:
- mylog.warning(
- "openOMD: File does not have valid iteration encoding: {}".format(handle.attrs["iterationEncoding"]))
-
- if len(list_iterations) == 0:
- mylog.warning("openOMD: No iterations found!")
- if u"groupBased" in handle.attrs["iterationEncoding"] and len(list_iterations) > 1:
- mylog.warning("openPMD: only choose to load one iteration ({})".format(handle["/data"].keys()[0]))
-
- self.base_path = "{}/{}/".format("/data", handle["/data"].keys()[0])
- self.meshes_path = self._handle["/"].attrs["meshesPath"]
- self.particles_path = self._handle["/"].attrs["particlesPath"]
-
- def _set_code_unit_attributes(self):
- """
- From yt doc:
- handle conversion between the different physical units and the code units
- """
- # We hardcode these to 1.0 since every dataset can have different code <-> physical scaling
- # We get the actual unit by multiplying with "unitSI" when getting our data from disk
- self.length_unit = self.quan(1.0, "m")
- self.mass_unit = self.quan(1.0, "kg")
- self.time_unit = self.quan(1.0, "s")
- self.velocity_unit = self.quan(1.0, "m/s")
- self.magnetic_unit = self.quan(1.0, "T")
-
- def _parse_parameter_file(self):
- """
- From yt doc:
- read in metadata describing the overall data on disk
- """
- f = self._handle
- bp = self.base_path
- mp = self.meshes_path
-
- self.unique_identifier = 0
- self.parameters = 0
-
- # We assume all fields to have the same shape
- try:
- mesh = f[bp + mp].keys()[0]
- axis = f[bp + mp + "/" + mesh].keys()[0]
- fshape = np.asarray(f[bp + mp + "/" + mesh + "/" + axis].shape, dtype=np.int64)
- except:
- fshape = np.array([1, 1, 1], dtype=np.int64)
- mylog.warning("Could not detect shape of simulated field! "
- "Assuming a single cell and thus setting fshape to [1, 1, 1]!")
- self.dimensionality = len(fshape)
-
- fshape = np.append(fshape, np.ones(3 - self.dimensionality))
- self.domain_dimensions = fshape
-
- self.domain_left_edge = np.zeros(3, dtype=np.float64)
- try:
- mesh = f[bp + mp].keys()[0]
- if self._nonstandard:
- width = f[bp].attrs['cell_width']
- height = f[bp].attrs['cell_height']
- depth = f[bp].attrs['cell_depth']
- spacing = np.asarray([width, height, depth])
- unitSI = f[bp].attrs['unit_length']
- else:
- spacing = np.asarray(f[bp + mp + "/" + mesh].attrs["gridSpacing"])
- unitSI = f[bp + mp + "/" + mesh].attrs["gridUnitSI"]
- self.domain_right_edge = self.domain_dimensions[:spacing.size] * unitSI * spacing
- self.domain_right_edge = np.append(self.domain_right_edge, np.ones(3 - self.domain_right_edge.size))
- except Exception as e:
- mylog.warning("The domain extent could not be calculated! ({}) Setting the field extent to 1m**3! "
- "This WILL break particle-overplotting!".format(e))
- self.domain_right_edge = np.ones(3, dtype=np.float64)
-
- if self._nonstandard:
- self.current_time = 0
- else:
- self.current_time = f[bp].attrs["time"]
-
- self.periodicity = np.zeros(3, dtype=np.bool)
- self.refine_by = 1
- self.cosmological_simulation = 0
-
- @classmethod
- def _is_valid(self, *args, **kwargs):
- """
- Checks whether the supplied file adheres to the required openPMD standards
- and thus can be read by this frontend
- """
- try:
- f = validator.open_file(args[0])
- except:
- return False
- verbose = False
- extension_pic = False
- # root attributes at "/"
- result_array = np.array([0, 0])
- result_array += validator.check_root_attr(f, verbose, extension_pic)
-
- # Go through all the iterations, checking both the particles
- # and the meshes
- result_array += validator.check_iterations(f, verbose, extension_pic)
-
- # this might still be a compatible file not fully respecting openPMD standards
- if result_array[0] != 0:
- try:
- if "/data" in f and f["/data"].keys()[0].isdigit():
- self._nonstandard = True
- mylog.info("Reading a file with the openPMD frontend that does not respect standards? "
- "Just understand that you're on your own for this!")
- return True
- except:
- return False
- return True
diff -r cb452bf1282587483afb193829c3cad67b1ee557 -r c6cf6d23e8544b19bb570d804a99d93bd3465978 yt/frontends/openPMD/definitions.py
--- a/yt/frontends/openPMD/definitions.py
+++ /dev/null
@@ -1,1 +0,0 @@
-# This file is often empty. It can hold definitions related to a frontend.
diff -r cb452bf1282587483afb193829c3cad67b1ee557 -r c6cf6d23e8544b19bb570d804a99d93bd3465978 yt/frontends/openPMD/fields.py
--- a/yt/frontends/openPMD/fields.py
+++ /dev/null
@@ -1,211 +0,0 @@
-"""
-openPMD-specific fields
-
-
-
-"""
-
-#-----------------------------------------------------------------------------
-# Copyright (c) 2013, yt Development Team.
-# Copyright (c) 2015, Daniel Grassinger (HZDR)
-# Copyright (c) 2016, Fabian Koller (HZDR)
-#
-# Distributed under the terms of the Modified BSD License.
-#
-# The full license is in the file COPYING.txt, distributed with this software.
-#-----------------------------------------------------------------------------
-
-import numpy as np
-from yt.funcs import mylog
-from yt.utilities.physical_constants import speed_of_light
-from yt.fields.field_info_container import \
- FieldInfoContainer
-from yt.fields.magnetic_field import \
- setup_magnetic_field_aliases
-from .misc import parse_unit_dimension
-import yt
-
-
-def setup_kinetic_energy(self, ptype):
- def _kin_en(field, data):
- p2 = (data[ptype, "particle_momentum_x"]**2 +
- data[ptype, "particle_momentum_y"]**2 +
- data[ptype, "particle_momentum_z"]**2)
- mass = data[ptype, "particle_mass"] * data[ptype, "particle_weighting"]
- return speed_of_light * np.sqrt(p2 + mass**2 * speed_of_light**2) - mass * speed_of_light**2
-
- self.add_field((ptype, "particle_kinetic_energy"),
- function=_kin_en,
- units="kg*m**2/s**2",
- particle_type=True)
-
-
-def setup_velocity(self, ptype):
- def _get_vel(axis):
- def velocity(field, data):
- c = speed_of_light
- momentum = data[ptype, "particle_momentum_{}".format(axis)]
- mass = data[ptype, "particle_mass"]
- weighting = data[ptype, "particle_weighting"]
- return momentum / (
- (mass * weighting)**2 +
- (momentum**2) / (c**2)
- ) ** 0.5
-
- return velocity
-
- for ax in "xyz":
- self.add_field((ptype, "particle_velocity_%s" % ax),
- function=_get_vel(ax),
- units="m/s",
- particle_type=True)
-
-
-def setup_poynting_vector(self):
- def _get_poyn(axis):
- def poynting(field, data):
- u = 79577.4715459 # = 1/magnetic permeability
- if axis in "x":
- return u * (data["E_y"] * data["magnetic_field_z"] - data["E_z"] * data["magnetic_field_y"])
- elif axis in "y":
- return u * (data["E_z"] * data["magnetic_field_x"] - data["E_x"] * data["magnetic_field_z"])
- elif axis in "z":
- return u * (data["E_x"] * data["magnetic_field_y"] - data["E_y"] * data["magnetic_field_x"])
-
- return poynting
-
- for ax in "xyz":
- self.add_field(("openPMD", "poynting_vector_%s" % ax),
- function=_get_poyn(ax),
- units="T*V/m")
-
-
-def setup_absolute_positions(self, ptype):
- def _abs_pos(axis):
- def ap(field, data):
- return np.add(data[ptype, "particle_positionCoarse_{}".format(axis)],
- data[ptype, "particle_positionOffset_{}".format(axis)])
-
- return ap
-
- for ax in "xyz":
- self.add_field((ptype, "particle_position_%s" % ax),
- function=_abs_pos(ax),
- units="m",
- particle_type=True)
-
-
-class OpenPMDFieldInfo(FieldInfoContainer):
- """
- We need to specify which fields we might have in our dataset. The field info
- container subclass here will define which fields it knows about. There are
- optionally methods on it that get called which can be subclassed.
-
- This class defines, which fields and particle fields could be in the HDF5-file
- The field names have to match the names in "openPMDHierarchy" in data_structures.py
- This also defines the units of the fields
- """
-
- _mag_fields = []
-
- def __init__(self, ds, field_list):
- f = ds._handle
- bp = ds.base_path
- mp = ds.meshes_path
- pp = ds.particles_path
- fields = f[bp + mp]
-
- for fname in fields.keys():
- field = fields.get(fname)
- if "dataset" in str(field).split(" ")[1]:
- # We have a dataset, don't consider axes. This appears to be a vector field of single dimensionality
- ytname = str("_".join([fname.replace("_", "-")]))
- if ds._nonstandard:
- parsed = ""
- else:
- parsed = parse_unit_dimension(np.asarray(field.attrs["unitDimension"], dtype="int"))
- unit = str(yt.YTQuantity(1, parsed).units)
- aliases = []
- # Save a list of magnetic fields for aliasing later on
- # We can not reasonably infer field type by name in openPMD
- if "T" in unit or "kg/(A*s**2)" in unit:
- self._mag_fields.append(ytname)
- self.known_other_fields += ((ytname, (unit, aliases, None)), )
- else:
- if ds._nonstandard:
- axes = "xyz" # naively assume all fields in non-standard files are 3D
- else:
- axes = field.attrs["axisLabels"]
- for axis in axes:
- ytname = str("_".join([fname.replace("_", "-"), axis]))
- if ds._nonstandard:
- parsed = ""
- else:
- parsed = parse_unit_dimension(np.asarray(field.attrs["unitDimension"], dtype="int"))
- unit = str(yt.YTQuantity(1, parsed).units)
- aliases = []
- # Save a list of magnetic fields for aliasing later on
- # We can not reasonably infer field type by name in openPMD
- if "T" in unit or "kg/(A*s**2)" in unit:
- self._mag_fields.append(ytname)
- self.known_other_fields += ((ytname, (unit, aliases, None)), )
- for i in self.known_other_fields:
- mylog.debug("oPMD - fields - known_other_fields - {}".format(i))
-
- particle_fields = ()
- particles = f[bp + pp]
- for species in particles.keys():
- for attrib in particles.get(species).keys():
- if "weighting" in attrib:
- particle_fields += (("particle_weighting", ("", [], None)),)
- continue
- try:
- if ds._nonstandard:
- if "globalCellIdx" in attrib or "position" in attrib:
- parsed = "m" # Required for spatial selection of particles
- else:
- parsed = ""
- else:
- parsed = parse_unit_dimension(
- np.asarray(particles.get(species).get(attrib).attrs["unitDimension"], dtype="int"))
- unit = str(yt.YTQuantity(1, parsed).units)
- name = ["particle", attrib]
- ytattrib = attrib
- # Symbolically rename position to preserve yt's interpretation of the pfield
- # particle_position is later derived in setup_absolute_positions
- if ytattrib in "position":
- ytattrib = "positionCoarse"
- for axis in particles.get(species).get(attrib).keys():
- aliases = []
- if axis in "rxyz":
- name = ["particle", ytattrib, axis]
- ytname = str("_".join(name))
- if ds._nonstandard and "globalCellIdx" in ytname:
- aliases.append(ytname.replace("globalCellIdx", "positionOffset"))
- particle_fields += ((ytname, (unit, aliases, None)), )
- except:
- mylog.info("{}_{} does not seem to have unitDimension".format(species, attrib))
- self.known_particle_fields = particle_fields
- for i in self.known_particle_fields:
- mylog.debug("oPMD - fields - known_particle_fields - {}".format(i))
- super(OpenPMDFieldInfo, self).__init__(ds, field_list)
-
- def setup_fluid_fields(self):
- """
- Here you can create functions to calculate out of existing fields the
- values of new fields e.g. calculate out of E-field and B-field the
- Poynting vector
- """
- # Set up aliases first so the setup for poynting can use them
- if len(self._mag_fields) > 0:
- setup_magnetic_field_aliases(self, "openPMD", self._mag_fields)
- setup_poynting_vector(self)
-
- def setup_particle_fields(self, ptype):
- """
- This will get called for every particle type.
- """
- setup_absolute_positions(self, ptype)
- setup_kinetic_energy(self, ptype)
- setup_velocity(self, ptype)
- super(OpenPMDFieldInfo, self).setup_particle_fields(ptype)
diff -r cb452bf1282587483afb193829c3cad67b1ee557 -r c6cf6d23e8544b19bb570d804a99d93bd3465978 yt/frontends/openPMD/io.py
--- a/yt/frontends/openPMD/io.py
+++ /dev/null
@@ -1,279 +0,0 @@
-"""
-openPMD-specific IO functions
-
-
-
-"""
-
-#-----------------------------------------------------------------------------
-# Copyright (c) 2013, yt Development Team.
-# Copyright (c) 2015, Daniel Grassinger (HZDR)
-# Copyright (c) 2016, Fabian Koller (HZDR)
-#
-# Distributed under the terms of the Modified BSD License.
-#
-# The full license is in the file COPYING.txt, distributed with this software.
-#-----------------------------------------------------------------------------
-
-from yt.utilities.io_handler import \
- BaseIOHandler
-from yt.utilities.logger import ytLogger as mylog
-import numpy as np
-
-
-class IOHandlerOpenPMD(BaseIOHandler):
- _field_dtype = "float32"
- _dataset_type = "openPMD"
-
- def __init__(self, ds, *args, **kwargs):
- self.ds = ds
- self._handle = ds._handle
- self.base_path = ds.base_path
- self.meshes_path = ds.meshes_path
- self.particles_path = ds.particles_path
- self._array_fields = {}
- self._cached_ptype = ""
-
- def _fill_cache(self, ptype, index=0, offset=None):
- if str((ptype, index, offset)) not in self._cached_ptype:
- # Get a particle species (e.g. /data/3500/particles/e/)
- if ptype in "io":
- spec = self._handle[self.base_path + self.particles_path].keys()[0]
- else:
- spec = ptype
- pds = self._handle[self.base_path + self.particles_path + "/" + spec]
- # Get 1D-Arrays for individual particle positions along axes
- axes = [str(ax) for ax in pds["position"].keys()]
- if self.ds._nonstandard:
- position_offset = "globalCellIdx/"
- else:
- position_offset = "positionOffset/"
- self.cache = np.empty((3, offset), dtype="float64")
- for i in range(3):
- ax = "xyz"[i]
- if ax in axes:
- np.add(self.get_component(pds, "position/" + ax, index, offset, self.ds._nonstandard),
- self.get_component(pds, position_offset + ax, index, offset, self.ds._nonstandard),
- self.cache[i])
- else:
- # Pad accordingly with zeros to make 1D/2D datasets compatible
- # These have to be the same shape as the existing axes since that equals the number of particles
- self.cache[i] = np.zeros(offset)
- self._cached_ptype = str((ptype, index, offset))
-
- def _read_particle_coords(self, chunks, ptf):
- """
- Reads coordinates for all specified particle-types from file.
-
- Parameters
- ----------
- chunks:
- A list of chunks
- A chunk is a list of grids
-
- ptf:
- A dictionary
- - keys are ptypes
- - values are lists of particle fields
-
- Yields
- ------
- A series of tuples of (ptype_name, (x_coords, y_coords, z_coords)).
- ptype_name is a type pf particle,
- x_coords, y_coords and z_coords are arrays of positions/coordinates of all particles of that type
- """
- chunks = list(chunks)
- f = self._handle
- ds = f[self.base_path]
- for chunk in chunks:
- for g in chunk.objs:
- if g.filename is None:
- continue
- for ptype, field_list in sorted(ptf.items()):
- if ptype in "io":
- spec = ds[self.particles_path].keys()[0]
- else:
- spec = ptype
- for gridptype, idx in g.particle_index:
- if str(gridptype) == str(spec):
- index = idx
- for gridptype, ofs in g.particle_offset:
- if str(gridptype) == str(spec):
- offset = ofs
- mylog.debug("openPMD - _read_particle_coords: (grid {}) {}, {} [{}:{}]".format(g, ptype, field_list, index, offset))
- if str((ptype, index, offset)) not in self._cached_ptype:
- self._fill_cache(ptype, index, offset)
- yield (ptype, (self.cache[0], self.cache[1], self.cache[2]))
-
- def _read_particle_fields(self, chunks, ptf, selector):
- """
- Reads particle fields masked by particle type and field out of a list of chunks from file.
-
- Parameters
- ----------
- chunks:
- A list of chunks
- A chunk is a list of grids
-
- ptf:
- A dictionary
- - keys are ptype
- - values are lists of (particle?) fields
-
-
- selector:
- yt-project.org/docs/dev/quickstart/data_inspection.html?highlight=selector#Examining-Data-in-Regions
- yt-project.org/doc/developing/creating_datatypes.html
- A region (in and/or outside your domain) specifying the field you want to read
- Selector objects have a .select_points(x,y,z) that returns a mask, so you need to do your masking here.
-
- Yields
- -------
- Tuples of structure ((ptype, field), data)
- where data is a numpy array of data masked to
- - a particle type p
- - in a specified field
- """
- chunks = list(chunks)
- f = self._handle
- ds = f[self.base_path]
- for chunk in chunks:
- for g in chunk.objs:
- if g.filename is None:
- continue
- for ptype, field_list in sorted(ptf.items()):
- # Get a particle species (e.g. /data/3500/particles/e/)
- if ptype in "io":
- spec = ds[self.particles_path].keys()[0]
- else:
- spec = ptype
- index = dict(g.particle_index).get(spec)
- offset = dict(g.particle_offset).get(spec)
- mylog.debug("openPMD - _read_particle_fields: (grid {}) {}, {} [{}:{}]".format(g, ptype, field_list, index, offset))
- if str((ptype, index, offset)) not in self._cached_ptype:
- self._fill_cache(ptype, index, offset)
- mask = selector.select_points(self.cache[0], self.cache[1], self.cache[2], 0.0)
- if mask is None:
- continue
- pds = ds[self.particles_path + "/" + spec]
- for field in field_list:
- nfield = "/".join(field.split("_")[1:]).replace("positionCoarse", "position")
- data = self.get_component(pds, nfield, index, offset, self.ds._nonstandard)
- yield ((ptype, field), data[mask])
-
- def _read_fluid_selection(self, chunks, selector, fields, size):
- """
- Reads selected fields of given size from file.
-
- From yt doc:
- Receives a collection of data "chunks", a selector describing which "chunks" you are concerned with,
- a list of fields, and the size of the data to read.
- It should create and return a dictionary whose keys are the fields,
- and whose values are numpy arrays containing the data.
- The data should actually be read via the _read_chunk_data() method.
-
- Parameters
- ----------
- chunks:
- A list of chunks
- A chunk is a list of grids
-
- selector:
- yt-project.org/docs/dev/quickstart/data_inspection.html?highlight=selector#Examining-Data-in-Regions
- yt-project.org/doc/developing/creating_datatypes.html
- A region (in and/or outside your domain) specifying the field you want to read
-
- fields:
- A list of (fname, ftype) tuples representing a field
-
- size:
- Size of the data arrays you want to read
-
- Returns
- -------
- A dictionary:
- - keys are (ftype, fname) tuples representing a field
- - values are numpy arrays with data form that field
- """
- mylog.info("_read_fluid_selection {} {} {} {}".format(chunks, selector, fields, size))
- f = self._handle
- bp = self.base_path
- mp = self.meshes_path
- rv = {}
- chunks = list(chunks)
-
- if selector.__class__.__name__ == "GridSelector":
- if not (len(chunks) == len(chunks[0].objs) == 1):
- raise RuntimeError
-
- if size is None:
- size = sum((g.count(selector) for chunk in chunks
- for g in chunk.objs))
- ind = {}
- for field in fields:
- rv[field] = np.empty(size, dtype="float64")
- ind[field] = 0
-
- for ftype, fname in fields:
- field = (ftype, fname)
- for chunk in chunks:
- for g in chunk.objs:
- ds = f[bp + mp]
- nfield = fname.replace("_", "/").replace("-","_")
- index = g.mesh_index
- offset = g.mesh_offset
- data = np.array(self.get_component(ds, nfield, index, offset, self.ds._nonstandard))
- # The following is a modified AMRGridPatch.select(...)
- mask = g._get_selector_mask(selector)
- data.shape = mask.shape # Workaround - casts a 2D (x,y) array to 3D (x,y,1)
- count = g.count(selector)
- rv[field][ind[field]:ind[field] + count] = data[mask]
- ind[field] += count
- for i in rv:
- rv[i].flatten()
- return rv
-
- def get_component(self, group, component_name, index=0, offset=None, nonstandard=False):
- """Grab a component from a group
-
- Parameters
- ----------
- group: hdf5 group to get whole/masked component from
- component_name: string of a component (relative path) inside the group
- index: (optional) start index of data to return
- offset: (optional) size of the data to return
- nonstandard: (optional) bool to signal whether to assume (non)standard PMD
-
- Returns
- -------
- n-dimensional numpy array filled with values of component
- """
-
- record_component = group[component_name]
- if nonstandard:
- try:
- unitSI = record_component.attrs["sim_unit"]
- if "globalCellIdx" in component_name:
- it = group["/data"].keys()[0]
- length = group["/data/" + it].attrs['unit_length']
- unitSI *= length
- except:
- unitSI = 1.0
- else:
- unitSI = record_component.attrs["unitSI"]
- # check whether component is constant
- if "value" in record_component.attrs.keys():
- if offset is not None:
- shape = offset
- else:
- shape = record_component.attrs["shape"] - index
- # component is constant, craft an array by hand
- mylog.debug("openPMD - misc - get_component (const): {}/{}({})".format(group.name, component_name, shape))
- return np.full(shape, record_component.attrs["value"] * unitSI)
- else:
- if offset is not None:
- offset += index
- # component is a dataset, return it (possibly masked)
- mylog.debug(
- "openPMD - misc - get_component: {}/{}[{}:{}]".format(group.name, component_name, index, offset))
- return np.multiply(record_component[index:offset], unitSI)
\ No newline at end of file
diff -r cb452bf1282587483afb193829c3cad67b1ee557 -r c6cf6d23e8544b19bb570d804a99d93bd3465978 yt/frontends/openPMD/misc.py
--- a/yt/frontends/openPMD/misc.py
+++ /dev/null
@@ -1,799 +0,0 @@
-#-----------------------------------------------------------------------------
-# Copyright (c) 2015, Axel Huebl, Remi Lehe
-# Copyright (c) 2016, Fabian Koller (HZDR)
-#
-# Permission to use, copy, modify, and/or distribute this software for any
-# purpose with or without fee is hereby granted, provided that the above
-# copyright notice and this permission notice appear in all copies.
-#
-# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
-# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
-# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
-# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
-# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
-# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
-# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-#
-#-----------------------------------------------------------------------------
-
-import h5py as h5
-import numpy as np
-import re
-import string
-import collections # for isinstance
-import sys, getopt, os.path
-from yt.utilities.logger import ytLogger as mylog
-
-openPMD = "1.0.0"
-
-ext_list = [["ED-PIC", np.uint32(1)]]
-
-
-def open_file(file_name):
- try:
- f = h5.File(file_name, "r")
- return(f)
- except:
- raise
-
-
-def get_attr(f, name):
- """
- Try to access the path `name` in the file `f`
- Return the corresponding attribute if it is present
- """
- if name in list(f.attrs.keys()):
- return(True, f.attrs[name])
- else:
- return(False, None)
-
-
-def test_record(g, r):
- """
- Checks if a record is valid
-
- Parameters
- ----------
- g : h5py.Group
- The group the record resides in
-
- r : string
- The name of the record.
-
- Returns
- -------
- An array with 2 elements :
- - The first element is 1 if an error occured, and 0 otherwise
- - The second element is 0 if a warning arised, and 0 otherwise
- """
- regEx = re.compile("^\w+$") # Python3 only: re.ASCII
- if regEx.match(r):
- # test component names
- result_array = np.array([0,0])
- if not is_scalar_record(g[r]) :
- for component_name in g[r]:
- if not regEx.match(component_name):
- mylog.warning("openPMD: Component %s of record %s is NOT" \
- " named properly (a-Z0-9_)!" %(component_name, g[r].name) )
- result_array += np.array([1,0])
- else:
- mylog.warning("openPMD: Record %s is NOT named properly (a-Z0-9_)!" \
- %(r.name) )
- result_array = np.array([1,0])
-
- return(result_array)
-
-
-def test_key(f, v, request, name):
- """
- Checks whether a key is present. A key can either be
- a h5py.Group or a h5py.Dataset.
- Returns an error if the key if absent and requested
- Returns a warning if the key if absent and recommended
-
- Parameters
- ----------
- f : an h5py.File or h5py.Group object
- The object in which to find the key
-
- v : bool
- Verbose option
-
- request : string
- Either "required", "recommended" or "optional"
-
- name : string
- The name of the key within this File, Group or DataSet
-
- Returns
- -------
- An array with 2 elements :
- - The first element is 1 if an error occured, and 0 otherwise
- - The second element is 0 if a warning arised, and 0 otherwise
- """
- valid = (name in list(f.keys()))
- if valid:
- if v:
- mylog.info("Key %s (%s) exists in `%s`!" %(name, request, str(f.name) ) )
- result_array = np.array([0,0])
- else:
- if request == "required":
- mylog.warning("openPMD: Key %s (%s) does NOT exist in `%s`!" \
- %(name, request, str(f.name)) )
- result_array = np.array([1, 0])
- elif request == "recommended":
- mylog.info("openPMD: Key %s (%s) does NOT exist in `%s`!" \
- %(name, request, str(f.name)) )
- result_array = np.array([0, 1])
- elif request == "optional":
- if v:
- mylog.info("openPMD: Key %s (%s) does NOT exist in `%s`!" \
- %(name, request, str(f.name)) )
- result_array = np.array([0, 0])
- else :
- raise ValueError("Unrecognized string for `request` : %s" %request)
-
- return(result_array)
-
-
-def test_attr(f, v, request, name, is_type=None, type_format=None):
- """
- Checks whether an attribute is present.
- Returns an error if the attribute if absent and requested
- Returns a warning if the attribute if absent and recommanded
-
- Parameters
- ----------
- f : an h5py.File, h5py.Group or h5py.DataSet object
- The object in which to find the key
-
- v : bool
- Verbose option
-
- request : string
- Either "required", "recommended" or "optional
-
- name : string
- The name of the attribute within this File, Group or DataSet
-
- is_type : (numpy or python) data type
- The type of the attribute. Default is "arbitrary" for None.
- Can be a list of data types where at least one data type must match
- but this list can not be combined with type_format.
-
- type_format: (numpy or python) data type
- Used with is_type to specify numpy ndarray dtypes or a
- base np.string_ format regex. Can be a list of data types
- for ndarrays where at least one data type must match.
-
- Returns
- -------
- An array with 2 elements :
- - The first element is 1 if an error occured, and 0 otherwise
- - The second element is 0 if a warning arised, and 0 otherwise
- """
- valid, value = get_attr(f, name)
- if valid:
- if v:
- mylog.info("openPMD: Attribute %s (%s) exists in `%s`! Type = %s, Value = %s" \
- %(name, request, str(f.name), type(value), str(value)) )
-
- # test type
- if is_type is not None:
- if not type_format is None and not is_type is np.string_ and \
- not isinstance(type_format, collections.Iterable):
- type_format = [type_format]
- type_format_names = map(lambda x: x.__name__, type_format)
- if not is_type is None and not isinstance(is_type, collections.Iterable):
- is_type = [is_type]
- is_type_names = map(lambda x: x.__name__, is_type)
- # add for each type in is_type -> wrong, need to add this at the comparison level!
- if type(value) in is_type:
- # np.string_ format or general ndarray dtype text
- if type(value) is np.string_ and type_format is not None:
- regEx = re.compile(type_format) # Python3 only: re.ASCII
- if regEx.match(value.decode()) :
- result_array = np.array([0,0])
- else:
- mylog.warning("openPMD: Attribute %s in `%s` does not satisfy " \
- "format ('%s' should be in format '%s')!" \
- %(name, str(f.name), value.decode(), type_format ) )
- result_array = np.array([1,0])
- # ndarray dtypes
- elif type(value) is np.ndarray:
- if value.dtype.type in type_format:
- result_array = np.array([0,0])
- elif type_format is None:
- result_array = np.array([0,0])
- else:
- mylog.warning("openPMD: Attribute %s in `%s` is not of type " \
- "ndarray of '%s' (is ndarray of '%s')!" \
- %(name, str(f.name), type_format_names, \
- value.dtype.type.__name__) )
- result_array = np.array([1,0])
- else:
- result_array = np.array([0,0])
- else:
- mylog.error(
- "Error: Attribute %s in `%s` is not of type '%s' (is '%s')!" \
- %(name, str(f.name), str(is_type_names), \
- type(value).__name__) )
- result_array = np.array([1,0])
- else: # is_type is None (== arbitrary)
- result_array = np.array([0,0])
- else:
- if request == "required":
- mylog.warning("openPMD: Attribute %s (%s) does NOT exist in `%s`!" \
- %(name, request, str(f.name)) )
- result_array = np.array([1, 0])
- elif request == "recommended":
- mylog.info("openPMD: Attribute %s (%s) does NOT exist in `%s`!" \
- %(name, request, str(f.name)) )
- result_array = np.array([0, 1])
- elif request == "optional":
- if v:
- mylog.info("openPMD: Attribute %s (%s) does NOT exist in `%s`!" \
- %(name, request, str(f.name)) )
- result_array = np.array([0, 0])
- else :
- raise ValueError("Unrecognized string for `request` : %s" %request)
-
- return(result_array)
-
-
-def is_scalar_record(r):
- """
- Checks if a record is a scalar record or not.
-
- Parameters
- ----------
- r : an h5py.Group or h5py.Dataset object
- the record that shall be tested
-
- Returns
- -------
- bool : true if the record is a scalar record, false if the record
- is either a vector or an other type of tensor record
- """
- if type(r) is h5.Group :
- # now it could be either a vector/tensor record
- # or a scalar record with a constant component
-
- valid, value = get_attr(r, "value")
- # constant components require a "value" and a "shape" attribute
- if valid :
- return True
- else:
- return False
- else :
- return True
-
-
-def test_component(c, v) :
- """
- Checks if a record component defines all required attributes.
-
- Parameters
- ----------
- c : an h5py.Group or h5py.Dataset object
- the record component that shall be tested
-
- v : bool
- Verbose option
-
- Returns
- -------
- An array with 2 elements :
- - The first element is the number of errors encountered
- - The second element is the number of warnings encountered
- """
- # Initialize the result array
- # First element : number of errors
- # Second element : number of warnings
- result_array = np.array([0,0])
-
- if type(c) is h5.Group :
- # since this check tests components, this must be a constant
- # component: requires "value" and "shape" attributes
- result_array += test_attr(c, v, "required", "value") # type can be arbitrary
- result_array += test_attr(c, v, "required", "shape", np.ndarray, np.uint64)
-
- # default attributes for all components
- result_array += test_attr(c, v, "required", "unitSI", np.float64)
-
- return(result_array)
-
-
-def check_root_attr(f, v, pic):
- """
- Scan the root of the file and make sure that all the attributes are present
-
- Parameters
- ----------
- f : an h5py.File object
- The HDF5 file in which to find the attribute
-
- v : bool
- Verbose option
-
- pic : bool
- Whether to check for the ED-PIC extension attributes
-
- Returns
- -------
- An array with 2 elements :
- - The first element is the number of errors encountered
- - The second element is the number of warnings encountered
- """
- # Initialize the result array
- # First element : number of errors
- # Second element : number of warnings
- result_array = np.array([0,0])
-
- # STANDARD.md
- # required
- result_array += test_attr(f, v, "required", "openPMD", np.string_, "^[0-9]+\.[0-9]+\.[0-9]+$")
- result_array += test_attr(f, v, "required", "openPMDextension", np.uint32)
- result_array += test_attr(f, v, "required", "basePath", np.string_, "^\/data\/\%T\/$")
- result_array += test_attr(f, v, "required", "meshesPath", np.string_)
- result_array += test_attr(f, v, "required", "particlesPath", np.string_)
- result_array += test_attr(f, v, "required", "iterationEncoding", np.string_, "^groupBased|fileBased$")
- result_array += test_attr(f, v, "required", "iterationFormat", np.string_)
-
- # groupBased iteration encoding needs to match basePath
- if result_array[0] == 0 :
- if f.attrs["iterationEncoding"].decode() == "groupBased" :
- if f.attrs["iterationFormat"].decode() != f.attrs["basePath"].decode() :
- mylog.warning("openPMD: for groupBased iterationEncoding the basePath "
- "and iterationFormat must match!")
- result_array += np.array([1,0])
-
- # recommended
- result_array += test_attr(f, v, "recommended", "author", np.string_)
- result_array += test_attr(f, v, "recommended", "software", np.string_)
- result_array += test_attr(f, v, "recommended",
- "softwareVersion", np.string_)
- result_array += test_attr(f, v, "recommended", "date", np.string_,
- "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [\+|-][0-9]{4}$")
-
- # optional
- result_array += test_attr(f, v, "optional", "comment", np.string_)
-
- # Extension: ED-PIC
- if pic:
- valid, extensionIDs = get_attr(f, "openPMDextension")
- if valid:
- if (ext_list[0][1] & extensionIDs) != extensionIDs:
- mylog.warning("openPMD: ID=%s for extension `%s` not found in " \
- "`openPMDextension` (is %s)!" \
- %(ext_list[0][1], ext_list[0][0], extensionIDs) )
- result_array += np.array([1,0])
-
- return(result_array)
-
-
-def check_iterations(f, v, pic) :
- """
- Scan all the iterations present in the file, checking both
- the meshes and the particles
-
- Parameters
- ----------
- f : an h5py.File object
- The HDF5 file in which to find the attribute
-
- v : bool
- Verbose option
-
- pic : bool
- Whether to check for the ED-PIC extension attributes
-
- Returns
- -------
- An array with 2 elements :
- - The first element is the number of errors encountered
- - The second element is the number of warnings encountered
- """
-
- # Find all the iterations
- format_error = False
- try :
- list_iterations = list(f['/data/'].keys())
- except KeyError :
- format_error = True
- else :
- # Check that these iterations are indeed encoded as integers
- for iteration in list_iterations :
- for character in iteration : # go through the string
- if not (character in string.digits) :
- format_error = True
- # Detect any error and interrupt execution if one is found
- if format_error == True :
- mylog.warning("openPMD: it seems that the path of the data within the HDF5 file "
- "is not of the form '/data/%T/', where %T corresponds to an "
- "actual integer.")
- return(np.array([1, 0]))
- else :
- mylog.info("openPMD: Found %d iteration(s)" % len(list_iterations) )
-
- # Initialize the result array
- # First element : number of errors
- # Second element : number of warnings
- result_array = np.array([ 0, 0])
-
- # Loop over the iterations and check the meshes and the particles
- for iteration in list_iterations :
- result_array += check_base_path(f, iteration, v, pic)
- # Go deeper only if there is no error at this point
- if result_array[0] == 0 :
- result_array += check_meshes(f, iteration, v, pic)
- result_array += check_particles(f, iteration, v, pic)
-
- return(result_array)
-
-
-def check_base_path(f, iteration, v, pic):
- """
- Scan the base_path that corresponds to this iteration
-
- Parameters
- ----------
- f : an h5py.File object
- The HDF5 file in which to find the attribute
-
- iteration : string representing an integer
- The iteration at which to scan the meshes
-
- v : bool
- Verbose option
-
- pic : bool
- Whether to check for the ED-PIC extension attributes
-
- Returns
- -------
- An array with 2 elements :
- - The first element is the number of errors encountered
- - The second element is the number of warnings encountered
- """
- # Initialize the result array
- # First element : number of errors
- # Second element : number of warnings
- result_array = np.array([ 0, 0])
-
- # Find the path to the data
- base_path = ("/data/%s/" % iteration).encode('ascii')
- bp = f[base_path]
-
- # Check for the attributes of the STANDARD.md
- result_array += test_attr(bp, v, "required", "time", [np.float32, np.float64])
- result_array += test_attr(bp, v, "required", "dt", [np.float32, np.float64])
- result_array += test_attr(bp, v, "required", "timeUnitSI", np.float64)
-
- return(result_array)
-
-
-def check_meshes(f, iteration, v, pic):
- """
- Scan all the meshes corresponding to one iteration
-
- Parameters
- ----------
- f : an h5py.File object
- The HDF5 file in which to find the attribute
-
- iteration : string representing an integer
- The iteration at which to scan the meshes
-
- v : bool
- Verbose option
-
- pic : bool
- Whether to check for the ED-PIC extension attributes
-
- Returns
- -------
- An array with 2 elements :
- - The first element is the number of errors encountered
- - The second element is the number of warnings encountered
- """
- # Initialize the result array
- # First element : number of errors
- # Second element : number of warnings
- result_array = np.array([ 0, 0])
-
- # Find the path to the data
- base_path = "/data/%s/" % iteration
- valid, meshes_path = get_attr(f, "meshesPath")
- if not valid :
- mylog.warning("openPMD: `meshesPath` is missing or malformed in '/'")
- return( np.array([1, 0]) )
- meshes_path = meshes_path.decode()
-
- if os.path.join( base_path, meshes_path) != ( base_path + meshes_path ):
- mylog.warning("openPMD: `basePath`+`meshesPath` seems to be malformed "
- "(is `basePath` absolute and ends on a `/` ?)")
- return( np.array([1, 0]) )
- else:
- full_meshes_path = (base_path + meshes_path).encode('ascii')
- # Find all the meshes
- try:
- list_meshes = list(f[full_meshes_path].keys())
- except KeyError:
- list_meshes = []
- mylog.debug( "Iteration %s : found %d meshes"
- %( iteration, len(list_meshes) ) )
-
- # Check for the attributes of the STANDARD.md
- for field_name in list_meshes :
- field = f[full_meshes_path + field_name.encode('ascii')]
-
- result_array += test_record(f[full_meshes_path], field_name)
-
- # General attributes of the record
- result_array += test_attr(field, v, "required",
- "unitDimension", np.ndarray, np.float64)
- result_array += test_attr(field, v, "required",
- "timeOffset", [np.float32, np.float64])
- result_array += test_attr(field, v, "required",
- "gridSpacing", np.ndarray, [np.float32, np.float64])
- result_array += test_attr(field, v, "required",
- "gridGlobalOffset", np.ndarray, [np.float32, np.float64])
- result_array += test_attr(field, v, "required",
- "gridUnitSI", np.float64)
- result_array += test_attr(field, v, "required",
- "dataOrder", np.string_)
- result_array += test_attr(field, v, "required",
- "axisLabels", np.ndarray, np.string_)
- # Specific check for geometry
- geometry_test = test_attr(field, v, "required", "geometry", np.string_)
- result_array += geometry_test
- # geometryParameters is required when using thetaMode
- if geometry_test[0] == 0 and field.attrs["geometry"] == b"thetaMode" :
- result_array += test_attr(field, v, "required",
- "geometryParameters", np.string_)
- # otherwise it is optional
- else :
- result_array += test_attr(field, v, "optional",
- "geometryParameters", np.string_)
-
- # Attributes of the record's components
- if is_scalar_record(field) : # If the record is a scalar field
- result_array += test_component(field, v)
- result_array += test_attr(field, v,
- "required", "position", np.ndarray, [np.float32, np.float64])
- else: # If the record is a vector field
- # Loop over the components
- for component_name in list(field.keys()) :
- component = field[component_name]
- result_array += test_component(component, v)
- result_array += test_attr(component, v,
- "required", "position", np.ndarray, [np.float32, np.float64])
-
- # Check for the attributes of the PIC extension,
- # if asked to do so by the user
- if pic:
-
- # Check the attributes associated with the field solver
- result_array += test_attr(f[full_meshes_path], v, "required",
- "fieldSolver", np.string_)
- valid, field_solver = get_attr(f[full_meshes_path], "fieldSolver")
- if (valid == True) and (field_solver in ["other", "GPSTD"]) :
- result_array += test_attr(f[full_meshes_path], v, "required",
- "fieldSolverParameters", np.string_)
-
- # Check for the attributes associated with the field boundaries
- result_array += test_attr(f[full_meshes_path], v, "required",
- "fieldBoundary", np.ndarray, np.string_)
- valid, field_boundary = get_attr(f[full_meshes_path], "fieldBoundary")
- if (valid == True) and (np.any(field_boundary == b"other")) :
- result_array += test_attr(f[full_meshes_path], v, "required",
- "fieldBoundaryParameters", np.ndarray, np.string_)
-
- # Check for the attributes associated with the field boundaries
- result_array += test_attr(f[full_meshes_path], v, "required",
- "particleBoundary", np.ndarray, np.string_)
- valid, particle_boundary = get_attr(f[full_meshes_path], "particleBoundary")
- if (valid == True) and (np.any(particle_boundary == b"other")) :
- result_array += test_attr(f[full_meshes_path], v, "required",
- "particleBoundaryParameters", np.ndarray, np.string_)
-
- # Check the attributes associated with the current smoothing
- result_array += test_attr(f[full_meshes_path], v, "required",
- "currentSmoothing", np.string_)
- valid, current_smoothing = get_attr(f[full_meshes_path], "currentSmoothing")
- if (valid == True) and (current_smoothing != b"none") :
- result_array += test_attr(f[full_meshes_path], v, "required",
- "currentSmoothingParameters", np.string_)
-
- # Check the attributes associated with the charge conservation
- result_array += test_attr(f[full_meshes_path], v, "required",
- "chargeCorrection", np.string_)
- valid, charge_correction = get_attr(f[full_meshes_path], "chargeCorrection")
- if valid == True and charge_correction != b"none":
- result_array += test_attr(f[full_meshes_path], v, "required",
- "chargeCorrectionParameters", np.string_)
-
- # Check for the attributes of each record
- for field_name in list_meshes :
- field = f[full_meshes_path + field_name.encode('ascii')]
- result_array + test_attr(field, v, "required",
- "fieldSmoothing", np.string_)
- valid, field_smoothing = get_attr(field, "fieldSmoothing")
- if (valid == True) and (field_smoothing != b"none") :
- result_array += test_attr(field,v, "required",
- "fieldSmoothingParameters", np.string_)
- return(result_array)
-
-
-def check_particles(f, iteration, v, pic) :
- """
- Scan all the particle data corresponding to one iteration
-
- Parameters
- ----------
- f : an h5py.File object
- The HDF5 file in which to find the attribute
-
- iteration : string representing an integer
- The iteration at which to scan the particle data
-
- v : bool
- Verbose option
-
- pic : bool
- Whether to check for the ED-PIC extension attributes
-
- Returns
- -------
- An array with 2 elements :
- - The first element is the number of errors encountered
- - The second element is the number of warnings encountered
- """
- # Initialize the result array
- # First element : number of errors
- # Second element : number of warnings
- result_array = np.array([ 0, 0])
-
- # Find the path to the data
- base_path = ("/data/%s/" % iteration).encode('ascii')
- valid, particles_path = get_attr(f, "particlesPath")
- if os.path.join( base_path, particles_path) != \
- ( base_path + particles_path ) :
- mylog.warning("openPMD: `basePath`+`meshesPath` seems to be malformed "
- "(is `basePath` absolute and ends on a `/` ?)")
- return( np.array([1, 0]) )
- else:
- full_particle_path = base_path + particles_path
- # Find all the particle species
- try:
- list_species = list(f[full_particle_path].keys())
- except KeyError:
- list_species = []
- mylog.debug( "Iteration %s : found %d particle species"
- %( iteration, len(list_species) ) )
-
- # Go through all the particle species
- for species_name in list_species :
- species = f[full_particle_path + species_name.encode('ascii')]
-
- # Check all records for this species
- for species_record_name in species :
- result_array += test_record(species, species_record_name)
-
- # Check the position record of the particles
- result_array += test_key(species, v, "required", "position")
-
- # Check the position offset record of the particles
- result_array += test_key(species, v, "required", "positionOffset")
- if result_array[0] == 0 :
- position_dimensions = len(species["position"].keys())
- positionOffset_dimensions = len(species["positionOffset"].keys())
- if position_dimensions != positionOffset_dimensions :
- mylog.warning("openPMD: `position` (ndim=%s) and `positionOffset` " \
- "(ndim=%s) do not have the same dimensions in " \
- "species `%s`!" \
- %(str(position_dimensions), \
- str(positionOffset_dimensions),
- species.name) )
- result_array += np.array([ 1, 0])
-
- # Check the particlePatches record of the particles
- patch_test = test_key(species, v, "recommended", "particlePatches")
- result_array += patch_test
- if result_array[0] == 0 and patch_test[1] == 0 :
- result_array += test_key(species["particlePatches"], v, "required",
- "numParticles")
- result_array += test_key(species["particlePatches"], v, "required",
- "numParticlesOffset")
- result_array += test_key(species["particlePatches"], v, "required",
- "offset")
- result_array += test_key(species["particlePatches"], v, "required",
- "extent")
- if result_array[0] == 0 :
- offset = species["particlePatches"]["offset"]
- extent = species["particlePatches"]["extent"]
- # Attributes of the components
- for component_name in list(species["position"].keys()) :
- result_array += test_key( offset, v, "required",
- component_name)
- result_array += test_key( extent, v, "required",
- component_name)
- if result_array[0] == 0 :
- dset_offset = offset[component_name]
- result_array += test_component(dset_offset, v)
- dset_extent = extent[component_name]
- result_array += test_component(dset_extent, v)
-
- # Check the records required by the PIC extension
- if pic :
- result_array += test_key(species, v, "required", "momentum")
- result_array += test_key(species, v, "required", "charge")
- result_array += test_key(species, v, "required", "mass")
- result_array += test_key(species, v, "required", "weighting")
- result_array += test_key(species, v, "optional", "boundElectrons")
- result_array += test_key(species, v, "optional", "protonNumber")
- result_array += test_key(species, v, "optional", "neutronNumber")
-
- # Check the attributes associated with the PIC extension
- if pic :
- result_array += test_attr(species, v, "required",
- "particleShape", [np.float32, np.float64])
- result_array += test_attr(species, v, "required",
- "currentDeposition", np.string_)
- result_array += test_attr(species, v, "required",
- "particlePush", np.string_)
- result_array += test_attr(species, v, "required",
- "particleInterpolation", np.string_)
-
- # Check for the attributes associated with the particle smoothing
- result_array += test_attr(species, v, "required",
- "particleSmoothing", np.string_)
- valid, particle_smoothing = get_attr(species, "particleSmoothing")
- if valid == True and particle_smoothing != b"none":
- result_array += test_attr(species, v, "required",
- "particleSmoothingParameters", np.string_)
-
- # Check attributes of each record of the particle
- for record in list(species.keys()):
- # all records (but particlePatches) require units
- if record != "particlePatches":
- result_array += test_attr(species[record], v,
- "required", "unitDimension", np.ndarray, np.float64)
- time_type = f[base_path].attrs["time"].dtype.type
- result_array += test_attr(species[record], v, "required",
- "timeOffset", time_type)
- if pic:
- result_array += test_attr(species[record], v, "required",
- "weightingPower", np.float64)
- result_array += test_attr(species[record], v, "required",
- "macroWeighted", np.uint32)
- # Attributes of the components
- if is_scalar_record(species[record]): # Scalar record
- dset = species[record]
- result_array += test_component(dset, v)
- else: # Vector record
- # Loop over the components
- for component_name in list(species[record].keys()):
- dset = species[os.path.join(record, component_name)]
- result_array += test_component(dset, v)
-
- return result_array
-
-
-def parse_unit_dimension(unit_dimension):
- if len(unit_dimension) is not 7:
- mylog.error("SI must have 7 base dimensions!")
- unit_dimension = np.asarray(unit_dimension, dtype='int')
- dim = []
- si = ["m",
- "kg",
- "s",
- "A",
- "C",
- "mol",
- "cd"]
- for i in range(7):
- if unit_dimension[i] != 0:
- dim.append("{}**{}".format(si[i], unit_dimension[i]))
- return "*".join(dim)
diff -r cb452bf1282587483afb193829c3cad67b1ee557 -r c6cf6d23e8544b19bb570d804a99d93bd3465978 yt/frontends/openPMD/setup.py
--- a/yt/frontends/openPMD/setup.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/usr/bin/env python
-"""
-
-Copyright (c) 2015, Daniel Grassinger (HZDR)
-
-Distributed under the terms of the Modified BSD License.
-
-The full license is in the file COPYING.txt, distributed with this software.
-"""
-
-
-def configuration(parent_package='', top_path=None):
- from numpy.distutils.misc_util import Configuration
- config = Configuration('openPMD', parent_package, top_path)
- config.make_config_py() # installs __config__.py
- # config.make_svn_version_py()
- return config
diff -r cb452bf1282587483afb193829c3cad67b1ee557 -r c6cf6d23e8544b19bb570d804a99d93bd3465978 yt/frontends/open_pmd/__init__.py
--- /dev/null
+++ b/yt/frontends/open_pmd/__init__.py
@@ -0,0 +1,15 @@
+"""
+API for yt.frontends.skeleton
+
+
+
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2013, yt Development Team.
+# Copyright (c) 2015, Daniel Grassinger (HZDR)
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
diff -r cb452bf1282587483afb193829c3cad67b1ee557 -r c6cf6d23e8544b19bb570d804a99d93bd3465978 yt/frontends/open_pmd/api.py
--- /dev/null
+++ b/yt/frontends/open_pmd/api.py
@@ -0,0 +1,26 @@
+"""
+API for yt.frontends._skeleton
+
+
+
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2013, yt Development Team.
+# Copyright (c) 2015, Daniel Grassinger (HZDR)
+# Copyright (c) 2016, Fabian Koller (HZDR)
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+from .data_structures import \
+ OpenPMDDataset, \
+ OpenPMDGrid, \
+ OpenPMDHierarchy
+
+from .fields import \
+ OpenPMDFieldInfo
+
+from .io import \
+ IOHandlerOpenPMD
This diff is so big that we needed to truncate the remainder.
https://bitbucket.org/yt_analysis/yt/commits/0a69a5e3a7a3/
Changeset: 0a69a5e3a7a3
Branch: yt
User: Fabian Koller
Date: 2016-08-05 12:53:00+00:00
Summary: Docstrings
Affected #: 1 file
diff -r c6cf6d23e8544b19bb570d804a99d93bd3465978 -r 0a69a5e3a7a30daadeb5e2d0da6c7a9f954a046d yt/frontends/open_pmd/data_structures.py
--- a/yt/frontends/open_pmd/data_structures.py
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -55,7 +55,7 @@
class OpenPMDGrid(AMRGridPatch):
- """Represents a disjoined chunk of data on-disk.
+ """Represents disjoint chunk of data on-disk and their relation.
The chunks are sliced off the original field along the x-axis.
This defines the index and offset for every mesh and particle type.
@@ -63,9 +63,10 @@
"""
_id_offset = 0
__slots__ = ["_level_id"]
- particle_index = [] # Every particle species might have distinct hdf5-indices
- particle_offset = [] # and offsets. Contain tuples (ptype, index) and (ptype, offset)
- # TODO (maybe) consider these for every ftype
+ # Every particle species might have different hdf5-indices and offsets
+ # These contain tuples (ptype, index) and (ptype, offset)
+ particle_index = []
+ particle_offset = []
mesh_index = 0
mesh_offset = 0
@@ -87,9 +88,9 @@
class OpenPMDHierarchy(GridIndex):
- """
- Defines which fields and particles are created and read from the hard disk
- Furthermore it defines the characteristics of the grids
+ """Defines which fields and particles are created and read from the hard disk.
+
+ Furthermore it defines the characteristics of the grids.
"""
grid = OpenPMDGrid
@@ -190,21 +191,18 @@
self.num_grids = int(ceil(np.max(self.numparts.values()) * ppg**-1))
def _parse_index(self):
+ """Fills each grid with appropriate properties (extent, dimensions, ...)
+
+ This calculates the properties of every OpenPMDGrid based on the total number of grids in the simulation.
+ The domain is divided into ``self.num_grids`` equally sized chunks along the x-axis.
+ ``grid_levels`` is always equal to 0 since we only have one level of refinement in openPMD.
+
+ Notes
+ -----
+ ``self.grid_dimensions`` is rounded to the nearest integer.
+ Furthermore the last grid might be smaller and have fewer particles than the others.
+ In general, NOT all particles in a grid will be inside the grid edges.
"""
- Parses dimensions from self._handle into self.
-
- From yt doc:
- this must fill in
- grid_left_edge,
- grid_right_edge,
- grid_particle_count,
- grid_dimensions and
- grid_levels
- with the appropriate information.
- Each of these variables is an array with an entry for each of the self.num_grids grids.
- Additionally, grids must be an array of AMRGridPatch objects that already know their IDs.
- """
- # There is only one refinement level in openPMD
self.grid_levels.flat[:] = 0
self.grids = np.empty(self.num_grids, dtype='object')
@@ -236,7 +234,7 @@
# The last grid need not be the same size as the previous ones
num = nrp[spec]
else:
- num = int(floor(self.numparts[spec] * self.num_grids ** -1))
+ num = int(floor(self.numparts[spec] * self.num_grids**-1))
particlecount += [(spec, num)]
nrp[spec] -= num
self.grid_particle_count[i] += num
@@ -252,12 +250,10 @@
remaining -= self.grid_dimensions[i][0]
def _populate_grid_objects(self):
- """
- This function initializes the grids
+ """This initializes all grids.
- From yt doc:
- this initializes the grids by calling _prepare_grid() and _setup_dx() on all of them.
- Additionally, it should set up Children and Parent lists on each grid object.
+ Additionally, it should set up Children and Parent lists on each grid object.
+ openPMD is not adaptive and thus there are no Children and Parents for any grid.
"""
for i in range(self.num_grids):
self.grids[i]._prepare_grid()
@@ -266,14 +262,7 @@
class OpenPMDDataset(Dataset):
- """
- A dataset object contains all the information of the simulation and
- is intialized with yt.load()
-
- TODO Ideally, a data set object should only contain a single data set.
- afaik, yt.load() can load multiple data sets and also supports
- multiple iteration-loading if done that way, e.g., from a prefix
- of files.
+ """Contains all the required information of a single iteration of the simulation.
"""
_index_class = OpenPMDHierarchy
_field_info_class = OpenPMDFieldInfo
@@ -284,65 +273,66 @@
units_override=None,
unit_system="mks"):
self._handle = HDF5FileHandler(filename)
- self._filepath = os.path.dirname(filename)
- if self._nonstandard:
- self._set_nonstandard_paths(self._handle)
- else:
- self._set_paths(self._handle, self._filepath)
+ self._set_paths(self._handle, os.path.dirname(filename), self._nonstandard)
Dataset.__init__(self, filename, dataset_type,
units_override=units_override,
unit_system=unit_system)
self.storage_filename = storage_filename
self.fluid_types += ('openPMD',)
- parts = tuple(str(c) for c in self._handle[self.base_path + self.particles_path].keys())
- if len(parts) > 1:
- # Only use infile particle names if there is more than one species
- self.particle_types = parts
- mylog.debug("open_pmd - self.particle_types: {}".format(self.particle_types))
+ particles = tuple(str(c) for c in self._handle[self.base_path + self.particles_path].keys())
+ if len(particles) > 1:
+ # Only use on-disk particle names if there is more than one species
+ self.particle_types = particles
+ mylog.debug("open_pmd: self.particle_types: {}".format(self.particle_types))
self.particle_types_raw = self.particle_types
self.particle_types = tuple(self.particle_types)
- def _set_nonstandard_paths(self, handle):
- iteration = handle["/data"].keys()[0]
- self.base_path = "/data/{}/".format(iteration)
- self.meshes_path = "fields/"
- self.particles_path = "particles/"
+ def _set_paths(self, handle, path, nonstandard):
+ """Parses relevant hdf5-paths out of ``handle``.
- def _set_paths(self, handle, filepath):
- list_iterations = []
- if u"groupBased" in handle.attrs["iterationEncoding"]:
- for i in list(handle["/data"].keys()):
- list_iterations.append(i)
- mylog.info("open_pmd: found {} iterations in file".format(len(list_iterations)))
- elif u"fileBased" in handle.attrs["iterationEncoding"]:
- regex = u"^" + handle.attrs["iterationFormat"].replace('%T', '[0-9]+') + u"$"
- if filepath is '':
- mylog.warning("open_pmd: For file based iterations, please use absolute file paths!")
- pass
- for filename in os.listdir(filepath):
- if re.match(regex, filename):
- list_iterations.append(filename)
- mylog.info("open_pmd: found {} iterations in directory".format(len(list_iterations)))
+ Parameters
+ ----------
+ handle : h5py._hl.files.File
+ path : str
+ nonstandard : bool
+ """
+ self.base_path = "/data/{}/".format(handle["/data"].keys()[0])
+ if nonstandard:
+ self.meshes_path = "fields/"
+ self.particles_path = "particles/"
else:
- mylog.warning(
- "openOMD: File does not have valid iteration encoding: {}".format(handle.attrs["iterationEncoding"]))
+ list_iterations = []
+ if "groupBased" in handle.attrs["iterationEncoding"]:
+ for i in list(handle["/data"].keys()):
+ list_iterations.append(i)
+ mylog.info("open_pmd: found {} iterations in file".format(len(list_iterations)))
+ elif "fileBased" in handle.attrs["iterationEncoding"]:
+ regex = "^" + handle.attrs["iterationFormat"].replace('%T', '[0-9]+') + "$"
+ if path is '':
+ mylog.warning("open_pmd: For file based iterations, please use absolute file paths!")
+ pass
+ for filename in os.listdir(path):
+ if re.match(regex, filename):
+ list_iterations.append(filename)
+ mylog.info("open_pmd: found {} iterations in directory".format(len(list_iterations)))
+ else:
+ mylog.warning(
+ "open_pmd: No valid iteration encoding: {}".format(handle.attrs["iterationEncoding"]))
- if len(list_iterations) == 0:
- mylog.warning("openOMD: No iterations found!")
- if u"groupBased" in handle.attrs["iterationEncoding"] and len(list_iterations) > 1:
- mylog.warning("open_pmd: only choose to load one iteration ({})".format(handle["/data"].keys()[0]))
+ if len(list_iterations) == 0:
+ mylog.warning("open_pmd: No iterations found!")
+ if "groupBased" in handle.attrs["iterationEncoding"] and len(list_iterations) > 1:
+ mylog.warning("open_pmd: only choose to load one iteration ({})".format(handle["/data"].keys()[0]))
- self.base_path = "{}/{}/".format("/data", handle["/data"].keys()[0])
- self.meshes_path = self._handle["/"].attrs["meshesPath"]
- self.particles_path = self._handle["/"].attrs["particlesPath"]
+ self.meshes_path = self._handle["/"].attrs["meshesPath"]
+ self.particles_path = self._handle["/"].attrs["particlesPath"]
def _set_code_unit_attributes(self):
+ """Handle conversion between different physical units and the code units.
+
+ These are hardcoded as 1.0. Every dataset in openPMD can have different code <-> physical scaling.
+ The individual factor is obtained by multiplying with "unitSI" reading getting data from disk.
"""
- From yt doc:
- handle conversion between the different physical units and the code units
- """
- # We hardcode these to 1.0 since every dataset can have different code <-> physical scaling
- # We get the actual unit by multiplying with "unitSI" when getting our data from disk
self.length_unit = self.quan(1.0, "m")
self.mass_unit = self.quan(1.0, "kg")
self.time_unit = self.quan(1.0, "s")
@@ -350,9 +340,11 @@
self.magnetic_unit = self.quan(1.0, "T")
def _parse_parameter_file(self):
- """
- From yt doc:
- read in metadata describing the overall data on disk
+ """Read in metadata describing the overall data on-disk.
+
+ Notes
+ -----
+ All meshes are assumed to have the same dimensions and size.
"""
f = self._handle
bp = self.base_path
@@ -360,8 +352,10 @@
self.unique_identifier = 0
self.parameters = 0
+ self.periodicity = np.zeros(3, dtype=np.bool)
+ self.refine_by = 1
+ self.cosmological_simulation = 0
- # We assume all fields to have the same shape
try:
mesh = f[bp + mp].keys()[0]
axis = f[bp + mp + "/" + mesh].keys()[0]
@@ -390,8 +384,8 @@
self.domain_right_edge = self.domain_dimensions[:spacing.size] * unitSI * spacing
self.domain_right_edge = np.append(self.domain_right_edge, np.ones(3 - self.domain_right_edge.size))
except Exception as e:
- mylog.warning("The domain extent could not be calculated! ({}) Setting the field extent to 1m**3! "
- "This WILL break particle-overplotting!".format(e))
+ mylog.warning(
+ "The domain extent could not be calculated! ({}) Setting the field extent to 1m**3!".format(e))
self.domain_right_edge = np.ones(3, dtype=np.float64)
if self._nonstandard:
@@ -399,15 +393,9 @@
else:
self.current_time = f[bp].attrs["time"]
- self.periodicity = np.zeros(3, dtype=np.bool)
- self.refine_by = 1
- self.cosmological_simulation = 0
-
@classmethod
def _is_valid(self, *args, **kwargs):
- """
- Checks whether the supplied file adheres to the required openPMD standards
- and thus can be read by this frontend
+ """Checks whether the supplied file can be read by this frontend.
"""
try:
f = validator.open_file(args[0])
@@ -426,10 +414,13 @@
# this might still be a compatible file not fully respecting openPMD standards
if result_array[0] != 0:
try:
- if "/data" in f and f["/data"].keys()[0].isdigit():
+ iteration = f["/data"].keys()[0]
+ if iteration.isdigit() \
+ and "fields" in f["/data/" + iteration].keys()\
+ and "particles" in f["/data/" + iteration].keys():
self._nonstandard = True
- mylog.info("Reading a file with the open_pmd frontend that does not respect standards? "
- "Just understand that you're on your own for this!")
+ mylog.info(
+ "open_pmd - Reading a file not compliant with the standard. No support will be guaranteed!")
return True
except:
return False
https://bitbucket.org/yt_analysis/yt/commits/a0271fa934ca/
Changeset: a0271fa934ca
Branch: yt
User: Fabian Koller
Date: 2016-08-05 15:49:04+00:00
Summary: Docstrings, formatting
Affected #: 8 files
diff -r 0a69a5e3a7a30daadeb5e2d0da6c7a9f954a046d -r a0271fa934ca3ada4865fdc495dc2b51089f7854 yt/frontends/open_pmd/__init__.py
--- a/yt/frontends/open_pmd/__init__.py
+++ b/yt/frontends/open_pmd/__init__.py
@@ -1,15 +1,15 @@
"""
-API for yt.frontends.skeleton
+API for yt.frontends.open_pmd
"""
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
# Copyright (c) 2013, yt Development Team.
# Copyright (c) 2015, Daniel Grassinger (HZDR)
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
diff -r 0a69a5e3a7a30daadeb5e2d0da6c7a9f954a046d -r a0271fa934ca3ada4865fdc495dc2b51089f7854 yt/frontends/open_pmd/api.py
--- a/yt/frontends/open_pmd/api.py
+++ b/yt/frontends/open_pmd/api.py
@@ -5,14 +5,14 @@
"""
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
# Copyright (c) 2013, yt Development Team.
# Copyright (c) 2015, Daniel Grassinger (HZDR)
# Copyright (c) 2016, Fabian Koller (HZDR)
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
from .data_structures import \
OpenPMDDataset, \
diff -r 0a69a5e3a7a30daadeb5e2d0da6c7a9f954a046d -r a0271fa934ca3ada4865fdc495dc2b51089f7854 yt/frontends/open_pmd/data_structures.py
--- a/yt/frontends/open_pmd/data_structures.py
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -4,7 +4,7 @@
"""
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
# Copyright (c) 2013, yt Development Team.
# Copyright (c) 2015, Daniel Grassinger (HZDR)
# Copyright (c) 2016, Fabian Koller (HZDR)
@@ -12,47 +12,20 @@
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
-from yt.data_objects.grid_patch import \
- AMRGridPatch
-from yt.geometry.grid_geometry_handler import \
- GridIndex
-from yt.data_objects.static_output import \
- Dataset
-from .fields import OpenPMDFieldInfo
-
-from yt.utilities.file_handler import \
- HDF5FileHandler
-
-import yt.frontends.open_pmd.misc as validator
+from yt.data_objects.grid_patch import AMRGridPatch
+from yt.geometry.grid_geometry_handler import GridIndex
+from yt.data_objects.static_output import Dataset
+from yt.utilities.file_handler import HDF5FileHandler
+from yt.utilities.logger import ytLogger as mylog
+from yt.frontends.open_pmd.fields import OpenPMDFieldInfo
+from yt.frontends.open_pmd.misc import is_const_component, check_root_attr, check_iterations
import numpy as np
import os
import re
from math import ceil, floor
-from yt.utilities.logger import ytLogger as mylog
-
-
-def is_const_component(record_component):
- """Determines whether a group or dataset in the HDF5 file is constant.
-
- Parameters
- ----------
- record_component : HDF5 group or dataset
-
- Returns
- -------
- bool
- True if constant, False otherwise
-
- References
- ----------
- .. https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md,
- section 'Constant Record Components'
- """
- return "value" in record_component.attrs.keys()
-
class OpenPMDGrid(AMRGridPatch):
"""Represents disjoint chunk of data on-disk and their relation.
@@ -70,8 +43,8 @@
mesh_index = 0
mesh_offset = 0
- def __init__(self, id, index, level=-1, pi=None, po=None, mi=0, mo=0):
- AMRGridPatch.__init__(self, id, filename=index.index_filename,
+ def __init__(self, gid, index, level=-1, pi=None, po=None, mi=0, mo=0):
+ AMRGridPatch.__init__(self, gid, filename=index.index_filename,
index=index)
if pi:
self.particle_index = pi
@@ -114,7 +87,7 @@
bp = self.dataset.base_path
mp = self.dataset.meshes_path
pp = self.dataset.particles_path
-
+
mesh_fields = []
try:
for field in f[bp + mp].keys():
@@ -123,7 +96,7 @@
mesh_fields.append(field + "_" + ax)
except AttributeError:
# This is for dataSets, they do not have keys
- mesh_fields.append(field.replace("_","-"))
+ mesh_fields.append(field.replace("_", "-"))
except KeyError:
# There are no mesh fields
pass
@@ -176,7 +149,7 @@
mp = self.dataset.meshes_path
pp = self.dataset.particles_path
- gridsize = 10 * 10**6 # Byte
+ gridsize = 10 * 10 ** 6 # Byte
self.numparts = {}
for species in f[bp + pp].keys():
@@ -186,9 +159,9 @@
else:
self.numparts[species] = f[bp + pp + species + "/position/" + axis].len()
# Limit particles per grid by resulting memory footprint
- ppg = int(gridsize/(self.dataset.dimensionality*4)) # 4 Byte per value per dimension (f32)
+ ppg = int(gridsize / (self.dataset.dimensionality * 4)) # 4 Byte per value per dimension (f32)
# Use an upper bound of equally sized grids, last one might be smaller
- self.num_grids = int(ceil(np.max(self.numparts.values()) * ppg**-1))
+ self.num_grids = int(ceil(np.max(self.numparts.values()) * ppg ** -1))
def _parse_index(self):
"""Fills each grid with appropriate properties (extent, dimensions, ...)
@@ -216,15 +189,14 @@
for i in range(self.num_grids):
self.grid_dimensions[i] = self.dataset.domain_dimensions
prev = remaining
- remaining -= self.grid_dimensions[i][0] * self.num_grids**-1
+ remaining -= self.grid_dimensions[i][0] * self.num_grids ** -1
self.grid_dimensions[i][0] = int(round(prev, 0) - round(remaining, 0))
self.grid_left_edge[i] = self.dataset.domain_left_edge.copy()
self.grid_left_edge[i][0] = meshedge
self.grid_right_edge[i] = self.dataset.domain_right_edge.copy()
- self.grid_right_edge[i][0] = self.grid_left_edge[i][0] \
- + self.grid_dimensions[i][0]\
- * self.dataset.domain_dimensions[0]**-1\
- * self.dataset.domain_right_edge[0]
+ self.grid_right_edge[i][0] = self.grid_left_edge[i][0] + self.grid_dimensions[i][0] * \
+ self.dataset.domain_dimensions[0] ** -1 * \
+ self.dataset.domain_right_edge[0]
meshedge = self.grid_right_edge[i][0]
particlecount = []
particleindex = []
@@ -234,7 +206,7 @@
# The last grid need not be the same size as the previous ones
num = nrp[spec]
else:
- num = int(floor(self.numparts[spec] * self.num_grids**-1))
+ num = int(floor(self.numparts[spec] * self.num_grids ** -1))
particlecount += [(spec, num)]
nrp[spec] -= num
self.grid_particle_count[i] += num
@@ -292,7 +264,7 @@
Parameters
----------
- handle : h5py._hl.files.File
+ handle : h5py.File
path : str
nonstandard : bool
"""
@@ -377,11 +349,11 @@
height = f[bp].attrs['cell_height']
depth = f[bp].attrs['cell_depth']
spacing = np.asarray([width, height, depth])
- unitSI = f[bp].attrs['unit_length']
+ unit_si = f[bp].attrs['unit_length']
else:
spacing = np.asarray(f[bp + mp + "/" + mesh].attrs["gridSpacing"])
- unitSI = f[bp + mp + "/" + mesh].attrs["gridUnitSI"]
- self.domain_right_edge = self.domain_dimensions[:spacing.size] * unitSI * spacing
+ unit_si = f[bp + mp + "/" + mesh].attrs["gridUnitSI"]
+ self.domain_right_edge = self.domain_dimensions[:spacing.size] * unit_si * spacing
self.domain_right_edge = np.append(self.domain_right_edge, np.ones(3 - self.domain_right_edge.size))
except Exception as e:
mylog.warning(
@@ -398,25 +370,25 @@
"""Checks whether the supplied file can be read by this frontend.
"""
try:
- f = validator.open_file(args[0])
+ f = HDF5FileHandler(args[0])
except:
return False
verbose = False
extension_pic = False
# root attributes at "/"
result_array = np.array([0, 0])
- result_array += validator.check_root_attr(f, verbose, extension_pic)
+ result_array += check_root_attr(f, verbose, extension_pic)
# Go through all the iterations, checking both the particles
# and the meshes
- result_array += validator.check_iterations(f, verbose, extension_pic)
+ result_array += check_iterations(f, verbose, extension_pic)
# this might still be a compatible file not fully respecting openPMD standards
if result_array[0] != 0:
try:
iteration = f["/data"].keys()[0]
if iteration.isdigit() \
- and "fields" in f["/data/" + iteration].keys()\
+ and "fields" in f["/data/" + iteration].keys() \
and "particles" in f["/data/" + iteration].keys():
self._nonstandard = True
mylog.info(
diff -r 0a69a5e3a7a30daadeb5e2d0da6c7a9f954a046d -r a0271fa934ca3ada4865fdc495dc2b51089f7854 yt/frontends/open_pmd/definitions.py
--- a/yt/frontends/open_pmd/definitions.py
+++ b/yt/frontends/open_pmd/definitions.py
@@ -1,1 +0,0 @@
-# This file is often empty. It can hold definitions related to a frontend.
diff -r 0a69a5e3a7a30daadeb5e2d0da6c7a9f954a046d -r a0271fa934ca3ada4865fdc495dc2b51089f7854 yt/frontends/open_pmd/fields.py
--- a/yt/frontends/open_pmd/fields.py
+++ b/yt/frontends/open_pmd/fields.py
@@ -5,7 +5,7 @@
"""
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
# Copyright (c) 2013, yt Development Team.
# Copyright (c) 2015, Daniel Grassinger (HZDR)
# Copyright (c) 2016, Fabian Koller (HZDR)
@@ -13,26 +13,25 @@
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+
+from yt.funcs import mylog
+from yt.utilities.physical_constants import speed_of_light
+from yt.fields.field_info_container import FieldInfoContainer
+from yt.units.yt_array import YTQuantity
+from yt.fields.magnetic_field import setup_magnetic_field_aliases
+from yt.frontends.open_pmd.misc import parse_unit_dimension
import numpy as np
-from yt.funcs import mylog
-from yt.utilities.physical_constants import speed_of_light
-from yt.fields.field_info_container import \
- FieldInfoContainer
-from yt.fields.magnetic_field import \
- setup_magnetic_field_aliases
-from .misc import parse_unit_dimension
-import yt
def setup_kinetic_energy(self, ptype):
def _kin_en(field, data):
- p2 = (data[ptype, "particle_momentum_x"]**2 +
- data[ptype, "particle_momentum_y"]**2 +
- data[ptype, "particle_momentum_z"]**2)
+ p2 = (data[ptype, "particle_momentum_x"] ** 2 +
+ data[ptype, "particle_momentum_y"] ** 2 +
+ data[ptype, "particle_momentum_z"] ** 2)
mass = data[ptype, "particle_mass"] * data[ptype, "particle_weighting"]
- return speed_of_light * np.sqrt(p2 + mass**2 * speed_of_light**2) - mass * speed_of_light**2
+ return speed_of_light * np.sqrt(p2 + mass ** 2 * speed_of_light ** 2) - mass * speed_of_light ** 2
self.add_field((ptype, "particle_kinetic_energy"),
function=_kin_en,
@@ -48,8 +47,8 @@
mass = data[ptype, "particle_mass"]
weighting = data[ptype, "particle_weighting"]
return momentum / (
- (mass * weighting)**2 +
- (momentum**2) / (c**2)
+ (mass * weighting) ** 2 +
+ (momentum ** 2) / (c ** 2)
) ** 0.5
return velocity
@@ -124,13 +123,13 @@
parsed = ""
else:
parsed = parse_unit_dimension(np.asarray(field.attrs["unitDimension"], dtype="int"))
- unit = str(yt.YTQuantity(1, parsed).units)
+ unit = str(YTQuantity(1, parsed).units)
aliases = []
# Save a list of magnetic fields for aliasing later on
# We can not reasonably infer field type by name in openPMD
if "T" in unit or "kg/(A*s**2)" in unit:
self._mag_fields.append(ytname)
- self.known_other_fields += ((ytname, (unit, aliases, None)), )
+ self.known_other_fields += ((ytname, (unit, aliases, None)),)
else:
if ds._nonstandard:
axes = "xyz" # naively assume all fields in non-standard files are 3D
@@ -142,13 +141,13 @@
parsed = ""
else:
parsed = parse_unit_dimension(np.asarray(field.attrs["unitDimension"], dtype="int"))
- unit = str(yt.YTQuantity(1, parsed).units)
+ unit = str(YTQuantity(1, parsed).units)
aliases = []
# Save a list of magnetic fields for aliasing later on
# We can not reasonably infer field type by name in openPMD
if "T" in unit or "kg/(A*s**2)" in unit:
self._mag_fields.append(ytname)
- self.known_other_fields += ((ytname, (unit, aliases, None)), )
+ self.known_other_fields += ((ytname, (unit, aliases, None)),)
for i in self.known_other_fields:
mylog.debug("oPMD - fields - known_other_fields - {}".format(i))
@@ -168,7 +167,7 @@
else:
parsed = parse_unit_dimension(
np.asarray(particles.get(species).get(attrib).attrs["unitDimension"], dtype="int"))
- unit = str(yt.YTQuantity(1, parsed).units)
+ unit = str(YTQuantity(1, parsed).units)
name = ["particle", attrib]
ytattrib = attrib
# Symbolically rename position to preserve yt's interpretation of the pfield
@@ -182,7 +181,7 @@
ytname = str("_".join(name))
if ds._nonstandard and "globalCellIdx" in ytname:
aliases.append(ytname.replace("globalCellIdx", "positionOffset"))
- particle_fields += ((ytname, (unit, aliases, None)), )
+ particle_fields += ((ytname, (unit, aliases, None)),)
except:
mylog.info("{}_{} does not seem to have unitDimension".format(species, attrib))
self.known_particle_fields = particle_fields
diff -r 0a69a5e3a7a30daadeb5e2d0da6c7a9f954a046d -r a0271fa934ca3ada4865fdc495dc2b51089f7854 yt/frontends/open_pmd/io.py
--- a/yt/frontends/open_pmd/io.py
+++ b/yt/frontends/open_pmd/io.py
@@ -5,7 +5,7 @@
"""
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
# Copyright (c) 2013, yt Development Team.
# Copyright (c) 2015, Daniel Grassinger (HZDR)
# Copyright (c) 2016, Fabian Koller (HZDR)
@@ -13,11 +13,12 @@
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
-from yt.utilities.io_handler import \
- BaseIOHandler
+from yt.utilities.io_handler import BaseIOHandler
from yt.utilities.logger import ytLogger as mylog
+from yt.frontends.open_pmd.misc import is_const_component
+
import numpy as np
@@ -35,20 +36,33 @@
self._cached_ptype = ""
def _fill_cache(self, ptype, index=0, offset=None):
+ """Fills the particle position cache for the ``ptype``.
+
+ Parameters
+ ----------
+ ptype : str
+ index : int, optional
+ offset : int, optional
+ """
if str((ptype, index, offset)) not in self._cached_ptype:
- # Get a particle species (e.g. /data/3500/particles/e/)
if ptype in "io":
spec = self._handle[self.base_path + self.particles_path].keys()[0]
else:
spec = ptype
pds = self._handle[self.base_path + self.particles_path + "/" + spec]
- # Get 1D-Arrays for individual particle positions along axes
axes = [str(ax) for ax in pds["position"].keys()]
if self.ds._nonstandard:
position_offset = "globalCellIdx/"
else:
position_offset = "positionOffset/"
- self.cache = np.empty((3, offset), dtype="float64")
+ if offset is None:
+ if is_const_component(pds["position/" + axes[0]]):
+ size = pds["position/" + axes[0]].attrs["shape"]
+ else:
+ size = pds["position/" + axes[0]].len()
+ else:
+ size = offset
+ self.cache = np.empty((3, size), dtype="float64")
for i in range(3):
ax = "xyz"[i]
if ax in axes:
@@ -58,30 +72,28 @@
else:
# Pad accordingly with zeros to make 1D/2D datasets compatible
# These have to be the same shape as the existing axes since that equals the number of particles
- self.cache[i] = np.zeros(offset)
+ self.cache[i] = np.zeros(size)
self._cached_ptype = str((ptype, index, offset))
def _read_particle_coords(self, chunks, ptf):
+ """Reads coordinates for given particle-types in given chunks from file.
+
+ Parameters
+ ----------
+ chunks
+ A list of chunks
+ A chunk is a list of grids
+ ptf : dict
+ keys are ptypes
+ values are lists of particle fields
+
+ Yields
+ ------
+ tuple : (str, ((N,) ndarray, (N,) ndarray, (N,) ndarray))
+ Tuple of ptype and tuple of coordinates (x, y, z) corresponding to coordinates of particles of that ptype
+ All coordinate arrays have the same length
"""
- Reads coordinates for all specified particle-types from file.
- Parameters
- ----------
- chunks:
- A list of chunks
- A chunk is a list of grids
-
- ptf:
- A dictionary
- - keys are ptypes
- - values are lists of particle fields
-
- Yields
- ------
- A series of tuples of (ptype_name, (x_coords, y_coords, z_coords)).
- ptype_name is a type pf particle,
- x_coords, y_coords and z_coords are arrays of positions/coordinates of all particles of that type
- """
chunks = list(chunks)
f = self._handle
ds = f[self.base_path]
@@ -94,45 +106,39 @@
spec = ds[self.particles_path].keys()[0]
else:
spec = ptype
- for gridptype, idx in g.particle_index:
- if str(gridptype) == str(spec):
- index = idx
- for gridptype, ofs in g.particle_offset:
- if str(gridptype) == str(spec):
- offset = ofs
- mylog.debug("open_pmd - _read_particle_coords: (grid {}) {}, {} [{}:{}]".format(g, ptype, field_list, index, offset))
+ index = dict(g.particle_index).get(spec)
+ offset = dict(g.particle_offset).get(spec)
+ mylog.debug(
+ "open_pmd - _read_particle_coords: (grid {}) {}, {} [{}:{}]".format(g, ptype, field_list, index,
+ offset))
if str((ptype, index, offset)) not in self._cached_ptype:
self._fill_cache(ptype, index, offset)
yield (ptype, (self.cache[0], self.cache[1], self.cache[2]))
def _read_particle_fields(self, chunks, ptf, selector):
- """
- Reads particle fields masked by particle type and field out of a list of chunks from file.
+ """Reads given fields for given particle types masked by a given selection.
- Parameters
- ----------
- chunks:
- A list of chunks
- A chunk is a list of grids
+ Parameters
+ ----------
+ chunks
+ A list of chunks
+ A chunk is a list of grids
+ ptf : dict
+ keys are ptype
+ values are lists of particle fields
+ selector
+ A region (inside your domain) specifying which parts of the field you want to read
+ See [1] and [2]
- ptf:
- A dictionary
- - keys are ptype
- - values are lists of (particle?) fields
+ References
+ ----------
+ .. [1] yt-project.org/docs/dev/quickstart/data_inspection.html?highlight=selector#Examining-Data-in-Regions
+ .. [2] yt-project.org/doc/developing/creating_datatypes.html
-
- selector:
- yt-project.org/docs/dev/quickstart/data_inspection.html?highlight=selector#Examining-Data-in-Regions
- yt-project.org/doc/developing/creating_datatypes.html
- A region (in and/or outside your domain) specifying the field you want to read
- Selector objects have a .select_points(x,y,z) that returns a mask, so you need to do your masking here.
-
- Yields
- -------
- Tuples of structure ((ptype, field), data)
- where data is a numpy array of data masked to
- - a particle type p
- - in a specified field
+ Yields
+ ------
+ tuple : ((str, str), (N,) ndarray)
+ Tuple of tuple (ptype, fieldname) and masked field-data
"""
chunks = list(chunks)
f = self._handle
@@ -149,7 +155,9 @@
spec = ptype
index = dict(g.particle_index).get(spec)
offset = dict(g.particle_offset).get(spec)
- mylog.debug("open_pmd - _read_particle_fields: (grid {}) {}, {} [{}:{}]".format(g, ptype, field_list, index, offset))
+ mylog.debug(
+ "open_pmd - _read_particle_fields: (grid {}) {}, {} [{}:{}]".format(g, ptype, field_list, index,
+ offset))
if str((ptype, index, offset)) not in self._cached_ptype:
self._fill_cache(ptype, index, offset)
mask = selector.select_points(self.cache[0], self.cache[1], self.cache[2], 0.0)
@@ -162,38 +170,26 @@
yield ((ptype, field), data[mask])
def _read_fluid_selection(self, chunks, selector, fields, size):
- """
- Reads selected fields of given size from file.
+ """Reads given fields for given meshes masked by a given selection.
- From yt doc:
- Receives a collection of data "chunks", a selector describing which "chunks" you are concerned with,
- a list of fields, and the size of the data to read.
- It should create and return a dictionary whose keys are the fields,
- and whose values are numpy arrays containing the data.
- The data should actually be read via the _read_chunk_data() method.
+ Parameters
+ ----------
+ chunks
+ A list of chunks
+ A chunk is a list of grids
+ selector
+ A region (inside your domain) specifying which parts of the field you want to read
+ See [1] and [2]
+ fields : array_like
+ tuples (fname, ftype) representing a field
+ size : int
+ Size of the data to read
- Parameters
- ----------
- chunks:
- A list of chunks
- A chunk is a list of grids
-
- selector:
- yt-project.org/docs/dev/quickstart/data_inspection.html?highlight=selector#Examining-Data-in-Regions
- yt-project.org/doc/developing/creating_datatypes.html
- A region (in and/or outside your domain) specifying the field you want to read
-
- fields:
- A list of (fname, ftype) tuples representing a field
-
- size:
- Size of the data arrays you want to read
-
- Returns
- -------
- A dictionary:
- - keys are (ftype, fname) tuples representing a field
- - values are numpy arrays with data form that field
+ Returns
+ -------
+ dict
+ keys are tuples (ftype, fname) representing a field
+ values are flat (``size``,) ndarrays with data from that field
"""
mylog.info("_read_fluid_selection {} {} {} {}".format(chunks, selector, fields, size))
f = self._handle
@@ -219,7 +215,7 @@
for chunk in chunks:
for g in chunk.objs:
ds = f[bp + mp]
- nfield = fname.replace("_", "/").replace("-","_")
+ nfield = fname.replace("_", "/").replace("-", "_")
index = g.mesh_index
offset = g.mesh_offset
data = np.array(self.get_component(ds, nfield, index, offset, self.ds._nonstandard))
@@ -234,21 +230,29 @@
return rv
def get_component(self, group, component_name, index=0, offset=None, nonstandard=False):
- """Grab a component from a group
+ """Grab a dataset component from a group as a whole or sliced.
Parameters
----------
- group: hdf5 group to get whole/masked component from
- component_name: string of a component (relative path) inside the group
- index: (optional) start index of data to return
- offset: (optional) size of the data to return
- nonstandard: (optional) bool to signal whether to assume (non)standard PMD
+ group : h5py.Group
+ component_name : str
+ relative path of the component in the group
+ index : int, optional
+ first entry along the first axis to read
+ offset : int, optional
+ number of entries to read
+ if not supplied, every entry after index is returned
+ nonstandard : bool, optional
+
+ Notes
+ -----
+ This scales every entry of the component with the respective "unitSI".
Returns
-------
- n-dimensional numpy array filled with values of component
+ (N,) ndarray
+
"""
-
record_component = group[component_name]
if nonstandard:
try:
@@ -276,4 +280,4 @@
# component is a dataset, return it (possibly masked)
mylog.debug(
"open_pmd - misc - get_component: {}/{}[{}:{}]".format(group.name, component_name, index, offset))
- return np.multiply(record_component[index:offset], unitSI)
\ No newline at end of file
+ return np.multiply(record_component[index:offset], unitSI)
diff -r 0a69a5e3a7a30daadeb5e2d0da6c7a9f954a046d -r a0271fa934ca3ada4865fdc495dc2b51089f7854 yt/frontends/open_pmd/misc.py
--- a/yt/frontends/open_pmd/misc.py
+++ b/yt/frontends/open_pmd/misc.py
@@ -1,4 +1,4 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
# Copyright (c) 2015, Axel Huebl, Remi Lehe
# Copyright (c) 2016, Fabian Koller (HZDR)
#
@@ -14,13 +14,13 @@
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
import h5py as h5
import numpy as np
import re
import string
-import collections # for isinstance
+import collections # for isinstance
import sys, getopt, os.path
from yt.utilities.logger import ytLogger as mylog
@@ -32,7 +32,7 @@
def open_file(file_name):
try:
f = h5.File(file_name, "r")
- return(f)
+ return (f)
except:
raise
@@ -43,9 +43,9 @@
Return the corresponding attribute if it is present
"""
if name in list(f.attrs.keys()):
- return(True, f.attrs[name])
+ return (True, f.attrs[name])
else:
- return(False, None)
+ return (False, None)
def test_record(g, r):
@@ -66,22 +66,22 @@
- The first element is 1 if an error occured, and 0 otherwise
- The second element is 0 if a warning arised, and 0 otherwise
"""
- regEx = re.compile("^\w+$") # Python3 only: re.ASCII
+ regEx = re.compile("^\w+$") # Python3 only: re.ASCII
if regEx.match(r):
# test component names
- result_array = np.array([0,0])
- if not is_scalar_record(g[r]) :
+ result_array = np.array([0, 0])
+ if not is_scalar_record(g[r]):
for component_name in g[r]:
if not regEx.match(component_name):
mylog.warning("open_pmd: Component %s of record %s is NOT" \
- " named properly (a-Z0-9_)!" %(component_name, g[r].name) )
- result_array += np.array([1,0])
+ " named properly (a-Z0-9_)!" % (component_name, g[r].name))
+ result_array += np.array([1, 0])
else:
mylog.warning("open_pmd: Record %s is NOT named properly (a-Z0-9_)!" \
- %(r.name) )
- result_array = np.array([1,0])
+ % (r.name))
+ result_array = np.array([1, 0])
- return(result_array)
+ return (result_array)
def test_key(f, v, request, name):
@@ -114,26 +114,26 @@
valid = (name in list(f.keys()))
if valid:
if v:
- mylog.info("Key %s (%s) exists in `%s`!" %(name, request, str(f.name) ) )
- result_array = np.array([0,0])
+ mylog.info("Key %s (%s) exists in `%s`!" % (name, request, str(f.name)))
+ result_array = np.array([0, 0])
else:
if request == "required":
mylog.warning("open_pmd: Key %s (%s) does NOT exist in `%s`!" \
- %(name, request, str(f.name)) )
+ % (name, request, str(f.name)))
result_array = np.array([1, 0])
elif request == "recommended":
mylog.info("open_pmd: Key %s (%s) does NOT exist in `%s`!" \
- %(name, request, str(f.name)) )
+ % (name, request, str(f.name)))
result_array = np.array([0, 1])
elif request == "optional":
if v:
- mylog.info("open_pmd: Key %s (%s) does NOT exist in `%s`!" \
- %(name, request, str(f.name)) )
+ mylog.info("open_pmd: Key %s (%s) does NOT exist in `%s`!" \
+ % (name, request, str(f.name)))
result_array = np.array([0, 0])
- else :
- raise ValueError("Unrecognized string for `request` : %s" %request)
+ else:
+ raise ValueError("Unrecognized string for `request` : %s" % request)
- return(result_array)
+ return (result_array)
def test_attr(f, v, request, name, is_type=None, type_format=None):
@@ -176,12 +176,12 @@
if valid:
if v:
mylog.info("open_pmd: Attribute %s (%s) exists in `%s`! Type = %s, Value = %s" \
- %(name, request, str(f.name), type(value), str(value)) )
+ % (name, request, str(f.name), type(value), str(value)))
# test type
if is_type is not None:
if not type_format is None and not is_type is np.string_ and \
- not isinstance(type_format, collections.Iterable):
+ not isinstance(type_format, collections.Iterable):
type_format = [type_format]
type_format_names = map(lambda x: x.__name__, type_format)
if not is_type is None and not isinstance(is_type, collections.Iterable):
@@ -191,54 +191,54 @@
if type(value) in is_type:
# np.string_ format or general ndarray dtype text
if type(value) is np.string_ and type_format is not None:
- regEx = re.compile(type_format) # Python3 only: re.ASCII
- if regEx.match(value.decode()) :
- result_array = np.array([0,0])
+ regEx = re.compile(type_format) # Python3 only: re.ASCII
+ if regEx.match(value.decode()):
+ result_array = np.array([0, 0])
else:
mylog.warning("open_pmd: Attribute %s in `%s` does not satisfy " \
- "format ('%s' should be in format '%s')!" \
- %(name, str(f.name), value.decode(), type_format ) )
- result_array = np.array([1,0])
+ "format ('%s' should be in format '%s')!" \
+ % (name, str(f.name), value.decode(), type_format))
+ result_array = np.array([1, 0])
# ndarray dtypes
elif type(value) is np.ndarray:
if value.dtype.type in type_format:
- result_array = np.array([0,0])
+ result_array = np.array([0, 0])
elif type_format is None:
- result_array = np.array([0,0])
+ result_array = np.array([0, 0])
else:
mylog.warning("open_pmd: Attribute %s in `%s` is not of type " \
- "ndarray of '%s' (is ndarray of '%s')!" \
- %(name, str(f.name), type_format_names, \
- value.dtype.type.__name__) )
- result_array = np.array([1,0])
+ "ndarray of '%s' (is ndarray of '%s')!" \
+ % (name, str(f.name), type_format_names, \
+ value.dtype.type.__name__))
+ result_array = np.array([1, 0])
else:
- result_array = np.array([0,0])
+ result_array = np.array([0, 0])
else:
mylog.error(
- "Error: Attribute %s in `%s` is not of type '%s' (is '%s')!" \
- %(name, str(f.name), str(is_type_names), \
- type(value).__name__) )
- result_array = np.array([1,0])
- else: # is_type is None (== arbitrary)
- result_array = np.array([0,0])
+ "Error: Attribute %s in `%s` is not of type '%s' (is '%s')!" \
+ % (name, str(f.name), str(is_type_names), \
+ type(value).__name__))
+ result_array = np.array([1, 0])
+ else: # is_type is None (== arbitrary)
+ result_array = np.array([0, 0])
else:
if request == "required":
mylog.warning("open_pmd: Attribute %s (%s) does NOT exist in `%s`!" \
- %(name, request, str(f.name)) )
+ % (name, request, str(f.name)))
result_array = np.array([1, 0])
elif request == "recommended":
mylog.info("open_pmd: Attribute %s (%s) does NOT exist in `%s`!" \
- %(name, request, str(f.name)) )
+ % (name, request, str(f.name)))
result_array = np.array([0, 1])
elif request == "optional":
if v:
- mylog.info("open_pmd: Attribute %s (%s) does NOT exist in `%s`!" \
- %(name, request, str(f.name)) )
+ mylog.info("open_pmd: Attribute %s (%s) does NOT exist in `%s`!" \
+ % (name, request, str(f.name)))
result_array = np.array([0, 0])
- else :
- raise ValueError("Unrecognized string for `request` : %s" %request)
+ else:
+ raise ValueError("Unrecognized string for `request` : %s" % request)
- return(result_array)
+ return (result_array)
def is_scalar_record(r):
@@ -255,21 +255,21 @@
bool : true if the record is a scalar record, false if the record
is either a vector or an other type of tensor record
"""
- if type(r) is h5.Group :
+ if type(r) is h5.Group:
# now it could be either a vector/tensor record
# or a scalar record with a constant component
valid, value = get_attr(r, "value")
# constant components require a "value" and a "shape" attribute
- if valid :
+ if valid:
return True
else:
return False
- else :
+ else:
return True
-def test_component(c, v) :
+def test_component(c, v):
"""
Checks if a record component defines all required attributes.
@@ -290,18 +290,18 @@
# Initialize the result array
# First element : number of errors
# Second element : number of warnings
- result_array = np.array([0,0])
+ result_array = np.array([0, 0])
- if type(c) is h5.Group :
+ if type(c) is h5.Group:
# since this check tests components, this must be a constant
# component: requires "value" and "shape" attributes
- result_array += test_attr(c, v, "required", "value") # type can be arbitrary
+ result_array += test_attr(c, v, "required", "value") # type can be arbitrary
result_array += test_attr(c, v, "required", "shape", np.ndarray, np.uint64)
# default attributes for all components
result_array += test_attr(c, v, "required", "unitSI", np.float64)
- return(result_array)
+ return (result_array)
def check_root_attr(f, v, pic):
@@ -328,7 +328,7 @@
# Initialize the result array
# First element : number of errors
# Second element : number of warnings
- result_array = np.array([0,0])
+ result_array = np.array([0, 0])
# STANDARD.md
# required
@@ -341,20 +341,20 @@
result_array += test_attr(f, v, "required", "iterationFormat", np.string_)
# groupBased iteration encoding needs to match basePath
- if result_array[0] == 0 :
- if f.attrs["iterationEncoding"].decode() == "groupBased" :
- if f.attrs["iterationFormat"].decode() != f.attrs["basePath"].decode() :
+ if result_array[0] == 0:
+ if f.attrs["iterationEncoding"].decode() == "groupBased":
+ if f.attrs["iterationFormat"].decode() != f.attrs["basePath"].decode():
mylog.warning("open_pmd: for groupBased iterationEncoding the basePath "
- "and iterationFormat must match!")
- result_array += np.array([1,0])
+ "and iterationFormat must match!")
+ result_array += np.array([1, 0])
- # recommended
+ # recommended
result_array += test_attr(f, v, "recommended", "author", np.string_)
result_array += test_attr(f, v, "recommended", "software", np.string_)
result_array += test_attr(f, v, "recommended",
"softwareVersion", np.string_)
result_array += test_attr(f, v, "recommended", "date", np.string_,
- "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [\+|-][0-9]{4}$")
+ "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [\+|-][0-9]{4}$")
# optional
result_array += test_attr(f, v, "optional", "comment", np.string_)
@@ -365,14 +365,14 @@
if valid:
if (ext_list[0][1] & extensionIDs) != extensionIDs:
mylog.warning("open_pmd: ID=%s for extension `%s` not found in " \
- "`openPMDextension` (is %s)!" \
- %(ext_list[0][1], ext_list[0][0], extensionIDs) )
- result_array += np.array([1,0])
+ "`openPMDextension` (is %s)!" \
+ % (ext_list[0][1], ext_list[0][0], extensionIDs))
+ result_array += np.array([1, 0])
- return(result_array)
+ return (result_array)
-def check_iterations(f, v, pic) :
+def check_iterations(f, v, pic):
"""
Scan all the iterations present in the file, checking both
the meshes and the particles
@@ -397,39 +397,39 @@
# Find all the iterations
format_error = False
- try :
+ try:
list_iterations = list(f['/data/'].keys())
- except KeyError :
+ except KeyError:
format_error = True
- else :
+ else:
# Check that these iterations are indeed encoded as integers
- for iteration in list_iterations :
- for character in iteration : # go through the string
- if not (character in string.digits) :
+ for iteration in list_iterations:
+ for character in iteration: # go through the string
+ if not (character in string.digits):
format_error = True
# Detect any error and interrupt execution if one is found
- if format_error == True :
+ if format_error == True:
mylog.warning("open_pmd: it seems that the path of the data within the HDF5 file "
- "is not of the form '/data/%T/', where %T corresponds to an "
- "actual integer.")
- return(np.array([1, 0]))
- else :
- mylog.info("open_pmd: Found %d iteration(s)" % len(list_iterations) )
+ "is not of the form '/data/%T/', where %T corresponds to an "
+ "actual integer.")
+ return (np.array([1, 0]))
+ else:
+ mylog.info("open_pmd: Found %d iteration(s)" % len(list_iterations))
# Initialize the result array
# First element : number of errors
# Second element : number of warnings
- result_array = np.array([ 0, 0])
+ result_array = np.array([0, 0])
# Loop over the iterations and check the meshes and the particles
- for iteration in list_iterations :
+ for iteration in list_iterations:
result_array += check_base_path(f, iteration, v, pic)
# Go deeper only if there is no error at this point
- if result_array[0] == 0 :
+ if result_array[0] == 0:
result_array += check_meshes(f, iteration, v, pic)
result_array += check_particles(f, iteration, v, pic)
- return(result_array)
+ return (result_array)
def check_base_path(f, iteration, v, pic):
@@ -459,7 +459,7 @@
# Initialize the result array
# First element : number of errors
# Second element : number of warnings
- result_array = np.array([ 0, 0])
+ result_array = np.array([0, 0])
# Find the path to the data
base_path = ("/data/%s/" % iteration).encode('ascii')
@@ -470,7 +470,7 @@
result_array += test_attr(bp, v, "required", "dt", [np.float32, np.float64])
result_array += test_attr(bp, v, "required", "timeUnitSI", np.float64)
- return(result_array)
+ return (result_array)
def check_meshes(f, iteration, v, pic):
@@ -500,20 +500,20 @@
# Initialize the result array
# First element : number of errors
# Second element : number of warnings
- result_array = np.array([ 0, 0])
+ result_array = np.array([0, 0])
# Find the path to the data
base_path = "/data/%s/" % iteration
valid, meshes_path = get_attr(f, "meshesPath")
- if not valid :
+ if not valid:
mylog.warning("open_pmd: `meshesPath` is missing or malformed in '/'")
- return( np.array([1, 0]) )
+ return (np.array([1, 0]))
meshes_path = meshes_path.decode()
- if os.path.join( base_path, meshes_path) != ( base_path + meshes_path ):
+ if os.path.join(base_path, meshes_path) != (base_path + meshes_path):
mylog.warning("open_pmd: `basePath`+`meshesPath` seems to be malformed "
- "(is `basePath` absolute and ends on a `/` ?)")
- return( np.array([1, 0]) )
+ "(is `basePath` absolute and ends on a `/` ?)")
+ return (np.array([1, 0]))
else:
full_meshes_path = (base_path + meshes_path).encode('ascii')
# Find all the meshes
@@ -521,11 +521,11 @@
list_meshes = list(f[full_meshes_path].keys())
except KeyError:
list_meshes = []
- mylog.debug( "Iteration %s : found %d meshes"
- %( iteration, len(list_meshes) ) )
+ mylog.debug("Iteration %s : found %d meshes"
+ % (iteration, len(list_meshes)))
# Check for the attributes of the STANDARD.md
- for field_name in list_meshes :
+ for field_name in list_meshes:
field = f[full_meshes_path + field_name.encode('ascii')]
result_array += test_record(f[full_meshes_path], field_name)
@@ -549,26 +549,26 @@
geometry_test = test_attr(field, v, "required", "geometry", np.string_)
result_array += geometry_test
# geometryParameters is required when using thetaMode
- if geometry_test[0] == 0 and field.attrs["geometry"] == b"thetaMode" :
+ if geometry_test[0] == 0 and field.attrs["geometry"] == b"thetaMode":
result_array += test_attr(field, v, "required",
- "geometryParameters", np.string_)
+ "geometryParameters", np.string_)
# otherwise it is optional
- else :
+ else:
result_array += test_attr(field, v, "optional",
- "geometryParameters", np.string_)
+ "geometryParameters", np.string_)
# Attributes of the record's components
- if is_scalar_record(field) : # If the record is a scalar field
+ if is_scalar_record(field): # If the record is a scalar field
result_array += test_component(field, v)
result_array += test_attr(field, v,
- "required", "position", np.ndarray, [np.float32, np.float64])
- else: # If the record is a vector field
+ "required", "position", np.ndarray, [np.float32, np.float64])
+ else: # If the record is a vector field
# Loop over the components
- for component_name in list(field.keys()) :
+ for component_name in list(field.keys()):
component = field[component_name]
result_array += test_component(component, v)
result_array += test_attr(component, v,
- "required", "position", np.ndarray, [np.float32, np.float64])
+ "required", "position", np.ndarray, [np.float32, np.float64])
# Check for the attributes of the PIC extension,
# if asked to do so by the user
@@ -578,33 +578,33 @@
result_array += test_attr(f[full_meshes_path], v, "required",
"fieldSolver", np.string_)
valid, field_solver = get_attr(f[full_meshes_path], "fieldSolver")
- if (valid == True) and (field_solver in ["other", "GPSTD"]) :
+ if (valid == True) and (field_solver in ["other", "GPSTD"]):
result_array += test_attr(f[full_meshes_path], v, "required",
"fieldSolverParameters", np.string_)
# Check for the attributes associated with the field boundaries
result_array += test_attr(f[full_meshes_path], v, "required",
- "fieldBoundary", np.ndarray, np.string_)
+ "fieldBoundary", np.ndarray, np.string_)
valid, field_boundary = get_attr(f[full_meshes_path], "fieldBoundary")
- if (valid == True) and (np.any(field_boundary == b"other")) :
+ if (valid == True) and (np.any(field_boundary == b"other")):
result_array += test_attr(f[full_meshes_path], v, "required",
- "fieldBoundaryParameters", np.ndarray, np.string_)
+ "fieldBoundaryParameters", np.ndarray, np.string_)
# Check for the attributes associated with the field boundaries
result_array += test_attr(f[full_meshes_path], v, "required",
- "particleBoundary", np.ndarray, np.string_)
+ "particleBoundary", np.ndarray, np.string_)
valid, particle_boundary = get_attr(f[full_meshes_path], "particleBoundary")
- if (valid == True) and (np.any(particle_boundary == b"other")) :
+ if (valid == True) and (np.any(particle_boundary == b"other")):
result_array += test_attr(f[full_meshes_path], v, "required",
- "particleBoundaryParameters", np.ndarray, np.string_)
+ "particleBoundaryParameters", np.ndarray, np.string_)
# Check the attributes associated with the current smoothing
result_array += test_attr(f[full_meshes_path], v, "required",
"currentSmoothing", np.string_)
valid, current_smoothing = get_attr(f[full_meshes_path], "currentSmoothing")
- if (valid == True) and (current_smoothing != b"none") :
+ if (valid == True) and (current_smoothing != b"none"):
result_array += test_attr(f[full_meshes_path], v, "required",
- "currentSmoothingParameters", np.string_)
+ "currentSmoothingParameters", np.string_)
# Check the attributes associated with the charge conservation
result_array += test_attr(f[full_meshes_path], v, "required",
@@ -612,21 +612,21 @@
valid, charge_correction = get_attr(f[full_meshes_path], "chargeCorrection")
if valid == True and charge_correction != b"none":
result_array += test_attr(f[full_meshes_path], v, "required",
- "chargeCorrectionParameters", np.string_)
+ "chargeCorrectionParameters", np.string_)
# Check for the attributes of each record
- for field_name in list_meshes :
+ for field_name in list_meshes:
field = f[full_meshes_path + field_name.encode('ascii')]
result_array + test_attr(field, v, "required",
"fieldSmoothing", np.string_)
valid, field_smoothing = get_attr(field, "fieldSmoothing")
- if (valid == True) and (field_smoothing != b"none") :
- result_array += test_attr(field,v, "required",
- "fieldSmoothingParameters", np.string_)
- return(result_array)
+ if (valid == True) and (field_smoothing != b"none"):
+ result_array += test_attr(field, v, "required",
+ "fieldSmoothingParameters", np.string_)
+ return (result_array)
-def check_particles(f, iteration, v, pic) :
+def check_particles(f, iteration, v, pic):
"""
Scan all the particle data corresponding to one iteration
@@ -653,16 +653,16 @@
# Initialize the result array
# First element : number of errors
# Second element : number of warnings
- result_array = np.array([ 0, 0])
+ result_array = np.array([0, 0])
# Find the path to the data
base_path = ("/data/%s/" % iteration).encode('ascii')
valid, particles_path = get_attr(f, "particlesPath")
- if os.path.join( base_path, particles_path) != \
- ( base_path + particles_path ) :
+ if os.path.join(base_path, particles_path) != \
+ (base_path + particles_path):
mylog.warning("open_pmd: `basePath`+`meshesPath` seems to be malformed "
- "(is `basePath` absolute and ends on a `/` ?)")
- return( np.array([1, 0]) )
+ "(is `basePath` absolute and ends on a `/` ?)")
+ return (np.array([1, 0]))
else:
full_particle_path = base_path + particles_path
# Find all the particle species
@@ -670,15 +670,15 @@
list_species = list(f[full_particle_path].keys())
except KeyError:
list_species = []
- mylog.debug( "Iteration %s : found %d particle species"
- %( iteration, len(list_species) ) )
+ mylog.debug("Iteration %s : found %d particle species"
+ % (iteration, len(list_species)))
# Go through all the particle species
- for species_name in list_species :
+ for species_name in list_species:
species = f[full_particle_path + species_name.encode('ascii')]
# Check all records for this species
- for species_record_name in species :
+ for species_record_name in species:
result_array += test_record(species, species_record_name)
# Check the position record of the particles
@@ -686,22 +686,22 @@
# Check the position offset record of the particles
result_array += test_key(species, v, "required", "positionOffset")
- if result_array[0] == 0 :
+ if result_array[0] == 0:
position_dimensions = len(species["position"].keys())
positionOffset_dimensions = len(species["positionOffset"].keys())
- if position_dimensions != positionOffset_dimensions :
+ if position_dimensions != positionOffset_dimensions:
mylog.warning("open_pmd: `position` (ndim=%s) and `positionOffset` " \
- "(ndim=%s) do not have the same dimensions in " \
- "species `%s`!" \
- %(str(position_dimensions), \
- str(positionOffset_dimensions),
- species.name) )
- result_array += np.array([ 1, 0])
+ "(ndim=%s) do not have the same dimensions in " \
+ "species `%s`!" \
+ % (str(position_dimensions), \
+ str(positionOffset_dimensions),
+ species.name))
+ result_array += np.array([1, 0])
# Check the particlePatches record of the particles
patch_test = test_key(species, v, "recommended", "particlePatches")
result_array += patch_test
- if result_array[0] == 0 and patch_test[1] == 0 :
+ if result_array[0] == 0 and patch_test[1] == 0:
result_array += test_key(species["particlePatches"], v, "required",
"numParticles")
result_array += test_key(species["particlePatches"], v, "required",
@@ -710,23 +710,23 @@
"offset")
result_array += test_key(species["particlePatches"], v, "required",
"extent")
- if result_array[0] == 0 :
+ if result_array[0] == 0:
offset = species["particlePatches"]["offset"]
extent = species["particlePatches"]["extent"]
# Attributes of the components
- for component_name in list(species["position"].keys()) :
- result_array += test_key( offset, v, "required",
- component_name)
- result_array += test_key( extent, v, "required",
- component_name)
- if result_array[0] == 0 :
+ for component_name in list(species["position"].keys()):
+ result_array += test_key(offset, v, "required",
+ component_name)
+ result_array += test_key(extent, v, "required",
+ component_name)
+ if result_array[0] == 0:
dset_offset = offset[component_name]
result_array += test_component(dset_offset, v)
dset_extent = extent[component_name]
result_array += test_component(dset_extent, v)
# Check the records required by the PIC extension
- if pic :
+ if pic:
result_array += test_key(species, v, "required", "momentum")
result_array += test_key(species, v, "required", "charge")
result_array += test_key(species, v, "required", "mass")
@@ -736,7 +736,7 @@
result_array += test_key(species, v, "optional", "neutronNumber")
# Check the attributes associated with the PIC extension
- if pic :
+ if pic:
result_array += test_attr(species, v, "required",
"particleShape", [np.float32, np.float64])
result_array += test_attr(species, v, "required",
@@ -752,14 +752,14 @@
valid, particle_smoothing = get_attr(species, "particleSmoothing")
if valid == True and particle_smoothing != b"none":
result_array += test_attr(species, v, "required",
- "particleSmoothingParameters", np.string_)
+ "particleSmoothingParameters", np.string_)
# Check attributes of each record of the particle
for record in list(species.keys()):
# all records (but particlePatches) require units
if record != "particlePatches":
result_array += test_attr(species[record], v,
- "required", "unitDimension", np.ndarray, np.float64)
+ "required", "unitDimension", np.ndarray, np.float64)
time_type = f[base_path].attrs["time"].dtype.type
result_array += test_attr(species[record], v, "required",
"timeOffset", time_type)
@@ -797,3 +797,23 @@
if unit_dimension[i] != 0:
dim.append("{}**{}".format(si[i], unit_dimension[i]))
return "*".join(dim)
+
+
+def is_const_component(record_component):
+ """Determines whether a group or dataset in the HDF5 file is constant.
+
+ Parameters
+ ----------
+ record_component : h5py.Group or h5py.Dataset
+
+ Returns
+ -------
+ bool
+ True if constant, False otherwise
+
+ References
+ ----------
+ .. https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md,
+ section 'Constant Record Components'
+ """
+ return "value" in record_component.attrs.keys()
diff -r 0a69a5e3a7a30daadeb5e2d0da6c7a9f954a046d -r a0271fa934ca3ada4865fdc495dc2b51089f7854 yt/frontends/open_pmd/setup.py
--- a/yt/frontends/open_pmd/setup.py
+++ b/yt/frontends/open_pmd/setup.py
@@ -1,12 +1,13 @@
#!/usr/bin/env python
-"""
-
-Copyright (c) 2015, Daniel Grassinger (HZDR)
-
-Distributed under the terms of the Modified BSD License.
-
-The full license is in the file COPYING.txt, distributed with this software.
-"""
+# -----------------------------------------------------------------------------
+# Copyright (c) 2013, yt Development Team.
+# Copyright (c) 2015, Daniel Grassinger (HZDR)
+# Copyright (c) 2016, Fabian Koller (HZDR)
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+# -----------------------------------------------------------------------------
def configuration(parent_package='', top_path=None):
https://bitbucket.org/yt_analysis/yt/commits/b8606643c597/
Changeset: b8606643c597
Branch: yt
User: Fabian Koller
Date: 2016-08-09 09:32:59+00:00
Summary: Refactor imports,
docstrings,
Affected #: 4 files
diff -r a0271fa934ca3ada4865fdc495dc2b51089f7854 -r b8606643c597735c1ebaf04a29ce40075dc8ab44 yt/frontends/open_pmd/data_structures.py
--- a/yt/frontends/open_pmd/data_structures.py
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -14,19 +14,21 @@
# The full license is in the file COPYING.txt, distributed with this software.
# -----------------------------------------------------------------------------
-from yt.data_objects.grid_patch import AMRGridPatch
-from yt.geometry.grid_geometry_handler import GridIndex
-from yt.data_objects.static_output import Dataset
-from yt.utilities.file_handler import HDF5FileHandler
-from yt.utilities.logger import ytLogger as mylog
-from yt.frontends.open_pmd.fields import OpenPMDFieldInfo
-from yt.frontends.open_pmd.misc import is_const_component, check_root_attr, check_iterations
-
-import numpy as np
import os
import re
from math import ceil, floor
+import numpy as np
+
+from yt.data_objects.grid_patch import AMRGridPatch
+from yt.data_objects.static_output import Dataset
+from yt.frontends.open_pmd.fields import OpenPMDFieldInfo
+from yt.frontends.open_pmd.misc import is_const_component, check_root_attr, check_iterations
+from yt.geometry.grid_geometry_handler import GridIndex
+from yt.utilities.file_handler import HDF5FileHandler
+from yt.utilities.logger import ytLogger as mylog
+
+
class OpenPMDGrid(AMRGridPatch):
"""Represents disjoint chunk of data on-disk and their relation.
@@ -266,6 +268,7 @@
----------
handle : h5py.File
path : str
+ (absolute) filepath for current hdf5 container
nonstandard : bool
"""
self.base_path = "/data/{}/".format(handle["/data"].keys()[0])
diff -r a0271fa934ca3ada4865fdc495dc2b51089f7854 -r b8606643c597735c1ebaf04a29ce40075dc8ab44 yt/frontends/open_pmd/fields.py
--- a/yt/frontends/open_pmd/fields.py
+++ b/yt/frontends/open_pmd/fields.py
@@ -15,49 +15,14 @@
# The full license is in the file COPYING.txt, distributed with this software.
# -----------------------------------------------------------------------------
-from yt.funcs import mylog
-from yt.utilities.physical_constants import speed_of_light
+import numpy as np
+
from yt.fields.field_info_container import FieldInfoContainer
-from yt.units.yt_array import YTQuantity
from yt.fields.magnetic_field import setup_magnetic_field_aliases
from yt.frontends.open_pmd.misc import parse_unit_dimension
-
-import numpy as np
-
-
-def setup_kinetic_energy(self, ptype):
- def _kin_en(field, data):
- p2 = (data[ptype, "particle_momentum_x"] ** 2 +
- data[ptype, "particle_momentum_y"] ** 2 +
- data[ptype, "particle_momentum_z"] ** 2)
- mass = data[ptype, "particle_mass"] * data[ptype, "particle_weighting"]
- return speed_of_light * np.sqrt(p2 + mass ** 2 * speed_of_light ** 2) - mass * speed_of_light ** 2
-
- self.add_field((ptype, "particle_kinetic_energy"),
- function=_kin_en,
- units="kg*m**2/s**2",
- particle_type=True)
-
-
-def setup_velocity(self, ptype):
- def _get_vel(axis):
- def velocity(field, data):
- c = speed_of_light
- momentum = data[ptype, "particle_momentum_{}".format(axis)]
- mass = data[ptype, "particle_mass"]
- weighting = data[ptype, "particle_weighting"]
- return momentum / (
- (mass * weighting) ** 2 +
- (momentum ** 2) / (c ** 2)
- ) ** 0.5
-
- return velocity
-
- for ax in "xyz":
- self.add_field((ptype, "particle_velocity_%s" % ax),
- function=_get_vel(ax),
- units="m/s",
- particle_type=True)
+from yt.units.yt_array import YTQuantity
+from yt.utilities.logger import ytLogger as mylog
+from yt.utilities.physical_constants import speed_of_light
def setup_poynting_vector(self):
@@ -79,6 +44,41 @@
units="T*V/m")
+def setup_kinetic_energy(self, ptype):
+ def _kin_en(field, data):
+ p2 = (data[ptype, "particle_momentum_x"] ** 2 +
+ data[ptype, "particle_momentum_y"] ** 2 +
+ data[ptype, "particle_momentum_z"] ** 2)
+ mass = data[ptype, "particle_mass"] * data[ptype, "particle_weighting"]
+ return speed_of_light * np.sqrt(p2 + mass ** 2 * speed_of_light ** 2) - mass * speed_of_light ** 2
+
+ self.add_field((ptype, "particle_kinetic_energy"),
+ function=_kin_en,
+ units="kg*m**2/s**2",
+ particle_type=True)
+
+
+def setup_velocity(self, ptype):
+ def _get_vel(axis):
+ def velocity(field, data):
+ c = speed_of_light
+ momentum = data[ptype, "particle_momentum_{}".format(axis)]
+ mass = data[ptype, "particle_mass"]
+ weighting = data[ptype, "particle_weighting"]
+ return momentum / np.sqrt(
+ (mass * weighting) ** 2 +
+ (momentum ** 2) / (c ** 2)
+ )
+
+ return velocity
+
+ for ax in "xyz":
+ self.add_field((ptype, "particle_velocity_%s" % ax),
+ function=_get_vel(ax),
+ units="m/s",
+ particle_type=True)
+
+
def setup_absolute_positions(self, ptype):
def _abs_pos(axis):
def ap(field, data):
@@ -95,16 +95,42 @@
class OpenPMDFieldInfo(FieldInfoContainer):
+ """Specifies which fields from the dataset yt should know about.
+
+ ``self.known_other_fields`` and ``self.known_particle_fields`` must be populated.
+ Entries for both of these lists must be tuples of the form
+ ("name", ("units", ["fields", "to", "alias"], "display_name"))
+ These fields will be represented and handled in yt in the way you define them here.
+ The fields defined in both ``self.known_other_fields`` and ``self.known_particle_fields`` will only be added
+ to a dataset (with units, aliases, etc), if they match any entry in the ``OpenPMDHierarchy``'s ``self.field_list``.
+
+ Notes
+ -----
+
+ Contrary to many other frontends, we dynamically obtain the known fields from the simulation output.
+ The openPMD markup is extremely flexible - names, dimensions and the number of individual datasets
+ can (and very likely will) vary.
+
+ openPMD states that names of records and their components are only allowed to contain the
+ characters a-Z,
+ the numbers 0-9
+ and the underscore _
+ (equivalently, the regex \w).
+ Since yt widely uses the underscore in field names, openPMD's underscores (_) are replaced by hyphen (-).
+
+ The constructor of the super-class is called after the fields have been dynamically parsed, so they are known when
+ needed during the call of ``setup_fluid_aliases`` .
+
+ Derived fields will automatically be set up, if names and units of your known on-disk (or manually derived)
+ fields match the ones in [1].
+
+ References
+ ----------
+ .. http://yt-project.org/docs/dev/analyzing/fields.html
+ .. http://yt-project.org/docs/dev/developing/creating_frontend.html#data-meaning-structures
+ .. https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md
+ .. [1] http://yt-project.org/docs/dev/reference/field_list.html#universal-fields
"""
- We need to specify which fields we might have in our dataset. The field info
- container subclass here will define which fields it knows about. There are
- optionally methods on it that get called which can be subclassed.
-
- This class defines, which fields and particle fields could be in the HDF5-file
- The field names have to match the names in "openPMDHierarchy" in data_structures.py
- This also defines the units of the fields
- """
-
_mag_fields = []
def __init__(self, ds, field_list):
@@ -126,8 +152,8 @@
unit = str(YTQuantity(1, parsed).units)
aliases = []
# Save a list of magnetic fields for aliasing later on
- # We can not reasonably infer field type by name in openPMD
- if "T" in unit or "kg/(A*s**2)" in unit:
+ # We can not reasonably infer field type/unit by name in openPMD
+ if unit in "T" or "kg/(A*s**2)" in unit:
self._mag_fields.append(ytname)
self.known_other_fields += ((ytname, (unit, aliases, None)),)
else:
@@ -145,7 +171,7 @@
aliases = []
# Save a list of magnetic fields for aliasing later on
# We can not reasonably infer field type by name in openPMD
- if "T" in unit or "kg/(A*s**2)" in unit:
+ if unit in "T" or "kg/(A*s**2)" in unit:
self._mag_fields.append(ytname)
self.known_other_fields += ((ytname, (unit, aliases, None)),)
for i in self.known_other_fields:
@@ -170,15 +196,15 @@
unit = str(YTQuantity(1, parsed).units)
name = ["particle", attrib]
ytattrib = attrib
- # Symbolically rename position to preserve yt's interpretation of the pfield
- # particle_position is later derived in setup_absolute_positions
if ytattrib in "position":
+ # Symbolically rename position to preserve yt's interpretation of the pfield
+ # particle_position is later derived in setup_absolute_positions in the way yt expects it
ytattrib = "positionCoarse"
for axis in particles.get(species).get(attrib).keys():
aliases = []
if axis in "rxyz":
name = ["particle", ytattrib, axis]
- ytname = str("_".join(name))
+ ytname = str("_".join([name.replace("_", "-")]))
if ds._nonstandard and "globalCellIdx" in ytname:
aliases.append(ytname.replace("globalCellIdx", "positionOffset"))
particle_fields += ((ytname, (unit, aliases, None)),)
@@ -190,10 +216,9 @@
super(OpenPMDFieldInfo, self).__init__(ds, field_list)
def setup_fluid_fields(self):
- """
- Here you can create functions to calculate out of existing fields the
- values of new fields e.g. calculate out of E-field and B-field the
- Poynting vector
+ """Defines which derived mesh fields to create.
+
+ If a field can not be calculated, it will simply be skipped.
"""
# Set up aliases first so the setup for poynting can use them
if len(self._mag_fields) > 0:
@@ -201,8 +226,10 @@
setup_poynting_vector(self)
def setup_particle_fields(self, ptype):
- """
- This will get called for every particle type.
+ """Defines which derived particle fields to create.
+
+ This will be called for every entry in `OpenPMDDataset``'s ``self.particle_types``.
+ If a field can not be calculated, it will simply be skipped.
"""
setup_absolute_positions(self, ptype)
setup_kinetic_energy(self, ptype)
diff -r a0271fa934ca3ada4865fdc495dc2b51089f7854 -r b8606643c597735c1ebaf04a29ce40075dc8ab44 yt/frontends/open_pmd/io.py
--- a/yt/frontends/open_pmd/io.py
+++ b/yt/frontends/open_pmd/io.py
@@ -15,11 +15,11 @@
# The full license is in the file COPYING.txt, distributed with this software.
# -----------------------------------------------------------------------------
+import numpy as np
+
+from yt.frontends.open_pmd.misc import is_const_component
from yt.utilities.io_handler import BaseIOHandler
from yt.utilities.logger import ytLogger as mylog
-from yt.frontends.open_pmd.misc import is_const_component
-
-import numpy as np
class IOHandlerOpenPMD(BaseIOHandler):
@@ -165,8 +165,9 @@
continue
pds = ds[self.particles_path + "/" + spec]
for field in field_list:
- nfield = "/".join(field.split("_")[1:]).replace("positionCoarse", "position")
- data = self.get_component(pds, nfield, index, offset, self.ds._nonstandard)
+ component = "/".join(field.split("_")[1:]).replace("positionCoarse",
+ "position").replace("-", "_")
+ data = self.get_component(pds, component, index, offset, self.ds._nonstandard)
yield ((ptype, field), data[mask])
def _read_fluid_selection(self, chunks, selector, fields, size):
@@ -215,10 +216,10 @@
for chunk in chunks:
for g in chunk.objs:
ds = f[bp + mp]
- nfield = fname.replace("_", "/").replace("-", "_")
+ component = fname.replace("_", "/").replace("-", "_")
index = g.mesh_index
offset = g.mesh_offset
- data = np.array(self.get_component(ds, nfield, index, offset, self.ds._nonstandard))
+ data = np.array(self.get_component(ds, component, index, offset, self.ds._nonstandard))
# The following is a modified AMRGridPatch.select(...)
mask = g._get_selector_mask(selector)
data.shape = mask.shape # Workaround - casts a 2D (x,y) array to 3D (x,y,1)
diff -r a0271fa934ca3ada4865fdc495dc2b51089f7854 -r b8606643c597735c1ebaf04a29ce40075dc8ab44 yt/frontends/open_pmd/misc.py
--- a/yt/frontends/open_pmd/misc.py
+++ b/yt/frontends/open_pmd/misc.py
@@ -16,12 +16,14 @@
#
# -----------------------------------------------------------------------------
-import h5py as h5
-import numpy as np
+import collections # for isinstance
+import os.path
import re
import string
-import collections # for isinstance
-import sys, getopt, os.path
+
+import numpy as np
+
+from yt.utilities.file_handler import HDF5FileHandler
from yt.utilities.logger import ytLogger as mylog
openPMD = "1.0.0"
@@ -31,7 +33,7 @@
def open_file(file_name):
try:
- f = h5.File(file_name, "r")
+ f = HDF5FileHandler.File(file_name, "r")
return (f)
except:
raise
https://bitbucket.org/yt_analysis/yt/commits/7c1974a7c397/
Changeset: 7c1974a7c397
Branch: yt
User: Fabian Koller
Date: 2016-08-14 15:10:33+00:00
Summary: Consider mesh-size as well when calculating num_grids,
detect hdf5 dataset/group by direct comparison with h5.Dataset/h5.Group,
unify frontend-output
Affected #: 4 files
diff -r b8606643c597735c1ebaf04a29ce40075dc8ab44 -r 7c1974a7c3973278e3426b590aecebadb1eb9cd7 yt/frontends/open_pmd/data_structures.py
--- a/yt/frontends/open_pmd/data_structures.py
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -18,6 +18,7 @@
import re
from math import ceil, floor
+import h5py as h5
import numpy as np
from yt.data_objects.grid_patch import AMRGridPatch
@@ -30,11 +31,12 @@
class OpenPMDGrid(AMRGridPatch):
- """Represents disjoint chunk of data on-disk and their relation.
+ """Represents disjoint chunk of data on-disk.
The chunks are sliced off the original field along the x-axis.
This defines the index and offset for every mesh and particle type.
- It also defines further required characteristics for every grid.
+ It also defines parents and children grids. Since openPMD does not have multiple levels of refinement,
+ there are no parents or children for any grid.
"""
_id_offset = 0
__slots__ = ["_level_id"]
@@ -94,15 +96,15 @@
try:
for field in f[bp + mp].keys():
try:
- for ax in f[bp + mp + field].keys():
- mesh_fields.append(field + "_" + ax)
+ for axis in f[bp + mp + field].keys():
+ mesh_fields.append(field + "_" + axis)
except AttributeError:
- # This is for dataSets, they do not have keys
+ # This is a h5.Dataset (i.e. no axes)
mesh_fields.append(field.replace("_", "-"))
except KeyError:
# There are no mesh fields
pass
- self.field_list = [("openPMD", str(c)) for c in mesh_fields]
+ self.field_list = [("openPMD", str(field)) for field in mesh_fields]
particle_fields = []
try:
@@ -118,23 +120,24 @@
axes = f[bp + pp + species + "/" + record].keys()
if record in "position":
record = "positionCoarse"
- for ax in axes:
- particle_fields.append(species + "_" + record + "_" + ax)
+ for axis in axes:
+ particle_fields.append(species + "_" + record + "_" + axis)
except AttributeError:
# Record is a dataset, does not have axes (e.g. weighting)
particle_fields.append(species + "_" + record)
pass
else:
- # We probably do not want particlePatches as accessible field lists
pass
if len(f[bp + pp].keys()) > 1:
# There is more than one particle species, use the specific names as field types
self.field_list.extend(
- [(str(c).split("_")[0], ("particle_" + "_".join(str(c).split("_")[1:]))) for c in particle_fields])
+ [(str(field).split("_")[0],
+ ("particle_" + "_".join(str(field).split("_")[1:]))) for field in particle_fields])
else:
# Only one particle species, fall back to "io"
self.field_list.extend(
- [("io", ("particle_" + "_".join(str(c).split("_")[1:]))) for c in particle_fields])
+ [("io",
+ ("particle_" + "_".join(str(field).split("_")[1:]))) for field in particle_fields])
except KeyError:
# There are no particle fields
pass
@@ -145,15 +148,24 @@
The number of grids is determined by their respective memory footprint.
You may change ``gridsize`` accordingly. Increase if you have few cores or run single-threaded.
"""
- # TODO Calculate the ppg not solely on particle count, also on meshsize
f = self.dataset._handle
bp = self.dataset.base_path
mp = self.dataset.meshes_path
pp = self.dataset.particles_path
- gridsize = 10 * 10 ** 6 # Byte
+ gridsize = 12 * 10 ** 6 # Byte
self.numparts = {}
+ meshsizes = {}
+ for mesh in f[bp + mp].keys():
+ if type(f[bp + mp + mesh]) is h5.Group:
+ for axis in f[bp + mp + mesh]:
+ meshsizes[mesh + axis] = f[bp + mp + mesh + "/" + axis].size
+ else:
+ meshsizes[mesh] = f[bp + mp + mesh].size
+ if not meshsizes.values()[1:] == meshsizes.values()[:-1]:
+ mylog.warning("open_pmd - This frontend assumes all meshes are equally shaped,"
+ " but meshes have different sizes!")
for species in f[bp + pp].keys():
axis = f[bp + pp + species + "/position"].keys()[0]
if is_const_component(f[bp + pp + species + "/position/" + axis]):
@@ -163,7 +175,8 @@
# Limit particles per grid by resulting memory footprint
ppg = int(gridsize / (self.dataset.dimensionality * 4)) # 4 Byte per value per dimension (f32)
# Use an upper bound of equally sized grids, last one might be smaller
- self.num_grids = int(ceil(np.max(self.numparts.values()) * ppg ** -1))
+ values = np.max((np.max(self.numparts.values()), np.max(meshsizes.values())))
+ self.num_grids = int(ceil(values * ppg ** -1))
def _parse_index(self):
"""Fills each grid with appropriate properties (extent, dimensions, ...)
@@ -200,7 +213,7 @@
self.dataset.domain_dimensions[0] ** -1 * \
self.dataset.domain_right_edge[0]
meshedge = self.grid_right_edge[i][0]
- particlecount = []
+ particleoffset = []
particleindex = []
for spec in self.numparts:
particleindex += [(spec, pci[spec])]
@@ -209,16 +222,16 @@
num = nrp[spec]
else:
num = int(floor(self.numparts[spec] * self.num_grids ** -1))
- particlecount += [(spec, num)]
+ particleoffset += [(spec, num)]
nrp[spec] -= num
self.grid_particle_count[i] += num
self.grids[i] = self.grid(
i, self, self.grid_levels[i, 0],
pi=particleindex,
- po=particlecount,
+ po=particleoffset,
mi=meshindex,
mo=self.grid_dimensions[i][0])
- for spec, val in particlecount:
+ for spec, val in particleoffset:
pci[spec] += val
meshindex += self.grid_dimensions[i][0]
remaining -= self.grid_dimensions[i][0]
@@ -257,7 +270,7 @@
if len(particles) > 1:
# Only use on-disk particle names if there is more than one species
self.particle_types = particles
- mylog.debug("open_pmd: self.particle_types: {}".format(self.particle_types))
+ mylog.debug("open_pmd - self.particle_types: {}".format(self.particle_types))
self.particle_types_raw = self.particle_types
self.particle_types = tuple(self.particle_types)
@@ -280,24 +293,24 @@
if "groupBased" in handle.attrs["iterationEncoding"]:
for i in list(handle["/data"].keys()):
list_iterations.append(i)
- mylog.info("open_pmd: found {} iterations in file".format(len(list_iterations)))
+ mylog.info("open_pmd - found {} iterations in file".format(len(list_iterations)))
elif "fileBased" in handle.attrs["iterationEncoding"]:
regex = "^" + handle.attrs["iterationFormat"].replace('%T', '[0-9]+') + "$"
if path is '':
- mylog.warning("open_pmd: For file based iterations, please use absolute file paths!")
+ mylog.warning("open_pmd - For file based iterations, please use absolute file paths!")
pass
for filename in os.listdir(path):
if re.match(regex, filename):
list_iterations.append(filename)
- mylog.info("open_pmd: found {} iterations in directory".format(len(list_iterations)))
+ mylog.info("open_pmd - found {} iterations in directory".format(len(list_iterations)))
else:
mylog.warning(
- "open_pmd: No valid iteration encoding: {}".format(handle.attrs["iterationEncoding"]))
+ "open_pmd - No valid iteration encoding: {}".format(handle.attrs["iterationEncoding"]))
if len(list_iterations) == 0:
- mylog.warning("open_pmd: No iterations found!")
+ mylog.warning("open_pmd - no iterations found!")
if "groupBased" in handle.attrs["iterationEncoding"] and len(list_iterations) > 1:
- mylog.warning("open_pmd: only choose to load one iteration ({})".format(handle["/data"].keys()[0]))
+ mylog.warning("open_pmd - only choose to load one iteration ({})".format(handle["/data"].keys()[0]))
self.meshes_path = self._handle["/"].attrs["meshesPath"]
self.particles_path = self._handle["/"].attrs["particlesPath"]
@@ -337,7 +350,7 @@
fshape = np.asarray(f[bp + mp + "/" + mesh + "/" + axis].shape, dtype=np.int64)
except:
fshape = np.array([1, 1, 1], dtype=np.int64)
- mylog.warning("Could not detect shape of simulated field! "
+ mylog.warning("open_pmd - Could not detect shape of simulated field! "
"Assuming a single cell and thus setting fshape to [1, 1, 1]!")
self.dimensionality = len(fshape)
@@ -360,7 +373,7 @@
self.domain_right_edge = np.append(self.domain_right_edge, np.ones(3 - self.domain_right_edge.size))
except Exception as e:
mylog.warning(
- "The domain extent could not be calculated! ({}) Setting the field extent to 1m**3!".format(e))
+ "open_pmd - The domain extent could not be calculated! ({}) Setting the field extent to 1m**3!".format(e))
self.domain_right_edge = np.ones(3, dtype=np.float64)
if self._nonstandard:
@@ -373,7 +386,7 @@
"""Checks whether the supplied file can be read by this frontend.
"""
try:
- f = HDF5FileHandler(args[0])
+ f = h5.File(args[0])
except:
return False
verbose = False
@@ -399,4 +412,5 @@
return True
except:
return False
+ f.close()
return True
diff -r b8606643c597735c1ebaf04a29ce40075dc8ab44 -r 7c1974a7c3973278e3426b590aecebadb1eb9cd7 yt/frontends/open_pmd/fields.py
--- a/yt/frontends/open_pmd/fields.py
+++ b/yt/frontends/open_pmd/fields.py
@@ -15,6 +15,7 @@
# The full license is in the file COPYING.txt, distributed with this software.
# -----------------------------------------------------------------------------
+import h5py as h5
import numpy as np
from yt.fields.field_info_container import FieldInfoContainer
@@ -141,9 +142,9 @@
fields = f[bp + mp]
for fname in fields.keys():
- field = fields.get(fname)
- if "dataset" in str(field).split(" ")[1]:
- # We have a dataset, don't consider axes. This appears to be a vector field of single dimensionality
+ field = fields[fname]
+ if type(field) is h5.Dataset:
+ # Don't consider axes. This appears to be a vector field of single dimensionality
ytname = str("_".join([fname.replace("_", "-")]))
if ds._nonstandard:
parsed = ""
@@ -175,15 +176,11 @@
self._mag_fields.append(ytname)
self.known_other_fields += ((ytname, (unit, aliases, None)),)
for i in self.known_other_fields:
- mylog.debug("oPMD - fields - known_other_fields - {}".format(i))
-
+ mylog.debug("open_pmd - known_other_fields - {}".format(i))
particle_fields = ()
particles = f[bp + pp]
for species in particles.keys():
- for attrib in particles.get(species).keys():
- if "weighting" in attrib:
- particle_fields += (("particle_weighting", ("", [], None)),)
- continue
+ for attrib in particles[species].keys():
try:
if ds._nonstandard:
if "globalCellIdx" in attrib or "position" in attrib:
@@ -200,19 +197,23 @@
# Symbolically rename position to preserve yt's interpretation of the pfield
# particle_position is later derived in setup_absolute_positions in the way yt expects it
ytattrib = "positionCoarse"
- for axis in particles.get(species).get(attrib).keys():
- aliases = []
- if axis in "rxyz":
- name = ["particle", ytattrib, axis]
- ytname = str("_".join([name.replace("_", "-")]))
- if ds._nonstandard and "globalCellIdx" in ytname:
- aliases.append(ytname.replace("globalCellIdx", "positionOffset"))
- particle_fields += ((ytname, (unit, aliases, None)),)
- except:
- mylog.info("{}_{} does not seem to have unitDimension".format(species, attrib))
+ pds = particles[species + "/" + attrib]
+ if type(pds) is h5.Dataset:
+ particle_fields += ((str("_".join(name)), (unit, [], None)),)
+ else:
+ for axis in pds.keys():
+ aliases = []
+ if axis in "rxyz":
+ name = ["particle", ytattrib.replace("_", "-"), axis]
+ ytname = str("_".join(name))
+ if ds._nonstandard and "globalCellIdx" in ytname:
+ aliases.append(ytname.replace("globalCellIdx", "positionOffset"))
+ particle_fields += ((ytname, (unit, aliases, None)),)
+ except KeyError:
+ mylog.info("open_pmd - {}_{} does not seem to have unitDimension".format(species, attrib))
self.known_particle_fields = particle_fields
for i in self.known_particle_fields:
- mylog.debug("oPMD - fields - known_particle_fields - {}".format(i))
+ mylog.debug("open_pmd - known_particle_fields - {}".format(i))
super(OpenPMDFieldInfo, self).__init__(ds, field_list)
def setup_fluid_fields(self):
@@ -220,6 +221,7 @@
If a field can not be calculated, it will simply be skipped.
"""
+
# Set up aliases first so the setup for poynting can use them
if len(self._mag_fields) > 0:
setup_magnetic_field_aliases(self, "openPMD", self._mag_fields)
diff -r b8606643c597735c1ebaf04a29ce40075dc8ab44 -r 7c1974a7c3973278e3426b590aecebadb1eb9cd7 yt/frontends/open_pmd/io.py
--- a/yt/frontends/open_pmd/io.py
+++ b/yt/frontends/open_pmd/io.py
@@ -192,7 +192,7 @@
keys are tuples (ftype, fname) representing a field
values are flat (``size``,) ndarrays with data from that field
"""
- mylog.info("_read_fluid_selection {} {} {} {}".format(chunks, selector, fields, size))
+ mylog.info("open_pmd - _read_fluid_selection {} {} {} {}".format(chunks, selector, fields, size))
f = self._handle
bp = self.base_path
mp = self.meshes_path
@@ -267,18 +267,18 @@
else:
unitSI = record_component.attrs["unitSI"]
# check whether component is constant
- if "value" in record_component.attrs.keys():
+ if is_const_component(record_component):
if offset is not None:
shape = offset
else:
shape = record_component.attrs["shape"] - index
# component is constant, craft an array by hand
- mylog.debug("open_pmd - misc - get_component (const): {}/{}({})".format(group.name, component_name, shape))
+ mylog.debug("open_pmd - get_component (const): {}/{}({})".format(group.name, component_name, shape))
return np.full(shape, record_component.attrs["value"] * unitSI)
else:
if offset is not None:
offset += index
# component is a dataset, return it (possibly masked)
mylog.debug(
- "open_pmd - misc - get_component: {}/{}[{}:{}]".format(group.name, component_name, index, offset))
+ "open_pmd - get_component: {}/{}[{}:{}]".format(group.name, component_name, index, offset))
return np.multiply(record_component[index:offset], unitSI)
diff -r b8606643c597735c1ebaf04a29ce40075dc8ab44 -r 7c1974a7c3973278e3426b590aecebadb1eb9cd7 yt/frontends/open_pmd/misc.py
--- a/yt/frontends/open_pmd/misc.py
+++ b/yt/frontends/open_pmd/misc.py
@@ -21,9 +21,9 @@
import re
import string
+import h5py as h5
import numpy as np
-from yt.utilities.file_handler import HDF5FileHandler
from yt.utilities.logger import ytLogger as mylog
openPMD = "1.0.0"
@@ -31,23 +31,15 @@
ext_list = [["ED-PIC", np.uint32(1)]]
-def open_file(file_name):
- try:
- f = HDF5FileHandler.File(file_name, "r")
- return (f)
- except:
- raise
-
-
def get_attr(f, name):
"""
Try to access the path `name` in the file `f`
Return the corresponding attribute if it is present
"""
if name in list(f.attrs.keys()):
- return (True, f.attrs[name])
+ return(True, f.attrs[name])
else:
- return (False, None)
+ return(False, None)
def test_record(g, r):
@@ -68,22 +60,22 @@
- The first element is 1 if an error occured, and 0 otherwise
- The second element is 0 if a warning arised, and 0 otherwise
"""
- regEx = re.compile("^\w+$") # Python3 only: re.ASCII
+ regEx = re.compile("^\w+$") # Python3 only: re.ASCII
if regEx.match(r):
# test component names
- result_array = np.array([0, 0])
- if not is_scalar_record(g[r]):
+ result_array = np.array([0,0])
+ if not is_scalar_record(g[r]) :
for component_name in g[r]:
if not regEx.match(component_name):
- mylog.warning("open_pmd: Component %s of record %s is NOT" \
- " named properly (a-Z0-9_)!" % (component_name, g[r].name))
- result_array += np.array([1, 0])
+ mylog.warning("open_pmd - Component %s of record %s is NOT" \
+ " named properly (a-Z0-9_)!" %(component_name, g[r].name) )
+ result_array += np.array([1,0])
else:
- mylog.warning("open_pmd: Record %s is NOT named properly (a-Z0-9_)!" \
- % (r.name))
- result_array = np.array([1, 0])
+ mylog.warning("open_pmd - Record %s is NOT named properly (a-Z0-9_)!" \
+ %(r.name) )
+ result_array = np.array([1,0])
- return (result_array)
+ return(result_array)
def test_key(f, v, request, name):
@@ -116,26 +108,26 @@
valid = (name in list(f.keys()))
if valid:
if v:
- mylog.info("Key %s (%s) exists in `%s`!" % (name, request, str(f.name)))
- result_array = np.array([0, 0])
+ mylog.info("open_pmd - Key %s (%s) exists in `%s`!" %(name, request, str(f.name) ) )
+ result_array = np.array([0,0])
else:
if request == "required":
- mylog.warning("open_pmd: Key %s (%s) does NOT exist in `%s`!" \
- % (name, request, str(f.name)))
+ mylog.warning("open_pmd - Key %s (%s) does NOT exist in `%s`!" \
+ %(name, request, str(f.name)) )
result_array = np.array([1, 0])
elif request == "recommended":
- mylog.info("open_pmd: Key %s (%s) does NOT exist in `%s`!" \
- % (name, request, str(f.name)))
+ mylog.info("open_pmd - Key %s (%s) does NOT exist in `%s`!" \
+ %(name, request, str(f.name)) )
result_array = np.array([0, 1])
elif request == "optional":
if v:
- mylog.info("open_pmd: Key %s (%s) does NOT exist in `%s`!" \
- % (name, request, str(f.name)))
+ mylog.info("open_pmd - Key %s (%s) does NOT exist in `%s`!" \
+ %(name, request, str(f.name)) )
result_array = np.array([0, 0])
else:
- raise ValueError("Unrecognized string for `request` : %s" % request)
+ raise ValueError("Unrecognized string for `request` : %s" %request)
- return (result_array)
+ return(result_array)
def test_attr(f, v, request, name, is_type=None, type_format=None):
@@ -177,13 +169,13 @@
valid, value = get_attr(f, name)
if valid:
if v:
- mylog.info("open_pmd: Attribute %s (%s) exists in `%s`! Type = %s, Value = %s" \
- % (name, request, str(f.name), type(value), str(value)))
+ mylog.info("open_pmd - Attribute %s (%s) exists in `%s`! Type = %s, Value = %s" \
+ %(name, request, str(f.name), type(value), str(value)) )
# test type
if is_type is not None:
if not type_format is None and not is_type is np.string_ and \
- not isinstance(type_format, collections.Iterable):
+ not isinstance(type_format, collections.Iterable):
type_format = [type_format]
type_format_names = map(lambda x: x.__name__, type_format)
if not is_type is None and not isinstance(is_type, collections.Iterable):
@@ -193,54 +185,54 @@
if type(value) in is_type:
# np.string_ format or general ndarray dtype text
if type(value) is np.string_ and type_format is not None:
- regEx = re.compile(type_format) # Python3 only: re.ASCII
- if regEx.match(value.decode()):
- result_array = np.array([0, 0])
+ regEx = re.compile(type_format) # Python3 only: re.ASCII
+ if regEx.match(value.decode()) :
+ result_array = np.array([0,0])
else:
- mylog.warning("open_pmd: Attribute %s in `%s` does not satisfy " \
- "format ('%s' should be in format '%s')!" \
- % (name, str(f.name), value.decode(), type_format))
- result_array = np.array([1, 0])
+ mylog.warning("open_pmd - Attribute %s in `%s` does not satisfy " \
+ "format ('%s' should be in format '%s')!" \
+ %(name, str(f.name), value.decode(), type_format ) )
+ result_array = np.array([1,0])
# ndarray dtypes
elif type(value) is np.ndarray:
if value.dtype.type in type_format:
- result_array = np.array([0, 0])
+ result_array = np.array([0,0])
elif type_format is None:
- result_array = np.array([0, 0])
+ result_array = np.array([0,0])
else:
- mylog.warning("open_pmd: Attribute %s in `%s` is not of type " \
- "ndarray of '%s' (is ndarray of '%s')!" \
- % (name, str(f.name), type_format_names, \
- value.dtype.type.__name__))
- result_array = np.array([1, 0])
+ mylog.warning("open_pmd - Attribute %s in `%s` is not of type " \
+ "ndarray of '%s' (is ndarray of '%s')!" \
+ %(name, str(f.name), type_format_names, \
+ value.dtype.type.__name__) )
+ result_array = np.array([1,0])
else:
- result_array = np.array([0, 0])
+ result_array = np.array([0,0])
else:
mylog.error(
- "Error: Attribute %s in `%s` is not of type '%s' (is '%s')!" \
- % (name, str(f.name), str(is_type_names), \
- type(value).__name__))
- result_array = np.array([1, 0])
- else: # is_type is None (== arbitrary)
- result_array = np.array([0, 0])
+ "open_pmd - Attribute %s in `%s` is not of type '%s' (is '%s')!" \
+ %(name, str(f.name), str(is_type_names), \
+ type(value).__name__) )
+ result_array = np.array([1,0])
+ else: # is_type is None (== arbitrary)
+ result_array = np.array([0,0])
else:
if request == "required":
- mylog.warning("open_pmd: Attribute %s (%s) does NOT exist in `%s`!" \
- % (name, request, str(f.name)))
+ mylog.warning("open_pmd - Attribute %s (%s) does NOT exist in `%s`!" \
+ %(name, request, str(f.name)) )
result_array = np.array([1, 0])
elif request == "recommended":
- mylog.info("open_pmd: Attribute %s (%s) does NOT exist in `%s`!" \
- % (name, request, str(f.name)))
+ mylog.info("open_pmd - Attribute %s (%s) does NOT exist in `%s`!" \
+ %(name, request, str(f.name)) )
result_array = np.array([0, 1])
elif request == "optional":
if v:
- mylog.info("open_pmd: Attribute %s (%s) does NOT exist in `%s`!" \
- % (name, request, str(f.name)))
+ mylog.info("open_pmd - Attribute %s (%s) does NOT exist in `%s`!" \
+ %(name, request, str(f.name)) )
result_array = np.array([0, 0])
- else:
- raise ValueError("Unrecognized string for `request` : %s" % request)
+ else :
+ raise ValueError("Unrecognized string for `request` : %s" %request)
- return (result_array)
+ return(result_array)
def is_scalar_record(r):
@@ -257,21 +249,21 @@
bool : true if the record is a scalar record, false if the record
is either a vector or an other type of tensor record
"""
- if type(r) is h5.Group:
+ if type(r) is h5.Group :
# now it could be either a vector/tensor record
# or a scalar record with a constant component
valid, value = get_attr(r, "value")
# constant components require a "value" and a "shape" attribute
- if valid:
+ if valid :
return True
else:
return False
- else:
+ else :
return True
-def test_component(c, v):
+def test_component(c, v) :
"""
Checks if a record component defines all required attributes.
@@ -292,18 +284,18 @@
# Initialize the result array
# First element : number of errors
# Second element : number of warnings
- result_array = np.array([0, 0])
+ result_array = np.array([0,0])
- if type(c) is h5.Group:
+ if type(c) is h5.Group :
# since this check tests components, this must be a constant
# component: requires "value" and "shape" attributes
- result_array += test_attr(c, v, "required", "value") # type can be arbitrary
+ result_array += test_attr(c, v, "required", "value") # type can be arbitrary
result_array += test_attr(c, v, "required", "shape", np.ndarray, np.uint64)
# default attributes for all components
result_array += test_attr(c, v, "required", "unitSI", np.float64)
- return (result_array)
+ return(result_array)
def check_root_attr(f, v, pic):
@@ -330,7 +322,7 @@
# Initialize the result array
# First element : number of errors
# Second element : number of warnings
- result_array = np.array([0, 0])
+ result_array = np.array([0,0])
# STANDARD.md
# required
@@ -343,20 +335,20 @@
result_array += test_attr(f, v, "required", "iterationFormat", np.string_)
# groupBased iteration encoding needs to match basePath
- if result_array[0] == 0:
- if f.attrs["iterationEncoding"].decode() == "groupBased":
- if f.attrs["iterationFormat"].decode() != f.attrs["basePath"].decode():
- mylog.warning("open_pmd: for groupBased iterationEncoding the basePath "
- "and iterationFormat must match!")
- result_array += np.array([1, 0])
+ if result_array[0] == 0 :
+ if f.attrs["iterationEncoding"].decode() == "groupBased" :
+ if f.attrs["iterationFormat"].decode() != f.attrs["basePath"].decode() :
+ mylog.warning("open_pmd - for groupBased iterationEncoding the basePath "
+ "and iterationFormat must match!")
+ result_array += np.array([1,0])
- # recommended
+ # recommended
result_array += test_attr(f, v, "recommended", "author", np.string_)
result_array += test_attr(f, v, "recommended", "software", np.string_)
result_array += test_attr(f, v, "recommended",
"softwareVersion", np.string_)
result_array += test_attr(f, v, "recommended", "date", np.string_,
- "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [\+|-][0-9]{4}$")
+ "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [\+|-][0-9]{4}$")
# optional
result_array += test_attr(f, v, "optional", "comment", np.string_)
@@ -366,15 +358,15 @@
valid, extensionIDs = get_attr(f, "openPMDextension")
if valid:
if (ext_list[0][1] & extensionIDs) != extensionIDs:
- mylog.warning("open_pmd: ID=%s for extension `%s` not found in " \
- "`openPMDextension` (is %s)!" \
- % (ext_list[0][1], ext_list[0][0], extensionIDs))
- result_array += np.array([1, 0])
+ mylog.warning("open_pmd - ID=%s for extension `%s` not found in " \
+ "`openPMDextension` (is %s)!" \
+ %(ext_list[0][1], ext_list[0][0], extensionIDs) )
+ result_array += np.array([1,0])
- return (result_array)
+ return(result_array)
-def check_iterations(f, v, pic):
+def check_iterations(f, v, pic) :
"""
Scan all the iterations present in the file, checking both
the meshes and the particles
@@ -399,39 +391,39 @@
# Find all the iterations
format_error = False
- try:
+ try :
list_iterations = list(f['/data/'].keys())
- except KeyError:
+ except KeyError :
format_error = True
- else:
+ else :
# Check that these iterations are indeed encoded as integers
- for iteration in list_iterations:
- for character in iteration: # go through the string
- if not (character in string.digits):
+ for iteration in list_iterations :
+ for character in iteration : # go through the string
+ if not (character in string.digits) :
format_error = True
# Detect any error and interrupt execution if one is found
- if format_error == True:
- mylog.warning("open_pmd: it seems that the path of the data within the HDF5 file "
- "is not of the form '/data/%T/', where %T corresponds to an "
- "actual integer.")
- return (np.array([1, 0]))
- else:
- mylog.info("open_pmd: Found %d iteration(s)" % len(list_iterations))
+ if format_error == True :
+ mylog.warning("open_pmd - it seems that the path of the data within the HDF5 file "
+ "is not of the form '/data/%T/', where %T corresponds to an "
+ "actual integer.")
+ return(np.array([1, 0]))
+ else :
+ mylog.info("open_pmd - Found %d iteration(s)" % len(list_iterations) )
# Initialize the result array
# First element : number of errors
# Second element : number of warnings
- result_array = np.array([0, 0])
+ result_array = np.array([ 0, 0])
# Loop over the iterations and check the meshes and the particles
- for iteration in list_iterations:
+ for iteration in list_iterations :
result_array += check_base_path(f, iteration, v, pic)
# Go deeper only if there is no error at this point
- if result_array[0] == 0:
+ if result_array[0] == 0 :
result_array += check_meshes(f, iteration, v, pic)
result_array += check_particles(f, iteration, v, pic)
- return (result_array)
+ return(result_array)
def check_base_path(f, iteration, v, pic):
@@ -461,7 +453,7 @@
# Initialize the result array
# First element : number of errors
# Second element : number of warnings
- result_array = np.array([0, 0])
+ result_array = np.array([ 0, 0])
# Find the path to the data
base_path = ("/data/%s/" % iteration).encode('ascii')
@@ -472,7 +464,7 @@
result_array += test_attr(bp, v, "required", "dt", [np.float32, np.float64])
result_array += test_attr(bp, v, "required", "timeUnitSI", np.float64)
- return (result_array)
+ return(result_array)
def check_meshes(f, iteration, v, pic):
@@ -502,20 +494,20 @@
# Initialize the result array
# First element : number of errors
# Second element : number of warnings
- result_array = np.array([0, 0])
+ result_array = np.array([ 0, 0])
# Find the path to the data
base_path = "/data/%s/" % iteration
valid, meshes_path = get_attr(f, "meshesPath")
- if not valid:
- mylog.warning("open_pmd: `meshesPath` is missing or malformed in '/'")
- return (np.array([1, 0]))
+ if not valid :
+ mylog.warning("open_pmd - `meshesPath` is missing or malformed in '/'")
+ return( np.array([1, 0]) )
meshes_path = meshes_path.decode()
- if os.path.join(base_path, meshes_path) != (base_path + meshes_path):
- mylog.warning("open_pmd: `basePath`+`meshesPath` seems to be malformed "
- "(is `basePath` absolute and ends on a `/` ?)")
- return (np.array([1, 0]))
+ if os.path.join( base_path, meshes_path) != ( base_path + meshes_path ):
+ mylog.warning("open_pmd - `basePath`+`meshesPath` seems to be malformed "
+ "(is `basePath` absolute and ends on a `/` ?)")
+ return( np.array([1, 0]) )
else:
full_meshes_path = (base_path + meshes_path).encode('ascii')
# Find all the meshes
@@ -523,11 +515,11 @@
list_meshes = list(f[full_meshes_path].keys())
except KeyError:
list_meshes = []
- mylog.debug("Iteration %s : found %d meshes"
- % (iteration, len(list_meshes)))
+ mylog.debug( "open_pmd - Iteration %s : found %d meshes"
+ %( iteration, len(list_meshes) ) )
# Check for the attributes of the STANDARD.md
- for field_name in list_meshes:
+ for field_name in list_meshes :
field = f[full_meshes_path + field_name.encode('ascii')]
result_array += test_record(f[full_meshes_path], field_name)
@@ -551,26 +543,26 @@
geometry_test = test_attr(field, v, "required", "geometry", np.string_)
result_array += geometry_test
# geometryParameters is required when using thetaMode
- if geometry_test[0] == 0 and field.attrs["geometry"] == b"thetaMode":
+ if geometry_test[0] == 0 and field.attrs["geometry"] == b"thetaMode" :
result_array += test_attr(field, v, "required",
- "geometryParameters", np.string_)
+ "geometryParameters", np.string_)
# otherwise it is optional
- else:
+ else :
result_array += test_attr(field, v, "optional",
- "geometryParameters", np.string_)
+ "geometryParameters", np.string_)
# Attributes of the record's components
- if is_scalar_record(field): # If the record is a scalar field
+ if is_scalar_record(field) : # If the record is a scalar field
result_array += test_component(field, v)
result_array += test_attr(field, v,
- "required", "position", np.ndarray, [np.float32, np.float64])
- else: # If the record is a vector field
+ "required", "position", np.ndarray, [np.float32, np.float64])
+ else: # If the record is a vector field
# Loop over the components
- for component_name in list(field.keys()):
+ for component_name in list(field.keys()) :
component = field[component_name]
result_array += test_component(component, v)
result_array += test_attr(component, v,
- "required", "position", np.ndarray, [np.float32, np.float64])
+ "required", "position", np.ndarray, [np.float32, np.float64])
# Check for the attributes of the PIC extension,
# if asked to do so by the user
@@ -580,33 +572,33 @@
result_array += test_attr(f[full_meshes_path], v, "required",
"fieldSolver", np.string_)
valid, field_solver = get_attr(f[full_meshes_path], "fieldSolver")
- if (valid == True) and (field_solver in ["other", "GPSTD"]):
+ if (valid == True) and (field_solver in ["other", "GPSTD"]) :
result_array += test_attr(f[full_meshes_path], v, "required",
"fieldSolverParameters", np.string_)
# Check for the attributes associated with the field boundaries
result_array += test_attr(f[full_meshes_path], v, "required",
- "fieldBoundary", np.ndarray, np.string_)
+ "fieldBoundary", np.ndarray, np.string_)
valid, field_boundary = get_attr(f[full_meshes_path], "fieldBoundary")
- if (valid == True) and (np.any(field_boundary == b"other")):
+ if (valid == True) and (np.any(field_boundary == b"other")) :
result_array += test_attr(f[full_meshes_path], v, "required",
- "fieldBoundaryParameters", np.ndarray, np.string_)
+ "fieldBoundaryParameters", np.ndarray, np.string_)
# Check for the attributes associated with the field boundaries
result_array += test_attr(f[full_meshes_path], v, "required",
- "particleBoundary", np.ndarray, np.string_)
+ "particleBoundary", np.ndarray, np.string_)
valid, particle_boundary = get_attr(f[full_meshes_path], "particleBoundary")
- if (valid == True) and (np.any(particle_boundary == b"other")):
+ if (valid == True) and (np.any(particle_boundary == b"other")) :
result_array += test_attr(f[full_meshes_path], v, "required",
- "particleBoundaryParameters", np.ndarray, np.string_)
+ "particleBoundaryParameters", np.ndarray, np.string_)
# Check the attributes associated with the current smoothing
result_array += test_attr(f[full_meshes_path], v, "required",
"currentSmoothing", np.string_)
valid, current_smoothing = get_attr(f[full_meshes_path], "currentSmoothing")
- if (valid == True) and (current_smoothing != b"none"):
+ if (valid == True) and (current_smoothing != b"none") :
result_array += test_attr(f[full_meshes_path], v, "required",
- "currentSmoothingParameters", np.string_)
+ "currentSmoothingParameters", np.string_)
# Check the attributes associated with the charge conservation
result_array += test_attr(f[full_meshes_path], v, "required",
@@ -614,21 +606,21 @@
valid, charge_correction = get_attr(f[full_meshes_path], "chargeCorrection")
if valid == True and charge_correction != b"none":
result_array += test_attr(f[full_meshes_path], v, "required",
- "chargeCorrectionParameters", np.string_)
+ "chargeCorrectionParameters", np.string_)
# Check for the attributes of each record
- for field_name in list_meshes:
+ for field_name in list_meshes :
field = f[full_meshes_path + field_name.encode('ascii')]
result_array + test_attr(field, v, "required",
"fieldSmoothing", np.string_)
valid, field_smoothing = get_attr(field, "fieldSmoothing")
- if (valid == True) and (field_smoothing != b"none"):
- result_array += test_attr(field, v, "required",
- "fieldSmoothingParameters", np.string_)
- return (result_array)
+ if (valid == True) and (field_smoothing != b"none") :
+ result_array += test_attr(field,v, "required",
+ "fieldSmoothingParameters", np.string_)
+ return(result_array)
-def check_particles(f, iteration, v, pic):
+def check_particles(f, iteration, v, pic) :
"""
Scan all the particle data corresponding to one iteration
@@ -655,16 +647,16 @@
# Initialize the result array
# First element : number of errors
# Second element : number of warnings
- result_array = np.array([0, 0])
+ result_array = np.array([ 0, 0])
# Find the path to the data
base_path = ("/data/%s/" % iteration).encode('ascii')
valid, particles_path = get_attr(f, "particlesPath")
- if os.path.join(base_path, particles_path) != \
- (base_path + particles_path):
- mylog.warning("open_pmd: `basePath`+`meshesPath` seems to be malformed "
- "(is `basePath` absolute and ends on a `/` ?)")
- return (np.array([1, 0]))
+ if os.path.join( base_path, particles_path) != \
+ ( base_path + particles_path ) :
+ mylog.warning("open_pmd - `basePath`+`meshesPath` seems to be malformed "
+ "(is `basePath` absolute and ends on a `/` ?)")
+ return( np.array([1, 0]) )
else:
full_particle_path = base_path + particles_path
# Find all the particle species
@@ -672,15 +664,15 @@
list_species = list(f[full_particle_path].keys())
except KeyError:
list_species = []
- mylog.debug("Iteration %s : found %d particle species"
- % (iteration, len(list_species)))
+ mylog.debug( "open_pmd - Iteration %s : found %d particle species"
+ %( iteration, len(list_species) ) )
# Go through all the particle species
- for species_name in list_species:
+ for species_name in list_species :
species = f[full_particle_path + species_name.encode('ascii')]
# Check all records for this species
- for species_record_name in species:
+ for species_record_name in species :
result_array += test_record(species, species_record_name)
# Check the position record of the particles
@@ -688,22 +680,22 @@
# Check the position offset record of the particles
result_array += test_key(species, v, "required", "positionOffset")
- if result_array[0] == 0:
+ if result_array[0] == 0 :
position_dimensions = len(species["position"].keys())
positionOffset_dimensions = len(species["positionOffset"].keys())
- if position_dimensions != positionOffset_dimensions:
- mylog.warning("open_pmd: `position` (ndim=%s) and `positionOffset` " \
- "(ndim=%s) do not have the same dimensions in " \
- "species `%s`!" \
- % (str(position_dimensions), \
- str(positionOffset_dimensions),
- species.name))
- result_array += np.array([1, 0])
+ if position_dimensions != positionOffset_dimensions :
+ mylog.warning("open_pmd - `position` (ndim=%s) and `positionOffset` " \
+ "(ndim=%s) do not have the same dimensions in " \
+ "species `%s`!" \
+ %(str(position_dimensions), \
+ str(positionOffset_dimensions),
+ species.name) )
+ result_array += np.array([ 1, 0])
# Check the particlePatches record of the particles
patch_test = test_key(species, v, "recommended", "particlePatches")
result_array += patch_test
- if result_array[0] == 0 and patch_test[1] == 0:
+ if result_array[0] == 0 and patch_test[1] == 0 :
result_array += test_key(species["particlePatches"], v, "required",
"numParticles")
result_array += test_key(species["particlePatches"], v, "required",
@@ -712,23 +704,23 @@
"offset")
result_array += test_key(species["particlePatches"], v, "required",
"extent")
- if result_array[0] == 0:
+ if result_array[0] == 0 :
offset = species["particlePatches"]["offset"]
extent = species["particlePatches"]["extent"]
# Attributes of the components
- for component_name in list(species["position"].keys()):
- result_array += test_key(offset, v, "required",
- component_name)
- result_array += test_key(extent, v, "required",
- component_name)
- if result_array[0] == 0:
+ for component_name in list(species["position"].keys()) :
+ result_array += test_key( offset, v, "required",
+ component_name)
+ result_array += test_key( extent, v, "required",
+ component_name)
+ if result_array[0] == 0 :
dset_offset = offset[component_name]
result_array += test_component(dset_offset, v)
dset_extent = extent[component_name]
result_array += test_component(dset_extent, v)
# Check the records required by the PIC extension
- if pic:
+ if pic :
result_array += test_key(species, v, "required", "momentum")
result_array += test_key(species, v, "required", "charge")
result_array += test_key(species, v, "required", "mass")
@@ -738,7 +730,7 @@
result_array += test_key(species, v, "optional", "neutronNumber")
# Check the attributes associated with the PIC extension
- if pic:
+ if pic :
result_array += test_attr(species, v, "required",
"particleShape", [np.float32, np.float64])
result_array += test_attr(species, v, "required",
@@ -754,14 +746,14 @@
valid, particle_smoothing = get_attr(species, "particleSmoothing")
if valid == True and particle_smoothing != b"none":
result_array += test_attr(species, v, "required",
- "particleSmoothingParameters", np.string_)
+ "particleSmoothingParameters", np.string_)
# Check attributes of each record of the particle
for record in list(species.keys()):
# all records (but particlePatches) require units
if record != "particlePatches":
result_array += test_attr(species[record], v,
- "required", "unitDimension", np.ndarray, np.float64)
+ "required", "unitDimension", np.ndarray, np.float64)
time_type = f[base_path].attrs["time"].dtype.type
result_array += test_attr(species[record], v, "required",
"timeOffset", time_type)
@@ -784,8 +776,41 @@
def parse_unit_dimension(unit_dimension):
+ """Transforms an openPMD unitDimension into a string.
+
+ Parameters
+ ----------
+ unit_dimension : array_like
+ integer array of length 7 with one entry for the dimensional component of every SI unit
+
+ [0] length L,
+ [1] mass M,
+ [2] time T,
+ [3] electric current I,
+ [4] thermodynamic temperature theta,
+ [5] amount of substance N,
+ [6] luminous intensity J
+
+ References
+ ----------
+ .. https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md#unit-systems-and-dimensionality
+
+ Returns
+ -------
+ str
+
+ Examples
+ --------
+ >>> velocity = [1., 0., -1., 0., 0., 0., 0.]
+ >>> print parse_unit_dimension(velocity)
+ 'm**1*s**-1'
+
+ >>> magnetic_field = [0., 1., -2., -1., 0., 0., 0.]
+ >>> print parse_unit_dimension(magnetic_field)
+ 'kg**1*s**-2*A**-1'
+ """
if len(unit_dimension) is not 7:
- mylog.error("SI must have 7 base dimensions!")
+ mylog.error("open_pmd - SI must have 7 base dimensions!")
unit_dimension = np.asarray(unit_dimension, dtype='int')
dim = []
si = ["m",
https://bitbucket.org/yt_analysis/yt/commits/4310439cba3f/
Changeset: 4310439cba3f
Branch: yt
User: Fabian Koller
Date: 2016-08-14 15:54:08+00:00
Summary: import h5py through yt
Affected #: 3 files
diff -r 7c1974a7c3973278e3426b590aecebadb1eb9cd7 -r 4310439cba3f4b685f1c10b153e326d9a25cbe6b yt/frontends/open_pmd/data_structures.py
--- a/yt/frontends/open_pmd/data_structures.py
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -18,7 +18,6 @@
import re
from math import ceil, floor
-import h5py as h5
import numpy as np
from yt.data_objects.grid_patch import AMRGridPatch
@@ -28,6 +27,7 @@
from yt.geometry.grid_geometry_handler import GridIndex
from yt.utilities.file_handler import HDF5FileHandler
from yt.utilities.logger import ytLogger as mylog
+from yt.utilities.on_demand_imports import _h5py as h5
class OpenPMDGrid(AMRGridPatch):
diff -r 7c1974a7c3973278e3426b590aecebadb1eb9cd7 -r 4310439cba3f4b685f1c10b153e326d9a25cbe6b yt/frontends/open_pmd/fields.py
--- a/yt/frontends/open_pmd/fields.py
+++ b/yt/frontends/open_pmd/fields.py
@@ -15,7 +15,6 @@
# The full license is in the file COPYING.txt, distributed with this software.
# -----------------------------------------------------------------------------
-import h5py as h5
import numpy as np
from yt.fields.field_info_container import FieldInfoContainer
@@ -23,6 +22,7 @@
from yt.frontends.open_pmd.misc import parse_unit_dimension
from yt.units.yt_array import YTQuantity
from yt.utilities.logger import ytLogger as mylog
+from yt.utilities.on_demand_imports import _h5py as h5
from yt.utilities.physical_constants import speed_of_light
diff -r 7c1974a7c3973278e3426b590aecebadb1eb9cd7 -r 4310439cba3f4b685f1c10b153e326d9a25cbe6b yt/frontends/open_pmd/misc.py
--- a/yt/frontends/open_pmd/misc.py
+++ b/yt/frontends/open_pmd/misc.py
@@ -21,10 +21,10 @@
import re
import string
-import h5py as h5
import numpy as np
from yt.utilities.logger import ytLogger as mylog
+from yt.utilities.on_demand_imports import _h5py as h5
openPMD = "1.0.0"
https://bitbucket.org/yt_analysis/yt/commits/dd823d27c61f/
Changeset: dd823d27c61f
Branch: yt
User: Fabian Koller
Date: 2016-08-15 17:03:08+00:00
Summary: make log less verbose
Affected #: 2 files
diff -r 4310439cba3f4b685f1c10b153e326d9a25cbe6b -r dd823d27c61f96f3149ec41233358e46be7a2642 yt/frontends/open_pmd/io.py
--- a/yt/frontends/open_pmd/io.py
+++ b/yt/frontends/open_pmd/io.py
@@ -192,7 +192,7 @@
keys are tuples (ftype, fname) representing a field
values are flat (``size``,) ndarrays with data from that field
"""
- mylog.info("open_pmd - _read_fluid_selection {} {} {} {}".format(chunks, selector, fields, size))
+ mylog.debug("open_pmd - _read_fluid_selection {} {} {} {}".format(chunks, selector, fields, size))
f = self._handle
bp = self.base_path
mp = self.meshes_path
diff -r 4310439cba3f4b685f1c10b153e326d9a25cbe6b -r dd823d27c61f96f3149ec41233358e46be7a2642 yt/frontends/open_pmd/misc.py
--- a/yt/frontends/open_pmd/misc.py
+++ b/yt/frontends/open_pmd/misc.py
@@ -408,7 +408,7 @@
"actual integer.")
return(np.array([1, 0]))
else :
- mylog.info("open_pmd - Found %d iteration(s)" % len(list_iterations) )
+ mylog.debug("open_pmd - Found %d iteration(s) in file" % len(list_iterations) )
# Initialize the result array
# First element : number of errors
https://bitbucket.org/yt_analysis/yt/commits/dc83e274e7bb/
Changeset: dc83e274e7bb
Branch: yt
User: Fabian Koller
Date: 2016-08-16 08:21:58+00:00
Summary: Change equality check for h5py dataset/group since yt's _h5py only offers group
Affected #: 1 file
diff -r dd823d27c61f96f3149ec41233358e46be7a2642 -r dc83e274e7bbdd1849ca739937f17fc45ea7c18c yt/frontends/open_pmd/fields.py
--- a/yt/frontends/open_pmd/fields.py
+++ b/yt/frontends/open_pmd/fields.py
@@ -143,7 +143,7 @@
for fname in fields.keys():
field = fields[fname]
- if type(field) is h5.Dataset:
+ if type(field) is not h5.Group: # yt.utilities.on_demand_imports._h5py only has Group
# Don't consider axes. This appears to be a vector field of single dimensionality
ytname = str("_".join([fname.replace("_", "-")]))
if ds._nonstandard:
@@ -198,7 +198,7 @@
# particle_position is later derived in setup_absolute_positions in the way yt expects it
ytattrib = "positionCoarse"
pds = particles[species + "/" + attrib]
- if type(pds) is h5.Dataset:
+ if type(field) is not h5.Group: # yt.utilities.on_demand_imports._h5py only has Group
particle_fields += ((str("_".join(name)), (unit, [], None)),)
else:
for axis in pds.keys():
https://bitbucket.org/yt_analysis/yt/commits/5f143a0d34b6/
Changeset: 5f143a0d34b6
Branch: yt
User: Fabian Koller
Date: 2016-08-16 09:25:41+00:00
Summary: Revert to manual h5py check
Affected #: 1 file
diff -r dc83e274e7bbdd1849ca739937f17fc45ea7c18c -r 5f143a0d34b641455abaa3ac116bbf02dd0fc2a4 yt/frontends/open_pmd/fields.py
--- a/yt/frontends/open_pmd/fields.py
+++ b/yt/frontends/open_pmd/fields.py
@@ -16,13 +16,13 @@
# -----------------------------------------------------------------------------
import numpy as np
+import h5py as h5
from yt.fields.field_info_container import FieldInfoContainer
from yt.fields.magnetic_field import setup_magnetic_field_aliases
from yt.frontends.open_pmd.misc import parse_unit_dimension
from yt.units.yt_array import YTQuantity
from yt.utilities.logger import ytLogger as mylog
-from yt.utilities.on_demand_imports import _h5py as h5
from yt.utilities.physical_constants import speed_of_light
@@ -143,7 +143,7 @@
for fname in fields.keys():
field = fields[fname]
- if type(field) is not h5.Group: # yt.utilities.on_demand_imports._h5py only has Group
+ if type(field) is h5.Dataset:
# Don't consider axes. This appears to be a vector field of single dimensionality
ytname = str("_".join([fname.replace("_", "-")]))
if ds._nonstandard:
@@ -198,7 +198,7 @@
# particle_position is later derived in setup_absolute_positions in the way yt expects it
ytattrib = "positionCoarse"
pds = particles[species + "/" + attrib]
- if type(field) is not h5.Group: # yt.utilities.on_demand_imports._h5py only has Group
+ if type(pds) is h5.Dataset:
particle_fields += ((str("_".join(name)), (unit, [], None)),)
else:
for axis in pds.keys():
https://bitbucket.org/yt_analysis/yt/commits/5ea6b6e8b460/
Changeset: 5ea6b6e8b460
Branch: yt
User: Fabian Koller
Date: 2016-08-16 13:10:40+00:00
Summary: Do not attempt to read mesh selection if the mask is none
Affected #: 1 file
diff -r 5f143a0d34b641455abaa3ac116bbf02dd0fc2a4 -r 5ea6b6e8b4606f8c7fcfca3bcc8114738dc03a6c yt/frontends/open_pmd/io.py
--- a/yt/frontends/open_pmd/io.py
+++ b/yt/frontends/open_pmd/io.py
@@ -196,6 +196,7 @@
f = self._handle
bp = self.base_path
mp = self.meshes_path
+ ds = f[bp + mp]
rv = {}
chunks = list(chunks)
@@ -215,17 +216,17 @@
field = (ftype, fname)
for chunk in chunks:
for g in chunk.objs:
- ds = f[bp + mp]
- component = fname.replace("_", "/").replace("-", "_")
- index = g.mesh_index
- offset = g.mesh_offset
- data = np.array(self.get_component(ds, component, index, offset, self.ds._nonstandard))
- # The following is a modified AMRGridPatch.select(...)
mask = g._get_selector_mask(selector)
- data.shape = mask.shape # Workaround - casts a 2D (x,y) array to 3D (x,y,1)
- count = g.count(selector)
- rv[field][ind[field]:ind[field] + count] = data[mask]
- ind[field] += count
+ if mask is not None:
+ component = fname.replace("_", "/").replace("-", "_")
+ index = g.mesh_index
+ offset = g.mesh_offset
+ data = np.array(self.get_component(ds, component, index, offset, self.ds._nonstandard))
+ # The following is a modified AMRGridPatch.select(...)
+ data.shape = mask.shape # Workaround - casts a 2D (x,y) array to 3D (x,y,1)
+ count = g.count(selector)
+ rv[field][ind[field]:ind[field] + count] = data[mask]
+ ind[field] += count
for i in rv:
rv[i].flatten()
return rv
https://bitbucket.org/yt_analysis/yt/commits/4422797274ad/
Changeset: 4422797274ad
Branch: yt
User: Fabian Koller
Date: 2016-08-18 08:42:45+00:00
Summary: Change the mechanism for setting default unit attributes,
calculate time with timeUnitSI,
simplify validity check for ease of maintenance
Affected #: 2 files
diff -r 5ea6b6e8b4606f8c7fcfca3bcc8114738dc03a6c -r 4422797274ad4947fcf9a44caf4a2f11ae1f6161 yt/frontends/open_pmd/data_structures.py
--- a/yt/frontends/open_pmd/data_structures.py
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -22,8 +22,9 @@
from yt.data_objects.grid_patch import AMRGridPatch
from yt.data_objects.static_output import Dataset
+from yt.funcs import setdefaultattr
from yt.frontends.open_pmd.fields import OpenPMDFieldInfo
-from yt.frontends.open_pmd.misc import is_const_component, check_root_attr, check_iterations
+from yt.frontends.open_pmd.misc import is_const_component
from yt.geometry.grid_geometry_handler import GridIndex
from yt.utilities.file_handler import HDF5FileHandler
from yt.utilities.logger import ytLogger as mylog
@@ -321,11 +322,11 @@
These are hardcoded as 1.0. Every dataset in openPMD can have different code <-> physical scaling.
The individual factor is obtained by multiplying with "unitSI" reading getting data from disk.
"""
- self.length_unit = self.quan(1.0, "m")
- self.mass_unit = self.quan(1.0, "kg")
- self.time_unit = self.quan(1.0, "s")
- self.velocity_unit = self.quan(1.0, "m/s")
- self.magnetic_unit = self.quan(1.0, "T")
+ setdefaultattr(self, 'length_unit', self.quan(1.0, "m"))
+ setdefaultattr(self, 'mass_unit', self.quan(1.0, "kg"))
+ setdefaultattr(self, 'time_unit', self.quan(1.0, "s"))
+ setdefaultattr(self, 'velocity_unit', self.quan(1.0, "m/s"))
+ setdefaultattr(self, 'magnetic_unit', self.quan(1.0, "T"))
def _parse_parameter_file(self):
"""Read in metadata describing the overall data on-disk.
@@ -379,7 +380,7 @@
if self._nonstandard:
self.current_time = 0
else:
- self.current_time = f[bp].attrs["time"]
+ self.current_time = f[bp].attrs["time"] * f[bp].attrs["timeUnitSI"]
@classmethod
def _is_valid(self, *args, **kwargs):
@@ -389,28 +390,16 @@
f = h5.File(args[0])
except:
return False
- verbose = False
- extension_pic = False
- # root attributes at "/"
- result_array = np.array([0, 0])
- result_array += check_root_attr(f, verbose, extension_pic)
- # Go through all the iterations, checking both the particles
- # and the meshes
- result_array += check_iterations(f, verbose, extension_pic)
+ requirements = ["openPMD", "basePath", "meshesPath", "particlesPath"]
+ for i in requirements:
+ if i not in f.attrs.keys():
+ f.close()
+ return False
- # this might still be a compatible file not fully respecting openPMD standards
- if result_array[0] != 0:
- try:
- iteration = f["/data"].keys()[0]
- if iteration.isdigit() \
- and "fields" in f["/data/" + iteration].keys() \
- and "particles" in f["/data/" + iteration].keys():
- self._nonstandard = True
- mylog.info(
- "open_pmd - Reading a file not compliant with the standard. No support will be guaranteed!")
- return True
- except:
- return False
+ if "1.0.0" != f.attrs["openPMD"]:
+ f.close()
+ return False
+
f.close()
return True
diff -r 5ea6b6e8b4606f8c7fcfca3bcc8114738dc03a6c -r 4422797274ad4947fcf9a44caf4a2f11ae1f6161 yt/frontends/open_pmd/misc.py
--- a/yt/frontends/open_pmd/misc.py
+++ b/yt/frontends/open_pmd/misc.py
@@ -1,778 +1,14 @@
# -----------------------------------------------------------------------------
-# Copyright (c) 2015, Axel Huebl, Remi Lehe
# Copyright (c) 2016, Fabian Koller (HZDR)
#
-# Permission to use, copy, modify, and/or distribute this software for any
-# purpose with or without fee is hereby granted, provided that the above
-# copyright notice and this permission notice appear in all copies.
+# Distributed under the terms of the Modified BSD License.
#
-# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
-# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
-# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
-# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
-# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
-# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
-# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-#
+# The full license is in the file COPYING.txt, distributed with this software.
# -----------------------------------------------------------------------------
-import collections # for isinstance
-import os.path
-import re
-import string
-
import numpy as np
from yt.utilities.logger import ytLogger as mylog
-from yt.utilities.on_demand_imports import _h5py as h5
-
-openPMD = "1.0.0"
-
-ext_list = [["ED-PIC", np.uint32(1)]]
-
-
-def get_attr(f, name):
- """
- Try to access the path `name` in the file `f`
- Return the corresponding attribute if it is present
- """
- if name in list(f.attrs.keys()):
- return(True, f.attrs[name])
- else:
- return(False, None)
-
-
-def test_record(g, r):
- """
- Checks if a record is valid
-
- Parameters
- ----------
- g : h5py.Group
- The group the record resides in
-
- r : string
- The name of the record.
-
- Returns
- -------
- An array with 2 elements :
- - The first element is 1 if an error occured, and 0 otherwise
- - The second element is 0 if a warning arised, and 0 otherwise
- """
- regEx = re.compile("^\w+$") # Python3 only: re.ASCII
- if regEx.match(r):
- # test component names
- result_array = np.array([0,0])
- if not is_scalar_record(g[r]) :
- for component_name in g[r]:
- if not regEx.match(component_name):
- mylog.warning("open_pmd - Component %s of record %s is NOT" \
- " named properly (a-Z0-9_)!" %(component_name, g[r].name) )
- result_array += np.array([1,0])
- else:
- mylog.warning("open_pmd - Record %s is NOT named properly (a-Z0-9_)!" \
- %(r.name) )
- result_array = np.array([1,0])
-
- return(result_array)
-
-
-def test_key(f, v, request, name):
- """
- Checks whether a key is present. A key can either be
- a h5py.Group or a h5py.Dataset.
- Returns an error if the key if absent and requested
- Returns a warning if the key if absent and recommended
-
- Parameters
- ----------
- f : an h5py.File or h5py.Group object
- The object in which to find the key
-
- v : bool
- Verbose option
-
- request : string
- Either "required", "recommended" or "optional"
-
- name : string
- The name of the key within this File, Group or DataSet
-
- Returns
- -------
- An array with 2 elements :
- - The first element is 1 if an error occured, and 0 otherwise
- - The second element is 0 if a warning arised, and 0 otherwise
- """
- valid = (name in list(f.keys()))
- if valid:
- if v:
- mylog.info("open_pmd - Key %s (%s) exists in `%s`!" %(name, request, str(f.name) ) )
- result_array = np.array([0,0])
- else:
- if request == "required":
- mylog.warning("open_pmd - Key %s (%s) does NOT exist in `%s`!" \
- %(name, request, str(f.name)) )
- result_array = np.array([1, 0])
- elif request == "recommended":
- mylog.info("open_pmd - Key %s (%s) does NOT exist in `%s`!" \
- %(name, request, str(f.name)) )
- result_array = np.array([0, 1])
- elif request == "optional":
- if v:
- mylog.info("open_pmd - Key %s (%s) does NOT exist in `%s`!" \
- %(name, request, str(f.name)) )
- result_array = np.array([0, 0])
- else:
- raise ValueError("Unrecognized string for `request` : %s" %request)
-
- return(result_array)
-
-
-def test_attr(f, v, request, name, is_type=None, type_format=None):
- """
- Checks whether an attribute is present.
- Returns an error if the attribute if absent and requested
- Returns a warning if the attribute if absent and recommanded
-
- Parameters
- ----------
- f : an h5py.File, h5py.Group or h5py.DataSet object
- The object in which to find the key
-
- v : bool
- Verbose option
-
- request : string
- Either "required", "recommended" or "optional
-
- name : string
- The name of the attribute within this File, Group or DataSet
-
- is_type : (numpy or python) data type
- The type of the attribute. Default is "arbitrary" for None.
- Can be a list of data types where at least one data type must match
- but this list can not be combined with type_format.
-
- type_format: (numpy or python) data type
- Used with is_type to specify numpy ndarray dtypes or a
- base np.string_ format regex. Can be a list of data types
- for ndarrays where at least one data type must match.
-
- Returns
- -------
- An array with 2 elements :
- - The first element is 1 if an error occured, and 0 otherwise
- - The second element is 0 if a warning arised, and 0 otherwise
- """
- valid, value = get_attr(f, name)
- if valid:
- if v:
- mylog.info("open_pmd - Attribute %s (%s) exists in `%s`! Type = %s, Value = %s" \
- %(name, request, str(f.name), type(value), str(value)) )
-
- # test type
- if is_type is not None:
- if not type_format is None and not is_type is np.string_ and \
- not isinstance(type_format, collections.Iterable):
- type_format = [type_format]
- type_format_names = map(lambda x: x.__name__, type_format)
- if not is_type is None and not isinstance(is_type, collections.Iterable):
- is_type = [is_type]
- is_type_names = map(lambda x: x.__name__, is_type)
- # add for each type in is_type -> wrong, need to add this at the comparison level!
- if type(value) in is_type:
- # np.string_ format or general ndarray dtype text
- if type(value) is np.string_ and type_format is not None:
- regEx = re.compile(type_format) # Python3 only: re.ASCII
- if regEx.match(value.decode()) :
- result_array = np.array([0,0])
- else:
- mylog.warning("open_pmd - Attribute %s in `%s` does not satisfy " \
- "format ('%s' should be in format '%s')!" \
- %(name, str(f.name), value.decode(), type_format ) )
- result_array = np.array([1,0])
- # ndarray dtypes
- elif type(value) is np.ndarray:
- if value.dtype.type in type_format:
- result_array = np.array([0,0])
- elif type_format is None:
- result_array = np.array([0,0])
- else:
- mylog.warning("open_pmd - Attribute %s in `%s` is not of type " \
- "ndarray of '%s' (is ndarray of '%s')!" \
- %(name, str(f.name), type_format_names, \
- value.dtype.type.__name__) )
- result_array = np.array([1,0])
- else:
- result_array = np.array([0,0])
- else:
- mylog.error(
- "open_pmd - Attribute %s in `%s` is not of type '%s' (is '%s')!" \
- %(name, str(f.name), str(is_type_names), \
- type(value).__name__) )
- result_array = np.array([1,0])
- else: # is_type is None (== arbitrary)
- result_array = np.array([0,0])
- else:
- if request == "required":
- mylog.warning("open_pmd - Attribute %s (%s) does NOT exist in `%s`!" \
- %(name, request, str(f.name)) )
- result_array = np.array([1, 0])
- elif request == "recommended":
- mylog.info("open_pmd - Attribute %s (%s) does NOT exist in `%s`!" \
- %(name, request, str(f.name)) )
- result_array = np.array([0, 1])
- elif request == "optional":
- if v:
- mylog.info("open_pmd - Attribute %s (%s) does NOT exist in `%s`!" \
- %(name, request, str(f.name)) )
- result_array = np.array([0, 0])
- else :
- raise ValueError("Unrecognized string for `request` : %s" %request)
-
- return(result_array)
-
-
-def is_scalar_record(r):
- """
- Checks if a record is a scalar record or not.
-
- Parameters
- ----------
- r : an h5py.Group or h5py.Dataset object
- the record that shall be tested
-
- Returns
- -------
- bool : true if the record is a scalar record, false if the record
- is either a vector or an other type of tensor record
- """
- if type(r) is h5.Group :
- # now it could be either a vector/tensor record
- # or a scalar record with a constant component
-
- valid, value = get_attr(r, "value")
- # constant components require a "value" and a "shape" attribute
- if valid :
- return True
- else:
- return False
- else :
- return True
-
-
-def test_component(c, v) :
- """
- Checks if a record component defines all required attributes.
-
- Parameters
- ----------
- c : an h5py.Group or h5py.Dataset object
- the record component that shall be tested
-
- v : bool
- Verbose option
-
- Returns
- -------
- An array with 2 elements :
- - The first element is the number of errors encountered
- - The second element is the number of warnings encountered
- """
- # Initialize the result array
- # First element : number of errors
- # Second element : number of warnings
- result_array = np.array([0,0])
-
- if type(c) is h5.Group :
- # since this check tests components, this must be a constant
- # component: requires "value" and "shape" attributes
- result_array += test_attr(c, v, "required", "value") # type can be arbitrary
- result_array += test_attr(c, v, "required", "shape", np.ndarray, np.uint64)
-
- # default attributes for all components
- result_array += test_attr(c, v, "required", "unitSI", np.float64)
-
- return(result_array)
-
-
-def check_root_attr(f, v, pic):
- """
- Scan the root of the file and make sure that all the attributes are present
-
- Parameters
- ----------
- f : an h5py.File object
- The HDF5 file in which to find the attribute
-
- v : bool
- Verbose option
-
- pic : bool
- Whether to check for the ED-PIC extension attributes
-
- Returns
- -------
- An array with 2 elements :
- - The first element is the number of errors encountered
- - The second element is the number of warnings encountered
- """
- # Initialize the result array
- # First element : number of errors
- # Second element : number of warnings
- result_array = np.array([0,0])
-
- # STANDARD.md
- # required
- result_array += test_attr(f, v, "required", "openPMD", np.string_, "^[0-9]+\.[0-9]+\.[0-9]+$")
- result_array += test_attr(f, v, "required", "openPMDextension", np.uint32)
- result_array += test_attr(f, v, "required", "basePath", np.string_, "^\/data\/\%T\/$")
- result_array += test_attr(f, v, "required", "meshesPath", np.string_)
- result_array += test_attr(f, v, "required", "particlesPath", np.string_)
- result_array += test_attr(f, v, "required", "iterationEncoding", np.string_, "^groupBased|fileBased$")
- result_array += test_attr(f, v, "required", "iterationFormat", np.string_)
-
- # groupBased iteration encoding needs to match basePath
- if result_array[0] == 0 :
- if f.attrs["iterationEncoding"].decode() == "groupBased" :
- if f.attrs["iterationFormat"].decode() != f.attrs["basePath"].decode() :
- mylog.warning("open_pmd - for groupBased iterationEncoding the basePath "
- "and iterationFormat must match!")
- result_array += np.array([1,0])
-
- # recommended
- result_array += test_attr(f, v, "recommended", "author", np.string_)
- result_array += test_attr(f, v, "recommended", "software", np.string_)
- result_array += test_attr(f, v, "recommended",
- "softwareVersion", np.string_)
- result_array += test_attr(f, v, "recommended", "date", np.string_,
- "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [\+|-][0-9]{4}$")
-
- # optional
- result_array += test_attr(f, v, "optional", "comment", np.string_)
-
- # Extension: ED-PIC
- if pic:
- valid, extensionIDs = get_attr(f, "openPMDextension")
- if valid:
- if (ext_list[0][1] & extensionIDs) != extensionIDs:
- mylog.warning("open_pmd - ID=%s for extension `%s` not found in " \
- "`openPMDextension` (is %s)!" \
- %(ext_list[0][1], ext_list[0][0], extensionIDs) )
- result_array += np.array([1,0])
-
- return(result_array)
-
-
-def check_iterations(f, v, pic) :
- """
- Scan all the iterations present in the file, checking both
- the meshes and the particles
-
- Parameters
- ----------
- f : an h5py.File object
- The HDF5 file in which to find the attribute
-
- v : bool
- Verbose option
-
- pic : bool
- Whether to check for the ED-PIC extension attributes
-
- Returns
- -------
- An array with 2 elements :
- - The first element is the number of errors encountered
- - The second element is the number of warnings encountered
- """
-
- # Find all the iterations
- format_error = False
- try :
- list_iterations = list(f['/data/'].keys())
- except KeyError :
- format_error = True
- else :
- # Check that these iterations are indeed encoded as integers
- for iteration in list_iterations :
- for character in iteration : # go through the string
- if not (character in string.digits) :
- format_error = True
- # Detect any error and interrupt execution if one is found
- if format_error == True :
- mylog.warning("open_pmd - it seems that the path of the data within the HDF5 file "
- "is not of the form '/data/%T/', where %T corresponds to an "
- "actual integer.")
- return(np.array([1, 0]))
- else :
- mylog.debug("open_pmd - Found %d iteration(s) in file" % len(list_iterations) )
-
- # Initialize the result array
- # First element : number of errors
- # Second element : number of warnings
- result_array = np.array([ 0, 0])
-
- # Loop over the iterations and check the meshes and the particles
- for iteration in list_iterations :
- result_array += check_base_path(f, iteration, v, pic)
- # Go deeper only if there is no error at this point
- if result_array[0] == 0 :
- result_array += check_meshes(f, iteration, v, pic)
- result_array += check_particles(f, iteration, v, pic)
-
- return(result_array)
-
-
-def check_base_path(f, iteration, v, pic):
- """
- Scan the base_path that corresponds to this iteration
-
- Parameters
- ----------
- f : an h5py.File object
- The HDF5 file in which to find the attribute
-
- iteration : string representing an integer
- The iteration at which to scan the meshes
-
- v : bool
- Verbose option
-
- pic : bool
- Whether to check for the ED-PIC extension attributes
-
- Returns
- -------
- An array with 2 elements :
- - The first element is the number of errors encountered
- - The second element is the number of warnings encountered
- """
- # Initialize the result array
- # First element : number of errors
- # Second element : number of warnings
- result_array = np.array([ 0, 0])
-
- # Find the path to the data
- base_path = ("/data/%s/" % iteration).encode('ascii')
- bp = f[base_path]
-
- # Check for the attributes of the STANDARD.md
- result_array += test_attr(bp, v, "required", "time", [np.float32, np.float64])
- result_array += test_attr(bp, v, "required", "dt", [np.float32, np.float64])
- result_array += test_attr(bp, v, "required", "timeUnitSI", np.float64)
-
- return(result_array)
-
-
-def check_meshes(f, iteration, v, pic):
- """
- Scan all the meshes corresponding to one iteration
-
- Parameters
- ----------
- f : an h5py.File object
- The HDF5 file in which to find the attribute
-
- iteration : string representing an integer
- The iteration at which to scan the meshes
-
- v : bool
- Verbose option
-
- pic : bool
- Whether to check for the ED-PIC extension attributes
-
- Returns
- -------
- An array with 2 elements :
- - The first element is the number of errors encountered
- - The second element is the number of warnings encountered
- """
- # Initialize the result array
- # First element : number of errors
- # Second element : number of warnings
- result_array = np.array([ 0, 0])
-
- # Find the path to the data
- base_path = "/data/%s/" % iteration
- valid, meshes_path = get_attr(f, "meshesPath")
- if not valid :
- mylog.warning("open_pmd - `meshesPath` is missing or malformed in '/'")
- return( np.array([1, 0]) )
- meshes_path = meshes_path.decode()
-
- if os.path.join( base_path, meshes_path) != ( base_path + meshes_path ):
- mylog.warning("open_pmd - `basePath`+`meshesPath` seems to be malformed "
- "(is `basePath` absolute and ends on a `/` ?)")
- return( np.array([1, 0]) )
- else:
- full_meshes_path = (base_path + meshes_path).encode('ascii')
- # Find all the meshes
- try:
- list_meshes = list(f[full_meshes_path].keys())
- except KeyError:
- list_meshes = []
- mylog.debug( "open_pmd - Iteration %s : found %d meshes"
- %( iteration, len(list_meshes) ) )
-
- # Check for the attributes of the STANDARD.md
- for field_name in list_meshes :
- field = f[full_meshes_path + field_name.encode('ascii')]
-
- result_array += test_record(f[full_meshes_path], field_name)
-
- # General attributes of the record
- result_array += test_attr(field, v, "required",
- "unitDimension", np.ndarray, np.float64)
- result_array += test_attr(field, v, "required",
- "timeOffset", [np.float32, np.float64])
- result_array += test_attr(field, v, "required",
- "gridSpacing", np.ndarray, [np.float32, np.float64])
- result_array += test_attr(field, v, "required",
- "gridGlobalOffset", np.ndarray, [np.float32, np.float64])
- result_array += test_attr(field, v, "required",
- "gridUnitSI", np.float64)
- result_array += test_attr(field, v, "required",
- "dataOrder", np.string_)
- result_array += test_attr(field, v, "required",
- "axisLabels", np.ndarray, np.string_)
- # Specific check for geometry
- geometry_test = test_attr(field, v, "required", "geometry", np.string_)
- result_array += geometry_test
- # geometryParameters is required when using thetaMode
- if geometry_test[0] == 0 and field.attrs["geometry"] == b"thetaMode" :
- result_array += test_attr(field, v, "required",
- "geometryParameters", np.string_)
- # otherwise it is optional
- else :
- result_array += test_attr(field, v, "optional",
- "geometryParameters", np.string_)
-
- # Attributes of the record's components
- if is_scalar_record(field) : # If the record is a scalar field
- result_array += test_component(field, v)
- result_array += test_attr(field, v,
- "required", "position", np.ndarray, [np.float32, np.float64])
- else: # If the record is a vector field
- # Loop over the components
- for component_name in list(field.keys()) :
- component = field[component_name]
- result_array += test_component(component, v)
- result_array += test_attr(component, v,
- "required", "position", np.ndarray, [np.float32, np.float64])
-
- # Check for the attributes of the PIC extension,
- # if asked to do so by the user
- if pic:
-
- # Check the attributes associated with the field solver
- result_array += test_attr(f[full_meshes_path], v, "required",
- "fieldSolver", np.string_)
- valid, field_solver = get_attr(f[full_meshes_path], "fieldSolver")
- if (valid == True) and (field_solver in ["other", "GPSTD"]) :
- result_array += test_attr(f[full_meshes_path], v, "required",
- "fieldSolverParameters", np.string_)
-
- # Check for the attributes associated with the field boundaries
- result_array += test_attr(f[full_meshes_path], v, "required",
- "fieldBoundary", np.ndarray, np.string_)
- valid, field_boundary = get_attr(f[full_meshes_path], "fieldBoundary")
- if (valid == True) and (np.any(field_boundary == b"other")) :
- result_array += test_attr(f[full_meshes_path], v, "required",
- "fieldBoundaryParameters", np.ndarray, np.string_)
-
- # Check for the attributes associated with the field boundaries
- result_array += test_attr(f[full_meshes_path], v, "required",
- "particleBoundary", np.ndarray, np.string_)
- valid, particle_boundary = get_attr(f[full_meshes_path], "particleBoundary")
- if (valid == True) and (np.any(particle_boundary == b"other")) :
- result_array += test_attr(f[full_meshes_path], v, "required",
- "particleBoundaryParameters", np.ndarray, np.string_)
-
- # Check the attributes associated with the current smoothing
- result_array += test_attr(f[full_meshes_path], v, "required",
- "currentSmoothing", np.string_)
- valid, current_smoothing = get_attr(f[full_meshes_path], "currentSmoothing")
- if (valid == True) and (current_smoothing != b"none") :
- result_array += test_attr(f[full_meshes_path], v, "required",
- "currentSmoothingParameters", np.string_)
-
- # Check the attributes associated with the charge conservation
- result_array += test_attr(f[full_meshes_path], v, "required",
- "chargeCorrection", np.string_)
- valid, charge_correction = get_attr(f[full_meshes_path], "chargeCorrection")
- if valid == True and charge_correction != b"none":
- result_array += test_attr(f[full_meshes_path], v, "required",
- "chargeCorrectionParameters", np.string_)
-
- # Check for the attributes of each record
- for field_name in list_meshes :
- field = f[full_meshes_path + field_name.encode('ascii')]
- result_array + test_attr(field, v, "required",
- "fieldSmoothing", np.string_)
- valid, field_smoothing = get_attr(field, "fieldSmoothing")
- if (valid == True) and (field_smoothing != b"none") :
- result_array += test_attr(field,v, "required",
- "fieldSmoothingParameters", np.string_)
- return(result_array)
-
-
-def check_particles(f, iteration, v, pic) :
- """
- Scan all the particle data corresponding to one iteration
-
- Parameters
- ----------
- f : an h5py.File object
- The HDF5 file in which to find the attribute
-
- iteration : string representing an integer
- The iteration at which to scan the particle data
-
- v : bool
- Verbose option
-
- pic : bool
- Whether to check for the ED-PIC extension attributes
-
- Returns
- -------
- An array with 2 elements :
- - The first element is the number of errors encountered
- - The second element is the number of warnings encountered
- """
- # Initialize the result array
- # First element : number of errors
- # Second element : number of warnings
- result_array = np.array([ 0, 0])
-
- # Find the path to the data
- base_path = ("/data/%s/" % iteration).encode('ascii')
- valid, particles_path = get_attr(f, "particlesPath")
- if os.path.join( base_path, particles_path) != \
- ( base_path + particles_path ) :
- mylog.warning("open_pmd - `basePath`+`meshesPath` seems to be malformed "
- "(is `basePath` absolute and ends on a `/` ?)")
- return( np.array([1, 0]) )
- else:
- full_particle_path = base_path + particles_path
- # Find all the particle species
- try:
- list_species = list(f[full_particle_path].keys())
- except KeyError:
- list_species = []
- mylog.debug( "open_pmd - Iteration %s : found %d particle species"
- %( iteration, len(list_species) ) )
-
- # Go through all the particle species
- for species_name in list_species :
- species = f[full_particle_path + species_name.encode('ascii')]
-
- # Check all records for this species
- for species_record_name in species :
- result_array += test_record(species, species_record_name)
-
- # Check the position record of the particles
- result_array += test_key(species, v, "required", "position")
-
- # Check the position offset record of the particles
- result_array += test_key(species, v, "required", "positionOffset")
- if result_array[0] == 0 :
- position_dimensions = len(species["position"].keys())
- positionOffset_dimensions = len(species["positionOffset"].keys())
- if position_dimensions != positionOffset_dimensions :
- mylog.warning("open_pmd - `position` (ndim=%s) and `positionOffset` " \
- "(ndim=%s) do not have the same dimensions in " \
- "species `%s`!" \
- %(str(position_dimensions), \
- str(positionOffset_dimensions),
- species.name) )
- result_array += np.array([ 1, 0])
-
- # Check the particlePatches record of the particles
- patch_test = test_key(species, v, "recommended", "particlePatches")
- result_array += patch_test
- if result_array[0] == 0 and patch_test[1] == 0 :
- result_array += test_key(species["particlePatches"], v, "required",
- "numParticles")
- result_array += test_key(species["particlePatches"], v, "required",
- "numParticlesOffset")
- result_array += test_key(species["particlePatches"], v, "required",
- "offset")
- result_array += test_key(species["particlePatches"], v, "required",
- "extent")
- if result_array[0] == 0 :
- offset = species["particlePatches"]["offset"]
- extent = species["particlePatches"]["extent"]
- # Attributes of the components
- for component_name in list(species["position"].keys()) :
- result_array += test_key( offset, v, "required",
- component_name)
- result_array += test_key( extent, v, "required",
- component_name)
- if result_array[0] == 0 :
- dset_offset = offset[component_name]
- result_array += test_component(dset_offset, v)
- dset_extent = extent[component_name]
- result_array += test_component(dset_extent, v)
-
- # Check the records required by the PIC extension
- if pic :
- result_array += test_key(species, v, "required", "momentum")
- result_array += test_key(species, v, "required", "charge")
- result_array += test_key(species, v, "required", "mass")
- result_array += test_key(species, v, "required", "weighting")
- result_array += test_key(species, v, "optional", "boundElectrons")
- result_array += test_key(species, v, "optional", "protonNumber")
- result_array += test_key(species, v, "optional", "neutronNumber")
-
- # Check the attributes associated with the PIC extension
- if pic :
- result_array += test_attr(species, v, "required",
- "particleShape", [np.float32, np.float64])
- result_array += test_attr(species, v, "required",
- "currentDeposition", np.string_)
- result_array += test_attr(species, v, "required",
- "particlePush", np.string_)
- result_array += test_attr(species, v, "required",
- "particleInterpolation", np.string_)
-
- # Check for the attributes associated with the particle smoothing
- result_array += test_attr(species, v, "required",
- "particleSmoothing", np.string_)
- valid, particle_smoothing = get_attr(species, "particleSmoothing")
- if valid == True and particle_smoothing != b"none":
- result_array += test_attr(species, v, "required",
- "particleSmoothingParameters", np.string_)
-
- # Check attributes of each record of the particle
- for record in list(species.keys()):
- # all records (but particlePatches) require units
- if record != "particlePatches":
- result_array += test_attr(species[record], v,
- "required", "unitDimension", np.ndarray, np.float64)
- time_type = f[base_path].attrs["time"].dtype.type
- result_array += test_attr(species[record], v, "required",
- "timeOffset", time_type)
- if pic:
- result_array += test_attr(species[record], v, "required",
- "weightingPower", np.float64)
- result_array += test_attr(species[record], v, "required",
- "macroWeighted", np.uint32)
- # Attributes of the components
- if is_scalar_record(species[record]): # Scalar record
- dset = species[record]
- result_array += test_component(dset, v)
- else: # Vector record
- # Loop over the components
- for component_name in list(species[record].keys()):
- dset = species[os.path.join(record, component_name)]
- result_array += test_component(dset, v)
-
- return result_array
def parse_unit_dimension(unit_dimension):
https://bitbucket.org/yt_analysis/yt/commits/6305c05948b6/
Changeset: 6305c05948b6
Branch: yt
User: Fabian Koller
Date: 2016-08-18 09:42:23+00:00
Summary: Completely remove code that handles non-standard PMD files,
unify quotes,
unify np dtypes
Affected #: 5 files
diff -r 4422797274ad4947fcf9a44caf4a2f11ae1f6161 -r 6305c05948b6a6daf74db504cfed7075d69160d3 yt/frontends/open_pmd/data_structures.py
--- a/yt/frontends/open_pmd/data_structures.py
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -72,7 +72,7 @@
"""
grid = OpenPMDGrid
- def __init__(self, ds, dataset_type='openPMD'):
+ def __init__(self, ds, dataset_type="openPMD"):
self.dataset_type = dataset_type
self.dataset = ds
self.index_filename = ds.parameter_filename
@@ -193,7 +193,7 @@
In general, NOT all particles in a grid will be inside the grid edges.
"""
self.grid_levels.flat[:] = 0
- self.grids = np.empty(self.num_grids, dtype='object')
+ self.grids = np.empty(self.num_grids, dtype="object")
nrp = self.numparts.copy() # Number of remaining particles from the dataset
pci = {} # Index for particle chunk
@@ -254,19 +254,18 @@
"""
_index_class = OpenPMDHierarchy
_field_info_class = OpenPMDFieldInfo
- _nonstandard = False
- def __init__(self, filename, dataset_type='openPMD',
+ def __init__(self, filename, dataset_type="openPMD",
storage_filename=None,
units_override=None,
unit_system="mks"):
self._handle = HDF5FileHandler(filename)
- self._set_paths(self._handle, os.path.dirname(filename), self._nonstandard)
+ self._set_paths(self._handle, os.path.dirname(filename))
Dataset.__init__(self, filename, dataset_type,
units_override=units_override,
unit_system=unit_system)
self.storage_filename = storage_filename
- self.fluid_types += ('openPMD',)
+ self.fluid_types += ("openPMD",)
particles = tuple(str(c) for c in self._handle[self.base_path + self.particles_path].keys())
if len(particles) > 1:
# Only use on-disk particle names if there is more than one species
@@ -275,7 +274,7 @@
self.particle_types_raw = self.particle_types
self.particle_types = tuple(self.particle_types)
- def _set_paths(self, handle, path, nonstandard):
+ def _set_paths(self, handle, path):
"""Parses relevant hdf5-paths out of ``handle``.
Parameters
@@ -283,38 +282,33 @@
handle : h5py.File
path : str
(absolute) filepath for current hdf5 container
- nonstandard : bool
"""
+ list_iterations = []
+ if "groupBased" in handle.attrs["iterationEncoding"]:
+ for i in list(handle["/data"].keys()):
+ list_iterations.append(i)
+ mylog.info("open_pmd - found {} iterations in file".format(len(list_iterations)))
+ elif "fileBased" in handle.attrs["iterationEncoding"]:
+ regex = "^" + handle.attrs["iterationFormat"].replace("%T", "[0-9]+") + "$"
+ if path is "":
+ mylog.warning("open_pmd - For file based iterations, please use absolute file paths!")
+ pass
+ for filename in os.listdir(path):
+ if re.match(regex, filename):
+ list_iterations.append(filename)
+ mylog.info("open_pmd - found {} iterations in directory".format(len(list_iterations)))
+ else:
+ mylog.warning(
+ "open_pmd - No valid iteration encoding: {}".format(handle.attrs["iterationEncoding"]))
+
+ if len(list_iterations) == 0:
+ mylog.warning("open_pmd - no iterations found!")
+ if "groupBased" in handle.attrs["iterationEncoding"] and len(list_iterations) > 1:
+ mylog.warning("open_pmd - only choose to load one iteration ({})".format(handle["/data"].keys()[0]))
+
self.base_path = "/data/{}/".format(handle["/data"].keys()[0])
- if nonstandard:
- self.meshes_path = "fields/"
- self.particles_path = "particles/"
- else:
- list_iterations = []
- if "groupBased" in handle.attrs["iterationEncoding"]:
- for i in list(handle["/data"].keys()):
- list_iterations.append(i)
- mylog.info("open_pmd - found {} iterations in file".format(len(list_iterations)))
- elif "fileBased" in handle.attrs["iterationEncoding"]:
- regex = "^" + handle.attrs["iterationFormat"].replace('%T', '[0-9]+') + "$"
- if path is '':
- mylog.warning("open_pmd - For file based iterations, please use absolute file paths!")
- pass
- for filename in os.listdir(path):
- if re.match(regex, filename):
- list_iterations.append(filename)
- mylog.info("open_pmd - found {} iterations in directory".format(len(list_iterations)))
- else:
- mylog.warning(
- "open_pmd - No valid iteration encoding: {}".format(handle.attrs["iterationEncoding"]))
-
- if len(list_iterations) == 0:
- mylog.warning("open_pmd - no iterations found!")
- if "groupBased" in handle.attrs["iterationEncoding"] and len(list_iterations) > 1:
- mylog.warning("open_pmd - only choose to load one iteration ({})".format(handle["/data"].keys()[0]))
-
- self.meshes_path = self._handle["/"].attrs["meshesPath"]
- self.particles_path = self._handle["/"].attrs["particlesPath"]
+ self.meshes_path = self._handle["/"].attrs["meshesPath"]
+ self.particles_path = self._handle["/"].attrs["particlesPath"]
def _set_code_unit_attributes(self):
"""Handle conversion between different physical units and the code units.
@@ -322,11 +316,11 @@
These are hardcoded as 1.0. Every dataset in openPMD can have different code <-> physical scaling.
The individual factor is obtained by multiplying with "unitSI" reading getting data from disk.
"""
- setdefaultattr(self, 'length_unit', self.quan(1.0, "m"))
- setdefaultattr(self, 'mass_unit', self.quan(1.0, "kg"))
- setdefaultattr(self, 'time_unit', self.quan(1.0, "s"))
- setdefaultattr(self, 'velocity_unit', self.quan(1.0, "m/s"))
- setdefaultattr(self, 'magnetic_unit', self.quan(1.0, "T"))
+ setdefaultattr(self, "length_unit", self.quan(1.0, "m"))
+ setdefaultattr(self, "mass_unit", self.quan(1.0, "kg"))
+ setdefaultattr(self, "time_unit", self.quan(1.0, "s"))
+ setdefaultattr(self, "velocity_unit", self.quan(1.0, "m/s"))
+ setdefaultattr(self, "magnetic_unit", self.quan(1.0, "T"))
def _parse_parameter_file(self):
"""Read in metadata describing the overall data on-disk.
@@ -361,15 +355,8 @@
self.domain_left_edge = np.zeros(3, dtype=np.float64)
try:
mesh = f[bp + mp].keys()[0]
- if self._nonstandard:
- width = f[bp].attrs['cell_width']
- height = f[bp].attrs['cell_height']
- depth = f[bp].attrs['cell_depth']
- spacing = np.asarray([width, height, depth])
- unit_si = f[bp].attrs['unit_length']
- else:
- spacing = np.asarray(f[bp + mp + "/" + mesh].attrs["gridSpacing"])
- unit_si = f[bp + mp + "/" + mesh].attrs["gridUnitSI"]
+ spacing = np.asarray(f[bp + mp + "/" + mesh].attrs["gridSpacing"])
+ unit_si = f[bp + mp + "/" + mesh].attrs["gridUnitSI"]
self.domain_right_edge = self.domain_dimensions[:spacing.size] * unit_si * spacing
self.domain_right_edge = np.append(self.domain_right_edge, np.ones(3 - self.domain_right_edge.size))
except Exception as e:
@@ -377,10 +364,7 @@
"open_pmd - The domain extent could not be calculated! ({}) Setting the field extent to 1m**3!".format(e))
self.domain_right_edge = np.ones(3, dtype=np.float64)
- if self._nonstandard:
- self.current_time = 0
- else:
- self.current_time = f[bp].attrs["time"] * f[bp].attrs["timeUnitSI"]
+ self.current_time = f[bp].attrs["time"] * f[bp].attrs["timeUnitSI"]
@classmethod
def _is_valid(self, *args, **kwargs):
diff -r 4422797274ad4947fcf9a44caf4a2f11ae1f6161 -r 6305c05948b6a6daf74db504cfed7075d69160d3 yt/frontends/open_pmd/fields.py
--- a/yt/frontends/open_pmd/fields.py
+++ b/yt/frontends/open_pmd/fields.py
@@ -146,10 +146,7 @@
if type(field) is h5.Dataset:
# Don't consider axes. This appears to be a vector field of single dimensionality
ytname = str("_".join([fname.replace("_", "-")]))
- if ds._nonstandard:
- parsed = ""
- else:
- parsed = parse_unit_dimension(np.asarray(field.attrs["unitDimension"], dtype="int"))
+ parsed = parse_unit_dimension(np.asarray(field.attrs["unitDimension"], dtype=np.int))
unit = str(YTQuantity(1, parsed).units)
aliases = []
# Save a list of magnetic fields for aliasing later on
@@ -158,16 +155,10 @@
self._mag_fields.append(ytname)
self.known_other_fields += ((ytname, (unit, aliases, None)),)
else:
- if ds._nonstandard:
- axes = "xyz" # naively assume all fields in non-standard files are 3D
- else:
- axes = field.attrs["axisLabels"]
+ axes = field.attrs["axisLabels"]
for axis in axes:
ytname = str("_".join([fname.replace("_", "-"), axis]))
- if ds._nonstandard:
- parsed = ""
- else:
- parsed = parse_unit_dimension(np.asarray(field.attrs["unitDimension"], dtype="int"))
+ parsed = parse_unit_dimension(np.asarray(field.attrs["unitDimension"], dtype=np.int))
unit = str(YTQuantity(1, parsed).units)
aliases = []
# Save a list of magnetic fields for aliasing later on
@@ -182,14 +173,9 @@
for species in particles.keys():
for attrib in particles[species].keys():
try:
- if ds._nonstandard:
- if "globalCellIdx" in attrib or "position" in attrib:
- parsed = "m" # Required for spatial selection of particles
- else:
- parsed = ""
- else:
- parsed = parse_unit_dimension(
- np.asarray(particles.get(species).get(attrib).attrs["unitDimension"], dtype="int"))
+ parsed = parse_unit_dimension(
+ np.asarray(particles.get(species).get(attrib).attrs["unitDimension"],
+ dtype=np.int))
unit = str(YTQuantity(1, parsed).units)
name = ["particle", attrib]
ytattrib = attrib
@@ -206,8 +192,6 @@
if axis in "rxyz":
name = ["particle", ytattrib.replace("_", "-"), axis]
ytname = str("_".join(name))
- if ds._nonstandard and "globalCellIdx" in ytname:
- aliases.append(ytname.replace("globalCellIdx", "positionOffset"))
particle_fields += ((ytname, (unit, aliases, None)),)
except KeyError:
mylog.info("open_pmd - {}_{} does not seem to have unitDimension".format(species, attrib))
diff -r 4422797274ad4947fcf9a44caf4a2f11ae1f6161 -r 6305c05948b6a6daf74db504cfed7075d69160d3 yt/frontends/open_pmd/io.py
--- a/yt/frontends/open_pmd/io.py
+++ b/yt/frontends/open_pmd/io.py
@@ -41,39 +41,30 @@
Parameters
----------
ptype : str
+ The on-disk name of the particle species
index : int, optional
offset : int, optional
"""
if str((ptype, index, offset)) not in self._cached_ptype:
- if ptype in "io":
- spec = self._handle[self.base_path + self.particles_path].keys()[0]
- else:
- spec = ptype
- pds = self._handle[self.base_path + self.particles_path + "/" + spec]
+ self._cached_ptype = str((ptype, index, offset))
+ pds = self._handle[self.base_path + self.particles_path + "/" + ptype]
axes = [str(ax) for ax in pds["position"].keys()]
- if self.ds._nonstandard:
- position_offset = "globalCellIdx/"
- else:
- position_offset = "positionOffset/"
if offset is None:
if is_const_component(pds["position/" + axes[0]]):
- size = pds["position/" + axes[0]].attrs["shape"]
+ offset = pds["position/" + axes[0]].attrs["shape"]
else:
- size = pds["position/" + axes[0]].len()
- else:
- size = offset
- self.cache = np.empty((3, size), dtype="float64")
+ offset = pds["position/" + axes[0]].len()
+ self.cache = np.empty((3, offset), dtype=np.float64)
for i in range(3):
ax = "xyz"[i]
if ax in axes:
- np.add(self.get_component(pds, "position/" + ax, index, offset, self.ds._nonstandard),
- self.get_component(pds, position_offset + ax, index, offset, self.ds._nonstandard),
+ np.add(self.get_component(pds, "position/" + ax, index, offset),
+ self.get_component(pds, "positionOffset/" + ax, index, offset),
self.cache[i])
else:
# Pad accordingly with zeros to make 1D/2D datasets compatible
# These have to be the same shape as the existing axes since that equals the number of particles
- self.cache[i] = np.zeros(size)
- self._cached_ptype = str((ptype, index, offset))
+ self.cache[i] = np.zeros(offset)
def _read_particle_coords(self, chunks, ptf):
"""Reads coordinates for given particle-types in given chunks from file.
@@ -109,10 +100,10 @@
index = dict(g.particle_index).get(spec)
offset = dict(g.particle_offset).get(spec)
mylog.debug(
- "open_pmd - _read_particle_coords: (grid {}) {}, {} [{}:{}]".format(g, ptype, field_list, index,
+ "open_pmd - _read_particle_coords: (grid {}) {}, {} [{}:{}]".format(g, spec, field_list, index,
offset))
- if str((ptype, index, offset)) not in self._cached_ptype:
- self._fill_cache(ptype, index, offset)
+ if str((spec, index, offset)) not in self._cached_ptype:
+ self._fill_cache(spec, index, offset)
yield (ptype, (self.cache[0], self.cache[1], self.cache[2]))
def _read_particle_fields(self, chunks, ptf, selector):
@@ -156,10 +147,10 @@
index = dict(g.particle_index).get(spec)
offset = dict(g.particle_offset).get(spec)
mylog.debug(
- "open_pmd - _read_particle_fields: (grid {}) {}, {} [{}:{}]".format(g, ptype, field_list, index,
+ "open_pmd - _read_particle_fields: (grid {}) {}, {} [{}:{}]".format(g, spec, field_list, index,
offset))
- if str((ptype, index, offset)) not in self._cached_ptype:
- self._fill_cache(ptype, index, offset)
+ if str((spec, index, offset)) not in self._cached_ptype:
+ self._fill_cache(spec, index, offset)
mask = selector.select_points(self.cache[0], self.cache[1], self.cache[2], 0.0)
if mask is None:
continue
@@ -167,7 +158,7 @@
for field in field_list:
component = "/".join(field.split("_")[1:]).replace("positionCoarse",
"position").replace("-", "_")
- data = self.get_component(pds, component, index, offset, self.ds._nonstandard)
+ data = self.get_component(pds, component, index, offset)
yield ((ptype, field), data[mask])
def _read_fluid_selection(self, chunks, selector, fields, size):
@@ -182,7 +173,7 @@
A region (inside your domain) specifying which parts of the field you want to read
See [1] and [2]
fields : array_like
- tuples (fname, ftype) representing a field
+ Tuples (fname, ftype) representing a field
size : int
Size of the data to read
@@ -209,7 +200,7 @@
for g in chunk.objs))
ind = {}
for field in fields:
- rv[field] = np.empty(size, dtype="float64")
+ rv[field] = np.empty(size, dtype=np.float64)
ind[field] = 0
for ftype, fname in fields:
@@ -217,21 +208,22 @@
for chunk in chunks:
for g in chunk.objs:
mask = g._get_selector_mask(selector)
- if mask is not None:
- component = fname.replace("_", "/").replace("-", "_")
- index = g.mesh_index
- offset = g.mesh_offset
- data = np.array(self.get_component(ds, component, index, offset, self.ds._nonstandard))
- # The following is a modified AMRGridPatch.select(...)
- data.shape = mask.shape # Workaround - casts a 2D (x,y) array to 3D (x,y,1)
- count = g.count(selector)
- rv[field][ind[field]:ind[field] + count] = data[mask]
- ind[field] += count
+ if mask is None:
+ continue
+ component = fname.replace("_", "/").replace("-", "_")
+ index = g.mesh_index
+ offset = g.mesh_offset
+ data = np.array(self.get_component(ds, component, index, offset))
+ # The following is a modified AMRGridPatch.select(...)
+ data.shape = mask.shape # Workaround - casts a 2D (x,y) array to 3D (x,y,1)
+ count = g.count(selector)
+ rv[field][ind[field]:ind[field] + count] = data[mask]
+ ind[field] += count
for i in rv:
rv[i].flatten()
return rv
- def get_component(self, group, component_name, index=0, offset=None, nonstandard=False):
+ def get_component(self, group, component_name, index=0, offset=None):
"""Grab a dataset component from a group as a whole or sliced.
Parameters
@@ -244,7 +236,6 @@
offset : int, optional
number of entries to read
if not supplied, every entry after index is returned
- nonstandard : bool, optional
Notes
-----
@@ -256,30 +247,17 @@
"""
record_component = group[component_name]
- if nonstandard:
- try:
- unitSI = record_component.attrs["sim_unit"]
- if "globalCellIdx" in component_name:
- it = group["/data"].keys()[0]
- length = group["/data/" + it].attrs['unit_length']
- unitSI *= length
- except:
- unitSI = 1.0
- else:
- unitSI = record_component.attrs["unitSI"]
- # check whether component is constant
+ unit_si = record_component.attrs["unitSI"]
if is_const_component(record_component):
- if offset is not None:
- shape = offset
- else:
- shape = record_component.attrs["shape"] - index
+ if offset is None:
+ offset = record_component.attrs["shape"] - index
# component is constant, craft an array by hand
- mylog.debug("open_pmd - get_component (const): {}/{}({})".format(group.name, component_name, shape))
- return np.full(shape, record_component.attrs["value"] * unitSI)
+ mylog.debug("open_pmd - get_component (const): {}/{}({})".format(group.name, component_name, offset))
+ return np.full(offset, record_component.attrs["value"] * unit_si)
else:
if offset is not None:
offset += index
# component is a dataset, return it (possibly masked)
mylog.debug(
"open_pmd - get_component: {}/{}[{}:{}]".format(group.name, component_name, index, offset))
- return np.multiply(record_component[index:offset], unitSI)
+ return np.multiply(record_component[index:offset], unit_si)
diff -r 4422797274ad4947fcf9a44caf4a2f11ae1f6161 -r 6305c05948b6a6daf74db504cfed7075d69160d3 yt/frontends/open_pmd/misc.py
--- a/yt/frontends/open_pmd/misc.py
+++ b/yt/frontends/open_pmd/misc.py
@@ -47,7 +47,7 @@
"""
if len(unit_dimension) is not 7:
mylog.error("open_pmd - SI must have 7 base dimensions!")
- unit_dimension = np.asarray(unit_dimension, dtype='int')
+ unit_dimension = np.asarray(unit_dimension, dtype=np.int)
dim = []
si = ["m",
"kg",
diff -r 4422797274ad4947fcf9a44caf4a2f11ae1f6161 -r 6305c05948b6a6daf74db504cfed7075d69160d3 yt/frontends/open_pmd/setup.py
--- a/yt/frontends/open_pmd/setup.py
+++ b/yt/frontends/open_pmd/setup.py
@@ -10,9 +10,9 @@
# -----------------------------------------------------------------------------
-def configuration(parent_package='', top_path=None):
+def configuration(parent_package="", top_path=None):
from numpy.distutils.misc_util import Configuration
- config = Configuration('open_pmd', parent_package, top_path)
+ config = Configuration("open_pmd", parent_package, top_path)
config.make_config_py() # installs __config__.py
# config.make_svn_version_py()
return config
https://bitbucket.org/yt_analysis/yt/commits/e792637050ec/
Changeset: e792637050ec
Branch: yt
User: Fabian Koller
Date: 2016-08-18 15:27:26+00:00
Summary: Set up frontend answer tests
Affected #: 2 files
diff -r 6305c05948b6a6daf74db504cfed7075d69160d3 -r e792637050ec118e6f5e4fe6ecae4edc58936164 yt/frontends/open_pmd/tests/test_outputs.py
--- /dev/null
+++ b/yt/frontends/open_pmd/tests/test_outputs.py
@@ -0,0 +1,68 @@
+"""
+openPMD frontend tests
+
+
+
+"""
+
+# -----------------------------------------------------------------------------
+# Copyright (c) 2016, Fabian Koller (HZDR).
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+# -----------------------------------------------------------------------------
+
+from yt.testing import \
+ assert_almost_equal, \
+ assert_equal, \
+ assert_array_equal, \
+ requires_file
+
+from yt.utilities.answer_testing.framework import \
+ FieldValuesTest, \
+ requires_ds, \
+ data_dir_load
+
+twoD = "/example-2d/hdf5/data00000100.h5"
+threeD = "/example-3d/hdf5/data00000100.h5"
+
+ at requires_file(threeD)
+def test_3d_out():
+ ds = data_dir_load(threeD)
+ field_list = [('all', 'particle_charge'),
+ ('all', 'particle_mass'),
+ ('all', 'particle_momentum_x'),
+ ('all', 'particle_momentum_y'),
+ ('all', 'particle_momentum_z'),
+ ('all', 'particle_positionCoarse_x'),
+ ('all', 'particle_positionCoarse_y'),
+ ('all', 'particle_positionCoarse_z'),
+ ('all', 'particle_positionOffset_x'),
+ ('all', 'particle_positionOffset_y'),
+ ('all', 'particle_positionOffset_z'),
+ ('all', 'particle_weighting'),
+ ('io', 'particle_charge'),
+ ('io', 'particle_mass'),
+ ('io', 'particle_momentum_x'),
+ ('io', 'particle_momentum_y'),
+ ('io', 'particle_momentum_z'),
+ ('io', 'particle_positionCoarse_x'),
+ ('io', 'particle_positionCoarse_y'),
+ ('io', 'particle_positionCoarse_z'),
+ ('io', 'particle_positionOffset_x'),
+ ('io', 'particle_positionOffset_y'),
+ ('io', 'particle_positionOffset_z'),
+ ('io', 'particle_weighting'),
+ ('openPMD', 'E_x'),
+ ('openPMD', 'E_y'),
+ ('openPMD', 'E_z'),
+ ('openPMD', 'rho')]
+ domain_domensions = [26, 26, 201]
+ domain_width = [2.08e-05, 2.08e-05, 2.01e-05]
+ yield assert_equal, str(ds), "data00000100.h5"
+ yield assert_equal, ds.dimensionality, 3
+ yield assert_equal, ds.current_time, 3.28471214521e-14
+ yield assert_array_equal, ds.field_list, field_list
+ yield assert_array_equal, ds.domain_domensions, domain_domensions
+ yield assert_almost_equal, ds.domain_right_edge - ds.domain_left_edge, domain_width
\ No newline at end of file
https://bitbucket.org/yt_analysis/yt/commits/293e1885c996/
Changeset: 293e1885c996
Branch: yt
User: Fabian Koller
Date: 2016-08-19 04:09:16+00:00
Summary: Parse unitDimension of constant records
Affected #: 2 files
diff -r e792637050ec118e6f5e4fe6ecae4edc58936164 -r 293e1885c996e94c078d6d79d98c2f53bedcdaef yt/frontends/open_pmd/fields.py
--- a/yt/frontends/open_pmd/fields.py
+++ b/yt/frontends/open_pmd/fields.py
@@ -20,7 +20,9 @@
from yt.fields.field_info_container import FieldInfoContainer
from yt.fields.magnetic_field import setup_magnetic_field_aliases
-from yt.frontends.open_pmd.misc import parse_unit_dimension
+from yt.frontends.open_pmd.misc import \
+ parse_unit_dimension, \
+ is_const_component
from yt.units.yt_array import YTQuantity
from yt.utilities.logger import ytLogger as mylog
from yt.utilities.physical_constants import speed_of_light
@@ -184,7 +186,7 @@
# particle_position is later derived in setup_absolute_positions in the way yt expects it
ytattrib = "positionCoarse"
pds = particles[species + "/" + attrib]
- if type(pds) is h5.Dataset:
+ if type(pds) is h5.Dataset or is_const_component(pds):
particle_fields += ((str("_".join(name)), (unit, [], None)),)
else:
for axis in pds.keys():
diff -r e792637050ec118e6f5e4fe6ecae4edc58936164 -r 293e1885c996e94c078d6d79d98c2f53bedcdaef yt/frontends/open_pmd/tests/test_outputs.py
--- a/yt/frontends/open_pmd/tests/test_outputs.py
+++ b/yt/frontends/open_pmd/tests/test_outputs.py
@@ -24,8 +24,11 @@
requires_ds, \
data_dir_load
-twoD = "/example-2d/hdf5/data00000100.h5"
-threeD = "/example-3d/hdf5/data00000100.h5"
+from yt.frontends.open_pmd.data_structures import \
+ OpenPMDDataset
+
+twoD = "example-2d/hdf5/data00000100.h5"
+threeD = "example-3d/hdf5/data00000100.h5"
@requires_file(threeD)
def test_3d_out():
@@ -60,9 +63,13 @@
('openPMD', 'rho')]
domain_domensions = [26, 26, 201]
domain_width = [2.08e-05, 2.08e-05, 2.01e-05]
+
+ assert isinstance(ds, OpenPMDDataset)
yield assert_equal, str(ds), "data00000100.h5"
yield assert_equal, ds.dimensionality, 3
yield assert_equal, ds.current_time, 3.28471214521e-14
+ yield assert_equal, ds.particle_types_raw, ('io',)
+ assert "all" in ds.particle_unions
yield assert_array_equal, ds.field_list, field_list
yield assert_array_equal, ds.domain_domensions, domain_domensions
yield assert_almost_equal, ds.domain_right_edge - ds.domain_left_edge, domain_width
\ No newline at end of file
https://bitbucket.org/yt_analysis/yt/commits/9c8b7c8acdd9/
Changeset: 9c8b7c8acdd9
Branch: yt
User: Fabian Koller
Date: 2016-08-19 08:45:07+00:00
Summary: Parse field list from file rather than use "axisLabels"
Affected #: 1 file
diff -r 293e1885c996e94c078d6d79d98c2f53bedcdaef -r 9c8b7c8acdd952f62e3d9725baaca865c0e2d11f yt/frontends/open_pmd/fields.py
--- a/yt/frontends/open_pmd/fields.py
+++ b/yt/frontends/open_pmd/fields.py
@@ -25,13 +25,16 @@
is_const_component
from yt.units.yt_array import YTQuantity
from yt.utilities.logger import ytLogger as mylog
-from yt.utilities.physical_constants import speed_of_light
+from yt.utilities.physical_constants import \
+ speed_of_light, \
+ mu_0
def setup_poynting_vector(self):
def _get_poyn(axis):
def poynting(field, data):
- u = 79577.4715459 # = 1/magnetic permeability
+ u = mu_0**-1
+ #u = 79577.4715459 # = 1/magnetic permeability
if axis in "x":
return u * (data["E_y"] * data["magnetic_field_z"] - data["E_z"] * data["magnetic_field_y"])
elif axis in "y":
@@ -44,7 +47,7 @@
for ax in "xyz":
self.add_field(("openPMD", "poynting_vector_%s" % ax),
function=_get_poyn(ax),
- units="T*V/m")
+ units="W/m**2")
def setup_kinetic_energy(self, ptype):
@@ -157,7 +160,7 @@
self._mag_fields.append(ytname)
self.known_other_fields += ((ytname, (unit, aliases, None)),)
else:
- axes = field.attrs["axisLabels"]
+ axes = field.keys()
for axis in axes:
ytname = str("_".join([fname.replace("_", "-"), axis]))
parsed = parse_unit_dimension(np.asarray(field.attrs["unitDimension"], dtype=np.int))
https://bitbucket.org/yt_analysis/yt/commits/33fe74896128/
Changeset: 33fe74896128
Branch: yt
User: Fabian Koller
Date: 2016-08-19 10:26:20+00:00
Summary: Remove dead code
Affected #: 2 files
diff -r 9c8b7c8acdd952f62e3d9725baaca865c0e2d11f -r 33fe748961289351c95e0a78feb485d0d61a0834 yt/frontends/open_pmd/fields.py
--- a/yt/frontends/open_pmd/fields.py
+++ b/yt/frontends/open_pmd/fields.py
@@ -34,7 +34,6 @@
def _get_poyn(axis):
def poynting(field, data):
u = mu_0**-1
- #u = 79577.4715459 # = 1/magnetic permeability
if axis in "x":
return u * (data["E_y"] * data["magnetic_field_z"] - data["E_z"] * data["magnetic_field_y"])
elif axis in "y":
diff -r 9c8b7c8acdd952f62e3d9725baaca865c0e2d11f -r 33fe748961289351c95e0a78feb485d0d61a0834 yt/frontends/open_pmd/io.py
--- a/yt/frontends/open_pmd/io.py
+++ b/yt/frontends/open_pmd/io.py
@@ -89,21 +89,18 @@
f = self._handle
ds = f[self.base_path]
for chunk in chunks:
- for g in chunk.objs:
- if g.filename is None:
- continue
+ for grid in chunk.objs:
for ptype, field_list in sorted(ptf.items()):
if ptype in "io":
spec = ds[self.particles_path].keys()[0]
else:
spec = ptype
- index = dict(g.particle_index).get(spec)
- offset = dict(g.particle_offset).get(spec)
+ index = dict(grid.particle_index).get(spec)
+ offset = dict(grid.particle_offset).get(spec)
mylog.debug(
- "open_pmd - _read_particle_coords: (grid {}) {}, {} [{}:{}]".format(g, spec, field_list, index,
+ "open_pmd - _read_particle_coords: (grid {}) {}, {} [{}:{}]".format(grid, spec, field_list, index,
offset))
- if str((spec, index, offset)) not in self._cached_ptype:
- self._fill_cache(spec, index, offset)
+ self._fill_cache(spec, index, offset)
yield (ptype, (self.cache[0], self.cache[1], self.cache[2]))
def _read_particle_fields(self, chunks, ptf, selector):
@@ -135,22 +132,19 @@
f = self._handle
ds = f[self.base_path]
for chunk in chunks:
- for g in chunk.objs:
- if g.filename is None:
- continue
+ for grid in chunk.objs:
for ptype, field_list in sorted(ptf.items()):
# Get a particle species (e.g. /data/3500/particles/e/)
if ptype in "io":
spec = ds[self.particles_path].keys()[0]
else:
spec = ptype
- index = dict(g.particle_index).get(spec)
- offset = dict(g.particle_offset).get(spec)
+ index = dict(grid.particle_index).get(spec)
+ offset = dict(grid.particle_offset).get(spec)
mylog.debug(
- "open_pmd - _read_particle_fields: (grid {}) {}, {} [{}:{}]".format(g, spec, field_list, index,
+ "open_pmd - _read_particle_fields: (grid {}) {}, {} [{}:{}]".format(grid, spec, field_list, index,
offset))
- if str((spec, index, offset)) not in self._cached_ptype:
- self._fill_cache(spec, index, offset)
+ self._fill_cache(spec, index, offset)
mask = selector.select_points(self.cache[0], self.cache[1], self.cache[2], 0.0)
if mask is None:
continue
https://bitbucket.org/yt_analysis/yt/commits/9136237dd388/
Changeset: 9136237dd388
Branch: yt
User: Fabian Koller
Date: 2016-08-19 15:55:13+00:00
Summary: Count _num_grids smarter (equally sized/positioned grids will be in one grid; particlePatches will be considered)
Affected #: 1 file
diff -r 33fe748961289351c95e0a78feb485d0d61a0834 -r 9136237dd388ca7a86839020f685e3b36f0efa4a yt/frontends/open_pmd/data_structures.py
--- a/yt/frontends/open_pmd/data_structures.py
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -17,6 +17,8 @@
import os
import re
from math import ceil, floor
+import functools
+from operator import mul
import numpy as np
@@ -43,20 +45,22 @@
__slots__ = ["_level_id"]
# Every particle species might have different hdf5-indices and offsets
# These contain tuples (ptype, index) and (ptype, offset)
- particle_index = []
- particle_offset = []
- mesh_index = 0
- mesh_offset = 0
+ ftypes=[]
+ ptypes=[]
+ findex = 0
+ foffset = 0
+ pindex = 0
+ poffset = 0
- def __init__(self, gid, index, level=-1, pi=None, po=None, mi=0, mo=0):
+ def __init__(self, gid, index, level=-1, fi=0, fo=0, pi=0, po=0, ft=[], pt=[]):
AMRGridPatch.__init__(self, gid, filename=index.index_filename,
index=index)
- if pi:
- self.particle_index = pi
- if po:
- self.particle_offset = po
- self.mesh_index = mi
- self.mesh_offset = mo
+ self.findex = fi
+ self.foffset = fo
+ self.pindex = pi
+ self.poffset = po
+ self.ftypes = ft
+ self.ptypes = pt
self.Parent = None
self.Children = []
self.Level = level
@@ -155,29 +159,37 @@
pp = self.dataset.particles_path
gridsize = 12 * 10 ** 6 # Byte
+ self.meshshapes = {}
self.numparts = {}
- meshsizes = {}
+
+ self.num_grids = 0
for mesh in f[bp + mp].keys():
if type(f[bp + mp + mesh]) is h5.Group:
- for axis in f[bp + mp + mesh]:
- meshsizes[mesh + axis] = f[bp + mp + mesh + "/" + axis].size
+ self.meshshapes[mesh] = f[bp + mp + mesh + "/" + f[bp + mp + mesh].keys()[0]].shape
else:
- meshsizes[mesh] = f[bp + mp + mesh].size
- if not meshsizes.values()[1:] == meshsizes.values()[:-1]:
- mylog.warning("open_pmd - This frontend assumes all meshes are equally shaped,"
- " but meshes have different sizes!")
+ self.meshshapes[mesh] = f[bp + mp + mesh].shape
for species in f[bp + pp].keys():
- axis = f[bp + pp + species + "/position"].keys()[0]
- if is_const_component(f[bp + pp + species + "/position/" + axis]):
- self.numparts[species] = f[bp + pp + species + "/position/" + axis].attrs["shape"]
+ if "particlePatches" in f[bp + pp + "/" + species].keys():
+ for patch, size in enumerate(f[bp + pp + "/" + species + "/particlePatches/numParticles"]):
+ self.numparts[species + str(patch)] = size
else:
- self.numparts[species] = f[bp + pp + species + "/position/" + axis].len()
- # Limit particles per grid by resulting memory footprint
- ppg = int(gridsize / (self.dataset.dimensionality * 4)) # 4 Byte per value per dimension (f32)
- # Use an upper bound of equally sized grids, last one might be smaller
- values = np.max((np.max(self.numparts.values()), np.max(meshsizes.values())))
- self.num_grids = int(ceil(values * ppg ** -1))
+ axis = f[bp + pp + species + "/position"].keys()[0]
+ if is_const_component(f[bp + pp + species + "/position/" + axis]):
+ self.numparts[species] = f[bp + pp + species + "/position/" + axis].attrs["shape"]
+ else:
+ self.numparts[species] = f[bp + pp + species + "/position/" + axis].len()
+
+ # Limit values per grid by resulting memory footprint
+ self.vpg = int(gridsize / (self.dataset.dimensionality * 4)) # 4 Byte per value per dimension (f32)
+
+ # Meshes of the same size do not need separate chunks
+ for shape in set(self.meshshapes.values()):
+ self.num_grids += min(shape[0], int(np.ceil(functools.reduce(mul, shape) * self.vpg**-1)))
+
+ # Same goes for particle chunks
+ for size in set(self.numparts.values()):
+ self.num_grids += int(np.ceil(size * self.vpg**-1))
def _parse_index(self):
"""Fills each grid with appropriate properties (extent, dimensions, ...)
@@ -195,6 +207,48 @@
self.grid_levels.flat[:] = 0
self.grids = np.empty(self.num_grids, dtype="object")
+ grid_index_total = 0
+
+ # Mesh grids
+ for shape in set(self.meshshapes.values()):
+ # Total dimension of this grid
+ domain_dimension = np.asarray(shape, dtype=np.int32)
+ domain_dimension = np.pad(domain_dimension,
+ (0, 3 - len(domain_dimension)),
+ 'constant', constant_values=(0, 1))
+ # Number of grids of this shape
+ num_grids = min(shape[0], int(np.ceil(functools.reduce(mul, shape) * self.vpg ** -1)))
+ gle = self.dataset.domain_left_edge # TODO Calculate based on mesh
+ gre = self.dataset.domain_right_edge # TODO Calculate based on mesh
+ grid_dim_offset = np.linspace(0, domain_dimension[0], num_grids + 1, dtype=np.int32)
+ grid_edge_offset = grid_dim_offset * np.float(domain_dimension[0]) ** -1 * (gre[0] - gle[0]) + gle[0]
+ # TODO add information about which meshes are contained, index, offset...
+ mesh_names = []
+ for (mname, mshape) in self.meshshapes.items():
+ if shape == mshape:
+ mesh_names.append(str(mname))
+ for grid in range(num_grids):
+ self.grid_dimensions[grid_index_total] = domain_dimension
+ self.grid_dimensions[grid_index_total][0] = grid_dim_offset[grid + 1] - grid_dim_offset[grid]
+ self.grid_left_edge[grid_index_total] = gle # TODO Calculate based on mesh
+ self.grid_left_edge[grid_index_total][0] = grid_edge_offset[grid]
+ self.grid_right_edge[grid_index_total] = gre # TODO Calculate based on mesh
+ self.grid_right_edge[grid_index_total][0] = grid_edge_offset[grid + 1]
+ self.grids[grid_index_total] = self.grid(grid_index_total, self, 0,
+ fi=0,
+ fo=self.grid_dimensions[grid_index_total][0],
+ ft=mesh_names)
+ grid_index_total += 1
+
+ # Particle grids
+ for species, count in self.numparts.items():
+ if "#" in species:
+ # This is part of a particle patch
+ pass
+ else:
+ for size in set(self.numparts.values()):
+ grid_count = int(np.ceil(size * self.vpg**-1))
+
nrp = self.numparts.copy() # Number of remaining particles from the dataset
pci = {} # Index for particle chunk
for spec in nrp:
@@ -381,7 +435,7 @@
f.close()
return False
- if "1.0.0" != f.attrs["openPMD"]:
+ if "1.0." in f.attrs["openPMD"]:
f.close()
return False
https://bitbucket.org/yt_analysis/yt/commits/a268110228d8/
Changeset: a268110228d8
Branch: yt
User: Fabian Koller
Date: 2016-08-20 19:33:16+00:00
Summary: Grids in particlePatches are set up as needed, meshes and non-Patch particles lack the correct grid_edges
Affected #: 1 file
diff -r 9136237dd388ca7a86839020f685e3b36f0efa4a -r a268110228d8e62bf956de878a22eac6d4fff777 yt/frontends/open_pmd/data_structures.py
--- a/yt/frontends/open_pmd/data_structures.py
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -204,11 +204,21 @@
Furthermore the last grid might be smaller and have fewer particles than the others.
In general, NOT all particles in a grid will be inside the grid edges.
"""
+ f = self.dataset._handle
+ bp = self.dataset.base_path
+ mp = self.dataset.meshes_path
+ pp = self.dataset.particles_path
+
self.grid_levels.flat[:] = 0
self.grids = np.empty(self.num_grids, dtype="object")
grid_index_total = 0
+ def get_component(group, component_name, index, offset):
+ record_component = group[component_name]
+ unit_si = record_component.attrs["unitSI"]
+ return np.multiply(record_component[index:offset], unit_si)
+
# Mesh grids
for shape in set(self.meshshapes.values()):
# Total dimension of this grid
@@ -218,36 +228,83 @@
'constant', constant_values=(0, 1))
# Number of grids of this shape
num_grids = min(shape[0], int(np.ceil(functools.reduce(mul, shape) * self.vpg ** -1)))
- gle = self.dataset.domain_left_edge # TODO Calculate based on mesh
- gre = self.dataset.domain_right_edge # TODO Calculate based on mesh
+ gle = [-1, -1, -1] # TODO Calculate based on mesh
+ gre = [1, 1, 1] # TODO Calculate based on mesh
grid_dim_offset = np.linspace(0, domain_dimension[0], num_grids + 1, dtype=np.int32)
grid_edge_offset = grid_dim_offset * np.float(domain_dimension[0]) ** -1 * (gre[0] - gle[0]) + gle[0]
- # TODO add information about which meshes are contained, index, offset...
mesh_names = []
for (mname, mshape) in self.meshshapes.items():
if shape == mshape:
mesh_names.append(str(mname))
+ prev = 0
for grid in range(num_grids):
self.grid_dimensions[grid_index_total] = domain_dimension
self.grid_dimensions[grid_index_total][0] = grid_dim_offset[grid + 1] - grid_dim_offset[grid]
- self.grid_left_edge[grid_index_total] = gle # TODO Calculate based on mesh
+ self.grid_left_edge[grid_index_total] = gle
self.grid_left_edge[grid_index_total][0] = grid_edge_offset[grid]
- self.grid_right_edge[grid_index_total] = gre # TODO Calculate based on mesh
+ self.grid_right_edge[grid_index_total] = gre
self.grid_right_edge[grid_index_total][0] = grid_edge_offset[grid + 1]
+ self.grid_particle_count[grid_index_total] = 0
self.grids[grid_index_total] = self.grid(grid_index_total, self, 0,
- fi=0,
+ fi=prev,
fo=self.grid_dimensions[grid_index_total][0],
ft=mesh_names)
+ prev = self.grid_dimensions[grid_index_total][0]
grid_index_total += 1
# Particle grids
for species, count in self.numparts.items():
if "#" in species:
- # This is part of a particle patch
- pass
+ spec = species.split("#")
+ patch = f[bp + pp + "/" + spec[0] + "/particlePatches"]
+ domain_dimension = []
+ for axis in patch["extent"].keys():
+ domain_dimension.append(patch["extent/" + axis].value.item(int(spec[1])))
+ domain_dimension = np.asarray(domain_dimension, dtype=np.int32)
+ domain_dimension = np.pad(domain_dimension,
+ (0, 3 - len(domain_dimension)),
+ 'constant', constant_values=(0, 1))
+ num_grids = int(np.ceil(count * self.vpg ** -1))
+ gle = []
+ for axis in patch["offset"].keys():
+ gle.append(get_component(patch, "offset/" + axis, int(spec[1]), 1)[0])
+ gle = np.asarray(gle)
+ gle = np.pad(gle,
+ (0, 3 - len(gle)),
+ 'constant', constant_values=(0, 0))
+ gre = []
+ for axis in patch["extent"].keys():
+ gre.append(get_component(patch, "extent/" + axis, int(spec[1]), 1)[0])
+ gre = np.asarray(gre)
+ gre = np.pad(gre,
+ (0, 3 - len(gre)),
+ 'constant', constant_values=(0, 1))
+ np.add(gle, gre, gre)
+ npo = patch["numParticlesOffset"].value.item(int(spec[1]))
+ particle_count = np.linspace(npo, npo + count, num_grids + 1,
+ dtype=np.int32)
+ particle_names = [str(pname.split("#")[0])]
else:
- for size in set(self.numparts.values()):
- grid_count = int(np.ceil(size * self.vpg**-1))
+ domain_dimension = self.dataset.domain_dimensions
+ num_grids = int(np.ceil(count * self.vpg ** -1))
+ gle = [-1, -1, -1] # TODO Calculate based on mesh
+ gre = [1, 1, 1] # TODO Calculate based on mesh
+ particle_count = np.linspace(0, count, num_grids + 1, dtype=np.int32)
+ particle_names = []
+ for (pname, size) in self.numparts.items():
+ if size == count:
+ particle_names.append(str(pname))
+ for grid in range(num_grids):
+ self.grid_dimensions[grid_index_total] = domain_dimension
+ self.grid_left_edge[grid_index_total] = gle
+ self.grid_right_edge[grid_index_total] = gre
+ self.grid_particle_count[grid_index_total] = (particle_count[grid + 1] - particle_count[grid]) * len(
+ particle_names)
+ self.grids[grid_index_total] = self.grid(grid_index_total, self, 0,
+ pi=particle_count[grid],
+ po=particle_count[grid + 1] - particle_count[grid],
+ pt=particle_names)
+ grid_index_total += 1
nrp = self.numparts.copy() # Number of remaining particles from the dataset
pci = {} # Index for particle chunk
https://bitbucket.org/yt_analysis/yt/commits/e60a7ab4e001/
Changeset: e60a7ab4e001
Branch: yt
User: Fabian Koller
Date: 2016-08-20 20:44:40+00:00
Summary: Reading data from disk with new indices, offsets works
Affected #: 2 files
diff -r a268110228d8e62bf956de878a22eac6d4fff777 -r e60a7ab4e001b183af8b8aa37b8879e17c89647d yt/frontends/open_pmd/data_structures.py
--- a/yt/frontends/open_pmd/data_structures.py
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -172,7 +172,7 @@
for species in f[bp + pp].keys():
if "particlePatches" in f[bp + pp + "/" + species].keys():
for patch, size in enumerate(f[bp + pp + "/" + species + "/particlePatches/numParticles"]):
- self.numparts[species + str(patch)] = size
+ self.numparts[species + "#" + str(patch)] = size
else:
axis = f[bp + pp + species + "/position"].keys()[0]
if is_const_component(f[bp + pp + species + "/position/" + axis]):
@@ -187,9 +187,18 @@
for shape in set(self.meshshapes.values()):
self.num_grids += min(shape[0], int(np.ceil(functools.reduce(mul, shape) * self.vpg**-1)))
- # Same goes for particle chunks
- for size in set(self.numparts.values()):
- self.num_grids += int(np.ceil(size * self.vpg**-1))
+ # Same goes for particle chunks if they are not inside particlePatches
+ patches = {}
+ no_patches = {}
+ for k, v in self.numparts.items():
+ if "#" in k:
+ patches[k] = v
+ else:
+ no_patches[k] = v
+ for size in set(no_patches.values()):
+ self.num_grids += int(np.ceil(size * self.vpg ** -1))
+ for size in patches.values():
+ self.num_grids += int(np.ceil(size * self.vpg ** -1))
def _parse_index(self):
"""Fills each grid with appropriate properties (extent, dimensions, ...)
@@ -217,7 +226,7 @@
def get_component(group, component_name, index, offset):
record_component = group[component_name]
unit_si = record_component.attrs["unitSI"]
- return np.multiply(record_component[index:offset], unit_si)
+ return np.multiply(record_component[index:index+offset], unit_si)
# Mesh grids
for shape in set(self.meshshapes.values()):
@@ -283,12 +292,12 @@
npo = patch["numParticlesOffset"].value.item(int(spec[1]))
particle_count = np.linspace(npo, npo + count, num_grids + 1,
dtype=np.int32)
- particle_names = [str(pname.split("#")[0])]
+ particle_names = [str(spec[0])]
else:
domain_dimension = self.dataset.domain_dimensions
num_grids = int(np.ceil(count * self.vpg ** -1))
- gle = [-1, -1, -1] # TODO Calculate based on mesh
- gre = [1, 1, 1] # TODO Calculate based on mesh
+ gle = self.dataset.domain_left_edge
+ gre = self.dataset.domain_right_edge
particle_count = np.linspace(0, count, num_grids + 1, dtype=np.int32)
particle_names = []
for (pname, size) in self.numparts.items():
@@ -305,7 +314,7 @@
po=particle_count[grid + 1] - particle_count[grid],
pt=particle_names)
grid_index_total += 1
-
+ """
nrp = self.numparts.copy() # Number of remaining particles from the dataset
pci = {} # Index for particle chunk
for spec in nrp:
@@ -347,6 +356,7 @@
pci[spec] += val
meshindex += self.grid_dimensions[i][0]
remaining -= self.grid_dimensions[i][0]
+ """
def _populate_grid_objects(self):
"""This initializes all grids.
@@ -487,14 +497,12 @@
return False
requirements = ["openPMD", "basePath", "meshesPath", "particlesPath"]
+ attrs = f["/"].attrs.keys()
for i in requirements:
- if i not in f.attrs.keys():
- f.close()
+ if i not in attrs:
return False
- if "1.0." in f.attrs["openPMD"]:
- f.close()
+ if "1.0." not in f.attrs["openPMD"]:
return False
- f.close()
return True
diff -r a268110228d8e62bf956de878a22eac6d4fff777 -r e60a7ab4e001b183af8b8aa37b8879e17c89647d yt/frontends/open_pmd/io.py
--- a/yt/frontends/open_pmd/io.py
+++ b/yt/frontends/open_pmd/io.py
@@ -95,12 +95,12 @@
spec = ds[self.particles_path].keys()[0]
else:
spec = ptype
- index = dict(grid.particle_index).get(spec)
- offset = dict(grid.particle_offset).get(spec)
+ if spec not in grid.ptypes:
+ continue
mylog.debug(
- "open_pmd - _read_particle_coords: (grid {}) {}, {} [{}:{}]".format(grid, spec, field_list, index,
- offset))
- self._fill_cache(spec, index, offset)
+ "open_pmd - _read_particle_coords: (grid {}) {}, {} [{}:{}]".format(grid, spec, field_list,
+ grid.pindex, grid.poffset))
+ self._fill_cache(spec, grid.pindex, grid.poffset)
yield (ptype, (self.cache[0], self.cache[1], self.cache[2]))
def _read_particle_fields(self, chunks, ptf, selector):
@@ -139,12 +139,12 @@
spec = ds[self.particles_path].keys()[0]
else:
spec = ptype
- index = dict(grid.particle_index).get(spec)
- offset = dict(grid.particle_offset).get(spec)
+ if spec not in grid.ptypes:
+ continue
mylog.debug(
- "open_pmd - _read_particle_fields: (grid {}) {}, {} [{}:{}]".format(grid, spec, field_list, index,
- offset))
- self._fill_cache(spec, index, offset)
+ "open_pmd - _read_particle_fields: (grid {}) {}, {} [{}:{}]".format(grid, spec, field_list,
+ grid.pindex, grid.poffset))
+ self._fill_cache(spec, grid.pindex, grid.poffset)
mask = selector.select_points(self.cache[0], self.cache[1], self.cache[2], 0.0)
if mask is None:
continue
@@ -152,7 +152,7 @@
for field in field_list:
component = "/".join(field.split("_")[1:]).replace("positionCoarse",
"position").replace("-", "_")
- data = self.get_component(pds, component, index, offset)
+ data = self.get_component(pds, component, grid.pindex, grid.poffset)
yield ((ptype, field), data[mask])
def _read_fluid_selection(self, chunks, selector, fields, size):
@@ -200,17 +200,17 @@
for ftype, fname in fields:
field = (ftype, fname)
for chunk in chunks:
- for g in chunk.objs:
- mask = g._get_selector_mask(selector)
+ for grid in chunk.objs:
+ if fname.split("_")[0] not in grid.ftypes:
+ continue
+ mask = grid._get_selector_mask(selector)
if mask is None:
continue
component = fname.replace("_", "/").replace("-", "_")
- index = g.mesh_index
- offset = g.mesh_offset
- data = np.array(self.get_component(ds, component, index, offset))
+ data = np.array(self.get_component(ds, component, grid.findex, grid.foffset))
# The following is a modified AMRGridPatch.select(...)
data.shape = mask.shape # Workaround - casts a 2D (x,y) array to 3D (x,y,1)
- count = g.count(selector)
+ count = grid.count(selector)
rv[field][ind[field]:ind[field] + count] = data[mask]
ind[field] += count
for i in rv:
https://bitbucket.org/yt_analysis/yt/commits/d15b4c031823/
Changeset: d15b4c031823
Branch: yt
User: Fabian Koller
Date: 2016-08-22 16:01:16+00:00
Summary: calulcate _particle_type_count based on file,
skip virtual grids with existing particle counts/indices/offsets,
increase speed of particle reading (more efficient use of positional cache)
Affected #: 2 files
diff -r e60a7ab4e001b183af8b8aa37b8879e17c89647d -r d15b4c0318233d4eebf45867c62d232cca9ca3a2 yt/frontends/open_pmd/data_structures.py
--- a/yt/frontends/open_pmd/data_structures.py
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -16,7 +16,6 @@
import os
import re
-from math import ceil, floor
import functools
from operator import mul
@@ -83,6 +82,24 @@
self.directory = os.path.dirname(self.index_filename)
GridIndex.__init__(self, ds, dataset_type)
+ def _get_particle_type_counts(self):
+ result = {}
+ f = self.dataset._handle
+ bp = self.dataset.base_path
+ pp = self.dataset.particles_path
+ for ptype in self.ds.particle_types_raw:
+ if ptype in "io":
+ spec = f[bp + pp].keys()[0]
+ else:
+ spec = ptype
+ axis = f[bp + pp + "/" + spec + "/position"].keys()[0]
+ pos = f[bp + pp + "/" + spec + "/position/" + axis]
+ if is_const_component(pos):
+ result[ptype] = pos.attrs["shape"]
+ else:
+ result[ptype] = pos.len()
+ return result
+
def _detect_output_fields(self):
"""Populates ``self.field_list`` with native fields (mesh and particle) on disk.
@@ -237,8 +254,8 @@
'constant', constant_values=(0, 1))
# Number of grids of this shape
num_grids = min(shape[0], int(np.ceil(functools.reduce(mul, shape) * self.vpg ** -1)))
- gle = [-1, -1, -1] # TODO Calculate based on mesh
- gre = [1, 1, 1] # TODO Calculate based on mesh
+ gle = self.dataset.domain_left_edge # TODO Calculate based on mesh
+ gre = self.dataset.domain_right_edge # TODO Calculate based on mesh
grid_dim_offset = np.linspace(0, domain_dimension[0], num_grids + 1, dtype=np.int32)
grid_edge_offset = grid_dim_offset * np.float(domain_dimension[0]) ** -1 * (gre[0] - gle[0]) + gle[0]
mesh_names = []
@@ -261,9 +278,12 @@
prev = self.grid_dimensions[grid_index_total][0]
grid_index_total += 1
+ handeled_ptypes = []
+
# Particle grids
for species, count in self.numparts.items():
if "#" in species:
+ # This is a particlePatch
spec = species.split("#")
patch = f[bp + pp + "/" + spec[0] + "/particlePatches"]
domain_dimension = []
@@ -293,7 +313,7 @@
particle_count = np.linspace(npo, npo + count, num_grids + 1,
dtype=np.int32)
particle_names = [str(spec[0])]
- else:
+ elif str(species) not in handeled_ptypes:
domain_dimension = self.dataset.domain_dimensions
num_grids = int(np.ceil(count * self.vpg ** -1))
gle = self.dataset.domain_left_edge
@@ -303,6 +323,10 @@
for (pname, size) in self.numparts.items():
if size == count:
particle_names.append(str(pname))
+ handeled_ptypes.append(str(pname))
+ else:
+ # A grid with this exact particle count has already been created
+ continue
for grid in range(num_grids):
self.grid_dimensions[grid_index_total] = domain_dimension
self.grid_left_edge[grid_index_total] = gle
@@ -314,49 +338,6 @@
po=particle_count[grid + 1] - particle_count[grid],
pt=particle_names)
grid_index_total += 1
- """
- nrp = self.numparts.copy() # Number of remaining particles from the dataset
- pci = {} # Index for particle chunk
- for spec in nrp:
- pci[spec] = 0
- remaining = self.dataset.domain_dimensions[0]
- meshindex = 0
- meshedge = self.dataset.domain_left_edge.copy()[0]
- for i in range(self.num_grids):
- self.grid_dimensions[i] = self.dataset.domain_dimensions
- prev = remaining
- remaining -= self.grid_dimensions[i][0] * self.num_grids ** -1
- self.grid_dimensions[i][0] = int(round(prev, 0) - round(remaining, 0))
- self.grid_left_edge[i] = self.dataset.domain_left_edge.copy()
- self.grid_left_edge[i][0] = meshedge
- self.grid_right_edge[i] = self.dataset.domain_right_edge.copy()
- self.grid_right_edge[i][0] = self.grid_left_edge[i][0] + self.grid_dimensions[i][0] * \
- self.dataset.domain_dimensions[0] ** -1 * \
- self.dataset.domain_right_edge[0]
- meshedge = self.grid_right_edge[i][0]
- particleoffset = []
- particleindex = []
- for spec in self.numparts:
- particleindex += [(spec, pci[spec])]
- if i is (self.num_grids - 1):
- # The last grid need not be the same size as the previous ones
- num = nrp[spec]
- else:
- num = int(floor(self.numparts[spec] * self.num_grids ** -1))
- particleoffset += [(spec, num)]
- nrp[spec] -= num
- self.grid_particle_count[i] += num
- self.grids[i] = self.grid(
- i, self, self.grid_levels[i, 0],
- pi=particleindex,
- po=particleoffset,
- mi=meshindex,
- mo=self.grid_dimensions[i][0])
- for spec, val in particleoffset:
- pci[spec] += val
- meshindex += self.grid_dimensions[i][0]
- remaining -= self.grid_dimensions[i][0]
- """
def _populate_grid_objects(self):
"""This initializes all grids.
@@ -502,7 +483,9 @@
if i not in attrs:
return False
- if "1.0." not in f.attrs["openPMD"]:
- return False
+ versions = ["1.0.0", "1.0.1"]
+ for i in versions:
+ if i in f.attrs["openPMD"]:
+ return True
- return True
+ return False
diff -r e60a7ab4e001b183af8b8aa37b8879e17c89647d -r d15b4c0318233d4eebf45867c62d232cca9ca3a2 yt/frontends/open_pmd/io.py
--- a/yt/frontends/open_pmd/io.py
+++ b/yt/frontends/open_pmd/io.py
@@ -21,6 +21,8 @@
from yt.utilities.io_handler import BaseIOHandler
from yt.utilities.logger import ytLogger as mylog
+from collections import defaultdict
+
class IOHandlerOpenPMD(BaseIOHandler):
_field_dtype = "float32"
@@ -66,94 +68,73 @@
# These have to be the same shape as the existing axes since that equals the number of particles
self.cache[i] = np.zeros(offset)
- def _read_particle_coords(self, chunks, ptf):
- """Reads coordinates for given particle-types in given chunks from file.
+ def _read_particle_selection(self, chunks, selector, fields):
+ """
Parameters
----------
chunks
- A list of chunks
- A chunk is a list of grids
- ptf : dict
- keys are ptypes
- values are lists of particle fields
+ selector
+ fields
- Yields
- ------
- tuple : (str, ((N,) ndarray, (N,) ndarray, (N,) ndarray))
- Tuple of ptype and tuple of coordinates (x, y, z) corresponding to coordinates of particles of that ptype
- All coordinate arrays have the same length
+ Returns
+ -------
+
"""
-
- chunks = list(chunks)
+ rv = {}
+ ind = {}
+ particle_count = {}
+ on_disk_ptypes_and_fields = defaultdict(list)
+ mapping_from_on_disk_to_request = defaultdict(list)
f = self._handle
ds = f[self.base_path]
- for chunk in chunks:
- for grid in chunk.objs:
- for ptype, field_list in sorted(ptf.items()):
+ unions = self.ds.particle_unions
+
+ for ptype, pname in fields:
+ pfield = ptype, pname
+ # Overestimate the size of all pfields so they include all particles, shrink it later
+ particle_count[pfield] = 0
+ if ptype in unions:
+ for pt in unions[ptype]:
+ particle_count[pfield] += self.ds.particle_type_counts[pt]
+ on_disk_ptypes_and_fields[pt].append(pname)
+ mapping_from_on_disk_to_request[pt, pname].append(pfield)
+ else:
+ particle_count[pfield] = self.ds.particle_type_counts[ptype]
+ on_disk_ptypes_and_fields[ptype].append(pname)
+ mapping_from_on_disk_to_request[pfield].append(pfield)
+ rv[pfield] = np.empty((particle_count[pfield],), dtype="float64")
+ ind[pfield] = 0
+
+ for ptype in on_disk_ptypes_and_fields:
+ for chunk in chunks:
+ for grid in chunk.objs:
if ptype in "io":
spec = ds[self.particles_path].keys()[0]
else:
spec = ptype
+ mylog.debug("spec {} grid.ptypes {}".format(spec, grid.ptypes))
if spec not in grid.ptypes:
continue
- mylog.debug(
- "open_pmd - _read_particle_coords: (grid {}) {}, {} [{}:{}]".format(grid, spec, field_list,
- grid.pindex, grid.poffset))
- self._fill_cache(spec, grid.pindex, grid.poffset)
- yield (ptype, (self.cache[0], self.cache[1], self.cache[2]))
-
- def _read_particle_fields(self, chunks, ptf, selector):
- """Reads given fields for given particle types masked by a given selection.
-
- Parameters
- ----------
- chunks
- A list of chunks
- A chunk is a list of grids
- ptf : dict
- keys are ptype
- values are lists of particle fields
- selector
- A region (inside your domain) specifying which parts of the field you want to read
- See [1] and [2]
-
- References
- ----------
- .. [1] yt-project.org/docs/dev/quickstart/data_inspection.html?highlight=selector#Examining-Data-in-Regions
- .. [2] yt-project.org/doc/developing/creating_datatypes.html
-
- Yields
- ------
- tuple : ((str, str), (N,) ndarray)
- Tuple of tuple (ptype, fieldname) and masked field-data
- """
- chunks = list(chunks)
- f = self._handle
- ds = f[self.base_path]
- for chunk in chunks:
- for grid in chunk.objs:
- for ptype, field_list in sorted(ptf.items()):
- # Get a particle species (e.g. /data/3500/particles/e/)
- if ptype in "io":
- spec = ds[self.particles_path].keys()[0]
- else:
- spec = ptype
- if spec not in grid.ptypes:
- continue
- mylog.debug(
- "open_pmd - _read_particle_fields: (grid {}) {}, {} [{}:{}]".format(grid, spec, field_list,
- grid.pindex, grid.poffset))
+ # read particle coords into cache
self._fill_cache(spec, grid.pindex, grid.poffset)
mask = selector.select_points(self.cache[0], self.cache[1], self.cache[2], 0.0)
if mask is None:
continue
pds = ds[self.particles_path + "/" + spec]
- for field in field_list:
- component = "/".join(field.split("_")[1:]).replace("positionCoarse",
- "position").replace("-", "_")
- data = self.get_component(pds, component, grid.pindex, grid.poffset)
- yield ((ptype, field), data[mask])
+ for field in on_disk_ptypes_and_fields[ptype]:
+ component = "/".join(field.split("_")[1:]).replace("positionCoarse", "position").replace("-",
+ "_")
+ data = self.get_component(pds, component, grid.pindex, grid.poffset)[mask]
+ for request_field in mapping_from_on_disk_to_request[(ptype, field)]:
+ my_ind = ind[request_field]
+ rv[request_field][my_ind:my_ind + data.shape[0], ...] = data
+ ind[request_field] += data.shape[0]
+
+ for field in fields:
+ rv[field] = rv[field][:ind[field]]
+
+ return rv
def _read_fluid_selection(self, chunks, selector, fields, size):
"""Reads given fields for given meshes masked by a given selection.
@@ -177,7 +158,6 @@
keys are tuples (ftype, fname) representing a field
values are flat (``size``,) ndarrays with data from that field
"""
- mylog.debug("open_pmd - _read_fluid_selection {} {} {} {}".format(chunks, selector, fields, size))
f = self._handle
bp = self.base_path
mp = self.meshes_path
@@ -201,6 +181,7 @@
field = (ftype, fname)
for chunk in chunks:
for grid in chunk.objs:
+ mylog.debug("open_pmd - _read_fluid_selection {} {} {}".format(grid, field, size))
if fname.split("_")[0] not in grid.ftypes:
continue
mask = grid._get_selector_mask(selector)
https://bitbucket.org/yt_analysis/yt/commits/d4186175f7da/
Changeset: d4186175f7da
Branch: yt
User: Fabian Koller
Date: 2016-08-23 09:40:25+00:00
Summary: Change dimension of grids with particles to 0 so they do not get counted towards data_size of meshes
Affected #: 2 files
diff -r d15b4c0318233d4eebf45867c62d232cca9ca3a2 -r d4186175f7daf0237f604fb9611b4da95f862c54 yt/frontends/open_pmd/data_structures.py
--- a/yt/frontends/open_pmd/data_structures.py
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -83,6 +83,14 @@
GridIndex.__init__(self, ds, dataset_type)
def _get_particle_type_counts(self):
+ """Sets the active number of particles for every species.
+
+ Returns
+ -------
+ dict
+ keys are ptypes
+ values are integer counts of the ptype
+ """
result = {}
f = self.dataset._handle
bp = self.dataset.base_path
@@ -221,30 +229,29 @@
"""Fills each grid with appropriate properties (extent, dimensions, ...)
This calculates the properties of every OpenPMDGrid based on the total number of grids in the simulation.
- The domain is divided into ``self.num_grids`` equally sized chunks along the x-axis.
+ The domain is divided into ``self.num_grids`` (roughly) equally sized chunks along the x-axis.
``grid_levels`` is always equal to 0 since we only have one level of refinement in openPMD.
Notes
-----
- ``self.grid_dimensions`` is rounded to the nearest integer.
+ ``self.grid_dimensions`` is rounded to the nearest integer. Grid edges are calculated from this dimension.
Furthermore the last grid might be smaller and have fewer particles than the others.
In general, NOT all particles in a grid will be inside the grid edges.
"""
f = self.dataset._handle
bp = self.dataset.base_path
- mp = self.dataset.meshes_path
pp = self.dataset.particles_path
self.grid_levels.flat[:] = 0
self.grids = np.empty(self.num_grids, dtype="object")
- grid_index_total = 0
-
def get_component(group, component_name, index, offset):
record_component = group[component_name]
unit_si = record_component.attrs["unitSI"]
return np.multiply(record_component[index:index+offset], unit_si)
+ grid_index_total = 0
+
# Mesh grids
for shape in set(self.meshshapes.values()):
# Total dimension of this grid
@@ -254,8 +261,8 @@
'constant', constant_values=(0, 1))
# Number of grids of this shape
num_grids = min(shape[0], int(np.ceil(functools.reduce(mul, shape) * self.vpg ** -1)))
- gle = self.dataset.domain_left_edge # TODO Calculate based on mesh
- gre = self.dataset.domain_right_edge # TODO Calculate based on mesh
+ gle = self.dataset.domain_left_edge
+ gre = self.dataset.domain_right_edge
grid_dim_offset = np.linspace(0, domain_dimension[0], num_grids + 1, dtype=np.int32)
grid_edge_offset = grid_dim_offset * np.float(domain_dimension[0]) ** -1 * (gre[0] - gle[0]) + gle[0]
mesh_names = []
@@ -275,10 +282,10 @@
fi=prev,
fo=self.grid_dimensions[grid_index_total][0],
ft=mesh_names)
- prev = self.grid_dimensions[grid_index_total][0]
+ prev += self.grid_dimensions[grid_index_total][0]
grid_index_total += 1
- handeled_ptypes = []
+ handled_ptypes = []
# Particle grids
for species, count in self.numparts.items():
@@ -286,13 +293,6 @@
# This is a particlePatch
spec = species.split("#")
patch = f[bp + pp + "/" + spec[0] + "/particlePatches"]
- domain_dimension = []
- for axis in patch["extent"].keys():
- domain_dimension.append(patch["extent/" + axis].value.item(int(spec[1])))
- domain_dimension = np.asarray(domain_dimension, dtype=np.int32)
- domain_dimension = np.pad(domain_dimension,
- (0, 3 - len(domain_dimension)),
- 'constant', constant_values=(0, 1))
num_grids = int(np.ceil(count * self.vpg ** -1))
gle = []
for axis in patch["offset"].keys():
@@ -313,8 +313,7 @@
particle_count = np.linspace(npo, npo + count, num_grids + 1,
dtype=np.int32)
particle_names = [str(spec[0])]
- elif str(species) not in handeled_ptypes:
- domain_dimension = self.dataset.domain_dimensions
+ elif str(species) not in handled_ptypes:
num_grids = int(np.ceil(count * self.vpg ** -1))
gle = self.dataset.domain_left_edge
gre = self.dataset.domain_right_edge
@@ -322,13 +321,14 @@
particle_names = []
for (pname, size) in self.numparts.items():
if size == count:
+ # Since this is not part of a particlePatch, we can include multiple same-sized ptypes
particle_names.append(str(pname))
- handeled_ptypes.append(str(pname))
+ handled_ptypes.append(str(pname))
else:
# A grid with this exact particle count has already been created
continue
for grid in range(num_grids):
- self.grid_dimensions[grid_index_total] = domain_dimension
+ self.grid_dimensions[grid_index_total] = [0, 0, 0] # Counted as mesh-size, thus no dimensional extent
self.grid_left_edge[grid_index_total] = gle
self.grid_right_edge[grid_index_total] = gre
self.grid_particle_count[grid_index_total] = (particle_count[grid + 1] - particle_count[grid]) * len(
@@ -454,16 +454,22 @@
fshape = np.append(fshape, np.ones(3 - self.dimensionality))
self.domain_dimensions = fshape
+
self.domain_left_edge = np.zeros(3, dtype=np.float64)
try:
mesh = f[bp + mp].keys()[0]
spacing = np.asarray(f[bp + mp + "/" + mesh].attrs["gridSpacing"])
- unit_si = f[bp + mp + "/" + mesh].attrs["gridUnitSI"]
+ #offset = np.asarray(f[bp + mp + "/" + mesh].attrs["gridGlobalOffset"])
+ unit_si = np.asarray(f[bp + mp + "/" + mesh].attrs["gridUnitSI"])
+ #self.domain_left_edge = self.domain_dimensions[:spacing.size] * unit_si * offset
self.domain_right_edge = self.domain_dimensions[:spacing.size] * unit_si * spacing
+ #self.domain_right_edge += self.domain_left_edge
+ #self.domain_left_edge = np.append(self.domain_left_edge, np.zeros(3 - self.domain_left_edge.size))
self.domain_right_edge = np.append(self.domain_right_edge, np.ones(3 - self.domain_right_edge.size))
except Exception as e:
mylog.warning(
"open_pmd - The domain extent could not be calculated! ({}) Setting the field extent to 1m**3!".format(e))
+ self.domain_left_edge = np.zeros(3, dtype=np.float64)
self.domain_right_edge = np.ones(3, dtype=np.float64)
self.current_time = f[bp].attrs["time"] * f[bp].attrs["timeUnitSI"]
diff -r d15b4c0318233d4eebf45867c62d232cca9ca3a2 -r d4186175f7daf0237f604fb9611b4da95f862c54 yt/frontends/open_pmd/io.py
--- a/yt/frontends/open_pmd/io.py
+++ b/yt/frontends/open_pmd/io.py
@@ -84,6 +84,7 @@
rv = {}
ind = {}
particle_count = {}
+ chunks = list(chunks) # chunks is a generator and thus depleted when you iterate over it once
on_disk_ptypes_and_fields = defaultdict(list)
mapping_from_on_disk_to_request = defaultdict(list)
f = self._handle
@@ -103,7 +104,7 @@
particle_count[pfield] = self.ds.particle_type_counts[ptype]
on_disk_ptypes_and_fields[ptype].append(pname)
mapping_from_on_disk_to_request[pfield].append(pfield)
- rv[pfield] = np.empty((particle_count[pfield],), dtype="float64")
+ rv[pfield] = np.empty((particle_count[pfield],), dtype=np.float64)
ind[pfield] = 0
for ptype in on_disk_ptypes_and_fields:
@@ -113,7 +114,6 @@
spec = ds[self.particles_path].keys()[0]
else:
spec = ptype
- mylog.debug("spec {} grid.ptypes {}".format(spec, grid.ptypes))
if spec not in grid.ptypes:
continue
# read particle coords into cache
@@ -194,8 +194,11 @@
count = grid.count(selector)
rv[field][ind[field]:ind[field] + count] = data[mask]
ind[field] += count
- for i in rv:
- rv[i].flatten()
+
+ for field in fields:
+ rv[field] = rv[field][:ind[field]]
+ rv[field].flatten()
+
return rv
def get_component(self, group, component_name, index=0, offset=None):
@@ -227,7 +230,7 @@
if offset is None:
offset = record_component.attrs["shape"] - index
# component is constant, craft an array by hand
- mylog.debug("open_pmd - get_component (const): {}/{}({})".format(group.name, component_name, offset))
+ mylog.debug("open_pmd - get_component (const): {}/{} ({})".format(group.name, component_name, offset))
return np.full(offset, record_component.attrs["value"] * unit_si)
else:
if offset is not None:
https://bitbucket.org/yt_analysis/yt/commits/befb7ffa14b6/
Changeset: befb7ffa14b6
Branch: yt
User: Fabian Koller
Date: 2016-08-23 12:42:25+00:00
Summary: Python 2/3 compatability,
make gridsize settable by the user (kwarg),
Affected #: 4 files
diff -r d4186175f7daf0237f604fb9611b4da95f862c54 -r befb7ffa14b6104c6c78ff76f0ce148103e1fcef yt/frontends/open_pmd/data_structures.py
--- a/yt/frontends/open_pmd/data_structures.py
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -16,7 +16,7 @@
import os
import re
-import functools
+from functools import reduce
from operator import mul
import numpy as np
@@ -97,10 +97,10 @@
pp = self.dataset.particles_path
for ptype in self.ds.particle_types_raw:
if ptype in "io":
- spec = f[bp + pp].keys()[0]
+ spec = list(f[bp + pp].keys())[0]
else:
spec = ptype
- axis = f[bp + pp + "/" + spec + "/position"].keys()[0]
+ axis = list(f[bp + pp + "/" + spec + "/position"].keys())[0]
pos = f[bp + pp + "/" + spec + "/position/" + axis]
if is_const_component(pos):
result[ptype] = pos.attrs["shape"]
@@ -124,9 +124,9 @@
mesh_fields = []
try:
- for field in f[bp + mp].keys():
+ for field in list(f[bp + mp].keys()):
try:
- for axis in f[bp + mp + field].keys():
+ for axis in list(f[bp + mp + field].keys()):
mesh_fields.append(field + "_" + axis)
except AttributeError:
# This is a h5.Dataset (i.e. no axes)
@@ -138,8 +138,8 @@
particle_fields = []
try:
- for species in f[bp + pp].keys():
- for record in f[bp + pp + species].keys():
+ for species in list(f[bp + pp].keys()):
+ for record in list(f[bp + pp + species].keys()):
if is_const_component(f[bp + pp + species + "/" + record]):
# Record itself (e.g. particle_mass) is constant
particle_fields.append(species + "_" + record)
@@ -147,7 +147,7 @@
try:
# Create a field for every axis (x,y,z) of every property (position)
# of every species (electrons)
- axes = f[bp + pp + species + "/" + record].keys()
+ axes = list(f[bp + pp + species + "/" + record].keys())
if record in "position":
record = "positionCoarse"
for axis in axes:
@@ -158,7 +158,7 @@
pass
else:
pass
- if len(f[bp + pp].keys()) > 1:
+ if len(list(f[bp + pp].keys())) > 1:
# There is more than one particle species, use the specific names as field types
self.field_list.extend(
[(str(field).split("_")[0],
@@ -183,46 +183,45 @@
mp = self.dataset.meshes_path
pp = self.dataset.particles_path
- gridsize = 12 * 10 ** 6 # Byte
self.meshshapes = {}
self.numparts = {}
self.num_grids = 0
- for mesh in f[bp + mp].keys():
+ for mesh in list(f[bp + mp].keys()):
if type(f[bp + mp + mesh]) is h5.Group:
- self.meshshapes[mesh] = f[bp + mp + mesh + "/" + f[bp + mp + mesh].keys()[0]].shape
+ self.meshshapes[mesh] = f[bp + mp + mesh + "/" + list(f[bp + mp + mesh].keys())[0]].shape
else:
self.meshshapes[mesh] = f[bp + mp + mesh].shape
- for species in f[bp + pp].keys():
- if "particlePatches" in f[bp + pp + "/" + species].keys():
+ for species in list(f[bp + pp].keys()):
+ if "particlePatches" in list(f[bp + pp + "/" + species].keys()):
for patch, size in enumerate(f[bp + pp + "/" + species + "/particlePatches/numParticles"]):
self.numparts[species + "#" + str(patch)] = size
else:
- axis = f[bp + pp + species + "/position"].keys()[0]
+ axis = list(f[bp + pp + species + "/position"].keys())[0]
if is_const_component(f[bp + pp + species + "/position/" + axis]):
self.numparts[species] = f[bp + pp + species + "/position/" + axis].attrs["shape"]
else:
self.numparts[species] = f[bp + pp + species + "/position/" + axis].len()
# Limit values per grid by resulting memory footprint
- self.vpg = int(gridsize / (self.dataset.dimensionality * 4)) # 4 Byte per value per dimension (f32)
+ self.vpg = int(self.dataset.gridsize / (self.dataset.dimensionality * 4)) # 4Byte per value per dimension (f32)
# Meshes of the same size do not need separate chunks
for shape in set(self.meshshapes.values()):
- self.num_grids += min(shape[0], int(np.ceil(functools.reduce(mul, shape) * self.vpg**-1)))
+ self.num_grids += min(shape[0], int(np.ceil(reduce(mul, shape) * self.vpg**-1)))
# Same goes for particle chunks if they are not inside particlePatches
patches = {}
no_patches = {}
- for k, v in self.numparts.items():
+ for k, v in list(self.numparts.items()):
if "#" in k:
patches[k] = v
else:
no_patches[k] = v
for size in set(no_patches.values()):
self.num_grids += int(np.ceil(size * self.vpg ** -1))
- for size in patches.values():
+ for size in list(patches.values()):
self.num_grids += int(np.ceil(size * self.vpg ** -1))
def _parse_index(self):
@@ -260,13 +259,13 @@
(0, 3 - len(domain_dimension)),
'constant', constant_values=(0, 1))
# Number of grids of this shape
- num_grids = min(shape[0], int(np.ceil(functools.reduce(mul, shape) * self.vpg ** -1)))
+ num_grids = min(shape[0], int(np.ceil(reduce(mul, shape) * self.vpg ** -1)))
gle = self.dataset.domain_left_edge
gre = self.dataset.domain_right_edge
grid_dim_offset = np.linspace(0, domain_dimension[0], num_grids + 1, dtype=np.int32)
grid_edge_offset = grid_dim_offset * np.float(domain_dimension[0]) ** -1 * (gre[0] - gle[0]) + gle[0]
mesh_names = []
- for (mname, mshape) in self.meshshapes.items():
+ for (mname, mshape) in list(self.meshshapes.items()):
if shape == mshape:
mesh_names.append(str(mname))
prev = 0
@@ -288,21 +287,21 @@
handled_ptypes = []
# Particle grids
- for species, count in self.numparts.items():
+ for species, count in list(self.numparts.items()):
if "#" in species:
# This is a particlePatch
spec = species.split("#")
patch = f[bp + pp + "/" + spec[0] + "/particlePatches"]
num_grids = int(np.ceil(count * self.vpg ** -1))
gle = []
- for axis in patch["offset"].keys():
+ for axis in list(patch["offset"].keys()):
gle.append(get_component(patch, "offset/" + axis, int(spec[1]), 1)[0])
gle = np.asarray(gle)
gle = np.pad(gle,
(0, 3 - len(gle)),
'constant', constant_values=(0, 0))
gre = []
- for axis in patch["extent"].keys():
+ for axis in list(patch["extent"].keys()):
gre.append(get_component(patch, "extent/" + axis, int(spec[1]), 1)[0])
gre = np.asarray(gre)
gre = np.pad(gre,
@@ -319,7 +318,7 @@
gre = self.dataset.domain_right_edge
particle_count = np.linspace(0, count, num_grids + 1, dtype=np.int32)
particle_names = []
- for (pname, size) in self.numparts.items():
+ for (pname, size) in list(self.numparts.items()):
if size == count:
# Since this is not part of a particlePatch, we can include multiple same-sized ptypes
particle_names.append(str(pname))
@@ -360,7 +359,7 @@
def __init__(self, filename, dataset_type="openPMD",
storage_filename=None,
units_override=None,
- unit_system="mks"):
+ unit_system="mks", **kwargs):
self._handle = HDF5FileHandler(filename)
self._set_paths(self._handle, os.path.dirname(filename))
Dataset.__init__(self, filename, dataset_type,
@@ -368,11 +367,12 @@
unit_system=unit_system)
self.storage_filename = storage_filename
self.fluid_types += ("openPMD",)
- particles = tuple(str(c) for c in self._handle[self.base_path + self.particles_path].keys())
+ particles = tuple(str(c) for c in list(self._handle[self.base_path + self.particles_path].keys()))
if len(particles) > 1:
# Only use on-disk particle names if there is more than one species
self.particle_types = particles
mylog.debug("open_pmd - self.particle_types: {}".format(self.particle_types))
+ self.gridsize = kwargs.pop("open_pmd_virtual_gridsize", 12*10**6)
self.particle_types_raw = self.particle_types
self.particle_types = tuple(self.particle_types)
@@ -385,37 +385,34 @@
path : str
(absolute) filepath for current hdf5 container
"""
- list_iterations = []
- if "groupBased" in handle.attrs["iterationEncoding"]:
- for i in list(handle["/data"].keys()):
- list_iterations.append(i)
- mylog.info("open_pmd - found {} iterations in file".format(len(list_iterations)))
- elif "fileBased" in handle.attrs["iterationEncoding"]:
- regex = "^" + handle.attrs["iterationFormat"].replace("%T", "[0-9]+") + "$"
+ iterations = []
+ encoding = handle.attrs["iterationEncoding"].decode()
+ if "groupBased" in encoding:
+ iterations = list(handle["/data"].keys())
+ mylog.info("open_pmd - found {} iterations in file".format(len(iterations)))
+ elif "fileBased" in encoding:
+ regex = "^" + handle.attrs["iterationFormat"].decode().replace("%T", "[0-9]+") + "$"
if path is "":
mylog.warning("open_pmd - For file based iterations, please use absolute file paths!")
pass
for filename in os.listdir(path):
if re.match(regex, filename):
- list_iterations.append(filename)
- mylog.info("open_pmd - found {} iterations in directory".format(len(list_iterations)))
- else:
- mylog.warning(
- "open_pmd - No valid iteration encoding: {}".format(handle.attrs["iterationEncoding"]))
+ iterations.append(filename)
+ mylog.info("open_pmd - found {} iterations in directory".format(len(iterations)))
- if len(list_iterations) == 0:
+ if len(iterations) == 0:
mylog.warning("open_pmd - no iterations found!")
- if "groupBased" in handle.attrs["iterationEncoding"] and len(list_iterations) > 1:
- mylog.warning("open_pmd - only choose to load one iteration ({})".format(handle["/data"].keys()[0]))
+ if "groupBased" in encoding and len(iterations) > 1:
+ mylog.warning("open_pmd - only choose to load one iteration ({})".format(list(handle["/data"].keys())[0]))
- self.base_path = "/data/{}/".format(handle["/data"].keys()[0])
- self.meshes_path = self._handle["/"].attrs["meshesPath"]
- self.particles_path = self._handle["/"].attrs["particlesPath"]
+ self.base_path = "/data/{}/".format(list(handle["/data"].keys())[0])
+ self.meshes_path = self._handle["/"].attrs["meshesPath"].decode()
+ self.particles_path = self._handle["/"].attrs["particlesPath"].decode()
def _set_code_unit_attributes(self):
"""Handle conversion between different physical units and the code units.
- These are hardcoded as 1.0. Every dataset in openPMD can have different code <-> physical scaling.
+ Every dataset in openPMD can have different code <-> physical scaling.
The individual factor is obtained by multiplying with "unitSI" reading getting data from disk.
"""
setdefaultattr(self, "length_unit", self.quan(1.0, "m"))
@@ -442,8 +439,8 @@
self.cosmological_simulation = 0
try:
- mesh = f[bp + mp].keys()[0]
- axis = f[bp + mp + "/" + mesh].keys()[0]
+ mesh = list(f[bp + mp].keys())[0]
+ axis = list(f[bp + mp + "/" + mesh].keys())[0]
fshape = np.asarray(f[bp + mp + "/" + mesh + "/" + axis].shape, dtype=np.int64)
except:
fshape = np.array([1, 1, 1], dtype=np.int64)
@@ -457,7 +454,7 @@
self.domain_left_edge = np.zeros(3, dtype=np.float64)
try:
- mesh = f[bp + mp].keys()[0]
+ mesh = list(f[bp + mp].keys())[0]
spacing = np.asarray(f[bp + mp + "/" + mesh].attrs["gridSpacing"])
#offset = np.asarray(f[bp + mp + "/" + mesh].attrs["gridGlobalOffset"])
unit_si = np.asarray(f[bp + mp + "/" + mesh].attrs["gridUnitSI"])
@@ -484,14 +481,14 @@
return False
requirements = ["openPMD", "basePath", "meshesPath", "particlesPath"]
- attrs = f["/"].attrs.keys()
+ attrs = list(f["/"].attrs.keys())
for i in requirements:
if i not in attrs:
return False
versions = ["1.0.0", "1.0.1"]
for i in versions:
- if i in f.attrs["openPMD"]:
+ if i in f.attrs["openPMD"].decode():
return True
return False
diff -r d4186175f7daf0237f604fb9611b4da95f862c54 -r befb7ffa14b6104c6c78ff76f0ce148103e1fcef yt/frontends/open_pmd/fields.py
--- a/yt/frontends/open_pmd/fields.py
+++ b/yt/frontends/open_pmd/fields.py
@@ -145,7 +145,7 @@
pp = ds.particles_path
fields = f[bp + mp]
- for fname in fields.keys():
+ for fname in list(fields.keys()):
field = fields[fname]
if type(field) is h5.Dataset:
# Don't consider axes. This appears to be a vector field of single dimensionality
@@ -159,7 +159,7 @@
self._mag_fields.append(ytname)
self.known_other_fields += ((ytname, (unit, aliases, None)),)
else:
- axes = field.keys()
+ axes = list(field.keys())
for axis in axes:
ytname = str("_".join([fname.replace("_", "-"), axis]))
parsed = parse_unit_dimension(np.asarray(field.attrs["unitDimension"], dtype=np.int))
@@ -174,8 +174,8 @@
mylog.debug("open_pmd - known_other_fields - {}".format(i))
particle_fields = ()
particles = f[bp + pp]
- for species in particles.keys():
- for attrib in particles[species].keys():
+ for species in list(particles.keys()):
+ for attrib in list(particles[species].keys()):
try:
parsed = parse_unit_dimension(
np.asarray(particles.get(species).get(attrib).attrs["unitDimension"],
@@ -191,7 +191,7 @@
if type(pds) is h5.Dataset or is_const_component(pds):
particle_fields += ((str("_".join(name)), (unit, [], None)),)
else:
- for axis in pds.keys():
+ for axis in list(pds.keys()):
aliases = []
if axis in "rxyz":
name = ["particle", ytattrib.replace("_", "-"), axis]
diff -r d4186175f7daf0237f604fb9611b4da95f862c54 -r befb7ffa14b6104c6c78ff76f0ce148103e1fcef yt/frontends/open_pmd/io.py
--- a/yt/frontends/open_pmd/io.py
+++ b/yt/frontends/open_pmd/io.py
@@ -50,7 +50,7 @@
if str((ptype, index, offset)) not in self._cached_ptype:
self._cached_ptype = str((ptype, index, offset))
pds = self._handle[self.base_path + self.particles_path + "/" + ptype]
- axes = [str(ax) for ax in pds["position"].keys()]
+ axes = [str(ax) for ax in list(pds["position"].keys())]
if offset is None:
if is_const_component(pds["position/" + axes[0]]):
offset = pds["position/" + axes[0]].attrs["shape"]
@@ -111,7 +111,7 @@
for chunk in chunks:
for grid in chunk.objs:
if ptype in "io":
- spec = ds[self.particles_path].keys()[0]
+ spec = list(ds[self.particles_path].keys())[0]
else:
spec = ptype
if spec not in grid.ptypes:
@@ -230,7 +230,7 @@
if offset is None:
offset = record_component.attrs["shape"] - index
# component is constant, craft an array by hand
- mylog.debug("open_pmd - get_component (const): {}/{} ({})".format(group.name, component_name, offset))
+ mylog.debug("open_pmd - get_component: {}/{} [const {}]".format(group.name, component_name, offset))
return np.full(offset, record_component.attrs["value"] * unit_si)
else:
if offset is not None:
diff -r d4186175f7daf0237f604fb9611b4da95f862c54 -r befb7ffa14b6104c6c78ff76f0ce148103e1fcef yt/frontends/open_pmd/misc.py
--- a/yt/frontends/open_pmd/misc.py
+++ b/yt/frontends/open_pmd/misc.py
@@ -79,4 +79,4 @@
.. https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md,
section 'Constant Record Components'
"""
- return "value" in record_component.attrs.keys()
+ return "value" in list(record_component.attrs.keys())
https://bitbucket.org/yt_analysis/yt/commits/58b99dc48b19/
Changeset: 58b99dc48b19
Branch: yt
User: Fabian Koller
Date: 2016-08-23 13:39:16+00:00
Summary: Optimize imports,
unify array padding
Affected #: 3 files
diff -r befb7ffa14b6104c6c78ff76f0ce148103e1fcef -r 58b99dc48b192a4f78b8d11768810d1af7c8e088 yt/frontends/open_pmd/data_structures.py
--- a/yt/frontends/open_pmd/data_structures.py
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -14,18 +14,20 @@
# The full license is in the file COPYING.txt, distributed with this software.
# -----------------------------------------------------------------------------
-import os
-import re
from functools import reduce
from operator import mul
+from os import \
+ path, \
+ listdir
+from re import match
import numpy as np
from yt.data_objects.grid_patch import AMRGridPatch
from yt.data_objects.static_output import Dataset
-from yt.funcs import setdefaultattr
from yt.frontends.open_pmd.fields import OpenPMDFieldInfo
from yt.frontends.open_pmd.misc import is_const_component
+from yt.funcs import setdefaultattr
from yt.geometry.grid_geometry_handler import GridIndex
from yt.utilities.file_handler import HDF5FileHandler
from yt.utilities.logger import ytLogger as mylog
@@ -79,7 +81,7 @@
self.dataset_type = dataset_type
self.dataset = ds
self.index_filename = ds.parameter_filename
- self.directory = os.path.dirname(self.index_filename)
+ self.directory = path.dirname(self.index_filename)
GridIndex.__init__(self, ds, dataset_type)
def _get_particle_type_counts(self):
@@ -255,9 +257,7 @@
for shape in set(self.meshshapes.values()):
# Total dimension of this grid
domain_dimension = np.asarray(shape, dtype=np.int32)
- domain_dimension = np.pad(domain_dimension,
- (0, 3 - len(domain_dimension)),
- 'constant', constant_values=(0, 1))
+ domain_dimension = np.append(domain_dimension, np.ones(3 - len(domain_dimension)))
# Number of grids of this shape
num_grids = min(shape[0], int(np.ceil(reduce(mul, shape) * self.vpg ** -1)))
gle = self.dataset.domain_left_edge
@@ -297,16 +297,12 @@
for axis in list(patch["offset"].keys()):
gle.append(get_component(patch, "offset/" + axis, int(spec[1]), 1)[0])
gle = np.asarray(gle)
- gle = np.pad(gle,
- (0, 3 - len(gle)),
- 'constant', constant_values=(0, 0))
+ gle = np.append(gle, np.zeros(3 - len(gle)))
gre = []
for axis in list(patch["extent"].keys()):
gre.append(get_component(patch, "extent/" + axis, int(spec[1]), 1)[0])
gre = np.asarray(gre)
- gre = np.pad(gre,
- (0, 3 - len(gre)),
- 'constant', constant_values=(0, 1))
+ gre = np.append(gre, np.ones(3 - len(gre)))
np.add(gle, gre, gre)
npo = patch["numParticlesOffset"].value.item(int(spec[1]))
particle_count = np.linspace(npo, npo + count, num_grids + 1,
@@ -361,7 +357,7 @@
units_override=None,
unit_system="mks", **kwargs):
self._handle = HDF5FileHandler(filename)
- self._set_paths(self._handle, os.path.dirname(filename))
+ self._set_paths(self._handle, path.dirname(filename))
Dataset.__init__(self, filename, dataset_type,
units_override=units_override,
unit_system=unit_system)
@@ -395,8 +391,8 @@
if path is "":
mylog.warning("open_pmd - For file based iterations, please use absolute file paths!")
pass
- for filename in os.listdir(path):
- if re.match(regex, filename):
+ for filename in listdir(path):
+ if match(regex, filename):
iterations.append(filename)
mylog.info("open_pmd - found {} iterations in directory".format(len(iterations)))
@@ -451,7 +447,6 @@
fshape = np.append(fshape, np.ones(3 - self.dimensionality))
self.domain_dimensions = fshape
-
self.domain_left_edge = np.zeros(3, dtype=np.float64)
try:
mesh = list(f[bp + mp].keys())[0]
@@ -463,9 +458,9 @@
#self.domain_right_edge += self.domain_left_edge
#self.domain_left_edge = np.append(self.domain_left_edge, np.zeros(3 - self.domain_left_edge.size))
self.domain_right_edge = np.append(self.domain_right_edge, np.ones(3 - self.domain_right_edge.size))
- except Exception as e:
+ except:
mylog.warning(
- "open_pmd - The domain extent could not be calculated! ({}) Setting the field extent to 1m**3!".format(e))
+ "open_pmd - The domain extent could not be calculated! Setting the field extent to 1m**3!")
self.domain_left_edge = np.zeros(3, dtype=np.float64)
self.domain_right_edge = np.ones(3, dtype=np.float64)
diff -r befb7ffa14b6104c6c78ff76f0ce148103e1fcef -r 58b99dc48b192a4f78b8d11768810d1af7c8e088 yt/frontends/open_pmd/fields.py
--- a/yt/frontends/open_pmd/fields.py
+++ b/yt/frontends/open_pmd/fields.py
@@ -15,8 +15,8 @@
# The full license is in the file COPYING.txt, distributed with this software.
# -----------------------------------------------------------------------------
+import h5py as h5
import numpy as np
-import h5py as h5
from yt.fields.field_info_container import FieldInfoContainer
from yt.fields.magnetic_field import setup_magnetic_field_aliases
diff -r befb7ffa14b6104c6c78ff76f0ce148103e1fcef -r 58b99dc48b192a4f78b8d11768810d1af7c8e088 yt/frontends/open_pmd/io.py
--- a/yt/frontends/open_pmd/io.py
+++ b/yt/frontends/open_pmd/io.py
@@ -15,14 +15,14 @@
# The full license is in the file COPYING.txt, distributed with this software.
# -----------------------------------------------------------------------------
+from collections import defaultdict
+
import numpy as np
from yt.frontends.open_pmd.misc import is_const_component
from yt.utilities.io_handler import BaseIOHandler
from yt.utilities.logger import ytLogger as mylog
-from collections import defaultdict
-
class IOHandlerOpenPMD(BaseIOHandler):
_field_dtype = "float32"
@@ -69,27 +69,35 @@
self.cache[i] = np.zeros(offset)
def _read_particle_selection(self, chunks, selector, fields):
- """
+ """Reads given particle fields for given particle species masked by a given selection.
Parameters
----------
chunks
+ A list of chunks
+ A chunk is a list of grids
selector
- fields
+ A region (inside your domain) specifying which parts of the field you want to read
+ See [1] and [2]
+ fields : array_like
+ Tuples (ptype, pfield) representing a field
Returns
-------
+ dict
+ keys are tuples (ptype, pfield) representing a field
+ values are (N,) ndarrays with data from that field
+ """
+ f = self._handle
+ ds = f[self.base_path]
+ unions = self.ds.particle_unions
+ chunks = list(chunks) # the argument chunks is a generator
- """
rv = {}
ind = {}
particle_count = {}
- chunks = list(chunks) # chunks is a generator and thus depleted when you iterate over it once
- on_disk_ptypes_and_fields = defaultdict(list)
- mapping_from_on_disk_to_request = defaultdict(list)
- f = self._handle
- ds = f[self.base_path]
- unions = self.ds.particle_unions
+ ptf = defaultdict(list) # ParticleTypes&Fields
+ rfm = defaultdict(list) # RequestFieldMapping
for ptype, pname in fields:
pfield = ptype, pname
@@ -98,16 +106,16 @@
if ptype in unions:
for pt in unions[ptype]:
particle_count[pfield] += self.ds.particle_type_counts[pt]
- on_disk_ptypes_and_fields[pt].append(pname)
- mapping_from_on_disk_to_request[pt, pname].append(pfield)
+ ptf[pt].append(pname)
+ rfm[pt, pname].append(pfield)
else:
particle_count[pfield] = self.ds.particle_type_counts[ptype]
- on_disk_ptypes_and_fields[ptype].append(pname)
- mapping_from_on_disk_to_request[pfield].append(pfield)
+ ptf[ptype].append(pname)
+ rfm[pfield].append(pfield)
rv[pfield] = np.empty((particle_count[pfield],), dtype=np.float64)
ind[pfield] = 0
- for ptype in on_disk_ptypes_and_fields:
+ for ptype in ptf:
for chunk in chunks:
for grid in chunk.objs:
if ptype in "io":
@@ -122,13 +130,12 @@
if mask is None:
continue
pds = ds[self.particles_path + "/" + spec]
- for field in on_disk_ptypes_and_fields[ptype]:
+ for field in ptf[ptype]:
component = "/".join(field.split("_")[1:]).replace("positionCoarse", "position").replace("-",
"_")
data = self.get_component(pds, component, grid.pindex, grid.poffset)[mask]
- for request_field in mapping_from_on_disk_to_request[(ptype, field)]:
- my_ind = ind[request_field]
- rv[request_field][my_ind:my_ind + data.shape[0], ...] = data
+ for request_field in rfm[(ptype, field)]:
+ rv[request_field][ind[request_field]:ind[request_field] + data.shape[0]] = data
ind[request_field] += data.shape[0]
for field in fields:
@@ -162,8 +169,10 @@
bp = self.base_path
mp = self.meshes_path
ds = f[bp + mp]
+ chunks = list(chunks)
+
rv = {}
- chunks = list(chunks)
+ ind = {}
if selector.__class__.__name__ == "GridSelector":
if not (len(chunks) == len(chunks[0].objs) == 1):
@@ -172,7 +181,6 @@
if size is None:
size = sum((g.count(selector) for chunk in chunks
for g in chunk.objs))
- ind = {}
for field in fields:
rv[field] = np.empty(size, dtype=np.float64)
ind[field] = 0
@@ -188,7 +196,7 @@
if mask is None:
continue
component = fname.replace("_", "/").replace("-", "_")
- data = np.array(self.get_component(ds, component, grid.findex, grid.foffset))
+ data = self.get_component(ds, component, grid.findex, grid.foffset)
# The following is a modified AMRGridPatch.select(...)
data.shape = mask.shape # Workaround - casts a 2D (x,y) array to 3D (x,y,1)
count = grid.count(selector)
@@ -222,7 +230,6 @@
Returns
-------
(N,) ndarray
-
"""
record_component = group[component_name]
unit_si = record_component.attrs["unitSI"]
https://bitbucket.org/yt_analysis/yt/commits/e5bdf7bae4cb/
Changeset: e5bdf7bae4cb
Branch: yt
User: Fabian Koller
Date: 2016-08-25 12:48:06+00:00
Summary: 2D answer test
Affected #: 4 files
diff -r 58b99dc48b192a4f78b8d11768810d1af7c8e088 -r e5bdf7bae4cb16750f0152ba2469098bc4d4df5b yt/frontends/open_pmd/data_structures.py
--- a/yt/frontends/open_pmd/data_structures.py
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -287,7 +287,7 @@
handled_ptypes = []
# Particle grids
- for species, count in list(self.numparts.items()):
+ for (species, count) in list(self.numparts.items()):
if "#" in species:
# This is a particlePatch
spec = species.split("#")
diff -r 58b99dc48b192a4f78b8d11768810d1af7c8e088 -r e5bdf7bae4cb16750f0152ba2469098bc4d4df5b yt/frontends/open_pmd/fields.py
--- a/yt/frontends/open_pmd/fields.py
+++ b/yt/frontends/open_pmd/fields.py
@@ -209,7 +209,6 @@
If a field can not be calculated, it will simply be skipped.
"""
-
# Set up aliases first so the setup for poynting can use them
if len(self._mag_fields) > 0:
setup_magnetic_field_aliases(self, "openPMD", self._mag_fields)
diff -r 58b99dc48b192a4f78b8d11768810d1af7c8e088 -r e5bdf7bae4cb16750f0152ba2469098bc4d4df5b yt/frontends/open_pmd/io.py
--- a/yt/frontends/open_pmd/io.py
+++ b/yt/frontends/open_pmd/io.py
@@ -89,9 +89,11 @@
values are (N,) ndarrays with data from that field
"""
f = self._handle
- ds = f[self.base_path]
+ bp = self.base_path
+ pp = self.particles_path
+ ds = f[bp + pp]
unions = self.ds.particle_unions
- chunks = list(chunks) # the argument chunks is a generator
+ chunks = list(chunks) # chunks is a generator
rv = {}
ind = {}
@@ -119,17 +121,17 @@
for chunk in chunks:
for grid in chunk.objs:
if ptype in "io":
- spec = list(ds[self.particles_path].keys())[0]
+ species = list(ds.keys())[0]
else:
- spec = ptype
- if spec not in grid.ptypes:
+ species = ptype
+ if species not in grid.ptypes:
continue
# read particle coords into cache
- self._fill_cache(spec, grid.pindex, grid.poffset)
+ self._fill_cache(species, grid.pindex, grid.poffset)
mask = selector.select_points(self.cache[0], self.cache[1], self.cache[2], 0.0)
if mask is None:
continue
- pds = ds[self.particles_path + "/" + spec]
+ pds = ds[species]
for field in ptf[ptype]:
component = "/".join(field.split("_")[1:]).replace("positionCoarse", "position").replace("-",
"_")
@@ -185,8 +187,8 @@
rv[field] = np.empty(size, dtype=np.float64)
ind[field] = 0
- for ftype, fname in fields:
- field = (ftype, fname)
+ for field in fields:
+ (ftype, fname) = field
for chunk in chunks:
for grid in chunk.objs:
mylog.debug("open_pmd - _read_fluid_selection {} {} {}".format(grid, field, size))
diff -r 58b99dc48b192a4f78b8d11768810d1af7c8e088 -r e5bdf7bae4cb16750f0152ba2469098bc4d4df5b yt/frontends/open_pmd/tests/test_outputs.py
--- a/yt/frontends/open_pmd/tests/test_outputs.py
+++ b/yt/frontends/open_pmd/tests/test_outputs.py
@@ -13,23 +13,20 @@
# The full license is in the file COPYING.txt, distributed with this software.
# -----------------------------------------------------------------------------
+from yt.frontends.open_pmd.data_structures import \
+ OpenPMDDataset
from yt.testing import \
assert_almost_equal, \
assert_equal, \
assert_array_equal, \
requires_file
-
from yt.utilities.answer_testing.framework import \
- FieldValuesTest, \
- requires_ds, \
data_dir_load
-from yt.frontends.open_pmd.data_structures import \
- OpenPMDDataset
-
twoD = "example-2d/hdf5/data00000100.h5"
threeD = "example-3d/hdf5/data00000100.h5"
+
@requires_file(threeD)
def test_3d_out():
ds = data_dir_load(threeD)
@@ -61,15 +58,78 @@
('openPMD', 'E_y'),
('openPMD', 'E_z'),
('openPMD', 'rho')]
- domain_domensions = [26, 26, 201]
+ domain_dimensions = [26, 26, 201]
domain_width = [2.08e-05, 2.08e-05, 2.01e-05]
assert isinstance(ds, OpenPMDDataset)
yield assert_equal, str(ds), "data00000100.h5"
yield assert_equal, ds.dimensionality, 3
- yield assert_equal, ds.current_time, 3.28471214521e-14
yield assert_equal, ds.particle_types_raw, ('io',)
assert "all" in ds.particle_unions
yield assert_array_equal, ds.field_list, field_list
- yield assert_array_equal, ds.domain_domensions, domain_domensions
- yield assert_almost_equal, ds.domain_right_edge - ds.domain_left_edge, domain_width
\ No newline at end of file
+ yield assert_array_equal, ds.domain_domensions, domain_dimensions
+ yield assert_almost_equal, ds.current_time, 3.28471214521e-14
+ yield assert_almost_equal, ds.domain_right_edge - ds.domain_left_edge, domain_width
+
+
+ at requires_file(twoD)
+def test_2d_out():
+ ds = data_dir_load(twoD)
+ field_list = [('Hydrogen1+', 'particle_charge'),
+ ('Hydrogen1+', 'particle_mass'),
+ ('Hydrogen1+', 'particle_momentum_x'),
+ ('Hydrogen1+', 'particle_momentum_y'),
+ ('Hydrogen1+', 'particle_momentum_z'),
+ ('Hydrogen1+', 'particle_positionCoarse_x'),
+ ('Hydrogen1+', 'particle_positionCoarse_y'),
+ ('Hydrogen1+', 'particle_positionCoarse_z'),
+ ('Hydrogen1+', 'particle_positionOffset_x'),
+ ('Hydrogen1+', 'particle_positionOffset_y'),
+ ('Hydrogen1+', 'particle_positionOffset_z'),
+ ('Hydrogen1+', 'particle_weighting'),
+ ('all', 'particle_charge'),
+ ('all', 'particle_mass'),
+ ('all', 'particle_momentum_x'),
+ ('all', 'particle_momentum_y'),
+ ('all', 'particle_momentum_z'),
+ ('all', 'particle_positionCoarse_x'),
+ ('all', 'particle_positionCoarse_y'),
+ ('all', 'particle_positionCoarse_z'),
+ ('all', 'particle_positionOffset_x'),
+ ('all', 'particle_positionOffset_y'),
+ ('all', 'particle_positionOffset_z'),
+ ('all', 'particle_weighting'),
+ ('electrons', 'particle_charge'),
+ ('electrons', 'particle_mass'),
+ ('electrons', 'particle_momentum_x'),
+ ('electrons', 'particle_momentum_y'),
+ ('electrons', 'particle_momentum_z'),
+ ('electrons', 'particle_positionCoarse_x'),
+ ('electrons', 'particle_positionCoarse_y'),
+ ('electrons', 'particle_positionCoarse_z'),
+ ('electrons', 'particle_positionOffset_x'),
+ ('electrons', 'particle_positionOffset_y'),
+ ('electrons', 'particle_positionOffset_z'),
+ ('electrons', 'particle_weighting'),
+ ('openPMD', 'B_x'),
+ ('openPMD', 'B_y'),
+ ('openPMD', 'B_z'),
+ ('openPMD', 'E_x'),
+ ('openPMD', 'E_y'),
+ ('openPMD', 'E_z'),
+ ('openPMD', 'J_x'),
+ ('openPMD', 'J_y'),
+ ('openPMD', 'J_z'),
+ ('openPMD', 'rho')]
+ domain_dimensions = [51, 201, 1]
+ domain_width = [3.06e-05, 2.01e-05, 1e+0]
+
+ assert isinstance(ds, OpenPMDDataset)
+ yield assert_equal, str(ds), "data00000100.h5"
+ yield assert_equal, ds.dimensionality, 2
+ yield assert_equal, ds.particle_types_raw, ('Hydrogen1+', 'electrons')
+ assert "all" in ds.particle_unions
+ yield assert_array_equal, ds.field_list, field_list
+ yield assert_array_equal, ds.domain_domensions, domain_dimensions
+ yield assert_almost_equal, ds.current_time, 3.29025596712e-14
+ yield assert_almost_equal, ds.domain_right_edge - ds.domain_left_edge, domain_width
https://bitbucket.org/yt_analysis/yt/commits/121736049e2f/
Changeset: 121736049e2f
Branch: yt
User: Fabian Koller
Date: 2016-08-25 13:02:03+00:00
Summary: Set deafult virtual gridsize to 1GB
Affected #: 1 file
diff -r e5bdf7bae4cb16750f0152ba2469098bc4d4df5b -r 121736049e2fb23ddbf403957e030e105548fbb7 yt/frontends/open_pmd/data_structures.py
--- a/yt/frontends/open_pmd/data_structures.py
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -368,7 +368,7 @@
# Only use on-disk particle names if there is more than one species
self.particle_types = particles
mylog.debug("open_pmd - self.particle_types: {}".format(self.particle_types))
- self.gridsize = kwargs.pop("open_pmd_virtual_gridsize", 12*10**6)
+ self.gridsize = kwargs.pop("open_pmd_virtual_gridsize", 10**9)
self.particle_types_raw = self.particle_types
self.particle_types = tuple(self.particle_types)
https://bitbucket.org/yt_analysis/yt/commits/74f6f4aec9df/
Changeset: 74f6f4aec9df
Branch: yt
User: Fabian Koller
Date: 2016-08-25 13:39:17+00:00
Summary: Enable continuous integration testing,
fix answer tests
Affected #: 2 files
diff -r 121736049e2fb23ddbf403957e030e105548fbb7 -r 74f6f4aec9dfcc2ca95f0646ac6800cff9b55e9f tests/tests.yaml
--- a/tests/tests.yaml
+++ b/tests/tests.yaml
@@ -35,6 +35,9 @@
- yt/frontends/owls_subfind/tests/test_outputs.py
- yt/frontends/gadget_fof/tests/test_outputs.py:test_fields_g5
- yt/frontends/gadget_fof/tests/test_outputs.py:test_fields_g42
+
+ local_open_pmd_000:
+ - yt/frontends/open_pmd/tests/test_ouputs.py
local_owls_000:
- yt/frontends/owls/tests/test_outputs.py
diff -r 121736049e2fb23ddbf403957e030e105548fbb7 -r 74f6f4aec9dfcc2ca95f0646ac6800cff9b55e9f yt/frontends/open_pmd/tests/test_outputs.py
--- a/yt/frontends/open_pmd/tests/test_outputs.py
+++ b/yt/frontends/open_pmd/tests/test_outputs.py
@@ -23,6 +23,8 @@
from yt.utilities.answer_testing.framework import \
data_dir_load
+import numpy as np
+
twoD = "example-2d/hdf5/data00000100.h5"
threeD = "example-3d/hdf5/data00000100.h5"
@@ -58,8 +60,8 @@
('openPMD', 'E_y'),
('openPMD', 'E_z'),
('openPMD', 'rho')]
- domain_dimensions = [26, 26, 201]
- domain_width = [2.08e-05, 2.08e-05, 2.01e-05]
+ domain_dimensions = [26, 26, 201] * np.ones_like(ds.domain_dimensions)
+ domain_width = [2.08e-05, 2.08e-05, 2.01e-05] * np.ones_like(ds.domain_left_edge)
assert isinstance(ds, OpenPMDDataset)
yield assert_equal, str(ds), "data00000100.h5"
@@ -67,8 +69,8 @@
yield assert_equal, ds.particle_types_raw, ('io',)
assert "all" in ds.particle_unions
yield assert_array_equal, ds.field_list, field_list
- yield assert_array_equal, ds.domain_domensions, domain_dimensions
- yield assert_almost_equal, ds.current_time, 3.28471214521e-14
+ yield assert_array_equal, ds.domain_dimensions, domain_dimensions
+ yield assert_almost_equal, ds.current_time, 3.28471214521e-14 * np.ones_like(ds.current_time)
yield assert_almost_equal, ds.domain_right_edge - ds.domain_left_edge, domain_width
@@ -121,8 +123,8 @@
('openPMD', 'J_y'),
('openPMD', 'J_z'),
('openPMD', 'rho')]
- domain_dimensions = [51, 201, 1]
- domain_width = [3.06e-05, 2.01e-05, 1e+0]
+ domain_dimensions = [51, 201, 1] * np.ones_like(ds.domain_dimensions)
+ domain_width = [3.06e-05, 2.01e-05, 1e+0] * np.ones_like(ds.domain_left_edge)
assert isinstance(ds, OpenPMDDataset)
yield assert_equal, str(ds), "data00000100.h5"
@@ -130,6 +132,6 @@
yield assert_equal, ds.particle_types_raw, ('Hydrogen1+', 'electrons')
assert "all" in ds.particle_unions
yield assert_array_equal, ds.field_list, field_list
- yield assert_array_equal, ds.domain_domensions, domain_dimensions
- yield assert_almost_equal, ds.current_time, 3.29025596712e-14
+ yield assert_array_equal, ds.domain_dimensions, domain_dimensions
+ yield assert_almost_equal, ds.current_time, 3.29025596712e-14 * np.ones_like(ds.current_time)
yield assert_almost_equal, ds.domain_right_edge - ds.domain_left_edge, domain_width
https://bitbucket.org/yt_analysis/yt/commits/b1bfb69b8b9e/
Changeset: b1bfb69b8b9e
Branch: yt
User: Fabian Koller
Date: 2016-08-25 15:02:38+00:00
Summary: Docstrings, formatting
Affected #: 4 files
diff -r 74f6f4aec9dfcc2ca95f0646ac6800cff9b55e9f -r b1bfb69b8b9e4f4a2fdc3a65fbc22cfa5dd9da90 .hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -85,4 +85,3 @@
doc/_temp/*
doc/source/bootcamp/.ipynb_checkpoints/
dist
-*.DS_Store
diff -r 74f6f4aec9dfcc2ca95f0646ac6800cff9b55e9f -r b1bfb69b8b9e4f4a2fdc3a65fbc22cfa5dd9da90 yt/fields/field_info_container.py
--- a/yt/fields/field_info_container.py
+++ b/yt/fields/field_info_container.py
@@ -364,8 +364,8 @@
if field in self._show_field_errors:
raise
if type(e) != YTFieldNotFound:
- mylog.debug("Raises %s (%) during field %s detection.",
- str(type(e)), e, field)
+ mylog.debug("Raises %s during field %s detection.",
+ str(type(e)), field)
self.pop(field)
continue
# This next bit checks that we can't somehow generate everything.
diff -r 74f6f4aec9dfcc2ca95f0646ac6800cff9b55e9f -r b1bfb69b8b9e4f4a2fdc3a65fbc22cfa5dd9da90 yt/frontends/open_pmd/data_structures.py
--- a/yt/frontends/open_pmd/data_structures.py
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -35,17 +35,15 @@
class OpenPMDGrid(AMRGridPatch):
- """Represents disjoint chunk of data on-disk.
+ """Represents chunk of data on-disk.
- The chunks are sliced off the original field along the x-axis.
This defines the index and offset for every mesh and particle type.
It also defines parents and children grids. Since openPMD does not have multiple levels of refinement,
there are no parents or children for any grid.
"""
_id_offset = 0
__slots__ = ["_level_id"]
- # Every particle species might have different hdf5-indices and offsets
- # These contain tuples (ptype, index) and (ptype, offset)
+ # Every particle species and mesh might have different hdf5-indices and offsets
ftypes=[]
ptypes=[]
findex = 0
@@ -71,7 +69,7 @@
class OpenPMDHierarchy(GridIndex):
- """Defines which fields and particles are created and read from the hard disk.
+ """Defines which fields and particles are created and read from disk.
Furthermore it defines the characteristics of the grids.
"""
@@ -178,7 +176,6 @@
"""Sets ``self.num_grids`` to be the total number of grids in the simulation.
The number of grids is determined by their respective memory footprint.
- You may change ``gridsize`` accordingly. Increase if you have few cores or run single-threaded.
"""
f = self.dataset._handle
bp = self.dataset.base_path
@@ -236,8 +233,7 @@
Notes
-----
``self.grid_dimensions`` is rounded to the nearest integer. Grid edges are calculated from this dimension.
- Furthermore the last grid might be smaller and have fewer particles than the others.
- In general, NOT all particles in a grid will be inside the grid edges.
+ Grids with dimensions [0, 0, 0] are particle only. The others do not have any particles affiliated with them.
"""
f = self.dataset._handle
bp = self.dataset.base_path
@@ -348,17 +344,28 @@
class OpenPMDDataset(Dataset):
"""Contains all the required information of a single iteration of the simulation.
+
+ Notes
+ -----
+ It is assumed that all meshes cover the same region. Their resolution can be different.
+ It is assumed that all particles reside in this same region exclusively.
+ It is assumed that the particle position is *relative* to the grid origin. (i.e. grid offset is **NOT** considered)
"""
_index_class = OpenPMDHierarchy
_field_info_class = OpenPMDFieldInfo
- def __init__(self, filename, dataset_type="openPMD",
+ def __init__(self,
+ filename,
+ dataset_type="openPMD",
storage_filename=None,
units_override=None,
- unit_system="mks", **kwargs):
+ unit_system="mks",
+ **kwargs):
self._handle = HDF5FileHandler(filename)
self._set_paths(self._handle, path.dirname(filename))
- Dataset.__init__(self, filename, dataset_type,
+ Dataset.__init__(self,
+ filename,
+ dataset_type,
units_override=units_override,
unit_system=unit_system)
self.storage_filename = storage_filename
@@ -419,10 +426,6 @@
def _parse_parameter_file(self):
"""Read in metadata describing the overall data on-disk.
-
- Notes
- -----
- All meshes are assumed to have the same dimensions and size.
"""
f = self._handle
bp = self.base_path
@@ -447,16 +450,16 @@
fshape = np.append(fshape, np.ones(3 - self.dimensionality))
self.domain_dimensions = fshape
- self.domain_left_edge = np.zeros(3, dtype=np.float64)
try:
mesh = list(f[bp + mp].keys())[0]
spacing = np.asarray(f[bp + mp + "/" + mesh].attrs["gridSpacing"])
- #offset = np.asarray(f[bp + mp + "/" + mesh].attrs["gridGlobalOffset"])
+ # offset = np.asarray(f[bp + mp + "/" + mesh].attrs["gridGlobalOffset"])
unit_si = np.asarray(f[bp + mp + "/" + mesh].attrs["gridUnitSI"])
- #self.domain_left_edge = self.domain_dimensions[:spacing.size] * unit_si * offset
+ # self.domain_left_edge = self.domain_dimensions[:spacing.size] * unit_si * offset
+ self.domain_left_edge = np.zeros(3, dtype=np.float64)
self.domain_right_edge = self.domain_dimensions[:spacing.size] * unit_si * spacing
- #self.domain_right_edge += self.domain_left_edge
- #self.domain_left_edge = np.append(self.domain_left_edge, np.zeros(3 - self.domain_left_edge.size))
+ # self.domain_right_edge += self.domain_left_edge
+ # self.domain_left_edge = np.append(self.domain_left_edge, np.zeros(3 - self.domain_left_edge.size))
self.domain_right_edge = np.append(self.domain_right_edge, np.ones(3 - self.domain_right_edge.size))
except:
mylog.warning(
diff -r 74f6f4aec9dfcc2ca95f0646ac6800cff9b55e9f -r b1bfb69b8b9e4f4a2fdc3a65fbc22cfa5dd9da90 yt/frontends/open_pmd/io.py
--- a/yt/frontends/open_pmd/io.py
+++ b/yt/frontends/open_pmd/io.py
@@ -146,7 +146,7 @@
return rv
def _read_fluid_selection(self, chunks, selector, fields, size):
- """Reads given fields for given meshes masked by a given selection.
+ """Reads given fields masked by a given selection.
Parameters
----------
@@ -212,7 +212,7 @@
return rv
def get_component(self, group, component_name, index=0, offset=None):
- """Grab a dataset component from a group as a whole or sliced.
+ """Grabs a dataset component from a group as a whole or sliced.
Parameters
----------
@@ -231,7 +231,9 @@
Returns
-------
- (N,) ndarray
+ ndarray
+ (N,) 1D in case of particle data
+ (O,P,Q) 1D/2D/3D in case of mesh data
"""
record_component = group[component_name]
unit_si = record_component.attrs["unitSI"]
https://bitbucket.org/yt_analysis/yt/commits/95cb2f97619f/
Changeset: 95cb2f97619f
Branch: yt
User: Fabian Koller
Date: 2016-08-26 08:38:06+00:00
Summary: Make IO non-verbose
Affected #: 1 file
diff -r b1bfb69b8b9e4f4a2fdc3a65fbc22cfa5dd9da90 -r 95cb2f97619f41dd3354f01073b991e6793274f9 yt/frontends/open_pmd/io.py
--- a/yt/frontends/open_pmd/io.py
+++ b/yt/frontends/open_pmd/io.py
@@ -191,7 +191,6 @@
(ftype, fname) = field
for chunk in chunks:
for grid in chunk.objs:
- mylog.debug("open_pmd - _read_fluid_selection {} {} {}".format(grid, field, size))
if fname.split("_")[0] not in grid.ftypes:
continue
mask = grid._get_selector_mask(selector)
@@ -241,12 +240,11 @@
if offset is None:
offset = record_component.attrs["shape"] - index
# component is constant, craft an array by hand
- mylog.debug("open_pmd - get_component: {}/{} [const {}]".format(group.name, component_name, offset))
+ # mylog.debug("open_pmd - get_component: {}/{} [const {}]".format(group.name, component_name, offset))
return np.full(offset, record_component.attrs["value"] * unit_si)
else:
if offset is not None:
offset += index
# component is a dataset, return it (possibly masked)
- mylog.debug(
- "open_pmd - get_component: {}/{}[{}:{}]".format(group.name, component_name, index, offset))
+ # mylog.debug("open_pmd - get_component: {}/{}[{}:{}]".format(group.name, component_name, index, offset))
return np.multiply(record_component[index:offset], unit_si)
https://bitbucket.org/yt_analysis/yt/commits/2801b9037846/
Changeset: 2801b9037846
Branch: yt
User: Fabian Koller
Date: 2016-08-26 08:53:21+00:00
Summary: Remove unused import
Affected #: 1 file
diff -r 95cb2f97619f41dd3354f01073b991e6793274f9 -r 2801b9037846bcd14730340ea5ff47f6a42d7d58 yt/frontends/open_pmd/io.py
--- a/yt/frontends/open_pmd/io.py
+++ b/yt/frontends/open_pmd/io.py
@@ -21,7 +21,6 @@
from yt.frontends.open_pmd.misc import is_const_component
from yt.utilities.io_handler import BaseIOHandler
-from yt.utilities.logger import ytLogger as mylog
class IOHandlerOpenPMD(BaseIOHandler):
https://bitbucket.org/yt_analysis/yt/commits/79e7cb821f49/
Changeset: 79e7cb821f49
Branch: yt
User: Fabian Koller
Date: 2016-08-27 11:28:27+00:00
Summary: Make frontend tests importable
Affected #: 1 file
diff -r 2801b9037846bcd14730340ea5ff47f6a42d7d58 -r 79e7cb821f499e4f005bd0ddde57e4a59020bbc6 yt/frontends/open_pmd/api.py
--- a/yt/frontends/open_pmd/api.py
+++ b/yt/frontends/open_pmd/api.py
@@ -1,5 +1,5 @@
"""
-API for yt.frontends._skeleton
+API for yt.frontends.open_pmd
@@ -24,3 +24,6 @@
from .io import \
IOHandlerOpenPMD
+
+from . import \
+ tests
https://bitbucket.org/yt_analysis/yt/commits/6bc25f2898a3/
Changeset: 6bc25f2898a3
Branch: yt
User: Fabian Koller
Date: 2016-08-29 16:04:09+00:00
Summary: Remove deprecated setup.py inside frontend
Affected #: 1 file
diff -r 79e7cb821f499e4f005bd0ddde57e4a59020bbc6 -r 6bc25f2898a3fbf920a1fd1d40c9f692c97dd29f yt/frontends/open_pmd/setup.py
--- a/yt/frontends/open_pmd/setup.py
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env python
-# -----------------------------------------------------------------------------
-# Copyright (c) 2013, yt Development Team.
-# Copyright (c) 2015, Daniel Grassinger (HZDR)
-# Copyright (c) 2016, Fabian Koller (HZDR)
-#
-# Distributed under the terms of the Modified BSD License.
-#
-# The full license is in the file COPYING.txt, distributed with this software.
-# -----------------------------------------------------------------------------
-
-
-def configuration(parent_package="", top_path=None):
- from numpy.distutils.misc_util import Configuration
- config = Configuration("open_pmd", parent_package, top_path)
- config.make_config_py() # installs __config__.py
- # config.make_svn_version_py()
- return config
https://bitbucket.org/yt_analysis/yt/commits/cd9d195547d8/
Changeset: cd9d195547d8
Branch: yt
User: ngoldbaum
Date: 2016-09-12 19:21:52+00:00
Summary: remove unnecessary OpenPMD entry in tests.yaml
Affected #: 1 file
diff -r 6bc25f2898a3fbf920a1fd1d40c9f692c97dd29f -r cd9d195547d8bd6d11c81a7d450a8a74fc7d9390 tests/tests.yaml
--- a/tests/tests.yaml
+++ b/tests/tests.yaml
@@ -36,9 +36,6 @@
- yt/frontends/gadget_fof/tests/test_outputs.py:test_fields_g5
- yt/frontends/gadget_fof/tests/test_outputs.py:test_fields_g42
- local_open_pmd_000:
- - yt/frontends/open_pmd/tests/test_ouputs.py
-
local_owls_000:
- yt/frontends/owls/tests/test_outputs.py
https://bitbucket.org/yt_analysis/yt/commits/1b343066b476/
Changeset: 1b343066b476
Branch: yt
User: C0nsultant
Date: 2016-09-13 08:57:34+00:00
Summary: Merged yt_analysis/yt into yt
Affected #: 2 files
diff -r cd9d195547d8bd6d11c81a7d450a8a74fc7d9390 -r 1b343066b4766c445a53d8e05a7420726b6b7578 yt/frontends/ramses/data_structures.py
--- a/yt/frontends/ramses/data_structures.py
+++ b/yt/frontends/ramses/data_structures.py
@@ -647,7 +647,7 @@
if self.cosmological_simulation == 0:
- self.current_time = self.parameters['time'] * self.parameters['unit_t']
+ self.current_time = self.parameters['time']
else :
self.tau_frw, self.t_frw, self.dtau, self.n_frw, self.time_tot = \
friedman( self.omega_matter, self.omega_lambda, 1. - self.omega_matter - self.omega_lambda )
diff -r cd9d195547d8bd6d11c81a7d450a8a74fc7d9390 -r 1b343066b4766c445a53d8e05a7420726b6b7578 yt/frontends/ramses/tests/test_outputs.py
--- a/yt/frontends/ramses/tests/test_outputs.py
+++ b/yt/frontends/ramses/tests/test_outputs.py
@@ -25,6 +25,8 @@
FieldValuesTest, \
create_obj
from yt.frontends.ramses.api import RAMSESDataset
+import os
+import yt
_fields = ("temperature", "density", "velocity_magnitude",
("deposit", "all_density"), ("deposit", "all_count"))
@@ -57,3 +59,15 @@
def test_units_override():
for test in units_override_check(output_00080):
yield test
+
+
+ramsesNonCosmo = 'DICEGalaxyDisk_nonCosmological/output_00002'
+ at requires_file(ramsesNonCosmo)
+def test_unit_non_cosmo():
+ ds = yt.load(os.path.join(ramsesNonCosmo, 'info_00002.txt'))
+
+ expected_raw_time = 0.0299468077820411 # in ramses unit
+ yield assert_equal, ds.current_time.value, expected_raw_time
+
+ expected_time = 14087886140997.336 # in seconds
+ assert_equal(ds.current_time.in_units('s').value, expected_time)
https://bitbucket.org/yt_analysis/yt/commits/9e91759f1451/
Changeset: 9e91759f1451
Branch: yt
User: Fabian Koller
Date: 2016-09-16 11:26:33+00:00
Summary: Include openPMD in sphinx doc
Treat mesh and particle positions as absolute (i.e. with globalOffset)
Transparently read constant mesh records
Affected #: 5 files
diff -r cd9d195547d8bd6d11c81a7d450a8a74fc7d9390 -r 9e91759f14515d40ffe09f50186bd773439be531 doc/source/examining/loading_data.rst
--- a/doc/source/examining/loading_data.rst
+++ b/doc/source/examining/loading_data.rst
@@ -1519,6 +1519,57 @@
# The halo mass
print(ad["FOF", "particle_mass"])
+.. _loading-openpmd-data:
+
+openPMD Data
+---------
+
+`openPMD <http://www.openpmd.org>`_ is an open source meta-standard and naming
+scheme for mesh based data and particle data. It does not actually define a file
+format.
+
+HDF5-containers respecting the minimal set of meta information from
+versions 1.0.0 and 1.0.1 of the standard are compatible.
+Support for the ED-PIC extension is not available. Mesh data in cartesian coordinates
+and particle data can be read by this frontend.
+
+To load the first in-file iteration of a openPMD datasets using the standard HDF5
+output format:
+
+.. code-block:: python
+
+ import yt
+ ds = yt.load('example-3d/hdf5/data00000100.h5')
+
+If you operate on large files, you may want to modify the virtual chunking behaviour through
+``open_pmd_virtual_gridsize``. The supplied value is an estimate of the size of a single read request
+for each particle attribute/mesh (in Byte).
+
+.. code-block:: python
+
+ import yt
+ ds = yt.load('example-3d/hdf5/data00000100.h5', open_pmd_virtual_gridsize=10e4)
+ sp = yt.SlicePlot(ds, 'x', 'rho')
+ sp.show()
+
+Particle data is fully supported:
+
+.. code-block:: python
+
+ import yt
+ ds = yt.load('example-3d/hdf5/data00000100.h5')
+ ad = f.all_data()
+ ppp = yt.ParticlePhasePlot(ad, 'particle_position_y', 'particle_momentum_y', 'particle_weighting')
+ ppp.show()
+
+.. rubric:: Caveats
+
+* 1D, 2D and 3D data is compatible, but lower dimensional data might yield
+ strange results since it gets padded and treated as 3D. Extraneous dimensions are
+ set to be of length 1.0m and have a width of one cell.
+* The frontend has hardcoded logic for renaming the openPMD ``position``
+ of particles to ``positionCoarse``
+
.. _loading-pyne-data:
PyNE Data
diff -r cd9d195547d8bd6d11c81a7d450a8a74fc7d9390 -r 9e91759f14515d40ffe09f50186bd773439be531 doc/source/reference/code_support.rst
--- a/doc/source/reference/code_support.rst
+++ b/doc/source/reference/code_support.rst
@@ -48,6 +48,8 @@
+-----------------------+------------+-----------+------------+-------+----------+----------+------------+----------+
| Nyx | Y | N | Y | Y | Y | Y | Y | Full |
+-----------------------+------------+-----------+------------+-------+----------+----------+------------+----------+
+| openPMD | Y | Y | N | Y | Y | Y | N | Partial |
++-----------------------+------------+-----------+------------+-------+----------+----------+------------+----------+
| Orion | Y | Y | Y | Y | Y | Y | Y | Full |
+-----------------------+------------+-----------+------------+-------+----------+----------+------------+----------+
| OWLS/EAGLE | Y | Y | Y | Y | Y [#f2]_ | Y | Y | Full |
diff -r cd9d195547d8bd6d11c81a7d450a8a74fc7d9390 -r 9e91759f14515d40ffe09f50186bd773439be531 yt/frontends/open_pmd/data_structures.py
--- a/yt/frontends/open_pmd/data_structures.py
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -204,7 +204,7 @@
self.numparts[species] = f[bp + pp + species + "/position/" + axis].len()
# Limit values per grid by resulting memory footprint
- self.vpg = int(self.dataset.gridsize / (self.dataset.dimensionality * 4)) # 4Byte per value per dimension (f32)
+ self.vpg = int(self.dataset.gridsize / 4) # 4Byte per value (f32)
# Meshes of the same size do not need separate chunks
for shape in set(self.meshshapes.values()):
@@ -349,7 +349,7 @@
-----
It is assumed that all meshes cover the same region. Their resolution can be different.
It is assumed that all particles reside in this same region exclusively.
- It is assumed that the particle position is *relative* to the grid origin. (i.e. grid offset is **NOT** considered)
+ It is assumed that the particle and mesh positions are *absolute* with respect to the simulation origin.
"""
_index_class = OpenPMDHierarchy
_field_info_class = OpenPMDFieldInfo
@@ -453,13 +453,13 @@
try:
mesh = list(f[bp + mp].keys())[0]
spacing = np.asarray(f[bp + mp + "/" + mesh].attrs["gridSpacing"])
- # offset = np.asarray(f[bp + mp + "/" + mesh].attrs["gridGlobalOffset"])
+ offset = np.asarray(f[bp + mp + "/" + mesh].attrs["gridGlobalOffset"])
unit_si = np.asarray(f[bp + mp + "/" + mesh].attrs["gridUnitSI"])
- # self.domain_left_edge = self.domain_dimensions[:spacing.size] * unit_si * offset
- self.domain_left_edge = np.zeros(3, dtype=np.float64)
+ self.domain_left_edge = offset * unit_si
+ # self.domain_left_edge = np.zeros(3, dtype=np.float64)
self.domain_right_edge = self.domain_dimensions[:spacing.size] * unit_si * spacing
- # self.domain_right_edge += self.domain_left_edge
- # self.domain_left_edge = np.append(self.domain_left_edge, np.zeros(3 - self.domain_left_edge.size))
+ self.domain_right_edge += self.domain_left_edge
+ self.domain_left_edge = np.append(self.domain_left_edge, np.zeros(3 - self.domain_left_edge.size))
self.domain_right_edge = np.append(self.domain_right_edge, np.ones(3 - self.domain_right_edge.size))
except:
mylog.warning(
diff -r cd9d195547d8bd6d11c81a7d450a8a74fc7d9390 -r 9e91759f14515d40ffe09f50186bd773439be531 yt/frontends/open_pmd/fields.py
--- a/yt/frontends/open_pmd/fields.py
+++ b/yt/frontends/open_pmd/fields.py
@@ -123,9 +123,6 @@
(equivalently, the regex \w).
Since yt widely uses the underscore in field names, openPMD's underscores (_) are replaced by hyphen (-).
- The constructor of the super-class is called after the fields have been dynamically parsed, so they are known when
- needed during the call of ``setup_fluid_aliases`` .
-
Derived fields will automatically be set up, if names and units of your known on-disk (or manually derived)
fields match the ones in [1].
diff -r cd9d195547d8bd6d11c81a7d450a8a74fc7d9390 -r 9e91759f14515d40ffe09f50186bd773439be531 yt/frontends/open_pmd/io.py
--- a/yt/frontends/open_pmd/io.py
+++ b/yt/frontends/open_pmd/io.py
@@ -100,8 +100,8 @@
ptf = defaultdict(list) # ParticleTypes&Fields
rfm = defaultdict(list) # RequestFieldMapping
- for ptype, pname in fields:
- pfield = ptype, pname
+ for (ptype, pname) in fields:
+ pfield = (ptype, pname)
# Overestimate the size of all pfields so they include all particles, shrink it later
particle_count[pfield] = 0
if ptype in unions:
@@ -186,8 +186,8 @@
rv[field] = np.empty(size, dtype=np.float64)
ind[field] = 0
- for field in fields:
- (ftype, fname) = field
+ for (ftype, fname) in fields:
+ field = (ftype, fname)
for chunk in chunks:
for grid in chunk.objs:
if fname.split("_")[0] not in grid.ftypes:
@@ -236,11 +236,14 @@
record_component = group[component_name]
unit_si = record_component.attrs["unitSI"]
if is_const_component(record_component):
+ shape = record_component.attrs["shape"]
if offset is None:
- offset = record_component.attrs["shape"] - index
+ shape[0] -= index # This makes constant meshes readable
+ else:
+ shape[0] = offset
# component is constant, craft an array by hand
- # mylog.debug("open_pmd - get_component: {}/{} [const {}]".format(group.name, component_name, offset))
- return np.full(offset, record_component.attrs["value"] * unit_si)
+ # mylog.debug("open_pmd - get_component: {}/{} [const {}]".format(group.name, component_name, shape))
+ return np.full(shape, record_component.attrs["value"] * unit_si)
else:
if offset is not None:
offset += index
https://bitbucket.org/yt_analysis/yt/commits/84a303612b34/
Changeset: 84a303612b34
Branch: yt
User: Fabian Koller
Date: 2016-09-16 11:27:09+00:00
Summary: Merge with yt upstream
Affected #: 2 files
diff -r 9e91759f14515d40ffe09f50186bd773439be531 -r 84a303612b3461ebaabc713fbc3fa3807c0fa9f3 yt/frontends/ramses/data_structures.py
--- a/yt/frontends/ramses/data_structures.py
+++ b/yt/frontends/ramses/data_structures.py
@@ -647,7 +647,7 @@
if self.cosmological_simulation == 0:
- self.current_time = self.parameters['time'] * self.parameters['unit_t']
+ self.current_time = self.parameters['time']
else :
self.tau_frw, self.t_frw, self.dtau, self.n_frw, self.time_tot = \
friedman( self.omega_matter, self.omega_lambda, 1. - self.omega_matter - self.omega_lambda )
diff -r 9e91759f14515d40ffe09f50186bd773439be531 -r 84a303612b3461ebaabc713fbc3fa3807c0fa9f3 yt/frontends/ramses/tests/test_outputs.py
--- a/yt/frontends/ramses/tests/test_outputs.py
+++ b/yt/frontends/ramses/tests/test_outputs.py
@@ -25,6 +25,8 @@
FieldValuesTest, \
create_obj
from yt.frontends.ramses.api import RAMSESDataset
+import os
+import yt
_fields = ("temperature", "density", "velocity_magnitude",
("deposit", "all_density"), ("deposit", "all_count"))
@@ -57,3 +59,15 @@
def test_units_override():
for test in units_override_check(output_00080):
yield test
+
+
+ramsesNonCosmo = 'DICEGalaxyDisk_nonCosmological/output_00002'
+ at requires_file(ramsesNonCosmo)
+def test_unit_non_cosmo():
+ ds = yt.load(os.path.join(ramsesNonCosmo, 'info_00002.txt'))
+
+ expected_raw_time = 0.0299468077820411 # in ramses unit
+ yield assert_equal, ds.current_time.value, expected_raw_time
+
+ expected_time = 14087886140997.336 # in seconds
+ assert_equal(ds.current_time.in_units('s').value, expected_time)
https://bitbucket.org/yt_analysis/yt/commits/d99b47bf0c26/
Changeset: d99b47bf0c26
Branch: yt
User: Fabian Koller
Date: 2016-09-16 15:08:28+00:00
Summary: Create a wrapper for h5py.Dataset in the on-demand import of h5py
Handle underscores in fieldnames better
Affected #: 3 files
diff -r 84a303612b3461ebaabc713fbc3fa3807c0fa9f3 -r d99b47bf0c26b9b7ac0f2218e5d71fa2c6e4e691 yt/frontends/open_pmd/fields.py
--- a/yt/frontends/open_pmd/fields.py
+++ b/yt/frontends/open_pmd/fields.py
@@ -15,7 +15,6 @@
# The full license is in the file COPYING.txt, distributed with this software.
# -----------------------------------------------------------------------------
-import h5py as h5
import numpy as np
from yt.fields.field_info_container import FieldInfoContainer
@@ -23,6 +22,7 @@
from yt.frontends.open_pmd.misc import \
parse_unit_dimension, \
is_const_component
+from yt.utilities.on_demand_imports import _h5py as h5
from yt.units.yt_array import YTQuantity
from yt.utilities.logger import ytLogger as mylog
from yt.utilities.physical_constants import \
diff -r 84a303612b3461ebaabc713fbc3fa3807c0fa9f3 -r d99b47bf0c26b9b7ac0f2218e5d71fa2c6e4e691 yt/frontends/open_pmd/io.py
--- a/yt/frontends/open_pmd/io.py
+++ b/yt/frontends/open_pmd/io.py
@@ -190,12 +190,12 @@
field = (ftype, fname)
for chunk in chunks:
for grid in chunk.objs:
- if fname.split("_")[0] not in grid.ftypes:
+ component = fname.replace("_", "/").replace("-", "_")
+ if component.split("/")[0] not in grid.ftypes:
continue
mask = grid._get_selector_mask(selector)
if mask is None:
continue
- component = fname.replace("_", "/").replace("-", "_")
data = self.get_component(ds, component, grid.findex, grid.foffset)
# The following is a modified AMRGridPatch.select(...)
data.shape = mask.shape # Workaround - casts a 2D (x,y) array to 3D (x,y,1)
diff -r 84a303612b3461ebaabc713fbc3fa3807c0fa9f3 -r d99b47bf0c26b9b7ac0f2218e5d71fa2c6e4e691 yt/utilities/on_demand_imports.py
--- a/yt/utilities/on_demand_imports.py
+++ b/yt/utilities/on_demand_imports.py
@@ -230,6 +230,19 @@
self._Group = Group
return self._Group
+ _Dataset = None
+ @property
+ def Dataset(self):
+ if self._err:
+ raise self._err
+ if self._Dataset is None:
+ try:
+ from h5py import Dataset
+ except ImportError:
+ Dataset = NotAModule(self._name)
+ self._Dataset = Dataset
+ return self._Dataset
+
___version__ = None
@property
def __version__(self):
https://bitbucket.org/yt_analysis/yt/commits/db01f716f54f/
Changeset: db01f716f54f
Branch: yt
User: Fabian Koller
Date: 2016-09-20 10:21:39+00:00
Summary: Grid creation - consider mesh positions
Component acquisition - make shape-handeling less error-prone
Affected #: 2 files
diff -r d99b47bf0c26b9b7ac0f2218e5d71fa2c6e4e691 -r db01f716f54f588e88a8e6599489b2521ba3d994 yt/frontends/open_pmd/data_structures.py
--- a/yt/frontends/open_pmd/data_structures.py
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -83,7 +83,7 @@
GridIndex.__init__(self, ds, dataset_type)
def _get_particle_type_counts(self):
- """Sets the active number of particles for every species.
+ """Reads the active number of particles for every species.
Returns
-------
@@ -189,9 +189,13 @@
for mesh in list(f[bp + mp].keys()):
if type(f[bp + mp + mesh]) is h5.Group:
- self.meshshapes[mesh] = f[bp + mp + mesh + "/" + list(f[bp + mp + mesh].keys())[0]].shape
+ shape = f[bp + mp + mesh + "/" + list(f[bp + mp + mesh].keys())[0]].shape
else:
- self.meshshapes[mesh] = f[bp + mp + mesh].shape
+ shape = f[bp + mp + mesh].shape
+ spacing = tuple(f[bp + mp + mesh].attrs["gridSpacing"])
+ offset = tuple(f[bp + mp + mesh].attrs["gridGlobalOffset"])
+ unit_si = f[bp + mp + mesh].attrs["gridUnitSI"]
+ self.meshshapes[mesh] = (shape, spacing, offset, unit_si)
for species in list(f[bp + pp].keys()):
if "particlePatches" in list(f[bp + pp + "/" + species].keys()):
for patch, size in enumerate(f[bp + pp + "/" + species + "/particlePatches/numParticles"]):
@@ -207,13 +211,13 @@
self.vpg = int(self.dataset.gridsize / 4) # 4Byte per value (f32)
# Meshes of the same size do not need separate chunks
- for shape in set(self.meshshapes.values()):
+ for (shape, spacing, offset, unit_si) in set(self.meshshapes.values()):
self.num_grids += min(shape[0], int(np.ceil(reduce(mul, shape) * self.vpg**-1)))
# Same goes for particle chunks if they are not inside particlePatches
patches = {}
no_patches = {}
- for k, v in list(self.numparts.items()):
+ for (k, v) in list(self.numparts.items()):
if "#" in k:
patches[k] = v
else:
@@ -242,27 +246,43 @@
self.grid_levels.flat[:] = 0
self.grids = np.empty(self.num_grids, dtype="object")
- def get_component(group, component_name, index, offset):
+ def get_component(group, component_name, _index=0, _offset=None):
record_component = group[component_name]
- unit_si = record_component.attrs["unitSI"]
- return np.multiply(record_component[index:index+offset], unit_si)
+ _unit_si = record_component.attrs["unitSI"]
+ if is_const_component(record_component):
+ _shape = tuple(record_component.attrs["shape"],)
+ if _offset is None:
+ _shape[0] -= _index
+ else:
+ _shape[0] = _offset
+ return np.full(_shape, record_component.attrs["value"] * _unit_si)
+ else:
+ if _offset is not None:
+ _offset += _index
+ return np.multiply(record_component[_index:_offset], _unit_si)
grid_index_total = 0
# Mesh grids
- for shape in set(self.meshshapes.values()):
+ for mesh in set(self.meshshapes.values()):
+ (shape, spacing, offset, unit_si) = mesh
+ shape = np.asarray(shape)
+ spacing = np.asarray(spacing)
+ offset = np.asarray(offset)
# Total dimension of this grid
domain_dimension = np.asarray(shape, dtype=np.int32)
domain_dimension = np.append(domain_dimension, np.ones(3 - len(domain_dimension)))
# Number of grids of this shape
num_grids = min(shape[0], int(np.ceil(reduce(mul, shape) * self.vpg ** -1)))
- gle = self.dataset.domain_left_edge
- gre = self.dataset.domain_right_edge
+ gle = offset * unit_si # self.dataset.domain_left_edge
+ gre = domain_dimension[:spacing.size] * unit_si * spacing + gle # self.dataset.domain_right_edge
+ gle = np.append(gle, np.zeros(3 - len(gle)))
+ gre = np.append(gre, np.ones(3 - len(gre)))
grid_dim_offset = np.linspace(0, domain_dimension[0], num_grids + 1, dtype=np.int32)
grid_edge_offset = grid_dim_offset * np.float(domain_dimension[0]) ** -1 * (gre[0] - gle[0]) + gle[0]
mesh_names = []
- for (mname, mshape) in list(self.meshshapes.items()):
- if shape == mshape:
+ for (mname, mdata) in list(self.meshshapes.items()):
+ if mesh == mdata:
mesh_names.append(str(mname))
prev = 0
for grid in range(num_grids):
@@ -394,7 +414,8 @@
iterations = list(handle["/data"].keys())
mylog.info("open_pmd - found {} iterations in file".format(len(iterations)))
elif "fileBased" in encoding:
- regex = "^" + handle.attrs["iterationFormat"].decode().replace("%T", "[0-9]+") + "$"
+ itformat = handle.attrs["iterationFormat"].split("/")[-1]
+ regex = "^" + itformat.replace("%T", "[0-9]+") + "$"
if path is "":
mylog.warning("open_pmd - For file based iterations, please use absolute file paths!")
pass
@@ -446,15 +467,13 @@
mylog.warning("open_pmd - Could not detect shape of simulated field! "
"Assuming a single cell and thus setting fshape to [1, 1, 1]!")
self.dimensionality = len(fshape)
-
- fshape = np.append(fshape, np.ones(3 - self.dimensionality))
- self.domain_dimensions = fshape
+ self.domain_dimensions = np.append(fshape, np.ones(3 - self.dimensionality))
try:
mesh = list(f[bp + mp].keys())[0]
spacing = np.asarray(f[bp + mp + "/" + mesh].attrs["gridSpacing"])
offset = np.asarray(f[bp + mp + "/" + mesh].attrs["gridGlobalOffset"])
- unit_si = np.asarray(f[bp + mp + "/" + mesh].attrs["gridUnitSI"])
+ unit_si = f[bp + mp + "/" + mesh].attrs["gridUnitSI"]
self.domain_left_edge = offset * unit_si
# self.domain_left_edge = np.zeros(3, dtype=np.float64)
self.domain_right_edge = self.domain_dimensions[:spacing.size] * unit_si * spacing
diff -r d99b47bf0c26b9b7ac0f2218e5d71fa2c6e4e691 -r db01f716f54f588e88a8e6599489b2521ba3d994 yt/frontends/open_pmd/io.py
--- a/yt/frontends/open_pmd/io.py
+++ b/yt/frontends/open_pmd/io.py
@@ -236,7 +236,7 @@
record_component = group[component_name]
unit_si = record_component.attrs["unitSI"]
if is_const_component(record_component):
- shape = record_component.attrs["shape"]
+ shape = tuple(record_component.attrs["shape"],)
if offset is None:
shape[0] -= index # This makes constant meshes readable
else:
https://bitbucket.org/yt_analysis/yt/commits/b9c231a50356/
Changeset: b9c231a50356
Branch: yt
User: Fabian Koller
Date: 2016-09-21 10:08:07+00:00
Summary: Restor python3 compatability
Affected #: 1 file
diff -r db01f716f54f588e88a8e6599489b2521ba3d994 -r b9c231a50356c2a29e9d81cc2338007767f9b625 yt/frontends/open_pmd/data_structures.py
--- a/yt/frontends/open_pmd/data_structures.py
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -414,7 +414,7 @@
iterations = list(handle["/data"].keys())
mylog.info("open_pmd - found {} iterations in file".format(len(iterations)))
elif "fileBased" in encoding:
- itformat = handle.attrs["iterationFormat"].split("/")[-1]
+ itformat = handle.attrs["iterationFormat"].decode().split("/")[-1]
regex = "^" + itformat.replace("%T", "[0-9]+") + "$"
if path is "":
mylog.warning("open_pmd - For file based iterations, please use absolute file paths!")
https://bitbucket.org/yt_analysis/yt/commits/2db00bdb6bfb/
Changeset: 2db00bdb6bfb
Branch: yt
User: Fabian Koller
Date: 2016-09-23 09:54:14+00:00
Summary: Improcve exception handeling,
make better use of on-demand h5py wrapper,
general refactor for better code readability,
Affected #: 6 files
diff -r b9c231a50356c2a29e9d81cc2338007767f9b625 -r 2db00bdb6bfb8806692228cb69f4b743fbf790ac doc/source/reference/api/api.rst
--- a/doc/source/reference/api/api.rst
+++ b/doc/source/reference/api/api.rst
@@ -323,6 +323,21 @@
~yt.frontends.moab.io.IOHandlerMoabH5MHex8
~yt.frontends.moab.io.IOHandlerMoabPyneHex8
+OpenPMD
+^^^^^^^
+
+.. autosummary::
+:toctree: generated/
+
+ ~yt.frontends.open_pmd.data_structures.OpenPMDGrid
+ ~yt.frontends.open_pmd.data_structures.OpenPMDHierarchy
+ ~yt.frontends.open_pmd.data_structures.OpenPMDDataset
+ ~yt.frontends.open_pmd.fields.OpenPMDFieldInfo
+ ~yt.frontends.open_pmd.io.IOHandlerOpenPMDHDF5
+ ~yt.frontends.open_pmd.misc.parse_unit_dimension
+ ~yt.frontends.open_pmd.misc.is_const_component
+ ~yt.frontends.open_pmd.misc.get_component
+
RAMSES
^^^^^^
diff -r b9c231a50356c2a29e9d81cc2338007767f9b625 -r 2db00bdb6bfb8806692228cb69f4b743fbf790ac yt/frontends/open_pmd/api.py
--- a/yt/frontends/open_pmd/api.py
+++ b/yt/frontends/open_pmd/api.py
@@ -23,7 +23,7 @@
OpenPMDFieldInfo
from .io import \
- IOHandlerOpenPMD
+ IOHandlerOpenPMDHDF5
from . import \
tests
diff -r b9c231a50356c2a29e9d81cc2338007767f9b625 -r 2db00bdb6bfb8806692228cb69f4b743fbf790ac yt/frontends/open_pmd/data_structures.py
--- a/yt/frontends/open_pmd/data_structures.py
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -14,6 +14,7 @@
# The full license is in the file COPYING.txt, distributed with this software.
# -----------------------------------------------------------------------------
+from distutils.version import StrictVersion
from functools import reduce
from operator import mul
from os import \
@@ -26,7 +27,9 @@
from yt.data_objects.grid_patch import AMRGridPatch
from yt.data_objects.static_output import Dataset
from yt.frontends.open_pmd.fields import OpenPMDFieldInfo
-from yt.frontends.open_pmd.misc import is_const_component
+from yt.frontends.open_pmd.misc import \
+ is_const_component, \
+ get_component
from yt.funcs import setdefaultattr
from yt.geometry.grid_geometry_handler import GridIndex
from yt.utilities.file_handler import HDF5FileHandler
@@ -96,11 +99,11 @@
bp = self.dataset.base_path
pp = self.dataset.particles_path
for ptype in self.ds.particle_types_raw:
- if ptype in "io":
- spec = list(f[bp + pp].keys())[0]
+ if str(ptype) == "io":
+ spec = f[bp + pp].keys()[0]
else:
spec = ptype
- axis = list(f[bp + pp + "/" + spec + "/position"].keys())[0]
+ axis = f[bp + pp + "/" + spec + "/position"].keys()[0]
pos = f[bp + pp + "/" + spec + "/position/" + axis]
if is_const_component(pos):
result[ptype] = pos.attrs["shape"]
@@ -124,10 +127,11 @@
mesh_fields = []
try:
- for field in list(f[bp + mp].keys()):
+ for field in f[bp + mp].keys():
try:
- for axis in list(f[bp + mp + field].keys()):
- mesh_fields.append(field + "_" + axis)
+ for axis in f[bp + mp + field].keys():
+ mesh_fields.append(field.replace("_", "-")
+ + "_" + axis)
except AttributeError:
# This is a h5.Dataset (i.e. no axes)
mesh_fields.append(field.replace("_", "-"))
@@ -138,27 +142,31 @@
particle_fields = []
try:
- for species in list(f[bp + pp].keys()):
- for record in list(f[bp + pp + species].keys()):
+ for species in f[bp + pp].keys():
+ for record in f[bp + pp + species].keys():
if is_const_component(f[bp + pp + species + "/" + record]):
# Record itself (e.g. particle_mass) is constant
- particle_fields.append(species + "_" + record)
+ particle_fields.append(species.replace("_", "-")
+ + "_" + record.replace("_", "-"))
elif "particlePatches" not in record:
try:
# Create a field for every axis (x,y,z) of every property (position)
# of every species (electrons)
- axes = list(f[bp + pp + species + "/" + record].keys())
- if record in "position":
+ axes = f[bp + pp + species + "/" + record].keys()
+ if str(record) == "position":
record = "positionCoarse"
for axis in axes:
- particle_fields.append(species + "_" + record + "_" + axis)
+ particle_fields.append(species.replace("_", "-")
+ + "_" + record.replace("_", "-")
+ + "_" + axis)
except AttributeError:
# Record is a dataset, does not have axes (e.g. weighting)
- particle_fields.append(species + "_" + record)
+ particle_fields.append(species.replace("_", "-")
+ + "_" + record.replace("_", "-"))
pass
else:
pass
- if len(list(f[bp + pp].keys())) > 1:
+ if len(f[bp + pp].keys()) > 1:
# There is more than one particle species, use the specific names as field types
self.field_list.extend(
[(str(field).split("_")[0],
@@ -187,21 +195,21 @@
self.num_grids = 0
- for mesh in list(f[bp + mp].keys()):
+ for mesh in f[bp + mp].keys():
if type(f[bp + mp + mesh]) is h5.Group:
- shape = f[bp + mp + mesh + "/" + list(f[bp + mp + mesh].keys())[0]].shape
+ shape = f[bp + mp + mesh + "/" + f[bp + mp + mesh].keys()[0]].shape
else:
shape = f[bp + mp + mesh].shape
spacing = tuple(f[bp + mp + mesh].attrs["gridSpacing"])
offset = tuple(f[bp + mp + mesh].attrs["gridGlobalOffset"])
unit_si = f[bp + mp + mesh].attrs["gridUnitSI"]
self.meshshapes[mesh] = (shape, spacing, offset, unit_si)
- for species in list(f[bp + pp].keys()):
- if "particlePatches" in list(f[bp + pp + "/" + species].keys()):
- for patch, size in enumerate(f[bp + pp + "/" + species + "/particlePatches/numParticles"]):
+ for species in f[bp + pp].keys():
+ if "particlePatches" in f[bp + pp + "/" + species].keys():
+ for (patch, size) in enumerate(f[bp + pp + "/" + species + "/particlePatches/numParticles"]):
self.numparts[species + "#" + str(patch)] = size
else:
- axis = list(f[bp + pp + species + "/position"].keys())[0]
+ axis = f[bp + pp + species + "/position"].keys()[0]
if is_const_component(f[bp + pp + species + "/position/" + axis]):
self.numparts[species] = f[bp + pp + species + "/position/" + axis].attrs["shape"]
else:
@@ -217,14 +225,14 @@
# Same goes for particle chunks if they are not inside particlePatches
patches = {}
no_patches = {}
- for (k, v) in list(self.numparts.items()):
+ for (k, v) in self.numparts.items():
if "#" in k:
patches[k] = v
else:
no_patches[k] = v
for size in set(no_patches.values()):
self.num_grids += int(np.ceil(size * self.vpg ** -1))
- for size in list(patches.values()):
+ for size in patches.values():
self.num_grids += int(np.ceil(size * self.vpg ** -1))
def _parse_index(self):
@@ -246,21 +254,6 @@
self.grid_levels.flat[:] = 0
self.grids = np.empty(self.num_grids, dtype="object")
- def get_component(group, component_name, _index=0, _offset=None):
- record_component = group[component_name]
- _unit_si = record_component.attrs["unitSI"]
- if is_const_component(record_component):
- _shape = tuple(record_component.attrs["shape"],)
- if _offset is None:
- _shape[0] -= _index
- else:
- _shape[0] = _offset
- return np.full(_shape, record_component.attrs["value"] * _unit_si)
- else:
- if _offset is not None:
- _offset += _index
- return np.multiply(record_component[_index:_offset], _unit_si)
-
grid_index_total = 0
# Mesh grids
@@ -281,11 +274,11 @@
grid_dim_offset = np.linspace(0, domain_dimension[0], num_grids + 1, dtype=np.int32)
grid_edge_offset = grid_dim_offset * np.float(domain_dimension[0]) ** -1 * (gre[0] - gle[0]) + gle[0]
mesh_names = []
- for (mname, mdata) in list(self.meshshapes.items()):
+ for (mname, mdata) in self.meshshapes.items():
if mesh == mdata:
mesh_names.append(str(mname))
prev = 0
- for grid in range(num_grids):
+ for grid in np.arange(num_grids):
self.grid_dimensions[grid_index_total] = domain_dimension
self.grid_dimensions[grid_index_total][0] = grid_dim_offset[grid + 1] - grid_dim_offset[grid]
self.grid_left_edge[grid_index_total] = gle
@@ -303,19 +296,19 @@
handled_ptypes = []
# Particle grids
- for (species, count) in list(self.numparts.items()):
+ for (species, count) in self.numparts.items():
if "#" in species:
# This is a particlePatch
spec = species.split("#")
patch = f[bp + pp + "/" + spec[0] + "/particlePatches"]
num_grids = int(np.ceil(count * self.vpg ** -1))
gle = []
- for axis in list(patch["offset"].keys()):
+ for axis in patch["offset"].keys():
gle.append(get_component(patch, "offset/" + axis, int(spec[1]), 1)[0])
gle = np.asarray(gle)
gle = np.append(gle, np.zeros(3 - len(gle)))
gre = []
- for axis in list(patch["extent"].keys()):
+ for axis in patch["extent"].keys():
gre.append(get_component(patch, "extent/" + axis, int(spec[1]), 1)[0])
gre = np.asarray(gre)
gre = np.append(gre, np.ones(3 - len(gre)))
@@ -330,7 +323,7 @@
gre = self.dataset.domain_right_edge
particle_count = np.linspace(0, count, num_grids + 1, dtype=np.int32)
particle_names = []
- for (pname, size) in list(self.numparts.items()):
+ for (pname, size) in self.numparts.items():
if size == count:
# Since this is not part of a particlePatch, we can include multiple same-sized ptypes
particle_names.append(str(pname))
@@ -338,7 +331,7 @@
else:
# A grid with this exact particle count has already been created
continue
- for grid in range(num_grids):
+ for grid in np.arange(num_grids):
self.grid_dimensions[grid_index_total] = [0, 0, 0] # Counted as mesh-size, thus no dimensional extent
self.grid_left_edge[grid_index_total] = gle
self.grid_right_edge[grid_index_total] = gre
@@ -356,7 +349,7 @@
Additionally, it should set up Children and Parent lists on each grid object.
openPMD is not adaptive and thus there are no Children and Parents for any grid.
"""
- for i in range(self.num_grids):
+ for i in np.arange(self.num_grids):
self.grids[i]._prepare_grid()
self.grids[i]._setup_dx()
self.max_level = 0
@@ -382,6 +375,7 @@
unit_system="mks",
**kwargs):
self._handle = HDF5FileHandler(filename)
+ self.gridsize = kwargs.pop("open_pmd_virtual_gridsize", 10**9)
self._set_paths(self._handle, path.dirname(filename))
Dataset.__init__(self,
filename,
@@ -390,12 +384,11 @@
unit_system=unit_system)
self.storage_filename = storage_filename
self.fluid_types += ("openPMD",)
- particles = tuple(str(c) for c in list(self._handle[self.base_path + self.particles_path].keys()))
+ particles = tuple(str(c) for c in self._handle[self.base_path + self.particles_path].keys())
if len(particles) > 1:
# Only use on-disk particle names if there is more than one species
self.particle_types = particles
mylog.debug("open_pmd - self.particle_types: {}".format(self.particle_types))
- self.gridsize = kwargs.pop("open_pmd_virtual_gridsize", 10**9)
self.particle_types_raw = self.particle_types
self.particle_types = tuple(self.particle_types)
@@ -411,7 +404,7 @@
iterations = []
encoding = handle.attrs["iterationEncoding"].decode()
if "groupBased" in encoding:
- iterations = list(handle["/data"].keys())
+ iterations = handle["/data"].keys()
mylog.info("open_pmd - found {} iterations in file".format(len(iterations)))
elif "fileBased" in encoding:
itformat = handle.attrs["iterationFormat"].decode().split("/")[-1]
@@ -427,9 +420,9 @@
if len(iterations) == 0:
mylog.warning("open_pmd - no iterations found!")
if "groupBased" in encoding and len(iterations) > 1:
- mylog.warning("open_pmd - only choose to load one iteration ({})".format(list(handle["/data"].keys())[0]))
+ mylog.warning("open_pmd - only choose to load one iteration ({})".format(handle["/data"].keys()[0]))
- self.base_path = "/data/{}/".format(list(handle["/data"].keys())[0])
+ self.base_path = "/data/{}/".format(handle["/data"].keys()[0])
self.meshes_path = self._handle["/"].attrs["meshesPath"].decode()
self.particles_path = self._handle["/"].attrs["particlesPath"].decode()
@@ -458,19 +451,21 @@
self.refine_by = 1
self.cosmological_simulation = 0
- try:
- mesh = list(f[bp + mp].keys())[0]
- axis = list(f[bp + mp + "/" + mesh].keys())[0]
- fshape = np.asarray(f[bp + mp + "/" + mesh + "/" + axis].shape, dtype=np.int64)
- except:
- fshape = np.array([1, 1, 1], dtype=np.int64)
- mylog.warning("open_pmd - Could not detect shape of simulated field! "
- "Assuming a single cell and thus setting fshape to [1, 1, 1]!")
+ shapes = [[1, 1, 1]]
+ for mesh in f[bp + mp].keys():
+ if type(f[bp + mp + mesh]) is h5.Group:
+ shapes.append(f[bp + mp + mesh + "/" + f[bp + mp + mesh].keys()[0]].shape)
+ else:
+ shapes.append(f[bp + mp + mesh].shape)
+ fshape = np.empty(3, dtype=np.float64)
+ shapes = np.asarray(shapes)
+ for i in [0, 1, 2]:
+ fshape[i] = np.max(shapes.transpose()[i])
self.dimensionality = len(fshape)
self.domain_dimensions = np.append(fshape, np.ones(3 - self.dimensionality))
try:
- mesh = list(f[bp + mp].keys())[0]
+ mesh = f[bp + mp].keys()[0]
spacing = np.asarray(f[bp + mp + "/" + mesh].attrs["gridSpacing"])
offset = np.asarray(f[bp + mp + "/" + mesh].attrs["gridGlobalOffset"])
unit_si = f[bp + mp + "/" + mesh].attrs["gridUnitSI"]
@@ -480,9 +475,8 @@
self.domain_right_edge += self.domain_left_edge
self.domain_left_edge = np.append(self.domain_left_edge, np.zeros(3 - self.domain_left_edge.size))
self.domain_right_edge = np.append(self.domain_right_edge, np.ones(3 - self.domain_right_edge.size))
- except:
- mylog.warning(
- "open_pmd - The domain extent could not be calculated! Setting the field extent to 1m**3!")
+ except IndexError:
+ mylog.warning("open_pmd - It seems your data does not contain meshes. Assuming domain extent of 1m^3!")
self.domain_left_edge = np.zeros(3, dtype=np.float64)
self.domain_right_edge = np.ones(3, dtype=np.float64)
@@ -493,19 +487,19 @@
"""Checks whether the supplied file can be read by this frontend.
"""
try:
- f = h5.File(args[0])
- except:
+ f = h5.File(args[0], "r")
+ except (IOError, OSError):
return False
requirements = ["openPMD", "basePath", "meshesPath", "particlesPath"]
- attrs = list(f["/"].attrs.keys())
+ attrs = f["/"].attrs.keys()
for i in requirements:
if i not in attrs:
return False
- versions = ["1.0.0", "1.0.1"]
- for i in versions:
- if i in f.attrs["openPMD"].decode():
- return True
-
- return False
+ known_versions = [StrictVersion("1.0.0"),
+ StrictVersion("1.0.1")]
+ if StrictVersion(f.attrs["openPMD"].decode()) in known_versions:
+ return True
+ else:
+ return False
diff -r b9c231a50356c2a29e9d81cc2338007767f9b625 -r 2db00bdb6bfb8806692228cb69f4b743fbf790ac yt/frontends/open_pmd/fields.py
--- a/yt/frontends/open_pmd/fields.py
+++ b/yt/frontends/open_pmd/fields.py
@@ -22,9 +22,9 @@
from yt.frontends.open_pmd.misc import \
parse_unit_dimension, \
is_const_component
-from yt.utilities.on_demand_imports import _h5py as h5
from yt.units.yt_array import YTQuantity
from yt.utilities.logger import ytLogger as mylog
+from yt.utilities.on_demand_imports import _h5py as h5
from yt.utilities.physical_constants import \
speed_of_light, \
mu_0
@@ -142,9 +142,9 @@
pp = ds.particles_path
fields = f[bp + mp]
- for fname in list(fields.keys()):
+ for fname in fields.keys():
field = fields[fname]
- if type(field) is h5.Dataset:
+ if type(field) is h5.Dataset or is_const_component(field):
# Don't consider axes. This appears to be a vector field of single dimensionality
ytname = str("_".join([fname.replace("_", "-")]))
parsed = parse_unit_dimension(np.asarray(field.attrs["unitDimension"], dtype=np.int))
@@ -152,51 +152,45 @@
aliases = []
# Save a list of magnetic fields for aliasing later on
# We can not reasonably infer field type/unit by name in openPMD
- if unit in "T" or "kg/(A*s**2)" in unit:
+ if unit == "T" or unit == "kg/(A*s**2)":
self._mag_fields.append(ytname)
self.known_other_fields += ((ytname, (unit, aliases, None)),)
else:
- axes = list(field.keys())
- for axis in axes:
+ for axis in field.keys():
ytname = str("_".join([fname.replace("_", "-"), axis]))
parsed = parse_unit_dimension(np.asarray(field.attrs["unitDimension"], dtype=np.int))
unit = str(YTQuantity(1, parsed).units)
aliases = []
# Save a list of magnetic fields for aliasing later on
# We can not reasonably infer field type by name in openPMD
- if unit in "T" or "kg/(A*s**2)" in unit:
+ if unit == "T" or unit == "kg/(A*s**2)":
self._mag_fields.append(ytname)
self.known_other_fields += ((ytname, (unit, aliases, None)),)
for i in self.known_other_fields:
mylog.debug("open_pmd - known_other_fields - {}".format(i))
- particle_fields = ()
particles = f[bp + pp]
- for species in list(particles.keys()):
- for attrib in list(particles[species].keys()):
+ for species in particles.keys():
+ for record in particles[species].keys():
try:
- parsed = parse_unit_dimension(
- np.asarray(particles.get(species).get(attrib).attrs["unitDimension"],
- dtype=np.int))
+ pds = particles[species + "/" + record]
+ parsed = parse_unit_dimension(pds.attrs["unitDimension"])
unit = str(YTQuantity(1, parsed).units)
- name = ["particle", attrib]
- ytattrib = attrib
- if ytattrib in "position":
+ ytattrib = str(record).replace("_", "-")
+ if ytattrib == "position":
# Symbolically rename position to preserve yt's interpretation of the pfield
# particle_position is later derived in setup_absolute_positions in the way yt expects it
ytattrib = "positionCoarse"
- pds = particles[species + "/" + attrib]
if type(pds) is h5.Dataset or is_const_component(pds):
- particle_fields += ((str("_".join(name)), (unit, [], None)),)
+ name = ["particle", ytattrib]
+ self.known_particle_fields += ((str("_".join(name)), (unit, [], None)),)
else:
- for axis in list(pds.keys()):
+ for axis in pds.keys():
aliases = []
- if axis in "rxyz":
- name = ["particle", ytattrib.replace("_", "-"), axis]
+ name = ["particle", ytattrib, axis]
ytname = str("_".join(name))
- particle_fields += ((ytname, (unit, aliases, None)),)
+ self.known_particle_fields += ((ytname, (unit, aliases, None)),)
except KeyError:
- mylog.info("open_pmd - {}_{} does not seem to have unitDimension".format(species, attrib))
- self.known_particle_fields = particle_fields
+ mylog.info("open_pmd - {}_{} does not seem to have unitDimension".format(species, record))
for i in self.known_particle_fields:
mylog.debug("open_pmd - known_particle_fields - {}".format(i))
super(OpenPMDFieldInfo, self).__init__(ds, field_list)
diff -r b9c231a50356c2a29e9d81cc2338007767f9b625 -r 2db00bdb6bfb8806692228cb69f4b743fbf790ac yt/frontends/open_pmd/io.py
--- a/yt/frontends/open_pmd/io.py
+++ b/yt/frontends/open_pmd/io.py
@@ -19,11 +19,13 @@
import numpy as np
-from yt.frontends.open_pmd.misc import is_const_component
+from yt.frontends.open_pmd.misc import \
+ is_const_component, \
+ get_component
from yt.utilities.io_handler import BaseIOHandler
-class IOHandlerOpenPMD(BaseIOHandler):
+class IOHandlerOpenPMDHDF5(BaseIOHandler):
_field_dtype = "float32"
_dataset_type = "openPMD"
@@ -49,18 +51,18 @@
if str((ptype, index, offset)) not in self._cached_ptype:
self._cached_ptype = str((ptype, index, offset))
pds = self._handle[self.base_path + self.particles_path + "/" + ptype]
- axes = [str(ax) for ax in list(pds["position"].keys())]
+ axes = [str(ax) for ax in pds["position"].keys()]
if offset is None:
if is_const_component(pds["position/" + axes[0]]):
offset = pds["position/" + axes[0]].attrs["shape"]
else:
offset = pds["position/" + axes[0]].len()
self.cache = np.empty((3, offset), dtype=np.float64)
- for i in range(3):
+ for i in np.arange(3):
ax = "xyz"[i]
if ax in axes:
- np.add(self.get_component(pds, "position/" + ax, index, offset),
- self.get_component(pds, "positionOffset/" + ax, index, offset),
+ np.add(get_component(pds, "position/" + ax, index, offset),
+ get_component(pds, "positionOffset/" + ax, index, offset),
self.cache[i])
else:
# Pad accordingly with zeros to make 1D/2D datasets compatible
@@ -119,8 +121,8 @@
for ptype in ptf:
for chunk in chunks:
for grid in chunk.objs:
- if ptype in "io":
- species = list(ds.keys())[0]
+ if str(ptype) == "io":
+ species = ds.keys()[0]
else:
species = ptype
if species not in grid.ptypes:
@@ -134,7 +136,7 @@
for field in ptf[ptype]:
component = "/".join(field.split("_")[1:]).replace("positionCoarse", "position").replace("-",
"_")
- data = self.get_component(pds, component, grid.pindex, grid.poffset)[mask]
+ data = get_component(pds, component, grid.pindex, grid.poffset)[mask]
for request_field in rfm[(ptype, field)]:
rv[request_field][ind[request_field]:ind[request_field] + data.shape[0]] = data
ind[request_field] += data.shape[0]
@@ -196,7 +198,7 @@
mask = grid._get_selector_mask(selector)
if mask is None:
continue
- data = self.get_component(ds, component, grid.findex, grid.foffset)
+ data = get_component(ds, component, grid.findex, grid.foffset)
# The following is a modified AMRGridPatch.select(...)
data.shape = mask.shape # Workaround - casts a 2D (x,y) array to 3D (x,y,1)
count = grid.count(selector)
@@ -208,45 +210,3 @@
rv[field].flatten()
return rv
-
- def get_component(self, group, component_name, index=0, offset=None):
- """Grabs a dataset component from a group as a whole or sliced.
-
- Parameters
- ----------
- group : h5py.Group
- component_name : str
- relative path of the component in the group
- index : int, optional
- first entry along the first axis to read
- offset : int, optional
- number of entries to read
- if not supplied, every entry after index is returned
-
- Notes
- -----
- This scales every entry of the component with the respective "unitSI".
-
- Returns
- -------
- ndarray
- (N,) 1D in case of particle data
- (O,P,Q) 1D/2D/3D in case of mesh data
- """
- record_component = group[component_name]
- unit_si = record_component.attrs["unitSI"]
- if is_const_component(record_component):
- shape = tuple(record_component.attrs["shape"],)
- if offset is None:
- shape[0] -= index # This makes constant meshes readable
- else:
- shape[0] = offset
- # component is constant, craft an array by hand
- # mylog.debug("open_pmd - get_component: {}/{} [const {}]".format(group.name, component_name, shape))
- return np.full(shape, record_component.attrs["value"] * unit_si)
- else:
- if offset is not None:
- offset += index
- # component is a dataset, return it (possibly masked)
- # mylog.debug("open_pmd - get_component: {}/{}[{}:{}]".format(group.name, component_name, index, offset))
- return np.multiply(record_component[index:offset], unit_si)
diff -r b9c231a50356c2a29e9d81cc2338007767f9b625 -r 2db00bdb6bfb8806692228cb69f4b743fbf790ac yt/frontends/open_pmd/misc.py
--- a/yt/frontends/open_pmd/misc.py
+++ b/yt/frontends/open_pmd/misc.py
@@ -56,7 +56,7 @@
"C",
"mol",
"cd"]
- for i in range(7):
+ for i in np.arange(7):
if unit_dimension[i] != 0:
dim.append("{}**{}".format(si[i], unit_dimension[i]))
return "*".join(dim)
@@ -79,4 +79,47 @@
.. https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md,
section 'Constant Record Components'
"""
- return "value" in list(record_component.attrs.keys())
+ return "value" in record_component.attrs.keys()
+
+
+def get_component(group, component_name, index=0, offset=None):
+ """Grabs a dataset component from a group as a whole or sliced.
+
+ Parameters
+ ----------
+ group : h5py.Group
+ component_name : str
+ relative path of the component in the group
+ index : int, optional
+ first entry along the first axis to read
+ offset : int, optional
+ number of entries to read
+ if not supplied, every entry after index is returned
+
+ Notes
+ -----
+ This scales every entry of the component with the respective "unitSI".
+
+ Returns
+ -------
+ ndarray
+ (N,) 1D in case of particle data
+ (O,P,Q) 1D/2D/3D in case of mesh data
+ """
+ record_component = group[component_name]
+ unit_si = record_component.attrs["unitSI"]
+ if is_const_component(record_component):
+ shape = np.asarray(record_component.attrs["shape"])
+ if offset is None:
+ shape[0] -= index
+ else:
+ shape[0] = offset
+ # component is constant, craft an array by hand
+ # mylog.debug("open_pmd - get_component: {}/{} [const {}]".format(group.name, component_name, shape))
+ return np.full(shape, record_component.attrs["value"] * unit_si)
+ else:
+ if offset is not None:
+ offset += index
+ # component is a dataset, return it (possibly masked)
+ # mylog.debug("open_pmd - get_component: {}/{}[{}:{}]".format(group.name, component_name, index, offset))
+ return np.multiply(record_component[index:offset], unit_si)
https://bitbucket.org/yt_analysis/yt/commits/882c4e9d9d35/
Changeset: 882c4e9d9d35
Branch: yt
User: Fabian Koller
Date: 2016-09-23 11:09:35+00:00
Summary: Restore python3 compatability
Affected #: 2 files
diff -r 2db00bdb6bfb8806692228cb69f4b743fbf790ac -r 882c4e9d9d357b932dc1a8a88c243b3c3ad7fdf9 yt/frontends/open_pmd/data_structures.py
--- a/yt/frontends/open_pmd/data_structures.py
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -100,10 +100,10 @@
pp = self.dataset.particles_path
for ptype in self.ds.particle_types_raw:
if str(ptype) == "io":
- spec = f[bp + pp].keys()[0]
+ spec = list(f[bp + pp].keys())[0]
else:
spec = ptype
- axis = f[bp + pp + "/" + spec + "/position"].keys()[0]
+ axis = list(f[bp + pp + "/" + spec + "/position"].keys())[0]
pos = f[bp + pp + "/" + spec + "/position/" + axis]
if is_const_component(pos):
result[ptype] = pos.attrs["shape"]
@@ -152,7 +152,7 @@
try:
# Create a field for every axis (x,y,z) of every property (position)
# of every species (electrons)
- axes = f[bp + pp + species + "/" + record].keys()
+ axes = list(f[bp + pp + species + "/" + record].keys())
if str(record) == "position":
record = "positionCoarse"
for axis in axes:
@@ -166,7 +166,7 @@
pass
else:
pass
- if len(f[bp + pp].keys()) > 1:
+ if len(list(f[bp + pp].keys())) > 1:
# There is more than one particle species, use the specific names as field types
self.field_list.extend(
[(str(field).split("_")[0],
@@ -197,7 +197,7 @@
for mesh in f[bp + mp].keys():
if type(f[bp + mp + mesh]) is h5.Group:
- shape = f[bp + mp + mesh + "/" + f[bp + mp + mesh].keys()[0]].shape
+ shape = f[bp + mp + mesh + "/" + list(f[bp + mp + mesh].keys())[0]].shape
else:
shape = f[bp + mp + mesh].shape
spacing = tuple(f[bp + mp + mesh].attrs["gridSpacing"])
@@ -209,7 +209,7 @@
for (patch, size) in enumerate(f[bp + pp + "/" + species + "/particlePatches/numParticles"]):
self.numparts[species + "#" + str(patch)] = size
else:
- axis = f[bp + pp + species + "/position"].keys()[0]
+ axis = list(f[bp + pp + species + "/position"].keys())[0]
if is_const_component(f[bp + pp + species + "/position/" + axis]):
self.numparts[species] = f[bp + pp + species + "/position/" + axis].attrs["shape"]
else:
@@ -404,7 +404,7 @@
iterations = []
encoding = handle.attrs["iterationEncoding"].decode()
if "groupBased" in encoding:
- iterations = handle["/data"].keys()
+ iterations = list(handle["/data"].keys())
mylog.info("open_pmd - found {} iterations in file".format(len(iterations)))
elif "fileBased" in encoding:
itformat = handle.attrs["iterationFormat"].decode().split("/")[-1]
@@ -420,9 +420,9 @@
if len(iterations) == 0:
mylog.warning("open_pmd - no iterations found!")
if "groupBased" in encoding and len(iterations) > 1:
- mylog.warning("open_pmd - only choose to load one iteration ({})".format(handle["/data"].keys()[0]))
+ mylog.warning("open_pmd - only choose to load one iteration ({})".format(list(handle["/data"].keys())[0]))
- self.base_path = "/data/{}/".format(handle["/data"].keys()[0])
+ self.base_path = "/data/{}/".format(list(handle["/data"].keys())[0])
self.meshes_path = self._handle["/"].attrs["meshesPath"].decode()
self.particles_path = self._handle["/"].attrs["particlesPath"].decode()
@@ -454,7 +454,7 @@
shapes = [[1, 1, 1]]
for mesh in f[bp + mp].keys():
if type(f[bp + mp + mesh]) is h5.Group:
- shapes.append(f[bp + mp + mesh + "/" + f[bp + mp + mesh].keys()[0]].shape)
+ shapes.append(f[bp + mp + mesh + "/" + list(f[bp + mp + mesh].keys())[0]].shape)
else:
shapes.append(f[bp + mp + mesh].shape)
fshape = np.empty(3, dtype=np.float64)
@@ -465,7 +465,7 @@
self.domain_dimensions = np.append(fshape, np.ones(3 - self.dimensionality))
try:
- mesh = f[bp + mp].keys()[0]
+ mesh = list(f[bp + mp].keys())[0]
spacing = np.asarray(f[bp + mp + "/" + mesh].attrs["gridSpacing"])
offset = np.asarray(f[bp + mp + "/" + mesh].attrs["gridGlobalOffset"])
unit_si = f[bp + mp + "/" + mesh].attrs["gridUnitSI"]
@@ -492,14 +492,17 @@
return False
requirements = ["openPMD", "basePath", "meshesPath", "particlesPath"]
- attrs = f["/"].attrs.keys()
+ attrs = list(f["/"].attrs.keys())
for i in requirements:
if i not in attrs:
+ f.close()
return False
known_versions = [StrictVersion("1.0.0"),
StrictVersion("1.0.1")]
if StrictVersion(f.attrs["openPMD"].decode()) in known_versions:
+ f.close()
return True
else:
+ f.close()
return False
diff -r 2db00bdb6bfb8806692228cb69f4b743fbf790ac -r 882c4e9d9d357b932dc1a8a88c243b3c3ad7fdf9 yt/frontends/open_pmd/io.py
--- a/yt/frontends/open_pmd/io.py
+++ b/yt/frontends/open_pmd/io.py
@@ -51,7 +51,7 @@
if str((ptype, index, offset)) not in self._cached_ptype:
self._cached_ptype = str((ptype, index, offset))
pds = self._handle[self.base_path + self.particles_path + "/" + ptype]
- axes = [str(ax) for ax in pds["position"].keys()]
+ axes = list(pds["position"].keys())
if offset is None:
if is_const_component(pds["position/" + axes[0]]):
offset = pds["position/" + axes[0]].attrs["shape"]
@@ -122,7 +122,7 @@
for chunk in chunks:
for grid in chunk.objs:
if str(ptype) == "io":
- species = ds.keys()[0]
+ species = list(ds.keys())[0]
else:
species = ptype
if species not in grid.ptypes:
@@ -134,8 +134,9 @@
continue
pds = ds[species]
for field in ptf[ptype]:
- component = "/".join(field.split("_")[1:]).replace("positionCoarse", "position").replace("-",
- "_")
+ component = "/".join(field.split("_")[1:])
+ component = component.replace("positionCoarse", "position")
+ component = component.replace("-", "_")
data = get_component(pds, component, grid.pindex, grid.poffset)[mask]
for request_field in rfm[(ptype, field)]:
rv[request_field][ind[request_field]:ind[request_field] + data.shape[0]] = data
https://bitbucket.org/yt_analysis/yt/commits/a5b20901c1cc/
Changeset: a5b20901c1cc
Branch: yt
User: Fabian Koller
Date: 2016-09-23 11:24:15+00:00
Summary: Fix API reference source indent
Affected #: 1 file
diff -r 882c4e9d9d357b932dc1a8a88c243b3c3ad7fdf9 -r a5b20901c1cccac0b8e0479b67e3c7ce7b93a8d8 doc/source/reference/api/api.rst
--- a/doc/source/reference/api/api.rst
+++ b/doc/source/reference/api/api.rst
@@ -327,16 +327,16 @@
^^^^^^^
.. autosummary::
-:toctree: generated/
+ :toctree: generated/
- ~yt.frontends.open_pmd.data_structures.OpenPMDGrid
- ~yt.frontends.open_pmd.data_structures.OpenPMDHierarchy
- ~yt.frontends.open_pmd.data_structures.OpenPMDDataset
- ~yt.frontends.open_pmd.fields.OpenPMDFieldInfo
- ~yt.frontends.open_pmd.io.IOHandlerOpenPMDHDF5
- ~yt.frontends.open_pmd.misc.parse_unit_dimension
- ~yt.frontends.open_pmd.misc.is_const_component
- ~yt.frontends.open_pmd.misc.get_component
+ ~yt.frontends.open_pmd.data_structures.OpenPMDGrid
+ ~yt.frontends.open_pmd.data_structures.OpenPMDHierarchy
+ ~yt.frontends.open_pmd.data_structures.OpenPMDDataset
+ ~yt.frontends.open_pmd.fields.OpenPMDFieldInfo
+ ~yt.frontends.open_pmd.io.IOHandlerOpenPMDHDF5
+ ~yt.frontends.open_pmd.misc.parse_unit_dimension
+ ~yt.frontends.open_pmd.misc.is_const_component
+ ~yt.frontends.open_pmd.misc.get_component
RAMSES
^^^^^^
https://bitbucket.org/yt_analysis/yt/commits/0c4431584e15/
Changeset: 0c4431584e15
Branch: yt
User: Fabian Koller
Date: 2016-09-23 14:57:47+00:00
Summary: Rework domain extent calculation
Affected #: 1 file
diff -r a5b20901c1cccac0b8e0479b67e3c7ce7b93a8d8 -r 0c4431584e15ddd5412b5d5f78f8c3b8a1b1c209 yt/frontends/open_pmd/data_structures.py
--- a/yt/frontends/open_pmd/data_structures.py
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -23,7 +23,6 @@
from re import match
import numpy as np
-
from yt.data_objects.grid_patch import AMRGridPatch
from yt.data_objects.static_output import Dataset
from yt.frontends.open_pmd.fields import OpenPMDFieldInfo
@@ -451,32 +450,42 @@
self.refine_by = 1
self.cosmological_simulation = 0
- shapes = [[1, 1, 1]]
- for mesh in f[bp + mp].keys():
- if type(f[bp + mp + mesh]) is h5.Group:
- shapes.append(f[bp + mp + mesh + "/" + list(f[bp + mp + mesh].keys())[0]].shape)
- else:
- shapes.append(f[bp + mp + mesh].shape)
- fshape = np.empty(3, dtype=np.float64)
- shapes = np.asarray(shapes)
- for i in [0, 1, 2]:
- fshape[i] = np.max(shapes.transpose()[i])
- self.dimensionality = len(fshape)
- self.domain_dimensions = np.append(fshape, np.ones(3 - self.dimensionality))
-
try:
- mesh = list(f[bp + mp].keys())[0]
- spacing = np.asarray(f[bp + mp + "/" + mesh].attrs["gridSpacing"])
- offset = np.asarray(f[bp + mp + "/" + mesh].attrs["gridGlobalOffset"])
- unit_si = f[bp + mp + "/" + mesh].attrs["gridUnitSI"]
- self.domain_left_edge = offset * unit_si
- # self.domain_left_edge = np.zeros(3, dtype=np.float64)
- self.domain_right_edge = self.domain_dimensions[:spacing.size] * unit_si * spacing
- self.domain_right_edge += self.domain_left_edge
- self.domain_left_edge = np.append(self.domain_left_edge, np.zeros(3 - self.domain_left_edge.size))
- self.domain_right_edge = np.append(self.domain_right_edge, np.ones(3 - self.domain_right_edge.size))
- except IndexError:
+ shapes = {}
+ left_edges = {}
+ right_edges = {}
+ for mesh in f[bp + mp].keys():
+ if type(f[bp + mp + mesh]) is h5.Group:
+ shape = np.asarray(f[bp + mp + mesh + "/" + list(f[bp + mp + mesh].keys())[0]].shape)
+ else:
+ shapes[mesh] = np.asarray(f[bp + mp + mesh].shape)
+ spacing = np.asarray(f[bp + mp + mesh].attrs["gridSpacing"])
+ offset = np.asarray(f[bp + mp + mesh].attrs["gridGlobalOffset"])
+ unit_si = np.asarray(f[bp + mp + mesh].attrs["gridUnitSI"])
+ le = offset * unit_si
+ re = le + shape * unit_si * spacing
+ shapes[mesh] = shape
+ left_edges[mesh] = le
+ right_edges[mesh] = re
+ lowest_dim = np.min([len(i) for i in shapes.values()])
+ shapes = np.asarray([i[:lowest_dim] for i in shapes.values()])
+ left_edges = np.asarray([i[:lowest_dim] for i in left_edges.values()])
+ right_edges = np.asarray([i[:lowest_dim] for i in right_edges.values()])
+ fs = []
+ dle = []
+ dre = []
+ for i in np.arange(lowest_dim):
+ fs.append(np.max(shapes.transpose()[i]))
+ dle.append(np.min(left_edges.transpose()[i]))
+ dre.append(np.min(right_edges.transpose()[i]))
+ self.dimensionality = len(fs)
+ self.domain_dimensions = np.append(fs, np.ones(3 - self.dimensionality))
+ self.domain_left_edge = np.append(dle, np.zeros(3 - len(dle)))
+ self.domain_right_edge = np.append(dre, np.ones(3 - len(dre)))
+ except ValueError:
mylog.warning("open_pmd - It seems your data does not contain meshes. Assuming domain extent of 1m^3!")
+ self.dimensionality = 3
+ self.domain_dimensions = np.ones(3, dtype=np.float64)
self.domain_left_edge = np.zeros(3, dtype=np.float64)
self.domain_right_edge = np.ones(3, dtype=np.float64)
https://bitbucket.org/yt_analysis/yt/commits/71960f84ae9d/
Changeset: 71960f84ae9d
Branch: yt
User: atmyers
Date: 2016-09-23 17:55:57+00:00
Summary: Merged in C0nsultant/openpmd (pull request #2376)
Add openPMD frontend
Affected #: 17 files
diff -r 068ad05bda58e31a1684d907398a61ce93fd595c -r 71960f84ae9d3e12d70759e342b2bc581581674c doc/source/examining/loading_data.rst
--- a/doc/source/examining/loading_data.rst
+++ b/doc/source/examining/loading_data.rst
@@ -1519,6 +1519,57 @@
# The halo mass
print(ad["FOF", "particle_mass"])
+.. _loading-openpmd-data:
+
+openPMD Data
+---------
+
+`openPMD <http://www.openpmd.org>`_ is an open source meta-standard and naming
+scheme for mesh based data and particle data. It does not actually define a file
+format.
+
+HDF5-containers respecting the minimal set of meta information from
+versions 1.0.0 and 1.0.1 of the standard are compatible.
+Support for the ED-PIC extension is not available. Mesh data in cartesian coordinates
+and particle data can be read by this frontend.
+
+To load the first in-file iteration of a openPMD datasets using the standard HDF5
+output format:
+
+.. code-block:: python
+
+ import yt
+ ds = yt.load('example-3d/hdf5/data00000100.h5')
+
+If you operate on large files, you may want to modify the virtual chunking behaviour through
+``open_pmd_virtual_gridsize``. The supplied value is an estimate of the size of a single read request
+for each particle attribute/mesh (in Byte).
+
+.. code-block:: python
+
+ import yt
+ ds = yt.load('example-3d/hdf5/data00000100.h5', open_pmd_virtual_gridsize=10e4)
+ sp = yt.SlicePlot(ds, 'x', 'rho')
+ sp.show()
+
+Particle data is fully supported:
+
+.. code-block:: python
+
+ import yt
+ ds = yt.load('example-3d/hdf5/data00000100.h5')
+ ad = f.all_data()
+ ppp = yt.ParticlePhasePlot(ad, 'particle_position_y', 'particle_momentum_y', 'particle_weighting')
+ ppp.show()
+
+.. rubric:: Caveats
+
+* 1D, 2D and 3D data is compatible, but lower dimensional data might yield
+ strange results since it gets padded and treated as 3D. Extraneous dimensions are
+ set to be of length 1.0m and have a width of one cell.
+* The frontend has hardcoded logic for renaming the openPMD ``position``
+ of particles to ``positionCoarse``
+
.. _loading-pyne-data:
PyNE Data
diff -r 068ad05bda58e31a1684d907398a61ce93fd595c -r 71960f84ae9d3e12d70759e342b2bc581581674c doc/source/reference/api/api.rst
--- a/doc/source/reference/api/api.rst
+++ b/doc/source/reference/api/api.rst
@@ -323,6 +323,21 @@
~yt.frontends.moab.io.IOHandlerMoabH5MHex8
~yt.frontends.moab.io.IOHandlerMoabPyneHex8
+OpenPMD
+^^^^^^^
+
+.. autosummary::
+ :toctree: generated/
+
+ ~yt.frontends.open_pmd.data_structures.OpenPMDGrid
+ ~yt.frontends.open_pmd.data_structures.OpenPMDHierarchy
+ ~yt.frontends.open_pmd.data_structures.OpenPMDDataset
+ ~yt.frontends.open_pmd.fields.OpenPMDFieldInfo
+ ~yt.frontends.open_pmd.io.IOHandlerOpenPMDHDF5
+ ~yt.frontends.open_pmd.misc.parse_unit_dimension
+ ~yt.frontends.open_pmd.misc.is_const_component
+ ~yt.frontends.open_pmd.misc.get_component
+
RAMSES
^^^^^^
diff -r 068ad05bda58e31a1684d907398a61ce93fd595c -r 71960f84ae9d3e12d70759e342b2bc581581674c doc/source/reference/code_support.rst
--- a/doc/source/reference/code_support.rst
+++ b/doc/source/reference/code_support.rst
@@ -48,6 +48,8 @@
+-----------------------+------------+-----------+------------+-------+----------+----------+------------+----------+
| Nyx | Y | N | Y | Y | Y | Y | Y | Full |
+-----------------------+------------+-----------+------------+-------+----------+----------+------------+----------+
+| openPMD | Y | Y | N | Y | Y | Y | N | Partial |
++-----------------------+------------+-----------+------------+-------+----------+----------+------------+----------+
| Orion | Y | Y | Y | Y | Y | Y | Y | Full |
+-----------------------+------------+-----------+------------+-------+----------+----------+------------+----------+
| OWLS/EAGLE | Y | Y | Y | Y | Y [#f2]_ | Y | Y | Full |
diff -r 068ad05bda58e31a1684d907398a61ce93fd595c -r 71960f84ae9d3e12d70759e342b2bc581581674c yt/frontends/api.py
--- a/yt/frontends/api.py
+++ b/yt/frontends/api.py
@@ -35,6 +35,7 @@
'halo_catalog',
'http_stream',
'moab',
+ 'open_pmd',
'owls',
'owls_subfind',
'ramses',
diff -r 068ad05bda58e31a1684d907398a61ce93fd595c -r 71960f84ae9d3e12d70759e342b2bc581581674c yt/frontends/open_pmd/__init__.py
--- /dev/null
+++ b/yt/frontends/open_pmd/__init__.py
@@ -0,0 +1,15 @@
+"""
+API for yt.frontends.open_pmd
+
+
+
+"""
+
+# -----------------------------------------------------------------------------
+# Copyright (c) 2013, yt Development Team.
+# Copyright (c) 2015, Daniel Grassinger (HZDR)
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+# -----------------------------------------------------------------------------
diff -r 068ad05bda58e31a1684d907398a61ce93fd595c -r 71960f84ae9d3e12d70759e342b2bc581581674c yt/frontends/open_pmd/api.py
--- /dev/null
+++ b/yt/frontends/open_pmd/api.py
@@ -0,0 +1,29 @@
+"""
+API for yt.frontends.open_pmd
+
+
+
+"""
+
+# -----------------------------------------------------------------------------
+# Copyright (c) 2013, yt Development Team.
+# Copyright (c) 2015, Daniel Grassinger (HZDR)
+# Copyright (c) 2016, Fabian Koller (HZDR)
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+# -----------------------------------------------------------------------------
+
+from .data_structures import \
+ OpenPMDDataset, \
+ OpenPMDGrid, \
+ OpenPMDHierarchy
+
+from .fields import \
+ OpenPMDFieldInfo
+
+from .io import \
+ IOHandlerOpenPMDHDF5
+
+from . import \
+ tests
diff -r 068ad05bda58e31a1684d907398a61ce93fd595c -r 71960f84ae9d3e12d70759e342b2bc581581674c yt/frontends/open_pmd/data_structures.py
--- /dev/null
+++ b/yt/frontends/open_pmd/data_structures.py
@@ -0,0 +1,517 @@
+"""
+openPMD data structures
+
+
+"""
+
+# -----------------------------------------------------------------------------
+# Copyright (c) 2013, yt Development Team.
+# Copyright (c) 2015, Daniel Grassinger (HZDR)
+# Copyright (c) 2016, Fabian Koller (HZDR)
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+# -----------------------------------------------------------------------------
+
+from distutils.version import StrictVersion
+from functools import reduce
+from operator import mul
+from os import \
+ path, \
+ listdir
+from re import match
+
+import numpy as np
+from yt.data_objects.grid_patch import AMRGridPatch
+from yt.data_objects.static_output import Dataset
+from yt.frontends.open_pmd.fields import OpenPMDFieldInfo
+from yt.frontends.open_pmd.misc import \
+ is_const_component, \
+ get_component
+from yt.funcs import setdefaultattr
+from yt.geometry.grid_geometry_handler import GridIndex
+from yt.utilities.file_handler import HDF5FileHandler
+from yt.utilities.logger import ytLogger as mylog
+from yt.utilities.on_demand_imports import _h5py as h5
+
+
+class OpenPMDGrid(AMRGridPatch):
+ """Represents chunk of data on-disk.
+
+ This defines the index and offset for every mesh and particle type.
+ It also defines parents and children grids. Since openPMD does not have multiple levels of refinement,
+ there are no parents or children for any grid.
+ """
+ _id_offset = 0
+ __slots__ = ["_level_id"]
+ # Every particle species and mesh might have different hdf5-indices and offsets
+ ftypes=[]
+ ptypes=[]
+ findex = 0
+ foffset = 0
+ pindex = 0
+ poffset = 0
+
+ def __init__(self, gid, index, level=-1, fi=0, fo=0, pi=0, po=0, ft=[], pt=[]):
+ AMRGridPatch.__init__(self, gid, filename=index.index_filename,
+ index=index)
+ self.findex = fi
+ self.foffset = fo
+ self.pindex = pi
+ self.poffset = po
+ self.ftypes = ft
+ self.ptypes = pt
+ self.Parent = None
+ self.Children = []
+ self.Level = level
+
+ def __repr__(self):
+ return "OpenPMDGrid_%04i (%s)" % (self.id, self.ActiveDimensions)
+
+
+class OpenPMDHierarchy(GridIndex):
+ """Defines which fields and particles are created and read from disk.
+
+ Furthermore it defines the characteristics of the grids.
+ """
+ grid = OpenPMDGrid
+
+ def __init__(self, ds, dataset_type="openPMD"):
+ self.dataset_type = dataset_type
+ self.dataset = ds
+ self.index_filename = ds.parameter_filename
+ self.directory = path.dirname(self.index_filename)
+ GridIndex.__init__(self, ds, dataset_type)
+
+ def _get_particle_type_counts(self):
+ """Reads the active number of particles for every species.
+
+ Returns
+ -------
+ dict
+ keys are ptypes
+ values are integer counts of the ptype
+ """
+ result = {}
+ f = self.dataset._handle
+ bp = self.dataset.base_path
+ pp = self.dataset.particles_path
+ for ptype in self.ds.particle_types_raw:
+ if str(ptype) == "io":
+ spec = list(f[bp + pp].keys())[0]
+ else:
+ spec = ptype
+ axis = list(f[bp + pp + "/" + spec + "/position"].keys())[0]
+ pos = f[bp + pp + "/" + spec + "/position/" + axis]
+ if is_const_component(pos):
+ result[ptype] = pos.attrs["shape"]
+ else:
+ result[ptype] = pos.len()
+ return result
+
+ def _detect_output_fields(self):
+ """Populates ``self.field_list`` with native fields (mesh and particle) on disk.
+
+ Each entry is a tuple of two strings. The first element is the on-disk fluid type or particle type.
+ The second element is the name of the field in yt. This string is later used for accessing the data.
+ Convention suggests that the on-disk fluid type should be "openPMD",
+ the on-disk particle type (for a single species of particles) is "io"
+ or (for multiple species of particles) the particle name on-disk.
+ """
+ f = self.dataset._handle
+ bp = self.dataset.base_path
+ mp = self.dataset.meshes_path
+ pp = self.dataset.particles_path
+
+ mesh_fields = []
+ try:
+ for field in f[bp + mp].keys():
+ try:
+ for axis in f[bp + mp + field].keys():
+ mesh_fields.append(field.replace("_", "-")
+ + "_" + axis)
+ except AttributeError:
+ # This is a h5.Dataset (i.e. no axes)
+ mesh_fields.append(field.replace("_", "-"))
+ except KeyError:
+ # There are no mesh fields
+ pass
+ self.field_list = [("openPMD", str(field)) for field in mesh_fields]
+
+ particle_fields = []
+ try:
+ for species in f[bp + pp].keys():
+ for record in f[bp + pp + species].keys():
+ if is_const_component(f[bp + pp + species + "/" + record]):
+ # Record itself (e.g. particle_mass) is constant
+ particle_fields.append(species.replace("_", "-")
+ + "_" + record.replace("_", "-"))
+ elif "particlePatches" not in record:
+ try:
+ # Create a field for every axis (x,y,z) of every property (position)
+ # of every species (electrons)
+ axes = list(f[bp + pp + species + "/" + record].keys())
+ if str(record) == "position":
+ record = "positionCoarse"
+ for axis in axes:
+ particle_fields.append(species.replace("_", "-")
+ + "_" + record.replace("_", "-")
+ + "_" + axis)
+ except AttributeError:
+ # Record is a dataset, does not have axes (e.g. weighting)
+ particle_fields.append(species.replace("_", "-")
+ + "_" + record.replace("_", "-"))
+ pass
+ else:
+ pass
+ if len(list(f[bp + pp].keys())) > 1:
+ # There is more than one particle species, use the specific names as field types
+ self.field_list.extend(
+ [(str(field).split("_")[0],
+ ("particle_" + "_".join(str(field).split("_")[1:]))) for field in particle_fields])
+ else:
+ # Only one particle species, fall back to "io"
+ self.field_list.extend(
+ [("io",
+ ("particle_" + "_".join(str(field).split("_")[1:]))) for field in particle_fields])
+ except KeyError:
+ # There are no particle fields
+ pass
+
+ def _count_grids(self):
+ """Sets ``self.num_grids`` to be the total number of grids in the simulation.
+
+ The number of grids is determined by their respective memory footprint.
+ """
+ f = self.dataset._handle
+ bp = self.dataset.base_path
+ mp = self.dataset.meshes_path
+ pp = self.dataset.particles_path
+
+ self.meshshapes = {}
+ self.numparts = {}
+
+ self.num_grids = 0
+
+ for mesh in f[bp + mp].keys():
+ if type(f[bp + mp + mesh]) is h5.Group:
+ shape = f[bp + mp + mesh + "/" + list(f[bp + mp + mesh].keys())[0]].shape
+ else:
+ shape = f[bp + mp + mesh].shape
+ spacing = tuple(f[bp + mp + mesh].attrs["gridSpacing"])
+ offset = tuple(f[bp + mp + mesh].attrs["gridGlobalOffset"])
+ unit_si = f[bp + mp + mesh].attrs["gridUnitSI"]
+ self.meshshapes[mesh] = (shape, spacing, offset, unit_si)
+ for species in f[bp + pp].keys():
+ if "particlePatches" in f[bp + pp + "/" + species].keys():
+ for (patch, size) in enumerate(f[bp + pp + "/" + species + "/particlePatches/numParticles"]):
+ self.numparts[species + "#" + str(patch)] = size
+ else:
+ axis = list(f[bp + pp + species + "/position"].keys())[0]
+ if is_const_component(f[bp + pp + species + "/position/" + axis]):
+ self.numparts[species] = f[bp + pp + species + "/position/" + axis].attrs["shape"]
+ else:
+ self.numparts[species] = f[bp + pp + species + "/position/" + axis].len()
+
+ # Limit values per grid by resulting memory footprint
+ self.vpg = int(self.dataset.gridsize / 4) # 4Byte per value (f32)
+
+ # Meshes of the same size do not need separate chunks
+ for (shape, spacing, offset, unit_si) in set(self.meshshapes.values()):
+ self.num_grids += min(shape[0], int(np.ceil(reduce(mul, shape) * self.vpg**-1)))
+
+ # Same goes for particle chunks if they are not inside particlePatches
+ patches = {}
+ no_patches = {}
+ for (k, v) in self.numparts.items():
+ if "#" in k:
+ patches[k] = v
+ else:
+ no_patches[k] = v
+ for size in set(no_patches.values()):
+ self.num_grids += int(np.ceil(size * self.vpg ** -1))
+ for size in patches.values():
+ self.num_grids += int(np.ceil(size * self.vpg ** -1))
+
+ def _parse_index(self):
+ """Fills each grid with appropriate properties (extent, dimensions, ...)
+
+ This calculates the properties of every OpenPMDGrid based on the total number of grids in the simulation.
+ The domain is divided into ``self.num_grids`` (roughly) equally sized chunks along the x-axis.
+ ``grid_levels`` is always equal to 0 since we only have one level of refinement in openPMD.
+
+ Notes
+ -----
+ ``self.grid_dimensions`` is rounded to the nearest integer. Grid edges are calculated from this dimension.
+ Grids with dimensions [0, 0, 0] are particle only. The others do not have any particles affiliated with them.
+ """
+ f = self.dataset._handle
+ bp = self.dataset.base_path
+ pp = self.dataset.particles_path
+
+ self.grid_levels.flat[:] = 0
+ self.grids = np.empty(self.num_grids, dtype="object")
+
+ grid_index_total = 0
+
+ # Mesh grids
+ for mesh in set(self.meshshapes.values()):
+ (shape, spacing, offset, unit_si) = mesh
+ shape = np.asarray(shape)
+ spacing = np.asarray(spacing)
+ offset = np.asarray(offset)
+ # Total dimension of this grid
+ domain_dimension = np.asarray(shape, dtype=np.int32)
+ domain_dimension = np.append(domain_dimension, np.ones(3 - len(domain_dimension)))
+ # Number of grids of this shape
+ num_grids = min(shape[0], int(np.ceil(reduce(mul, shape) * self.vpg ** -1)))
+ gle = offset * unit_si # self.dataset.domain_left_edge
+ gre = domain_dimension[:spacing.size] * unit_si * spacing + gle # self.dataset.domain_right_edge
+ gle = np.append(gle, np.zeros(3 - len(gle)))
+ gre = np.append(gre, np.ones(3 - len(gre)))
+ grid_dim_offset = np.linspace(0, domain_dimension[0], num_grids + 1, dtype=np.int32)
+ grid_edge_offset = grid_dim_offset * np.float(domain_dimension[0]) ** -1 * (gre[0] - gle[0]) + gle[0]
+ mesh_names = []
+ for (mname, mdata) in self.meshshapes.items():
+ if mesh == mdata:
+ mesh_names.append(str(mname))
+ prev = 0
+ for grid in np.arange(num_grids):
+ self.grid_dimensions[grid_index_total] = domain_dimension
+ self.grid_dimensions[grid_index_total][0] = grid_dim_offset[grid + 1] - grid_dim_offset[grid]
+ self.grid_left_edge[grid_index_total] = gle
+ self.grid_left_edge[grid_index_total][0] = grid_edge_offset[grid]
+ self.grid_right_edge[grid_index_total] = gre
+ self.grid_right_edge[grid_index_total][0] = grid_edge_offset[grid + 1]
+ self.grid_particle_count[grid_index_total] = 0
+ self.grids[grid_index_total] = self.grid(grid_index_total, self, 0,
+ fi=prev,
+ fo=self.grid_dimensions[grid_index_total][0],
+ ft=mesh_names)
+ prev += self.grid_dimensions[grid_index_total][0]
+ grid_index_total += 1
+
+ handled_ptypes = []
+
+ # Particle grids
+ for (species, count) in self.numparts.items():
+ if "#" in species:
+ # This is a particlePatch
+ spec = species.split("#")
+ patch = f[bp + pp + "/" + spec[0] + "/particlePatches"]
+ num_grids = int(np.ceil(count * self.vpg ** -1))
+ gle = []
+ for axis in patch["offset"].keys():
+ gle.append(get_component(patch, "offset/" + axis, int(spec[1]), 1)[0])
+ gle = np.asarray(gle)
+ gle = np.append(gle, np.zeros(3 - len(gle)))
+ gre = []
+ for axis in patch["extent"].keys():
+ gre.append(get_component(patch, "extent/" + axis, int(spec[1]), 1)[0])
+ gre = np.asarray(gre)
+ gre = np.append(gre, np.ones(3 - len(gre)))
+ np.add(gle, gre, gre)
+ npo = patch["numParticlesOffset"].value.item(int(spec[1]))
+ particle_count = np.linspace(npo, npo + count, num_grids + 1,
+ dtype=np.int32)
+ particle_names = [str(spec[0])]
+ elif str(species) not in handled_ptypes:
+ num_grids = int(np.ceil(count * self.vpg ** -1))
+ gle = self.dataset.domain_left_edge
+ gre = self.dataset.domain_right_edge
+ particle_count = np.linspace(0, count, num_grids + 1, dtype=np.int32)
+ particle_names = []
+ for (pname, size) in self.numparts.items():
+ if size == count:
+ # Since this is not part of a particlePatch, we can include multiple same-sized ptypes
+ particle_names.append(str(pname))
+ handled_ptypes.append(str(pname))
+ else:
+ # A grid with this exact particle count has already been created
+ continue
+ for grid in np.arange(num_grids):
+ self.grid_dimensions[grid_index_total] = [0, 0, 0] # Counted as mesh-size, thus no dimensional extent
+ self.grid_left_edge[grid_index_total] = gle
+ self.grid_right_edge[grid_index_total] = gre
+ self.grid_particle_count[grid_index_total] = (particle_count[grid + 1] - particle_count[grid]) * len(
+ particle_names)
+ self.grids[grid_index_total] = self.grid(grid_index_total, self, 0,
+ pi=particle_count[grid],
+ po=particle_count[grid + 1] - particle_count[grid],
+ pt=particle_names)
+ grid_index_total += 1
+
+ def _populate_grid_objects(self):
+ """This initializes all grids.
+
+ Additionally, it should set up Children and Parent lists on each grid object.
+ openPMD is not adaptive and thus there are no Children and Parents for any grid.
+ """
+ for i in np.arange(self.num_grids):
+ self.grids[i]._prepare_grid()
+ self.grids[i]._setup_dx()
+ self.max_level = 0
+
+
+class OpenPMDDataset(Dataset):
+ """Contains all the required information of a single iteration of the simulation.
+
+ Notes
+ -----
+ It is assumed that all meshes cover the same region. Their resolution can be different.
+ It is assumed that all particles reside in this same region exclusively.
+ It is assumed that the particle and mesh positions are *absolute* with respect to the simulation origin.
+ """
+ _index_class = OpenPMDHierarchy
+ _field_info_class = OpenPMDFieldInfo
+
+ def __init__(self,
+ filename,
+ dataset_type="openPMD",
+ storage_filename=None,
+ units_override=None,
+ unit_system="mks",
+ **kwargs):
+ self._handle = HDF5FileHandler(filename)
+ self.gridsize = kwargs.pop("open_pmd_virtual_gridsize", 10**9)
+ self._set_paths(self._handle, path.dirname(filename))
+ Dataset.__init__(self,
+ filename,
+ dataset_type,
+ units_override=units_override,
+ unit_system=unit_system)
+ self.storage_filename = storage_filename
+ self.fluid_types += ("openPMD",)
+ particles = tuple(str(c) for c in self._handle[self.base_path + self.particles_path].keys())
+ if len(particles) > 1:
+ # Only use on-disk particle names if there is more than one species
+ self.particle_types = particles
+ mylog.debug("open_pmd - self.particle_types: {}".format(self.particle_types))
+ self.particle_types_raw = self.particle_types
+ self.particle_types = tuple(self.particle_types)
+
+ def _set_paths(self, handle, path):
+ """Parses relevant hdf5-paths out of ``handle``.
+
+ Parameters
+ ----------
+ handle : h5py.File
+ path : str
+ (absolute) filepath for current hdf5 container
+ """
+ iterations = []
+ encoding = handle.attrs["iterationEncoding"].decode()
+ if "groupBased" in encoding:
+ iterations = list(handle["/data"].keys())
+ mylog.info("open_pmd - found {} iterations in file".format(len(iterations)))
+ elif "fileBased" in encoding:
+ itformat = handle.attrs["iterationFormat"].decode().split("/")[-1]
+ regex = "^" + itformat.replace("%T", "[0-9]+") + "$"
+ if path is "":
+ mylog.warning("open_pmd - For file based iterations, please use absolute file paths!")
+ pass
+ for filename in listdir(path):
+ if match(regex, filename):
+ iterations.append(filename)
+ mylog.info("open_pmd - found {} iterations in directory".format(len(iterations)))
+
+ if len(iterations) == 0:
+ mylog.warning("open_pmd - no iterations found!")
+ if "groupBased" in encoding and len(iterations) > 1:
+ mylog.warning("open_pmd - only choose to load one iteration ({})".format(list(handle["/data"].keys())[0]))
+
+ self.base_path = "/data/{}/".format(list(handle["/data"].keys())[0])
+ self.meshes_path = self._handle["/"].attrs["meshesPath"].decode()
+ self.particles_path = self._handle["/"].attrs["particlesPath"].decode()
+
+ def _set_code_unit_attributes(self):
+ """Handle conversion between different physical units and the code units.
+
+ Every dataset in openPMD can have different code <-> physical scaling.
+ The individual factor is obtained by multiplying with "unitSI" reading getting data from disk.
+ """
+ setdefaultattr(self, "length_unit", self.quan(1.0, "m"))
+ setdefaultattr(self, "mass_unit", self.quan(1.0, "kg"))
+ setdefaultattr(self, "time_unit", self.quan(1.0, "s"))
+ setdefaultattr(self, "velocity_unit", self.quan(1.0, "m/s"))
+ setdefaultattr(self, "magnetic_unit", self.quan(1.0, "T"))
+
+ def _parse_parameter_file(self):
+ """Read in metadata describing the overall data on-disk.
+ """
+ f = self._handle
+ bp = self.base_path
+ mp = self.meshes_path
+
+ self.unique_identifier = 0
+ self.parameters = 0
+ self.periodicity = np.zeros(3, dtype=np.bool)
+ self.refine_by = 1
+ self.cosmological_simulation = 0
+
+ try:
+ shapes = {}
+ left_edges = {}
+ right_edges = {}
+ for mesh in f[bp + mp].keys():
+ if type(f[bp + mp + mesh]) is h5.Group:
+ shape = np.asarray(f[bp + mp + mesh + "/" + list(f[bp + mp + mesh].keys())[0]].shape)
+ else:
+ shapes[mesh] = np.asarray(f[bp + mp + mesh].shape)
+ spacing = np.asarray(f[bp + mp + mesh].attrs["gridSpacing"])
+ offset = np.asarray(f[bp + mp + mesh].attrs["gridGlobalOffset"])
+ unit_si = np.asarray(f[bp + mp + mesh].attrs["gridUnitSI"])
+ le = offset * unit_si
+ re = le + shape * unit_si * spacing
+ shapes[mesh] = shape
+ left_edges[mesh] = le
+ right_edges[mesh] = re
+ lowest_dim = np.min([len(i) for i in shapes.values()])
+ shapes = np.asarray([i[:lowest_dim] for i in shapes.values()])
+ left_edges = np.asarray([i[:lowest_dim] for i in left_edges.values()])
+ right_edges = np.asarray([i[:lowest_dim] for i in right_edges.values()])
+ fs = []
+ dle = []
+ dre = []
+ for i in np.arange(lowest_dim):
+ fs.append(np.max(shapes.transpose()[i]))
+ dle.append(np.min(left_edges.transpose()[i]))
+ dre.append(np.min(right_edges.transpose()[i]))
+ self.dimensionality = len(fs)
+ self.domain_dimensions = np.append(fs, np.ones(3 - self.dimensionality))
+ self.domain_left_edge = np.append(dle, np.zeros(3 - len(dle)))
+ self.domain_right_edge = np.append(dre, np.ones(3 - len(dre)))
+ except ValueError:
+ mylog.warning("open_pmd - It seems your data does not contain meshes. Assuming domain extent of 1m^3!")
+ self.dimensionality = 3
+ self.domain_dimensions = np.ones(3, dtype=np.float64)
+ self.domain_left_edge = np.zeros(3, dtype=np.float64)
+ self.domain_right_edge = np.ones(3, dtype=np.float64)
+
+ self.current_time = f[bp].attrs["time"] * f[bp].attrs["timeUnitSI"]
+
+ @classmethod
+ def _is_valid(self, *args, **kwargs):
+ """Checks whether the supplied file can be read by this frontend.
+ """
+ try:
+ f = h5.File(args[0], "r")
+ except (IOError, OSError):
+ return False
+
+ requirements = ["openPMD", "basePath", "meshesPath", "particlesPath"]
+ attrs = list(f["/"].attrs.keys())
+ for i in requirements:
+ if i not in attrs:
+ f.close()
+ return False
+
+ known_versions = [StrictVersion("1.0.0"),
+ StrictVersion("1.0.1")]
+ if StrictVersion(f.attrs["openPMD"].decode()) in known_versions:
+ f.close()
+ return True
+ else:
+ f.close()
+ return False
diff -r 068ad05bda58e31a1684d907398a61ce93fd595c -r 71960f84ae9d3e12d70759e342b2bc581581674c yt/frontends/open_pmd/fields.py
--- /dev/null
+++ b/yt/frontends/open_pmd/fields.py
@@ -0,0 +1,217 @@
+"""
+openPMD-specific fields
+
+
+
+"""
+
+# -----------------------------------------------------------------------------
+# Copyright (c) 2013, yt Development Team.
+# Copyright (c) 2015, Daniel Grassinger (HZDR)
+# Copyright (c) 2016, Fabian Koller (HZDR)
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+# -----------------------------------------------------------------------------
+
+import numpy as np
+
+from yt.fields.field_info_container import FieldInfoContainer
+from yt.fields.magnetic_field import setup_magnetic_field_aliases
+from yt.frontends.open_pmd.misc import \
+ parse_unit_dimension, \
+ is_const_component
+from yt.units.yt_array import YTQuantity
+from yt.utilities.logger import ytLogger as mylog
+from yt.utilities.on_demand_imports import _h5py as h5
+from yt.utilities.physical_constants import \
+ speed_of_light, \
+ mu_0
+
+
+def setup_poynting_vector(self):
+ def _get_poyn(axis):
+ def poynting(field, data):
+ u = mu_0**-1
+ if axis in "x":
+ return u * (data["E_y"] * data["magnetic_field_z"] - data["E_z"] * data["magnetic_field_y"])
+ elif axis in "y":
+ return u * (data["E_z"] * data["magnetic_field_x"] - data["E_x"] * data["magnetic_field_z"])
+ elif axis in "z":
+ return u * (data["E_x"] * data["magnetic_field_y"] - data["E_y"] * data["magnetic_field_x"])
+
+ return poynting
+
+ for ax in "xyz":
+ self.add_field(("openPMD", "poynting_vector_%s" % ax),
+ function=_get_poyn(ax),
+ units="W/m**2")
+
+
+def setup_kinetic_energy(self, ptype):
+ def _kin_en(field, data):
+ p2 = (data[ptype, "particle_momentum_x"] ** 2 +
+ data[ptype, "particle_momentum_y"] ** 2 +
+ data[ptype, "particle_momentum_z"] ** 2)
+ mass = data[ptype, "particle_mass"] * data[ptype, "particle_weighting"]
+ return speed_of_light * np.sqrt(p2 + mass ** 2 * speed_of_light ** 2) - mass * speed_of_light ** 2
+
+ self.add_field((ptype, "particle_kinetic_energy"),
+ function=_kin_en,
+ units="kg*m**2/s**2",
+ particle_type=True)
+
+
+def setup_velocity(self, ptype):
+ def _get_vel(axis):
+ def velocity(field, data):
+ c = speed_of_light
+ momentum = data[ptype, "particle_momentum_{}".format(axis)]
+ mass = data[ptype, "particle_mass"]
+ weighting = data[ptype, "particle_weighting"]
+ return momentum / np.sqrt(
+ (mass * weighting) ** 2 +
+ (momentum ** 2) / (c ** 2)
+ )
+
+ return velocity
+
+ for ax in "xyz":
+ self.add_field((ptype, "particle_velocity_%s" % ax),
+ function=_get_vel(ax),
+ units="m/s",
+ particle_type=True)
+
+
+def setup_absolute_positions(self, ptype):
+ def _abs_pos(axis):
+ def ap(field, data):
+ return np.add(data[ptype, "particle_positionCoarse_{}".format(axis)],
+ data[ptype, "particle_positionOffset_{}".format(axis)])
+
+ return ap
+
+ for ax in "xyz":
+ self.add_field((ptype, "particle_position_%s" % ax),
+ function=_abs_pos(ax),
+ units="m",
+ particle_type=True)
+
+
+class OpenPMDFieldInfo(FieldInfoContainer):
+ """Specifies which fields from the dataset yt should know about.
+
+ ``self.known_other_fields`` and ``self.known_particle_fields`` must be populated.
+ Entries for both of these lists must be tuples of the form
+ ("name", ("units", ["fields", "to", "alias"], "display_name"))
+ These fields will be represented and handled in yt in the way you define them here.
+ The fields defined in both ``self.known_other_fields`` and ``self.known_particle_fields`` will only be added
+ to a dataset (with units, aliases, etc), if they match any entry in the ``OpenPMDHierarchy``'s ``self.field_list``.
+
+ Notes
+ -----
+
+ Contrary to many other frontends, we dynamically obtain the known fields from the simulation output.
+ The openPMD markup is extremely flexible - names, dimensions and the number of individual datasets
+ can (and very likely will) vary.
+
+ openPMD states that names of records and their components are only allowed to contain the
+ characters a-Z,
+ the numbers 0-9
+ and the underscore _
+ (equivalently, the regex \w).
+ Since yt widely uses the underscore in field names, openPMD's underscores (_) are replaced by hyphen (-).
+
+ Derived fields will automatically be set up, if names and units of your known on-disk (or manually derived)
+ fields match the ones in [1].
+
+ References
+ ----------
+ .. http://yt-project.org/docs/dev/analyzing/fields.html
+ .. http://yt-project.org/docs/dev/developing/creating_frontend.html#data-meaning-structures
+ .. https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md
+ .. [1] http://yt-project.org/docs/dev/reference/field_list.html#universal-fields
+ """
+ _mag_fields = []
+
+ def __init__(self, ds, field_list):
+ f = ds._handle
+ bp = ds.base_path
+ mp = ds.meshes_path
+ pp = ds.particles_path
+ fields = f[bp + mp]
+
+ for fname in fields.keys():
+ field = fields[fname]
+ if type(field) is h5.Dataset or is_const_component(field):
+ # Don't consider axes. This appears to be a vector field of single dimensionality
+ ytname = str("_".join([fname.replace("_", "-")]))
+ parsed = parse_unit_dimension(np.asarray(field.attrs["unitDimension"], dtype=np.int))
+ unit = str(YTQuantity(1, parsed).units)
+ aliases = []
+ # Save a list of magnetic fields for aliasing later on
+ # We can not reasonably infer field type/unit by name in openPMD
+ if unit == "T" or unit == "kg/(A*s**2)":
+ self._mag_fields.append(ytname)
+ self.known_other_fields += ((ytname, (unit, aliases, None)),)
+ else:
+ for axis in field.keys():
+ ytname = str("_".join([fname.replace("_", "-"), axis]))
+ parsed = parse_unit_dimension(np.asarray(field.attrs["unitDimension"], dtype=np.int))
+ unit = str(YTQuantity(1, parsed).units)
+ aliases = []
+ # Save a list of magnetic fields for aliasing later on
+ # We can not reasonably infer field type by name in openPMD
+ if unit == "T" or unit == "kg/(A*s**2)":
+ self._mag_fields.append(ytname)
+ self.known_other_fields += ((ytname, (unit, aliases, None)),)
+ for i in self.known_other_fields:
+ mylog.debug("open_pmd - known_other_fields - {}".format(i))
+ particles = f[bp + pp]
+ for species in particles.keys():
+ for record in particles[species].keys():
+ try:
+ pds = particles[species + "/" + record]
+ parsed = parse_unit_dimension(pds.attrs["unitDimension"])
+ unit = str(YTQuantity(1, parsed).units)
+ ytattrib = str(record).replace("_", "-")
+ if ytattrib == "position":
+ # Symbolically rename position to preserve yt's interpretation of the pfield
+ # particle_position is later derived in setup_absolute_positions in the way yt expects it
+ ytattrib = "positionCoarse"
+ if type(pds) is h5.Dataset or is_const_component(pds):
+ name = ["particle", ytattrib]
+ self.known_particle_fields += ((str("_".join(name)), (unit, [], None)),)
+ else:
+ for axis in pds.keys():
+ aliases = []
+ name = ["particle", ytattrib, axis]
+ ytname = str("_".join(name))
+ self.known_particle_fields += ((ytname, (unit, aliases, None)),)
+ except KeyError:
+ mylog.info("open_pmd - {}_{} does not seem to have unitDimension".format(species, record))
+ for i in self.known_particle_fields:
+ mylog.debug("open_pmd - known_particle_fields - {}".format(i))
+ super(OpenPMDFieldInfo, self).__init__(ds, field_list)
+
+ def setup_fluid_fields(self):
+ """Defines which derived mesh fields to create.
+
+ If a field can not be calculated, it will simply be skipped.
+ """
+ # Set up aliases first so the setup for poynting can use them
+ if len(self._mag_fields) > 0:
+ setup_magnetic_field_aliases(self, "openPMD", self._mag_fields)
+ setup_poynting_vector(self)
+
+ def setup_particle_fields(self, ptype):
+ """Defines which derived particle fields to create.
+
+ This will be called for every entry in `OpenPMDDataset``'s ``self.particle_types``.
+ If a field can not be calculated, it will simply be skipped.
+ """
+ setup_absolute_positions(self, ptype)
+ setup_kinetic_energy(self, ptype)
+ setup_velocity(self, ptype)
+ super(OpenPMDFieldInfo, self).setup_particle_fields(ptype)
diff -r 068ad05bda58e31a1684d907398a61ce93fd595c -r 71960f84ae9d3e12d70759e342b2bc581581674c yt/frontends/open_pmd/io.py
--- /dev/null
+++ b/yt/frontends/open_pmd/io.py
@@ -0,0 +1,213 @@
+"""
+openPMD-specific IO functions
+
+
+
+"""
+
+# -----------------------------------------------------------------------------
+# Copyright (c) 2013, yt Development Team.
+# Copyright (c) 2015, Daniel Grassinger (HZDR)
+# Copyright (c) 2016, Fabian Koller (HZDR)
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+# -----------------------------------------------------------------------------
+
+from collections import defaultdict
+
+import numpy as np
+
+from yt.frontends.open_pmd.misc import \
+ is_const_component, \
+ get_component
+from yt.utilities.io_handler import BaseIOHandler
+
+
+class IOHandlerOpenPMDHDF5(BaseIOHandler):
+ _field_dtype = "float32"
+ _dataset_type = "openPMD"
+
+ def __init__(self, ds, *args, **kwargs):
+ self.ds = ds
+ self._handle = ds._handle
+ self.base_path = ds.base_path
+ self.meshes_path = ds.meshes_path
+ self.particles_path = ds.particles_path
+ self._array_fields = {}
+ self._cached_ptype = ""
+
+ def _fill_cache(self, ptype, index=0, offset=None):
+ """Fills the particle position cache for the ``ptype``.
+
+ Parameters
+ ----------
+ ptype : str
+ The on-disk name of the particle species
+ index : int, optional
+ offset : int, optional
+ """
+ if str((ptype, index, offset)) not in self._cached_ptype:
+ self._cached_ptype = str((ptype, index, offset))
+ pds = self._handle[self.base_path + self.particles_path + "/" + ptype]
+ axes = list(pds["position"].keys())
+ if offset is None:
+ if is_const_component(pds["position/" + axes[0]]):
+ offset = pds["position/" + axes[0]].attrs["shape"]
+ else:
+ offset = pds["position/" + axes[0]].len()
+ self.cache = np.empty((3, offset), dtype=np.float64)
+ for i in np.arange(3):
+ ax = "xyz"[i]
+ if ax in axes:
+ np.add(get_component(pds, "position/" + ax, index, offset),
+ get_component(pds, "positionOffset/" + ax, index, offset),
+ self.cache[i])
+ else:
+ # Pad accordingly with zeros to make 1D/2D datasets compatible
+ # These have to be the same shape as the existing axes since that equals the number of particles
+ self.cache[i] = np.zeros(offset)
+
+ def _read_particle_selection(self, chunks, selector, fields):
+ """Reads given particle fields for given particle species masked by a given selection.
+
+ Parameters
+ ----------
+ chunks
+ A list of chunks
+ A chunk is a list of grids
+ selector
+ A region (inside your domain) specifying which parts of the field you want to read
+ See [1] and [2]
+ fields : array_like
+ Tuples (ptype, pfield) representing a field
+
+ Returns
+ -------
+ dict
+ keys are tuples (ptype, pfield) representing a field
+ values are (N,) ndarrays with data from that field
+ """
+ f = self._handle
+ bp = self.base_path
+ pp = self.particles_path
+ ds = f[bp + pp]
+ unions = self.ds.particle_unions
+ chunks = list(chunks) # chunks is a generator
+
+ rv = {}
+ ind = {}
+ particle_count = {}
+ ptf = defaultdict(list) # ParticleTypes&Fields
+ rfm = defaultdict(list) # RequestFieldMapping
+
+ for (ptype, pname) in fields:
+ pfield = (ptype, pname)
+ # Overestimate the size of all pfields so they include all particles, shrink it later
+ particle_count[pfield] = 0
+ if ptype in unions:
+ for pt in unions[ptype]:
+ particle_count[pfield] += self.ds.particle_type_counts[pt]
+ ptf[pt].append(pname)
+ rfm[pt, pname].append(pfield)
+ else:
+ particle_count[pfield] = self.ds.particle_type_counts[ptype]
+ ptf[ptype].append(pname)
+ rfm[pfield].append(pfield)
+ rv[pfield] = np.empty((particle_count[pfield],), dtype=np.float64)
+ ind[pfield] = 0
+
+ for ptype in ptf:
+ for chunk in chunks:
+ for grid in chunk.objs:
+ if str(ptype) == "io":
+ species = list(ds.keys())[0]
+ else:
+ species = ptype
+ if species not in grid.ptypes:
+ continue
+ # read particle coords into cache
+ self._fill_cache(species, grid.pindex, grid.poffset)
+ mask = selector.select_points(self.cache[0], self.cache[1], self.cache[2], 0.0)
+ if mask is None:
+ continue
+ pds = ds[species]
+ for field in ptf[ptype]:
+ component = "/".join(field.split("_")[1:])
+ component = component.replace("positionCoarse", "position")
+ component = component.replace("-", "_")
+ data = get_component(pds, component, grid.pindex, grid.poffset)[mask]
+ for request_field in rfm[(ptype, field)]:
+ rv[request_field][ind[request_field]:ind[request_field] + data.shape[0]] = data
+ ind[request_field] += data.shape[0]
+
+ for field in fields:
+ rv[field] = rv[field][:ind[field]]
+
+ return rv
+
+ def _read_fluid_selection(self, chunks, selector, fields, size):
+ """Reads given fields masked by a given selection.
+
+ Parameters
+ ----------
+ chunks
+ A list of chunks
+ A chunk is a list of grids
+ selector
+ A region (inside your domain) specifying which parts of the field you want to read
+ See [1] and [2]
+ fields : array_like
+ Tuples (fname, ftype) representing a field
+ size : int
+ Size of the data to read
+
+ Returns
+ -------
+ dict
+ keys are tuples (ftype, fname) representing a field
+ values are flat (``size``,) ndarrays with data from that field
+ """
+ f = self._handle
+ bp = self.base_path
+ mp = self.meshes_path
+ ds = f[bp + mp]
+ chunks = list(chunks)
+
+ rv = {}
+ ind = {}
+
+ if selector.__class__.__name__ == "GridSelector":
+ if not (len(chunks) == len(chunks[0].objs) == 1):
+ raise RuntimeError
+
+ if size is None:
+ size = sum((g.count(selector) for chunk in chunks
+ for g in chunk.objs))
+ for field in fields:
+ rv[field] = np.empty(size, dtype=np.float64)
+ ind[field] = 0
+
+ for (ftype, fname) in fields:
+ field = (ftype, fname)
+ for chunk in chunks:
+ for grid in chunk.objs:
+ component = fname.replace("_", "/").replace("-", "_")
+ if component.split("/")[0] not in grid.ftypes:
+ continue
+ mask = grid._get_selector_mask(selector)
+ if mask is None:
+ continue
+ data = get_component(ds, component, grid.findex, grid.foffset)
+ # The following is a modified AMRGridPatch.select(...)
+ data.shape = mask.shape # Workaround - casts a 2D (x,y) array to 3D (x,y,1)
+ count = grid.count(selector)
+ rv[field][ind[field]:ind[field] + count] = data[mask]
+ ind[field] += count
+
+ for field in fields:
+ rv[field] = rv[field][:ind[field]]
+ rv[field].flatten()
+
+ return rv
diff -r 068ad05bda58e31a1684d907398a61ce93fd595c -r 71960f84ae9d3e12d70759e342b2bc581581674c yt/frontends/open_pmd/misc.py
--- /dev/null
+++ b/yt/frontends/open_pmd/misc.py
@@ -0,0 +1,125 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) 2016, Fabian Koller (HZDR)
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+# -----------------------------------------------------------------------------
+
+import numpy as np
+
+from yt.utilities.logger import ytLogger as mylog
+
+
+def parse_unit_dimension(unit_dimension):
+ """Transforms an openPMD unitDimension into a string.
+
+ Parameters
+ ----------
+ unit_dimension : array_like
+ integer array of length 7 with one entry for the dimensional component of every SI unit
+
+ [0] length L,
+ [1] mass M,
+ [2] time T,
+ [3] electric current I,
+ [4] thermodynamic temperature theta,
+ [5] amount of substance N,
+ [6] luminous intensity J
+
+ References
+ ----------
+ .. https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md#unit-systems-and-dimensionality
+
+ Returns
+ -------
+ str
+
+ Examples
+ --------
+ >>> velocity = [1., 0., -1., 0., 0., 0., 0.]
+ >>> print parse_unit_dimension(velocity)
+ 'm**1*s**-1'
+
+ >>> magnetic_field = [0., 1., -2., -1., 0., 0., 0.]
+ >>> print parse_unit_dimension(magnetic_field)
+ 'kg**1*s**-2*A**-1'
+ """
+ if len(unit_dimension) is not 7:
+ mylog.error("open_pmd - SI must have 7 base dimensions!")
+ unit_dimension = np.asarray(unit_dimension, dtype=np.int)
+ dim = []
+ si = ["m",
+ "kg",
+ "s",
+ "A",
+ "C",
+ "mol",
+ "cd"]
+ for i in np.arange(7):
+ if unit_dimension[i] != 0:
+ dim.append("{}**{}".format(si[i], unit_dimension[i]))
+ return "*".join(dim)
+
+
+def is_const_component(record_component):
+ """Determines whether a group or dataset in the HDF5 file is constant.
+
+ Parameters
+ ----------
+ record_component : h5py.Group or h5py.Dataset
+
+ Returns
+ -------
+ bool
+ True if constant, False otherwise
+
+ References
+ ----------
+ .. https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md,
+ section 'Constant Record Components'
+ """
+ return "value" in record_component.attrs.keys()
+
+
+def get_component(group, component_name, index=0, offset=None):
+ """Grabs a dataset component from a group as a whole or sliced.
+
+ Parameters
+ ----------
+ group : h5py.Group
+ component_name : str
+ relative path of the component in the group
+ index : int, optional
+ first entry along the first axis to read
+ offset : int, optional
+ number of entries to read
+ if not supplied, every entry after index is returned
+
+ Notes
+ -----
+ This scales every entry of the component with the respective "unitSI".
+
+ Returns
+ -------
+ ndarray
+ (N,) 1D in case of particle data
+ (O,P,Q) 1D/2D/3D in case of mesh data
+ """
+ record_component = group[component_name]
+ unit_si = record_component.attrs["unitSI"]
+ if is_const_component(record_component):
+ shape = np.asarray(record_component.attrs["shape"])
+ if offset is None:
+ shape[0] -= index
+ else:
+ shape[0] = offset
+ # component is constant, craft an array by hand
+ # mylog.debug("open_pmd - get_component: {}/{} [const {}]".format(group.name, component_name, shape))
+ return np.full(shape, record_component.attrs["value"] * unit_si)
+ else:
+ if offset is not None:
+ offset += index
+ # component is a dataset, return it (possibly masked)
+ # mylog.debug("open_pmd - get_component: {}/{}[{}:{}]".format(group.name, component_name, index, offset))
+ return np.multiply(record_component[index:offset], unit_si)
diff -r 068ad05bda58e31a1684d907398a61ce93fd595c -r 71960f84ae9d3e12d70759e342b2bc581581674c yt/frontends/open_pmd/tests/test_outputs.py
--- /dev/null
+++ b/yt/frontends/open_pmd/tests/test_outputs.py
@@ -0,0 +1,137 @@
+"""
+openPMD frontend tests
+
+
+
+"""
+
+# -----------------------------------------------------------------------------
+# Copyright (c) 2016, Fabian Koller (HZDR).
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+# -----------------------------------------------------------------------------
+
+from yt.frontends.open_pmd.data_structures import \
+ OpenPMDDataset
+from yt.testing import \
+ assert_almost_equal, \
+ assert_equal, \
+ assert_array_equal, \
+ requires_file
+from yt.utilities.answer_testing.framework import \
+ data_dir_load
+
+import numpy as np
+
+twoD = "example-2d/hdf5/data00000100.h5"
+threeD = "example-3d/hdf5/data00000100.h5"
+
+
+ at requires_file(threeD)
+def test_3d_out():
+ ds = data_dir_load(threeD)
+ field_list = [('all', 'particle_charge'),
+ ('all', 'particle_mass'),
+ ('all', 'particle_momentum_x'),
+ ('all', 'particle_momentum_y'),
+ ('all', 'particle_momentum_z'),
+ ('all', 'particle_positionCoarse_x'),
+ ('all', 'particle_positionCoarse_y'),
+ ('all', 'particle_positionCoarse_z'),
+ ('all', 'particle_positionOffset_x'),
+ ('all', 'particle_positionOffset_y'),
+ ('all', 'particle_positionOffset_z'),
+ ('all', 'particle_weighting'),
+ ('io', 'particle_charge'),
+ ('io', 'particle_mass'),
+ ('io', 'particle_momentum_x'),
+ ('io', 'particle_momentum_y'),
+ ('io', 'particle_momentum_z'),
+ ('io', 'particle_positionCoarse_x'),
+ ('io', 'particle_positionCoarse_y'),
+ ('io', 'particle_positionCoarse_z'),
+ ('io', 'particle_positionOffset_x'),
+ ('io', 'particle_positionOffset_y'),
+ ('io', 'particle_positionOffset_z'),
+ ('io', 'particle_weighting'),
+ ('openPMD', 'E_x'),
+ ('openPMD', 'E_y'),
+ ('openPMD', 'E_z'),
+ ('openPMD', 'rho')]
+ domain_dimensions = [26, 26, 201] * np.ones_like(ds.domain_dimensions)
+ domain_width = [2.08e-05, 2.08e-05, 2.01e-05] * np.ones_like(ds.domain_left_edge)
+
+ assert isinstance(ds, OpenPMDDataset)
+ yield assert_equal, str(ds), "data00000100.h5"
+ yield assert_equal, ds.dimensionality, 3
+ yield assert_equal, ds.particle_types_raw, ('io',)
+ assert "all" in ds.particle_unions
+ yield assert_array_equal, ds.field_list, field_list
+ yield assert_array_equal, ds.domain_dimensions, domain_dimensions
+ yield assert_almost_equal, ds.current_time, 3.28471214521e-14 * np.ones_like(ds.current_time)
+ yield assert_almost_equal, ds.domain_right_edge - ds.domain_left_edge, domain_width
+
+
+ at requires_file(twoD)
+def test_2d_out():
+ ds = data_dir_load(twoD)
+ field_list = [('Hydrogen1+', 'particle_charge'),
+ ('Hydrogen1+', 'particle_mass'),
+ ('Hydrogen1+', 'particle_momentum_x'),
+ ('Hydrogen1+', 'particle_momentum_y'),
+ ('Hydrogen1+', 'particle_momentum_z'),
+ ('Hydrogen1+', 'particle_positionCoarse_x'),
+ ('Hydrogen1+', 'particle_positionCoarse_y'),
+ ('Hydrogen1+', 'particle_positionCoarse_z'),
+ ('Hydrogen1+', 'particle_positionOffset_x'),
+ ('Hydrogen1+', 'particle_positionOffset_y'),
+ ('Hydrogen1+', 'particle_positionOffset_z'),
+ ('Hydrogen1+', 'particle_weighting'),
+ ('all', 'particle_charge'),
+ ('all', 'particle_mass'),
+ ('all', 'particle_momentum_x'),
+ ('all', 'particle_momentum_y'),
+ ('all', 'particle_momentum_z'),
+ ('all', 'particle_positionCoarse_x'),
+ ('all', 'particle_positionCoarse_y'),
+ ('all', 'particle_positionCoarse_z'),
+ ('all', 'particle_positionOffset_x'),
+ ('all', 'particle_positionOffset_y'),
+ ('all', 'particle_positionOffset_z'),
+ ('all', 'particle_weighting'),
+ ('electrons', 'particle_charge'),
+ ('electrons', 'particle_mass'),
+ ('electrons', 'particle_momentum_x'),
+ ('electrons', 'particle_momentum_y'),
+ ('electrons', 'particle_momentum_z'),
+ ('electrons', 'particle_positionCoarse_x'),
+ ('electrons', 'particle_positionCoarse_y'),
+ ('electrons', 'particle_positionCoarse_z'),
+ ('electrons', 'particle_positionOffset_x'),
+ ('electrons', 'particle_positionOffset_y'),
+ ('electrons', 'particle_positionOffset_z'),
+ ('electrons', 'particle_weighting'),
+ ('openPMD', 'B_x'),
+ ('openPMD', 'B_y'),
+ ('openPMD', 'B_z'),
+ ('openPMD', 'E_x'),
+ ('openPMD', 'E_y'),
+ ('openPMD', 'E_z'),
+ ('openPMD', 'J_x'),
+ ('openPMD', 'J_y'),
+ ('openPMD', 'J_z'),
+ ('openPMD', 'rho')]
+ domain_dimensions = [51, 201, 1] * np.ones_like(ds.domain_dimensions)
+ domain_width = [3.06e-05, 2.01e-05, 1e+0] * np.ones_like(ds.domain_left_edge)
+
+ assert isinstance(ds, OpenPMDDataset)
+ yield assert_equal, str(ds), "data00000100.h5"
+ yield assert_equal, ds.dimensionality, 2
+ yield assert_equal, ds.particle_types_raw, ('Hydrogen1+', 'electrons')
+ assert "all" in ds.particle_unions
+ yield assert_array_equal, ds.field_list, field_list
+ yield assert_array_equal, ds.domain_dimensions, domain_dimensions
+ yield assert_almost_equal, ds.current_time, 3.29025596712e-14 * np.ones_like(ds.current_time)
+ yield assert_almost_equal, ds.domain_right_edge - ds.domain_left_edge, domain_width
diff -r 068ad05bda58e31a1684d907398a61ce93fd595c -r 71960f84ae9d3e12d70759e342b2bc581581674c yt/utilities/on_demand_imports.py
--- a/yt/utilities/on_demand_imports.py
+++ b/yt/utilities/on_demand_imports.py
@@ -230,6 +230,19 @@
self._Group = Group
return self._Group
+ _Dataset = None
+ @property
+ def Dataset(self):
+ if self._err:
+ raise self._err
+ if self._Dataset is None:
+ try:
+ from h5py import Dataset
+ except ImportError:
+ Dataset = NotAModule(self._name)
+ self._Dataset = Dataset
+ return self._Dataset
+
___version__ = None
@property
def __version__(self):
Repository URL: https://bitbucket.org/yt_analysis/yt/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the yt-svn
mailing list