[yt-svn] commit/yt: xarthisius: Merged in brittonsmith/yt (pull request #2360)
commits-noreply at bitbucket.org
commits-noreply at bitbucket.org
Sat Sep 24 14:15:20 PDT 2016
1 new commit in yt:
https://bitbucket.org/yt_analysis/yt/commits/6d05ffb05f3f/
Changeset: 6d05ffb05f3f
Branch: yt
User: xarthisius
Date: 2016-09-24 21:14:53+00:00
Summary: Merged in brittonsmith/yt (pull request #2360)
Refactor HaloCatalog saving
Affected #: 6 files
diff -r af34b428166a02df3f069fa33f5df3ca8cf58545 -r 6d05ffb05f3fbe2295f0fcce5db0b82837b03401 yt/analysis_modules/halo_analysis/halo_catalog.py
--- a/yt/analysis_modules/halo_analysis/halo_catalog.py
+++ b/yt/analysis_modules/halo_analysis/halo_catalog.py
@@ -13,10 +13,11 @@
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
-from yt.utilities.on_demand_imports import _h5py as h5py
import numpy as np
import os
+from yt.frontends.ytdata.utilities import \
+ save_as_dataset
from yt.funcs import \
ensure_dir, \
mylog
@@ -431,13 +432,16 @@
key, quantity = action
if quantity in self.halos_ds.field_info:
new_halo.quantities[key] = \
- self.data_source[quantity][int(i)].in_cgs()
+ self.data_source[quantity][int(i)]
elif callable(quantity):
new_halo.quantities[key] = quantity(new_halo)
else:
- raise RuntimeError("Action must be a callback, filter, or quantity.")
+ raise RuntimeError(
+ "Action must be a callback, filter, or quantity.")
if halo_filter:
+ for quantity in new_halo.quantities.values():
+ quantity.convert_to_base()
self.catalog.append(new_halo.quantities)
if save_halos and halo_filter:
@@ -458,27 +462,20 @@
mylog.info("Saving halo catalog (%d halos) to %s." %
(n_halos, os.path.join(self.output_dir,
self.output_prefix)))
- out_file = h5py.File(filename, 'w')
- for attr in ["current_redshift", "current_time",
- "domain_dimensions",
- "cosmological_simulation", "omega_lambda",
- "omega_matter", "hubble_constant"]:
- out_file.attrs[attr] = getattr(self.halos_ds, attr)
- for attr in ["domain_left_edge", "domain_right_edge"]:
- out_file.attrs[attr] = getattr(self.halos_ds, attr).in_cgs()
- out_file.attrs["data_type"] = "halo_catalog"
- out_file.attrs["num_halos"] = n_halos
+ extra_attrs = {"data_type": "halo_catalog",
+ "num_halos": n_halos}
+ data = {}
+ ftypes = {}
if n_halos > 0:
- field_data = np.empty(n_halos)
for key in self.quantities:
- units = ""
- if hasattr(self.catalog[0][key], "units"):
- units = str(self.catalog[0][key].units)
- for i in range(n_halos):
- field_data[i] = self.catalog[i][key]
- dataset = out_file.create_dataset(str(key), data=field_data)
- dataset.attrs["units"] = units
- out_file.close()
+ # This sets each field to be saved in the root hdf5 group,
+ # as per the HaloCatalog format.
+ ftypes[key] = "."
+ data[key] = self.halos_ds.arr(
+ [halo[key] for halo in self.catalog])
+
+ save_as_dataset(self.halos_ds, filename, data,
+ field_types=ftypes, extra_attrs=extra_attrs)
def add_default_quantities(self, field_type='halos'):
self.add_quantity("particle_identifier", field_type=field_type,prepend=True)
diff -r af34b428166a02df3f069fa33f5df3ca8cf58545 -r 6d05ffb05f3fbe2295f0fcce5db0b82837b03401 yt/analysis_modules/halo_analysis/tests/test_halo_finders.py
--- a/yt/analysis_modules/halo_analysis/tests/test_halo_finders.py
+++ b/yt/analysis_modules/halo_analysis/tests/test_halo_finders.py
@@ -2,6 +2,8 @@
import sys
from yt.convenience import load
+from yt.frontends.halo_catalog.data_structures import \
+ HaloCatalogDataset
from yt.utilities.answer_testing.framework import \
FieldValuesTest, \
requires_ds
@@ -30,6 +32,7 @@
"halo_catalogs", method,
"%s.0.h5" % method)
ds = load(fn)
+ assert isinstance(ds, HaloCatalogDataset)
for field in _fields:
yield FieldValuesTest(ds, field, particle_type=True,
decimals=decimals[method])
diff -r af34b428166a02df3f069fa33f5df3ca8cf58545 -r 6d05ffb05f3fbe2295f0fcce5db0b82837b03401 yt/frontends/halo_catalog/data_structures.py
--- a/yt/frontends/halo_catalog/data_structures.py
+++ b/yt/frontends/halo_catalog/data_structures.py
@@ -24,6 +24,7 @@
HaloCatalogFieldInfo
from yt.funcs import \
+ parse_h5_attr, \
setdefaultattr
from yt.geometry.particle_geometry_handler import \
ParticleIndex
@@ -34,7 +35,7 @@
class HaloCatalogHDF5File(ParticleFile):
def __init__(self, ds, io, filename, file_id):
with h5py.File(filename, "r") as f:
- self.header = dict((field, f.attrs[field]) \
+ self.header = dict((field, parse_h5_attr(f, field)) \
for field in f.attrs.keys())
super(HaloCatalogHDF5File, self).__init__(ds, io, filename, file_id)
@@ -56,7 +57,7 @@
def _parse_parameter_file(self):
with h5py.File(self.parameter_filename, "r") as f:
- hvals = dict((key, f.attrs[key]) for key in f.attrs.keys())
+ hvals = dict((key, parse_h5_attr(f, key)) for key in f.attrs.keys())
self.dimensionality = 3
self.refine_by = 2
self.unique_identifier = \
@@ -88,6 +89,6 @@
if not args[0].endswith(".h5"): return False
with h5py.File(args[0], "r") as f:
if "data_type" in f.attrs and \
- f.attrs["data_type"] == "halo_catalog":
+ parse_h5_attr(f, "data_type") == "halo_catalog":
return True
return False
diff -r af34b428166a02df3f069fa33f5df3ca8cf58545 -r 6d05ffb05f3fbe2295f0fcce5db0b82837b03401 yt/frontends/halo_catalog/io.py
--- a/yt/frontends/halo_catalog/io.py
+++ b/yt/frontends/halo_catalog/io.py
@@ -78,6 +78,7 @@
mylog.debug("Initializing index % 5i (% 7i particles)",
data_file.file_id, pcount)
ind = 0
+ if pcount == 0: return None
with h5py.File(data_file.filename, "r") as f:
if not f.keys(): return None
pos = np.empty((pcount, 3), dtype="float64")
diff -r af34b428166a02df3f069fa33f5df3ca8cf58545 -r 6d05ffb05f3fbe2295f0fcce5db0b82837b03401 yt/frontends/ytdata/data_structures.py
--- a/yt/frontends/ytdata/data_structures.py
+++ b/yt/frontends/ytdata/data_structures.py
@@ -41,7 +41,8 @@
from yt.extern.six import \
string_types
from yt.funcs import \
- is_root
+ is_root, \
+ parse_h5_attr
from yt.geometry.grid_geometry_handler import \
GridIndex
from yt.geometry.particle_geometry_handler import \
@@ -67,13 +68,6 @@
"covering_grid",
"smoothed_covering_grid"]
-def parse_h5_attr(f, attr):
- val = f.attrs.get(attr, None)
- if isinstance(val, bytes):
- return val.decode('utf8')
- else:
- return val
-
class YTDataset(Dataset):
"""Base dataset class for all ytdata datasets."""
def _parse_parameter_file(self):
diff -r af34b428166a02df3f069fa33f5df3ca8cf58545 -r 6d05ffb05f3fbe2295f0fcce5db0b82837b03401 yt/funcs.py
--- a/yt/funcs.py
+++ b/yt/funcs.py
@@ -1024,3 +1024,14 @@
if not hasattr(obj, name):
setattr(obj, name, value)
return getattr(obj, name)
+
+def parse_h5_attr(f, attr):
+ """A Python3-safe function for getting hdf5 attributes.
+
+ If an attribute is supposed to be a string, this will return it as such.
+ """
+ val = f.attrs.get(attr, None)
+ if isinstance(val, bytes):
+ return val.decode('utf8')
+ else:
+ return val
Repository URL: https://bitbucket.org/yt_analysis/yt/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the yt-svn
mailing list