[yt-svn] commit/yt: 2 new changesets
commits-noreply at bitbucket.org
commits-noreply at bitbucket.org
Mon Mar 24 10:20:06 PDT 2014
2 new commits in yt:
https://bitbucket.org/yt_analysis/yt/commits/ad194dc3a326/
Changeset: ad194dc3a326
Branch: yt-3.0
User: BW Keller
Date: 2014-03-24 18:01:55
Summary: Added support for tipsy auxiliary files. Auxiliary fields
are automatically detected and are read in when requested.
Affected #: 1 file
diff -r 1e9d46dad91a659cd47c18bea9d938e6a13b0c59 -r ad194dc3a326f9e77d6deaef3b1d5064458cde0f yt/frontends/sph/io.py
--- a/yt/frontends/sph/io.py
+++ b/yt/frontends/sph/io.py
@@ -14,6 +14,7 @@
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
+import glob
import h5py
import numpy as np
from .definitions import gadget_ptypes, ghdf5_ptypes
@@ -357,6 +358,7 @@
"DarkMatter",
"Stars" )
+ _aux_fields = []
_fields = ( ("Gas", "Mass"),
("Gas", "Coordinates"),
("Gas", "Velocities"),
@@ -382,7 +384,50 @@
def _read_fluid_selection(self, chunks, selector, fields, size):
raise NotImplementedError
- def _fill_fields(self, fields, vals, mask):
+ def _read_aux_fields(self, field, mask, data_file):
+ """
+ Read in auxiliary files from gasoline/pkdgrav
+ """
+ filename = data_file.filename+'.'+field
+ dtype = None
+ # We need to do some fairly ugly detection to see what format the auxiliary
+ # files are in. They can be either ascii or binary, and the binary files can be
+ # either floats, ints, or doubles. We're going to use a try-catch cascade to
+ # determine the format.
+ try:#ASCII
+ auxdata = np.genfromtxt(filename, skip_header=1)
+ if auxdata.size != np.sum(data_file.total_particles.values()):
+ print "Error reading auxiliary tipsy file"
+ raise RuntimeError
+ except ValueError:#binary/xdr
+ f = open(filename, 'rb')
+ l = struct.unpack(data_file.pf.endian+"i", f.read(4))[0]
+ if l != np.sum(data_file.total_particles.values()):
+ print "Error reading auxiliary tipsy file"
+ raise RuntimeError
+ dtype = 'd'
+ if field in ('iord', 'igasorder', 'grp'):#These fields are integers
+ dtype = 'i'
+ try:# If we try loading doubles by default, we can catch an exception and try floats next
+ auxdata = np.array(struct.unpack(data_file.pf.endian+(l*dtype), f.read()))
+ except struct.error:
+ f.seek(4)
+ dtype = 'f'
+ try:
+ auxdata = np.array(struct.unpack(data_file.pf.endian+(l*dtype), f.read()))
+ except struct.error: # None of the binary attempts to read succeeded
+ print "Error reading auxiliary tipsy file"
+ raise RuntimeError
+
+ # Use the mask to slice out the appropriate particle type data
+ if mask.size == data_file.total_particles['DarkMatter']:
+ return auxdata[:data_file.total_particles['DarkMatter']]
+ elif mask.size == data_file.total_particles['Gas']:
+ return auxdata[data_file.total_particles['DarkMatter']:data_file.total_particles['Stars']]
+ else:
+ return auxdata[data_file.total_particles['Stars']:]
+
+ def _fill_fields(self, fields, vals, mask, data_file):
if mask is None:
size = 0
else:
@@ -390,7 +435,9 @@
rv = {}
for field in fields:
mylog.debug("Allocating %s values for %s", size, field)
- if field in self._vector_fields:
+ if field in self._aux_fields: #Read each of the auxiliary fields
+ rv[field] = self._read_aux_fields(field, mask, data_file)
+ elif field in self._vector_fields:
rv[field] = np.empty((size, 3), dtype="float64")
if size == 0: continue
rv[field][:,0] = vals[field]['x'][mask]
@@ -408,6 +455,7 @@
self.domain_right_edge[i] - eps)
return rv
+
def _read_particle_coords(self, chunks, ptf):
data_files = set([])
for chunk in chunks:
@@ -443,7 +491,7 @@
p["Coordinates"]['y'].astype("float64"),
p["Coordinates"]['z'].astype("float64"))
if mask is None: continue
- tf = self._fill_fields(field_list, p, mask)
+ tf = self._fill_fields(field_list, p, mask, data_file)
for field in field_list:
yield (ptype, field), tf.pop(field)
f.close()
@@ -510,6 +558,8 @@
pds = {}
field_list = []
tp = data_file.total_particles
+ aux_filenames = glob.glob(data_file.filename+'.*') # Find out which auxiliaries we have
+ self._aux_fields = [f[1+len(data_file.filename):] for f in aux_filenames]
for ptype, field in self._fields:
pfields = []
if tp[ptype] == 0: continue
@@ -523,6 +573,12 @@
field_list.append((ptype, field))
for ptype in pds:
self._pdtypes[ptype] = np.dtype(pds[ptype])
+ if any(["Gas"==f[0] for f in field_list]): #Add the auxiliary fields to each ptype we have
+ field_list += [("Gas",a) for a in self._aux_fields]
+ if any(["DarkMatter"==f[0] for f in field_list]):
+ field_list += [("DarkMatter",a) for a in self._aux_fields]
+ if any(["Stars"==f[0] for f in field_list]):
+ field_list += [("Stars",a) for a in self._aux_fields]
self._field_list = field_list
return self._field_list
https://bitbucket.org/yt_analysis/yt/commits/8998ed219450/
Changeset: 8998ed219450
Branch: yt-3.0
User: MatthewTurk
Date: 2014-03-24 18:19:57
Summary: Merged in bwkeller/yt/yt-3.0 (pull request #750)
Added support for tipsy auxiliary files. Auxiliary fields
Affected #: 1 file
diff -r d25600daca2f3d21df48c4b97abdb782eb673169 -r 8998ed2194500cd405198102eb90239d20e73b75 yt/frontends/sph/io.py
--- a/yt/frontends/sph/io.py
+++ b/yt/frontends/sph/io.py
@@ -14,6 +14,7 @@
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
+import glob
import h5py
import numpy as np
from .definitions import gadget_ptypes, ghdf5_ptypes
@@ -357,6 +358,7 @@
"DarkMatter",
"Stars" )
+ _aux_fields = []
_fields = ( ("Gas", "Mass"),
("Gas", "Coordinates"),
("Gas", "Velocities"),
@@ -382,7 +384,50 @@
def _read_fluid_selection(self, chunks, selector, fields, size):
raise NotImplementedError
- def _fill_fields(self, fields, vals, mask):
+ def _read_aux_fields(self, field, mask, data_file):
+ """
+ Read in auxiliary files from gasoline/pkdgrav
+ """
+ filename = data_file.filename+'.'+field
+ dtype = None
+ # We need to do some fairly ugly detection to see what format the auxiliary
+ # files are in. They can be either ascii or binary, and the binary files can be
+ # either floats, ints, or doubles. We're going to use a try-catch cascade to
+ # determine the format.
+ try:#ASCII
+ auxdata = np.genfromtxt(filename, skip_header=1)
+ if auxdata.size != np.sum(data_file.total_particles.values()):
+ print "Error reading auxiliary tipsy file"
+ raise RuntimeError
+ except ValueError:#binary/xdr
+ f = open(filename, 'rb')
+ l = struct.unpack(data_file.pf.endian+"i", f.read(4))[0]
+ if l != np.sum(data_file.total_particles.values()):
+ print "Error reading auxiliary tipsy file"
+ raise RuntimeError
+ dtype = 'd'
+ if field in ('iord', 'igasorder', 'grp'):#These fields are integers
+ dtype = 'i'
+ try:# If we try loading doubles by default, we can catch an exception and try floats next
+ auxdata = np.array(struct.unpack(data_file.pf.endian+(l*dtype), f.read()))
+ except struct.error:
+ f.seek(4)
+ dtype = 'f'
+ try:
+ auxdata = np.array(struct.unpack(data_file.pf.endian+(l*dtype), f.read()))
+ except struct.error: # None of the binary attempts to read succeeded
+ print "Error reading auxiliary tipsy file"
+ raise RuntimeError
+
+ # Use the mask to slice out the appropriate particle type data
+ if mask.size == data_file.total_particles['DarkMatter']:
+ return auxdata[:data_file.total_particles['DarkMatter']]
+ elif mask.size == data_file.total_particles['Gas']:
+ return auxdata[data_file.total_particles['DarkMatter']:data_file.total_particles['Stars']]
+ else:
+ return auxdata[data_file.total_particles['Stars']:]
+
+ def _fill_fields(self, fields, vals, mask, data_file):
if mask is None:
size = 0
else:
@@ -390,7 +435,9 @@
rv = {}
for field in fields:
mylog.debug("Allocating %s values for %s", size, field)
- if field in self._vector_fields:
+ if field in self._aux_fields: #Read each of the auxiliary fields
+ rv[field] = self._read_aux_fields(field, mask, data_file)
+ elif field in self._vector_fields:
rv[field] = np.empty((size, 3), dtype="float64")
if size == 0: continue
rv[field][:,0] = vals[field]['x'][mask]
@@ -408,6 +455,7 @@
self.domain_right_edge[i] - eps)
return rv
+
def _read_particle_coords(self, chunks, ptf):
data_files = set([])
for chunk in chunks:
@@ -443,7 +491,7 @@
p["Coordinates"]['y'].astype("float64"),
p["Coordinates"]['z'].astype("float64"))
if mask is None: continue
- tf = self._fill_fields(field_list, p, mask)
+ tf = self._fill_fields(field_list, p, mask, data_file)
for field in field_list:
yield (ptype, field), tf.pop(field)
f.close()
@@ -510,6 +558,8 @@
pds = {}
field_list = []
tp = data_file.total_particles
+ aux_filenames = glob.glob(data_file.filename+'.*') # Find out which auxiliaries we have
+ self._aux_fields = [f[1+len(data_file.filename):] for f in aux_filenames]
for ptype, field in self._fields:
pfields = []
if tp[ptype] == 0: continue
@@ -523,6 +573,12 @@
field_list.append((ptype, field))
for ptype in pds:
self._pdtypes[ptype] = np.dtype(pds[ptype])
+ if any(["Gas"==f[0] for f in field_list]): #Add the auxiliary fields to each ptype we have
+ field_list += [("Gas",a) for a in self._aux_fields]
+ if any(["DarkMatter"==f[0] for f in field_list]):
+ field_list += [("DarkMatter",a) for a in self._aux_fields]
+ if any(["Stars"==f[0] for f in field_list]):
+ field_list += [("Stars",a) for a in self._aux_fields]
self._field_list = field_list
return self._field_list
Repository URL: https://bitbucket.org/yt_analysis/yt/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the yt-svn
mailing list