[yt-svn] commit/yt: 4 new changesets
commits-noreply at bitbucket.org
commits-noreply at bitbucket.org
Mon Apr 14 06:26:24 PDT 2014
4 new commits in yt:
https://bitbucket.org/yt_analysis/yt/commits/d95a6b0a2066/
Changeset: d95a6b0a2066
Branch: yt-3.0
User: BW Keller
Date: 2014-04-12 01:02:06
Summary: Fixed the ultra-slow tipsy _validate_header method.
Affected #: 1 file
diff -r cc6b873a4471e049b6fef33146e92615c72f1243 -r d95a6b0a206691d9314f7f69edb84a7e50a90416 yt/frontends/sph/data_structures.py
--- a/yt/frontends/sph/data_structures.py
+++ b/yt/frontends/sph/data_structures.py
@@ -388,7 +388,7 @@
self.over_refine_factor = over_refine_factor
if field_dtypes is None:
field_dtypes = {}
- success, self.endian = self._validate_header(filename, field_dtypes)
+ success, self.endian = self._validate_header(filename)
if not success:
print "SOMETHING HAS GONE WRONG. NBODIES != SUM PARTICLES."
print "%s != (%s == %s + %s + %s)" % (
@@ -522,7 +522,7 @@
self.time_unit = 1.0 / np.sqrt(G * density_unit)
@staticmethod
- def _validate_header(filename, field_dtypes):
+ def _validate_header(filename):
'''
This method automatically detects whether the tipsy file is big/little endian
and is not corrupt/invalid. It returns a tuple of (Valid, endianswap) where
@@ -534,7 +534,9 @@
except:
return False, 1
try:
- fs = len(f.read())
+ f.seek(0, os.SEEK_END)
+ fs = f.tell()
+ f.seek(0, os.SEEK_SET)
except IOError:
return False, 1
f.seek(0)
@@ -546,16 +548,11 @@
endianswap = ">"
f.seek(0)
t, n, ndim, ng, nd, ns = struct.unpack(">diiiii", f.read(28))
- # Now we construct the sizes of each of the particles.
- dtypes = IOHandlerTipsyBinary._compute_dtypes(field_dtypes, endianswap)
- #Catch for 4 byte padding
- gas_size = dtypes["Gas"].itemsize
- dm_size = dtypes["DarkMatter"].itemsize
- star_size = dtypes["Stars"].itemsize
- if (fs == 32+gas_size*ng+dm_size*nd+star_size*ns):
+ #Catch for 4 byte padding with floats and doubles
+ if (fs == 32+48*ng+36*nd+44*ns or fs == 32+2*(48*ng+36*nd+44*ns)):
f.read(4)
#File is borked if this is true
- elif (fs != 28+gas_size*ng+dm_size*nd+star_size*ns):
+ elif (fs != 28+48*ng+36*nd+44*ns and fs != 28+2*(48*ng+36*nd+44*ns)):
f.close()
return False, 0
f.close()
@@ -563,9 +560,7 @@
@classmethod
def _is_valid(self, *args, **kwargs):
- return False
- field_dtypes = kwargs.get("field_dtypes", {})
- return TipsyDataset._validate_header(args[0], field_dtypes)[0]
+ return TipsyDataset._validate_header(args[0])[0]
class HTTPParticleFile(ParticleFile):
pass
https://bitbucket.org/yt_analysis/yt/commits/5394fa52467e/
Changeset: 5394fa52467e
Branch: yt-3.0
User: BW Keller
Date: 2014-04-12 01:28:24
Summary: Alright, we can now *QUICKLY* read all formats of Tipsy files: gasoline,
pkdGrav, and ChaNGa generated.
Affected #: 1 file
diff -r d95a6b0a206691d9314f7f69edb84a7e50a90416 -r 5394fa52467e743c9908d07b5dacb6bff196378c yt/frontends/sph/data_structures.py
--- a/yt/frontends/sph/data_structures.py
+++ b/yt/frontends/sph/data_structures.py
@@ -539,7 +539,6 @@
f.seek(0, os.SEEK_SET)
except IOError:
return False, 1
- f.seek(0)
#Read in the header
t, n, ndim, ng, nd, ns = struct.unpack("<diiiii", f.read(28))
endianswap = "<"
@@ -548,11 +547,9 @@
endianswap = ">"
f.seek(0)
t, n, ndim, ng, nd, ns = struct.unpack(">diiiii", f.read(28))
- #Catch for 4 byte padding with floats and doubles
- if (fs == 32+48*ng+36*nd+44*ns or fs == 32+2*(48*ng+36*nd+44*ns)):
- f.read(4)
#File is borked if this is true
- elif (fs != 28+48*ng+36*nd+44*ns and fs != 28+2*(48*ng+36*nd+44*ns)):
+ if (fs != 28+48*ng+36*nd+44*ns and fs != 28+60*ng+48*nd+56*ns and
+ fs != 32+48*ng+36*nd+44*ns and fs != 32+60*ng+48*nd+56*ns):
f.close()
return False, 0
f.close()
https://bitbucket.org/yt_analysis/yt/commits/e479bc4baf0b/
Changeset: e479bc4baf0b
Branch: yt-3.0
User: BW Keller
Date: 2014-04-12 01:58:28
Summary: Commented on the weird Tipsy filesize constants.
Affected #: 1 file
diff -r 5394fa52467e743c9908d07b5dacb6bff196378c -r e479bc4baf0bd16cf3f9a5767486f48ecbb4b107 yt/frontends/sph/data_structures.py
--- a/yt/frontends/sph/data_structures.py
+++ b/yt/frontends/sph/data_structures.py
@@ -547,7 +547,11 @@
endianswap = ">"
f.seek(0)
t, n, ndim, ng, nd, ns = struct.unpack(">diiiii", f.read(28))
- #File is borked if this is true
+ # File is borked if this is true. The header is 28 bytes, and may
+ # Be followed by a 4 byte pad. Next comes gas particles, which use
+ # 48 bytes, followed by 36 bytes per dark matter particle, and 44 bytes
+ # per star particle. If positions are stored as doubles, each of these
+ # sizes is increased by 12 bytes.
if (fs != 28+48*ng+36*nd+44*ns and fs != 28+60*ng+48*nd+56*ns and
fs != 32+48*ng+36*nd+44*ns and fs != 32+60*ng+48*nd+56*ns):
f.close()
https://bitbucket.org/yt_analysis/yt/commits/491a75603dea/
Changeset: 491a75603dea
Branch: yt-3.0
User: MatthewTurk
Date: 2014-04-14 15:26:15
Summary: Merged in bwkeller/yt/yt-3.0 (pull request #811)
Quickly validate tipsy files
Affected #: 1 file
diff -r 6f790e40d53b5c3274215f9e2f4f09fa1157b82c -r 491a75603dead69deea5ea11c382ba2f25ad068d yt/frontends/sph/data_structures.py
--- a/yt/frontends/sph/data_structures.py
+++ b/yt/frontends/sph/data_structures.py
@@ -388,7 +388,7 @@
self.over_refine_factor = over_refine_factor
if field_dtypes is None:
field_dtypes = {}
- success, self.endian = self._validate_header(filename, field_dtypes)
+ success, self.endian = self._validate_header(filename)
if not success:
print "SOMETHING HAS GONE WRONG. NBODIES != SUM PARTICLES."
print "%s != (%s == %s + %s + %s)" % (
@@ -522,7 +522,7 @@
self.time_unit = 1.0 / np.sqrt(G * density_unit)
@staticmethod
- def _validate_header(filename, field_dtypes):
+ def _validate_header(filename):
'''
This method automatically detects whether the tipsy file is big/little endian
and is not corrupt/invalid. It returns a tuple of (Valid, endianswap) where
@@ -534,10 +534,11 @@
except:
return False, 1
try:
- fs = len(f.read())
+ f.seek(0, os.SEEK_END)
+ fs = f.tell()
+ f.seek(0, os.SEEK_SET)
except IOError:
return False, 1
- f.seek(0)
#Read in the header
t, n, ndim, ng, nd, ns = struct.unpack("<diiiii", f.read(28))
endianswap = "<"
@@ -546,16 +547,13 @@
endianswap = ">"
f.seek(0)
t, n, ndim, ng, nd, ns = struct.unpack(">diiiii", f.read(28))
- # Now we construct the sizes of each of the particles.
- dtypes = IOHandlerTipsyBinary._compute_dtypes(field_dtypes, endianswap)
- #Catch for 4 byte padding
- gas_size = dtypes["Gas"].itemsize
- dm_size = dtypes["DarkMatter"].itemsize
- star_size = dtypes["Stars"].itemsize
- if (fs == 32+gas_size*ng+dm_size*nd+star_size*ns):
- f.read(4)
- #File is borked if this is true
- elif (fs != 28+gas_size*ng+dm_size*nd+star_size*ns):
+ # File is borked if this is true. The header is 28 bytes, and may
+ # Be followed by a 4 byte pad. Next comes gas particles, which use
+ # 48 bytes, followed by 36 bytes per dark matter particle, and 44 bytes
+ # per star particle. If positions are stored as doubles, each of these
+ # sizes is increased by 12 bytes.
+ if (fs != 28+48*ng+36*nd+44*ns and fs != 28+60*ng+48*nd+56*ns and
+ fs != 32+48*ng+36*nd+44*ns and fs != 32+60*ng+48*nd+56*ns):
f.close()
return False, 0
f.close()
@@ -563,9 +561,7 @@
@classmethod
def _is_valid(self, *args, **kwargs):
- return False
- field_dtypes = kwargs.get("field_dtypes", {})
- return TipsyDataset._validate_header(args[0], field_dtypes)[0]
+ return TipsyDataset._validate_header(args[0])[0]
class HTTPParticleFile(ParticleFile):
pass
Repository URL: https://bitbucket.org/yt_analysis/yt/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the yt-svn
mailing list