[yt-svn] commit/yt-3.0: 5 new changesets
commits-noreply at bitbucket.org
commits-noreply at bitbucket.org
Wed Apr 24 11:17:06 PDT 2013
5 new commits in yt-3.0:
https://bitbucket.org/yt_analysis/yt-3.0/commits/494770bb6ed2/
Changeset: 494770bb6ed2
Branch: yt-3.0
User: sskory
Date: 2013-03-21 20:13:03
Summary: Speeding up field detection for enzo.
Affected #: 2 files
diff -r 6bdd0c4ba1619072b110248e87d120eb2e14d30f -r 494770bb6ed2d1689fa64e84da63103c6dfb1787 yt/frontends/enzo/data_structures.py
--- a/yt/frontends/enzo/data_structures.py
+++ b/yt/frontends/enzo/data_structures.py
@@ -30,6 +30,7 @@
import stat
import string
import re
+import multiprocessing
from itertools import izip
@@ -57,6 +58,8 @@
from yt.utilities.parallel_tools.parallel_analysis_interface import \
parallel_blocking_call
+from data_structures_helper import get_field_names_helper
+
class EnzoGrid(AMRGridPatch):
"""
Class representing a single Enzo Grid instance.
@@ -288,7 +291,7 @@
if self.parameter_file.parameters["VersionNumber"] > 2.0:
active_particles = True
nap = {}
- for type in self.parameters.get("AppendActiveParticleType", []):
+ for type in self.parameters["AppendActiveParticleType"]:
nap[type] = []
else:
active_particles = False
@@ -309,7 +312,7 @@
if active_particles:
ptypes = _next_token_line("PresentParticleTypes", f)
counts = [int(c) for c in _next_token_line("ParticleTypeCounts", f)]
- for ptype in self.parameters.get("AppendActiveParticleType", []):
+ for ptype in self.parameters["AppendActiveParticleType"]:
if ptype in ptypes:
nap[ptype].append(counts[ptypes.index(ptype)])
else:
@@ -401,29 +404,28 @@
self.max_level = self.grid_levels.max()
def _detect_active_particle_fields(self):
- select_grids = np.zeros(len(self.grids), dtype='int32')
- for ptype in self.parameter_file["AppendActiveParticleType"]:
- select_grids += self.grid_active_particle_count[ptype].flat
- gs = self.grids[select_grids > 0]
- grids = sorted((g for g in gs), key = lambda a: a.filename)
- handle = last = None
ap_list = self.parameter_file["AppendActiveParticleType"]
_fields = dict((ap, []) for ap in ap_list)
fields = []
- for g in grids:
- # We inspect every grid, for now, until we have a list of
- # attributes in a defined location.
- if last != g.filename:
- if handle is not None: handle.close()
- handle = h5py.File(g.filename, "r")
- node = handle["/Grid%08i/Particles/" % g.id]
- for ptype in (str(p) for p in node):
- if ptype not in _fields: continue
- for field in (str(f) for f in node[ptype]):
- _fields[ptype].append(field)
- fields += [(ptype, field) for field in _fields.pop(ptype)]
- if len(_fields) == 0: break
- if handle is not None: handle.close()
+ for ptype in self.parameter_file["AppendActiveParticleType"]:
+ select_grids = self.grid_active_particle_count[ptype].flat
+ gs = self.grids[select_grids > 0]
+ grids = sorted((g for g in gs), key = lambda a: a.filename)
+ handle = last = None
+ for g in grids:
+ # We inspect every grid, for now, until we have a list of
+ # attributes in a defined location.
+ if last != g.filename:
+ if handle is not None: handle.close()
+ handle = h5py.File(g.filename)
+ node = handle["/Grid%08i/Particles/" % g.id]
+ for ptype in (str(p) for p in node):
+ if ptype not in _fields: continue
+ for field in (str(f) for f in node[ptype]):
+ _fields[ptype].append(field)
+ fields += [(ptype, field) for field in _fields.pop(ptype)]
+ if len(_fields) == 0: break
+ if handle is not None: handle.close()
return set(fields)
def _setup_derived_fields(self):
@@ -448,15 +450,19 @@
mylog.info("Gathering a field list (this may take a moment.)")
field_list = set()
random_sample = self._generate_random_grids()
+ jobs = []
+ result_queue = multiprocessing.Queue()
for grid in random_sample:
if not hasattr(grid, 'filename'): continue
- try:
- gf = self.io._read_field_names(grid)
- except self.io._read_exception:
- mylog.debug("Grid %s is a bit funky?", grid.id)
- continue
- mylog.debug("Grid %s has: %s", grid.id, gf)
- field_list = field_list.union(gf)
+ p = multiprocessing.Process(target=get_field_names_helper,
+ args = (grid.filename, grid.id, result_queue))
+ jobs.append(p)
+ p.start()
+ for grid in random_sample:
+ res = result_queue.get()
+ mylog.debug(res[1])
+ if res[0] is not None:
+ field_list = field_list.union(res[0])
if "AppendActiveParticleType" in self.parameter_file.parameters:
ap_fields = self._detect_active_particle_fields()
field_list = list(set(field_list).union(ap_fields))
@@ -998,7 +1004,6 @@
for p, v in self._conversion_override.items():
self.conversion_factors[p] = v
self.refine_by = self.parameters["RefineBy"]
- self.periodicity = ensure_tuple(self.parameters["LeftFaceBoundaryCondition"] == 3)
self.dimensionality = self.parameters["TopGridRank"]
self.domain_dimensions = self.parameters["TopGridDimensions"]
self.current_time = self.parameters["InitialTime"]
diff -r 6bdd0c4ba1619072b110248e87d120eb2e14d30f -r 494770bb6ed2d1689fa64e84da63103c6dfb1787 yt/frontends/enzo/data_structures_helper.py
--- /dev/null
+++ b/yt/frontends/enzo/data_structures_helper.py
@@ -0,0 +1,34 @@
+"""
+Data structures helper for Enzo
+
+Author: Stephen Skory <s at skory.us>
+Affiliation: Univ of Colorado
+Homepage: http://yt-project.org/
+License:
+ Copyright (C) 2007-2013 Matthew Turk. All Rights Reserved.
+
+ This file is part of yt.
+
+ yt is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+
+from yt.utilities import hdf5_light_reader
+
+def get_field_names_helper(filename, id, results):
+ try:
+ names= hdf5_light_reader.ReadListOfDatasets(
+ filename, "/Grid%08i" % id)
+ results.put((names, "Grid %s has: %s" % (grid.id, names)))
+ except:
+ results.put((None, "Grid %s is a bit funky?" % id))
https://bitbucket.org/yt_analysis/yt-3.0/commits/1b21c904aae6/
Changeset: 1b21c904aae6
Branch: yt-3.0
User: sskory
Date: 2013-03-21 20:16:24
Summary: Restoring some stuff I accidentally changed (?).
Affected #: 1 file
diff -r 494770bb6ed2d1689fa64e84da63103c6dfb1787 -r 1b21c904aae6d287a00d354f6e83075fceb6aa0d yt/frontends/enzo/data_structures.py
--- a/yt/frontends/enzo/data_structures.py
+++ b/yt/frontends/enzo/data_structures.py
@@ -291,7 +291,7 @@
if self.parameter_file.parameters["VersionNumber"] > 2.0:
active_particles = True
nap = {}
- for type in self.parameters["AppendActiveParticleType"]:
+ for type in self.parameters.get("AppendActiveParticleType", []):
nap[type] = []
else:
active_particles = False
@@ -312,7 +312,7 @@
if active_particles:
ptypes = _next_token_line("PresentParticleTypes", f)
counts = [int(c) for c in _next_token_line("ParticleTypeCounts", f)]
- for ptype in self.parameters["AppendActiveParticleType"]:
+ for ptype in self.parameters.get("AppendActiveParticleType", []):
if ptype in ptypes:
nap[ptype].append(counts[ptypes.index(ptype)])
else:
@@ -1005,6 +1005,7 @@
self.conversion_factors[p] = v
self.refine_by = self.parameters["RefineBy"]
self.dimensionality = self.parameters["TopGridRank"]
+ self.periodicity = ensure_tuple(self.parameters["LeftFaceBoundaryCondition"] == 3)
self.domain_dimensions = self.parameters["TopGridDimensions"]
self.current_time = self.parameters["InitialTime"]
if "CurrentTimeIdentifier" in self.parameters:
https://bitbucket.org/yt_analysis/yt-3.0/commits/c2667b210a3b/
Changeset: c2667b210a3b
Branch: yt-3.0
User: sskory
Date: 2013-03-22 01:02:31
Summary: Replaced multithreading with threading.
Affected #: 2 files
diff -r 1b21c904aae6d287a00d354f6e83075fceb6aa0d -r c2667b210a3b44c37379ba6fd38420e8793feb81 yt/frontends/enzo/data_structures.py
--- a/yt/frontends/enzo/data_structures.py
+++ b/yt/frontends/enzo/data_structures.py
@@ -30,7 +30,9 @@
import stat
import string
import re
-import multiprocessing
+
+from threading import Thread
+import Queue
from itertools import izip
@@ -58,7 +60,13 @@
from yt.utilities.parallel_tools.parallel_analysis_interface import \
parallel_blocking_call
-from data_structures_helper import get_field_names_helper
+def get_field_names_helper(filename, id, results):
+ try:
+ names = hdf5_light_reader.ReadListOfDatasets(
+ filename, "/Grid%08i" % id)
+ results.put((names, "Grid %s has: %s" % (id, names)))
+ except:
+ results.put((None, "Grid %s is a bit funky?" % id))
class EnzoGrid(AMRGridPatch):
"""
@@ -451,13 +459,17 @@
field_list = set()
random_sample = self._generate_random_grids()
jobs = []
- result_queue = multiprocessing.Queue()
+ result_queue = Queue.Queue()
+ # Start threads
for grid in random_sample:
if not hasattr(grid, 'filename'): continue
- p = multiprocessing.Process(target=get_field_names_helper,
+ helper = Thread(target = get_field_names_helper,
args = (grid.filename, grid.id, result_queue))
- jobs.append(p)
- p.start()
+ jobs.append(helper)
+ helper.start()
+ # Here we make sure they're finished.
+ for helper in jobs:
+ helper.join()
for grid in random_sample:
res = result_queue.get()
mylog.debug(res[1])
diff -r 1b21c904aae6d287a00d354f6e83075fceb6aa0d -r c2667b210a3b44c37379ba6fd38420e8793feb81 yt/frontends/enzo/data_structures_helper.py
--- a/yt/frontends/enzo/data_structures_helper.py
+++ /dev/null
@@ -1,34 +0,0 @@
-"""
-Data structures helper for Enzo
-
-Author: Stephen Skory <s at skory.us>
-Affiliation: Univ of Colorado
-Homepage: http://yt-project.org/
-License:
- Copyright (C) 2007-2013 Matthew Turk. All Rights Reserved.
-
- This file is part of yt.
-
- yt is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>.
-"""
-
-from yt.utilities import hdf5_light_reader
-
-def get_field_names_helper(filename, id, results):
- try:
- names= hdf5_light_reader.ReadListOfDatasets(
- filename, "/Grid%08i" % id)
- results.put((names, "Grid %s has: %s" % (grid.id, names)))
- except:
- results.put((None, "Grid %s is a bit funky?" % id))
https://bitbucket.org/yt_analysis/yt-3.0/commits/36524214f584/
Changeset: 36524214f584
Branch: yt-3.0
User: sskory
Date: 2013-04-24 01:01:05
Summary: Making field detection threading optional, and off by default.
Affected #: 2 files
diff -r c2667b210a3b44c37379ba6fd38420e8793feb81 -r 36524214f584e412d6d4bf2bc8a81aa5b6df5ba9 yt/config.py
--- a/yt/config.py
+++ b/yt/config.py
@@ -64,7 +64,8 @@
answer_testing_bitwise = 'False',
gold_standard_filename = 'gold006',
local_standard_filename = 'local001',
- sketchfab_api_key = 'None'
+ sketchfab_api_key = 'None',
+ thread_field_detection = 'False'
)
# Here is the upgrade. We're actually going to parse the file in its entirety
# here. Then, if it has any of the Forbidden Sections, it will be rewritten
diff -r c2667b210a3b44c37379ba6fd38420e8793feb81 -r 36524214f584e412d6d4bf2bc8a81aa5b6df5ba9 yt/frontends/enzo/data_structures.py
--- a/yt/frontends/enzo/data_structures.py
+++ b/yt/frontends/enzo/data_structures.py
@@ -65,7 +65,7 @@
names = hdf5_light_reader.ReadListOfDatasets(
filename, "/Grid%08i" % id)
results.put((names, "Grid %s has: %s" % (id, names)))
- except:
+ except (exceptions.KeyError, hdf5_light_reader.ReadingError):
results.put((None, "Grid %s is a bit funky?" % id))
class EnzoGrid(AMRGridPatch):
@@ -418,22 +418,15 @@
for ptype in self.parameter_file["AppendActiveParticleType"]:
select_grids = self.grid_active_particle_count[ptype].flat
gs = self.grids[select_grids > 0]
- grids = sorted((g for g in gs), key = lambda a: a.filename)
- handle = last = None
- for g in grids:
- # We inspect every grid, for now, until we have a list of
- # attributes in a defined location.
- if last != g.filename:
- if handle is not None: handle.close()
- handle = h5py.File(g.filename)
- node = handle["/Grid%08i/Particles/" % g.id]
- for ptype in (str(p) for p in node):
- if ptype not in _fields: continue
- for field in (str(f) for f in node[ptype]):
- _fields[ptype].append(field)
- fields += [(ptype, field) for field in _fields.pop(ptype)]
- if len(_fields) == 0: break
- if handle is not None: handle.close()
+ g = gs[0]
+ handle = h5py.File(g.filename)
+ node = handle["/Grid%08i/Particles/" % g.id]
+ for ptype in (str(p) for p in node):
+ if ptype not in _fields: continue
+ for field in (str(f) for f in node[ptype]):
+ _fields[ptype].append(field)
+ fields += [(ptype, field) for field in _fields.pop(ptype)]
+ handle.close()
return set(fields)
def _setup_derived_fields(self):
@@ -458,23 +451,35 @@
mylog.info("Gathering a field list (this may take a moment.)")
field_list = set()
random_sample = self._generate_random_grids()
- jobs = []
- result_queue = Queue.Queue()
- # Start threads
- for grid in random_sample:
- if not hasattr(grid, 'filename'): continue
- helper = Thread(target = get_field_names_helper,
- args = (grid.filename, grid.id, result_queue))
- jobs.append(helper)
- helper.start()
- # Here we make sure they're finished.
- for helper in jobs:
- helper.join()
- for grid in random_sample:
- res = result_queue.get()
- mylog.debug(res[1])
- if res[0] is not None:
- field_list = field_list.union(res[0])
+ tothread = ytcfg.getboolean("yt","thread_field_detection")
+ if tothread:
+ jobs = []
+ result_queue = Queue.Queue()
+ # Start threads
+ for grid in random_sample:
+ if not hasattr(grid, 'filename'): continue
+ helper = Thread(target = get_field_names_helper,
+ args = (grid.filename, grid.id, result_queue))
+ jobs.append(helper)
+ helper.start()
+ # Here we make sure they're finished.
+ for helper in jobs:
+ helper.join()
+ for grid in random_sample:
+ res = result_queue.get()
+ mylog.debug(res[1])
+ if res[0] is not None:
+ field_list = field_list.union(res[0])
+ else:
+ for grid in random_sample:
+ if not hasattr(grid, 'filename'): continue
+ try:
+ gf = self.io._read_field_names(grid)
+ except self.io._read_exception:
+ mylog.debug("Grid %s is a bit funky?", grid.id)
+ continue
+ mylog.debug("Grid %s has: %s", grid.id, gf)
+ field_list = field_list.union(gf)
if "AppendActiveParticleType" in self.parameter_file.parameters:
ap_fields = self._detect_active_particle_fields()
field_list = list(set(field_list).union(ap_fields))
https://bitbucket.org/yt_analysis/yt-3.0/commits/5e02fade47bd/
Changeset: 5e02fade47bd
Branch: yt-3.0
User: sskory
Date: 2013-04-24 01:01:48
Summary: Merge.
Affected #: 19 files
Diff not available.
Repository URL: https://bitbucket.org/yt_analysis/yt-3.0/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the yt-svn
mailing list