[yt-svn] commit/yt: 6 new changesets
commits-noreply at bitbucket.org
commits-noreply at bitbucket.org
Sat Jan 3 21:17:43 PST 2015
6 new commits in yt:
https://bitbucket.org/yt_analysis/yt/commits/f6e4ba01af4c/
Changeset: f6e4ba01af4c
Branch: yt
User: ngoldbaum
Date: 2014-12-04 06:38:57+00:00
Summary: Iterate over blocks instead of grids in the isocontour flux calculation.
Affected #: 1 file
diff -r 9d96ac2cfc308468239e99e88552a704168291e0 -r f6e4ba01af4c3ee90b27814dba748adfcaa30539 yt/data_objects/data_containers.py
--- a/yt/data_objects/data_containers.py
+++ b/yt/data_objects/data_containers.py
@@ -1025,16 +1025,13 @@
"""
verts = []
samples = []
- pb = get_pbar("Extracting ", len(list(self._get_grid_objs())))
- for i, g in enumerate(self._get_grid_objs()):
- pb.update(i)
+ for block, mask in self.blocks:
my_verts = self._extract_isocontours_from_grid(
- g, field, value, sample_values)
+ block, mask, field, value, sample_values)
if sample_values is not None:
my_verts, svals = my_verts
samples.append(svals)
verts.append(my_verts)
- pb.finish()
verts = np.concatenate(verts).transpose()
verts = self.comm.par_combine_object(verts, op='cat', datatype='array')
verts = verts.transpose()
@@ -1058,11 +1055,9 @@
return verts, samples
return verts
-
- def _extract_isocontours_from_grid(self, grid, field, value,
- sample_values = None):
- mask = self._get_cut_mask(grid) * grid.child_mask
- vals = grid.get_vertex_centered_data(field, no_ghost = False)
+ def _extract_isocontours_from_grid(self, grid, mask, field, value,
+ sample_values=None):
+ vals = grid.get_vertex_centered_data(field, no_ghost=False)
if sample_values is not None:
svals = grid.get_vertex_centered_data(sample_values)
else:
@@ -1136,15 +1131,14 @@
... "velocity_x", "velocity_y", "velocity_z", "Metal_Density")
"""
flux = 0.0
- for g in self._get_grid_objs():
- flux += self._calculate_flux_in_grid(g, field, value,
- field_x, field_y, field_z, fluxing_field)
+ for block, mask in self.blocks:
+ flux += self._calculate_flux_in_grid(block, mask, field, value, field_x,
+ field_y, field_z, fluxing_field)
flux = self.comm.mpi_allreduce(flux, op="sum")
return flux
- def _calculate_flux_in_grid(self, grid, field, value,
+ def _calculate_flux_in_grid(self, grid, mask, field, value,
field_x, field_y, field_z, fluxing_field = None):
- mask = self._get_cut_mask(grid) * grid.child_mask
vals = grid.get_vertex_centered_data(field)
if fluxing_field is None:
ff = np.ones(vals.shape, dtype="float64")
https://bitbucket.org/yt_analysis/yt/commits/7349f544374a/
Changeset: 7349f544374a
Branch: yt
User: MatthewTurk
Date: 2014-12-09 19:27:10+00:00
Summary: First implementation of preloading for IO chunks.
Affected #: 6 files
diff -r f6e4ba01af4c3ee90b27814dba748adfcaa30539 -r 7349f544374accd760d4d37638b56dce658779d1 yt/data_objects/construction_data_containers.py
--- a/yt/data_objects/construction_data_containers.py
+++ b/yt/data_objects/construction_data_containers.py
@@ -466,7 +466,7 @@
self.domain_width = np.rint((self.ds.domain_right_edge -
self.ds.domain_left_edge)/self.dds).astype('int64')
self._setup_data_source()
-
+
@property
def icoords(self):
ic = np.indices(self.ActiveDimensions).astype("int64")
@@ -935,14 +935,18 @@
mylog.info("Extracting (sampling: %s)" % (fields,))
verts = []
samples = []
- for block, mask in parallel_objects(self.data_source.blocks):
- my_verts = self._extract_isocontours_from_grid(
- block, self.surface_field, self.field_value,
- mask, fields, sample_type)
- if fields is not None:
- my_verts, svals = my_verts
- samples.append(svals)
- verts.append(my_verts)
+ deps = self._determine_fields(self.surface_field)
+ deps = self._identify_dependencies(deps, spatial=True)
+ for io_chunk in parallel_objects(self.data_source.chunks(deps, "io",
+ preload_fields = deps)):
+ for block, mask in self.data_source.blocks:
+ my_verts = self._extract_isocontours_from_grid(
+ block, self.surface_field, self.field_value,
+ mask, fields, sample_type)
+ if fields is not None:
+ my_verts, svals = my_verts
+ samples.append(svals)
+ verts.append(my_verts)
verts = np.concatenate(verts).transpose()
verts = self.comm.par_combine_object(verts, op='cat', datatype='array')
self.vertices = verts
@@ -1025,21 +1029,27 @@
"""
flux = 0.0
mylog.info("Fluxing %s", fluxing_field)
- for block, mask in parallel_objects(self.data_source.blocks):
- flux += self._calculate_flux_in_grid(block, mask,
- field_x, field_y, field_z, fluxing_field)
+ deps = [field_x, field_y, field_z]
+ if fluxing_field is not None: deps.append(fluxing_field)
+ deps = self._determine_fields(deps)
+ deps = self._identify_dependencies(deps)
+ for io_chunk in parallel_objects(self.data_source.chunks(deps, "io",
+ preload_fields = deps)):
+ for block, mask in self.data_source.blocks:
+ flux += self._calculate_flux_in_grid(block, mask,
+ field_x, field_y, field_z, fluxing_field)
flux = self.comm.mpi_allreduce(flux, op="sum")
return flux
def _calculate_flux_in_grid(self, grid, mask,
- field_x, field_y, field_z, fluxing_field = None):
+ field_x, field_y, field_z, fluxing_field = None):
vals = grid.get_vertex_centered_data(self.surface_field)
if fluxing_field is None:
ff = np.ones(vals.shape, dtype="float64")
else:
ff = grid.get_vertex_centered_data(fluxing_field)
- xv, yv, zv = [grid.get_vertex_centered_data(f) for f in
- [field_x, field_y, field_z]]
+ xv, yv, zv = [grid.get_vertex_centered_data(f)
+ for f in [field_x, field_y, field_z]]
return march_cubes_grid_flux(self.field_value, vals, xv, yv, zv,
ff, mask, grid.LeftEdge, grid.dds)
diff -r f6e4ba01af4c3ee90b27814dba748adfcaa30539 -r 7349f544374accd760d4d37638b56dce658779d1 yt/frontends/enzo/io.py
--- a/yt/frontends/enzo/io.py
+++ b/yt/frontends/enzo/io.py
@@ -14,6 +14,7 @@
#-----------------------------------------------------------------------------
import os
+from contextlib import contextmanager
from yt.utilities.io_handler import \
BaseIOHandler, _axis_ids
@@ -130,8 +131,17 @@
raise RuntimeError
g = chunks[0].objs[0]
f = h5py.File(g.filename.encode('ascii'), 'r')
+ if g.id in self._cached_fields:
+ gf = self._cached_fields[g.id]
+ rv.update(gf)
+ if len(rv) == len(fields): return rv
gds = f.get("/Grid%08i" % g.id)
- for ftype, fname in fields:
+ for field in fields:
+ if field in rv:
+ self._hits += 1
+ continue
+ self._misses += 1
+ ftype, fname = field
if fname in gds:
rv[(ftype, fname)] = gds.get(fname).value.swapaxes(0,2)
else:
@@ -155,10 +165,16 @@
if g.filename is None: continue
if fid is None:
fid = h5py.h5f.open(g.filename.encode('ascii'), h5py.h5f.ACC_RDONLY)
+ gf = self._cached_fields.get(g.id, {})
data = np.empty(g.ActiveDimensions[::-1], dtype="float64")
data_view = data.swapaxes(0,2)
nd = 0
for field in fields:
+ if field in gf:
+ nd = g.select(selector, gf[field], rv[field], ind)
+ self._hits += 1
+ continue
+ self._misses += 1
ftype, fname = field
try:
node = "/Grid%08i/%s" % (g.id, fname)
@@ -172,6 +188,29 @@
if fid: fid.close()
return rv
+ @contextmanager
+ def preload(self, chunk, fields, mn = None):
+ if len(fields) == 0:
+ yield self
+ return
+ self.mn = mn
+ old_cache_on = self._cache_on
+ old_cached_fields = self._cached_fields
+ self._cached_fields = cf = {}
+ for gid in old_cached_fields:
+ # Will not copy numpy arrays, which is good!
+ cf[gid] = old_cached_fields[gid].copy()
+ self._hits = self._misses = 0
+ self._cached_fields = self._read_chunk_data(chunk, fields)
+ mylog.debug("(1st) Hits = % 10i Misses = % 10i",
+ self._hits, self._misses)
+ self._hits = self._misses = 0
+ yield self
+ mylog.debug("(2nd) Hits = % 10i Misses = % 10i",
+ self._hits, self._misses)
+ self._cached_fields = old_cached_fields
+ self._cache_on = old_cache_on
+
def _read_chunk_data(self, chunk, fields):
fid = fn = None
rv = {}
@@ -190,6 +229,8 @@
if len(fluid_fields) == 0: return rv
for g in chunk.objs:
rv[g.id] = gf = {}
+ if g.id in self._cached_fields:
+ rv[g.id].update(self._cached_fields[g.id])
if g.filename is None: continue
elif g.filename != fn:
if fid is not None: fid.close()
@@ -200,6 +241,10 @@
data = np.empty(g.ActiveDimensions[::-1], dtype="float64")
data_view = data.swapaxes(0,2)
for field in fluid_fields:
+ if field in gf:
+ self._hits += 1
+ continue
+ self._misses += 1
ftype, fname = field
try:
node = "/Grid%08i/%s" % (g.id, fname)
diff -r f6e4ba01af4c3ee90b27814dba748adfcaa30539 -r 7349f544374accd760d4d37638b56dce658779d1 yt/geometry/geometry_handler.py
--- a/yt/geometry/geometry_handler.py
+++ b/yt/geometry/geometry_handler.py
@@ -252,7 +252,6 @@
chunk_size)
return fields_to_return, fields_to_generate
-
def _chunk(self, dobj, chunking_style, ngz = 0, **kwargs):
# A chunk is either None or (grids, size)
if dobj._current_chunk is None:
diff -r f6e4ba01af4c3ee90b27814dba748adfcaa30539 -r 7349f544374accd760d4d37638b56dce658779d1 yt/geometry/grid_geometry_handler.py
--- a/yt/geometry/grid_geometry_handler.py
+++ b/yt/geometry/grid_geometry_handler.py
@@ -323,9 +323,13 @@
yield YTDataChunk(dobj, "spatial", [g], size, cache = False)
_grid_chunksize = 1000
- def _chunk_io(self, dobj, cache = True, local_only = False):
+ def _chunk_io(self, dobj, cache = True, local_only = False,
+ preload_fields = None):
# local_only is only useful for inline datasets and requires
# implementation by subclasses.
+ if preload_fields is None:
+ preload_fields = []
+ preload_fields, _ = self._split_fields(preload_fields)
gfiles = defaultdict(list)
gobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
for g in gobjs:
@@ -338,7 +342,8 @@
for grids in (gs[pos:pos + size] for pos
in xrange(0, len(gs), size)):
- yield YTDataChunk(dobj, "io", grids,
+ dc = YTDataChunk(dobj, "io", grids,
self._count_selection(dobj, grids),
cache = cache)
-
+ with self.io.preload(dc, preload_fields, fn):
+ yield dc
diff -r f6e4ba01af4c3ee90b27814dba748adfcaa30539 -r 7349f544374accd760d4d37638b56dce658779d1 yt/utilities/io_handler.py
--- a/yt/utilities/io_handler.py
+++ b/yt/utilities/io_handler.py
@@ -14,6 +14,7 @@
#-----------------------------------------------------------------------------
from collections import defaultdict
+from contextlib import contextmanager
from yt.funcs import mylog
import cPickle
@@ -37,6 +38,7 @@
_vector_fields = ()
_dataset_type = None
_particle_reader = False
+ _cache_on = False
def __init__(self, ds):
self.queue = defaultdict(dict)
@@ -44,12 +46,14 @@
self._last_selector_id = None
self._last_selector_counts = None
self._array_fields = {}
+ self._cached_fields = {}
# We need a function for reading a list of sets
# and a function for *popping* from a queue all the appropriate sets
- def preload(self, grids, sets):
- pass
+ @contextmanager
+ def preload(self, chunk, fields):
+ yield self
def pop(self, grid, field):
if grid.id in self.queue and field in self.queue[grid.id]:
diff -r f6e4ba01af4c3ee90b27814dba748adfcaa30539 -r 7349f544374accd760d4d37638b56dce658779d1 yt/utilities/parallel_tools/parallel_analysis_interface.py
--- a/yt/utilities/parallel_tools/parallel_analysis_interface.py
+++ b/yt/utilities/parallel_tools/parallel_analysis_interface.py
@@ -781,12 +781,8 @@
return data
def preload(self, grids, fields, io_handler):
- # This will preload if it detects we are parallel capable and
- # if so, we load *everything* that we need. Use with some care.
- if len(fields) == 0: return
- mylog.debug("Preloading %s from %s grids", fields, len(grids))
- if not self._distributed: return
- io_handler.preload(grids, fields)
+ # This is non-functional.
+ return
@parallel_passthrough
def mpi_allreduce(self, data, dtype=None, op='sum'):
https://bitbucket.org/yt_analysis/yt/commits/ce9d121083ba/
Changeset: ce9d121083ba
Branch: yt
User: MatthewTurk
Date: 2014-12-09 20:02:25+00:00
Summary: Adding an accumulation for caching.
Affected #: 3 files
diff -r 7349f544374accd760d4d37638b56dce658779d1 -r ce9d121083ba92930a11a735fd7b4bee1555ea74 yt/frontends/enzo/io.py
--- a/yt/frontends/enzo/io.py
+++ b/yt/frontends/enzo/io.py
@@ -14,6 +14,7 @@
#-----------------------------------------------------------------------------
import os
+import random
from contextlib import contextmanager
from yt.utilities.io_handler import \
@@ -146,6 +147,10 @@
rv[(ftype, fname)] = gds.get(fname).value.swapaxes(0,2)
else:
rv[(ftype, fname)] = np.zeros(g.ActiveDimensions)
+ if self._cache_on:
+ for gid in rv:
+ self._cached_fields.setdefault(gid, {})
+ self._cached_fields[gid].update(rv[gid])
f.close()
return rv
if size is None:
@@ -183,20 +188,24 @@
if fname == "Dark_Matter_Density": continue
raise
dg.read(h5py.h5s.ALL, h5py.h5s.ALL, data)
+ if self._cache_on:
+ self._cached_fields.setdefault(g.id, {})
+ # Copy because it's a view into an empty temp array
+ self._cached_fields[g.id][field] = data_view.copy()
nd = g.select(selector, data_view, rv[field], ind) # caches
ind += nd
if fid: fid.close()
return rv
@contextmanager
- def preload(self, chunk, fields, mn = None):
+ def preload(self, chunk, fields, max_size):
if len(fields) == 0:
yield self
return
- self.mn = mn
old_cache_on = self._cache_on
old_cached_fields = self._cached_fields
self._cached_fields = cf = {}
+ self._cache_on = True
for gid in old_cached_fields:
# Will not copy numpy arrays, which is good!
cf[gid] = old_cached_fields[gid].copy()
@@ -210,6 +219,20 @@
self._hits, self._misses)
self._cached_fields = old_cached_fields
self._cache_on = old_cache_on
+ # Randomly remove some grids from the cache. Note that we're doing
+ # this on a grid basis, not a field basis. Performance will be
+ # slightly non-deterministic as a result of this, but it should roughly
+ # be statistically alright, assuming (as we do) that this will get
+ # called during largely unbalanced stuff.
+ if len(self._cached_fields) > max_size:
+ to_remove = random.sample(self._cached_fields.keys(),
+ len(self._cached_fields) - max_size)
+ mylog.debug("Purging from cache %s", len(to_remove))
+ for k in to_remove:
+ self._cached_fields.pop(k)
+ else:
+ mylog.warning("Cache size % 10i (max % 10i)",
+ len(self._cached_fields), max_size)
def _read_chunk_data(self, chunk, fields):
fid = fn = None
@@ -255,6 +278,10 @@
dg.read(h5py.h5s.ALL, h5py.h5s.ALL, data)
gf[field] = data_view.copy()
if fid: fid.close()
+ if self._cache_on:
+ for gid in rv:
+ self._cached_fields.setdefault(gid, {})
+ self._cached_fields[gid].update(rv[gid])
return rv
class IOHandlerPackedHDF5GhostZones(IOHandlerPackedHDF5):
diff -r 7349f544374accd760d4d37638b56dce658779d1 -r ce9d121083ba92930a11a735fd7b4bee1555ea74 yt/geometry/grid_geometry_handler.py
--- a/yt/geometry/grid_geometry_handler.py
+++ b/yt/geometry/grid_geometry_handler.py
@@ -345,5 +345,7 @@
dc = YTDataChunk(dobj, "io", grids,
self._count_selection(dobj, grids),
cache = cache)
- with self.io.preload(dc, preload_fields, fn):
+ # We allow four full chunks to be included.
+ with self.io.preload(dc, preload_fields,
+ 4.0 * self._grid_chunksize):
yield dc
diff -r 7349f544374accd760d4d37638b56dce658779d1 -r ce9d121083ba92930a11a735fd7b4bee1555ea74 yt/utilities/io_handler.py
--- a/yt/utilities/io_handler.py
+++ b/yt/utilities/io_handler.py
@@ -52,7 +52,7 @@
# and a function for *popping* from a queue all the appropriate sets
@contextmanager
- def preload(self, chunk, fields):
+ def preload(self, chunk, fields, max_size):
yield self
def pop(self, grid, field):
https://bitbucket.org/yt_analysis/yt/commits/f6fc1de3b75b/
Changeset: f6fc1de3b75b
Branch: yt
User: MatthewTurk
Date: 2014-12-09 20:12:29+00:00
Summary: Initialize hits and misses
Affected #: 1 file
diff -r ce9d121083ba92930a11a735fd7b4bee1555ea74 -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 yt/utilities/io_handler.py
--- a/yt/utilities/io_handler.py
+++ b/yt/utilities/io_handler.py
@@ -39,6 +39,8 @@
_dataset_type = None
_particle_reader = False
_cache_on = False
+ _misses = 0
+ _hits = 0
def __init__(self, ds):
self.queue = defaultdict(dict)
https://bitbucket.org/yt_analysis/yt/commits/74e0b4906416/
Changeset: 74e0b4906416
Branch: yt
User: MatthewTurk
Date: 2014-12-31 03:05:49+00:00
Summary: Merging
Affected #: 105 files
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d distribute_setup.py
--- a/distribute_setup.py
+++ b/distribute_setup.py
@@ -269,7 +269,7 @@
def _remove_flat_installation(placeholder):
if not os.path.isdir(placeholder):
- log.warn('Unkown installation at %s', placeholder)
+ log.warn('Unknown installation at %s', placeholder)
return False
found = False
for file in os.listdir(placeholder):
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/extensions/notebook_sphinxext.py
--- a/doc/extensions/notebook_sphinxext.py
+++ b/doc/extensions/notebook_sphinxext.py
@@ -1,9 +1,16 @@
-import os, shutil, string, glob, re
+import errno
+import os
+import shutil
+import string
+import re
+import tempfile
+import uuid
from sphinx.util.compat import Directive
from docutils import nodes
from docutils.parsers.rst import directives
+from IPython.config import Config
from IPython.nbconvert import html, python
-from IPython.nbformat.current import read, write
+from IPython.nbformat import current as nbformat
from runipy.notebook_runner import NotebookRunner, NotebookError
class NotebookDirective(Directive):
@@ -14,7 +21,7 @@
"""
required_arguments = 1
optional_arguments = 1
- option_spec = {'skip_exceptions' : directives.flag}
+ option_spec = {'skip_exceptions': directives.flag}
final_argument_whitespace = True
def run(self): # check if there are spaces in the notebook name
@@ -26,9 +33,11 @@
if not self.state.document.settings.raw_enabled:
raise self.warning('"%s" directive disabled.' % self.name)
+ cwd = os.getcwd()
+ tmpdir = tempfile.mkdtemp()
+ os.chdir(tmpdir)
+
# get path to notebook
- source_dir = os.path.dirname(
- os.path.abspath(self.state.document.current_source))
nb_filename = self.arguments[0]
nb_basename = os.path.basename(nb_filename)
rst_file = self.state_machine.document.attributes['source']
@@ -37,26 +46,24 @@
# Move files around.
rel_dir = os.path.relpath(rst_dir, setup.confdir)
- rel_path = os.path.join(rel_dir, nb_basename)
dest_dir = os.path.join(setup.app.builder.outdir, rel_dir)
dest_path = os.path.join(dest_dir, nb_basename)
- if not os.path.exists(dest_dir):
- os.makedirs(dest_dir)
+ image_dir, image_rel_dir = make_image_dir(setup, rst_dir)
- # Copy unevaluated script
- try:
- shutil.copyfile(nb_abs_path, dest_path)
- except IOError:
- raise RuntimeError("Unable to copy notebook to build destination.")
+ # Ensure desination build directory exists
+ thread_safe_mkdir(os.path.dirname(dest_path))
+ # Copy unevaluated notebook
+ shutil.copyfile(nb_abs_path, dest_path)
+
+ # Construct paths to versions getting copied over
dest_path_eval = string.replace(dest_path, '.ipynb', '_evaluated.ipynb')
dest_path_script = string.replace(dest_path, '.ipynb', '.py')
rel_path_eval = string.replace(nb_basename, '.ipynb', '_evaluated.ipynb')
rel_path_script = string.replace(nb_basename, '.ipynb', '.py')
# Create python script vesion
- unevaluated_text = nb_to_html(nb_abs_path)
script_text = nb_to_python(nb_abs_path)
f = open(dest_path_script, 'w')
f.write(script_text.encode('utf8'))
@@ -64,8 +71,11 @@
skip_exceptions = 'skip_exceptions' in self.options
- evaluated_text = evaluate_notebook(nb_abs_path, dest_path_eval,
- skip_exceptions=skip_exceptions)
+ evaluated_text, resources = evaluate_notebook(
+ nb_abs_path, dest_path_eval, skip_exceptions=skip_exceptions)
+
+ evaluated_text = write_notebook_output(
+ resources, image_dir, image_rel_dir, evaluated_text)
# Create link to notebook and script files
link_rst = "(" + \
@@ -85,12 +95,9 @@
# add dependency
self.state.document.settings.record_dependencies.add(nb_abs_path)
- # clean up png files left behind by notebooks.
- png_files = glob.glob("*.png")
- fits_files = glob.glob("*.fits")
- h5_files = glob.glob("*.h5")
- for file in png_files:
- os.remove(file)
+ # clean up
+ os.chdir(cwd)
+ shutil.rmtree(tmpdir, True)
return [nb_node]
@@ -106,14 +113,18 @@
def nb_to_html(nb_path):
"""convert notebook to html"""
- exporter = html.HTMLExporter(template_file='full')
- output, resources = exporter.from_filename(nb_path)
+ c = Config({'ExtractOutputPreprocessor':{'enabled':True}})
+
+ exporter = html.HTMLExporter(template_file='full', config=c)
+ notebook = nbformat.read(open(nb_path), 'json')
+ output, resources = exporter.from_notebook_node(notebook)
header = output.split('<head>', 1)[1].split('</head>',1)[0]
body = output.split('<body>', 1)[1].split('</body>',1)[0]
# http://imgur.com/eR9bMRH
header = header.replace('<style', '<style scoped="scoped"')
- header = header.replace('body {\n overflow: visible;\n padding: 8px;\n}\n', '')
+ header = header.replace('body {\n overflow: visible;\n padding: 8px;\n}\n',
+ '')
header = header.replace("code,pre{", "code{")
# Filter out styles that conflict with the sphinx theme.
@@ -124,17 +135,19 @@
'uneditable-input{',
'collapse{',
]
+
filter_strings.extend(['h%s{' % (i+1) for i in range(6)])
- line_begin_strings = [
+ line_begin = [
'pre{',
'p{margin'
- ]
+ ]
- header_lines = filter(
- lambda x: not any([s in x for s in filter_strings]), header.split('\n'))
- header_lines = filter(
- lambda x: not any([x.startswith(s) for s in line_begin_strings]), header_lines)
+ filterfunc = lambda x: not any([s in x for s in filter_strings])
+ header_lines = filter(filterfunc, header.split('\n'))
+
+ filterfunc = lambda x: not any([x.startswith(s) for s in line_begin])
+ header_lines = filter(filterfunc, header_lines)
header = '\n'.join(header_lines)
@@ -143,13 +156,11 @@
lines.append(header)
lines.append(body)
lines.append('</div>')
- return '\n'.join(lines)
+ return '\n'.join(lines), resources
def evaluate_notebook(nb_path, dest_path=None, skip_exceptions=False):
# Create evaluated version and save it to the dest path.
- # Always use --pylab so figures appear inline
- # perhaps this is questionable?
- notebook = read(open(nb_path), 'json')
+ notebook = nbformat.read(open(nb_path), 'json')
nb_runner = NotebookRunner(notebook, pylab=False)
try:
nb_runner.run_notebook(skip_exceptions=skip_exceptions)
@@ -158,11 +169,14 @@
print e
# Return the traceback, filtering out ANSI color codes.
# http://stackoverflow.com/questions/13506033/filtering-out-ansi-escape-sequences
- return 'Notebook conversion failed with the following traceback: \n%s' % \
- re.sub(r'\\033[\[\]]([0-9]{1,2}([;@][0-9]{0,2})*)*[mKP]?', '', str(e))
+ return "Notebook conversion failed with the " \
+ "following traceback: \n%s" % \
+ re.sub(r'\\033[\[\]]([0-9]{1,2}([;@][0-9]{0,2})*)*[mKP]?', '',
+ str(e))
+
if dest_path is None:
dest_path = 'temp_evaluated.ipynb'
- write(nb_runner.nb, open(dest_path, 'w'), 'json')
+ nbformat.write(nb_runner.nb, open(dest_path, 'w'), 'json')
ret = nb_to_html(dest_path)
if dest_path is 'temp_evaluated.ipynb':
os.remove(dest_path)
@@ -186,3 +200,37 @@
html=(visit_notebook_node, depart_notebook_node))
app.add_directive('notebook', NotebookDirective)
+
+ retdict = dict(
+ version='0.1',
+ parallel_read_safe=True,
+ parallel_write_safe=True
+ )
+
+ return retdict
+
+def make_image_dir(setup, rst_dir):
+ image_dir = setup.app.builder.outdir + os.path.sep + '_images'
+ rel_dir = os.path.relpath(setup.confdir, rst_dir)
+ image_rel_dir = rel_dir + os.path.sep + '_images'
+ thread_safe_mkdir(image_dir)
+ return image_dir, image_rel_dir
+
+def write_notebook_output(resources, image_dir, image_rel_dir, evaluated_text):
+ my_uuid = uuid.uuid4().hex
+
+ for output in resources['outputs']:
+ new_name = image_dir + os.path.sep + my_uuid + output
+ new_relative_name = image_rel_dir + os.path.sep + my_uuid + output
+ evaluated_text = evaluated_text.replace(output, new_relative_name)
+ with open(new_name, 'wb') as f:
+ f.write(resources['outputs'][output])
+ return evaluated_text
+
+def thread_safe_mkdir(dirname):
+ try:
+ os.makedirs(dirname)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+ pass
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/extensions/notebookcell_sphinxext.py
--- a/doc/extensions/notebookcell_sphinxext.py
+++ b/doc/extensions/notebookcell_sphinxext.py
@@ -1,14 +1,14 @@
-import os, shutil, string, glob, io
+import os
+import shutil
+import io
+import tempfile
from sphinx.util.compat import Directive
from docutils.parsers.rst import directives
-from IPython.nbconvert import html, python
from IPython.nbformat import current
-from runipy.notebook_runner import NotebookRunner
-from jinja2 import FileSystemLoader
from notebook_sphinxext import \
- notebook_node, nb_to_html, nb_to_python, \
- visit_notebook_node, depart_notebook_node, \
- evaluate_notebook
+ notebook_node, visit_notebook_node, depart_notebook_node, \
+ evaluate_notebook, make_image_dir, write_notebook_output
+
class NotebookCellDirective(Directive):
"""Insert an evaluated notebook cell into a document
@@ -19,13 +19,22 @@
required_arguments = 0
optional_arguments = 1
has_content = True
- option_spec = {'skip_exceptions' : directives.flag}
+ option_spec = {'skip_exceptions': directives.flag}
def run(self):
# check if raw html is supported
if not self.state.document.settings.raw_enabled:
raise self.warning('"%s" directive disabled.' % self.name)
+ cwd = os.getcwd()
+ tmpdir = tempfile.mkdtemp()
+ os.chdir(tmpdir)
+
+ rst_file = self.state_machine.document.attributes['source']
+ rst_dir = os.path.abspath(os.path.dirname(rst_file))
+
+ image_dir, image_rel_dir = make_image_dir(setup, rst_dir)
+
# Construct notebook from cell content
content = "\n".join(self.content)
with open("temp.py", "w") as f:
@@ -35,7 +44,11 @@
skip_exceptions = 'skip_exceptions' in self.options
- evaluated_text = evaluate_notebook('temp.ipynb', skip_exceptions=skip_exceptions)
+ evaluated_text, resources = evaluate_notebook(
+ 'temp.ipynb', skip_exceptions=skip_exceptions)
+
+ evaluated_text = write_notebook_output(
+ resources, image_dir, image_rel_dir, evaluated_text)
# create notebook node
attributes = {'format': 'html', 'source': 'nb_path'}
@@ -44,9 +57,8 @@
self.state_machine.get_source_and_line(self.lineno)
# clean up
- files = glob.glob("*.png") + ['temp.py', 'temp.ipynb']
- for file in files:
- os.remove(file)
+ os.chdir(cwd)
+ shutil.rmtree(tmpdir, True)
return [nb_node]
@@ -60,6 +72,14 @@
app.add_directive('notebook-cell', NotebookCellDirective)
+ retdict = dict(
+ version='0.1',
+ parallel_read_safe=True,
+ parallel_write_safe=True
+ )
+
+ return retdict
+
def convert_to_ipynb(py_file, ipynb_file):
with io.open(py_file, 'r', encoding='utf-8') as f:
notebook = current.reads(f.read(), format='py')
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/extensions/numpydocmod/numpydoc.py
--- a/doc/extensions/numpydocmod/numpydoc.py
+++ b/doc/extensions/numpydocmod/numpydoc.py
@@ -99,6 +99,15 @@
app.add_domain(NumpyPythonDomain)
app.add_domain(NumpyCDomain)
+ retdict = dict(
+ version='0.1',
+ parallel_read_safe=True,
+ parallel_write_safe=True
+ )
+
+ return retdict
+
+
#------------------------------------------------------------------------------
# Docstring-mangling domains
#------------------------------------------------------------------------------
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/extensions/pythonscript_sphinxext.py
--- a/doc/extensions/pythonscript_sphinxext.py
+++ b/doc/extensions/pythonscript_sphinxext.py
@@ -1,8 +1,13 @@
+import tempfile
+import os
+import glob
+import shutil
+import subprocess
+import uuid
from sphinx.util.compat import Directive
-from subprocess import Popen,PIPE
-from docutils.parsers.rst import directives
from docutils import nodes
-import os, glob, base64
+from notebook_sphinxext import make_image_dir
+
class PythonScriptDirective(Directive):
"""Execute an inline python script and display images.
@@ -17,6 +22,15 @@
has_content = True
def run(self):
+ cwd = os.getcwd()
+ tmpdir = tempfile.mkdtemp()
+ os.chdir(tmpdir)
+
+ rst_file = self.state_machine.document.attributes['source']
+ rst_dir = os.path.abspath(os.path.dirname(rst_file))
+
+ image_dir, image_rel_dir = make_image_dir(setup, rst_dir)
+
# Construct script from cell content
content = "\n".join(self.content)
with open("temp.py", "w") as f:
@@ -27,35 +41,44 @@
print content
print ""
- codeproc = Popen(['python', 'temp.py'], stdout=PIPE)
- out = codeproc.stdout.read()
+ subprocess.call(['python', 'temp.py'])
- images = sorted(glob.glob("*.png"))
- fns = []
text = ''
- for im in images:
- text += get_image_tag(im)
- os.remove(im)
-
- os.remove("temp.py")
+ for im in sorted(glob.glob("*.png")):
+ text += get_image_tag(im, image_dir, image_rel_dir)
code = content
- literal = nodes.literal_block(code,code)
+ literal = nodes.literal_block(code, code)
literal['language'] = 'python'
attributes = {'format': 'html'}
img_node = nodes.raw('', text, **attributes)
-
+
+ # clean up
+ os.chdir(cwd)
+ shutil.rmtree(tmpdir, True)
+
return [literal, img_node]
+
def setup(app):
app.add_directive('python-script', PythonScriptDirective)
setup.app = app
setup.config = app.config
setup.confdir = app.confdir
-def get_image_tag(filename):
- with open(filename, "rb") as image_file:
- encoded_string = base64.b64encode(image_file.read())
- return '<img src="data:image/png;base64,%s" width="600"><br>' % encoded_string
+ retdict = dict(
+ version='0.1',
+ parallel_read_safe=True,
+ parallel_write_safe=True
+ )
+
+ return retdict
+
+
+def get_image_tag(filename, image_dir, image_rel_dir):
+ my_uuid = uuid.uuid4().hex
+ shutil.move(filename, image_dir + os.path.sep + my_uuid + filename)
+ relative_filename = image_rel_dir + os.path.sep + my_uuid + filename
+ return '<img src="%s" width="600"><br>' % relative_filename
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/extensions/youtube.py
--- a/doc/extensions/youtube.py
+++ /dev/null
@@ -1,43 +0,0 @@
-# http://countergram.com/youtube-in-rst
-
-from docutils import nodes
-from docutils.parsers.rst import directives
-
-CODE = """\
-<object type="application/x-shockwave-flash"
- width="%(width)s"
- height="%(height)s"
- class="youtube-embed"
- data="http://www.youtube.com/v/%(yid)s">
- <param name="movie" value="http://www.youtube.com/v/%(yid)s"></param>
- <param name="wmode" value="transparent"></param>%(extra)s
-</object>
-"""
-
-PARAM = """\n <param name="%s" value="%s"></param>"""
-
-def youtube(name, args, options, content, lineno,
- contentOffset, blockText, state, stateMachine):
- """ Restructured text extension for inserting youtube embedded videos """
- if len(content) == 0:
- return
- string_vars = {
- 'yid': content[0],
- 'width': 425,
- 'height': 344,
- 'extra': ''
- }
- extra_args = content[1:] # Because content[0] is ID
- extra_args = [ea.strip().split("=") for ea in extra_args] # key=value
- extra_args = [ea for ea in extra_args if len(ea) == 2] # drop bad lines
- extra_args = dict(extra_args)
- if 'width' in extra_args:
- string_vars['width'] = extra_args.pop('width')
- if 'height' in extra_args:
- string_vars['height'] = extra_args.pop('height')
- if extra_args:
- params = [PARAM % (key, extra_args[key]) for key in extra_args]
- string_vars['extra'] = "".join(params)
- return [nodes.raw('', CODE % (string_vars), format='html')]
-youtube.content = True
-directives.register_directive('youtube', youtube)
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/extensions/yt_colormaps.py
--- a/doc/extensions/yt_colormaps.py
+++ b/doc/extensions/yt_colormaps.py
@@ -15,6 +15,14 @@
setup.config = app.config
setup.confdir = app.confdir
+ retdict = dict(
+ version='0.1',
+ parallel_read_safe=True,
+ parallel_write_safe=True
+ )
+
+ return retdict
+
class ColormapScript(Directive):
required_arguments = 1
optional_arguments = 0
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/extensions/yt_cookbook.py
--- a/doc/extensions/yt_cookbook.py
+++ b/doc/extensions/yt_cookbook.py
@@ -15,6 +15,14 @@
setup.config = app.config
setup.confdir = app.confdir
+ retdict = dict(
+ version='0.1',
+ parallel_read_safe=True,
+ parallel_write_safe=True
+ )
+
+ return retdict
+
data_patterns = ["*.h5", "*.out", "*.dat"]
class CookbookScript(Directive):
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/helper_scripts/show_fields.py
--- a/doc/helper_scripts/show_fields.py
+++ b/doc/helper_scripts/show_fields.py
@@ -126,7 +126,7 @@
if in_cgs:
unit_object = unit_object.get_cgs_equivalent()
latex = unit_object.latex_representation()
- return latex.replace('\/','~')
+ return latex.replace('\/', '~')
def print_all_fields(fl):
for fn in sorted(fl):
@@ -204,6 +204,10 @@
field_info_names.append("CastroFieldInfo")
else:
field_info_names.append("BoxlibFieldInfo")
+ elif frontend == "chombo":
+ # remove low dimensional field info containters for ChomboPIC
+ field_info_names = [f for f in field_info_names if '1D' not in f
+ and '2D' not in f]
for dset_name, fi_name in zip(dataset_names, field_info_names):
fi = getattr(this_f, fi_name)
@@ -271,5 +275,6 @@
dp=f.dname, dw=len_disp)
print div
+ print ""
print footer
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/how_to_develop_yt.txt
--- a/doc/how_to_develop_yt.txt
+++ /dev/null
@@ -1,157 +0,0 @@
-How To Develop yt
-=================
-
-We are very happy to accept patches, features, and bugfixes from any member of
-the community! yt is developed using mercurial, primarily because it enables
-very easy and straightforward submission of changesets. We're eager to hear
-from you, and if you are developing yt, we encourage you to subscribe to the
-developer mailing list:
-
-http://lists.spacepope.org/listinfo.cgi/yt-dev-spacepope.org
-
-Please feel free to hack around, commit changes, and send them upstream. If
-you're new to Mercurial, these three resources are pretty great for learning
-the ins and outs:
-
- * http://hginit.com
- * http://hgbook.red-bean.com/read/
- * http://mercurial.selenic.com/
-
-Keep in touch, and happy hacking! We also provide doc/coding_styleguide.txt
-and an example of a fiducial docstring in doc/docstring_example.txt. Please
-read them before hacking on the codebase, and feel free to email any of the
-mailing lists for help with the codebase.
-
-Licenses
---------
-
-All code in yt should be under the BSD 3-clause license.
-
-How To Get The Source Code
---------------------------
-
-yt is hosted on BitBucket, and you can see all of the yt repositories at
-http://hg.yt-project.org/ . With the yt installation script you should have a
-copy of Mercurial. You can clone the repository like so:
-
- $ hg clone http://hg.yt-project.org/yt/
-
-You can update to any branch or revision by executing the command:
-
- $ hg up -C some_revision_specifier
-
-Specifying a branch name in the revspec will update to the latest revision on
-that branch. If you ran the installation script, you can tell Python to use a
-different version of the library by executing:
-
- $ python2.6 setup.py develop
-
-This will rebuild all C modules as well.
-
-How To Submit Changes
----------------------
-
-You can submit changes a couple different ways, but the easiest is to use the
-"fork" mechanism on BitBucket. Just go here:
-
-http://hg.yt-project.org/yt/fork
-
-and you're all set, ready to go. You'll have to either clone a new copy of the
-repository or edit .hg/hgrc to point to the location of your new fork, first,
-though.
-
-When you're ready to submit them to the main repository, simply go to:
-
-http://hg.yt-project.org/yt/fork
-
-Make sure you notify "yt_analysis" and put in a little description. That'll
-notify the core developers that you've got something ready to submit, and we
-will review it an (hopefully!) merge it in. If it goes well, you may end up
-with push access to the main repository.
-
-How To Read The Source Code
----------------------------
-
-yt is organized into several sub-packages, each of which governs a different
-conceptual regime.
-
- frontends
- This is where interfaces to codes are created. Within each subdirectory of
- yt/frontends/ there must exist the following files, even if empty:
-
- * data_structures.py, where subclasses of AMRGridPatch, Dataset and
- GridIndex are defined.
- * io.py, where a subclass of IOHandler is defined.
- * misc.py, where any miscellaneous functions or classes are defined.
- * definitions.py, where any definitions specific to the frontend are
- defined. (i.e., header formats, etc.)
-
- visualization
- This is where all visualization modules are stored. This includes plot
- collections, the volume rendering interface, and pixelization frontends.
-
- data_objects
- All objects that handle data, processed or unprocessed, not explicitly
- defined as visualization are located in here. This includes the base
- classes for data regions, covering grids, time series, and so on. This
- also includes derived fields and derived quantities.
-
- astro_objects
- This is where all objects that represent astrophysical objects should
- live -- for instance, galaxies, halos, disks, clusters and so on. These
- can be expressive, provide astrophysical analysis functionality and will
- in general be more user-modifiable, user-tweakable, and much less of a
- black box that data_objects.
-
- analysis_modules
- This is where all mechanisms for processing data live. This includes
- things like clump finding, halo profiling, halo finding, and so on. This
- is something of a catchall, but it serves as a level of greater
- abstraction that simply data selection and modification.
-
- gui
- This is where all GUI components go. Typically this will be some small
- tool used for one or two things, which contains a launching mechanism on
- the command line.
-
- utilities
- All broadly useful code that doesn't clearly fit in one of the other
- categories goes here.
-
-How To Use Branching
---------------------
-
-If you are planning on making a large change to the code base that may not be
-ready for many commits, or if you are planning on breaking some functionality
-and rewriting it, you are encouraged to create a new named branch. You can
-mark the current repository as a new named branch by executing:
-
- $ hg branch new_feature_name
-
-The next commit and all subsequent commits will be contained within that named
-branch. At this point, add your branch here:
-
-http://yt-project.org/wiki/ExistingBranches
-
-To merge changes in from another branch, you would execute:
-
- $ hg merge some_other_branch
-
-Note also that you can use revision specifiers instead of "some_other_branch".
-When you are ready to merge back into the main branch, execute this process:
-
- $ hg merge name_of_main_branch
- $ hg commit --close-branch
- $ hg up -C name_of_main_branch
- $ hg merge name_of_feature_branch
- $ hg commit
-
-When you execute the merge you may have to resolve conflicts. Once you resolve
-conflicts in a file, you can mark it as resolved by doing:
-
- $ hg resolve -m path/to/conflicting/file.py
-
-Please be careful when resolving conflicts in files.
-
-Once your branch has been merged in, mark it as closed on the wiki page.
-
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/install_script.sh
--- a/doc/install_script.sh
+++ b/doc/install_script.sh
@@ -264,8 +264,8 @@
echo "Alternatively, download the Xcode command line tools from"
echo "the Apple developer tools website."
echo
- echo "OS X 10.8.4 and 10.9: download Xcode 5.02 from the mac app store."
- echo "(search for Xcode)."
+ echo "OS X 10.8.4, 10.9, and 10.10: download the appropriate version of"
+ echo "Xcode from the mac app store (search for Xcode)."
echo
echo "Additionally, you will have to manually install the Xcode"
echo "command line tools."
@@ -273,7 +273,7 @@
echo "For OS X 10.8, see:"
echo "http://stackoverflow.com/questions/9353444"
echo
- echo "For OS X 10.9, the command line tools can be installed"
+ echo "For OS X 10.9 and 10.10, the command line tools can be installed"
echo "with the following command:"
echo " xcode-select --install"
echo
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/source/_static/custom.css
--- a/doc/source/_static/custom.css
+++ b/doc/source/_static/custom.css
@@ -99,3 +99,13 @@
height: 45px;
visibility: hidden;
}
+
+/*
+
+Make tables span only half the page.
+
+*/
+
+.table {
+ width: 50%
+}
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/source/about/index.rst
--- /dev/null
+++ b/doc/source/about/index.rst
@@ -0,0 +1,88 @@
+.. _aboutyt:
+
+About yt
+========
+
+.. contents::
+ :depth: 1
+ :local:
+ :backlinks: none
+
+What is yt?
+-----------
+
+yt is a toolkit for analyzing and visualizing quantitative data. Originally
+written to analyze 3D grid-based astrophysical simulation data,
+it has grown to handle any kind of data represented in a 2D or 3D volume.
+yt is an Python-based open source project and is open for anyone to use or
+contribute code. The entire source code and history is available to all
+at https://bitbucket.org/yt_analysis/yt .
+
+.. _who-is-yt:
+
+Who is yt?
+----------
+
+As an open-source project, yt has a large number of user-developers.
+In September of 2014, the yt developer community collectively decided to endow
+the title of *member* on individuals who had contributed in a significant way
+to the project. For a list of those members and a description of their
+contributions to the code, see
+`our members website. <http://yt-project.org/members.html>`_
+
+For an up-to-date list of everyone who has contributed to the yt codebase,
+see the current `CREDITS <http://hg.yt-project.org/yt/src/yt/CREDITS>`_ file.
+For a more detailed breakup of contributions made by individual users, see out
+`Open HUB page <https://www.openhub.net/p/yt_amr/contributors?query=&sort=commits>`_.
+
+History of yt
+-------------
+
+yt was originally begun by Matthew Turk in 2007 in the course of his graduate
+studies in computational astrophysics. The code was developed
+as a simple data-reader and exporter for grid-based hydrodynamical simulation
+data outputs from the *Enzo* code. Over the next few years, he invited
+collaborators and friends to contribute and use yt. As the community grew,
+so did the capabilities of yt. It is now a community-developed project with
+contributions from many people, the hospitality of several institutions, and
+benefiting from numerous grants. With this community-driven approach
+and contributions from a sizeable population of developers, it has evolved
+into a fully-featured toolkit for analysis and visualization of
+multidimensional data. It relies on no proprietary software -- although it
+can be and has been extended to interface with proprietary software and
+libraries -- and has been designed from the ground up to enable users to be
+as immersed in the data as they desire.
+
+How do I contact yt?
+--------------------
+
+If you have any questions about the code, please contact the `yt users email
+list <http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org>`_. If
+you're having other problems, please follow the steps in
+:ref:`asking-for-help`.
+
+How do I cite yt?
+-----------------
+
+If you use yt in a publication, we'd very much appreciate a citation! You
+should feel free to cite the `ApJS paper
+<http://adsabs.harvard.edu/abs/2011ApJS..192....9T>`_ with the following BibTeX
+entry: ::
+
+ @ARTICLE{2011ApJS..192....9T,
+ author = {{Turk}, M.~J. and {Smith}, B.~D. and {Oishi}, J.~S. and {Skory}, S. and
+ {Skillman}, S.~W. and {Abel}, T. and {Norman}, M.~L.},
+ title = "{yt: A Multi-code Analysis Toolkit for Astrophysical Simulation Data}",
+ journal = {\apjs},
+ archivePrefix = "arXiv",
+ eprint = {1011.3514},
+ primaryClass = "astro-ph.IM",
+ keywords = {cosmology: theory, methods: data analysis, methods: numerical },
+ year = 2011,
+ month = jan,
+ volume = 192,
+ pages = {9-+},
+ doi = {10.1088/0067-0049/192/1/9},
+ adsurl = {http://adsabs.harvard.edu/abs/2011ApJS..192....9T},
+ adsnote = {Provided by the SAO/NASA Astrophysics Data System}
+ }
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/source/analyzing/analysis_modules/halo_finders.rst
--- a/doc/source/analyzing/analysis_modules/halo_finders.rst
+++ b/doc/source/analyzing/analysis_modules/halo_finders.rst
@@ -68,15 +68,17 @@
but also `separately available <http://code.google.com/p/rockstar>`_. The lead
developer is Peter Behroozi, and the methods are described in `Behroozi
et al. 2011 <http://rockstar.googlecode.com/files/rockstar_ap101911.pdf>`_.
+In order to run the Rockstar halo finder in yt, make sure you've
+:ref:`installed it so that it can integrate with yt <rockstar-installation>`.
-.. note:: At the moment, Rockstar does not support multiple particle masses,
- instead using a fixed particle mass. This will not affect most dark matter
- simulations, but does make it less useful for finding halos from the stellar
- mass. In simulations where the highest-resolution particles all have the
- same mass (ie: zoom-in grid based simulations), one can set up a particle
- filter to select the lowest mass particles and perform the halo finding
- only on those. See the this cookbook recipe for an example:
- :ref:`cookbook-rockstar-nested-grid`.
+At the moment, Rockstar does not support multiple particle masses,
+instead using a fixed particle mass. This will not affect most dark matter
+simulations, but does make it less useful for finding halos from the stellar
+mass. In simulations where the highest-resolution particles all have the
+same mass (ie: zoom-in grid based simulations), one can set up a particle
+filter to select the lowest mass particles and perform the halo finding
+only on those. See the this cookbook recipe for an example:
+:ref:`cookbook-rockstar-nested-grid`.
To run the Rockstar Halo finding, you must launch python with MPI and
parallelization enabled. While Rockstar itself does not require MPI to run,
@@ -201,11 +203,29 @@
halo finder at a few paddings to find the right amount, especially
if you're analyzing many similar datasets.
+.. _rockstar-installation:
+
Rockstar Installation
---------------------
-The Rockstar is slightly patched and modified to run as a library inside of
-yt. By default it will be built with yt using the ``install_script.sh``.
-If it wasn't installed, please make sure that the installation setting
-``INST_ROCKSTAR=1`` is defined in the ``install_script.sh`` and re-run
-the installation script.
+Because of changes in the Rockstar API over time, yt only currently works with
+a slightly older version of Rockstar. This version of Rockstar has been
+slightly patched and modified to run as a library inside of yt. By default it
+is not installed with yt, but installation is very easy. The
+:ref:`install-script` used to install yt from source has a line:
+``INST_ROCKSTAR=0`` that must be changed to ``INST_ROCKSTAR=1``. You can
+rerun this installer script over the top of an existing installation, and
+it will only install components missing from the existing installation.
+You can do this as follows. Put your freshly modified install_script in
+the parent directory of the yt installation directory (e.g. the parent of
+``$YT_DEST``, ``yt-x86_64``, ``yt-i386``, etc.), and rerun the installer:
+
+.. code-block:: bash
+
+ cd $YT_DEST
+ cd ..
+ vi install_script.sh // or your favorite editor to change INST_ROCKSTAR=1
+ bash < install_script.sh
+
+This will download Rockstar and install it as a library in yt. You should now
+be able to use Rockstar and yt together.
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/source/analyzing/analysis_modules/index.rst
--- a/doc/source/analyzing/analysis_modules/index.rst
+++ b/doc/source/analyzing/analysis_modules/index.rst
@@ -1,12 +1,13 @@
.. _analysis-modules:
-Analysis Modules
-================
+Topic-Specific Analysis Modules
+===============================
-These are "canned" analysis modules that can operate on datasets, performing a
-sequence of operations that result in a final result. This functionality
-interoperates with yt, but one needs to import the functions associated
-with each specific analysis module into python before using them.
+These semi-autonomous analysis modules are unique to specific subject matter
+like tracking halos, generating synthetic observations, exporting output to
+external visualization routines, and more. Because they are somewhat
+specialized, they exist in their own corners of yt, and they do not get loaded
+by default when you "import yt". Read up on these advanced tools below.
.. toctree::
:maxdepth: 2
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/source/analyzing/analysis_modules/radmc3d_export.rst
--- a/doc/source/analyzing/analysis_modules/radmc3d_export.rst
+++ b/doc/source/analyzing/analysis_modules/radmc3d_export.rst
@@ -6,17 +6,11 @@
.. sectionauthor:: Andrew Myers <atmyers2 at gmail.com>
.. versionadded:: 2.6
-.. note::
-
- As of :code:`yt-3.0`, the radial column density analysis module is not
- currently functional. This functionality is still available in
- :code:`yt-2.x`. If you would like to use these features in :code:`yt-3.x`,
- help is needed to port them over. Contact the yt-users mailing list if you
- are interested in doing this.
-
`RADMC-3D
-<http://www.ita.uni-heidelberg.de/~dullemond/software/radmc-3d/>`_ is a three-dimensional Monte-Carlo radiative transfer code
-that is capable of handling both line and continuum emission. The :class:`~yt.analysis_modules.radmc3d_export.RadMC3DInterface.RadMC3DWriter`
+<http://www.ita.uni-heidelberg.de/~dullemond/software/radmc-3d/>`_ is a
+three-dimensional Monte-Carlo radiative transfer code that is capable of
+handling both line and continuum emission. The
+:class:`~yt.analysis_modules.radmc3d_export.RadMC3DInterface.RadMC3DWriter`
class saves AMR data to disk in an ACSII format that RADMC-3D can read.
In principle, this allows one to use RADMC-3D to make synthetic observations
from any simulation data format that yt recognizes.
@@ -31,74 +25,77 @@
.. code-block:: python
- from yt.mods import *
- from yt.analysis_modules.radmc3d_export.api import *
+ import yt
+ from yt.analysis_modules.radmc3d_export.api import RadMC3DWriter
-Then, define a field that calculates the dust density in each cell. Here, we assume
-a constant dust-to-gas mass ratio of 0.01:
+Then, define a field that calculates the dust density in each cell. Here, we
+assume a constant dust-to-gas mass ratio of 0.01:
.. code-block:: python
dust_to_gas = 0.01
def _DustDensity(field, data):
- return dust_to_gas*data["density"]
- add_field("DustDensity", function=_DustDensity)
+ return dust_to_gas * data["density"]
+ yt.add_field(("gas", "dust_density"), function=_DustDensity, units="g/cm**3")
Now load up a dataset and call the
:class:`~yt.analysis_modules.radmc3d_export.RadMC3DInterface.RadMC3DWriter`:
.. code-block:: python
- ds = load("galaxy0030/galaxy0030")
+ ds = yt.load("IsolatedGalaxy/galaxy0030/galaxy0030")
writer = RadMC3DWriter(ds)
writer.write_amr_grid()
- writer.write_dust_file("DustDensity", "dust_density.inp")
+ writer.write_dust_file(("gas", "dust_density"), "dust_density.inp")
-The method write_amr_grid() creates an "amr_grid.inp" file that tells RADMC-3D how
-to interpret the rest of the data, while "dust_density.inp" contains the actual data field.
+The method write_amr_grid() creates an "amr_grid.inp" file that tells RADMC-3D
+how to interpret the rest of the data, while "dust_density.inp" contains the
+actual data field.
-We can also supply temperature information. The following code creates a "DustTemperature"
-field that is constant at 10 K, and saves it into a file called "dust_temperature.inp"
+We can also supply temperature information. The following code creates a
+"dust_temperature" field that is constant at 10 K, and saves it into a file
+called "dust_temperature.inp"
.. code-block:: python
def _DustTemperature(field, data):
- return 10.0*data["Ones"]
- add_field("DustTemperature", function=_DustTemperature)
+ return 0. * data["temperature"] + data.ds.quan(10, 'K')
+ yt.add_field(("gas", "dust_temperature"), function=_DustTemperature, units="K")
- writer.write_dust_file("DustTemperature", "dust_temperature.inp")
+ writer.write_dust_file(("gas", "dust_temperature"), "dust_temperature.inp")
-With the "amr_grid.inp", "dust_density.inp", and "dust_temperature.inp" files, RADMC-3D
-has everything it needs to compute the thermal dust emission (you may also have to include
-the location and spectra of any sources, which currently must be done manually).
-The result is something that looks like this:
+With the "amr_grid.inp", "dust_density.inp", and "dust_temperature.inp" files,
+RADMC-3D has everything it needs to compute the thermal dust emission (you may
+also have to include the location and spectra of any sources, which currently
+must be done manually). The result is something that looks like this:
.. image:: _images/31micron.png
Line Emission
-------------
-The file format required for line emission is slightly different. The following script will generate
-two files, one called "numderdens_co.inp", which contains the number density of CO molecules
-for every cell in the index, and another called "gas-velocity.inp", which is useful if you want
-to include doppler broadening.
+The file format required for line emission is slightly different. The
+following script will generate two files, one called "numderdens_co.inp",
+which contains the number density of CO molecules for every cell in the index,
+and another called "gas-velocity.inp", which is useful if you want to include
+doppler broadening.
.. code-block:: python
- from yt.mods import *
- from yt.analysis_modules.radmc3d_export.api import *
+ import yt
+ from yt.analysis_modules.radmc3d_export.api import RadMC3DWriter
x_co = 1.0e-4
mu_h = 2.34e-24
def _NumberDensityCO(field, data):
return (x_co/mu_h)*data["density"]
- add_field("NumberDensityCO", function=_NumberDensityCO)
+ yt.add_field(("gas", "number_density_CO"), function=_NumberDensityCO, units="cm**-3")
- ds = load("galaxy0030/galaxy0030")
+ ds = yt.load("IsolatedGalaxy/galaxy0030/galaxy0030")
writer = RadMC3DWriter(ds)
writer.write_amr_grid()
- writer.write_line_file("NumberDensityCO", "numberdens_co.inp")
+ writer.write_line_file(("gas", "number_density_CO"), "numberdens_co.inp")
velocity_fields = ["velocity_x", "velocity_y", "velocity_z"]
writer.write_line_file(velocity_fields, "gas_velocity.inp")
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/source/analyzing/analysis_modules/sunyaev_zeldovich.rst
--- a/doc/source/analyzing/analysis_modules/sunyaev_zeldovich.rst
+++ b/doc/source/analyzing/analysis_modules/sunyaev_zeldovich.rst
@@ -1,3 +1,5 @@
+.. _sunyaev-zeldovich:
+
Mock Observations of the Sunyaev-Zeldovich Effect
-------------------------------------------------
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/source/analyzing/analysis_modules/synthetic_observation.rst
--- a/doc/source/analyzing/analysis_modules/synthetic_observation.rst
+++ b/doc/source/analyzing/analysis_modules/synthetic_observation.rst
@@ -1,3 +1,5 @@
+.. _synthetic-observations:
+
Synthetic Observation
=====================
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/source/analyzing/external_analysis.rst
--- a/doc/source/analyzing/external_analysis.rst
+++ /dev/null
@@ -1,411 +0,0 @@
-Using yt with External Analysis Tools
-=====================================
-
-yt can be used as a ``glue`` code between simulation data and other methods of
-analyzing data. Its facilities for understanding units, disk IO and data
-selection set it up ideally to use other mechanisms for analyzing, processing
-and visualizing data.
-
-Calling External Python Codes
------------------------------
-
-Calling external Python codes very straightforward. For instance, if you had a
-Python code that accepted a set of structured meshes and then post-processed
-them to apply radiative feedback, one could imagine calling it directly:
-
-.. code-block:: python
-
- import yt
- import radtrans
-
- ds = yt.load("DD0010/DD0010")
- rt_grids = []
-
- for grid in ds.index.grids:
- rt_grid = radtrans.RegularBox(
- grid.LeftEdge, grid.RightEdge,
- grid["density"], grid["temperature"], grid["metallicity"])
- rt_grids.append(rt_grid)
- grid.clear_data()
-
- radtrans.process(rt_grids)
-
-Or if you wanted to run a population synthesis module on a set of star
-particles (and you could fit them all into memory) it might look something like
-this:
-
-.. code-block:: python
-
- import yt
- import pop_synthesis
-
- ds = yt.load("DD0010/DD0010")
- ad = ds.all_data()
- star_masses = ad["StarMassMsun"]
- star_metals = ad["StarMetals"]
-
- pop_synthesis.CalculateSED(star_masses, star_metals)
-
-If you have a code that's written in Python that you are having trouble getting
-data into from yt, please feel encouraged to email the users list and we'll
-help out.
-
-Calling Non-Python External Codes
----------------------------------
-
-Independent of its ability to process, analyze and visualize data, yt can also
-serve as a mechanism for reading and selecting simulation data. In this way,
-it can be used to supply data to an external analysis routine written in
-Fortran, C or C++. This document describes how to supply that data, using the
-example of a simple code that calculates the best axes that describe a
-distribution of particles as a starting point. (The underlying method is left
-as an exercise for the reader; we're only currently interested in the function
-specification and structs.)
-
-If you have written a piece of code that performs some analysis function, and
-you would like to include it in the base distribution of yt, we would be happy
-to do so; drop us a line or see :ref:`contributing-code` for more information.
-
-To accomplish the process of linking Python with our external code, we will be
-using a language called `Cython <http://www.cython.org/>`_, which is
-essentially a superset of Python that compiles down to C. It is aware of NumPy
-arrays, and it is able to massage data between the interpreted language Python
-and C, Fortran or C++. It will be much easier to utilize routines and analysis
-code that have been separated into subroutines that accept data structures, so
-we will assume that our halo axis calculator accepts a set of structs.
-
-Our Example Code
-++++++++++++++++
-
-Here is the ``axes.h`` file in our imaginary code, which we will then wrap:
-
-.. code-block:: c
-
- typedef struct structParticleCollection {
- long npart;
- double *xpos;
- double *ypos;
- double *zpos;
- } ParticleCollection;
-
- void calculate_axes(ParticleCollection *part,
- double *ax1, double *ax2, double *ax3);
-
-There are several components to this analysis routine which we will have to
-wrap.
-
-#. We have to wrap the creation of an instance of ``ParticleCollection``.
-#. We have to transform a set of NumPy arrays into pointers to doubles.
-#. We have to create a set of doubles into which ``calculate_axes`` will be
- placing the values of the axes it calculates.
-#. We have to turn the return values back into Python objects.
-
-Each of these steps can be handled in turn, and we'll be doing it using Cython
-as our interface code.
-
-Setting Up and Building Our Wrapper
-+++++++++++++++++++++++++++++++++++
-
-To get started, we'll need to create two files:
-
-.. code-block:: bash
-
- axes_calculator.pyx
- axes_calculator_setup.py
-
-These can go anywhere, but it might be useful to put them in their own
-directory. The contents of ``axes_calculator.pyx`` will be left for the next
-section, but we will need to put some boilerplate code into
-``axes_calculator_setup.pyx``. As a quick sidenote, you should call these
-whatever is most appropriate for the external code you are wrapping;
-``axes_calculator`` is probably not the best bet.
-
-Here's a rough outline of what should go in ``axes_calculator_setup.py``:
-
-.. code-block:: python
-
- NAME = "axes_calculator"
- EXT_SOURCES = []
- EXT_LIBRARIES = ["axes_utils", "m"]
- EXT_LIBRARY_DIRS = ["/home/rincewind/axes_calculator/"]
- EXT_INCLUDE_DIRS = []
- DEFINES = []
-
- from distutils.core import setup
- from distutils.extension import Extension
- from Cython.Distutils import build_ext
-
- ext_modules = [Extension(NAME,
- [NAME+".pyx"] + EXT_SOURCES,
- libraries = EXT_LIBRARIES,
- library_dirs = EXT_LIBRARY_DIRS,
- include_dirs = EXT_INCLUDE_DIRS,
- define_macros = DEFINES)
- ]
-
- setup(
- name = NAME,
- cmdclass = {'build_ext': build_ext},
- ext_modules = ext_modules
- )
-
-The only variables you should have to change in this are the first six, and
-possibly only the first one. We'll go through these variables one at a time.
-
-``NAME``
- This is the name of our source file, minus the ``.pyx``. We're also
- mandating that it be the name of the module we import. You're free to
- modify this.
-``EXT_SOURCES``
- Any additional sources can be listed here. For instance, if you are only
- linking against a single ``.c`` file, you could list it here -- if our axes
- calculator were fully contained within a file called ``calculate_my_axes.c``
- we could link against it using this variable, and then we would not have to
- specify any libraries. This is usually the simplest way to do things, and in
- fact, yt makes use of this itself for things like HEALPix and interpolation
- functions.
-``EXT_LIBRARIES``
- Any libraries that will need to be linked against (like ``m``!) should be
- listed here. Note that these are the name of the library minus the leading
- ``lib`` and without the trailing ``.so``. So ``libm.so`` would become ``m``
- and ``libluggage.so`` would become ``luggage``.
-``EXT_LIBRARY_DIRS``
- If the libraries listed in ``EXT_LIBRARIES`` reside in some other directory
- or directories, those directories should be listed here. For instance,
- ``["/usr/local/lib", "/home/rincewind/luggage/"]`` .
-``EXT_INCLUDE_DIRS``
- If any header files have been included that live in external directories,
- those directories should be included here.
-``DEFINES``
- Any define macros that should be passed to the C compiler should be listed
- here; if they just need to be defined, then they should be specified to be
- defined as "None." For instance, if you wanted to pass ``-DTWOFLOWER``, you
- would set this to equal: ``[("TWOFLOWER", None)]``.
-
-To build our extension, we would run:
-
-.. code-block:: bash
-
- $ python2.7 axes_calculator_setup.py build_ext -i
-
-Note that since we don't yet have an ``axes_calculator.pyx``, this will fail.
-But once we have it, it ought to run.
-
-Writing and Calling our Wrapper
-+++++++++++++++++++++++++++++++
-
-Now we begin the tricky part, of writing our wrapper code. We've already
-figured out how to build it, which is halfway to being able to test that it
-works, and we now need to start writing Cython code.
-
-For a more detailed introduction to Cython, see the Cython documentation at
-http://docs.cython.org/ . We'll cover a few of the basics for wrapping code
-however.
-
-To start out with, we need to open up and edit our file,
-``axes_calculator.pyx``. Open this in your favorite version of vi (mine is
-vim) and we will get started by declaring the struct we need to pass in. But
-first, we need to include some header information:
-
-.. code-block:: cython
-
- import numpy as np
- cimport numpy as np
- cimport cython
- from stdlib cimport malloc, free
-
-These lines simply import and "Cython import" some common routines. For more
-information about what is already available, see the Cython documentation. For
-now, we need to start translating our data.
-
-To do so, we tell Cython both where the struct should come from, and then we
-describe the struct itself. One fun thing to note is that if you don't need to
-set or access all the values in a struct, and it just needs to be passed around
-opaquely, you don't have to include them in the definition. For an example of
-this, see the ``png_writer.pyx`` file in the yt repository. Here's the syntax
-for pulling in (from a file called ``axes_calculator.h``) a struct like the one
-described above:
-
-.. code-block:: cython
-
- cdef extern from "axes_calculator.h":
- ctypedef struct ParticleCollection:
- long npart
- double *xpos
- double *ypos
- double *zpos
-
-So far, pretty easy! We've basically just translated the declaration from the
-``.h`` file. Now that we have done so, any other Cython code can create and
-manipulate these ``ParticleCollection`` structs -- which we'll do shortly.
-Next up, we need to declare the function we're going to call, which looks
-nearly exactly like the one in the ``.h`` file. (One common problem is that
-Cython doesn't know what ``const`` means, so just remove it wherever you see
-it.) Declare it like so:
-
-.. code-block:: cython
-
- void calculate_axes(ParticleCollection *part,
- double *ax1, double *ax2, double *ax3)
-
-Note that this is indented one level, to indicate that it, too, comes from
-``axes_calculator.h``. The next step is to create a function that accepts
-arrays and converts them to the format the struct likes. We declare our
-function just like we would a normal Python function, using ``def``. You can
-also use ``cdef`` if you only want to call a function from within Cython. We
-want to call it from Python, too, so we just use ``def``. Note that we don't
-here specify types for the various arguments. In a moment we'll refine this to
-have better argument types.
-
-.. code-block:: cython
-
- def examine_axes(xpos, ypos, zpos):
- cdef double ax1[3], ax2[3], ax3[3]
- cdef ParticleCollection particles
- cdef int i
-
- particles.npart = len(xpos)
- particles.xpos = <double *> malloc(particles.npart * sizeof(double))
- particles.ypos = <double *> malloc(particles.npart * sizeof(double))
- particles.zpos = <double *> malloc(particles.npart * sizeof(double))
-
- for i in range(particles.npart):
- particles.xpos[i] = xpos[i]
- particles.ypos[i] = ypos[i]
- particles.zpos[i] = zpos[i]
-
- calculate_axes(&particles, ax1, ax2, ax3)
-
- free(particles.xpos)
- free(particles.ypos)
- free(particles.zpos)
-
- return ( (ax1[0], ax1[1], ax1[2]),
- (ax2[0], ax2[1], ax2[2]),
- (ax3[0], ax3[1], ax3[2]) )
-
-This does the rest. Note that we've weaved in C-type declarations (ax1, ax2,
-ax3) and Python access to the variables fed in. This function will probably be
-quite slow -- because it doesn't know anything about the variables xpos, ypos,
-zpos, it won't be able to speed up access to them. Now we will see what we can
-do by declaring them to be of array-type before we start handling them at all.
-We can do that by annotating in the function argument list. But first, let's
-test that it works. From the directory in which you placed these files, run:
-
-.. code-block:: bash
-
- $ python2.6 setup.py build_ext -i
-
-Now, create a sample file that feeds in the particles:
-
-.. code-block:: python
-
- import axes_calculator
- axes_calculator.examine_axes(xpos, ypos, zpos)
-
-Most of the time in that function is spent in converting the data. So now we
-can go back and we'll try again, rewriting our converter function to believe
-that its being fed arrays from NumPy:
-
-.. code-block:: cython
-
- def examine_axes(np.ndarray[np.float64_t, ndim=1] xpos,
- np.ndarray[np.float64_t, ndim=1] ypos,
- np.ndarray[np.float64_t, ndim=1] zpos):
- cdef double ax1[3], ax2[3], ax3[3]
- cdef ParticleCollection particles
- cdef int i
-
- particles.npart = len(xpos)
- particles.xpos = <double *> malloc(particles.npart * sizeof(double))
- particles.ypos = <double *> malloc(particles.npart * sizeof(double))
- particles.zpos = <double *> malloc(particles.npart * sizeof(double))
-
- for i in range(particles.npart):
- particles.xpos[i] = xpos[i]
- particles.ypos[i] = ypos[i]
- particles.zpos[i] = zpos[i]
-
- calculate_axes(&particles, ax1, ax2, ax3)
-
- free(particles.xpos)
- free(particles.ypos)
- free(particles.zpos)
-
- return ( (ax1[0], ax1[1], ax1[2]),
- (ax2[0], ax2[1], ax2[2]),
- (ax3[0], ax3[1], ax3[2]) )
-
-This should be substantially faster, assuming you feed it arrays.
-
-Now, there's one last thing we can try. If we know our function won't modify
-our arrays, and they are C-Contiguous, we can simply grab pointers to the data:
-
-.. code-block:: cython
-
- def examine_axes(np.ndarray[np.float64_t, ndim=1] xpos,
- np.ndarray[np.float64_t, ndim=1] ypos,
- np.ndarray[np.float64_t, ndim=1] zpos):
- cdef double ax1[3], ax2[3], ax3[3]
- cdef ParticleCollection particles
- cdef int i
-
- particles.npart = len(xpos)
- particles.xpos = <double *> xpos.data
- particles.ypos = <double *> ypos.data
- particles.zpos = <double *> zpos.data
-
- for i in range(particles.npart):
- particles.xpos[i] = xpos[i]
- particles.ypos[i] = ypos[i]
- particles.zpos[i] = zpos[i]
-
- calculate_axes(&particles, ax1, ax2, ax3)
-
- return ( (ax1[0], ax1[1], ax1[2]),
- (ax2[0], ax2[1], ax2[2]),
- (ax3[0], ax3[1], ax3[2]) )
-
-But note! This will break or do weird things if you feed it arrays that are
-non-contiguous.
-
-At this point, you should have a mostly working piece of wrapper code. And it
-was pretty easy! Let us know if you run into any problems, or if you are
-interested in distributing your code with yt.
-
-A complete set of files is available with this documentation. These are
-slightly different, so that the whole thing will simply compile, but they
-provide a useful example.
-
- * `axes.c <../_static/axes.c>`_
- * `axes.h <../_static/axes.h>`_
- * `axes_calculator.pyx <../_static/axes_calculator.pyx>`_
- * `axes_calculator_setup.py <../_static/axes_calculator_setup.txt>`_
-
-Exporting Data from yt
-----------------------
-
-yt is installed alongside h5py. If you need to export your data from yt, to
-share it with people or to use it inside another code, h5py is a good way to do
-so. You can write out complete datasets with just a few commands. You have to
-import, and then save things out into a file.
-
-.. code-block:: python
-
- import h5py
- f = h5py.File("some_file.h5")
- f.create_dataset("/data", data=some_data)
-
-This will create ``some_file.h5`` if necessary and add a new dataset
-(``/data``) to it. Writing out in ASCII should be relatively straightforward.
-For instance:
-
-.. code-block:: python
-
- f = open("my_file.txt", "w")
- for halo in halos:
- x, y, z = halo.center_of_mass()
- f.write("%0.2f %0.2f %0.2f\n", x, y, z)
- f.close()
-
-This example could be extended to work with any data object's fields, as well.
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/source/analyzing/filtering.rst
--- a/doc/source/analyzing/filtering.rst
+++ b/doc/source/analyzing/filtering.rst
@@ -201,7 +201,7 @@
prj.show()
- slc = yt.SlicePlot(ds, "x", "density", center=c, width=(50, "Mpc"),
+ slc = yt.SlicePlot(ds, "x", "density", center=center, width=(50, "Mpc"),
data_source=sp)
slc.show()
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/source/analyzing/index.rst
--- a/doc/source/analyzing/index.rst
+++ b/doc/source/analyzing/index.rst
@@ -1,7 +1,15 @@
.. _analyzing:
-Analyzing Data
-==============
+General Data Analysis
+=====================
+
+This documentation describes much of the yt infrastructure for manipulating
+one's data to extract the relevant information. Fields, data objects, and
+units are at the heart of how yt represents data. Beyond this, we provide
+a full description for how to filter your datasets based on specific criteria,
+how to analyze chronological datasets from the same underlying simulation or
+source (i.e. time series analysis), and how to run yt in parallel on
+multiple processors to accomplish tasks faster.
.. toctree::
:maxdepth: 2
@@ -13,5 +21,3 @@
generating_processed_data
time_series_analysis
parallel_computation
- analysis_modules/index
- external_analysis
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/source/analyzing/objects.rst
--- a/doc/source/analyzing/objects.rst
+++ b/doc/source/analyzing/objects.rst
@@ -83,6 +83,8 @@
If you want to create your own custom data object type, see
:ref:`creating-objects`.
+.. _geometric-objects:
+
Geometric Objects
^^^^^^^^^^^^^^^^^
@@ -107,12 +109,15 @@
| Usage: ``ortho_ray(axis, coord, ds=None, field_parameters=None, data_source=None)``
| A line (of data cells) stretching through the full domain
aligned with one of the x,y,z axes. Defined by an axis and a point
- to be intersected.
+ to be intersected. Please see this
+ :ref:`note about ray data value ordering <ray-data-ordering>`.
**Ray (Arbitrarily-Aligned)**
| Class :class:`~yt.data_objects.selection_data_containers.YTRayBase`
| Usage: ``ray(start_coord, end_coord, ds=None, field_parameters=None, data_source=None)``
| A line (of data cells) defined by arbitrary start and end coordinates.
+ Please see this
+ :ref:`note about ray data value ordering <ray-data-ordering>`.
2D Objects
""""""""""
@@ -170,6 +175,7 @@
| Usage: ``sphere(center, radius, ds=None, field_parameters=None, data_source=None)``
| A sphere defined by a central coordinate and a radius.
+.. _collection-objects:
Filtering and Collection Objects
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -202,6 +208,8 @@
| A ``data_collection`` is a list of data objects that can be
sampled and processed as a whole in a single data object.
+.. _construction-objects:
+
Construction Objects
^^^^^^^^^^^^^^^^^^^^
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/source/analyzing/parallel_computation.rst
--- a/doc/source/analyzing/parallel_computation.rst
+++ b/doc/source/analyzing/parallel_computation.rst
@@ -105,6 +105,14 @@
If you run into problems, the you can use :ref:`remote-debugging` to examine
what went wrong.
+How do I run my yt job on a subset of available processes
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+You can set the ``communicator`` keyword in the
+:func:`~yt.utilities.parallel_tools.parallel_analysis_interface.enable_parallelism`
+call to a specific MPI communicator to specify a subset of availble MPI
+processes. If none is specified, it defaults to ``COMM_WORLD``.
+
Creating Parallel and Serial Sections in a Script
+++++++++++++++++++++++++++++++++++++++++++++++++
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/source/analyzing/units/comoving_units_and_code_units.rst
--- a/doc/source/analyzing/units/comoving_units_and_code_units.rst
+++ b/doc/source/analyzing/units/comoving_units_and_code_units.rst
@@ -4,3 +4,33 @@
=============================
.. notebook:: 3)_Comoving_units_and_code_units.ipynb
+
+.. _cosmological-units:
+
+Units for Cosmological Datasets
+-------------------------------
+
+yt has additional capabilities to handle the comoving coordinate system used
+internally in cosmological simulations. Simulations that use comoving
+coordinates, all length units have three other counterparts correspoding to
+comoving units, scaled comoving units, and scaled proper units. In all cases
+'scaled' units refer to scaling by the reduced Hubble parameter - i.e. the length
+unit is what it would be in a universe where Hubble's parameter is 100 km/s/Mpc.
+
+To access these different units, yt has a common naming system. Scaled units are denoted by
+dividing by the scaled Hubble parameter ``h`` (which is in itself a unit). Comoving
+units are denoted by appending ``cm`` to the end of the unit name.
+
+Using the parsec as an example,
+
+``pc``
+ Proper parsecs, :math:`\rm{pc}`.
+
+``pccm``
+ Comoving parsecs, :math:`\rm{pc}/(1+z)`.
+
+``pccm/h``
+ Comoving parsecs normalized by the scaled hubble constant, :math:`\rm{pc}/h/(1+z)`.
+
+``pc/h``
+ Proper parsecs, normalized by the scaled hubble constant, :math:`\rm{pc}/h`.
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -31,7 +31,7 @@
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx',
'sphinx.ext.pngmath', 'sphinx.ext.viewcode',
- 'numpydocmod', 'youtube', 'yt_cookbook', 'yt_colormaps']
+ 'numpydocmod', 'yt_cookbook', 'yt_colormaps']
if not on_rtd:
extensions.append('sphinx.ext.autosummary')
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/source/cookbook/Halo_Analysis.ipynb
--- a/doc/source/cookbook/Halo_Analysis.ipynb
+++ b/doc/source/cookbook/Halo_Analysis.ipynb
@@ -1,7 +1,7 @@
{
"metadata": {
"name": "",
- "signature": "sha256:083fded18194cc4b6d9ebf6d04e24a7f1d90cf57831ff0155e99868c0ace73b7"
+ "signature": "sha256:cb4a8114d92def67d5f948ec8326e8af0b20a5340dc8fd54d8d583e9c646886e"
},
"nbformat": 3,
"nbformat_minor": 0,
@@ -226,7 +226,7 @@
"cell_type": "code",
"collapsed": false,
"input": [
- "hc.add_callback('profile', 'virial_radius', [('gas','temperature')],\n",
+ "hc.add_callback('profile', 'virial_radius_fraction', [('gas','temperature')],\n",
" storage='virial_profiles',\n",
" weight_field='cell_mass', \n",
" accumulation=False, output_dir='profiles')\n"
@@ -368,7 +368,7 @@
"input": [
"halo = hc_reloaded.halo_list[0]\n",
"\n",
- "radius = halo.virial_profiles['virial_radius']\n",
+ "radius = halo.virial_profiles[u\"('index', 'virial_radius_fraction')\"]\n",
"temperature = halo.virial_profiles[u\"('gas', 'temperature')\"]\n",
"\n",
"# Remove output files, that are no longer needed\n",
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/source/cookbook/cosmological_analysis.rst
--- a/doc/source/cookbook/cosmological_analysis.rst
+++ b/doc/source/cookbook/cosmological_analysis.rst
@@ -25,7 +25,7 @@
zoom datasets. This recipe uses Rockstar in two different ways to generate a
HaloCatalog from the highest resolution dark matter particles (the ones
inside the zoom region). It then overlays some of those halos on a projection
-as a demonstration. See :ref:`halo-analysis` and :ref:`annotate-halos` for
+as a demonstration. See :ref:`rockstar` and :ref:`annotate-halos` for
more information.
.. yt_cookbook:: rockstar_nest.py
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/source/cookbook/index.rst
--- a/doc/source/cookbook/index.rst
+++ b/doc/source/cookbook/index.rst
@@ -32,6 +32,8 @@
cosmological_analysis
constructing_data_objects
+.. _example-notebooks:
+
Example Notebooks
-----------------
.. toctree::
@@ -41,6 +43,7 @@
custom_colorbar_tickmarks
embedded_javascript_animation
embedded_webm_animation
+ gadget_notebook
../analyzing/analysis_modules/sunyaev_zeldovich
fits_radio_cubes
fits_xray_images
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/source/developing/building_the_docs.rst
--- a/doc/source/developing/building_the_docs.rst
+++ b/doc/source/developing/building_the_docs.rst
@@ -26,15 +26,17 @@
* Visualizing
* Analyzing
+* Analysis Modules
* Examining
* Cookbook
* Quickstart
* Developing
* Reference
+* FAQ
* Help
You will have to figure out where your new/modified doc fits into this, but
-browsing through the pre-built documentation is a good way to sort that out.
+browsing through the existing documentation is a good way to sort that out.
All the source for the documentation is written in
`Sphinx <http://sphinx-doc.org/>`_, which uses ReST for markup. ReST is very
@@ -83,6 +85,14 @@
build time by sphinx. We also use sphinx to run code snippets (e.g. the
cookbook and the notebooks) and embed resulting images and example data.
+You will want to make sure you have both Sphinx and the sphinx bootstrap theme
+installed. This installation is easily performed by running this at the
+command line:
+
+.. code-block:: bash
+
+ pip install sphinx sphinx_bootstrap_theme
+
Quick versus Full Documentation Builds
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/source/developing/creating_derived_fields.rst
--- a/doc/source/developing/creating_derived_fields.rst
+++ b/doc/source/developing/creating_derived_fields.rst
@@ -36,20 +36,27 @@
number of fairly specific parameters that can be passed in, but here we'll only
look at the most basic ones needed for a simple scalar baryon field.
+.. note::
+
+ There are two different :func:`add_field` functions. For the differences,
+ see :ref:`faq-add-field-diffs`.
+
.. code-block:: python
yt.add_field("pressure", function=_pressure, units="dyne/cm**2")
We feed it the name of the field, the name of the function, and the
-units. Note that the units parameter is a "raw" string, in the format that yt uses
-in its `symbolic units implementation <units>`_ (e.g., employing only unit names, numbers,
-and mathematical operators in the string, and using ``"**"`` for exponentiation). We suggest
-that you name the function that creates a derived field with the intended field name prefixed
-by a single underscore, as in the ``_pressure`` example above.
+units. Note that the units parameter is a "raw" string, in the format that yt
+uses in its `symbolic units implementation <units>`_ (e.g., employing only
+unit names, numbers, and mathematical operators in the string, and using
+``"**"`` for exponentiation). For cosmological datasets and fields, see
+:ref:`cosmological-units`. We suggest that you name the function that creates
+a derived field with the intended field name prefixed by a single underscore,
+as in the ``_pressure`` example above.
-:func:`add_field` can be invoked in two other ways. The first is by the function
-decorator :func:`derived_field`. The following code is equivalent to the previous
-example:
+:func:`add_field` can be invoked in two other ways. The first is by the
+function decorator :func:`derived_field`. The following code is equivalent to
+the previous example:
.. code-block:: python
@@ -60,15 +67,16 @@
return (data.ds.gamma - 1.0) * \
data["density"] * data["thermal_energy"]
-The :func:`derived_field` decorator takes the same arguments as :func:`add_field`,
-and is often a more convenient shorthand in cases where you want to quickly set up
-a new field.
+The :func:`derived_field` decorator takes the same arguments as
+:func:`add_field`, and is often a more convenient shorthand in cases where
+you want to quickly set up a new field.
-Defining derived fields in the above fashion must be done before a dataset is loaded,
-in order for the dataset to recognize it. If you want to set up a derived field after you
-have loaded a dataset, or if you only want to set up a derived field for a particular
-dataset, there is an :func:`~yt.data_objects.static_output.Dataset.add_field`
-method that hangs off dataset objects. The calling syntax is the same:
+Defining derived fields in the above fashion must be done before a dataset is
+loaded, in order for the dataset to recognize it. If you want to set up a
+derived field after you have loaded a dataset, or if you only want to set up
+a derived field for a particular dataset, there is an
+:func:`~yt.data_objects.static_output.Dataset.add_field` method that hangs off
+dataset objects. The calling syntax is the same:
.. code-block:: python
@@ -210,33 +218,3 @@
And now, when that derived field is actually used, you will be placed into a
debugger.
-
-Units for Cosmological Datasets
--------------------------------
-
-yt has additional capabilities to handle the comoving coordinate system used
-internally in cosmological simulations. Simulations that use comoving
-coordinates, all length units have three other counterparts correspoding to
-comoving units, scaled comoving units, and scaled proper units. In all cases
-'scaled' units refer to scaling by the reduced Hubble parameter - i.e. the length
-unit is what it would be in a universe where Hubble's parameter is 100 km/s/Mpc.
-
-To access these different units, yt has a common naming system. Scaled units are denoted by
-dividing by the scaled Hubble parameter ``h`` (which is in itself a unit). Comoving
-units are denoted by appending ``cm`` to the end of the unit name.
-
-Using the parsec as an example,
-
-``pc``
- Proper parsecs, :math:`\rm{pc}`.
-
-``pccm``
- Comoving parsecs, :math:`\rm{pc}/(1+z)`.
-
-``pccm/h``
- Comoving parsecs normalized by the scaled hubble constant, :math:`\rm{pc}/h/(1+z)`.
-
-``pc/h``
- Proper parsecs, normalized by the scaled hubble constant, :math:`\rm{pc}/h`.
-
-Further examples of this functionality are shown in :ref:`comoving_units_and_code_units`.
diff -r f6fc1de3b75b7b575307459233d0bbc3d52a23c6 -r 74e0b4906416272d8474181d1a263012c8f03a4d doc/source/developing/developing.rst
--- a/doc/source/developing/developing.rst
+++ b/doc/source/developing/developing.rst
@@ -220,6 +220,12 @@
#. Commit these changes, using ``hg commit``. This can take an argument
which is a series of filenames, if you have some changes you do not want
to commit.
+#. Remember that this is a large development effort and to keep the code
+ accessible to everyone, good documentation is a must. Add in source code
+ comments for what you are doing. Add in docstrings
+ if you are adding a new function or class or keyword to a function.
+ Add documentation to the appropriate section of the online docs so that
+ people other than yourself know how to use your new code.
#. If your changes include new functionality or cover an untested area of the
code, add a test. (See :ref:`testing` for more information.) Commit
these changes as well.
@@ -244,6 +250,9 @@
#. Issue a pull request at
https://bitbucket.org/YourUsername/yt/pull-request/new
+ A pull request is essentially just asking people to review and accept the
+ modifications you have made to your personal version of the code.
+
During the course of your pull request you may be asked to make changes. These
changes may be related to style issues, correctness issues, or even requesting
@@ -253,7 +262,7 @@
#. Make requested changes, or leave a comment indicating why you don't think
they should be made.
#. Commit those changes to your local repository.
-#. Push the changes to your fork::
+#. Push the changes to your fork:
hg push https://bitbucket.org/YourUsername/yt/
This diff is so big that we needed to truncate the remainder.
https://bitbucket.org/yt_analysis/yt/commits/20d4fcc40283/
Changeset: 20d4fcc40283
Branch: yt
User: xarthisius
Date: 2015-01-04 05:17:36+00:00
Summary: Merged in MatthewTurk/yt (pull request #1350)
Cache data for grid frontends in IO chunking
Affected #: 7 files
diff -r ea716eba60f6a36dfee7274b9503822c70af0ffe -r 20d4fcc40283217c698762168f98fbe9eb2e29ec yt/data_objects/construction_data_containers.py
--- a/yt/data_objects/construction_data_containers.py
+++ b/yt/data_objects/construction_data_containers.py
@@ -470,7 +470,7 @@
self.domain_width = np.rint((self.ds.domain_right_edge -
self.ds.domain_left_edge)/self.dds).astype('int64')
self._setup_data_source()
-
+
@property
def icoords(self):
ic = np.indices(self.ActiveDimensions).astype("int64")
@@ -950,14 +950,18 @@
mylog.info("Extracting (sampling: %s)" % (fields,))
verts = []
samples = []
- for block, mask in parallel_objects(self.data_source.blocks):
- my_verts = self._extract_isocontours_from_grid(
- block, self.surface_field, self.field_value,
- mask, fields, sample_type)
- if fields is not None:
- my_verts, svals = my_verts
- samples.append(svals)
- verts.append(my_verts)
+ deps = self._determine_fields(self.surface_field)
+ deps = self._identify_dependencies(deps, spatial=True)
+ for io_chunk in parallel_objects(self.data_source.chunks(deps, "io",
+ preload_fields = deps)):
+ for block, mask in self.data_source.blocks:
+ my_verts = self._extract_isocontours_from_grid(
+ block, self.surface_field, self.field_value,
+ mask, fields, sample_type)
+ if fields is not None:
+ my_verts, svals = my_verts
+ samples.append(svals)
+ verts.append(my_verts)
verts = np.concatenate(verts).transpose()
verts = self.comm.par_combine_object(verts, op='cat', datatype='array')
self.vertices = verts
@@ -1040,21 +1044,27 @@
"""
flux = 0.0
mylog.info("Fluxing %s", fluxing_field)
- for block, mask in parallel_objects(self.data_source.blocks):
- flux += self._calculate_flux_in_grid(block, mask,
- field_x, field_y, field_z, fluxing_field)
+ deps = [field_x, field_y, field_z]
+ if fluxing_field is not None: deps.append(fluxing_field)
+ deps = self._determine_fields(deps)
+ deps = self._identify_dependencies(deps)
+ for io_chunk in parallel_objects(self.data_source.chunks(deps, "io",
+ preload_fields = deps)):
+ for block, mask in self.data_source.blocks:
+ flux += self._calculate_flux_in_grid(block, mask,
+ field_x, field_y, field_z, fluxing_field)
flux = self.comm.mpi_allreduce(flux, op="sum")
return flux
def _calculate_flux_in_grid(self, grid, mask,
- field_x, field_y, field_z, fluxing_field = None):
+ field_x, field_y, field_z, fluxing_field = None):
vals = grid.get_vertex_centered_data(self.surface_field)
if fluxing_field is None:
ff = np.ones(vals.shape, dtype="float64")
else:
ff = grid.get_vertex_centered_data(fluxing_field)
- xv, yv, zv = [grid.get_vertex_centered_data(f) for f in
- [field_x, field_y, field_z]]
+ xv, yv, zv = [grid.get_vertex_centered_data(f)
+ for f in [field_x, field_y, field_z]]
return march_cubes_grid_flux(self.field_value, vals, xv, yv, zv,
ff, mask, grid.LeftEdge, grid.dds)
diff -r ea716eba60f6a36dfee7274b9503822c70af0ffe -r 20d4fcc40283217c698762168f98fbe9eb2e29ec yt/data_objects/data_containers.py
--- a/yt/data_objects/data_containers.py
+++ b/yt/data_objects/data_containers.py
@@ -1019,16 +1019,13 @@
"""
verts = []
samples = []
- pb = get_pbar("Extracting ", len(list(self._get_grid_objs())))
- for i, g in enumerate(self._get_grid_objs()):
- pb.update(i)
+ for block, mask in self.blocks:
my_verts = self._extract_isocontours_from_grid(
- g, field, value, sample_values)
+ block, mask, field, value, sample_values)
if sample_values is not None:
my_verts, svals = my_verts
samples.append(svals)
verts.append(my_verts)
- pb.finish()
verts = np.concatenate(verts).transpose()
verts = self.comm.par_combine_object(verts, op='cat', datatype='array')
verts = verts.transpose()
@@ -1052,11 +1049,9 @@
return verts, samples
return verts
-
- def _extract_isocontours_from_grid(self, grid, field, value,
- sample_values = None):
- mask = self._get_cut_mask(grid) * grid.child_mask
- vals = grid.get_vertex_centered_data(field, no_ghost = False)
+ def _extract_isocontours_from_grid(self, grid, mask, field, value,
+ sample_values=None):
+ vals = grid.get_vertex_centered_data(field, no_ghost=False)
if sample_values is not None:
svals = grid.get_vertex_centered_data(sample_values)
else:
@@ -1130,15 +1125,14 @@
... "velocity_x", "velocity_y", "velocity_z", "Metal_Density")
"""
flux = 0.0
- for g in self._get_grid_objs():
- flux += self._calculate_flux_in_grid(g, field, value,
- field_x, field_y, field_z, fluxing_field)
+ for block, mask in self.blocks:
+ flux += self._calculate_flux_in_grid(block, mask, field, value, field_x,
+ field_y, field_z, fluxing_field)
flux = self.comm.mpi_allreduce(flux, op="sum")
return flux
- def _calculate_flux_in_grid(self, grid, field, value,
+ def _calculate_flux_in_grid(self, grid, mask, field, value,
field_x, field_y, field_z, fluxing_field = None):
- mask = self._get_cut_mask(grid) * grid.child_mask
vals = grid.get_vertex_centered_data(field)
if fluxing_field is None:
ff = np.ones(vals.shape, dtype="float64")
diff -r ea716eba60f6a36dfee7274b9503822c70af0ffe -r 20d4fcc40283217c698762168f98fbe9eb2e29ec yt/frontends/enzo/io.py
--- a/yt/frontends/enzo/io.py
+++ b/yt/frontends/enzo/io.py
@@ -14,6 +14,8 @@
#-----------------------------------------------------------------------------
import os
+import random
+from contextlib import contextmanager
from yt.utilities.io_handler import \
BaseIOHandler, _axis_ids
@@ -130,12 +132,25 @@
raise RuntimeError
g = chunks[0].objs[0]
f = h5py.File(g.filename.encode('ascii'), 'r')
+ if g.id in self._cached_fields:
+ gf = self._cached_fields[g.id]
+ rv.update(gf)
+ if len(rv) == len(fields): return rv
gds = f.get("/Grid%08i" % g.id)
- for ftype, fname in fields:
+ for field in fields:
+ if field in rv:
+ self._hits += 1
+ continue
+ self._misses += 1
+ ftype, fname = field
if fname in gds:
rv[(ftype, fname)] = gds.get(fname).value.swapaxes(0,2)
else:
rv[(ftype, fname)] = np.zeros(g.ActiveDimensions)
+ if self._cache_on:
+ for gid in rv:
+ self._cached_fields.setdefault(gid, {})
+ self._cached_fields[gid].update(rv[gid])
f.close()
return rv
if size is None:
@@ -155,10 +170,16 @@
if g.filename is None: continue
if fid is None:
fid = h5py.h5f.open(g.filename.encode('ascii'), h5py.h5f.ACC_RDONLY)
+ gf = self._cached_fields.get(g.id, {})
data = np.empty(g.ActiveDimensions[::-1], dtype="float64")
data_view = data.swapaxes(0,2)
nd = 0
for field in fields:
+ if field in gf:
+ nd = g.select(selector, gf[field], rv[field], ind)
+ self._hits += 1
+ continue
+ self._misses += 1
ftype, fname = field
try:
node = "/Grid%08i/%s" % (g.id, fname)
@@ -167,11 +188,52 @@
if fname == "Dark_Matter_Density": continue
raise
dg.read(h5py.h5s.ALL, h5py.h5s.ALL, data)
+ if self._cache_on:
+ self._cached_fields.setdefault(g.id, {})
+ # Copy because it's a view into an empty temp array
+ self._cached_fields[g.id][field] = data_view.copy()
nd = g.select(selector, data_view, rv[field], ind) # caches
ind += nd
if fid: fid.close()
return rv
+ @contextmanager
+ def preload(self, chunk, fields, max_size):
+ if len(fields) == 0:
+ yield self
+ return
+ old_cache_on = self._cache_on
+ old_cached_fields = self._cached_fields
+ self._cached_fields = cf = {}
+ self._cache_on = True
+ for gid in old_cached_fields:
+ # Will not copy numpy arrays, which is good!
+ cf[gid] = old_cached_fields[gid].copy()
+ self._hits = self._misses = 0
+ self._cached_fields = self._read_chunk_data(chunk, fields)
+ mylog.debug("(1st) Hits = % 10i Misses = % 10i",
+ self._hits, self._misses)
+ self._hits = self._misses = 0
+ yield self
+ mylog.debug("(2nd) Hits = % 10i Misses = % 10i",
+ self._hits, self._misses)
+ self._cached_fields = old_cached_fields
+ self._cache_on = old_cache_on
+ # Randomly remove some grids from the cache. Note that we're doing
+ # this on a grid basis, not a field basis. Performance will be
+ # slightly non-deterministic as a result of this, but it should roughly
+ # be statistically alright, assuming (as we do) that this will get
+ # called during largely unbalanced stuff.
+ if len(self._cached_fields) > max_size:
+ to_remove = random.sample(self._cached_fields.keys(),
+ len(self._cached_fields) - max_size)
+ mylog.debug("Purging from cache %s", len(to_remove))
+ for k in to_remove:
+ self._cached_fields.pop(k)
+ else:
+ mylog.warning("Cache size % 10i (max % 10i)",
+ len(self._cached_fields), max_size)
+
def _read_chunk_data(self, chunk, fields):
fid = fn = None
rv = {}
@@ -190,6 +252,8 @@
if len(fluid_fields) == 0: return rv
for g in chunk.objs:
rv[g.id] = gf = {}
+ if g.id in self._cached_fields:
+ rv[g.id].update(self._cached_fields[g.id])
if g.filename is None: continue
elif g.filename != fn:
if fid is not None: fid.close()
@@ -200,6 +264,10 @@
data = np.empty(g.ActiveDimensions[::-1], dtype="float64")
data_view = data.swapaxes(0,2)
for field in fluid_fields:
+ if field in gf:
+ self._hits += 1
+ continue
+ self._misses += 1
ftype, fname = field
try:
node = "/Grid%08i/%s" % (g.id, fname)
@@ -210,6 +278,10 @@
dg.read(h5py.h5s.ALL, h5py.h5s.ALL, data)
gf[field] = data_view.copy()
if fid: fid.close()
+ if self._cache_on:
+ for gid in rv:
+ self._cached_fields.setdefault(gid, {})
+ self._cached_fields[gid].update(rv[gid])
return rv
class IOHandlerPackedHDF5GhostZones(IOHandlerPackedHDF5):
diff -r ea716eba60f6a36dfee7274b9503822c70af0ffe -r 20d4fcc40283217c698762168f98fbe9eb2e29ec yt/geometry/geometry_handler.py
--- a/yt/geometry/geometry_handler.py
+++ b/yt/geometry/geometry_handler.py
@@ -252,7 +252,6 @@
chunk_size)
return fields_to_return, fields_to_generate
-
def _chunk(self, dobj, chunking_style, ngz = 0, **kwargs):
# A chunk is either None or (grids, size)
if dobj._current_chunk is None:
diff -r ea716eba60f6a36dfee7274b9503822c70af0ffe -r 20d4fcc40283217c698762168f98fbe9eb2e29ec yt/geometry/grid_geometry_handler.py
--- a/yt/geometry/grid_geometry_handler.py
+++ b/yt/geometry/grid_geometry_handler.py
@@ -323,9 +323,13 @@
yield YTDataChunk(dobj, "spatial", [g], size, cache = False)
_grid_chunksize = 1000
- def _chunk_io(self, dobj, cache = True, local_only = False):
+ def _chunk_io(self, dobj, cache = True, local_only = False,
+ preload_fields = None):
# local_only is only useful for inline datasets and requires
# implementation by subclasses.
+ if preload_fields is None:
+ preload_fields = []
+ preload_fields, _ = self._split_fields(preload_fields)
gfiles = defaultdict(list)
gobjs = getattr(dobj._current_chunk, "objs", dobj._chunk_info)
for g in gobjs:
@@ -338,7 +342,10 @@
for grids in (gs[pos:pos + size] for pos
in xrange(0, len(gs), size)):
- yield YTDataChunk(dobj, "io", grids,
+ dc = YTDataChunk(dobj, "io", grids,
self._count_selection(dobj, grids),
cache = cache)
-
+ # We allow four full chunks to be included.
+ with self.io.preload(dc, preload_fields,
+ 4.0 * self._grid_chunksize):
+ yield dc
diff -r ea716eba60f6a36dfee7274b9503822c70af0ffe -r 20d4fcc40283217c698762168f98fbe9eb2e29ec yt/utilities/io_handler.py
--- a/yt/utilities/io_handler.py
+++ b/yt/utilities/io_handler.py
@@ -14,6 +14,7 @@
#-----------------------------------------------------------------------------
from collections import defaultdict
+from contextlib import contextmanager
from yt.funcs import mylog
import cPickle
@@ -37,6 +38,9 @@
_vector_fields = ()
_dataset_type = None
_particle_reader = False
+ _cache_on = False
+ _misses = 0
+ _hits = 0
def __init__(self, ds):
self.queue = defaultdict(dict)
@@ -44,12 +48,14 @@
self._last_selector_id = None
self._last_selector_counts = None
self._array_fields = {}
+ self._cached_fields = {}
# We need a function for reading a list of sets
# and a function for *popping* from a queue all the appropriate sets
- def preload(self, grids, sets):
- pass
+ @contextmanager
+ def preload(self, chunk, fields, max_size):
+ yield self
def pop(self, grid, field):
if grid.id in self.queue and field in self.queue[grid.id]:
diff -r ea716eba60f6a36dfee7274b9503822c70af0ffe -r 20d4fcc40283217c698762168f98fbe9eb2e29ec yt/utilities/parallel_tools/parallel_analysis_interface.py
--- a/yt/utilities/parallel_tools/parallel_analysis_interface.py
+++ b/yt/utilities/parallel_tools/parallel_analysis_interface.py
@@ -814,12 +814,8 @@
return data
def preload(self, grids, fields, io_handler):
- # This will preload if it detects we are parallel capable and
- # if so, we load *everything* that we need. Use with some care.
- if len(fields) == 0: return
- mylog.debug("Preloading %s from %s grids", fields, len(grids))
- if not self._distributed: return
- io_handler.preload(grids, fields)
+ # This is non-functional.
+ return
@parallel_passthrough
def mpi_allreduce(self, data, dtype=None, op='sum'):
Repository URL: https://bitbucket.org/yt_analysis/yt/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the yt-svn
mailing list