[Yt-svn] yt: 3 new changesets
hg at spacepope.org
hg at spacepope.org
Fri Jan 21 10:29:24 PST 2011
hg Repository: yt
details: yt/rev/bce1c775bb84
changeset: 3680:bce1c775bb84
user: Matthew Turk <matthewturk at gmail.com>
date:
Mon Jan 17 13:16:50 2011 -0500
description:
This commit marks a change in how Cython is used in yt. Previously we checked
in generated sources. However, this has simply become too burdensome -- the
diffs whenever we re-generated the Cython code were simply too large, and this
bloated the repository unnecessarily. This adds support for direct generation
and compilation using Cython.
This starts by removing .c and .cpp files generated by Cython and adding the
appropriate information about how to generate the .c and .cpp files to the
setup.py files for RAMSES and the amr_utils directories. Additionally,
dependencies are included in these setup.py files.
However, to get Cython to play nicely with NumPy, I've had to add in a routine
that I found written by Matthew Brett. I updated it to include some C++
support.
Additionally, as a safeguard for those systems were Cython is not already
included, I have added a brief routine that checks whether or not Cython is
installed. If it is not, it attempts to install it. This will still fail on
systems where superuser privs are required, but that should not be the case in
the majority of yt installations that are not on OSX. On OSX this will require
sudo-ing the installation.
hg Repository: yt
details: yt/rev/3b4dcd63cba6
changeset: 3681:3b4dcd63cba6
user: Matthew Turk <matthewturk at gmail.com>
date:
Fri Jan 21 13:17:21 2011 -0500
description:
Merging back from development to get pydot changes and fix for Maestro. Also
added the results directory (possibly only on my machine as of right now) to
.hgignore.
hg Repository: yt
details: yt/rev/8e477b655c96
changeset: 3682:8e477b655c96
user: Matthew Turk <matthewturk at gmail.com>
date:
Fri Jan 21 13:28:34 2011 -0500
description:
Merging Cython changes into main development branch.
See email to yt-users titled "Development branch now requires Cython" for more
information.
diffstat:
.hgignore | 3 +-
setup.py | 68 +-
yt/analysis_modules/halo_merger_tree/enzofof_merger_tree.py | 79 +-
yt/analysis_modules/halo_merger_tree/merger_tree.py | 116 +-
yt/data_objects/data_containers.py | 12 +-
yt/frontends/maestro/data_structures.py | 4 +
yt/frontends/ramses/_ramses_reader.cpp | 11579 ---
yt/frontends/ramses/setup.py | 9 +-
yt/utilities/amr_utils.c | 35083 ----------
yt/utilities/pydot.py | 1995 +
yt/utilities/setup.py | 10 +-
yt/visualization/plot_collection.py | 9 +-
yt/visualization/plot_modifications.py | 1 -
13 files changed, 2176 insertions(+), 46792 deletions(-)
diffs (truncated from 49178 to 300 lines):
diff -r 0ab612d207a3 -r 8e477b655c96 .hgignore
--- a/.hgignore Mon Jan 17 13:00:30 2011 -0500
+++ b/.hgignore Fri Jan 21 13:28:34 2011 -0500
@@ -4,4 +4,5 @@
syntax: glob
*.pyc
.*.swp
-*.so
\ No newline at end of file
+*.so
+tests/results/*
diff -r 0ab612d207a3 -r 8e477b655c96 setup.py
--- a/setup.py Mon Jan 17 13:00:30 2011 -0500
+++ b/setup.py Fri Jan 21 13:28:34 2011 -0500
@@ -5,10 +5,76 @@
import distribute_setup
distribute_setup.use_setuptools()
+from numpy.distutils.misc_util import appendpath
+from numpy.distutils import log
+
+# Verify that we have Cython installed
+try:
+ import Cython
+except ImportError as e:
+ print "Received error on importing Cython:"
+ print e
+ print "Now attempting to install Cython"
+ import pip
+ rv = pip.main(["install",
+ "http://yt.enzotools.org/dependencies/Cython-latest.tar.gz"])
+ if rv == 1:
+ print "Unable to install Cython. Please report this bug to yt-users."
+ sys.exit(1)
+
+######
+# This next bit comes from Matthew Brett, to get Cython working with NumPy
+# distutils. I added a bit to get C++ Cython working.
+from os.path import join as pjoin, dirname
+from distutils.dep_util import newer_group
+from distutils.errors import DistutilsError
+
+
+def generate_a_pyrex_source(self, base, ext_name, source, extension):
+ ''' Monkey patch for numpy build_src.build_src method
+
+ Uses Cython instead of Pyrex.
+
+ Assumes Cython is present
+ '''
+ if self.inplace:
+ target_dir = dirname(base)
+ else:
+ target_dir = appendpath(self.build_src, dirname(base))
+ if extension.language == "c++":
+ cplus = True
+ file_ext = ".cpp"
+ else:
+ cplus = False
+ file_ext = ".c"
+ target_file = pjoin(target_dir, ext_name + file_ext)
+ depends = [source] + extension.depends
+ if self.force or newer_group(depends, target_file, 'newer'):
+ import Cython.Compiler.Main
+ log.info("cythonc:> %s" % (target_file))
+ self.mkpath(target_dir)
+ options = Cython.Compiler.Main.CompilationOptions(
+ defaults=Cython.Compiler.Main.default_options,
+ include_path=extension.include_dirs,
+ language=extension.language, cplus = cplus,
+ output_file=target_file)
+ cython_result = Cython.Compiler.Main.compile(source,
+ options=options)
+ if cython_result.num_errors != 0:
+ raise DistutilsError("%d errors while compiling %r with Cython" \
+ % (cython_result.num_errors, source))
+ return target_file
+
+
+from numpy.distutils.command import build_src
+build_src.build_src.generate_a_pyrex_source = generate_a_pyrex_source
+# End snippet
+######
+
import setuptools
DATA_FILES = []
-VERSION = "2.0dev"
+VERSION = "2.1dev"
if os.path.exists('MANIFEST'): os.remove('MANIFEST')
diff -r 0ab612d207a3 -r 8e477b655c96 yt/analysis_modules/halo_merger_tree/enzofof_merger_tree.py
--- a/yt/analysis_modules/halo_merger_tree/enzofof_merger_tree.py Mon Jan 17 13:00:30 2011 -0500
+++ b/yt/analysis_modules/halo_merger_tree/enzofof_merger_tree.py Fri Jan 21 13:28:34 2011 -0500
@@ -50,6 +50,7 @@
from yt.funcs import *
from yt.utilities.pykdtree import KDTree
+import yt.utilities.pydot as pydot
# We don't currently use this, but we may again find a use for it in the
# future.
@@ -338,23 +339,29 @@
print "--> Halo %8.8d :: fraction = %g" % (c[0], c[1])
def write_dot(self, filename=None):
- r"""Writes merger tree to a GraphViz file.
-
- User is responsible for creating an image file from it, e.g.
- dot -Tpng tree_halo00000.dot > image.png
+ r"""Writes merger tree to a GraphViz or image file.
Parameters
----------
filename : str, optional
Filename to write the GraphViz file. Default will be
- tree_halo%05i.dat.
+ tree_halo%05i.gv, which is a text file in the GraphViz format.
+ If filename is an image (e.g. "MergerTree.png") the output will
+ be in the appropriate image format made by calling GraphViz
+ automatically. See GraphViz (e.g. "dot -v")
+ for a list of available output formats.
"""
- if filename == None: filename = "tree_halo%5.5d.dot" % self.halonum
- fp = open(filename, "w")
- fp.write("digraph G {\n")
- fp.write(" node [shape=rect];\n")
+ if filename == None: filename = "tree_halo%5.5d.gv" % self.halonum
+ # Create the pydot graph object.
+ self.graph = pydot.Dot('galaxy', graph_type='digraph')
+ self.halo_shape = "rect"
+ self.z_shape = "plaintext"
+ # Subgraphs to align levels
+ self.subgs = {}
+ for num in self.numbers:
+ self.subgs[num] = pydot.Subgraph('', rank = 'same')
+ self.graph.add_subgraph(self.subgs[num])
sorted_lvl = sorted(self.levels, reverse=True)
- printed = []
for ii,lvl in enumerate(sorted_lvl):
# Since we get the cycle number from the key, it won't
# exist for the last level, i.e. children of last level.
@@ -366,10 +373,12 @@
for br in self.levels[lvl]:
for c in br.children:
color = "red" if c[0] == br.progenitor else "black"
- line = " C%d_H%d -> C%d_H%d [color=%s];\n" % \
- (lvl, br.halo_id, next_lvl, c[0], color)
+ self.graph.add_edge(pydot.Edge("C%d_H%d" %(lvl, br.halo_id),
+ "C%d_H%d" % (next_lvl, c[0]), color=color))
+ #line = " C%d_H%d -> C%d_H%d [color=%s];\n" % \
+ # (lvl, br.halo_id, next_lvl, c[0], color)
- fp.write(line)
+ #fp.write(line)
last_level = (ii,lvl)
for ii,lvl in enumerate(sorted_lvl):
npart_max = 0
@@ -378,35 +387,21 @@
for br in self.levels[lvl]:
halo_str = "C%d_H%d" % (lvl, br.halo_id)
style = "filled" if br.npart == npart_max else "solid"
- line = "%s [label=\"Halo %d\\n%d particles\",style=%s]\n" % \
- (halo_str, br.halo_id, br.npart, style)
- fp.write(line)
- # Last level, annotate children because they have no associated branches
- npart_max = 0
- for br in self.levels[last_level[1]]:
- for c in br.children:
- if c[2] > npart_max: npart_max = c[2]
- for br in self.levels[last_level[1]]:
- for c in br.children:
- lvl = self.numbers[0]
- style = "filled" if c[2] == npart_max else "solid"
- halo_str = "C%d_H%d" % (lvl, c[0])
- line = "%s [label=\"Halo %d\\n%d particles\",style=%s]\n" % \
- (halo_str, c[0], c[2], style)
- fp.write(line)
- # Output redshifts
- fp.write("\n")
- fp.write("node [shape=plaintext]\n")
- fp.write("edge [style=invis]\n")
- line = ""
- for k in sorted(self.redshifts, reverse=True):
- line = line + "\"z = %0.3f\"" % (self.redshifts[k]) + " -> "
- if k == self.numbers[0]: break
- line = line[:-4] # Remove last arrow
- fp.write("\n%s\n" % line)
-
- fp.write("}\n")
- fp.close()
+ self.graph.add_node(pydot.Node(halo_str,
+ label = "Halo %d\\n%d particles" % (br.halo_id, br.npart),
+ style = style, shape = self.halo_shape))
+ # Add this node to the correct level subgraph.
+ self.subgs[lvl].add_node(pydot.Node(halo_str))
+ for lvl in self.numbers:
+ # Don't add the z if there are no halos already in the subgraph.
+ if len(self.subgs[lvl].get_node_list()) == 0: continue
+ self.subgs[lvl].add_node(pydot.Node("%1.5e" % self.redshifts[lvl],
+ shape = self.z_shape, label = "z=%0.3f" % self.redshifts[lvl]))
+ # Based on the suffix of the file name, write out the result to a file.
+ suffix = filename.split(".")[-1]
+ if suffix == "gv": suffix = "raw"
+ mylog.info("Writing %s format %s to disk." % (suffix, filename))
+ self.graph.write("%s" % filename, format=suffix)
def find_halo_relationships(output1_id, output2_id, output_basename = None,
radius = 0.10):
diff -r 0ab612d207a3 -r 8e477b655c96 yt/analysis_modules/halo_merger_tree/merger_tree.py
--- a/yt/analysis_modules/halo_merger_tree/merger_tree.py Mon Jan 17 13:00:30 2011 -0500
+++ b/yt/analysis_modules/halo_merger_tree/merger_tree.py Fri Jan 21 13:28:34 2011 -0500
@@ -36,6 +36,7 @@
HaloProfiler
from yt.convenience import load
from yt.utilities.logger import ytLogger as mylog
+import yt.utilities.pydot as pydot
try:
from yt.utilities.kdtree import *
except ImportError:
@@ -939,7 +940,13 @@
database : String
The name of the database file. Default = 'halos.db'.
dotfile : String
- The name of the file to write to. Default = 'MergerTree.gv'
+ The name of the file to write to. Default = 'MergerTree.gv'.
+ The suffix of this name gives the format of the output file,
+ so 'MergerTree.jpg' would output a jpg file. "dot -v" (from the
+ command line) will print
+ a list of image formats supported on the system. The default
+ suffix '.gv' will output the results to a text file in the Graphviz
+ markup language.
current_time : Integer
The SnapCurrentTimeIdentifier for the snapshot for the halos to
be tracked. This is identical to the CurrentTimeIdentifier in
@@ -969,15 +976,18 @@
if current_time is not None:
halos = self._translate_haloIDs(halos, current_time)
newhalos = set(halos)
- # A key is the GlobalHaloID for this halo, and the content is a
- # Node object.
- self.nodes = {}
- # A key is the GlobalHaloID for the parent in the relationship,
- # and the content is a Link ojbect.
- self.links = defaultdict(Link)
- # Record which halos are at the same z level for convenience.
- # They key is a z value, and the content a list of co-leveled halo IDs.
- self.levels = defaultdict(list)
+ # Create the pydot graph object.
+ self.graph = pydot.Dot('galaxy', graph_type='digraph')
+ # Build some initially empty subgraphs, which are used to identify
+ # nodes that are on the same rank (redshift).
+ line = "SELECT DISTINCT SnapZ FROM Halos;"
+ self.cursor.execute(line)
+ self.subgs = {}
+ result = self.cursor.fetchone()
+ while result:
+ self.subgs[result[0]] = pydot.Subgraph('', rank = 'same')
+ self.graph.add_subgraph(self.subgs[result[0]])
+ result = self.cursor.fetchone()
# For the first set of halos.
self._add_nodes(newhalos)
# Recurse over parents.
@@ -985,13 +995,7 @@
mylog.info("Finding parents for %d children." % len(newhalos))
newhalos = self._find_parents(newhalos)
self._add_nodes(newhalos)
- mylog.info("Writing out %s to disk." % dotfile)
- self._open_dot(dotfile)
- self._write_nodes()
- self._write_links()
- self._write_levels()
- self._close_dot()
- self._close_database()
+ self._write_dotfile(dotfile)
return None
def _translate_haloIDs(self, halos, current_time):
@@ -1032,9 +1036,10 @@
if pair[1] <= self.link_min or pair[0] != halo:
continue
else:
- self.nodes[halo].parentIDs.append(pID)
- self.links[pID].childIDs.append(halo)
- self.links[pID].fractions.append(pair[1])
+ self.graph.add_edge(pydot.Edge(pID, halo,
+ label = "%3.2f%%" % float(pair[1]*100),
+ color = "blue",
+ fontsize = "10"))
newhalos.add(pID)
result = self.cursor.fetchone()
return newhalos
@@ -1063,57 +1068,28 @@
value = (halo,)
self.cursor.execute(line, value)
result = self.cursor.fetchone()
- self.nodes[halo] = Node(na.array([result[2],result[3],result[4]]),
- result[1], [], result[0], 1. - float(result[5])/(maxID+1)) #+1 to prevent /0
- self.levels[result[0]].append(halo)
More information about the yt-svn
mailing list