[yt-svn] commit/yt: 4 new changesets
Bitbucket
commits-noreply at bitbucket.org
Mon Mar 26 10:39:00 PDT 2012
4 new commits in yt:
https://bitbucket.org/yt_analysis/yt/changeset/ffd65827bfa1/
changeset: ffd65827bfa1
branch: yt
user: sskory
date: 2012-03-24 00:17:17
summary: Modifying the merger tree to save the halo relationships at every
cycle of pairs, rather than just at the end. It also now has a check
to see just how far along its gone so work is not repeated.
affected #: 1 file
diff -r b9613e068053ec6b24315ee51a540007d68a518f -r ffd65827bfa1ab6894b17196eac8d65b03b68184 yt/analysis_modules/halo_merger_tree/merger_tree.py
--- a/yt/analysis_modules/halo_merger_tree/merger_tree.py
+++ b/yt/analysis_modules/halo_merger_tree/merger_tree.py
@@ -162,6 +162,8 @@
self.FOF_link_length= FOF_link_length # For FOF
self.dm_only = dm_only
self.refresh = refresh
+ self.index = index
+ self.zs = {}
# MPI stuff
if self.comm.rank is None:
self.comm.rank = 0
@@ -173,12 +175,19 @@
os.unlink(self.database)
except:
pass
- self._open_create_database()
- self._create_halo_table()
+ if self.comm.rank == 0:
+ self._open_create_database()
+ self._create_halo_table()
self._run_halo_finder_add_to_db()
# Find the h5 file names for all the halos.
for snap in self.restart_files:
self._build_h5_refs(snap)
+ # Find out how much work is already stored in the database.
+ if self.comm.rank == 0:
+ z_progress = self._find_progress()
+ else:
+ z_progress = None
+ z_progress = self.comm.mpi_bcast(z_progress)
# Loop over the pairs of snapshots to locate likely neighbors, and
# then use those likely neighbors to compute fractional contributions.
last = None
@@ -187,16 +196,22 @@
for snap, pair in enumerate(zip(self.restart_files[:-1], self.restart_files[1:])):
if not self.with_halos[snap] or not self.with_halos[snap+1]:
continue
+ if self.zs[pair[0]] > z_progress:
+ continue
self._find_likely_children(pair[0], pair[1])
# last is the data for the parent dataset, which can be supplied
# as the child from the previous round for all but the first loop.
last = self._compute_child_fraction(pair[0], pair[1], last)
+ if self.comm.rank == 0:
+ mylog.info("Updating database with parent-child relationships.")
+ self._copy_and_update_db()
+ # This has to happen because we delete the old database above.
+ self._open_create_database()
del last
- # Now update the database with all the writes.
- mylog.info("Updating database with parent-child relationships.")
if self.comm.rank == 0:
- self._copy_and_update_db()
- self.comm.barrier()
+ if self.index:
+ self._write_index()
+ self._close_database()
self.comm.barrier()
mylog.info("Done!")
@@ -210,6 +225,7 @@
for cycle, file in enumerate(self.restart_files):
gc.collect()
pf = load(file)
+ self.zs[file] = pf.current_redshift
self.period = pf.domain_right_edge - pf.domain_left_edge
# If the halos are already found, skip this data step, unless
# refresh is True.
@@ -281,30 +297,28 @@
def _open_create_database(self):
# open the database. This creates the database file on disk if it
# doesn't already exist. Open it on root only.
- if self.comm.rank == 0:
- self.conn = sql.connect(self.database)
- self.cursor = self.conn.cursor()
+ self.conn = sql.connect(self.database)
+ self.cursor = self.conn.cursor()
def _create_halo_table(self):
- if self.comm.rank == 0:
- # Handle the error if it already exists.
- try:
- # Create the table that will store the halo data.
- line = "CREATE TABLE Halos (GlobalHaloID INTEGER PRIMARY KEY,\
- SnapCurrentTimeIdentifier INTEGER, SnapZ FLOAT, SnapHaloID INTEGER, \
- HaloMass FLOAT,\
- NumPart INTEGER, CenMassX FLOAT, CenMassY FLOAT,\
- CenMassZ FLOAT, BulkVelX FLOAT, BulkVelY FLOAT, BulkVelZ FLOAT,\
- MaxRad FLOAT,\
- ChildHaloID0 INTEGER, ChildHaloFrac0 FLOAT, \
- ChildHaloID1 INTEGER, ChildHaloFrac1 FLOAT, \
- ChildHaloID2 INTEGER, ChildHaloFrac2 FLOAT, \
- ChildHaloID3 INTEGER, ChildHaloFrac3 FLOAT, \
- ChildHaloID4 INTEGER, ChildHaloFrac4 FLOAT);"
- self.cursor.execute(line)
- self.conn.commit()
- except sql.OperationalError:
- pass
+ # Handle the error if it already exists.
+ try:
+ # Create the table that will store the halo data.
+ line = "CREATE TABLE Halos (GlobalHaloID INTEGER PRIMARY KEY,\
+ SnapCurrentTimeIdentifier INTEGER, SnapZ FLOAT, SnapHaloID INTEGER, \
+ HaloMass FLOAT,\
+ NumPart INTEGER, CenMassX FLOAT, CenMassY FLOAT,\
+ CenMassZ FLOAT, BulkVelX FLOAT, BulkVelY FLOAT, BulkVelZ FLOAT,\
+ MaxRad FLOAT,\
+ ChildHaloID0 INTEGER, ChildHaloFrac0 FLOAT, \
+ ChildHaloID1 INTEGER, ChildHaloFrac1 FLOAT, \
+ ChildHaloID2 INTEGER, ChildHaloFrac2 FLOAT, \
+ ChildHaloID3 INTEGER, ChildHaloFrac3 FLOAT, \
+ ChildHaloID4 INTEGER, ChildHaloFrac4 FLOAT);"
+ self.cursor.execute(line)
+ self.conn.commit()
+ except sql.OperationalError:
+ pass
def _find_likely_children(self, parentfile, childfile):
# For each halo in the parent list, identify likely children in the
@@ -680,16 +694,33 @@
for insert in to_write:
temp_cursor.execute(line, insert)
temp_conn.commit()
+ temp_cursor.close()
+ temp_conn.close()
+ self._close_database()
+ os.rename(temp_name, self.database)
+
+ def _write_index(self):
mylog.info("Creating database index.")
line = "CREATE INDEX IF NOT EXISTS HalosIndex ON Halos ("
for name in columns:
line += name +","
line = line[:-1] + ");"
- temp_cursor.execute(line)
- temp_cursor.close()
- temp_conn.close()
- self._close_database()
- os.rename(temp_name, self.database)
+ self.cursor.execute(line)
+
+ def _find_progress(self):
+ # This queries the database to see how far along work has already come
+ # to identify parent->child relationships.
+ line = """SELECT ChildHaloID0, SnapZ from halos WHERE SnapHaloID = 0
+ ORDER BY SnapZ DESC;"""
+ self.cursor.execute(line)
+ results = self.cursor.fetchone()
+ while results:
+ results = list(results)
+ if results[0] == -1:
+ # We've hit a dump that does not have relationships. Save this.
+ return results[1] # the SnapZ.
+ results = self.cursor.fetchone()
+ return 0.
class MergerTreeConnect(DatabaseFunctions):
def __init__(self, database='halos.db'):
https://bitbucket.org/yt_analysis/yt/changeset/29b96b75361c/
changeset: 29b96b75361c
branch: yt
user: sskory
date: 2012-03-26 17:37:17
summary: Merging in recent changes.
affected #: 10 files
diff -r ffd65827bfa1ab6894b17196eac8d65b03b68184 -r 29b96b75361ceb3b96ae07950bb638cf74c991d4 yt/config.py
--- a/yt/config.py
+++ b/yt/config.py
@@ -52,7 +52,7 @@
pasteboard_repo = '',
test_storage_dir = '/does/not/exist',
enzo_db = '',
- hub_url = 'https://127.0.0.1:5000/',
+ hub_url = 'https://data.yt-project.org/upload',
hub_api_key = '',
)
# Here is the upgrade. We're actually going to parse the file in its entirety
diff -r ffd65827bfa1ab6894b17196eac8d65b03b68184 -r 29b96b75361ceb3b96ae07950bb638cf74c991d4 yt/convenience.py
--- a/yt/convenience.py
+++ b/yt/convenience.py
@@ -86,6 +86,12 @@
candidates = []
args = [os.path.expanduser(arg) if isinstance(arg, types.StringTypes)
else arg for arg in args]
+ valid_file = [os.path.isfile(arg) if isinstance(arg, types.StringTypes)
+ else False for arg in args]
+ if not any(valid_file):
+ mylog.error("None of the arguments provided to load() is a valid file")
+ mylog.error("Please check that you have used a correct path")
+ return None
for n, c in output_type_registry.items():
if n is None: continue
if c._is_valid(*args, **kwargs): candidates.append(n)
diff -r ffd65827bfa1ab6894b17196eac8d65b03b68184 -r 29b96b75361ceb3b96ae07950bb638cf74c991d4 yt/data_objects/static_output.py
--- a/yt/data_objects/static_output.py
+++ b/yt/data_objects/static_output.py
@@ -205,23 +205,16 @@
v = getattr(self, a)
mylog.info("Parameters: %-25s = %s", a, v)
- _field_info = None
def create_field_info(self):
- if getattr(self, "_field_info", None) is None:
+ if getattr(self, "field_info", None) is None:
# The setting up of fields occurs in the hierarchy, which is only
# instantiated once. So we have to double check to make sure that,
# in the event of double-loads of a parameter file, we do not blow
# away the exising field_info.
- self._field_info = FieldInfoContainer.create_with_fallback(
+ self.field_info = FieldInfoContainer.create_with_fallback(
self._fieldinfo_fallback)
- _get_hierarchy = True
- @property
- def field_info(self):
- if self._get_hierarchy:
- self._get_hierarchy=False
- self.hierarchy
- return self._field_info
+
def _reconstruct_pf(*args, **kwargs):
pfs = ParameterFileStore()
diff -r ffd65827bfa1ab6894b17196eac8d65b03b68184 -r 29b96b75361ceb3b96ae07950bb638cf74c991d4 yt/frontends/enzo/data_structures.py
--- a/yt/frontends/enzo/data_structures.py
+++ b/yt/frontends/enzo/data_structures.py
@@ -481,7 +481,7 @@
mylog.debug("Added additional grid %s", first_grid)
mylog.debug("Checking grids: %s", random_sample.tolist())
else:
- random_sample = na.mgrid[0:max(len(self.grids)-1,1)].astype("int32")
+ random_sample = na.mgrid[0:max(len(self.grids),1)].astype("int32")
return self.grids[(random_sample,)]
def find_particles_by_type(self, ptype, max_num=None, additional_fields=None):
diff -r ffd65827bfa1ab6894b17196eac8d65b03b68184 -r 29b96b75361ceb3b96ae07950bb638cf74c991d4 yt/utilities/minimal_representation.py
--- a/yt/utilities/minimal_representation.py
+++ b/yt/utilities/minimal_representation.py
@@ -31,18 +31,18 @@
from yt.config import ytcfg
from yt.funcs import *
-try:
- from poster.streaminghttp import register_openers
- from poster.encode import multipart_encode
- register_openers()
-except ImportError:
- pass
+from .poster.streaminghttp import register_openers
+from .poster.encode import multipart_encode
+register_openers()
class UploaderBar(object):
pbar = None
+ def __init__(self, my_name = ""):
+ self.my_name = my_name
+
def __call__(self, name, prog, total):
if self.pbar is None:
- self.pbar = get_pbar("Uploading %s" % name, total)
+ self.pbar = get_pbar("Uploading %s " % self.my_name, total)
self.pbar.update(prog)
if prog == total:
self.pbar.finish()
@@ -113,12 +113,12 @@
rv = urllib2.urlopen(request).read()
uploader_info = json.loads(rv)
new_url = url + "/handler/%s" % uploader_info['handler_uuid']
- for cn, cv in chunks:
+ for i, (cn, cv) in enumerate(chunks):
remaining = cv.size * cv.itemsize
f = TemporaryFile()
na.save(f, cv)
f.seek(0)
- pbar = UploaderBar()
+ pbar = UploaderBar("%s, % 2i/% 2i" % (self.type, i+1, len(chunks)))
datagen, headers = multipart_encode({'chunk_data' : f}, cb = pbar)
request = urllib2.Request(new_url, datagen, headers)
rv = urllib2.urlopen(request).read()
@@ -152,8 +152,7 @@
class MinimalMappableData(MinimalRepresentation):
- weight = "None"
- _attr_list = ("field_data", "field", "weight", "axis", "output_hash",
+ _attr_list = ("field_data", "field", "weight_field", "axis", "output_hash",
"vm_type")
def _generate_post(self):
@@ -169,6 +168,7 @@
class MinimalSliceData(MinimalMappableData):
type = 'slice'
vm_type = "Slice"
+ weight_field = "None"
class MinimalImageCollectionData(MinimalRepresentation):
type = "image_collection"
diff -r ffd65827bfa1ab6894b17196eac8d65b03b68184 -r 29b96b75361ceb3b96ae07950bb638cf74c991d4 yt/utilities/poster/README
--- /dev/null
+++ b/yt/utilities/poster/README
@@ -0,0 +1,4 @@
+Poster is a module by Chris AtLee, licensed under the MIT License, included
+here. For more information, see the poster home page:
+
+http://atlee.ca/software/poster
diff -r ffd65827bfa1ab6894b17196eac8d65b03b68184 -r 29b96b75361ceb3b96ae07950bb638cf74c991d4 yt/utilities/poster/__init__.py
--- /dev/null
+++ b/yt/utilities/poster/__init__.py
@@ -0,0 +1,32 @@
+# Copyright (c) 2011 Chris AtLee
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+"""poster module
+
+Support for streaming HTTP uploads, and multipart/form-data encoding
+
+```poster.version``` is a 3-tuple of integers representing the version number.
+New releases of poster will always have a version number that compares greater
+than an older version of poster.
+New in version 0.6."""
+
+import streaminghttp
+import encode
+
+version = (0, 8, 1) # Thanks JP!
diff -r ffd65827bfa1ab6894b17196eac8d65b03b68184 -r 29b96b75361ceb3b96ae07950bb638cf74c991d4 yt/utilities/poster/encode.py
--- /dev/null
+++ b/yt/utilities/poster/encode.py
@@ -0,0 +1,414 @@
+"""multipart/form-data encoding module
+
+This module provides functions that faciliate encoding name/value pairs
+as multipart/form-data suitable for a HTTP POST or PUT request.
+
+multipart/form-data is the standard way to upload files over HTTP"""
+
+__all__ = ['gen_boundary', 'encode_and_quote', 'MultipartParam',
+ 'encode_string', 'encode_file_header', 'get_body_size', 'get_headers',
+ 'multipart_encode']
+
+try:
+ import uuid
+ def gen_boundary():
+ """Returns a random string to use as the boundary for a message"""
+ return uuid.uuid4().hex
+except ImportError:
+ import random, sha
+ def gen_boundary():
+ """Returns a random string to use as the boundary for a message"""
+ bits = random.getrandbits(160)
+ return sha.new(str(bits)).hexdigest()
+
+import urllib, re, os, mimetypes
+try:
+ from email.header import Header
+except ImportError:
+ # Python 2.4
+ from email.Header import Header
+
+def encode_and_quote(data):
+ """If ``data`` is unicode, return urllib.quote_plus(data.encode("utf-8"))
+ otherwise return urllib.quote_plus(data)"""
+ if data is None:
+ return None
+
+ if isinstance(data, unicode):
+ data = data.encode("utf-8")
+ return urllib.quote_plus(data)
+
+def _strify(s):
+ """If s is a unicode string, encode it to UTF-8 and return the results,
+ otherwise return str(s), or None if s is None"""
+ if s is None:
+ return None
+ if isinstance(s, unicode):
+ return s.encode("utf-8")
+ return str(s)
+
+class MultipartParam(object):
+ """Represents a single parameter in a multipart/form-data request
+
+ ``name`` is the name of this parameter.
+
+ If ``value`` is set, it must be a string or unicode object to use as the
+ data for this parameter.
+
+ If ``filename`` is set, it is what to say that this parameter's filename
+ is. Note that this does not have to be the actual filename any local file.
+
+ If ``filetype`` is set, it is used as the Content-Type for this parameter.
+ If unset it defaults to "text/plain; charset=utf8"
+
+ If ``filesize`` is set, it specifies the length of the file ``fileobj``
+
+ If ``fileobj`` is set, it must be a file-like object that supports
+ .read().
+
+ Both ``value`` and ``fileobj`` must not be set, doing so will
+ raise a ValueError assertion.
+
+ If ``fileobj`` is set, and ``filesize`` is not specified, then
+ the file's size will be determined first by stat'ing ``fileobj``'s
+ file descriptor, and if that fails, by seeking to the end of the file,
+ recording the current position as the size, and then by seeking back to the
+ beginning of the file.
+
+ ``cb`` is a callable which will be called from iter_encode with (self,
+ current, total), representing the current parameter, current amount
+ transferred, and the total size.
+ """
+ def __init__(self, name, value=None, filename=None, filetype=None,
+ filesize=None, fileobj=None, cb=None):
+ self.name = Header(name).encode()
+ self.value = _strify(value)
+ if filename is None:
+ self.filename = None
+ else:
+ if isinstance(filename, unicode):
+ # Encode with XML entities
+ self.filename = filename.encode("ascii", "xmlcharrefreplace")
+ else:
+ self.filename = str(filename)
+ self.filename = self.filename.encode("string_escape").\
+ replace('"', '\\"')
+ self.filetype = _strify(filetype)
+
+ self.filesize = filesize
+ self.fileobj = fileobj
+ self.cb = cb
+
+ if self.value is not None and self.fileobj is not None:
+ raise ValueError("Only one of value or fileobj may be specified")
+
+ if fileobj is not None and filesize is None:
+ # Try and determine the file size
+ try:
+ self.filesize = os.fstat(fileobj.fileno()).st_size
+ except (OSError, AttributeError):
+ try:
+ fileobj.seek(0, 2)
+ self.filesize = fileobj.tell()
+ fileobj.seek(0)
+ except:
+ raise ValueError("Could not determine filesize")
+
+ def __cmp__(self, other):
+ attrs = ['name', 'value', 'filename', 'filetype', 'filesize', 'fileobj']
+ myattrs = [getattr(self, a) for a in attrs]
+ oattrs = [getattr(other, a) for a in attrs]
+ return cmp(myattrs, oattrs)
+
+ def reset(self):
+ if self.fileobj is not None:
+ self.fileobj.seek(0)
+ elif self.value is None:
+ raise ValueError("Don't know how to reset this parameter")
+
+ @classmethod
+ def from_file(cls, paramname, filename):
+ """Returns a new MultipartParam object constructed from the local
+ file at ``filename``.
+
+ ``filesize`` is determined by os.path.getsize(``filename``)
+
+ ``filetype`` is determined by mimetypes.guess_type(``filename``)[0]
+
+ ``filename`` is set to os.path.basename(``filename``)
+ """
+
+ return cls(paramname, filename=os.path.basename(filename),
+ filetype=mimetypes.guess_type(filename)[0],
+ filesize=os.path.getsize(filename),
+ fileobj=open(filename, "rb"))
+
+ @classmethod
+ def from_params(cls, params):
+ """Returns a list of MultipartParam objects from a sequence of
+ name, value pairs, MultipartParam instances,
+ or from a mapping of names to values
+
+ The values may be strings or file objects, or MultipartParam objects.
+ MultipartParam object names must match the given names in the
+ name,value pairs or mapping, if applicable."""
+ if hasattr(params, 'items'):
+ params = params.items()
+
+ retval = []
+ for item in params:
+ if isinstance(item, cls):
+ retval.append(item)
+ continue
+ name, value = item
+ if isinstance(value, cls):
+ assert value.name == name
+ retval.append(value)
+ continue
+ if hasattr(value, 'read'):
+ # Looks like a file object
+ filename = getattr(value, 'name', None)
+ if filename is not None:
+ filetype = mimetypes.guess_type(filename)[0]
+ else:
+ filetype = None
+
+ retval.append(cls(name=name, filename=filename,
+ filetype=filetype, fileobj=value))
+ else:
+ retval.append(cls(name, value))
+ return retval
+
+ def encode_hdr(self, boundary):
+ """Returns the header of the encoding of this parameter"""
+ boundary = encode_and_quote(boundary)
+
+ headers = ["--%s" % boundary]
+
+ if self.filename:
+ disposition = 'form-data; name="%s"; filename="%s"' % (self.name,
+ self.filename)
+ else:
+ disposition = 'form-data; name="%s"' % self.name
+
+ headers.append("Content-Disposition: %s" % disposition)
+
+ if self.filetype:
+ filetype = self.filetype
+ else:
+ filetype = "text/plain; charset=utf-8"
+
+ headers.append("Content-Type: %s" % filetype)
+
+ headers.append("")
+ headers.append("")
+
+ return "\r\n".join(headers)
+
+ def encode(self, boundary):
+ """Returns the string encoding of this parameter"""
+ if self.value is None:
+ value = self.fileobj.read()
+ else:
+ value = self.value
+
+ if re.search("^--%s$" % re.escape(boundary), value, re.M):
+ raise ValueError("boundary found in encoded string")
+
+ return "%s%s\r\n" % (self.encode_hdr(boundary), value)
+
+ def iter_encode(self, boundary, blocksize=4096):
+ """Yields the encoding of this parameter
+ If self.fileobj is set, then blocks of ``blocksize`` bytes are read and
+ yielded."""
+ total = self.get_size(boundary)
+ current = 0
+ if self.value is not None:
+ block = self.encode(boundary)
+ current += len(block)
+ yield block
+ if self.cb:
+ self.cb(self, current, total)
+ else:
+ block = self.encode_hdr(boundary)
+ current += len(block)
+ yield block
+ if self.cb:
+ self.cb(self, current, total)
+ last_block = ""
+ encoded_boundary = "--%s" % encode_and_quote(boundary)
+ boundary_exp = re.compile("^%s$" % re.escape(encoded_boundary),
+ re.M)
+ while True:
+ block = self.fileobj.read(blocksize)
+ if not block:
+ current += 2
+ yield "\r\n"
+ if self.cb:
+ self.cb(self, current, total)
+ break
+ last_block += block
+ if boundary_exp.search(last_block):
+ raise ValueError("boundary found in file data")
+ last_block = last_block[-len(encoded_boundary)-2:]
+ current += len(block)
+ yield block
+ if self.cb:
+ self.cb(self, current, total)
+
+ def get_size(self, boundary):
+ """Returns the size in bytes that this param will be when encoded
+ with the given boundary."""
+ if self.filesize is not None:
+ valuesize = self.filesize
+ else:
+ valuesize = len(self.value)
+
+ return len(self.encode_hdr(boundary)) + 2 + valuesize
+
+def encode_string(boundary, name, value):
+ """Returns ``name`` and ``value`` encoded as a multipart/form-data
+ variable. ``boundary`` is the boundary string used throughout
+ a single request to separate variables."""
+
+ return MultipartParam(name, value).encode(boundary)
+
+def encode_file_header(boundary, paramname, filesize, filename=None,
+ filetype=None):
+ """Returns the leading data for a multipart/form-data field that contains
+ file data.
+
+ ``boundary`` is the boundary string used throughout a single request to
+ separate variables.
+
+ ``paramname`` is the name of the variable in this request.
+
+ ``filesize`` is the size of the file data.
+
+ ``filename`` if specified is the filename to give to this field. This
+ field is only useful to the server for determining the original filename.
+
+ ``filetype`` if specified is the MIME type of this file.
+
+ The actual file data should be sent after this header has been sent.
+ """
+
+ return MultipartParam(paramname, filesize=filesize, filename=filename,
+ filetype=filetype).encode_hdr(boundary)
+
+def get_body_size(params, boundary):
+ """Returns the number of bytes that the multipart/form-data encoding
+ of ``params`` will be."""
+ size = sum(p.get_size(boundary) for p in MultipartParam.from_params(params))
+ return size + len(boundary) + 6
+
+def get_headers(params, boundary):
+ """Returns a dictionary with Content-Type and Content-Length headers
+ for the multipart/form-data encoding of ``params``."""
+ headers = {}
+ boundary = urllib.quote_plus(boundary)
+ headers['Content-Type'] = "multipart/form-data; boundary=%s" % boundary
+ headers['Content-Length'] = str(get_body_size(params, boundary))
+ return headers
+
+class multipart_yielder:
+ def __init__(self, params, boundary, cb):
+ self.params = params
+ self.boundary = boundary
+ self.cb = cb
+
+ self.i = 0
+ self.p = None
+ self.param_iter = None
+ self.current = 0
+ self.total = get_body_size(params, boundary)
+
+ def __iter__(self):
+ return self
+
+ def next(self):
+ """generator function to yield multipart/form-data representation
+ of parameters"""
+ if self.param_iter is not None:
+ try:
+ block = self.param_iter.next()
+ self.current += len(block)
+ if self.cb:
+ self.cb(self.p, self.current, self.total)
+ return block
+ except StopIteration:
+ self.p = None
+ self.param_iter = None
+
+ if self.i is None:
+ raise StopIteration
+ elif self.i >= len(self.params):
+ self.param_iter = None
+ self.p = None
+ self.i = None
+ block = "--%s--\r\n" % self.boundary
+ self.current += len(block)
+ if self.cb:
+ self.cb(self.p, self.current, self.total)
+ return block
+
+ self.p = self.params[self.i]
+ self.param_iter = self.p.iter_encode(self.boundary)
+ self.i += 1
+ return self.next()
+
+ def reset(self):
+ self.i = 0
+ self.current = 0
+ for param in self.params:
+ param.reset()
+
+def multipart_encode(params, boundary=None, cb=None):
+ """Encode ``params`` as multipart/form-data.
+
+ ``params`` should be a sequence of (name, value) pairs or MultipartParam
+ objects, or a mapping of names to values.
+ Values are either strings parameter values, or file-like objects to use as
+ the parameter value. The file-like objects must support .read() and either
+ .fileno() or both .seek() and .tell().
+
+ If ``boundary`` is set, then it as used as the MIME boundary. Otherwise
+ a randomly generated boundary will be used. In either case, if the
+ boundary string appears in the parameter values a ValueError will be
+ raised.
+
+ If ``cb`` is set, it should be a callback which will get called as blocks
+ of data are encoded. It will be called with (param, current, total),
+ indicating the current parameter being encoded, the current amount encoded,
+ and the total amount to encode.
+
+ Returns a tuple of `datagen`, `headers`, where `datagen` is a
+ generator that will yield blocks of data that make up the encoded
+ parameters, and `headers` is a dictionary with the assoicated
+ Content-Type and Content-Length headers.
+
+ Examples:
+
+ >>> datagen, headers = multipart_encode( [("key", "value1"), ("key", "value2")] )
+ >>> s = "".join(datagen)
+ >>> assert "value2" in s and "value1" in s
+
+ >>> p = MultipartParam("key", "value2")
+ >>> datagen, headers = multipart_encode( [("key", "value1"), p] )
+ >>> s = "".join(datagen)
+ >>> assert "value2" in s and "value1" in s
+
+ >>> datagen, headers = multipart_encode( {"key": "value1"} )
+ >>> s = "".join(datagen)
+ >>> assert "value2" not in s and "value1" in s
+
+ """
+ if boundary is None:
+ boundary = gen_boundary()
+ else:
+ boundary = urllib.quote_plus(boundary)
+
+ headers = get_headers(params, boundary)
+ params = MultipartParam.from_params(params)
+
+ return multipart_yielder(params, boundary, cb), headers
diff -r ffd65827bfa1ab6894b17196eac8d65b03b68184 -r 29b96b75361ceb3b96ae07950bb638cf74c991d4 yt/utilities/poster/streaminghttp.py
--- /dev/null
+++ b/yt/utilities/poster/streaminghttp.py
@@ -0,0 +1,199 @@
+"""Streaming HTTP uploads module.
+
+This module extends the standard httplib and urllib2 objects so that
+iterable objects can be used in the body of HTTP requests.
+
+In most cases all one should have to do is call :func:`register_openers()`
+to register the new streaming http handlers which will take priority over
+the default handlers, and then you can use iterable objects in the body
+of HTTP requests.
+
+**N.B.** You must specify a Content-Length header if using an iterable object
+since there is no way to determine in advance the total size that will be
+yielded, and there is no way to reset an interator.
+
+Example usage:
+
+>>> from StringIO import StringIO
+>>> import urllib2, poster.streaminghttp
+
+>>> opener = poster.streaminghttp.register_openers()
+
+>>> s = "Test file data"
+>>> f = StringIO(s)
+
+>>> req = urllib2.Request("http://localhost:5000", f,
+... {'Content-Length': str(len(s))})
+"""
+
+import httplib, urllib2, socket
+from httplib import NotConnected
+
+__all__ = ['StreamingHTTPConnection', 'StreamingHTTPRedirectHandler',
+ 'StreamingHTTPHandler', 'register_openers']
+
+if hasattr(httplib, 'HTTPS'):
+ __all__.extend(['StreamingHTTPSHandler', 'StreamingHTTPSConnection'])
+
+class _StreamingHTTPMixin:
+ """Mixin class for HTTP and HTTPS connections that implements a streaming
+ send method."""
+ def send(self, value):
+ """Send ``value`` to the server.
+
+ ``value`` can be a string object, a file-like object that supports
+ a .read() method, or an iterable object that supports a .next()
+ method.
+ """
+ # Based on python 2.6's httplib.HTTPConnection.send()
+ if self.sock is None:
+ if self.auto_open:
+ self.connect()
+ else:
+ raise NotConnected()
+
+ # send the data to the server. if we get a broken pipe, then close
+ # the socket. we want to reconnect when somebody tries to send again.
+ #
+ # NOTE: we DO propagate the error, though, because we cannot simply
+ # ignore the error... the caller will know if they can retry.
+ if self.debuglevel > 0:
+ print "send:", repr(value)
+ try:
+ blocksize = 8192
+ if hasattr(value, 'read') :
+ if hasattr(value, 'seek'):
+ value.seek(0)
+ if self.debuglevel > 0:
+ print "sendIng a read()able"
+ data = value.read(blocksize)
+ while data:
+ self.sock.sendall(data)
+ data = value.read(blocksize)
+ elif hasattr(value, 'next'):
+ if hasattr(value, 'reset'):
+ value.reset()
+ if self.debuglevel > 0:
+ print "sendIng an iterable"
+ for data in value:
+ self.sock.sendall(data)
+ else:
+ self.sock.sendall(value)
+ except socket.error, v:
+ if v[0] == 32: # Broken pipe
+ self.close()
+ raise
+
+class StreamingHTTPConnection(_StreamingHTTPMixin, httplib.HTTPConnection):
+ """Subclass of `httplib.HTTPConnection` that overrides the `send()` method
+ to support iterable body objects"""
+
+class StreamingHTTPRedirectHandler(urllib2.HTTPRedirectHandler):
+ """Subclass of `urllib2.HTTPRedirectHandler` that overrides the
+ `redirect_request` method to properly handle redirected POST requests
+
+ This class is required because python 2.5's HTTPRedirectHandler does
+ not remove the Content-Type or Content-Length headers when requesting
+ the new resource, but the body of the original request is not preserved.
+ """
+
+ handler_order = urllib2.HTTPRedirectHandler.handler_order - 1
+
+ # From python2.6 urllib2's HTTPRedirectHandler
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
+ """Return a Request or None in response to a redirect.
+
+ This is called by the http_error_30x methods when a
+ redirection response is received. If a redirection should
+ take place, return a new Request to allow http_error_30x to
+ perform the redirect. Otherwise, raise HTTPError if no-one
+ else should try to handle this url. Return None if you can't
+ but another Handler might.
+ """
+ m = req.get_method()
+ if (code in (301, 302, 303, 307) and m in ("GET", "HEAD")
+ or code in (301, 302, 303) and m == "POST"):
+ # Strictly (according to RFC 2616), 301 or 302 in response
+ # to a POST MUST NOT cause a redirection without confirmation
+ # from the user (of urllib2, in this case). In practice,
+ # essentially all clients do redirect in this case, so we
+ # do the same.
+ # be conciliant with URIs containing a space
+ newurl = newurl.replace(' ', '%20')
+ newheaders = dict((k, v) for k, v in req.headers.items()
+ if k.lower() not in (
+ "content-length", "content-type")
+ )
+ return urllib2.Request(newurl,
+ headers=newheaders,
+ origin_req_host=req.get_origin_req_host(),
+ unverifiable=True)
+ else:
+ raise urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)
+
+class StreamingHTTPHandler(urllib2.HTTPHandler):
+ """Subclass of `urllib2.HTTPHandler` that uses
+ StreamingHTTPConnection as its http connection class."""
+
+ handler_order = urllib2.HTTPHandler.handler_order - 1
+
+ def http_open(self, req):
+ """Open a StreamingHTTPConnection for the given request"""
+ return self.do_open(StreamingHTTPConnection, req)
+
+ def http_request(self, req):
+ """Handle a HTTP request. Make sure that Content-Length is specified
+ if we're using an interable value"""
+ # Make sure that if we're using an iterable object as the request
+ # body, that we've also specified Content-Length
+ if req.has_data():
+ data = req.get_data()
+ if hasattr(data, 'read') or hasattr(data, 'next'):
+ if not req.has_header('Content-length'):
+ raise ValueError(
+ "No Content-Length specified for iterable body")
+ return urllib2.HTTPHandler.do_request_(self, req)
+
+if hasattr(httplib, 'HTTPS'):
+ class StreamingHTTPSConnection(_StreamingHTTPMixin,
+ httplib.HTTPSConnection):
+ """Subclass of `httplib.HTTSConnection` that overrides the `send()`
+ method to support iterable body objects"""
+
+ class StreamingHTTPSHandler(urllib2.HTTPSHandler):
+ """Subclass of `urllib2.HTTPSHandler` that uses
+ StreamingHTTPSConnection as its http connection class."""
+
+ handler_order = urllib2.HTTPSHandler.handler_order - 1
+
+ def https_open(self, req):
+ return self.do_open(StreamingHTTPSConnection, req)
+
+ def https_request(self, req):
+ # Make sure that if we're using an iterable object as the request
+ # body, that we've also specified Content-Length
+ if req.has_data():
+ data = req.get_data()
+ if hasattr(data, 'read') or hasattr(data, 'next'):
+ if not req.has_header('Content-length'):
+ raise ValueError(
+ "No Content-Length specified for iterable body")
+ return urllib2.HTTPSHandler.do_request_(self, req)
+
+
+def get_handlers():
+ handlers = [StreamingHTTPHandler, StreamingHTTPRedirectHandler]
+ if hasattr(httplib, "HTTPS"):
+ handlers.append(StreamingHTTPSHandler)
+ return handlers
+
+def register_openers():
+ """Register the streaming http handlers in the global urllib2 default
+ opener object.
+
+ Returns the created OpenerDirector object."""
+ opener = urllib2.build_opener(*get_handlers())
+
+ urllib2.install_opener(opener)
+
+ return opener
diff -r ffd65827bfa1ab6894b17196eac8d65b03b68184 -r 29b96b75361ceb3b96ae07950bb638cf74c991d4 yt/visualization/volume_rendering/image_handling.py
--- a/yt/visualization/volume_rendering/image_handling.py
+++ b/yt/visualization/volume_rendering/image_handling.py
@@ -26,7 +26,6 @@
try: import pyfits
except: pass
import numpy as na
-import matplotlib; from matplotlib import pylab
from yt.funcs import *
@@ -82,6 +81,8 @@
elements. Optionally, *label*, *label_color* and *label_size* may be
specified.
"""
+ import matplotlib
+ import pylab
Nvec = image.shape[0]
image[na.isnan(image)] = 0.0
ma = image[image>0.0].max()
@@ -110,6 +111,7 @@
with "_rgb.png." *label*, *label_color* and *label_size* may also be
specified.
"""
+ import pylab
Nvec = image.shape[0]
image[na.isnan(image)] = 0.0
if image.shape[2] >= 4:
https://bitbucket.org/yt_analysis/yt/changeset/625d3a93fe00/
changeset: 625d3a93fe00
branch: yt
user: sskory
date: 2012-03-26 19:34:39
summary: Making the table creation SQL line auto-generated.
affected #: 1 file
diff -r 29b96b75361ceb3b96ae07950bb638cf74c991d4 -r 625d3a93fe00728571f1a2a1d19d54d474d5fb84 yt/analysis_modules/halo_merger_tree/merger_tree.py
--- a/yt/analysis_modules/halo_merger_tree/merger_tree.py
+++ b/yt/analysis_modules/halo_merger_tree/merger_tree.py
@@ -86,6 +86,22 @@
"ChildHaloID3", "ChildHaloFrac3",
"ChildHaloID4", "ChildHaloFrac4"]
+# Below we make the SQL command that creates the table "Halos" in the
+# database. This table is where all the data is stored.
+# Each column of data is named and its datatype is specified.
+# The GlobalHaloID is given the PRIMARY KEY property, which means that
+# the SQLite machinery assigns a consecutive and unique integer value
+# to that field automatically as each new entry is entered (that is,
+# if GlobalHaloID isn't specified already).
+create_db_line = "CREATE TABLE Halos ("
+for i, col in enumerate(columns):
+ if i == 0:
+ create_db_line += "%s %s PRIMARY KEY," % (col, column_types[col])
+ else:
+ create_db_line += " %s %s," % (col, column_types[col])
+# Clean of trailing comma, and closing stuff.
+create_db_line = create_db_line[:-1] + ");"
+
NumNeighbors = 15
NumDB = 5
@@ -301,21 +317,9 @@
self.cursor = self.conn.cursor()
def _create_halo_table(self):
- # Handle the error if it already exists.
+ # Handle the error if the table already exists by doing nothing.
try:
- # Create the table that will store the halo data.
- line = "CREATE TABLE Halos (GlobalHaloID INTEGER PRIMARY KEY,\
- SnapCurrentTimeIdentifier INTEGER, SnapZ FLOAT, SnapHaloID INTEGER, \
- HaloMass FLOAT,\
- NumPart INTEGER, CenMassX FLOAT, CenMassY FLOAT,\
- CenMassZ FLOAT, BulkVelX FLOAT, BulkVelY FLOAT, BulkVelZ FLOAT,\
- MaxRad FLOAT,\
- ChildHaloID0 INTEGER, ChildHaloFrac0 FLOAT, \
- ChildHaloID1 INTEGER, ChildHaloFrac1 FLOAT, \
- ChildHaloID2 INTEGER, ChildHaloFrac2 FLOAT, \
- ChildHaloID3 INTEGER, ChildHaloFrac3 FLOAT, \
- ChildHaloID4 INTEGER, ChildHaloFrac4 FLOAT);"
- self.cursor.execute(line)
+ self.cursor.execute(create_db_line)
self.conn.commit()
except sql.OperationalError:
pass
https://bitbucket.org/yt_analysis/yt/changeset/7dd26c1361e0/
changeset: 7dd26c1361e0
branch: yt
user: sskory
date: 2012-03-26 19:34:45
summary: Merge.
affected #: 1 file
diff -r 625d3a93fe00728571f1a2a1d19d54d474d5fb84 -r 7dd26c1361e0039222ffd6b93adaee70007dc145 yt/utilities/setup.py
--- a/yt/utilities/setup.py
+++ b/yt/utilities/setup.py
@@ -162,6 +162,7 @@
# https://bugzilla.redhat.com/show_bug.cgi?id=494579 ) we simply disable
# support for setjmp.
config.add_subpackage("amr_kdtree")
+ config.add_subpackage("poster")
config.add_subpackage("answer_testing")
config.add_subpackage("delaunay") # From SciPy, written by Robert Kern
config.add_subpackage("kdtree")
Repository URL: https://bitbucket.org/yt_analysis/yt/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the yt-svn
mailing list