[yt-svn] commit/yt: MatthewTurk: Adding config options for hub upload, as well as a simple hub upload command to

[Bitbucket]

1 new commit in yt:


https://bitbucket.org/yt_analysis/yt/changeset/02f940c0353b/
changeset:   02f940c0353b
branch:      yt
user:        MatthewTurk
date:        2012-02-13 23:08:59
summary:     Adding config options for hub upload, as well as a simple hub upload command to
the minimal representation objects.  These are not meant to be called directly,
but instead through an as-yet unwritten helper function that ensures both the
static output and the projection get uploaded.
affected #:  2 files

diff -r 3cb64c09895e4098d51f5072cf1d3fb7122ebb5a -r 02f940c0353bf4bface9c4d8fd0934ee9f56afbb yt/config.py
--- a/yt/config.py
+++ b/yt/config.py
@@ -52,6 +52,8 @@
     pasteboard_repo = '',
     test_storage_dir = '/does/not/exist',
     enzo_db = '',
+    hub_url = 'https://127.0.0.1:5000/',
+    hub_api_key = '',
     )
 # Here is the upgrade.  We're actually going to parse the file in its entirety
 # here.  Then, if it has any of the Forbidden Sections, it will be rewritten


diff -r 3cb64c09895e4098d51f5072cf1d3fb7122ebb5a -r 02f940c0353bf4bface9c4d8fd0934ee9f56afbb yt/utilities/minimal_representation.py
--- a/yt/utilities/minimal_representation.py
+++ b/yt/utilities/minimal_representation.py
@@ -23,7 +23,29 @@
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
+import numpy as na
 import abc
+import json
+import urllib2
+from tempfile import TemporaryFile
+from yt.config import ytcfg
+from yt.funcs import *
+
+try:
+    from poster.streaminghttp import register_openers
+    from poster.encode import multipart_encode
+    register_openers()
+except ImportError:
+    pass
+
+class UploaderBar(object):
+    pbar = None
+    def __call__(self, name, prog, total):
+        if self.pbar is None:
+            self.pbar = get_pbar("Uploading %s" % name, total)
+        self.pbar.update(prog)
+        if prog == total:
+            self.pbar.finish()
 
 class ContainerClass(object):
     pass
@@ -67,6 +89,45 @@
             setattr(cc, a, v)
         return cls(cc)
 
+    def upload(self):
+        api_key = ytcfg.get("yt","hub_api_key")
+        url = ytcfg.get("yt","hub_url")
+        metadata, (final_name, chunks) = self._generate_post()
+        for i in metadata:
+            if isinstance(metadata[i], na.ndarray):
+                metadata[i] = metadata[i].tolist()
+        metadata['obj_type'] = self.type
+        if len(chunks) == 0:
+            chunk_info = {'chunks': []}
+        else:
+            chunk_info = {'final_name' : final_name, 'chunks': []}
+            for cn, cv in chunks:
+                chunk_info['chunks'].append((cn, cv.size * cv.itemsize))
+        metadata = json.dumps(metadata)
+        chunk_info = json.dumps(chunk_info)
+        datagen, headers = multipart_encode({'metadata' : metadata,
+                                             'chunk_info' : chunk_info,
+                                             'api_key' : api_key})
+        request = urllib2.Request(url, datagen, headers)
+        # Actually do the request, and get the response
+        rv = urllib2.urlopen(request).read()
+        uploader_info = json.loads(rv)
+        new_url = url + "/handler/%s" % uploader_info['handler_uuid']
+        for cn, cv in chunks:
+            remaining = cv.size * cv.itemsize
+            f = TemporaryFile()
+            na.save(f, cv)
+            f.seek(0)
+            pbar = UploaderBar()
+            datagen, headers = multipart_encode({'chunk_data' : f}, cb = pbar)
+            request = urllib2.Request(new_url, datagen, headers)
+            rv = urllib2.urlopen(request).read()
+
+        datagen, headers = multipart_encode({'status' : 'FINAL'})
+        request = urllib2.Request(new_url, datagen, headers)
+        rv = urllib2.urlopen(request).read()
+        return json.loads(rv)
+
 class FilteredRepresentation(MinimalRepresentation):
     def _generate_post(self):
         raise RuntimeError
@@ -77,6 +138,7 @@
                   "unique_identifier", "current_redshift", "output_hash",
                   "cosmological_simulation", "omega_matter", "omega_lambda",
                   "hubble_constant", "name")
+    type = 'simulation_output'
 
     def __init__(self, obj):
         super(MinimalStaticOutput, self).__init__(obj)
@@ -86,7 +148,7 @@
     def _generate_post(self):
         metadata = self._attrs
         chunks = []
-        return metadata, chunks
+        return (metadata, (None, chunks))
 
 class MinimalMappableData(MinimalRepresentation):
 
@@ -97,10 +159,7 @@
         nobj = self._return_filtered_object(("field_data",))
         metadata = nobj._attrs
         chunks = [(arr, self.field_data[arr]) for arr in self.field_data]
-        return (metadata, chunks)
+        return (metadata, ('field_data', chunks))
 
 class MinimalProjectionData(MinimalMappableData):
-
-    def __init__(self, obj):
-        super(MinimalProjectionData, self).__init__(obj)
-        self.type = "proj"
+    type = 'proj'

Repository URL: https://bitbucket.org/yt_analysis/yt/

--

This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.