[yt-svn] commit/yt: ngoldbaum: Merged in xarthisius/yt (pull request #2495)

[commits-noreply at bitbucket.org]

1 new commit in yt:

https://bitbucket.org/yt_analysis/yt/commits/c23ff278bd32/
Changeset:   c23ff278bd32
Branch:      yt
User:        ngoldbaum
Date:        2017-01-23 21:36:22+00:00
Summary:     Merged in xarthisius/yt (pull request #2495)

Add option for listing available datasets in 'yt download'
Affected #:  2 files

diff -r b8437fc05d8a13b5bdc6df93fb52a1f85f084cc6 -r c23ff278bd328ac2f2944502c46acdb072be10b5 yt/funcs.py
--- a/yt/funcs.py
+++ b/yt/funcs.py
@@ -597,6 +597,35 @@
     return contents
 
 def download_file(url, filename):
+    requests = get_requests()
+    if requests is None:
+        return simple_download_file(url, filename)
+    else:
+        return fancy_download_file(url, filename, requests)
+
+def fancy_download_file(url, filename, requests=None):
+    response = requests.get(url, stream=True)
+    total_length = response.headers.get('content-length')
+
+    with open(filename, 'wb') as fh:
+        if total_length is None:
+            fh.write(response.content)
+        else:
+            blocksize = 4 * 1024 ** 2
+            iterations = int(float(total_length)/float(blocksize))
+
+            pbar = get_pbar(
+                'Downloading %s to %s ' % os.path.split(filename)[::-1],
+                iterations)
+            iteration = 0
+            for chunk in response.iter_content(chunk_size=blocksize):
+                fh.write(chunk)
+                iteration += 1
+                pbar.update(iteration)
+            pbar.finish()
+    return filename
+
+def simple_download_file(url, filename):
     class MyURLopener(urllib.request.FancyURLopener):
         def http_error_default(self, url, fp, errcode, errmsg, headers):
             raise RuntimeError("Attempt to download file from %s failed with error %s: %s." % \

diff -r b8437fc05d8a13b5bdc6df93fb52a1f85f084cc6 -r c23ff278bd328ac2f2944502c46acdb072be10b5 yt/utilities/command_line.py
--- a/yt/utilities/command_line.py
+++ b/yt/utilities/command_line.py
@@ -25,6 +25,7 @@
 import tempfile
 import json
 import pprint
+import textwrap
 
 from yt.config import ytcfg, CURRENT_CONFIG_FILE
 ytcfg["yt","__command_line"] = "True"
@@ -1313,16 +1314,21 @@
 
 class YTDownloadData(YTCommand):
 
-    args = (dict(short="filename", action="store", type=str,
-                 help="The name of the file to download"), 
-            dict(short="location", action="store", type=str,
-                 help="The location in which to place the file, can be "
-                      "\"supp_data_dir\", \"test_data_dir\", or any valid "
-                      "path on disk. "),
-            dict(longname="--clobber", short="-c",
-                 help="Output full contents of parameter file",
-                 action="store_true", default=False),
-            )
+    args = (
+        dict(short="filename", action="store", type=str,
+             help="The name of the file to download", nargs='?',
+             default=''), 
+        dict(short="location", action="store", type=str, nargs='?',
+             help="The location in which to place the file, can be "
+                  "\"supp_data_dir\", \"test_data_dir\", or any valid "
+                  "path on disk. ", default=''),
+        dict(longname="--overwrite", short="-c",
+             help="Overwrite existing file.",
+             action="store_true", default=False),
+        dict(longname="--list", short="-l",
+             help="Display all available files.",
+             action="store_true", default=False),
+    )
     description = \
         """
         Download a file from http://yt-project.org/data and save it to a 
@@ -1333,6 +1339,16 @@
     name = "download"
 
     def __call__(self, args):
+        if args.list:
+            self.get_list()
+            return
+        if not args.filename:
+            raise RuntimeError('You need to provide a filename. See --help '
+                               'for details or use --list to get available '
+                               'datasets.')
+        elif not args.location:
+            raise RuntimeError('You need to specify download location. See '
+                               '--help for details.')
         data_url = "http://yt-project.org/data/%s" % args.filename
         if args.location in ["test_data_dir", "supp_data_dir"]:
             data_dir = ytcfg.get("yt", args.location)
@@ -1344,11 +1360,27 @@
             print("The directory '%s' does not exist. Creating..." % data_dir)
             os.mkdir(data_dir)
         data_file = os.path.join(data_dir, args.filename)
-        if os.path.exists(data_file) and not args.clobber:
-            raise IOError("File '%s' exists and clobber=False!" % data_file)
+        if os.path.exists(data_file) and not args.overwrite:
+            raise IOError("File '%s' exists and overwrite=False!" % data_file)
+        print("Attempting to download file: %s" % args.filename)
         fn = download_file(data_url, data_file)
+
         if not os.path.exists(fn):
             raise IOError("The file '%s' did not download!!" % args.filename)
+        print("File: %s downloaded successfully to %s" %
+              (args.filename, data_file))
+
+    def get_list(self):
+        data = urllib.request.urlopen(
+            'http://yt-project.org/data/datafiles.json').read().decode('utf8')
+        data = json.loads(data)
+        for key in data:
+            for ds in data[key]:
+                ds['fullname'] = ds['url'].replace(
+                    'http://yt-project.org/data/', '')
+                print('{fullname} ({size}) type: {code}'.format(**ds))
+                for line in textwrap.wrap(ds['description']):
+                    print('\t', line)
 
 def run_main():
     args = parser.parse_args()

Repository URL: https://bitbucket.org/yt_analysis/yt/

--

This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.