[yt-svn] commit/yt: 7 new changesets
commits-noreply at bitbucket.org
commits-noreply at bitbucket.org
Mon Jan 23 13:36:50 PST 2017
7 new commits in yt:
https://bitbucket.org/yt_analysis/yt/commits/6d50be80ec68/
Changeset: 6d50be80ec68
Branch: yt
User: xarthisius
Date: 2017-01-17 21:19:29+00:00
Summary: Add option for listing available datasets in 'yt download'
Affected #: 1 file
diff -r fc3d1369fd2821a203b0bae4b94b53915254f468 -r 6d50be80ec68c6859287c0ba9282689f35b44c7f yt/utilities/command_line.py
--- a/yt/utilities/command_line.py
+++ b/yt/utilities/command_line.py
@@ -25,6 +25,7 @@
import tempfile
import json
import pprint
+import textwrap
from yt.config import ytcfg, CURRENT_CONFIG_FILE
ytcfg["yt","__command_line"] = "True"
@@ -1313,16 +1314,21 @@
class YTDownloadData(YTCommand):
- args = (dict(short="filename", action="store", type=str,
- help="The name of the file to download"),
- dict(short="location", action="store", type=str,
- help="The location in which to place the file, can be "
- "\"supp_data_dir\", \"test_data_dir\", or any valid "
- "path on disk. "),
- dict(longname="--clobber", short="-c",
- help="Output full contents of parameter file",
- action="store_true", default=False),
- )
+ args = (
+ dict(short="filename", action="store", type=str,
+ help="The name of the file to download", nargs='?',
+ default=''),
+ dict(short="location", action="store", type=str, nargs='?',
+ help="The location in which to place the file, can be "
+ "\"supp_data_dir\", \"test_data_dir\", or any valid "
+ "path on disk. ", default=''),
+ dict(longname="--clobber", short="-c",
+ help="Overwrite existing file.",
+ action="store_true", default=False),
+ dict(longname="--list", short="-l",
+ help="Display all available files.",
+ action="store_true", default=False),
+ )
description = \
"""
Download a file from http://yt-project.org/data and save it to a
@@ -1333,6 +1339,16 @@
name = "download"
def __call__(self, args):
+ if args.list:
+ self.get_list()
+ return
+ if not args.filename:
+ raise RuntimeError('You need to provide a filename. See --help '
+ 'for details or use --list to get available '
+ 'datasets.')
+ elif not args.location:
+ raise RuntimeError('You need to specify download location. See '
+ '--help for details.')
data_url = "http://yt-project.org/data/%s" % args.filename
if args.location in ["test_data_dir", "supp_data_dir"]:
data_dir = ytcfg.get("yt", args.location)
@@ -1350,6 +1366,16 @@
if not os.path.exists(fn):
raise IOError("The file '%s' did not download!!" % args.filename)
+ def get_list(self):
+ data = urllib.request.urlopen(
+ 'http://yt-project.org/data/datafiles.json').read().decode('utf8')
+ data = json.loads(data)
+ for key in data.keys():
+ for ds in data[key]:
+ print(ds['filename'] + '.tar.gz')
+ for line in textwrap.wrap(ds['description']):
+ print('\t', line)
+
def run_main():
args = parser.parse_args()
# The following is a workaround for a nasty Python 3 bug:
https://bitbucket.org/yt_analysis/yt/commits/5d4fcfad488f/
Changeset: 5d4fcfad488f
Branch: yt
User: xarthisius
Date: 2017-01-17 21:32:29+00:00
Summary: Include more data, drop pointless .keys() in loop
Affected #: 1 file
diff -r 6d50be80ec68c6859287c0ba9282689f35b44c7f -r 5d4fcfad488f964a1a5172ddec5a8b0270665fc4 yt/utilities/command_line.py
--- a/yt/utilities/command_line.py
+++ b/yt/utilities/command_line.py
@@ -1370,9 +1370,9 @@
data = urllib.request.urlopen(
'http://yt-project.org/data/datafiles.json').read().decode('utf8')
data = json.loads(data)
- for key in data.keys():
+ for key in data:
for ds in data[key]:
- print(ds['filename'] + '.tar.gz')
+ print('{filename}.tar.gz ({size}) type: {code}'.format(**ds))
for line in textwrap.wrap(ds['description']):
print('\t', line)
https://bitbucket.org/yt_analysis/yt/commits/436d36b6a543/
Changeset: 436d36b6a543
Branch: yt
User: xarthisius
Date: 2017-01-17 21:34:35+00:00
Summary: Rename '--clobber' to '--overwrite'
Affected #: 1 file
diff -r 5d4fcfad488f964a1a5172ddec5a8b0270665fc4 -r 436d36b6a5439cf3645c9ab01f9d0302cd369bdd yt/utilities/command_line.py
--- a/yt/utilities/command_line.py
+++ b/yt/utilities/command_line.py
@@ -1322,7 +1322,7 @@
help="The location in which to place the file, can be "
"\"supp_data_dir\", \"test_data_dir\", or any valid "
"path on disk. ", default=''),
- dict(longname="--clobber", short="-c",
+ dict(longname="--overwrite", short="-c",
help="Overwrite existing file.",
action="store_true", default=False),
dict(longname="--list", short="-l",
@@ -1360,8 +1360,8 @@
print("The directory '%s' does not exist. Creating..." % data_dir)
os.mkdir(data_dir)
data_file = os.path.join(data_dir, args.filename)
- if os.path.exists(data_file) and not args.clobber:
- raise IOError("File '%s' exists and clobber=False!" % data_file)
+ if os.path.exists(data_file) and not args.overwrite:
+ raise IOError("File '%s' exists and overwrite=False!" % data_file)
fn = download_file(data_url, data_file)
if not os.path.exists(fn):
raise IOError("The file '%s' did not download!!" % args.filename)
https://bitbucket.org/yt_analysis/yt/commits/ae8ee7c2b6de/
Changeset: ae8ee7c2b6de
Branch: yt
User: xarthisius
Date: 2017-01-17 22:01:54+00:00
Summary: Add download helper with progress bar
Affected #: 1 file
diff -r 436d36b6a5439cf3645c9ab01f9d0302cd369bdd -r ae8ee7c2b6de419d33797a178c2ab60f62c826d8 yt/funcs.py
--- a/yt/funcs.py
+++ b/yt/funcs.py
@@ -597,6 +597,35 @@
return contents
def download_file(url, filename):
+ requests = get_requests()
+ if requests is None:
+ return simple_download_file(url, filename)
+ else:
+ return fancy_download_file(url, filename, requests)
+
+def fancy_download_file(url, filename, requests=None):
+ response = requests.get(url, stream=True)
+ total_length = response.headers.get('content-length')
+
+ with open(filename, 'wb') as fh:
+ if total_length is None:
+ fh.write(response.content)
+ else:
+ blocksize = 4 * 1024 ** 2
+ iterations = int(float(total_length)/float(blocksize))
+
+ pbar = get_pbar(
+ 'Downloading %s to %s ' % os.path.split(filename)[::-1],
+ iterations)
+ iteration = 0
+ for chunk in response.iter_content(chunk_size=blocksize):
+ fh.write(chunk)
+ iteration += 1
+ pbar.update(iteration)
+ pbar.finish()
+ return filename
+
+def simple_download_file(url, filename):
class MyURLopener(urllib.request.FancyURLopener):
def http_error_default(self, url, fp, errcode, errmsg, headers):
raise RuntimeError("Attempt to download file from %s failed with error %s: %s." % \
https://bitbucket.org/yt_analysis/yt/commits/6f4c7a5dcc30/
Changeset: 6f4c7a5dcc30
Branch: yt
User: xarthisius
Date: 2017-01-17 22:12:12+00:00
Summary: Add messages at the beginning and successful finish of a file download
Affected #: 1 file
diff -r ae8ee7c2b6de419d33797a178c2ab60f62c826d8 -r 6f4c7a5dcc30dbcaaf140d36eea518ed4b6a43c0 yt/utilities/command_line.py
--- a/yt/utilities/command_line.py
+++ b/yt/utilities/command_line.py
@@ -1362,9 +1362,13 @@
data_file = os.path.join(data_dir, args.filename)
if os.path.exists(data_file) and not args.overwrite:
raise IOError("File '%s' exists and overwrite=False!" % data_file)
+ print("Attempting to download file: %s" % args.filename)
fn = download_file(data_url, data_file)
+
if not os.path.exists(fn):
raise IOError("The file '%s' did not download!!" % args.filename)
+ print("File: %s downloaded successfully to %s" %
+ (args.filename, data_file))
def get_list(self):
data = urllib.request.urlopen(
https://bitbucket.org/yt_analysis/yt/commits/b231bd306cbd/
Changeset: b231bd306cbd
Branch: yt
User: xarthisius
Date: 2017-01-23 15:18:07+00:00
Summary: Read the filename from the 'url' attribute instead of making assumptions about the suffix
Affected #: 1 file
diff -r 6f4c7a5dcc30dbcaaf140d36eea518ed4b6a43c0 -r b231bd306cbd020647cfa99b0891811486d74264 yt/utilities/command_line.py
--- a/yt/utilities/command_line.py
+++ b/yt/utilities/command_line.py
@@ -1376,7 +1376,9 @@
data = json.loads(data)
for key in data:
for ds in data[key]:
- print('{filename}.tar.gz ({size}) type: {code}'.format(**ds))
+ ds['fullname'] = ds['url'].replace(
+ 'http://yt-project.org/data/', '')
+ print('{fullname} ({size}) type: {code}'.format(**ds))
for line in textwrap.wrap(ds['description']):
print('\t', line)
https://bitbucket.org/yt_analysis/yt/commits/c23ff278bd32/
Changeset: c23ff278bd32
Branch: yt
User: ngoldbaum
Date: 2017-01-23 21:36:22+00:00
Summary: Merged in xarthisius/yt (pull request #2495)
Add option for listing available datasets in 'yt download'
Affected #: 2 files
diff -r b8437fc05d8a13b5bdc6df93fb52a1f85f084cc6 -r c23ff278bd328ac2f2944502c46acdb072be10b5 yt/funcs.py
--- a/yt/funcs.py
+++ b/yt/funcs.py
@@ -597,6 +597,35 @@
return contents
def download_file(url, filename):
+ requests = get_requests()
+ if requests is None:
+ return simple_download_file(url, filename)
+ else:
+ return fancy_download_file(url, filename, requests)
+
+def fancy_download_file(url, filename, requests=None):
+ response = requests.get(url, stream=True)
+ total_length = response.headers.get('content-length')
+
+ with open(filename, 'wb') as fh:
+ if total_length is None:
+ fh.write(response.content)
+ else:
+ blocksize = 4 * 1024 ** 2
+ iterations = int(float(total_length)/float(blocksize))
+
+ pbar = get_pbar(
+ 'Downloading %s to %s ' % os.path.split(filename)[::-1],
+ iterations)
+ iteration = 0
+ for chunk in response.iter_content(chunk_size=blocksize):
+ fh.write(chunk)
+ iteration += 1
+ pbar.update(iteration)
+ pbar.finish()
+ return filename
+
+def simple_download_file(url, filename):
class MyURLopener(urllib.request.FancyURLopener):
def http_error_default(self, url, fp, errcode, errmsg, headers):
raise RuntimeError("Attempt to download file from %s failed with error %s: %s." % \
diff -r b8437fc05d8a13b5bdc6df93fb52a1f85f084cc6 -r c23ff278bd328ac2f2944502c46acdb072be10b5 yt/utilities/command_line.py
--- a/yt/utilities/command_line.py
+++ b/yt/utilities/command_line.py
@@ -25,6 +25,7 @@
import tempfile
import json
import pprint
+import textwrap
from yt.config import ytcfg, CURRENT_CONFIG_FILE
ytcfg["yt","__command_line"] = "True"
@@ -1313,16 +1314,21 @@
class YTDownloadData(YTCommand):
- args = (dict(short="filename", action="store", type=str,
- help="The name of the file to download"),
- dict(short="location", action="store", type=str,
- help="The location in which to place the file, can be "
- "\"supp_data_dir\", \"test_data_dir\", or any valid "
- "path on disk. "),
- dict(longname="--clobber", short="-c",
- help="Output full contents of parameter file",
- action="store_true", default=False),
- )
+ args = (
+ dict(short="filename", action="store", type=str,
+ help="The name of the file to download", nargs='?',
+ default=''),
+ dict(short="location", action="store", type=str, nargs='?',
+ help="The location in which to place the file, can be "
+ "\"supp_data_dir\", \"test_data_dir\", or any valid "
+ "path on disk. ", default=''),
+ dict(longname="--overwrite", short="-c",
+ help="Overwrite existing file.",
+ action="store_true", default=False),
+ dict(longname="--list", short="-l",
+ help="Display all available files.",
+ action="store_true", default=False),
+ )
description = \
"""
Download a file from http://yt-project.org/data and save it to a
@@ -1333,6 +1339,16 @@
name = "download"
def __call__(self, args):
+ if args.list:
+ self.get_list()
+ return
+ if not args.filename:
+ raise RuntimeError('You need to provide a filename. See --help '
+ 'for details or use --list to get available '
+ 'datasets.')
+ elif not args.location:
+ raise RuntimeError('You need to specify download location. See '
+ '--help for details.')
data_url = "http://yt-project.org/data/%s" % args.filename
if args.location in ["test_data_dir", "supp_data_dir"]:
data_dir = ytcfg.get("yt", args.location)
@@ -1344,11 +1360,27 @@
print("The directory '%s' does not exist. Creating..." % data_dir)
os.mkdir(data_dir)
data_file = os.path.join(data_dir, args.filename)
- if os.path.exists(data_file) and not args.clobber:
- raise IOError("File '%s' exists and clobber=False!" % data_file)
+ if os.path.exists(data_file) and not args.overwrite:
+ raise IOError("File '%s' exists and overwrite=False!" % data_file)
+ print("Attempting to download file: %s" % args.filename)
fn = download_file(data_url, data_file)
+
if not os.path.exists(fn):
raise IOError("The file '%s' did not download!!" % args.filename)
+ print("File: %s downloaded successfully to %s" %
+ (args.filename, data_file))
+
+ def get_list(self):
+ data = urllib.request.urlopen(
+ 'http://yt-project.org/data/datafiles.json').read().decode('utf8')
+ data = json.loads(data)
+ for key in data:
+ for ds in data[key]:
+ ds['fullname'] = ds['url'].replace(
+ 'http://yt-project.org/data/', '')
+ print('{fullname} ({size}) type: {code}'.format(**ds))
+ for line in textwrap.wrap(ds['description']):
+ print('\t', line)
def run_main():
args = parser.parse_args()
Repository URL: https://bitbucket.org/yt_analysis/yt/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the yt-svn
mailing list