[yt-svn] commit/yt: MatthewTurk: Merged in xarthisius/yt (pull request #2289)
commits-noreply at bitbucket.org
commits-noreply at bitbucket.org
Tue Aug 9 09:14:02 PDT 2016
1 new commit in yt:
https://bitbucket.org/yt_analysis/yt/commits/42d65f24f2d8/
Changeset: 42d65f24f2d8
Branch: yt
User: MatthewTurk
Date: 2016-08-09 16:13:24+00:00
Summary: Merged in xarthisius/yt (pull request #2289)
Update config file handling and 'yt hub_register/upload_notebook/hubstart'
Affected #: 20 files
diff -r b1165fbb7bd24dcdbf61093a685f540567a7de87 -r 42d65f24f2d8f10b60704b33038c976cdcf0c580 doc/extensions/config_help.py
--- /dev/null
+++ b/doc/extensions/config_help.py
@@ -0,0 +1,34 @@
+import re
+import subprocess
+from docutils import statemachine
+from sphinx.util.compat import Directive
+
+def setup(app):
+ app.add_directive('config_help', GetConfigHelp)
+ setup.app = app
+ setup.config = app.config
+ setup.confdir = app.confdir
+
+ retdict = dict(
+ version='1.0',
+ parallel_read_safe=True,
+ parallel_write_safe=True
+ )
+
+ return retdict
+
+class GetConfigHelp(Directive):
+ required_arguments = 1
+ optional_arguments = 0
+ final_argument_whitespace = True
+
+ def run(self):
+ rst_file = self.state_machine.document.attributes['source']
+ data = subprocess.check_output(
+ self.arguments[0].split(" ") + ['-h']).decode('utf8').split('\n')
+ ind = next((i for i, val in enumerate(data)
+ if re.match('\s{0,3}\{.*\}\s*$', val)))
+ lines = ['.. code-block:: none', ''] + data[ind + 1:]
+ self.state_machine.insert_input(
+ statemachine.string2lines("\n".join(lines)), rst_file)
+ return []
diff -r b1165fbb7bd24dcdbf61093a685f540567a7de87 -r 42d65f24f2d8f10b60704b33038c976cdcf0c580 doc/source/_static/apiKey01.jpg
Binary file doc/source/_static/apiKey01.jpg has changed
diff -r b1165fbb7bd24dcdbf61093a685f540567a7de87 -r 42d65f24f2d8f10b60704b33038c976cdcf0c580 doc/source/_static/apiKey02.jpg
Binary file doc/source/_static/apiKey02.jpg has changed
diff -r b1165fbb7bd24dcdbf61093a685f540567a7de87 -r 42d65f24f2d8f10b60704b33038c976cdcf0c580 doc/source/_static/apiKey03.jpg
Binary file doc/source/_static/apiKey03.jpg has changed
diff -r b1165fbb7bd24dcdbf61093a685f540567a7de87 -r 42d65f24f2d8f10b60704b33038c976cdcf0c580 doc/source/_static/apiKey04.jpg
Binary file doc/source/_static/apiKey04.jpg has changed
diff -r b1165fbb7bd24dcdbf61093a685f540567a7de87 -r 42d65f24f2d8f10b60704b33038c976cdcf0c580 doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -31,7 +31,8 @@
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx',
'sphinx.ext.pngmath', 'sphinx.ext.viewcode',
- 'sphinx.ext.napoleon', 'yt_cookbook', 'yt_colormaps']
+ 'sphinx.ext.napoleon', 'yt_cookbook', 'yt_colormaps',
+ 'config_help']
if not on_rtd:
extensions.append('sphinx.ext.autosummary')
diff -r b1165fbb7bd24dcdbf61093a685f540567a7de87 -r 42d65f24f2d8f10b60704b33038c976cdcf0c580 doc/source/cookbook/yt_gadget_owls_analysis.ipynb
--- a/doc/source/cookbook/yt_gadget_owls_analysis.ipynb
+++ b/doc/source/cookbook/yt_gadget_owls_analysis.ipynb
@@ -20,7 +20,7 @@
"source": [
"The first thing you will need to run these examples is a working installation of yt. The author or these examples followed the instructions under \"Get yt: from source\" at http://yt-project.org/ to install an up to date development version of yt.\n",
"\n",
- "Next you should set the default ``test_data_dir`` in the ``.yt/config`` file in your home directory. Note that you may have to create the directory and file if it doesn't exist already.\n",
+ "Next you should set the default ``test_data_dir`` in the ``~/.config/yt/ytrc`` file in your home directory. Note that you may have to create the directory and file if it doesn't exist already.\n",
"\n",
"> [yt]\n",
"\n",
diff -r b1165fbb7bd24dcdbf61093a685f540567a7de87 -r 42d65f24f2d8f10b60704b33038c976cdcf0c580 doc/source/developing/testing.rst
--- a/doc/source/developing/testing.rst
+++ b/doc/source/developing/testing.rst
@@ -285,15 +285,12 @@
These datasets are available at http://yt-project.org/data/.
-Next, modify the file ``~/.yt/config`` to include a section ``[yt]``
-with the parameter ``test_data_dir``. Set this to point to the
-directory with the test data you want to test with. Here is an example
-config file:
+Next, add the config parameter ``test_data_dir`` pointing to
+directory with the test data you want to test with, e.g.:
.. code-block:: none
- [yt]
- test_data_dir = /Users/tomservo/src/yt-data
+ $ yt config set yt test_data_dir /Users/tomservo/src/yt-data
More data will be added over time. To run the answer tests, you must first
generate a set of test answers locally on a "known good" revision, then update
@@ -313,7 +310,7 @@
This command will create a set of local answers from the tipsy frontend tests
and store them in ``$HOME/Documents/test`` (this can but does not have to be the
same directory as the ``test_data_dir`` configuration variable defined in your
-``.yt/config`` file) in a file named ``local-tipsy``. To run the tipsy
+``~/.config/yt/ytrc`` file) in a file named ``local-tipsy``. To run the tipsy
frontend's answer tests using a different yt changeset, update to that
changeset, recompile if necessary, and run the tests using the following
command:
diff -r b1165fbb7bd24dcdbf61093a685f540567a7de87 -r 42d65f24f2d8f10b60704b33038c976cdcf0c580 doc/source/faq/index.rst
--- a/doc/source/faq/index.rst
+++ b/doc/source/faq/index.rst
@@ -388,10 +388,10 @@
To make things easier to load these sample datasets, you can add the parent
directory to your downloaded sample data to your *yt path*.
If you set the option ``test_data_dir``, in the section ``[yt]``,
-in ``~/.yt/config``, yt will search this path for them.
+in ``~/.config/yt/ytrc``, yt will search this path for them.
This means you can download these datasets to ``/big_drive/data_for_yt`` , add
-the appropriate item to ``~/.yt/config``, and no matter which directory you are
+the appropriate item to ``~/.config/yt/ytrc``, and no matter which directory you are
in when running yt, it will also check in *that* directory.
@@ -437,12 +437,11 @@
hand, you may want it to output a lot more, since you can't figure out exactly what's going
wrong, and you want to output some debugging information. The yt log level can be
changed using the :ref:`configuration-file`, either by setting it in the
-``$HOME/.yt/config`` file:
+``$HOME/.config/yt/ytrc`` file:
.. code-block:: bash
- [yt]
- loglevel = 10 # This sets the log level to "DEBUG"
+ $ yt config set yt loglevel 10 # This sets the log level to "DEBUG"
which would produce debug (as well as info, warning, and error) messages, or at runtime:
diff -r b1165fbb7bd24dcdbf61093a685f540567a7de87 -r 42d65f24f2d8f10b60704b33038c976cdcf0c580 doc/source/index.rst
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -133,6 +133,16 @@
<tr valign="top"><td width="25%"><p>
+ <a href="sharing_data.html">Sharing Data</a>
+ </p>
+ </td>
+ <td width="75%">
+ <p class="linkdescr">The yt Hub</p>
+ </td>
+ </tr>
+ <tr valign="top">
+ <td width="25%">
+ <p><a href="reference/index.html">Reference Materials</a></p></td>
@@ -185,6 +195,7 @@
analyzing/analysis_modules/index
examining/index
developing/index
+ sharing_data
reference/index
faq/index
Getting Help <help/index>
diff -r b1165fbb7bd24dcdbf61093a685f540567a7de87 -r 42d65f24f2d8f10b60704b33038c976cdcf0c580 doc/source/reference/command-line.rst
--- a/doc/source/reference/command-line.rst
+++ b/doc/source/reference/command-line.rst
@@ -54,35 +54,7 @@
This will print the list of available subcommands,
-.. code-block:: bash
-
- help Print help message
- bootstrap_dev Bootstrap a yt development environment
- bugreport Report a bug in yt
- hub_register Register a user on the Hub: http://hub.yt-project.org/
- hub_submit Submit a mercurial repository to the yt Hub
- (http://hub.yt-project.org/), creating a BitBucket
- repo in the process if necessary.
- instinfo Get some information about the yt installation
- version Get some information about the yt installation (this
- is an alias for instinfo).
- load Load a single dataset into an IPython instance
- mapserver Serve a plot in a GMaps-style interface
- pastebin Post a script to an anonymous pastebin
- pastebin_grab Print an online pastebin to STDOUT for local use.
- upload_notebook Upload an IPython notebook to hub.yt-project.org.
- plot Create a set of images
- rpdb Connect to a currently running (on localhost) rpd
- session. Commands run with --rpdb will trigger an rpdb
- session with any uncaught exceptions.
- notebook Run the IPython Notebook
- stats Print stats and max/min value of a given field (if
- requested), for one or more datasets (default field is
- Density)
- update Update the yt installation to the most recent version
- delete_image Delete image from imgur.com.
- upload_image Upload an image to imgur.com. Must be PNG.
-
+.. config_help:: yt
To execute any such function, simply run:
@@ -217,13 +189,12 @@
This command will accept the filename of a ``.ipynb`` file (generated from an
IPython notebook session) and upload it to the `yt hub
-<http://hub.yt-project.org/>` where others will be able to view it, and
+<https://hub.yt/>`__ where others will be able to view it, and
download it. This is an easy method for recording a sequence of commands,
their output, narrative information, and then sharing that with others. These
notebooks will be viewable online, and the appropriate URLs will be returned on
the command line.
-
rpdb
++++
@@ -272,3 +243,95 @@
The image uploaded using ``upload_image`` is assigned with a unique hash that
can be used to remove it. This subcommand provides an easy way to send a delete
request directly to the `imgur.com <http://imgur.com/>`_.
+
+Hub helper
+~~~~~~~~~~
+
+The :code:`yt hub` command-line tool allows to interact with the `yt hub
+<https://hub.yt>`__. The following subcommands are currently available:
+
+.. config_help:: yt hub
+
+register
+++++++++
+
+This subcommand starts an interactive process of creating an account on the `yt
+hub <https://hub.yt/>`__. Please note that the yt Hub also supports multiple OAuth
+providers such as Google, Bitbucket and GitHub for authentication.
+See :ref:`hub-APIkey` for more information.
+
+start
++++++
+
+This subcommand launches the Jupyter Notebook on the `yt Hub <https://hub.yt>`__
+with a chosen Hub folder mounted to the ``/data`` directory inside the notebook.
+If no path is given all the `example yt datasets
+<https://yt-project.org/data>`_ are mounted by default. The appropriate URL
+allowing to access the Notebook will be returned on the commandline.
+
+Example:
+
+.. code-block:: bash
+
+ $ yt hub start
+ $ yt hub start /user/xarthisius/Public
+
+
+Config helper
+~~~~~~~~~~~~~
+
+The :code:`yt config` command-line tool allows you to modify and access yt's
+configuration without manually locating and opening the config file in an editor.
+To get a quick list of available commands, just type:
+
+.. code-block:: bash
+
+ yt config -h
+
+This will print the list of available subcommands:
+
+.. config_help:: yt config
+
+Since the yt version 3.3.2, the previous location of the configuration file
+(``$HOME/.yt/config``) has been deprecated in favor of a location adhering to the
+`XDG Base Directory Specification
+<https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html>`_.
+(``$XDG_HOME_CONFIG/yt/ytrc``). In order to perform an automatic migration of
+the old config, you are encouraged to run:
+
+.. code-block:: bash
+
+ yt config migrate
+
+that will copy your current config file to the new location and store a backup
+copy as ``$HOME/.yt/config.bak``.
+
+Examples
+++++++++
+
+Listing current content of the config file:
+
+.. code-block:: bash
+
+ $ yt config list
+ [yt]
+ loglevel = 50
+
+Obtaining a single config value by name:
+
+.. code-block:: bash
+
+ $ yt config get yt loglevel
+ 50
+
+Changing a single config value:
+
+.. code-block:: bash
+
+ $ yt config set yt loglevel 10
+
+Removing a single config entry:
+
+.. code-block:: bash
+
+ $ yt config rm yt loglevel
diff -r b1165fbb7bd24dcdbf61093a685f540567a7de87 -r 42d65f24f2d8f10b60704b33038c976cdcf0c580 doc/source/reference/configuration.rst
--- a/doc/source/reference/configuration.rst
+++ b/doc/source/reference/configuration.rst
@@ -18,9 +18,9 @@
Configuration File Format
^^^^^^^^^^^^^^^^^^^^^^^^^
-yt will look for and recognize the file ``$HOME/.yt/config`` as a configuration
+yt will look for and recognize the file ``$HOME/.config/yt/ytrc`` as a configuration
file, containing several options that can be modified and adjusted to control
-runtime behavior. For example, a sample ``$HOME/.yt/config`` file could look
+runtime behavior. For example, a sample ``$HOME/.config/yt/ytrc`` file could look
like:
.. code-block:: none
@@ -31,7 +31,17 @@
This configuration file would set the logging threshold much lower, enabling
much more voluminous output from yt. Additionally, it increases the number of
-datasets tracked between instantiations of yt.
+datasets tracked between instantiations of yt. The configuration file can be
+managed using the ``yt config`` helper. It can list, add, modify and remove
+options from the configuration file, e.g.:
+
+.. code-block:: none
+
+ $ yt config -h
+ $ yt config list
+ $ yt config set yt loglevel 1
+ $ yt config rm yt maximumstoreddatasets
+
Configuration Options At Runtime
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff -r b1165fbb7bd24dcdbf61093a685f540567a7de87 -r 42d65f24f2d8f10b60704b33038c976cdcf0c580 doc/source/sharing_data.rst
--- /dev/null
+++ b/doc/source/sharing_data.rst
@@ -0,0 +1,117 @@
+.. _sharing-data:
+
+The yt Hub
+==========
+
+.. contents::
+ :depth: 2
+ :local:
+ :backlinks: none
+
+What is the yt Hub?
+-------------------
+
+The yt data Hub is a mechanism by which images, data objects and projects can be
+shared with other people. For instance, one can upload a dataset and allow other
+people to remotely analyze it with a jupyter notebook or upload notebooks and
+view them from any web browser.
+
+.. note:: All items posted on the hub are public!
+
+Over time, more widgets will be added, and more datatypes will be able to be
+uploaded. If you are interested in adding more ways of sharing data, please
+email the developers' list. We would like to add support for 3D widgets such
+as isocontours as well as interactive binning and rebinning of data from yt
+data objects, to be displayed as phase plots and profiles.
+
+.. note:: Working with the Hub requires additional dependencies to be installed.
+ You can obtain them by running: ``pip install yt[hub]``.
+
+.. _hub-APIkey:
+
+Obtaining an API key
+--------------------
+
+In order to interact with the yt Hub, you need to obtain API key, which is
+available only for authenticated users. You can `log into
+<https://girder.hub.yt/#?dialog=login>`_ the Hub using your Google, GitHub or
+Bitbucket account. After you log in, an API key can be generated under the *My
+account* page, which can be accessed through the dropdown menu in the upper
+right corner.
+
+.. image:: _static/apiKey01.jpg
+ :width: 50 %
+
+Select the *API keys* tab and press *Create new key* button:
+
+.. image:: _static/apiKey02.jpg
+ :width: 50 %
+
+By convention, the *Name* field of API keys can be used to specify what
+application is making use of the key in a human-readable way e.g. ``yt
+command``, although you may name your key however you want.
+
+.. image:: _static/apiKey03.jpg
+ :width: 50 %
+
+After the API Key is created you can obtain it by clicking *show* link:
+
+.. image:: _static/apiKey04.jpg
+ :width: 50 %
+
+For more information about API keys please see `this document
+<http://girder.readthedocs.io/en/latest/user-guide.html?highlight=API%20keys#api-keys>`__.
+
+After you have gotten your API key, update your config file:
+
+.. code-block:: none
+
+ $ yt config set yt hub_api_key 3fd1de56c2114c13a2de4dd51g10974b
+
+Replace ``3fd1de56c2114c13a2de4dd51g10974b`` with your API key.
+
+Registering a User
+^^^^^^^^^^^^^^^^^^
+
+If you do not wish to use OAuth authentication, you can create a Hub account
+using ``yt`` command. To register a user:
+
+.. code-block:: bash
+
+ $ yt hub register
+
+This will walk you through the process of registering. You will need to supply
+a name, a username, a password and an email address. Apart from creating a new
+user account, it will also generate an API key and append it to the yt's config
+file. At this point, you're ready to go!
+
+What Can Be Uploaded
+--------------------
+
+Currently, the yt hub can accept these types of data:
+
+ * Raw data files, scripts.
+ * IPython notebooks: these are stored on the hub and are made available for
+ download and via the IPython `nbviewer <http://nbviewer.ipython.org/>`_
+ service.
+
+How to Upload Data
+------------------
+
+Uploading data can be performed using the ``girder-cli`` command tool or
+directly via the web interface. Please refer to ``girder-cli`` `documentation page
+<http://girder.readthedocs.io/en/latest/python-client.html>`_ for additional
+information.
+
+Uploading Notebooks
+^^^^^^^^^^^^^^^^^^^
+
+Notebooks can be uploaded from the bash command line:
+
+.. code-block:: bash
+
+ yt upload_notebook notebook_file.ipynb
+
+After the notebook is finished uploading, yt will print a link to the raw
+notebook as well as an nbviewer link to the same notebook. Your notebooks will
+be stored under your hub Public directory.
diff -r b1165fbb7bd24dcdbf61093a685f540567a7de87 -r 42d65f24f2d8f10b60704b33038c976cdcf0c580 doc/source/visualizing/sketchfab.rst
--- a/doc/source/visualizing/sketchfab.rst
+++ b/doc/source/visualizing/sketchfab.rst
@@ -105,7 +105,7 @@
but it requires that you get an API key first. You can get this API key by
creating an account and then going to your "dashboard," where it will be listed
on the right hand side. Once you've obtained it, put it into your
-``~/.yt/config`` file under the heading ``[yt]`` as the variable
+``~/.config/yt/ytrc`` file under the heading ``[yt]`` as the variable
``sketchfab_api_key``. If you don't want to do this, you can also supply it as
an argument to the function ``export_sketchfab``.
diff -r b1165fbb7bd24dcdbf61093a685f540567a7de87 -r 42d65f24f2d8f10b60704b33038c976cdcf0c580 setup.py
--- a/setup.py
+++ b/setup.py
@@ -380,6 +380,9 @@
'IPython',
'cython',
],
+ extras_require = {
+ 'hub': ["girder_client"]
+ },
cmdclass={'sdist': sdist, 'build_ext': build_ext, 'build_py': build_py},
author="The yt project",
author_email="yt-dev at lists.spacepope.org",
diff -r b1165fbb7bd24dcdbf61093a685f540567a7de87 -r 42d65f24f2d8f10b60704b33038c976cdcf0c580 yt/config.py
--- a/yt/config.py
+++ b/yt/config.py
@@ -16,6 +16,7 @@
#-----------------------------------------------------------------------------
import os
+import warnings
from yt.extern.six.moves import configparser
ytcfg_defaults = dict(
@@ -48,8 +49,9 @@
test_storage_dir = '/does/not/exist',
test_data_dir = '/does/not/exist',
enzo_db = '',
- hub_url = 'https://hub.yt-project.org/upload',
+ hub_url = 'https://girder.hub.yt/api/v1',
hub_api_key = '',
+ hub_sandbox = '/collection/yt_sandbox/data',
notebook_password = '',
answer_testing_tolerance = '3',
answer_testing_bitwise = 'False',
@@ -67,20 +69,28 @@
default_colormap = 'arbre',
ray_tracing_engine = 'embree',
)
+
+CONFIG_DIR = os.environ.get(
+ 'XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config', 'yt'))
+if not os.path.exists(CONFIG_DIR):
+ os.makedirs(CONFIG_DIR)
+
+CURRENT_CONFIG_FILE = os.path.join(CONFIG_DIR, 'ytrc')
+_OLD_CONFIG_FILE = os.path.join(os.path.expanduser('~'), '.yt', 'config')
+
# Here is the upgrade. We're actually going to parse the file in its entirety
# here. Then, if it has any of the Forbidden Sections, it will be rewritten
# without them.
-__fn = os.path.expanduser("~/.yt/config")
-if os.path.exists(__fn):
- f = open(__fn).read()
+if os.path.exists(_OLD_CONFIG_FILE):
+ f = open(_OLD_CONFIG_FILE).read()
if any(header in f for header in ["[lagos]","[raven]","[fido]","[enki]"]):
print("***********************************************************")
print("* Upgrading configuration file to new format; saving old. *")
print("***********************************************************")
# This is of the old format
cp = configparser.ConfigParser()
- cp.read(__fn)
+ cp.read(_OLD_CONFIG_FILE)
# NOTE: To avoid having the 'DEFAULT' section here,
# we are not passing in ytcfg_defaults to the constructor.
new_cp = configparser.ConfigParser()
@@ -91,16 +101,21 @@
if option.lower() in ytcfg_defaults:
new_cp.set("yt", option, cp.get(section, option))
print("Setting %s to %s" % (option, cp.get(section, option)))
- open(__fn + ".old", "w").write(f)
- new_cp.write(open(__fn, "w"))
-# Pathological check for Kraken
-#elif os.path.exists("~/"):
-# if not os.path.exists("~/.yt"):
-# print "yt is creating a new directory, ~/.yt ."
-# os.mkdir(os.path.exists("~/.yt/"))
-# # Now we can read in and write out ...
-# new_cp = configparser.ConfigParser(ytcfg_defaults)
-# new_cp.write(__fn)
+ open(_OLD_CONFIG_FILE + ".old", "w").write(f)
+ new_cp.write(open(_OLD_CONFIG_FILE, "w"))
+
+ msg = (
+ "The configuration file {} is deprecated. "
+ "Please migrate your config to {} by running: "
+ "'yt config migrate'"
+ )
+ warnings.warn(msg.format(_OLD_CONFIG_FILE, CURRENT_CONFIG_FILE))
+
+if not os.path.exists(CURRENT_CONFIG_FILE):
+ cp = configparser.ConfigParser()
+ cp.add_section("yt")
+ with open(CURRENT_CONFIG_FILE, 'w') as new_cfg:
+ cp.write(new_cfg)
class YTConfigParser(configparser.ConfigParser):
def __setitem__(self, key, val):
@@ -108,12 +123,8 @@
def __getitem__(self, key):
self.get(key[0], key[1])
-if os.path.exists(os.path.expanduser("~/.yt/config")):
- ytcfg = YTConfigParser(ytcfg_defaults)
- ytcfg.read(['yt.cfg', os.path.expanduser('~/.yt/config')])
-else:
- ytcfg = YTConfigParser(ytcfg_defaults)
- ytcfg.read(['yt.cfg'])
+ytcfg = YTConfigParser(ytcfg_defaults)
+ytcfg.read([_OLD_CONFIG_FILE, CURRENT_CONFIG_FILE, 'yt.cfg'])
if not ytcfg.has_section("yt"):
ytcfg.add_section("yt")
diff -r b1165fbb7bd24dcdbf61093a685f540567a7de87 -r 42d65f24f2d8f10b60704b33038c976cdcf0c580 yt/utilities/command_line.py
--- a/yt/utilities/command_line.py
+++ b/yt/utilities/command_line.py
@@ -26,7 +26,7 @@
import json
import pprint
-from yt.config import ytcfg
+from yt.config import ytcfg, CURRENT_CONFIG_FILE
ytcfg["yt","__command_line"] = "True"
from yt.startup_tasks import parser, subparsers
from yt.funcs import \
@@ -39,11 +39,13 @@
enable_plugins
from yt.extern.six import add_metaclass, string_types
from yt.extern.six.moves import urllib, input
+from yt.extern.six.moves.urllib.parse import urlparse
from yt.convenience import load
from yt.visualization.plot_window import \
SlicePlot, \
ProjectionPlot
from yt.utilities.metadata import get_metadata
+from yt.utilities.configure import set_config
from yt.utilities.exceptions import \
YTOutputNotIdentified, YTFieldNotParseable
@@ -117,16 +119,48 @@
print("Changeset = %s" % vstring.strip().decode("utf-8"))
print("---")
return vstring
+
+def _get_girder_client():
+ try:
+ import girder_client
+ except ImportError:
+ print("this command requires girder_client to be installed")
+ print("Please install them using your python package manager, e.g.:")
+ print(" pip install girder_client --user")
+ exit()
+
+ hub_url = urlparse(ytcfg.get("yt", "hub_url"))
+ gc = girder_client.GirderClient(apiUrl=hub_url.geturl())
+ gc.authenticate(apiKey=ytcfg.get("yt", "hub_api_key"))
+ return gc
+
+
+_subparsers = {None: subparsers}
+_subparsers_description = {
+ 'config': 'Get and set configuration values for yt',
+ 'hub': 'Interact with the yt Hub'
+}
class YTCommandSubtype(type):
def __init__(cls, name, b, d):
type.__init__(cls, name, b, d)
if cls.name is not None:
names = ensure_list(cls.name)
+ if cls.subparser not in _subparsers:
+ try:
+ description = _subparsers_description[cls.subparser]
+ except KeyError:
+ description = cls.subparser
+ parent_parser = argparse.ArgumentParser(add_help=False)
+ p = subparsers.add_parser(cls.subparser, help=description,
+ description=description,
+ parents=[parent_parser])
+ _subparsers[cls.subparser] = p.add_subparsers(
+ title=cls.subparser, dest=cls.subparser)
+ sp = _subparsers[cls.subparser]
for name in names:
- sc = subparsers.add_parser(name,
- description = cls.description,
- help = cls.description)
+ sc = sp.add_parser(name, description=cls.description,
+ help=cls.description)
sc.set_defaults(func=cls.run)
for arg in cls.args:
_add_arg(sc, arg)
@@ -138,6 +172,7 @@
description = ""
aliases = ()
ndatasets = 1
+ subparser = None
@classmethod
def run(cls, args):
@@ -557,25 +592,27 @@
class YTHubRegisterCmd(YTCommand):
- name = "hub_register"
+ subparser = "hub"
+ name = "register"
description = \
"""
- Register a user on the Hub: http://hub.yt-project.org/
+ Register a user on the yt Hub: http://hub.yt/
"""
def __call__(self, args):
- # We need these pieces of information:
- # 1. Name
- # 2. Email
- # 3. Username
- # 4. Password (and password2)
- # 5. (optional) URL
- # 6. "Secret" key to make it epsilon harder for spammers
- if ytcfg.get("yt","hub_api_key") != "":
+ try:
+ import requests
+ except ImportError:
+ print("yt {} requires requests to be installed".format(self.name))
+ print("Please install them using your python package manager, e.g.:")
+ print(" pip install requests --user")
+ exit()
+ if ytcfg.get("yt", "hub_api_key") != "":
print("You seem to already have an API key for the hub in")
- print("~/.yt/config . Delete this if you want to force a")
+ print("{} . Delete this if you want to force a".format(CURRENT_CONFIG_FILE))
print("new user registration.")
+ exit()
print("Awesome! Let's start by registering a new user for you.")
- print("Here's the URL, for reference: http://hub.yt-project.org/ ")
+ print("Here's the URL, for reference: http://hub.yt/ ")
print()
print("As always, bail out with Ctrl-C at any time.")
print()
@@ -586,8 +623,11 @@
print()
print("To start out, what's your name?")
print()
- name = input("Name? ")
- if len(name) == 0: sys.exit(1)
+ first_name = input("First Name? ")
+ if len(first_name) == 0: sys.exit(1)
+ print()
+ last_name = input("Last Name? ")
+ if len(last_name) == 0: sys.exit(1)
print()
print("And your email address?")
print()
@@ -604,33 +644,32 @@
print("Sorry, they didn't match! Let's try again.")
print()
print()
- print("Would you like a URL displayed for your user?")
- print("Leave blank if no.")
- print()
- url = input("URL? ")
- print()
print("Okay, press enter to register. You should receive a welcome")
print("message at %s when this is complete." % email)
print()
input()
- data = dict(name = name, email = email, username = username,
- password = password1, password2 = password2,
- url = url, zap = "rowsdower")
- data = urllib.parse.urlencode(data)
- hub_url = "https://hub.yt-project.org/create_user"
- req = urllib.request.Request(hub_url, data)
- try:
- urllib.request.urlopen(req).read()
- except urllib.error.HTTPError as exc:
- if exc.code == 400:
- print("Sorry, the Hub couldn't create your user.")
- print("You can't register duplicate users, which is the most")
- print("common cause of this error. All values for username,")
- print("name, and email must be unique in our system.")
- sys.exit(1)
- except urllib.URLError as exc:
- print("Something has gone wrong. Here's the error message.")
- raise exc
+
+ data = dict(firstName=first_name, email=email, login=username,
+ password=password1, lastName=last_name, admin=False)
+ hub_url = ytcfg.get("yt", "hub_url")
+ req = requests.post(hub_url + "/user", data=data)
+
+ if req.ok:
+ headers = {'Girder-Token': req.json()['authToken']['token']}
+ else:
+ if req.status_code == 400:
+ print("Registration failed with 'Bad request':")
+ print(req.json()["message"])
+ exit(1)
+ print("User registration successful")
+ print("Obtaining API key...")
+ req = requests.post(hub_url + "/api_key", headers=headers,
+ data={'name': 'ytcmd', 'active': True})
+ apiKey = req.json()["key"]
+
+ print("Storing API key in configuration file")
+ set_config("yt", "hub_api_key", apiKey)
+
print()
print("SUCCESS!")
print()
@@ -810,40 +849,60 @@
import yt.utilities.lodgeit as lo
lo.main( None, download=args.number )
+class YTHubStartNotebook(YTCommand):
+ args = (
+ dict(dest="folderId", default=ytcfg.get("yt", "hub_sandbox"),
+ nargs="?",
+ help="(Optional) Hub folder to mount inside the Notebook"),
+ )
+ description = \
+ """
+ Start the Jupyter Notebook on the yt Hub.
+ """
+ subparser = "hub"
+ name = "start"
+ def __call__(self, args):
+ gc = _get_girder_client()
+
+ # TODO: should happen server-side
+ _id = gc._checkResourcePath(args.folderId)
+
+ resp = gc.post("/notebook/{}".format(_id))
+ try:
+ print("Launched! Please visit this URL:")
+ print(" https://tmpnb.hub.yt" + resp['url'])
+ print()
+ except (KeyError, TypeError):
+ print("Something went wrong. The yt Hub responded with : ")
+ print(resp)
+
class YTNotebookUploadCmd(YTCommand):
args = (dict(short="file", type=str),)
description = \
"""
- Upload an IPython notebook to hub.yt-project.org.
+ Upload an IPython Notebook to the yt Hub.
"""
name = "upload_notebook"
def __call__(self, args):
- filename = args.file
- if not os.path.isfile(filename):
- raise IOError(filename)
- if not filename.endswith(".ipynb"):
- print("File must be an IPython notebook!")
- return 1
- import json
- try:
- t = json.loads(open(filename).read())['metadata']['name']
- except (ValueError, KeyError):
- print("File does not appear to be an IPython notebook.")
- if len(t) == 0:
- t = filename.strip(".ipynb")
- from yt.utilities.minimal_representation import MinimalNotebook
- mn = MinimalNotebook(filename, t)
- rv = mn.upload()
+ gc = _get_girder_client()
+ username = gc.get("/user/me")["login"]
+ gc.upload(args.file, "/user/{}/Public".format(username))
+
+ _id = gc.resourceLookup(
+ "/user/{}/Public/{}".format(username, args.file))["_id"]
+ _fid = next(gc.listFile(_id))["_id"]
+ hub_url = urlparse(ytcfg.get("yt", "hub_url"))
print("Upload successful!")
print()
print("To access your raw notebook go here:")
print()
- print(" %s" % (rv['url']))
+ print(" {}://{}/#item/{}".format(hub_url.scheme, hub_url.netloc, _id))
print()
print("To view your notebook go here:")
print()
- print(" %s" % (rv['url'].replace("/go/", "/nb/")))
+ print(" http://nbviewer.jupyter.org/urls/{}/file/{}/download".format(
+ hub_url.netloc + hub_url.path, _fid))
print()
class YTPlotCmd(YTCommand):
@@ -947,7 +1006,7 @@
)
description = \
"""
- Run the IPython Notebook
+ Start the Jupyter Notebook locally.
"""
def __call__(self, args):
kwargs = {}
@@ -1141,6 +1200,61 @@
print()
pprint.pprint(rv)
+
+class YTConfigGetCmd(YTCommand):
+ subparser = 'config'
+ name = 'get'
+ description = 'get a config value'
+ args = (dict(short='section', help='The section containing the option.'),
+ dict(short='option', help='The option to retrieve.'))
+ def __call__(self, args):
+ from yt.utilities.configure import get_config
+ print(get_config(args.section, args.option))
+
+
+class YTConfigSetCmd(YTCommand):
+ subparser = 'config'
+ name = 'set'
+ description = 'set a config value'
+ args = (dict(short='section', help='The section containing the option.'),
+ dict(short='option', help='The option to set.'),
+ dict(short='value', help='The value to set the option to.'))
+ def __call__(self, args):
+ from yt.utilities.configure import set_config
+ set_config(args.section, args.option, args.value)
+
+
+class YTConfigRemoveCmd(YTCommand):
+ subparser = 'config'
+ name = 'rm'
+ description = 'remove a config option'
+ args = (dict(short='section', help='The section containing the option.'),
+ dict(short='option', help='The option to remove.'))
+ def __call__(self, args):
+ from yt.utilities.configure import rm_config
+ rm_config(args.section, args.option)
+
+
+class YTConfigListCmd(YTCommand):
+ subparser = 'config'
+ name = 'list'
+ description = 'show the config content'
+ args = ()
+ def __call__(self, args):
+ from yt.utilities.configure import write_config
+ write_config(sys.stdout)
+
+
+class YTConfigMigrateCmd(YTCommand):
+ subparser = 'config'
+ name = 'migrate'
+ description = 'migrate old config file'
+ args = ()
+ def __call__(self, args):
+ from yt.utilities.configure import migrate_config
+ migrate_config()
+
+
class YTSearchCmd(YTCommand):
args = (dict(short="-o", longname="--output",
action="store", type=str,
diff -r b1165fbb7bd24dcdbf61093a685f540567a7de87 -r 42d65f24f2d8f10b60704b33038c976cdcf0c580 yt/utilities/configure.py
--- /dev/null
+++ b/yt/utilities/configure.py
@@ -0,0 +1,92 @@
+# -*- coding: UTF-8 -*-
+#-----------------------------------------------------------------------------
+# Copyright (c) 2016, yt Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+import os
+import sys
+import argparse
+from yt.config import CURRENT_CONFIG_FILE, _OLD_CONFIG_FILE
+from yt.extern.six.moves import configparser
+
+CONFIG = configparser.SafeConfigParser()
+CONFIG.read([CURRENT_CONFIG_FILE])
+
+
+def get_config(section, option):
+ return CONFIG.get(section, option)
+
+
+def set_config(section, option, value):
+ if not CONFIG.has_section(section):
+ CONFIG.add_section(section)
+ CONFIG.set(section, option, value)
+ write_config()
+
+
+def write_config(fd=None):
+ if fd is None:
+ with open(CURRENT_CONFIG_FILE, 'w') as fd:
+ CONFIG.write(fd)
+ else:
+ CONFIG.write(fd)
+
+def migrate_config():
+ if not os.path.exists(_OLD_CONFIG_FILE):
+ print("Old config not found.")
+ sys.exit()
+ CONFIG.read(_OLD_CONFIG_FILE)
+ print("Writing a new config file to: {}".format(CURRENT_CONFIG_FILE))
+ write_config()
+ print("Backing up the old config file: {}.bak".format(_OLD_CONFIG_FILE))
+ os.rename(_OLD_CONFIG_FILE, _OLD_CONFIG_FILE + '.bak')
+
+
+def rm_config(section, option):
+ CONFIG.remove_option(section, option)
+ write_config()
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ description='Get and set configuration values for yt')
+ subparsers = parser.add_subparsers(help='sub-command help', dest='cmd')
+
+ get_parser = subparsers.add_parser('get', help='get a config value')
+ set_parser = subparsers.add_parser('set', help='set a config value')
+ rm_parser = subparsers.add_parser('rm', help='remove a config option')
+ subparsers.add_parser('migrate', help='migrate old config file')
+ subparsers.add_parser('list', help='show all config values')
+
+ get_parser.add_argument(
+ 'section', help='The section containing the option.')
+ get_parser.add_argument('option', help='The option to retrieve.')
+
+ set_parser.add_argument(
+ 'section', help='The section containing the option.')
+ set_parser.add_argument('option', help='The option to set.')
+ set_parser.add_argument('value', help='The value to set the option to.')
+
+ rm_parser.add_argument(
+ 'section', help='The section containing the option to remove.')
+ rm_parser.add_argument('option', help='The option to remove.')
+
+ args = parser.parse_args()
+
+ if args.cmd == 'get':
+ print(get_config(args.section, args.option))
+ elif args.cmd == 'set':
+ set_config(args.section, args.option, args.value)
+ elif args.cmd == 'list':
+ write_config(sys.stdout)
+ elif args.cmd == 'migrate':
+ migrate_config()
+ elif args.cmd == 'rm':
+ rm_config(args.section, args.option)
+
+if __name__ == '__main__':
+ main() # pragma: no cover
diff -r b1165fbb7bd24dcdbf61093a685f540567a7de87 -r 42d65f24f2d8f10b60704b33038c976cdcf0c580 yt/utilities/tests/test_config.py
--- /dev/null
+++ b/yt/utilities/tests/test_config.py
@@ -0,0 +1,142 @@
+# -*- coding: UTF-8 -*-
+#-----------------------------------------------------------------------------
+# Copyright (c) 2016, yt Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+import contextlib
+import mock
+import os
+import sys
+import unittest
+import yt.utilities.command_line
+import yt.config
+from yt.config import \
+ CURRENT_CONFIG_FILE, _OLD_CONFIG_FILE
+from yt.extern.six import StringIO
+from yt.extern.six.moves.configparser import NoOptionError, SafeConfigParser
+
+
+_DUMMY_CFG = ['[yt]', 'loglevel = 49']
+
+
+ at contextlib.contextmanager
+def captureOutput():
+ oldout, olderr = sys.stdout, sys.stderr
+ try:
+ out = [StringIO(), StringIO()]
+ sys.stdout, sys.stderr = out
+ yield out
+ finally:
+ sys.stdout, sys.stderr = oldout, olderr
+ out[0] = out[0].getvalue()
+ out[1] = out[1].getvalue()
+
+
+class SysExitException(Exception):
+ pass
+
+
+def setUpModule():
+ for cfgfile in (CURRENT_CONFIG_FILE, _OLD_CONFIG_FILE):
+ if os.path.exists(cfgfile):
+ os.rename(cfgfile, cfgfile + '.bak_test')
+
+ if cfgfile == CURRENT_CONFIG_FILE:
+ yt.utilities.configure.CONFIG = SafeConfigParser()
+ if not yt.utilities.configure.CONFIG.has_section('yt'):
+ yt.utilities.configure.CONFIG.add_section('yt')
+
+
+def tearDownModule():
+ for cfgfile in (CURRENT_CONFIG_FILE, _OLD_CONFIG_FILE):
+ if os.path.exists(cfgfile + '.bak_test'):
+ os.rename(cfgfile + '.bak_test', cfgfile)
+
+
+class TestYTConfig(unittest.TestCase):
+ def _runYTConfig(self, args):
+ args = ['yt', 'config'] + args
+ retcode = 0
+
+ with mock.patch.object(sys, 'argv', args),\
+ mock.patch('sys.exit', side_effect=SysExitException) as exit,\
+ captureOutput() as output:
+ try:
+ yt.utilities.command_line.run_main()
+ except SysExitException:
+ args = exit.mock_calls[0][1]
+ retcode = args[0] if len(args) else 0
+ return {
+ 'rc': retcode,
+ 'stdout': output[0],
+ 'stderr': output[1]
+ }
+
+class TestYTConfigCommands(TestYTConfig):
+ def testConfigCommands(self):
+ self.assertFalse(os.path.exists(CURRENT_CONFIG_FILE))
+
+ info = self._runYTConfig(['--help'])
+ self.assertEqual(info['rc'], 0)
+ self.assertEqual(info['stderr'], '')
+ self.assertIn('Get and set configuration values for yt',
+ info['stdout'])
+
+ info = self._runYTConfig(['list'])
+ self.assertEqual(info['rc'], 0)
+ self.assertIn('[yt]', info['stdout'])
+
+ info = self._runYTConfig(['set', 'yt', '__parallel', 'True'])
+ self.assertEqual(info['rc'], 0)
+
+ info = self._runYTConfig(['get', 'yt', '__parallel'])
+ self.assertEqual(info['rc'], 0)
+ self.assertEqual(info['stdout'].strip(), 'True')
+
+ info = self._runYTConfig(['rm', 'yt', '__parallel'])
+ self.assertEqual(info['rc'], 0)
+
+ with self.assertRaises(NoOptionError):
+ self._runYTConfig(['get', 'yt', 'foo'])
+
+ def tearDown(self):
+ if os.path.exists(CURRENT_CONFIG_FILE):
+ os.remove(CURRENT_CONFIG_FILE)
+
+class TestYTConfigMigration(TestYTConfig):
+
+ def setUp(self):
+ if not os.path.exists(os.path.dirname(_OLD_CONFIG_FILE)):
+ os.makedirs(os.path.dirname(_OLD_CONFIG_FILE))
+
+ with open(_OLD_CONFIG_FILE, 'w') as fh:
+ for line in _DUMMY_CFG:
+ fh.write('{}\n'.format(line))
+
+ if os.path.exists(CURRENT_CONFIG_FILE):
+ os.remove(CURRENT_CONFIG_FILE)
+
+ def tearDown(self):
+ if os.path.exists(CURRENT_CONFIG_FILE):
+ os.remove(CURRENT_CONFIG_FILE)
+ if os.path.exists(_OLD_CONFIG_FILE + '.bak'):
+ os.remove(_OLD_CONFIG_FILE + '.bak')
+
+ def testConfigMigration(self):
+ self.assertFalse(os.path.exists(CURRENT_CONFIG_FILE))
+ self.assertTrue(os.path.exists(_OLD_CONFIG_FILE))
+
+ info = self._runYTConfig(['migrate'])
+ self.assertEqual(info['rc'], 0)
+
+ self.assertTrue(os.path.exists(CURRENT_CONFIG_FILE))
+ self.assertFalse(os.path.exists(_OLD_CONFIG_FILE))
+ self.assertTrue(os.path.exists(_OLD_CONFIG_FILE + '.bak'))
+
+ with open(CURRENT_CONFIG_FILE, 'r') as fh:
+ new_cfg = ''.join(fh.readlines())
+ self.assertEqual(new_cfg.strip().split('\n'), _DUMMY_CFG)
Repository URL: https://bitbucket.org/yt_analysis/yt/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the yt-svn
mailing list