# Copyright (c) 2013, 2018 National Technology and Engineering Solutions of Sandia, LLC.
# Under the terms of Contract DE-NA0003525 with National Technology and Engineering
# Solutions of Sandia, LLC, the U.S. Government retains certain rights in this software.
# This module supports interacting with the Slycat server from Python and the command line.
# standard libraries
# parse arguments
import argparse
import os
import shlex
# authenticate to server
import getpass
import json
# encode/decode functions for string data (for http communication)
import base64
# miscellaneous utilities
import sys
import time
import math
# stream to server
import io
# logging
import logging
# url/IP
from urllib.parse import urlparse
import socket
# 3rd party libraries
# http requests & kerberos
import requests
import requests.exceptions as exceptions
from requests_kerberos import HTTPKerberosAuth, OPTIONAL
from requests_kerberos.exceptions import KerberosExchangeError
# turn off warning for insecure connections
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
# handle arrays for Slycat requests
import numpy
# web server interaction
import cherrypy
# local libraries
# handle arrays for Slycat requests
import slycat.darray
# set up logging
log = logging.getLogger("slycat.web.client")
log.setLevel(logging.INFO)
log.addHandler(logging.StreamHandler())
log.handlers[0].setFormatter(logging.Formatter("%(levelname)s - %(message)s"))
log.propagate = False
# default file slice size (10 MB)
FILE_SLICE_DEFAULT = 10_000_000
# default Slycat server
HOST_DEFAULT = "https://localhost"
# private function (denoted by _name)
def _require_array_ranges(ranges):
"""Validates a range object (hyperslice) for transmission to the server."""
if ranges is None:
return None
elif isinstance(ranges, int):
return [(0, ranges)]
elif isinstance(ranges, tuple):
return [ranges]
elif isinstance(ranges, list):
return ranges
else:
cherrypy.log.error("slycat.web.client.__init__.py", "Not a valid ranges object.")
raise Exception("Not a valid ranges object.")
# print iterations progress bar -- from stack overflow
# https://stackoverflow.com/questions/3173320/text-progress-bar-in-the-console
def _print_progress_bar (iteration, total, prefix = '', suffix = '',
decimals = 1, length = 100, fill = '█', printEnd = "\r"):
# Call in a loop to create terminal progress bar
# @params:
# iteration - Required : current iteration (Int)
# total - Required : total iterations (Int)
# prefix - Optional : prefix string (Str)
# suffix - Optional : suffix string (Str)
# decimals - Optional : positive number of decimals in percent complete (Int)
# length - Optional : character length of bar (Int)
# fill - Optional : bar fill character (Str)
# printEnd - Optional : end character (e.g. "\r", "\r\n") (Str)
percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
filledLength = int(length * iteration // total)
bar = fill * filledLength + '-' * (length - filledLength)
print(f'\r{prefix} |{bar}| {percent}% {suffix}', end = printEnd)
# print new line on complete
if iteration == total:
print()
[docs]
class ArgumentParser(argparse.ArgumentParser):
"""Return an instance of argparse.ArgumentParser, pre-configured with arguments to
connect to a Slycat server.
Pre-configured options are:
* **-\\-host** -- the URL of the Slycat server.
* **-\\-port** -- the port of the Slycat server.
* **-\\-http-proxy** -- the URL for an http proxy.
* **-\\-https-proxy** -- the URL for an https proxy.
* **-\\-verify** -- an SSL certificate to verify https host.
* **-\\-no-verify** -- disable https verification.
* **-\\-file-slice-size** -- maximum number of bytes to upload at once.
* **-\\-user** -- user name.
* **-\\-passowrd** -- user password.
* **-\\-kerberos** -- enable kerberos authentication.
* **-\\-log-level** -- log detail to display.
"""
def __init__(self, *arguments, **keywords):
argparse.ArgumentParser.__init__(self, *arguments, **keywords)
# arguments for connecting to Slycat server
self.add_argument("--host", default=HOST_DEFAULT,
help="Root URL of the Slycat server. Default: '%(default)s'.")
self.add_argument("--port", default=None,
help="Port of the Slycat server.")
self.add_argument("--http-proxy", default="",
help="HTTP proxy URL. Default: '%(default)s'.")
self.add_argument("--https-proxy", default="",
help="HTTPS proxy URL. Default: '%(default)s'.")
self.add_argument("--verify", default=None,
help="Specify a certificate to use for HTTPS host certificate verification.")
self.add_argument("--no-verify", default=False, action="store_true",
help="Disable HTTPS host certificate verification.")
self.add_argument('--file-slice-size', default=FILE_SLICE_DEFAULT, type=int,
help="Maximum number of bytes to upload before slicing files (i.e. before "
"uploading in chunks). Default: %(default)s.")
self.add_argument("--user", default=getpass.getuser(),
help="Slycat username. Default: '%(default)s'")
self.add_argument("--password", default=None, help="User password.")
self.add_argument("--kerberos", default=False, action="store_true",
help="Use Kerberos authentication. Default: %(default)s.")
self.add_argument("--log-level", default="info",
choices=["debug", "info", "warning", "error", "critical"],
help="Log level. Default: '%(default)s'.")
[docs]
def parse_args(self, list_input=None):
"""Overrides argparse parse_args command. Parses slycat.web.client arguments
in addition to any arguments added by users of the class. Can also be used to
parse a list of arguments passed from a Python function, such as
>>> parser = slycat.web.client.ArgumentParser()
>>> parser.parse_args(["--host", "slycat.sandia.gov", "--kerberos"]).
"""
if "SLYCAT" in os.environ:
sys.argv += shlex.split(os.environ["SLYCAT"])
# parse arguments
arguments = argparse.ArgumentParser.parse_args(self, list_input)
# log level
if arguments.log_level == "debug":
log.setLevel(logging.DEBUG)
elif arguments.log_level == "info":
log.setLevel(logging.INFO)
elif arguments.log_level == "warning":
log.setLevel(logging.WARNING)
elif arguments.log_level == "error":
log.setLevel(logging.ERROR)
elif arguments.log_level == "critical":
log.setLevel(logging.CRITICAL)
return arguments
[docs]
class Connection(object):
"""Provides a class to facilitate communication with the Slycat web server.
To use, open a connection and submit requests. For example:
>>> parser = slycat.web.client.ArgumentParser()
>>> arguments = parser.parse_args()
>>> connection = slycat.web.client.connect(arguments)
>>> projects = connection.get_projects()
"""
def __init__(self, host=HOST_DEFAULT, port=None, kerberos=False,
file_slice_size=FILE_SLICE_DEFAULT, **keywords):
# proxies default to ""
proxies = keywords.get("proxies", {"http": "", "https": ""})
# certificate verification is disabled unless specifically requested
verify = True
if keywords.get("verify") == "False":
verify = False
elif not keywords.get("verify"):
verify = False
# resolve host alias
host_alias = urlparse(host)
host_alias = host_alias.netloc
host_ip = socket.gethostbyname(host_alias)
host_name = socket.gethostbyaddr(host_ip)
host_name = host_name[0]
# host and port
self.host = host.replace(host_alias, host_name)
if port:
self.host = self.host + ':' + port
# using kerberos authentication?
self.kerberos = kerberos
# set up session
self.keywords = keywords
self.session = requests.Session()
# set up max file slice size
self.file_slice_size = file_slice_size
# check for Kerberos authentication
if self.kerberos:
# get Kerberos ticket granting ticket
self.session.auth = HTTPKerberosAuth(mutual_authentication=OPTIONAL,
force_preemptive=True)
# check that ticket is valid using list markings
url = self.host + "/api/configuration/markings"
try:
response = self.session.get(url, proxies=proxies, verify=verify,
auth=self.session.auth)
except KerberosExchangeError:
cherrypy.log.error("Could not find Kerberos ticket, try running 'kinit'.")
sys.exit(1)
if response.status_code == 401:
cherrypy.log.error(
"User %s is not Kerberos authenticated, try running 'kinit'." %
keywords.get("auth", ("",""))[0])
sys.exit(1)
else:
# get user name and password, if supplied
user_name, password = keywords.get("auth", ("",""))
# if password not supplied, prompt user
if not password:
password = getpass.getpass("%s password: " % user_name)
# encode as base 64
user_name_b64 = base64.b64encode(user_name.encode("ascii"))
password_b64 = base64.b64encode(password.encode("ascii"))
# change back to ascii for JSON
user_name_str = user_name_b64.decode('ascii')
password_str = password_b64.decode('ascii')
# authentication information
data = {"user_name":user_name_str, "password":password_str}
# login url
url = self.host + "/api/login"
# connect to server
self.session.post(url, json=data, proxies=proxies, verify=verify)
# session error, assume bad password
if len(list(self.session.cookies.keys())) == 0:
raise NameError('bad username or password:%s, for username:%s' %
(user_name, password))
[docs]
def request(self, method, path, **keywords):
"""Basic request to Slycat server using open session. To make a request
provide the HTTP method and path, and this method will return the body of
the response. Additional keyword arguments must be compatible with the
Python requests library.
Parameters
----------
method: string, required
The HTTP method to use, e.g. "GET", "PUT", etc.
path: string, required
The extension to the URL for the Slycat server, e.g. "/api/models/(mid)".
Returns
-------
response: the body of the Server response.
"""
# combine per-request and per-connection keyword arguments
keywords.update(self.keywords)
# combine host and path to produce the final request URI
uri = self.host + path
# create log message for this request
log_message = "{} {} {}".format(keywords.get("auth", ("", ""))[0], method, uri)
# send request
try:
response = self.session.request(method, uri, **keywords)
# add response to log message
log_message += " => {} {}".format(response.status_code, response.raw.reason)
# check to see if an error occured
response.raise_for_status()
# parse and return body of response
body = None
if response.headers["content-type"].startswith("application/json"):
body = response.json()
else:
body = response.content
# show request in debug log
log.debug(log_message)
return body
# log any errors to both slycat.web.client and cherrypy
except:
log.debug(log_message)
cherrypy.log.error("slycat.web.client.__init__.py request", "%s" % log_message)
raise
#######################################################################
# Low-level functions that map directly to the underlying RESTful API #
#######################################################################
[docs]
def delete_model(self, mid):
"""Delete an existing model.
Parameters
----------
mid: string, required
The unique model identifier.
See Also
--------
:http:delete:`/api/models/(mid)`
"""
self.request("DELETE", "/api/models/%s" % (mid))
[docs]
def delete_project(self, pid):
"""Delete an existing project.
Parameters
----------
pid: string, required
The unique project identifier.
See Also
--------
:http:delete:`/api/projects/(pid)`
"""
self.request("DELETE", "/api/projects/%s" % (pid))
[docs]
def delete_project_cache_object(self, pid, key):
"""Delete an existing project cache object.
Parameters
----------
pid: string, required
The unique project identifier.
key: string, required
Unique cache object key.
See Also
--------
:http:delete:`/api/projects/(pid)/cache/(key)`
"""
self.request("DELETE", "/api/projects/%s/cache/%s" % (pid, key))
[docs]
def delete_reference(self, rid):
"""Delete an existing reference.
Parameters
----------
rid: string, required
The unique reference identifier.
See Also
--------
:http:delete:`/api/references/(rid)`
"""
self.request("DELETE", "/api/references/%s" % (rid))
[docs]
def delete_remote(self, sid):
"""Delete an existing remote session.
Parameters
----------
sid: string, required
The unique remote session identifier.
See Also
--------
:http:delete:`/api/remotes/(hostname)`
"""
self.request("DELETE", "/api/remotes/%s" % (sid))
[docs]
def get_bookmark(self, bid):
"""Retrieve an existing bookmark.
Parameters
----------
bid: string, required
The unique bookmark identifier.
Returns
-------
bookmark: object
The bookmark object, which is an arbitrary collection of
JSON-compatible data.
See Also
--------
:http:get:`/api/bookmarks/(bid)`
"""
return self.request("GET", "/api/bookmarks/%s" % (bid))
[docs]
def get_configuration_markings(self):
"""Retrieve marking information from the server.
Returns
-------
markings: server marking information.
See Also
--------
:http:get:`/api/configuration/markings`
"""
return self.request("GET", "/api/configuration/markings",
headers={"accept":"application/json"})
[docs]
def get_configuration_parsers(self):
"""Retrieve parser plugin information from the server.
Returns
-------
parsers: server parser plugin information.
See Also
--------
:http:get:`/api/configuration/parsers`
"""
return self.request("GET", "/api/configuration/parsers",
headers={"accept":"application/json"})
[docs]
def get_configuration_remote_hosts(self):
"""Retrieve remote host information from the server.
Returns
-------
remote_hosts: server remote host information.
See Also
--------
:http:get:`/api/configuration/remote-hosts`
"""
return self.request("GET", "/api/configuration/remote-hosts",
headers={"accept":"application/json"})
[docs]
def get_configuration_support_email(self):
"""Retrieve support email information from the server.
Returns
-------
email: server support email information.
See Also
--------
:http:get:`/api/configuration/support-email`
"""
return self.request("GET", "/api/configuration/support-email",
headers={"accept":"application/json"})
[docs]
def get_configuration_version(self):
"""Retrieve version information from the server.
Returns
-------
version: server version information.
See Also
--------
:http:get:`/api/configuration/version`
"""
return self.request("GET", "/api/configuration/version",
headers={"accept":"application/json"})
[docs]
def get_configuration_wizards(self):
"""Retrieve wizard plugin information from the server.
Returns
-------
wizards: server wizard plugin information.
See Also
--------
:http:get:`/api/configuration/wizards`
"""
return self.request("GET", "/api/configuration/wizards",
headers={"accept":"application/json"})
# the following three calls seem to be defunct
# def get_global_resource(self, resource):
# return self.request("GET", "/resources/global/%s" % resource)
# def get_model_resource(self, mtype, resource):
# return self.request("GET", "/resources/pages/%s/%s" % (mtype, resource))
# def get_wizard_resource(self, wtype, resource):
# return self.request("GET", "/resources/wizards/%s/%s" % (wtype, resource))
[docs]
def get_model(self, mid):
"""Retrieve an existing model.
Parameters
----------
mid: string, required
The unique model identifier
Returns
-------
model: object
The model object, which is an arbitrary collection of
JSON-compatible data.
See Also
--------
:http:get:`/api/models/(mid)`
"""
return self.request("GET", "/api/models/%s" % mid,
headers={"accept":"application/json"})
[docs]
def get_model_file(self, mid, aid):
"""Retrieves the file corresponding to a given model and artifacts.
Parameters
----------
mid: string, required
Unique model identifier.
aid: string, required
Unique (with the model) artifact id.
Returns
-------
file: File data corresponding to artifact.
See Also
--------
:http:get:`/api/models/(mid)/files/(aid)`
"""
return self.request("GET", "/api/models/%s/files/%s" % (mid, aid))
[docs]
def get_model_parameter(self, mid, aid):
"""Retrieve a model parameter artifact.
Model parameters are JSON objects of arbitrary complexity. They are stored directly within the model
as part of its database record, so they should be limited in size (larger data should be stored using
arraysets or files).
Parameters
----------
mid: string, required
Unique model identifier.
aid: string, required
Unique (within the model) artifact id.
Returns
-------
parameter: JSON-compatible object.
See Also
--------
:http:put:`/api/models/(mid)/parameters/(aid)`
"""
return self.request("GET", "/api/models/%s/parameters/%s" %
(mid, aid), headers={"accept":"application/json"})
[docs]
def get_project_models(self, pid):
"""Retrieve every model in a oroject.
Parameters
----------
pid: string, required
Unique project identifier.
Returns
-------
models: JSON-compatible object.
See Also
--------
:http:get:`/api/projects/(pid)/models`
"""
return self.request("GET", "/api/projects/%s/models" % pid,
headers={"accept":"application/json"})
[docs]
def get_project_references(self, pid):
"""Returns every reference in a project.
Parameters
----------
pid: string, required
Unique project identifier.
Returns
-------
references: JSON-compatible object.
See Also
--------
:http:get:`/api/projects/(pid)/references`
"""
return self.request("GET", "/api/projects/%s/references" % pid,
headers={"accept":"application/json"})
[docs]
def get_project(self, pid):
"""Retrieve an existing project.
Parameters
----------
pid: string, required
Unique project identifier.
Returns
-------
project: Arbitrary collection of JSON-compatible data.
See Also
--------
:http:get:`/api/projects/(pid)`
"""
return self.request("GET", "/api/projects/%s" % pid,
headers={"accept":"application/json"})
[docs]
def get_project_cache_object(self, pid, key):
"""Retrieve an object from a project cache.
Parameters
----------
pid: string, required
Unique project identifier.
key: string, required
Cache object identifier.
Returns
-------
content: Cached object content.
See Also
--------
:http:get:`/api/projects/(pid)/cache/(key)`
"""
return self.request("GET", "/api/projects/%s/cache/%s" % (pid, key))
[docs]
def get_projects(self):
"""Retrieve all projects.
Returns
-------
projects: List of projects. Each project is an arbitrary collection of JSON-compatible data.
See Also
--------
:http:get:`/api/projects`
"""
return self.request("GET", "/api/projects_list", headers={"accept":"application/json"})
[docs]
def get_remote_file(self, sid, path, cache=None, project=None, key=None):
"""Retrieve a file using a remote session.
Parameters
----------
sid: string, required
Unique remote session identifier.
path: string, required
Remote filesystem path (must be absolute).
cache: string, optional
Optional server-side cache for the retrieved file. Must be `None` or "project".
project: string, optional
If `cache` is set to "project", this must specify a unique project identifier.
key: string, optional
if `cache` is set to "project", this must specify a unique key for the cached object.
Returns
-------
file: Remote file contents.
See Also
--------
:http:get:`/api/remotes/(hostname)/file(path)`
"""
return self.request("GET", "/api/remotes/%s/file%s" % (sid, path),
params={"cache": cache, "project": project, "key": key})
[docs]
def get_remote_image(self, sid, path, cache=None, project=None, key=None):
"""Retrieve an image using a remote session.
Parameters
----------
sid: string, required
Unique remote session identifier.
path: string, required
Remote filesystem path (must be absolute).
cache: string, optional
Optional server-side cache for the retrieved image. Must be `None` or "project".
project: string, optional
If `cache` is set to "project", this must specify a unique project identifier.
key: string, optional
if `cache` is set to "project", this must specify a unique key for the cached object.
Returns
-------
image: Remote image contents.
See Also
--------
:http:get:`/api/remotes/(hostname)/image(path)`
"""
return self.request("GET", "/api/remotes/%s/image%s" % (sid, path),
params={"cache": cache, "project": project, "key": key})
[docs]
def get_user(self, uid=None):
"""Retrieve directory information about an existing user.
Parameters
----------
uid: string, optional
Unique user identifier. If unspecified, returns information about the user making the call.
Returns
-------
user: Arbitrary collection of JSON-compatible data.
See Also
--------
:http:get:`/api/users/(uid)`
"""
return self.request("GET", "/api/users/%s" % ("-" if uid is None else uid),
headers={"accept":"application/json"})
[docs]
def post_events(self, path, parameters={}):
"""Post event to be logged on Slycat server.
Parameters
----------
path: string, required
Path-like URI describing event to be logged.
parameters: dictionary, optional
JSON type object describing event.
See Also
--------
:http:post:`/api/events/(event)`
"""
self.request("POST", "/api/events/%s" % path, params=parameters)
[docs]
def post_model_files(self, mid, aids, files, parser, input=True, parameters={}):
"""Stores model file artifacts.
Parameters
----------
mid: string, required
Unique model identifier.
aids: array, required
Artifact IDs for model storage.
files: array, required
Local files for upload.
parser: string, required
Name of Slycat parser that will process the files.
input: boolean, optional
Set as true (default) to store as model artifacts.
parametrs: dictionary, optional
Additional data to pass to parser.
See Also
--------
:http:post:`/api/models/(mid)/files`
"""
data = parameters
data.update({
"input": json.dumps(input),
"aids": aids,
"parser": parser
})
files = [("files", ("blob", file)) for file in files]
self.request("POST", "/api/models/%s/files" % mid, data=data, files=files)
[docs]
def post_model_finish(self, mid):
"""Notify the server that a model is fully initialized.
When called, the server will perform one-time computation
for the given model type.
Parameters
----------
mid: string, required
Unique model identifier.
See Also
--------
:http:post:`/api/models/(mid)/finish`
"""
self.request("POST", "/api/models/%s/finish" % (mid))
[docs]
def post_project_bookmarks(self, pid, bookmark):
"""Store a bookmark.
Parameters
----------
pid: string, required
Unique project identifier.
bookmark: object
Arbitrary collection of JSON-compatible data.
Returns
-------
bid: string
Unique bookmark identifier.
See Also
--------
:http:post:`/api/projects/(pid)/bookmarks`
"""
return self.request("POST", "/api/projects/%s/bookmarks" % (pid),
headers={"content-type":"application/json"}, data=json.dumps(bookmark))["id"]
[docs]
def post_project_models(self, pid, mtype, name, marking="", description=""):
"""Creates a new model, returning the model ID.
Parameters
----------
pid: string, required
Unique project identifier.
mtype: string, required
Model type.
name: string, required
Model name.
marking: string, optional
Model marking.
description: string, optional
Description of model.
Returns
-------
mid: string
New model identifier.
See Also
--------
:http:post:`/api/projects/(pid)/models`
"""
return self.request("POST", "/api/projects/%s/models" % (pid),
headers={"content-type":"application/json"},
data=json.dumps({"model-type":mtype, "name":name, "marking":marking,
"description":description}))["id"]
[docs]
def post_project_references(self, pid, name, mtype=None, mid=None, bid=None):
"""Store a project reference.
Parameters
----------
pid: string, required
Unique project identifier.
name: string, required
Reference name.
mtype: string, optional
Optional model type.
mid: string, optional
Optional model identifier.
bid: string, optional
Optional bookmark identifier.
Returns
-------
rid: string
Unique reference identifier.
See Also
--------
:http:post:`/api/projects/(pid)/references`
"""
return self.request("POST", "/api/projects/%s/references" % (pid),
headers={"content-type":"application/json"}, data=json.dumps({"name":name, "model-type":mtype, "mid":mid, "bid":bid}))["id"]
[docs]
def post_projects(self, name, description=""):
"""Creates a new project, returning the project ID.
Parameters
----------
name: string, required
Name of project to be created.
description: string, optional
Description of new project.
Returns
-------
pid: string
Unique project identifier.
See Also
--------
:http:post:`/api/projects`
"""
return self.request("POST", "/api/projects",
headers={"content-type":"application/json"},
data=json.dumps({"name":name, "description":description}))["id"]
[docs]
def post_remotes(self, hostname, username, password, agent=None):
"""Creates a new remote connection from the Slycat server to another host.
Parameters
----------
hostname: string, required
Name of remote host.
username: string, required
User name for connection.
password: string, required
Password to authenticate connection.
agent: boolean, optional
Create an agent upon establishing connection.
Returns
-------
sid: string
Session ID for connection.
See Also
--------
:http:post:`/api/remotes`
"""
return self.request("POST", "/api/remotes",
headers={"content-type":"application/json"},
data=json.dumps({"hostname":hostname, "username":username,
"password":password, "agent": agent}))["sid"]
[docs]
def post_remote_browse(self, sid, path, file_reject=None, file_allow=None,
directory_allow=None, directory_reject=None):
"""Uses an existing remote session to retrieve remote filesystem information.
Parameters
----------
sid: string, required
Session ID for connection.
path: string, required
Remote file system path (must be absolute).
file_reject: string, optional
Regular expression for rejecting files.
file_allow: string, optional
Regular expression for retaining files.
directory_reject: string, optional
Regular expression for rejecting directories.
directory_allow: string, optional
Regular expression for retaining directories.
Returns
-------
response_body: JSON like object.
See Also
--------
:http:post:`/api/remotes/(hostname)/browse(path)`
"""
body = {}
if file_reject is not None:
body["file-reject"] = file_reject
if file_allow is not None:
body["file-allow"] = file_allow
if directory_reject is not None:
body["directory-reject"] = directory_reject
if directory_allow is not None:
body["directory-allow"] = directory_allow
return self.request("POST", "/api/remotes/" + sid + "/browse" + path,
headers={"content-type":"application/json"}, data=json.dumps(body))
[docs]
def put_model(self, mid, model):
"""Modify a Slycat model.
Parameters
----------
mid: string, required
Model identifier.
model: dictionary, required
JSON like dictionary with fields to modified model including:
* name (*string, optional*)
* description (*string, optional*)
* state (*string, optional*)
* progress (*float, optional*)
* message (*string, optional*)
See Also
--------
:http:put:`/api/models/(mid)`
"""
self.request("PUT", "/api/models/%s" % (mid),
headers={"content-type":"application/json"}, data=json.dumps(model))
[docs]
def put_model_arrayset_data(self, mid, aid, hyperchunks, data, force_json=False):
"""Write data to an arrayset artifact on the server.
Parameters
----------
mid: string, required
Unique model identifier.
aid: string, required
Unique (to the model) arrayset artifact id.
hyperchunks: string, required
Specifies where the data will be stored, in :ref:`Hyperchunks` format.
data: iterable, required)
A collection of numpy.ndarray data chunks to be uploaded. The number of
data chunks must match the number implied by the `hyperchunks` parameter.
force_json: bool, optional)
Force the client to upload data using JSON instead of the binary format.
See Also
--------
:http:put:`/api/models/(mid)/arraysets/(aid)/data`
"""
# Sanity check arguments
if not isinstance(mid, str):
cherrypy.log.error("slycat.web.client.__init__.py put_model_arrayset_data",
"Model id must be a string")
raise ValueError("Model id must be a string.")
if not isinstance(aid, str):
cherrypy.log.error("slycat.web.client.__init__.py put_model_arrayset_data",
"Artifact id must be a string")
raise ValueError("Artifact id must be a string.")
if not isinstance(hyperchunks, str):
cherrypy.log.error("slycat.web.client.__init__.py put_model_arrayset_data",
"Hyperchunks specification must be a string.")
raise ValueError("Hyperchunks specification must be a string.")
for chunk in data:
if not isinstance(chunk, numpy.ndarray):
cherrypy.log.error("slycat.web.client.__init__.py put_model_arrayset_data",
"Data chunk must be a numpy array.")
raise ValueError("Data chunk must be a numpy array.")
# Mark whether every data chunk is numeric ... if so, we can send the data in binary form.
use_binary = numpy.all([chunk.dtype.char != "U" for chunk in data]) and not force_json
# Build-up the request
request_data = {}
request_data["hyperchunks"] = hyperchunks
if use_binary:
request_data["byteorder"] = sys.byteorder
if use_binary:
# binary data
request_buffer = io.BytesIO()
for chunk in data:
request_buffer.write(chunk.tostring(order="C"))
else:
# string data
request_buffer = io.StringIO()
request_buffer.write(json.dumps([chunk.tolist() for chunk in data]))
# Send the request to the server ...
self.request("PUT", "/api/models/%s/arraysets/%s/data" % (mid, aid),
data=request_data, files={"data":request_buffer.getvalue()})
[docs]
def put_model_arrayset_array(self, mid, aid, array, dimensions, attributes):
"""Starts a new array set array, ready to receive data.
Parameters
----------
mid: string, required
Unique model identifier.
aid: string, required
Unique artifact identifier.
array: int, required
Unique array index.
dimensions: array, required
Array dimensions.
attributes: array, required
Array attributes (data types).
See Also
--------
:http:put:`/api/models/(mid)/arraysets/(aid)/arrays/(array)`
"""
stub = slycat.darray.Stub(dimensions, attributes)
self.request("PUT", "/api/models/%s/arraysets/%s/arrays/%s" % (mid, aid, array),
headers={"content-type":"application/json"},
data=json.dumps({"dimensions":stub.dimensions, "attributes":stub.attributes}))
[docs]
def put_model_arrayset(self, mid, aid, input=True):
"""Starts a new model array set artifact, ready to receive data.
Parameters
----------
mid: string, required
Unique model identifier.
aid: string, required
Unique artifact identifier.
input: boolean, optional
Set to true (default) if array set is a model input.
See Also
--------
:http:put:`/api/models/(mid)/arraysets/(aid)`
"""
self.request("PUT", "/api/models/%s/arraysets/%s" % (mid, aid),
headers={"content-type":"application/json"}, data=json.dumps({"input":input}))
[docs]
def put_model_parameter(self, mid, aid, value, input=True):
"""Store a model parameter artifact.
Model parameters are JSON objects of arbitrary complexity. They are stored directly within the model
as part of its database record, so they should be limited in size (larger data should be stored using
arraysets or files).
To get the value of a parameter artifact, use :func:`get_model` and read the value
directly from the model record. An artifact id `foo` will be accessible in the
record as `model["artifact:foo"]`.
Parameters
----------
mid: string, required
Unique model identifier.
aid: string, required
Unique (within the model) artifact id.
value: object, required
An arbitrary collection of JSON-compatible data.
input: boolean, optional
Marks whether this artifact is a model input.
See Also
--------
:http:put:`/api/models/(mid)/parameters/(aid)`
"""
self.request("PUT", "/api/models/%s/parameters/%s" % (mid, aid),
headers={"content-type":"application/json"}, data=json.dumps({"value":value, "input":input}))
[docs]
def put_project(self, pid, project):
"""Modifies a project.
Parameters
----------
pid: string, required
Unique project identifier.
project: dictionary, required
JSON like dictionary with fields to modified model including:
* name (*string, optional*)
* description (*string, optional*)
* acl (*string, optional*) -- access control list
See Also
--------
:http:put:`/api/projects/(pid)`
"""
return self.request("PUT", "/api/projects/%s" % pid,
headers={"content-type":"application/json"}, data=json.dumps(project))
##################################################################################
# Convenience functions that layer additional functionality atop the RESTful API #
##################################################################################
[docs]
def find_project(self, name):
"""Return a project identified by name.
Parameters
----------
name: string, required
The name of the project to return.
Returns
-------
project: The matching project, which is an arbitrary collection of JSON-compatible data.
Raises
------
Exception
If a project with a matching name can't be found, or more than one project matches the name.
See Also
--------
:func:`find_or_create_project`, :func:`get_projects`
"""
projects = [project for project in self.get_projects()["projects"]
if project["name"] == name]
if len(projects) > 1:
cherrypy.log.error("slycat.web.client.__init__.py find_project",
"More than one project matched the given name.")
raise Exception("More than one project matched the given name.")
elif len(projects) == 1:
return projects[0]
else:
cherrypy.log.error("slycat.web.client.__init__.py find_project",
"No project matched the given name.")
raise Exception("No project matched the given name.")
[docs]
def find_or_create_project(self, name, description=""):
"""Return a project identified by name, or newly created.
Parameters
----------
name: string, required
The name of the project to return (or create).
description: string, optional
Description to use for the new project (if a new project is created).
Returns
-------
pid: string
Unique identifier of the matching (or newly created) project.
Raises
------
Exception
If more than one project matches the given name.
See Also
--------
:func:`post_projects`
"""
projects = [project for project in self.get_projects()["projects"]
if project["name"] == name]
if len(projects) > 1:
cherrypy.log.error("slycat.web.client.__init__.py find_or_create_project",
"More than one project matched the given name. Try using a " +
"different project name instead.")
raise Exception("More than one project matched the given name. " +
"Try using a different project name instead.")
elif len(projects) == 1:
return projects[0]["_id"]
else:
return self.post_projects(name, description)
[docs]
def update_model(self, mid, **kwargs):
"""Update model state.
This function provides a more convenient alternative to :func:`put_model`.
See Also
--------
:func:`put_model`
"""
model = {key: value for key, value in list(kwargs.items()) if value is not None}
self.put_model(mid, model)
[docs]
def join_model(self, mid):
"""Wait for a model to complete before returning.
A Slycat model goes through several distinct phases over its lifetime:
1. The model is created.
2. Input artifacts are pushed into the model.
3. The model is marked "finished".
4. Optional one-time computation is performed on the server, storing output artifacts.
5. The model is complete and ready to be viewed.
Use this function in scripts that have performed steps 1, 2, and 3 and need to wait until
step 4 completes.
Parameters
----------
mid: string, required
Unique model identifier.
Notes
-----
A model that hasn't been finished will never complete - you should
ensure that post_model_finish() is called successfully before calling
join_model().
See Also
--------
:func:`post_model_finish`
"""
while True:
model = self.request("GET", "/api/models/%s" % (mid),
headers={"accept":"application/json"})
if "state" in model and model["state"] not in ["waiting", "running"]:
return
time.sleep(1.0)
##############################################################
# Functions that manage file uploading using the RESTful API #
##############################################################
[docs]
def post_uploads(self, mid, parser, aids, input=True):
"""Create Slycat file upload session.
Create an upload session used to upload files for storage as model artifacts.
Once an upload session has been created, use upload_file put_upload_file_part
to upload files directly from the client to the server.
Parameters
----------
mid: string, required
Unique model identifier.
parser: string,
Name of parser to call after completion of upload.
aids: array, required
Artifact IDs for storage.
input: boolean, optional
True to create input artifacts for the model.
Returns
-------
uid: string
Upload session ID.
See Also
--------
:http:post:`/api/uploads`
"""
return self.request("POST", "/api/uploads", headers={"content-type":"application/json"},
data=json.dumps({"mid":mid, "input":input, "parser":parser, "aids":aids}))["id"]
[docs]
def put_upload_file_part(self, uid, pid, fid, file_slice):
"""Upload a file or part of a file to Slycat.
Upload a file (or part of a file) as part of an upload session created with
post_uploads.
Use the “pid” and “fid” parameters to specify that the data being uploaded is
for part M of file N. To upload a file from the client, specify the “file” parameter.
Parameters
----------
uid: string, required
Unique file upload session ID.
pid: string, required
Zero-based part ID of file being uploaded.
fid: string, required
Zero-based file ID for file being uploaded.
file: file part in bytes, required
File part to upload.
See Also
--------
:http:put:`/api/uploads/(uid)/files/(fid)/parts/(pid)`
"""
return self.request("PUT", "/api/uploads/%s/files/%s/parts/%s" % (uid, fid, pid),
data={'file': base64.b64encode(file_slice)})
[docs]
def post_upload_finished(self, uid, file_parts):
"""Notify Slycat server that file upload is finished
Notify the server that all files have been uploaded for the given upload
session, and processing can begin. The request must include the uploaded
parameter, which specifies the number of files that were uploaded,
and the number of parts in each file. The server uses this information to
validate that it received every part of every file that the client sent.
Parameters
----------
uid: string, required
Unique file upload session ID.
file_parts: array, required
Array of length number of files,
with each entry number of slices for that file.
See Also
--------
:http:post:`/api/uploads/(uid)/finished`
"""
return self.request("POST", "/api/uploads/%s/finished" % uid,
headers={"content-type":"application/json"},
data=json.dumps({'uploaded': file_parts}))
[docs]
def delete_upload(self, uid):
"""Delete uploaded files from Slycat server
Delete an upload session used to upload files for storage as model artifacts.
This function must be called once the client no longer needs the session,
whether the upload(s) have been completed successfully or the client is cancelling
an incomplete session.
Note that you can examine the return codes to see if the files have been completely
parsed by Slycat.
Parameters
----------
uid: string, required
Unique file upload session ID.
Returns
-------
status_code: int
Http status code:
* 204 No Content – The upload session and temporary storage has been deleted.
* 409 Conflict – The upload session cannot be deleted, parsing is in progress.
See Also
--------
:http:post:`/api/uploads`
"""
# call directly to get response code
response = self.session.delete(self.host + "/api/uploads/%s" % uid,
proxies=self.keywords.get("proxies"),
verify=self.keywords.get("verify"))
# return code
return response.status_code
[docs]
def upload_files(self, mid, file_list, parser, parser_parms, progress=True):
"""Upload a list of files for a given model to the Slycat server.
The files will be parsed by specified parser. This is not a direct call
to the API, but rather a convenience call to load/parse a list of files.
Parameters
----------
mid: string, required
Model ID associated with the files to be uploaded.
file_list: string array, required
Local files to be uploaded to the Slycat server. This can be one file,
but it should be passed as an array with one file.
parser: string, required
Name of parser to use on the Slycat server.
parser_parms: array, required
Any parameters to be passed to the parser (passed using aids when the
upload session is established).
progress: boolean, optional
Display a download progress indicator using standard out (will not
be logged to the log file).
"""
# how many files?
num_files = len(file_list)
# get number of parts for progress bar
file_slices_to_upload = []
for fid in range(num_files):
num_slices = math.ceil(os.path.getsize(file_list[fid]) / self.file_slice_size)
file_slices_to_upload.append(num_slices)
# create upload session
uid = self.post_uploads(mid, parser, parser_parms)
# keep track of slices uploaded
file_slices_uploaded = [0] * num_files
# upload each file
for fid in range(num_files):
# print progress bar if desired
if progress:
print('Uploading "%s":' % file_list[fid])
# split each file into slices
with open(file_list[fid], "rb") as file:
# get file slice
file_slice = file.read(self.file_slice_size)
while (file_slice != b''):
# upload slice
self.put_upload_file_part(uid, file_slices_uploaded[fid], fid, file_slice)
# advance to next slice
file_slice = file.read(self.file_slice_size)
file_slices_uploaded[fid] += 1
if progress:
_print_progress_bar(file_slices_uploaded[fid], file_slices_to_upload[fid],
prefix = 'Progress:', suffix = 'Complete', length = 50)
# finish upload
self.post_upload_finished(uid, file_slices_uploaded)
# wait for parsing to finish
while True:
if self.delete_upload(uid) == 409:
time.sleep(1.0)
else:
break
[docs]
def connect(arguments, **keywords):
"""Factory function for client connections that takes an option parser as input.
Parameters
----------
arguments: argument parser object, required
Parsed command line arguments.
keywords: dictionary, optional
Additional options to pass to requests.
Returns
-------
connection: session for submitting requests to Slycat sever.
"""
# arguments are from the command line parser,
# keywords get passed into the actual requests
# check for --no-verify or security certificate
if arguments.no_verify:
keywords["verify"] = False
elif arguments.verify is not None:
keywords["verify"] = arguments.verify
return Connection(auth=(arguments.user, arguments.password),
host=arguments.host, port=arguments.port, kerberos=arguments.kerberos,
proxies={"http":arguments.http_proxy, "https":arguments.https_proxy},
file_slice_size=arguments.file_slice_size, **keywords)