Merge remote-tracking branch 'origin/master' into send-workspace-with-raw-post

# Conflicts:
#	paperspace/client.py
#	paperspace/commands/jobs.py
#	paperspace/workspace.py

Author: kossak

paperspace/cli/__init__.py

@@ -10,6 +10,7 @@
 import paperspace.cli.machines
 import paperspace.cli.models
 import paperspace.cli.projects
+import paperspace.cli.run
 
 
 def show(self, file=None):

paperspace/cli/common.py

@@ -1,7 +1,7 @@
+import functools
 import getpass
 
 import click
-from click import group
 from click_didyoumean import DYMMixin
 from click_help_colors import HelpColorsGroup
 
@@ -32,10 +32,10 @@ def group(self, *args, **kwargs):
             aliases.append(kwargs.pop('alias'))
 
         def decorator(f):
-            cmd = group(*_args, **kwargs)(f)
+            cmd = click.group(*_args, **kwargs)(f)
             self.add_command(cmd)
             for alias in set(aliases):
-                alias_cmd = group(alias, **kwargs)(f)
+                alias_cmd = click.group(alias, **kwargs)(f)
                 self.add_command(alias_cmd)
                 alias_cmd.commands = cmd.commands
             return cmd
@@ -51,3 +51,20 @@ def callback_fun(ctx, param, value):
         return value
 
     return callback_fun
+
+
+def deprecated(version="1.0.0"):
+    deprecated_invoke_notice = """DeprecatedWarning: \nWARNING: This command will not be included in version %s .
+For more information, please see:
+
+https://docs.paperspace.com
+If you depend on functionality not listed there, please file an issue.""" % version
+
+    def new_invoke(self, ctx):
+        click.echo(click.style(deprecated_invoke_notice, fg='red'), err=True)
+        super(type(self), self).invoke(ctx)
+
+    def decorator(f):
+        f.invoke = functools.partial(new_invoke, f)
+
+    return decorator

paperspace/cli/run.py

@@ -0,0 +1,24 @@
+import click
+
+from paperspace import client, config
+from paperspace.cli import common
+from paperspace.cli.cli import cli
+from paperspace.cli.common import del_if_value_is_none, deprecated
+from paperspace.cli.jobs import common_jobs_create_options
+from paperspace.commands.run import RunCommand
+from paperspace.constants import RunMode
+
+
+@deprecated(version="0.6.0")
+@cli.command("run", help="Run script or command on remote cluster")
+@click.option("-c", "--python-command", "mode", flag_value=RunMode.RUN_MODE_PYTHON_COMMAND)
+@click.option("-m", "--module", "mode", flag_value=RunMode.RUN_MODE_PYTHON_MODULE)
+@click.option("-s", "--shell", "mode", flag_value=RunMode.RUN_MODE_SHELL_COMMAND)
+@common_jobs_create_options
+@click.argument("script", nargs=-1, required=True)
+@common.api_key_option
+def run(api_key, **kwargs):
+    del_if_value_is_none(kwargs)
+    jobs_api = client.API(config.CONFIG_HOST, api_key=api_key)
+    command = RunCommand(api=jobs_api)
+    command.execute(**kwargs)

paperspace/commands/jobs.py

@@ -145,7 +145,6 @@ def _make_table(self, logs, table, table_data):
 class CreateJobCommand(JobsCommandBase):
     def __init__(self, workspace_handler=None, **kwargs):
         super(CreateJobCommand, self).__init__(**kwargs)
-        # experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=kwargs.get('api_key'))
         self._workspace_handler = workspace_handler or WorkspaceHandler(logger=self.logger)
 
     def execute(self, json_):

paperspace/commands/run.py

@@ -0,0 +1,57 @@
+import os
+import sys
+
+from paperspace import client, config, logger
+from paperspace.commands.jobs import CreateJobCommand
+from paperspace.constants import RunMode
+from paperspace.workspace import WorkspaceHandler
+
+
+class RunCommand(object):
+
+    def __init__(self, api=None, logger_=logger):
+        self.api = api
+        self.logger = logger_
+
+    @staticmethod
+    def _get_executor(mode, python_version=None):
+        python_version = python_version or str(sys.version_info[0])  # defaults locally running version
+        python_bin = 'python{v}'.format(v=python_version)
+        executors = {
+            RunMode.RUN_MODE_DEFAULT: python_bin,
+            RunMode.RUN_MODE_PYTHON_COMMAND: '{python} -c'.format(python=python_bin),
+            RunMode.RUN_MODE_SHELL_COMMAND: '',
+            RunMode.RUN_MODE_PYTHON_MODULE: '{python} -m'.format(python=python_bin),
+        }
+        return executors[mode]
+
+    @staticmethod
+    def _clear_script_name(script_name, mode):
+        if mode == RunMode.RUN_MODE_DEFAULT:
+            return os.path.basename(script_name)
+        return script_name
+
+    def _create_command(self, mode, script, python_version=None):
+        command_parts = []
+        executor = self._get_executor(mode, python_version)
+        if executor:
+            command_parts.append(executor)
+
+        script_name = self._clear_script_name(script[0], mode)
+        if script_name:
+            command_parts.append(script_name)
+
+        script_params = ' '.join(script[1:])
+        if script_params:
+            command_parts.append(script_params)
+
+        command = ' '.join(command_parts)
+        return command
+
+    def execute(self, mode=None, script=None, **json_):
+        mode = mode or RunMode.RUN_MODE_DEFAULT
+        command = self._create_command(mode, script)
+        json_['command'] = command
+
+        command = CreateJobCommand(api=self.api, workspace_handler=WorkspaceHandler())
+        command.execute(json_)

paperspace/constants.py

@@ -59,3 +59,10 @@ class Region(object):
                  "C1", "C2", "C3", "C4", "C5", "C6", "C7", "C8", "C9", "C10")
 
 BILLING_TYPES = ["hourly", "monthly"]
+
+
+class RunMode:
+    RUN_MODE_DEFAULT = 1
+    RUN_MODE_PYTHON_COMMAND = 2
+    RUN_MODE_SHELL_COMMAND = 3
+    RUN_MODE_PYTHON_MODULE = 4

paperspace/workspace.py

@@ -144,10 +144,10 @@ def __init__(self, experiments_api, logger=None):
         self.experiments_api = experiments_api
 
     def handle(self, input_data):
-        archive_path = super(S3WorkspaceHandler, self).handle(input_data)
-        if archive_path in ['none', None]:
-            return archive_path
-
+        workspace = super(S3WorkspaceHandler, self).handle(input_data)
+        if not self.archive_path:
+            return workspace
+        archive_path = workspace
         file_name = os.path.basename(archive_path)
         project_handle = input_data['projectHandle']
 

tests/functional/test_run.py

@@ -0,0 +1,87 @@
+import sys
+
+import mock
+from click.testing import CliRunner
+
+from paperspace.cli import cli
+from paperspace.client import default_headers
+from tests import MockResponse
+
+
+class TestRunCommand(object):
+    command_name = 'run'
+    common_commands = ["--name", "test", "--projectId", "projectId", "--apiKey", "some_key"]
+    url = "https://api.paperspace.io/jobs/createJob/"
+    headers = default_headers.copy()
+    headers["X-API-Key"] = "some_key"
+
+    @mock.patch("paperspace.client.requests.post")
+    @mock.patch("paperspace.workspace.WorkspaceHandler._zip_workspace")
+    @mock.patch("paperspace.commands.jobs.CreateJobCommand._get_files_dict")
+    def test_run_simple_file_with_args(self, get_files_patched, workspace_zip_patched, post_patched):
+        get_files_patched.return_value = mock.MagicMock()
+        workspace_zip_patched.return_value = '/foo/bar'
+        post_patched.return_value = MockResponse(status_code=200)
+
+        runner = CliRunner()
+        result = runner.invoke(cli.cli, [self.command_name] + self.common_commands + ["/myscript.py", "a", "b"])
+
+        expected_headers = self.headers.copy()
+        expected_headers.update({
+            'Content-Type': "multipart/form-data"
+        })
+        post_patched.assert_called_with(self.url,
+                                        params={'name': u'test', 'projectId': u'projectId',
+                                                'workspaceFileName': 'bar',
+                                                'command': 'python{} myscript.py a b'.format(str(sys.version_info[0])),
+                                                'projectHandle': u'projectId',
+                                                'container': u'paperspace/tensorflow-python'},
+                                        data=None,
+                                        files=mock.ANY,
+                                        headers=expected_headers,
+                                        json=None)
+
+    @mock.patch("paperspace.client.requests.post")
+    def test_run_python_command_with_args_and_no_workspace(self, post_patched):
+        post_patched.return_value = MockResponse(status_code=200)
+
+        runner = CliRunner()
+        result = runner.invoke(cli.cli,
+                               [self.command_name] + self.common_commands + ["-c", "print(foo)", "--workspace", "none"])
+
+        expected_headers = self.headers.copy()
+        post_patched.assert_called_with(self.url,
+                                        params={'name': u'test', 'projectId': u'projectId',
+                                                'workspaceFileName': 'none',
+                                                'workspace': 'none',
+                                                'command': 'python{} -c print(foo)'.format(str(sys.version_info[0])),
+                                                'projectHandle': u'projectId',
+                                                'container': u'paperspace/tensorflow-python'},
+                                        data=None,
+                                        files=None,
+                                        headers=expected_headers,
+                                        json=None)
+
+    @mock.patch("paperspace.client.requests.post")
+    @mock.patch("paperspace.workspace.WorkspaceHandler._zip_workspace")
+    def test_run_shell_command_with_args_with_s3_workspace(self, workspace_zip_patched, post_patched):
+        workspace_zip_patched.return_value = '/foo/bar'
+        post_patched.return_value = MockResponse(status_code=200)
+
+        runner = CliRunner()
+        result = runner.invoke(cli.cli,
+                               [self.command_name] + self.common_commands + ["-s", "echo foo", "--workspaceUrl",
+                                                                             "s3://bucket/object"])
+
+        expected_headers = self.headers.copy()
+        post_patched.assert_called_with(self.url,
+                                        params={'name': u'test', 'projectId': u'projectId',
+                                                'workspaceFileName': 's3://bucket/object',
+                                                'workspaceUrl': 's3://bucket/object',
+                                                'command': 'echo foo',
+                                                'projectHandle': u'projectId',
+                                                'container': u'paperspace/tensorflow-python'},
+                                        data=None,
+                                        files=None,
+                                        headers=expected_headers,
+                                        json=None)