Merge pull request #111 from Paperspace/run-restored

Command `run` restored

Author: mkulaczkowski

paperspace/cli/__init__.py

@@ -10,6 +10,7 @@
 import paperspace.cli.machines
 import paperspace.cli.models
 import paperspace.cli.projects
+import paperspace.cli.run
 
 
 def show(self, file=None):

paperspace/cli/common.py

@@ -1,7 +1,7 @@
+import functools
 import getpass
 
 import click
-from click import group
 from click_didyoumean import DYMMixin
 from click_help_colors import HelpColorsGroup
 
@@ -32,10 +32,10 @@ def group(self, *args, **kwargs):
             aliases.append(kwargs.pop('alias'))
 
         def decorator(f):
-            cmd = group(*_args, **kwargs)(f)
+            cmd = click.group(*_args, **kwargs)(f)
             self.add_command(cmd)
             for alias in set(aliases):
-                alias_cmd = group(alias, **kwargs)(f)
+                alias_cmd = click.group(alias, **kwargs)(f)
                 self.add_command(alias_cmd)
                 alias_cmd.commands = cmd.commands
             return cmd
@@ -51,3 +51,20 @@ def callback_fun(ctx, param, value):
         return value
 
     return callback_fun
+
+
+def deprecated(version="1.0.0"):
+    deprecated_invoke_notice = """DeprecatedWarning: \nWARNING: This command will not be included in version %s .
+For more information, please see:
+
+https://docs.paperspace.com
+If you depend on functionality not listed there, please file an issue.""" % version
+
+    def new_invoke(self, ctx):
+        click.echo(click.style(deprecated_invoke_notice, fg='red'), err=True)
+        super(type(self), self).invoke(ctx)
+
+    def decorator(f):
+        f.invoke = functools.partial(new_invoke, f)
+
+    return decorator

paperspace/cli/jobs.py

@@ -1,3 +1,5 @@
+import functools
+
 import click
 
 from paperspace import client, config
@@ -64,28 +66,35 @@ def list_jobs(api_key, **filters):
     command.execute(filters=filters)
 
 
+def common_jobs_create_options(f):
+    options = [
+        click.option("--name", "name", help="Job name", required=True),
+        click.option("--machineType", "machineType", help="Virtual machine type"),
+        click.option("--container", "container", default="paperspace/tensorflow-python", help="Docker container"),
+        click.option("--command", "command", help="Job command/entrypoint"),
+        click.option("--ports", "ports", help="Mapped ports"),
+        click.option("--isPublic", "isPublic", help="Flag: is job public"),
+        click.option("--workspace", "workspace", required=False, help="Path to workspace directory"),
+        click.option("--workspaceArchive", "workspaceArchive", required=False, help="Path to workspace archive"),
+        click.option("--workspaceUrl", "workspaceUrl", required=False, help="Project git repository url"),
+        click.option("--workingDirectory", "workingDirectory", help="Working directory for the experiment", ),
+        click.option("--ignoreFiles", "ignore_files", help="Ignore certain files from uploading"),
+        click.option("--experimentId", "experimentId", help="Experiment Id"),
+        click.option("--jobEnv", "envVars", type=json_string, help="Environmental variables "),
+        click.option("--useDockerfile", "useDockerfile", help="Flag: using Dockerfile"),
+        click.option("--isPreemptible", "isPreemptible", help="Flag: isPreemptible"),
+        click.option("--project", "project", help="Project name"),
+        click.option("--projectId", "projectHandle", help="Project ID"),
+        click.option("--startedByUserId", "startedByUserId", help="User ID"),
+        click.option("--relDockerfilePath", "relDockerfilePath", help="Relative path to Dockerfile"),
+        click.option("--registryUsername", "registryUsername", help="Docker registry username"),
+        click.option("--registryPassword", "registryPassword", help="Docker registry password"),
+    ]
+    return functools.reduce(lambda x, opt: opt(x), reversed(options), f)
+
+
 @jobs_group.command("create", help="Create job")
-@click.option("--name", "name", help="Job name", required=True)
-@click.option("--machineType", "machineType", help="Virtual machine type")
-@click.option("--container", "container", help="Docker container")
-@click.option("--command", "command", help="Job command/entrypoint")
-@click.option("--ports", "ports", help="Mapped ports")
-@click.option("--isPublic", "isPublic", help="Flag: is job public")
-@click.option("--workspace", "workspace", required=False, help="Path to workspace directory")
-@click.option("--workspaceArchive", "workspaceArchive", required=False, help="Path to workspace archive")
-@click.option("--workspaceUrl", "workspaceUrl", required=False, help="Project git repository url")
-@click.option("--workingDirectory", "workingDirectory", help="Working directory for the experiment")
-@click.option("--ignoreFiles", "ignore_files", help="Ignore certain files from uploading")
-@click.option("--experimentId", "experimentId", help="Experiment Id")
-@click.option("--jobEnv", "envVars", type=json_string, help="Environmental variables ")
-@click.option("--useDockerfile", "useDockerfile", help="Flag: using Dockerfile")
-@click.option("--isPreemptible", "isPreemptible", help="Flag: isPreemptible")
-@click.option("--project", "project", help="Project name")
-@click.option("--projectId", "projectHandle", help="Project ID", required=True)
-@click.option("--startedByUserId", "startedByUserId", help="User ID")
-@click.option("--relDockerfilePath", "relDockerfilePath", help="Relative path to Dockerfile")
-@click.option("--registryUsername", "registryUsername", help="Docker registry username")
-@click.option("--registryPassword", "registryPassword", help="Docker registry password")
+@common_jobs_create_options
 @api_key_option
 @click.pass_context
 def create_job(ctx, api_key, **kwargs):

paperspace/cli/run.py

@@ -0,0 +1,24 @@
+import click
+
+from paperspace import client, config
+from paperspace.cli import common
+from paperspace.cli.cli import cli
+from paperspace.cli.common import del_if_value_is_none, deprecated
+from paperspace.cli.jobs import common_jobs_create_options
+from paperspace.commands.run import RunCommand
+from paperspace.constants import RunMode
+
+
+@deprecated(version="0.6.0")
+@cli.command("run", help="Run script or command on remote cluster")
+@click.option("-c", "--python-command", "mode", flag_value=RunMode.RUN_MODE_PYTHON_COMMAND)
+@click.option("-m", "--module", "mode", flag_value=RunMode.RUN_MODE_PYTHON_MODULE)
+@click.option("-s", "--shell", "mode", flag_value=RunMode.RUN_MODE_SHELL_COMMAND)
+@common_jobs_create_options
+@click.argument("script", nargs=-1, required=True)
+@common.api_key_option
+def run(api_key, **kwargs):
+    del_if_value_is_none(kwargs)
+    jobs_api = client.API(config.CONFIG_HOST, api_key=api_key)
+    command = RunCommand(api=jobs_api)
+    command.execute(**kwargs)

paperspace/client.py

@@ -28,11 +28,11 @@ def get_path(self, url):
         template = "{}{}" if url.startswith("/") else "{}/{}"
         return template.format(api_url, url)
 
-    def post(self, url, json=None, params=None, files=None):
+    def post(self, url, json=None, params=None, files=None, data=None):
         path = self.get_path(url)
-        logger.debug("POST request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}"
-                     .format(path, self.headers, json, params))
-        response = requests.post(path, json=json, params=params, headers=self.headers, files=files)
+        logger.debug("POST request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}\n\tdata: {}"
+                     .format(path, self.headers, json, params, data))
+        response = requests.post(path, json=json, params=params, headers=self.headers, files=files, data=data)
         logger.debug("Response status code: {}".format(response.status_code))
         logger.debug("Response content: {}".format(response.content))
         return response

paperspace/commands/experiments.py

@@ -34,7 +34,7 @@ def _log_create_experiment(self, response, success_msg_template, error_msg):
 class CreateExperimentCommand(ExperimentCommand):
 
     def execute(self, json_):
-        workspace_url = self._workspace_handler.upload_workspace(json_)
+        workspace_url = self._workspace_handler.handle(json_)
         if workspace_url:
             json_['workspaceUrl'] = workspace_url
 
@@ -47,7 +47,7 @@ def execute(self, json_):
 
 class CreateAndStartExperimentCommand(ExperimentCommand):
     def execute(self, json_):
-        workspace_url = self._workspace_handler.upload_workspace(json_)
+        workspace_url = self._workspace_handler.handle(json_)
         if workspace_url:
             json_['workspaceUrl'] = workspace_url
 

paperspace/commands/jobs.py

@@ -3,11 +3,10 @@
 import terminaltables
 from click import style
 
-from paperspace import config, client
 from paperspace.commands import common
 from paperspace.exceptions import BadResponseError
 from paperspace.utils import get_terminal_lines
-from paperspace.workspace import S3WorkspaceHandler
+from paperspace.workspace import WorkspaceHandler
 
 
 class JobsCommandBase(common.CommandBase):
@@ -146,22 +145,41 @@ def _make_table(self, logs, table, table_data):
 class CreateJobCommand(JobsCommandBase):
     def __init__(self, workspace_handler=None, **kwargs):
         super(CreateJobCommand, self).__init__(**kwargs)
-        experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=kwargs.get('api_key'))
-        self._workspace_handler = workspace_handler or S3WorkspaceHandler(experiments_api=experiments_api,
-                                                                          logger=self.logger)
+        self._workspace_handler = workspace_handler or WorkspaceHandler(logger=self.logger)
 
     def execute(self, json_):
         url = "/jobs/createJob/"
-
-        workspace_url = self._workspace_handler.upload_workspace(json_)
+        files = None
+        workspace_url = self._workspace_handler.handle(json_)
         if workspace_url:
-            json_['workspaceFileName'] = workspace_url
-        json_['projectId'] = json_.get('projectId', json_.get('projectHandle'))
-        response = self.api.post(url, json_)
+            if self._workspace_handler.archive_path:
+                archive_basename = self._workspace_handler.archive_basename
+                json_["workspaceFileName"] = archive_basename
+                self.api.headers["Content-Type"] = "multipart/form-data"
+                files = self._get_files_dict(workspace_url)
+            else:
+                json_["workspaceFileName"] = workspace_url
+
+        self.set_project(json_)
+
+        response = self.api.post(url, params=json_, files=files)
         self._log_message(response,
                           "Job created",
                           "Unknown error while creating job")
 
+    @staticmethod
+    def _get_files_dict(workspace_url):
+        files = {"file": open(workspace_url, "rb")}
+        return files
+
+    @staticmethod
+    def set_project(json_):
+        project_id = json_.get("projectId", json_.get("projectHandle"))
+        if not project_id:
+            json_["project"] = "paperspace-python"
+        else:
+            json_["projectId"] = project_id
+
 
 class ArtifactsDestroyCommand(JobsCommandBase):
     def execute(self, job_id, files=None):

paperspace/commands/run.py

@@ -0,0 +1,57 @@
+import os
+import sys
+
+from paperspace import client, config, logger
+from paperspace.commands.jobs import CreateJobCommand
+from paperspace.constants import RunMode
+from paperspace.workspace import WorkspaceHandler
+
+
+class RunCommand(object):
+
+    def __init__(self, api=None, logger_=logger):
+        self.api = api
+        self.logger = logger_
+
+    @staticmethod
+    def _get_executor(mode, python_version=None):
+        python_version = python_version or str(sys.version_info[0])  # defaults locally running version
+        python_bin = 'python{v}'.format(v=python_version)
+        executors = {
+            RunMode.RUN_MODE_DEFAULT: python_bin,
+            RunMode.RUN_MODE_PYTHON_COMMAND: '{python} -c'.format(python=python_bin),
+            RunMode.RUN_MODE_SHELL_COMMAND: '',
+            RunMode.RUN_MODE_PYTHON_MODULE: '{python} -m'.format(python=python_bin),
+        }
+        return executors[mode]
+
+    @staticmethod
+    def _clear_script_name(script_name, mode):
+        if mode == RunMode.RUN_MODE_DEFAULT:
+            return os.path.basename(script_name)
+        return script_name
+
+    def _create_command(self, mode, script, python_version=None):
+        command_parts = []
+        executor = self._get_executor(mode, python_version)
+        if executor:
+            command_parts.append(executor)
+
+        script_name = self._clear_script_name(script[0], mode)
+        if script_name:
+            command_parts.append(script_name)
+
+        script_params = ' '.join(script[1:])
+        if script_params:
+            command_parts.append(script_params)
+
+        command = ' '.join(command_parts)
+        return command
+
+    def execute(self, mode=None, script=None, **json_):
+        mode = mode or RunMode.RUN_MODE_DEFAULT
+        command = self._create_command(mode, script)
+        json_['command'] = command
+
+        command = CreateJobCommand(api=self.api, workspace_handler=WorkspaceHandler())
+        command.execute(json_)

paperspace/constants.py

@@ -59,3 +59,10 @@ class Region(object):
                  "C1", "C2", "C3", "C4", "C5", "C6", "C7", "C8", "C9", "C10")
 
 BILLING_TYPES = ["hourly", "monthly"]
+
+
+class RunMode:
+    RUN_MODE_DEFAULT = 1
+    RUN_MODE_PYTHON_COMMAND = 2
+    RUN_MODE_SHELL_COMMAND = 3
+    RUN_MODE_PYTHON_MODULE = 4