run command, restoring default files upload on job creation, restoring default values for project and container

Author: kossak

paperspace/cli/__init__.py

@@ -10,6 +10,7 @@
 import paperspace.cli.machines
 import paperspace.cli.models
 import paperspace.cli.projects
+import paperspace.cli.run
 
 
 def show(self, file=None):

paperspace/cli/jobs.py

@@ -1,3 +1,5 @@
+import functools
+
 import click
 
 from paperspace import client, config
@@ -64,28 +66,35 @@ def list_jobs(api_key, **filters):
     command.execute(filters=filters)
 
 
+def common_jobs_create_options(f):
+    options = [
+        click.option("--name", "name", help="Job name", required=True),
+        click.option("--machineType", "machineType", help="Virtual machine type"),
+        click.option("--container", "container", default="paperspace/tensorflow-python", help="Docker container"),
+        click.option("--command", "command", help="Job command/entrypoint"),
+        click.option("--ports", "ports", help="Mapped ports"),
+        click.option("--isPublic", "isPublic", help="Flag: is job public"),
+        click.option("--workspace", "workspace", required=False, help="Path to workspace directory"),
+        click.option("--workspaceArchive", "workspaceArchive", required=False, help="Path to workspace archive"),
+        click.option("--workspaceUrl", "workspaceUrl", required=False, help="Project git repository url"),
+        click.option("--workingDirectory", "workingDirectory", help="Working directory for the experiment", ),
+        click.option("--ignoreFiles", "ignore_files", help="Ignore certain files from uploading"),
+        click.option("--experimentId", "experimentId", help="Experiment Id"),
+        click.option("--jobEnv", "envVars", type=json_string, help="Environmental variables "),
+        click.option("--useDockerfile", "useDockerfile", help="Flag: using Dockerfile"),
+        click.option("--isPreemptible", "isPreemptible", help="Flag: isPreemptible"),
+        click.option("--project", "project", help="Project name"),
+        click.option("--projectId", "projectHandle", help="Project ID"),
+        click.option("--startedByUserId", "startedByUserId", help="User ID"),
+        click.option("--relDockerfilePath", "relDockerfilePath", help="Relative path to Dockerfile"),
+        click.option("--registryUsername", "registryUsername", help="Docker registry username"),
+        click.option("--registryPassword", "registryPassword", help="Docker registry password"),
+    ]
+    return functools.reduce(lambda x, opt: opt(x), reversed(options), f)
+
+
 @jobs_group.command("create", help="Create job")
-@click.option("--name", "name", help="Job name", required=True)
-@click.option("--machineType", "machineType", help="Virtual machine type")
-@click.option("--container", "container", help="Docker container")
-@click.option("--command", "command", help="Job command/entrypoint")
-@click.option("--ports", "ports", help="Mapped ports")
-@click.option("--isPublic", "isPublic", help="Flag: is job public")
-@click.option("--workspace", "workspace", required=False, help="Path to workspace directory")
-@click.option("--workspaceArchive", "workspaceArchive", required=False, help="Path to workspace archive")
-@click.option("--workspaceUrl", "workspaceUrl", required=False, help="Project git repository url")
-@click.option("--workingDirectory", "workingDirectory", help="Working directory for the experiment")
-@click.option("--ignoreFiles", "ignore_files", help="Ignore certain files from uploading")
-@click.option("--experimentId", "experimentId", help="Experiment Id")
-@click.option("--jobEnv", "envVars", type=json_string, help="Environmental variables ")
-@click.option("--useDockerfile", "useDockerfile", help="Flag: using Dockerfile")
-@click.option("--isPreemptible", "isPreemptible", help="Flag: isPreemptible")
-@click.option("--project", "project", help="Project name")
-@click.option("--projectId", "projectHandle", help="Project ID", required=True)
-@click.option("--startedByUserId", "startedByUserId", help="User ID")
-@click.option("--relDockerfilePath", "relDockerfilePath", help="Relative path to Dockerfile")
-@click.option("--registryUsername", "registryUsername", help="Docker registry username")
-@click.option("--registryPassword", "registryPassword", help="Docker registry password")
+@common_jobs_create_options
 @api_key_option
 def create_job(api_key, **kwargs):
     del_if_value_is_none(kwargs)

paperspace/cli/run.py

@@ -0,0 +1,63 @@
+import os
+import sys
+
+import click
+
+from paperspace import client, config
+from paperspace.cli import common
+from paperspace.cli.cli import cli
+from paperspace.cli.common import del_if_value_is_none
+from paperspace.cli.jobs import common_jobs_create_options
+from paperspace.commands.jobs import CreateJobCommand
+from paperspace.workspace import WorkspaceHandler
+
+RUN_MODE_DEFAULT = 1
+RUN_MODE_PYTHON_COMMAND = 2
+RUN_MODE_SHELL_COMMAND = 3
+RUN_MODE_PYTHON_MODULE = 4
+
+
+def get_executor(mode, python_version=None):
+    python_version = python_version or str(sys.version_info[0])  # defaults locally running version
+    python_bin = 'python{v}'.format(v=python_version)
+    executors = {
+        RUN_MODE_DEFAULT: python_bin,
+        RUN_MODE_PYTHON_COMMAND: '{python} -c'.format(python=python_bin),
+        RUN_MODE_SHELL_COMMAND: '',
+        RUN_MODE_PYTHON_MODULE: '{python} -m'.format(python=python_bin),
+    }
+    return executors[mode]
+
+
+def clear_script_name(script_name, mode):
+    if mode == RUN_MODE_DEFAULT:
+        return os.path.basename(script_name)
+    return script_name
+
+
+def create_command(mode, script, python_version=None):
+    executor = get_executor(mode, python_version)
+    script_name = clear_script_name(script[0], mode)
+    script_params = ' '.join(script[1:])
+    command = '{executor} {script_name} {script_params}'.format(executor=executor, script_name=script_name,
+                                                                script_params=script_params)
+    return command
+
+
+@cli.command("new-run")
+@click.option("-c", "--python-command", "mode", flag_value=RUN_MODE_PYTHON_COMMAND)
+@click.option("-m", "--module", "mode", flag_value=RUN_MODE_PYTHON_MODULE)
+@click.option("-s", "--shell", "mode", flag_value=RUN_MODE_SHELL_COMMAND)
+@common_jobs_create_options
+@click.argument("script", nargs=-1)
+@common.api_key_option
+def run(script, api_key, mode, **kwargs):
+    del_if_value_is_none(kwargs)
+
+    mode = mode or RUN_MODE_DEFAULT
+    command = create_command(mode, script)
+    kwargs['command'] = command
+
+    jobs_api = client.API(config.CONFIG_HOST, api_key=api_key)
+    command = CreateJobCommand(api=jobs_api, workspace_handler=WorkspaceHandler())
+    command.execute(kwargs)

paperspace/client.py

@@ -28,11 +28,11 @@ def get_path(self, url):
         template = "{}{}" if url.startswith("/") else "{}/{}"
         return template.format(api_url, url)
 
-    def post(self, url, json=None, params=None, files=None):
+    def post(self, url, json=None, params=None, files=None, data=None):
         path = self.get_path(url)
-        logger.debug("POST request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}"
-                     .format(path, self.headers, json, params))
-        response = requests.post(path, json=json, params=params, headers=self.headers, files=files)
+        logger.debug("POST request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}\n\tdata: {}"
+                     .format(path, self.headers, json, params, data))
+        response = requests.post(path, json=json, params=params, headers=self.headers, files=files, data=data)
         logger.debug("Response status code: {}".format(response.status_code))
         logger.debug("Response content: {}".format(response.content))
         return response

paperspace/commands/experiments.py

@@ -30,7 +30,7 @@ def _log_create_experiment(self, response, success_msg_template, error_msg):
 class CreateExperimentCommand(ExperimentCommand):
 
     def execute(self, json_):
-        workspace_url = self._workspace_handler.upload_workspace(json_)
+        workspace_url = self._workspace_handler.handle(json_)
         if workspace_url:
             json_['workspaceUrl'] = workspace_url
 
@@ -43,7 +43,7 @@ def execute(self, json_):
 
 class CreateAndStartExperimentCommand(ExperimentCommand):
     def execute(self, json_):
-        workspace_url = self._workspace_handler.upload_workspace(json_)
+        workspace_url = self._workspace_handler.handle(json_)
         if workspace_url:
             json_['workspaceUrl'] = workspace_url
 

paperspace/commands/jobs.py

@@ -7,7 +7,7 @@
 from paperspace.commands import common
 from paperspace.exceptions import BadResponseError
 from paperspace.utils import get_terminal_lines
-from paperspace.workspace import S3WorkspaceHandler
+from paperspace.workspace import S3WorkspaceHandler, WorkspaceHandler
 
 
 class JobsCommandBase(common.CommandBase):
@@ -126,22 +126,43 @@ def _make_table(self, logs, table, table_data):
 class CreateJobCommand(JobsCommandBase):
     def __init__(self, workspace_handler=None, **kwargs):
         super(CreateJobCommand, self).__init__(**kwargs)
-        experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=kwargs.get('api_key'))
-        self._workspace_handler = workspace_handler or S3WorkspaceHandler(experiments_api=experiments_api,
-                                                                          logger=self.logger)
+        # experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=kwargs.get('api_key'))
+        self._workspace_handler = workspace_handler or WorkspaceHandler(logger=self.logger)
 
     def execute(self, json_):
         url = "/jobs/createJob/"
-
-        workspace_url = self._workspace_handler.upload_workspace(json_)
+        files = None
+        workspace_url = self._workspace_handler.handle(json_)
         if workspace_url:
-            json_['workspaceFileName'] = workspace_url
-        json_['projectId'] = json_.get('projectId', json_.get('projectHandle'))
-        response = self.api.post(url, json_)
+            if self._workspace_handler.archive_path:
+                archive_basename = self._workspace_handler.archive_basename
+                json_["workspaceFileName"] = archive_basename
+                # json_["file"] = open(workspace_url, "rb")
+                # monitor = MultipartEncoder(json_).get_monitor()
+                # # s3_encoder = encoder.MultipartEncoder(fields={"file": open(workspace_url, "rb")})
+                # # monitor = encoder.MultipartEncoderMonitor(s3_encoder, callback=self._create_callback(s3_encoder))
+                # self.api.headers["Content-Type"] = monitor.content_type
+                self.api.headers["Content-Type"] = "multipart/form-data"
+                # data = monitor
+                files = {"file": open(workspace_url, "rb")}
+            else:
+                json_["workspaceFileName"] = workspace_url
+
+        self.set_project(json_)
+
+        response = self.api.post(url, params=json_, files=files)
         self._log_message(response,
                           "Job created",
                           "Unknown error while creating job")
 
+    @staticmethod
+    def set_project(json_):
+        project_id = json_.get("projectId", json_.get("projectHandle"))
+        if not project_id:
+            json_["project"] = "paperspace-python"
+        else:
+            json_["projectId"] = project_id
+
 
 class ArtifactsDestroyCommand(JobsCommandBase):
     def execute(self, job_id, files=None):

paperspace/workspace.py

@@ -13,15 +13,34 @@
     PresignedUrlMalformedResponseError, PresignedUrlError
 
 
-class S3WorkspaceHandler:
-    def __init__(self, experiments_api, logger=None):
+class MultipartEncoder(object):
+    def __init__(self, fields):
+        s3_encoder = encoder.MultipartEncoder(fields=fields)
+        self.monitor = encoder.MultipartEncoderMonitor(s3_encoder, callback=self._create_callback(s3_encoder))
+
+    def get_monitor(self):
+        return self.monitor
+
+    @staticmethod
+    def _create_callback(encoder_obj):
+        bar = progressbar.ProgressBar(max_value=encoder_obj.len)
+
+        def callback(monitor):
+            bar.update(monitor.bytes_read)
+
+        return callback
+
+
+class WorkspaceHandler(object):
+    def __init__(self, logger=None):
         """
 
         :param experiments_api: paperspace.client.API
         :param logger: paperspace.logger
         """
-        self.experiments_api = experiments_api
         self.logger = logger or default_logger
+        self.archive_path = None
+        self.archive_basename = None
 
     @staticmethod
     def _retrieve_file_paths(dir_name, ignored_files=None):
@@ -77,27 +96,49 @@ def _zip_workspace(self, workspace_path, ignore_files):
         self.logger.log('\nFinished creating archive: %s' % zip_file_name)
         return zip_file_path
 
-    @staticmethod
-    def _create_callback(encoder_obj):
-        bar = progressbar.ProgressBar(max_value=encoder_obj.len)
+    def handle(self, input_data):
+        workspace_archive, workspace_path, workspace_url = self._validate_input(input_data)
+        ignore_files = input_data.get('ignore_files')
 
-        def callback(monitor):
-            bar.update(monitor.bytes_read)
+        if workspace_url:
+            return  # nothing to do
 
-        return callback
+        if workspace_archive:
+            archive_path = os.path.abspath(workspace_archive)
+        else:
+            archive_path = self._zip_workspace(workspace_path, ignore_files)
+        self.archive_path = archive_path
+        self.archive_basename = os.path.basename(archive_path)
+        return archive_path
 
-    def upload_workspace(self, input_data):
+    @staticmethod
+    def _validate_input(input_data):
         workspace_url = input_data.get('workspaceUrl')
         workspace_path = input_data.get('workspace')
         workspace_archive = input_data.get('workspaceArchive')
-        ignore_files = input_data.get('ignore_files')
 
         if (workspace_archive and workspace_path) or (workspace_archive and workspace_url) or (
                 workspace_path and workspace_url):
             raise click.UsageError("Use either:\n\t--workspaceUrl to point repository URL"
                                    "\n\t--workspace to point on project directory"
                                    "\n\t--workspaceArchive to point on project .zip archive"
                                    "\n or neither to use current directory")
+        return workspace_archive, workspace_path, workspace_url
+
+
+class S3WorkspaceHandler(WorkspaceHandler):
+    def __init__(self, experiments_api, logger=None):
+        """
+
+        :param experiments_api: paperspace.client.API
+        :param logger: paperspace.logger
+        """
+        super(S3WorkspaceHandler, self).__init__(logger=logger)
+        self.experiments_api = experiments_api
+
+    def handle(self, input_data):
+        workspace_archive, workspace_path, workspace_url = self._validate_input(input_data)
+        ignore_files = input_data.get('ignore_files')
 
         if workspace_url:
             return  # nothing to do
@@ -133,8 +174,7 @@ def _upload(self, archive_path, s3_upload_data):
         fields = OrderedDict(s3_upload_data['fields'])
         fields.update(files)
 
-        s3_encoder = encoder.MultipartEncoder(fields=fields)
-        monitor = encoder.MultipartEncoderMonitor(s3_encoder, callback=self._create_callback(s3_encoder))
+        monitor = MultipartEncoder(fields).get_monitor()
         s3_response = requests.post(s3_upload_data['url'], data=monitor, headers={'Content-Type': monitor.content_type})
         self.logger.debug("S3 upload response: {}".format(s3_response.headers))
         if not s3_response.ok: