Skip to content
This repository was archived by the owner on Aug 11, 2020. It is now read-only.

Commit b1a6809

Browse files
kossakkossak
committed
restoring default files upload on job creation, restoring default values for project and container
(cherry picked from restoring `run` branch)
1 parent 954e826 commit b1a6809

File tree

11 files changed

+267
-144
lines changed

11 files changed

+267
-144
lines changed

‎paperspace/cli/jobs.py‎

Lines changed: 30 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import functools
2+
13
import click
24

35
from paperspace import client, config
@@ -64,28 +66,35 @@ def list_jobs(api_key, **filters):
6466
command.execute(filters=filters)
6567

6668

69+
def common_jobs_create_options(f):
70+
options = [
71+
click.option("--name", "name", help="Job name", required=True),
72+
click.option("--machineType", "machineType", help="Virtual machine type"),
73+
click.option("--container", "container", default="paperspace/tensorflow-python", help="Docker container"),
74+
click.option("--command", "command", help="Job command/entrypoint"),
75+
click.option("--ports", "ports", help="Mapped ports"),
76+
click.option("--isPublic", "isPublic", help="Flag: is job public"),
77+
click.option("--workspace", "workspace", required=False, help="Path to workspace directory"),
78+
click.option("--workspaceArchive", "workspaceArchive", required=False, help="Path to workspace archive"),
79+
click.option("--workspaceUrl", "workspaceUrl", required=False, help="Project git repository url"),
80+
click.option("--workingDirectory", "workingDirectory", help="Working directory for the experiment", ),
81+
click.option("--ignoreFiles", "ignore_files", help="Ignore certain files from uploading"),
82+
click.option("--experimentId", "experimentId", help="Experiment Id"),
83+
click.option("--jobEnv", "envVars", type=json_string, help="Environmental variables "),
84+
click.option("--useDockerfile", "useDockerfile", help="Flag: using Dockerfile"),
85+
click.option("--isPreemptible", "isPreemptible", help="Flag: isPreemptible"),
86+
click.option("--project", "project", help="Project name"),
87+
click.option("--projectId", "projectHandle", help="Project ID"),
88+
click.option("--startedByUserId", "startedByUserId", help="User ID"),
89+
click.option("--relDockerfilePath", "relDockerfilePath", help="Relative path to Dockerfile"),
90+
click.option("--registryUsername", "registryUsername", help="Docker registry username"),
91+
click.option("--registryPassword", "registryPassword", help="Docker registry password"),
92+
]
93+
return functools.reduce(lambda x, opt: opt(x), reversed(options), f)
94+
95+
6796
@jobs_group.command("create", help="Create job")
68-
@click.option("--name", "name", help="Job name", required=True)
69-
@click.option("--machineType", "machineType", help="Virtual machine type")
70-
@click.option("--container", "container", help="Docker container")
71-
@click.option("--command", "command", help="Job command/entrypoint")
72-
@click.option("--ports", "ports", help="Mapped ports")
73-
@click.option("--isPublic", "isPublic", help="Flag: is job public")
74-
@click.option("--workspace", "workspace", required=False, help="Path to workspace directory")
75-
@click.option("--workspaceArchive", "workspaceArchive", required=False, help="Path to workspace archive")
76-
@click.option("--workspaceUrl", "workspaceUrl", required=False, help="Project git repository url")
77-
@click.option("--workingDirectory", "workingDirectory", help="Working directory for the experiment")
78-
@click.option("--ignoreFiles", "ignore_files", help="Ignore certain files from uploading")
79-
@click.option("--experimentId", "experimentId", help="Experiment Id")
80-
@click.option("--jobEnv", "envVars", type=json_string, help="Environmental variables ")
81-
@click.option("--useDockerfile", "useDockerfile", help="Flag: using Dockerfile")
82-
@click.option("--isPreemptible", "isPreemptible", help="Flag: isPreemptible")
83-
@click.option("--project", "project", help="Project name")
84-
@click.option("--projectId", "projectHandle", help="Project ID", required=True)
85-
@click.option("--startedByUserId", "startedByUserId", help="User ID")
86-
@click.option("--relDockerfilePath", "relDockerfilePath", help="Relative path to Dockerfile")
87-
@click.option("--registryUsername", "registryUsername", help="Docker registry username")
88-
@click.option("--registryPassword", "registryPassword", help="Docker registry password")
97+
@common_jobs_create_options
8998
@api_key_option
9099
def create_job(api_key, **kwargs):
91100
del_if_value_is_none(kwargs)

‎paperspace/client.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,11 @@ def get_path(self, url):
2828
template = "{}{}" if url.startswith("/") else "{}/{}"
2929
return template.format(api_url, url)
3030

31-
def post(self, url, json=None, params=None, files=None):
31+
def post(self, url, json=None, params=None, files=None, data=None):
3232
path = self.get_path(url)
33-
logger.debug("POST request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}"
34-
.format(path, self.headers, json, params))
35-
response = requests.post(path, json=json, params=params, headers=self.headers, files=files)
33+
logger.debug("POST request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}\n\tdata: {}"
34+
.format(path, self.headers, json, params, data))
35+
response = requests.post(path, json=json, params=params, headers=self.headers, files=files, data=data)
3636
logger.debug("Response status code: {}".format(response.status_code))
3737
logger.debug("Response content: {}".format(response.content))
3838
return response

‎paperspace/commands/experiments.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ def _log_create_experiment(self, response, success_msg_template, error_msg):
3030
class CreateExperimentCommand(ExperimentCommand):
3131

3232
def execute(self, json_):
33-
workspace_url = self._workspace_handler.upload_workspace(json_)
33+
workspace_url = self._workspace_handler.handle(json_)
3434
if workspace_url:
3535
json_['workspaceUrl'] = workspace_url
3636

@@ -43,7 +43,7 @@ def execute(self, json_):
4343

4444
class CreateAndStartExperimentCommand(ExperimentCommand):
4545
def execute(self, json_):
46-
workspace_url = self._workspace_handler.upload_workspace(json_)
46+
workspace_url = self._workspace_handler.handle(json_)
4747
if workspace_url:
4848
json_['workspaceUrl'] = workspace_url
4949

‎paperspace/commands/jobs.py‎

Lines changed: 24 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from paperspace.commands import common
88
from paperspace.exceptions import BadResponseError
99
from paperspace.utils import get_terminal_lines
10-
from paperspace.workspace import S3WorkspaceHandler
10+
from paperspace.workspace import S3WorkspaceHandler, WorkspaceHandler
1111

1212

1313
class JobsCommandBase(common.CommandBase):
@@ -126,22 +126,37 @@ def _make_table(self, logs, table, table_data):
126126
class CreateJobCommand(JobsCommandBase):
127127
def __init__(self, workspace_handler=None, **kwargs):
128128
super(CreateJobCommand, self).__init__(**kwargs)
129-
experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=kwargs.get('api_key'))
130-
self._workspace_handler = workspace_handler or S3WorkspaceHandler(experiments_api=experiments_api,
131-
logger=self.logger)
129+
# experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=kwargs.get('api_key'))
130+
self._workspace_handler = workspace_handler or WorkspaceHandler(logger=self.logger)
132131

133132
def execute(self, json_):
134133
url = "/jobs/createJob/"
135-
136-
workspace_url = self._workspace_handler.upload_workspace(json_)
134+
files = None
135+
workspace_url = self._workspace_handler.handle(json_)
137136
if workspace_url:
138-
json_['workspaceFileName'] = workspace_url
139-
json_['projectId'] = json_.get('projectId', json_.get('projectHandle'))
140-
response = self.api.post(url, json_)
137+
if self._workspace_handler.archive_path:
138+
archive_basename = self._workspace_handler.archive_basename
139+
json_["workspaceFileName"] = archive_basename
140+
self.api.headers["Content-Type"] = "multipart/form-data"
141+
files = {"file": open(workspace_url, "rb")}
142+
else:
143+
json_["workspaceFileName"] = workspace_url
144+
145+
self.set_project(json_)
146+
147+
response = self.api.post(url, params=json_, files=files)
141148
self._log_message(response,
142149
"Job created",
143150
"Unknown error while creating job")
144151

152+
@staticmethod
153+
def set_project(json_):
154+
project_id = json_.get("projectId", json_.get("projectHandle"))
155+
if not project_id:
156+
json_["project"] = "paperspace-python"
157+
else:
158+
json_["projectId"] = project_id
159+
145160

146161
class ArtifactsDestroyCommand(JobsCommandBase):
147162
def execute(self, job_id, files=None):

‎paperspace/workspace.py‎

Lines changed: 53 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,34 @@
1313
PresignedUrlMalformedResponseError, PresignedUrlError
1414

1515

16-
class S3WorkspaceHandler:
17-
def __init__(self, experiments_api, logger=None):
16+
class MultipartEncoder(object):
17+
def __init__(self, fields):
18+
s3_encoder = encoder.MultipartEncoder(fields=fields)
19+
self.monitor = encoder.MultipartEncoderMonitor(s3_encoder, callback=self._create_callback(s3_encoder))
20+
21+
def get_monitor(self):
22+
return self.monitor
23+
24+
@staticmethod
25+
def _create_callback(encoder_obj):
26+
bar = progressbar.ProgressBar(max_value=encoder_obj.len)
27+
28+
def callback(monitor):
29+
bar.update(monitor.bytes_read)
30+
31+
return callback
32+
33+
34+
class WorkspaceHandler(object):
35+
def __init__(self, logger=None):
1836
"""
1937
2038
:param experiments_api: paperspace.client.API
2139
:param logger: paperspace.logger
2240
"""
23-
self.experiments_api = experiments_api
2441
self.logger = logger or default_logger
42+
self.archive_path = None
43+
self.archive_basename = None
2544

2645
@staticmethod
2746
def _retrieve_file_paths(dir_name, ignored_files=None):
@@ -77,27 +96,49 @@ def _zip_workspace(self, workspace_path, ignore_files):
7796
self.logger.log('\nFinished creating archive: %s' % zip_file_name)
7897
return zip_file_path
7998

80-
@staticmethod
81-
def _create_callback(encoder_obj):
82-
bar = progressbar.ProgressBar(max_value=encoder_obj.len)
99+
def handle(self, input_data):
100+
workspace_archive, workspace_path, workspace_url = self._validate_input(input_data)
101+
ignore_files = input_data.get('ignore_files')
83102

84-
def callback(monitor):
85-
bar.update(monitor.bytes_read)
103+
if workspace_url:
104+
return # nothing to do
86105

87-
return callback
106+
if workspace_archive:
107+
archive_path = os.path.abspath(workspace_archive)
108+
else:
109+
archive_path = self._zip_workspace(workspace_path, ignore_files)
110+
self.archive_path = archive_path
111+
self.archive_basename = os.path.basename(archive_path)
112+
return archive_path
88113

89-
def upload_workspace(self, input_data):
114+
@staticmethod
115+
def _validate_input(input_data):
90116
workspace_url = input_data.get('workspaceUrl')
91117
workspace_path = input_data.get('workspace')
92118
workspace_archive = input_data.get('workspaceArchive')
93-
ignore_files = input_data.get('ignore_files')
94119

95120
if (workspace_archive and workspace_path) or (workspace_archive and workspace_url) or (
96121
workspace_path and workspace_url):
97122
raise click.UsageError("Use either:\n\t--workspaceUrl to point repository URL"
98123
"\n\t--workspace to point on project directory"
99124
"\n\t--workspaceArchive to point on project .zip archive"
100125
"\n or neither to use current directory")
126+
return workspace_archive, workspace_path, workspace_url
127+
128+
129+
class S3WorkspaceHandler(WorkspaceHandler):
130+
def __init__(self, experiments_api, logger=None):
131+
"""
132+
133+
:param experiments_api: paperspace.client.API
134+
:param logger: paperspace.logger
135+
"""
136+
super(S3WorkspaceHandler, self).__init__(logger=logger)
137+
self.experiments_api = experiments_api
138+
139+
def handle(self, input_data):
140+
workspace_archive, workspace_path, workspace_url = self._validate_input(input_data)
141+
ignore_files = input_data.get('ignore_files')
101142

102143
if workspace_url:
103144
return # nothing to do
@@ -133,8 +174,7 @@ def _upload(self, archive_path, s3_upload_data):
133174
fields = OrderedDict(s3_upload_data['fields'])
134175
fields.update(files)
135176

136-
s3_encoder = encoder.MultipartEncoder(fields=fields)
137-
monitor = encoder.MultipartEncoderMonitor(s3_encoder, callback=self._create_callback(s3_encoder))
177+
monitor = MultipartEncoder(fields).get_monitor()
138178
s3_response = requests.post(s3_upload_data['url'], data=monitor, headers={'Content-Type': monitor.content_type})
139179
self.logger.debug("S3 upload response: {}".format(s3_response.headers))
140180
if not s3_response.ok:

‎tests/functional/test_deployments.py‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,8 @@ def test_should_send_proper_data_and_print_message_when_create_deployment_with_b
5858
headers=EXPECTED_HEADERS,
5959
json=self.BASIC_OPTIONS_REQUEST,
6060
params=None,
61-
files=None)
61+
files=None,
62+
data=None)
6263
assert result.output == self.EXPECTED_STDOUT
6364
assert result.exit_code == 0
6465

@@ -73,7 +74,8 @@ def test_should_send_different_api_key_when_api_key_parameter_was_used(self, pos
7374
headers=self.EXPECTED_HEADERS_WITH_CHANGED_API_KEY,
7475
json=self.BASIC_OPTIONS_REQUEST,
7576
params=None,
76-
files=None)
77+
files=None,
78+
data=None)
7779
assert result.output == self.EXPECTED_STDOUT
7880
assert result.exit_code == 0
7981

@@ -89,7 +91,8 @@ def test_should_send_proper_data_and_print_message_when_create_wrong_model_id_wa
8991
headers=EXPECTED_HEADERS,
9092
json=self.BASIC_OPTIONS_REQUEST,
9193
params=None,
92-
files=None)
94+
files=None,
95+
data=None)
9396
assert result.output == self.EXPECTED_STDOUT_MODEL_NOT_FOUND
9497
assert result.exit_code == 0
9598

@@ -223,7 +226,8 @@ def test_should_send_proper_data_and_print_message_when_deployments_start_was_us
223226
headers=EXPECTED_HEADERS,
224227
json=self.REQUEST_JSON,
225228
params=None,
226-
files=None)
229+
files=None,
230+
data=None)
227231
assert result.output == self.EXPECTED_STDOUT
228232
assert result.exit_code == 0
229233

@@ -257,7 +261,8 @@ def test_should_send_proper_data_and_print_message_when_deployments_stop_was_use
257261
headers=EXPECTED_HEADERS,
258262
json=self.REQUEST_JSON,
259263
params=None,
260-
files=None)
264+
files=None,
265+
data=None)
261266
assert result.output == self.EXPECTED_STDOUT
262267
assert result.exit_code == 0
263268

@@ -272,7 +277,8 @@ def test_should_send_proper_data_with_custom_api_key_when_api_key_parameter_was_
272277
headers=self.EXPECTED_HEADERS_WITH_CHANGED_API_KEY,
273278
json=self.REQUEST_JSON,
274279
params=None,
275-
files=None)
280+
files=None,
281+
data=None)
276282
assert result.output == self.EXPECTED_STDOUT
277283
assert result.exit_code == 0
278284

@@ -287,6 +293,7 @@ def test_should_send_proper_data_and_print_message_when_deployments_stop_used_wi
287293
headers=EXPECTED_HEADERS,
288294
json=self.REQUEST_JSON,
289295
params=None,
290-
files=None)
296+
files=None,
297+
data=None)
291298
assert result.output == self.EXPECTED_STDOUT_WITH_WRONG_ID
292299
assert result.exit_code == 0

‎tests/functional/test_experiments.py‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,8 @@ def test_should_send_proper_data_and_print_message_when_create_experiment_was_ru
8585
headers=self.EXPECTED_HEADERS,
8686
json=self.BASIC_OPTIONS_REQUEST,
8787
params=None,
88-
files=None)
88+
files=None,
89+
data=None)
8990
assert result.output == self.EXPECTED_STDOUT
9091
assert result.exit_code == 0
9192

@@ -101,7 +102,8 @@ def test_should_send_proper_data_and_print_message_when_create_experiment_was_ru
101102
headers=self.EXPECTED_HEADERS_WITH_CHANGED_API_KEY,
102103
json=self.FULL_OPTIONS_REQUEST,
103104
params=None,
104-
files=None)
105+
files=None,
106+
data=None)
105107
assert result.output == self.EXPECTED_STDOUT
106108
assert result.exit_code == 0
107109
assert self.EXPECTED_HEADERS_WITH_CHANGED_API_KEY["X-API-Key"] == "some_key"
@@ -118,7 +120,8 @@ def test_should_send_proper_data_and_print_message_when_create_wrong_project_id_
118120
headers=self.EXPECTED_HEADERS,
119121
json=self.BASIC_OPTIONS_REQUEST,
120122
params=None,
121-
files=None)
123+
files=None,
124+
data=None)
122125
assert result.output == self.EXPECTED_STDOUT_PROJECT_NOT_FOUND
123126
assert result.exit_code == 0
124127

@@ -227,7 +230,8 @@ def test_should_send_proper_data_and_print_message_when_create_experiment_was_ru
227230
headers=self.EXPECTED_HEADERS,
228231
json=self.BASIC_OPTIONS_REQUEST,
229232
params=None,
230-
files=None)
233+
files=None,
234+
data=None)
231235
assert result.output == self.EXPECTED_STDOUT
232236
assert result.exit_code == 0
233237

@@ -243,7 +247,8 @@ def test_should_send_proper_data_and_print_message_when_create_experiment_was_ru
243247
headers=self.EXPECTED_HEADERS_WITH_CHANGED_API_KEY,
244248
json=self.FULL_OPTIONS_REQUEST,
245249
params=None,
246-
files=None)
250+
files=None,
251+
data=None)
247252
assert result.output == self.EXPECTED_STDOUT
248253
assert result.exit_code == 0
249254

0 commit comments

Comments
 (0)