Skip to content
This repository was archived by the owner on Aug 11, 2020. It is now read-only.

Commit 1a3b48e

Browse files
sanfilipsanfilip
committed
create package; reorg jobs methods; separate out config obj
1 parent 5e5e09f commit 1a3b48e

File tree

6 files changed

+128
-73
lines changed

6 files changed

+128
-73
lines changed

‎hello.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
import paperspace
2+
3+
paperspace.config.PAPERSPACE_API_KEY = '14a4bc1cbc414...'
4+
5+
paperspace.jobs.runas_job()
6+
7+
print('hello paperspace-python!')

‎paperspace/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
from .config import *
2+
from . import jobs

‎paperspace/config.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
import os
2+
3+
class config: pass
4+
5+
config.PAPERSPACE_API_KEY = ''
6+
config.CONFIG_HOST = 'https://api.paperspace.io'
7+
config.CONFIG_LOG_HOST = 'https://logs.paperspace.io'
8+
9+
if 'PAPERSPACE_API_KEY' in os.environ:
10+
config.PAPERSPACE_API_KEY = os.environ['PAPERSPACE_API_KEY']

paperspace.py renamed to paperspace/jobs.py

Lines changed: 66 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,11 @@
1010
import botocore
1111
import requests
1212

13-
PAPERSPACE_API_KEY = ''
14-
CONFIG_HOST = 'https://api.paperspace.io'
15-
CONFIG_LOG_HOST = 'https://logs.paperspace.io'
16-
17-
if 'PAPERSPACE_API_KEY' in os.environ:
18-
PAPERSPACE_API_KEY = os.environ['PAPERSPACE_API_KEY']
13+
from . import config
1914

15+
print('ps PAPERSPACE_API_KEY: ' + config.PAPERSPACE_API_KEY)
16+
print('ps CONFIG_HOST: ' + config.CONFIG_HOST)
17+
print('ps CONFIG_LOG_HOST: ' + config.CONFIG_LOG_HOST)
2018

2119
def zip_to_tmp(obj_name):
2220
zipname = os.path.join(tempfile.gettempdir(),
@@ -43,7 +41,7 @@ def print_json_pretty(res):
4341
print(json.dumps(res, indent=2, sort_keys=True))
4442

4543

46-
def paperspace(category, method, params):
44+
def method(category, method, params):
4745

4846
if method in ['artifactsGet', 'artifactsList', 'getJob', 'getJobs',
4947
'getLogs']:
@@ -78,14 +76,42 @@ def paperspace(category, method, params):
7876
params['workspaceFileName'] = os.path.basename(workspace_file)
7977
del params['workspace']
8078

81-
r = requests.request(http_method, CONFIG_HOST + path,
82-
headers={'x-api-key': PAPERSPACE_API_KEY},
79+
r = requests.request(http_method, config.CONFIG_HOST + path,
80+
headers={'x-api-key': config.PAPERSPACE_API_KEY},
8381
params=params, files=files)
8482

8583
return r.json()
8684

8785

88-
def jobs_logs(params, tail=False, json=False):
86+
def getJobs(params):
87+
return method('jobs', 'getJobs', params)
88+
89+
90+
def artifactsList(params):
91+
return method('jobs', 'artifactsList', params)
92+
93+
94+
def artifactsDestroy(params):
95+
return method('jobs', 'artifactsDestroy', params)
96+
97+
98+
def getJob(params):
99+
return method('jobs', 'getJob', params)
100+
101+
102+
def clone(params):
103+
return method('jobs', 'clone', params)
104+
105+
106+
def stop(params):
107+
return method('jobs', 'stop', params)
108+
109+
110+
def destroy(params):
111+
return method('jobs', 'destroy', params)
112+
113+
114+
def logs(params, tail=False, json=False):
89115
last_line = 0
90116
PSEOF = False
91117
json_res = []
@@ -96,8 +122,8 @@ def jobs_logs(params, tail=False, json=False):
96122
params['line'] = 0
97123

98124
while True:
99-
r = requests.request('GET', CONFIG_LOG_HOST + '/jobs/logs',
100-
headers={'x-api-key': PAPERSPACE_API_KEY},
125+
r = requests.request('GET', config.CONFIG_LOG_HOST + '/jobs/logs',
126+
headers={'x-api-key': config.PAPERSPACE_API_KEY},
101127
params=params)
102128
try:
103129
res = r.json()
@@ -138,9 +164,9 @@ def jobs_logs(params, tail=False, json=False):
138164
return True
139165

140166

141-
def jobs_waitfor(params):
167+
def waitfor(params):
142168
while True:
143-
job = paperspace('jobs', 'getJob', params)
169+
job = method('jobs', 'getJob', params)
144170
if 'state' not in job:
145171
return job
146172
state = job['state']
@@ -154,8 +180,8 @@ def jobs_waitfor(params):
154180
time.sleep(5)
155181

156182

157-
def jobs_create(params):
158-
job = paperspace('jobs', 'createJob', params)
183+
def create(params):
184+
job = method('jobs', 'createJob', params)
159185
if 'id' not in job:
160186
print_json_pretty(job)
161187
return job
@@ -165,15 +191,15 @@ def jobs_create(params):
165191

166192
if job['state'] == 'Pending':
167193
print('Waiting for job to run...')
168-
job = jobs_waitfor({'jobId': jobId, 'state': 'Running'})
194+
job = waitfor({'jobId': jobId, 'state': 'Running'})
169195
if 'state' not in job:
170196
print_json_pretty(job)
171197
return job
172198

173199
if job['state'] != 'Error':
174200
print('Awaiting logs...')
175-
jobs_logs({'jobId': jobId}, tail=True)
176-
job = paperspace('jobs', 'getJob', {'jobId': jobId})
201+
logs({'jobId': jobId}, tail=True)
202+
job = method('jobs', 'getJob', {'jobId': jobId})
177203
if 'state' not in job:
178204
print_json_pretty(job)
179205
return job
@@ -185,7 +211,7 @@ def jobs_create(params):
185211
return job
186212

187213

188-
def jobs_artifactsGet(params):
214+
def artifactsGet(params):
189215
if 'dest' in params:
190216
dest = os.path.abspath(os.path.expanduser(params['dest']))
191217
if not os.path.exists(dest):
@@ -198,10 +224,10 @@ def jobs_artifactsGet(params):
198224
else:
199225
dest = os.getcwd()
200226

201-
artifacts_list = paperspace('jobs', 'artifactsList', params)
227+
artifacts_list = method('jobs', 'artifactsList', params)
202228
if artifacts_list:
203-
204-
creds = paperspace('jobs', 'artifactsGet', params)
229+
230+
creds = method('jobs', 'artifactsGet', params)
205231
if creds:
206232
bucket = creds['bucket']
207233
folder = creds['folder']
@@ -240,7 +266,6 @@ def jobs_artifactsGet(params):
240266

241267

242268
# TO DO:
243-
# create/use project config
244269
# deal with timeouts/server unreachable
245270
# deal with returned errors
246271
# deal with invalid directories, e.g. root for workspace
@@ -249,6 +274,11 @@ def jobs_artifactsGet(params):
249274

250275

251276
def runas_job(params={}):
277+
278+
print('runas_job PAPERSPACE_API_KEY: ' + config.PAPERSPACE_API_KEY)
279+
print('runas_job CONFIG_HOST: ' + config.CONFIG_HOST)
280+
print('runas_job CONFIG_LOG_HOST: ' + config.CONFIG_LOG_HOST)
281+
252282
if 'PAPERSPACE_JOB_RUNNER' in os.environ:
253283
return
254284

@@ -260,11 +290,11 @@ def runas_job(params={}):
260290
# TO DO: remove these replacements once we are auto importing paperspace on the job runner
261291
# and have defined the PAPERSPACE_JOB_RUNNER env var and passed it into the container
262292
src = src.replace('import paperspace', '# import paperspace')
263-
src = src.replace('paperspace.PAPERSPACE_API_KEY', 'pass # paperspace.PAPERSPACE_API_KEY')
264-
src = src.replace('paperspace.CONFIG_HOST', 'pass # paperspace.CONFIG_HOST')
265-
src = src.replace('paperspace.CONFIG_LOG_HOST', 'pass # paperspace.CONFIG_LOG_HOST')
266-
src = src.replace('paperspace.runas_job', 'paperspace_null_runas_job')
267-
src = "\ndef paperspace_null_runas_job(*args, **kwargs):\n return None\n" + src
293+
src = src.replace('paperspace.config.PAPERSPACE_API_KEY', '_paperspace_config_PAPERSPACE_API_KEY')
294+
src = src.replace('paperspace.config.CONFIG_HOST', '_paperspace_config_CONFIG_HOST')
295+
src = src.replace('paperspace.config.CONFIG_LOG_HOST', '_paperspace_config_CONFIG_LOG_HOST')
296+
src = src.replace('paperspace.jobs.runas_job', '_paperspace_jobs_null_runas_job')
297+
src = "def _paperspace_jobs_null_runas_job(*args, **kwargs): return None\n" + src
268298

269299
src_path = os.path.join(tempfile.gettempdir(), src_file)
270300
with open(src_path, "w") as file:
@@ -279,11 +309,17 @@ def runas_job(params={}):
279309
params['command'] = 'python3 ' + src_file
280310
params['workspace'] = src_path
281311

282-
jobs_create(params)
312+
create(params)
283313
sys.exit(0)
284314

285315

286316
# TO DO:
287317
# automatic install of imported dependencies
288318
# make console logging optional
289319
# allow return results
320+
# prevent interactive use
321+
# combine local workspace with source
322+
# detect/use python environment
323+
# set PAPERSPACE_JOB_RUNNER within job runner
324+
# allow specification of apiKey within runas_job call
325+
# allow artibitrary command with args

‎remote_test.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
import os
22
import paperspace
33

4-
paperspace.PAPERSPACE_API_KEY = '14a4bc1cbc414...'
4+
paperspace.config.PAPERSPACE_API_KEY = '14a4bc1cbc414...'
55

6-
paperspace.runas_job({'project': 'myprojec8', 'machineType': 'GPU+', 'container': 'Test-Container'})
6+
paperspace.jobs.runas_job({'project': 'myprojec', 'machineType': 'GPU+', 'container': 'Test-Container'})
77

88
print(os.getcwd())
99
print('something useful')

‎test.py‎

Lines changed: 41 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -2,74 +2,74 @@
22

33
# Tests:
44

5-
paperspace.PAPERSPACE_API_KEY = '14a4bc1cbc414...'
5+
paperspace.config.PAPERSPACE_API_KEY = '14a4bc1cbc414...'
66

7-
project = 'myproject7'
7+
project = 'myproject'
88
print('project: %s' % project)
99

10-
print("paperspace.paperspace('jobs', 'getJobs', {'project': project})")
11-
jobs = paperspace.paperspace('jobs', 'getJobs', {'project': project})
10+
print("paperspace.jobs.getJobs({'project': project})")
11+
jobs = paperspace.jobs.getJobs({'project': project})
1212
if 'error' in jobs:
13-
paperspace.print_json_pretty(jobs)
13+
paperspace.jobs.print_json_pretty(jobs)
1414
else:
1515
for job in jobs:
1616
print(job['id'])
1717

18-
print("jobs_create({'project': project, 'machineType': 'GPU+', 'container': 'Test-Container', 'command': './do.sh', 'workspace': '~/myproject3'})")
19-
job = paperspace.jobs_create({'project': project,
18+
print("jobs.create({'project': project, 'machineType': 'GPU+', 'container': 'Test-Container', 'command': './do.sh', 'workspace': '~/myproject3'})")
19+
job = paperspace.jobs.create({'project': project,
2020
'machineType': 'GPU+', 'container': 'Test-Container',
2121
'command': './do.sh', 'workspace': '~/myproject3'})
2222
jobId = job['id']
2323

24-
print("paperspace.paperspace('jobs', 'artifactsList', {'jobId': jobId, 'links': True})")
25-
artifacts = paperspace.paperspace('jobs', 'artifactsList', {'jobId': jobId, 'links': True})
24+
print("paperspace.jobs.artifactsList({'jobId': jobId, 'links': True})")
25+
artifacts = paperspace.jobs.artifactsList({'jobId': jobId, 'links': True})
2626
if artifacts:
27-
paperspace.print_json_pretty(artifacts)
27+
paperspace.jobs.print_json_pretty(artifacts)
2828

29-
print("paperspace.jobs_artifactsGet({'jobId': jobId, 'dest': '~/temp1'})")
30-
paperspace.jobs_artifactsGet({'jobId': jobId, 'dest': '~/temp1'})
29+
print("paperspace.jobs.artifactsGet({'jobId': jobId, 'dest': '~/temp1'})")
30+
paperspace.jobs.artifactsGet({'jobId': jobId, 'dest': '~/temp1'})
3131

32-
print("paperspace.paperspace('jobs', 'getJob', {'jobId': jobId})")
33-
job = paperspace.paperspace('jobs', 'getJob', {'jobId': jobId})
34-
paperspace.print_json_pretty(job)
32+
print("paperspace.jobs.getJob({'jobId': jobId})")
33+
job = paperspace.jobs.getJob({'jobId': jobId})
34+
paperspace.jobs.print_json_pretty(job)
3535

36-
print("paperspace.jobs_logs({'jobId': jobId, 'limit': 4}, tail=True)")
37-
paperspace.jobs_logs({'jobId': jobId, 'limit': 4}, tail=True)
36+
print("paperspace.jobs.logs({'jobId': jobId, 'limit': 4}, tail=True)")
37+
paperspace.jobs.logs({'jobId': jobId, 'limit': 4}, tail=True)
3838

39-
print("paperspace.paperspace('jobs', 'stop', {'jobId': jobId})")
40-
res = paperspace.paperspace('jobs', 'stop', {'jobId': jobId})
41-
paperspace.print_json_pretty(res)
39+
print("paperspace.jobs.stop({'jobId': jobId})")
40+
res = paperspace.jobs.stop({'jobId': jobId})
41+
paperspace.jobs.print_json_pretty(res)
4242

43-
print("paperspace.paperspace('jobs', 'clone', {'jobId': jobId})")
44-
clonedJob = paperspace.paperspace('jobs', 'clone', {'jobId': jobId})
45-
paperspace.print_json_pretty(clonedJob)
43+
print("paperspace.jobs.clone({'jobId': jobId})")
44+
clonedJob = paperspace.jobs.clone({'jobId': jobId})
45+
paperspace.jobs.print_json_pretty(clonedJob)
4646

47-
print("paperspace.jobs_waitfor({'jobId': clonedJob['id'], 'state': 'Stopped'})")
48-
waitforJob = paperspace.jobs_waitfor({'jobId': clonedJob['id'], 'state': 'Stopped'})
49-
paperspace.print_json_pretty(waitforJob)
47+
print("paperspace.jobs.waitfor({'jobId': clonedJob['id'], 'state': 'Stopped'})")
48+
waitforJob = paperspace.jobs.waitfor({'jobId': clonedJob['id'], 'state': 'Stopped'})
49+
paperspace.jobs.print_json_pretty(waitforJob)
5050

51-
print("paperspace.paperspace('jobs', 'artifactsList', {'jobId': clonedJob['id']})")
52-
artifacts = paperspace.paperspace('jobs', 'artifactsList', {'jobId': clonedJob['id']})
51+
print("paperspace.jobs.artifactsList({'jobId': clonedJob['id']})")
52+
artifacts = paperspace.jobs.artifactsList({'jobId': clonedJob['id']})
5353
if artifacts:
54-
paperspace.print_json_pretty(artifacts)
55-
print("paperspace.paperspace('jobs', 'artifactsDestroy', {'jobId': clonedJob['id']})")
56-
paperspace.paperspace('jobs', 'artifactsDestroy', {'jobId': clonedJob['id']})
54+
paperspace.jobs.print_json_pretty(artifacts)
55+
print("paperspace.jobs.artifactsDestroy({'jobId': clonedJob['id']})")
56+
paperspace.jobs.artifactsDestroy({'jobId': clonedJob['id']})
5757

58-
print("paperspace.paperspace('jobs', 'artifactsList', {'jobId': clonedJob['id']})")
59-
artifacts = paperspace.paperspace('jobs', 'artifactsList', {'jobId': clonedJob['id']})
58+
print("paperspace.jobs.artifactsList({'jobId': clonedJob['id']})")
59+
artifacts = paperspace.jobs.artifactsList({'jobId': clonedJob['id']})
6060
if artifacts:
61-
paperspace.print_json_pretty(artifacts)
61+
paperspace.jobs.print_json_pretty(artifacts)
6262

63-
print("paperspace.paperspace('jobs', 'getJobs', {'project': project})")
64-
jobs = paperspace.paperspace('jobs', 'getJobs', {'project': project})
63+
print("paperspace.jobs.getJobs({'project': project})")
64+
jobs = paperspace.jobs.getJobs({'project': project})
6565
for job in jobs:
6666
print(job['id'])
6767

68-
print("paperspace.paperspace('jobs', 'destroy', {'jobId': clonedJob['id']})")
69-
res = paperspace.paperspace('jobs', 'destroy', {'jobId': clonedJob['id']})
70-
paperspace.print_json_pretty(res)
68+
print("paperspace.jobs.destroy({'jobId': clonedJob['id']})")
69+
res = paperspace.jobs.destroy({'jobId': clonedJob['id']})
70+
paperspace.jobs.print_json_pretty(res)
7171

72-
print("paperspace.paperspace('jobs', 'getJobs', {'project': project})")
73-
jobs = paperspace.paperspace('jobs', 'getJobs', {'project': project})
72+
print("paperspace.jobs.getJobs({'project': project})")
73+
jobs = paperspace.jobs.getJobs({'project': project})
7474
for job in jobs:
7575
print(job['id'])

0 commit comments

Comments
 (0)