tweak: Clean up experiment logs and propagate changes to job logs.

Author: colin-welch

paperspace/cli/jobs.py

@@ -87,11 +87,14 @@ def list_jobs(api_key, **filters):
 @click.option("--registryUsername", "registryUsername", help="Docker registry username")
 @click.option("--registryPassword", "registryPassword", help="Docker registry password")
 @api_key_option
-def create_job(api_key, **kwargs):
+@click.pass_context
+def create_job(ctx, api_key, **kwargs):
     del_if_value_is_none(kwargs)
     jobs_api = client.API(config.CONFIG_HOST, api_key=api_key)
     command = jobs_commands.CreateJobCommand(api=jobs_api)
-    command.execute(kwargs)
+    job = command.execute(kwargs)
+    if job is not None:
+        ctx.invoke(list_logs, job_id=job["handle"], line=0, limit=100, follow=True, api_key=api_key)
 
 
 @jobs_group.command("logs", help="List job logs")

paperspace/commands/experiments.py

@@ -179,8 +179,7 @@ def get_experiment_details(experiment_id, api=experiments_api):
 
 class ExperimentLogsCommand(common.CommandBase):
     last_line_number = 0
-    base_url = "/jobs/logs?experimentId={}&line={}&limit={}"
-    other_url = "/jobs/logs"
+    base_url = "/jobs/logs"
 
     is_logs_complete = False
 
@@ -213,13 +212,12 @@ def execute(self, experiment_id, line, limit, follow):
                 self.is_logs_complete = True
 
     def _get_logs(self, experiment_id, line, limit):
-        url = self.base_url.format(experiment_id, line, limit)
         params = {
             'experimentId': experiment_id,
             'line': line,
             'limit': limit
         };
-        return self.api.get(self.other_url, params=params)
+        return self.api.get(self.base_url, params=params)
 
     def _log_logs_list(self, data, table, table_data, follow):
         if not data:

paperspace/commands/jobs.py

@@ -72,11 +72,14 @@ def _get_table_data(self, jobs):
 
 class JobLogsCommand(common.CommandBase):
     last_line_number = 0
-    base_url = "/jobs/logs?jobId={}&line={}&limit={}"
+    base_url = "/jobs/logs"
 
     is_logs_complete = False
 
     def execute(self, job_id, line, limit, follow):
+        if follow:
+            self.logger.log("Awaiting logs...")
+
         self.last_line_number = line
         table_title = "Job %s logs" % job_id
         table_data = [("LINE", "MESSAGE")]
@@ -96,24 +99,37 @@ def execute(self, job_id, line, limit, follow):
                 self.logger.log("Error while parsing response data: {}".format(e))
                 return
             else:
-                self._log_logs_list(data, table, table_data)
+                self._log_logs_list(data, table, table_data, follow)
 
             if not follow:
                 self.is_logs_complete = True
 
     def _get_logs(self, job_id, line, limit):
-        url = self.base_url.format(job_id, line, limit)
-        return self.api.get(url)
-
-    def _log_logs_list(self, data, table, table_data):
+        params = {
+            'jobId': job_id,
+            'line': line,
+            'limit': limit
+        };
+        return self.api.get(self.base_url, params=params)
+
+    def _log_logs_list(self, data, table, table_data, follow):
         if not data:
             self.logger.log("No Logs found")
         else:
-            table_str = self._make_table(data, table, table_data)
-            if len(table_str.splitlines()) > get_terminal_lines():
-                pydoc.pager(table_str)
+            if follow:
+                if data[-1].get("message") == "PSEOF":
+                    self.is_logs_complete = True
+                else:
+                    self.last_line_number = data[-1].get("line")
+                for log in data:
+                    log_str = "{}\t{}\t{}"
+                    self.logger.log(log_str.format(style(fg="blue", text=str(log.get("jobId"))), style(fg="red", text=str(log.get("line"))), log.get("message")))
             else:
-                self.logger.log(table_str)
+                table_str = self._make_table(data, table, table_data)
+                if len(table_str.splitlines()) > get_terminal_lines():
+                    pydoc.pager(table_str)
+                else:
+                    self.logger.log(table_str)
 
     def _make_table(self, logs, table, table_data):
         if logs[-1].get("message") == "PSEOF":