feat: Logging for experiments.

Author: colin-welch

paperspace/cli/experiments.py

@@ -308,3 +308,34 @@ def list_experiments(project_ids, api_key):
 def get_experiment_details(experiment_id, api_key):
     experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
     experiments_commands.get_experiment_details(experiment_id, api=experiments_api)
+
+
+@experiments.command("logs", help="List experiment logs")
+@click.option(
+    "--experimentId",
+    "experiment_id",
+    required=True
+)
+@click.option(
+    "--line",
+    "line",
+    required=False,
+    default=0
+)
+@click.option(
+    "--limit",
+    "limit",
+    required=False,
+    default=10000
+)
+@click.option(
+    "--follow",
+    "follow",
+    required=False,
+    default=False
+)
+@api_key_option
+def list_logs(experiment_id, line, limit, follow, api_key=None):
+    logs_api = client.API(config.CONFIG_LOG_HOST, api_key=api_key)
+    command = experiments_commands.ExperimentLogsCommand(api=logs_api)
+    command.execute(experiment_id, line, limit, follow)

paperspace/cli/jobs.py

@@ -94,17 +94,35 @@ def create_job(api_key, **kwargs):
     command.execute(kwargs)
 
 
-@jobs_group.command("log", help="List job logs")
+@jobs_group.command("logs", help="List job logs")
 @click.option(
     "--jobId",
     "job_id",
     required=True
 )
+@click.option(
+    "--line",
+    "line",
+    required=False,
+    default=0
+)
+@click.option(
+    "--limit",
+    "limit",
+    required=False,
+    default=10000
+)
+@click.option(
+    "--follow",
+    "follow",
+    required=False,
+    default=False
+)
 @api_key_option
-def list_logs(job_id, api_key=None):
+def list_logs(job_id, line, limit, follow, api_key=None):
     logs_api = client.API(config.CONFIG_LOG_HOST, api_key=api_key)
     command = jobs_commands.JobLogsCommand(api=logs_api)
-    command.execute(job_id)
+    command.execute(job_id, line, limit, follow)
 
 
 @jobs_group.group("artifacts", help="Manage jobs' artifacts", cls=ClickGroup)

paperspace/commands/experiments.py

@@ -1,9 +1,13 @@
+import pydoc
+
 import terminaltables
+from click import style
 
 from paperspace import logger, constants, client, config
 from paperspace.logger import log_response
-from paperspace.workspace import S3WorkspaceHandler
 from paperspace.commands import common
+from paperspace.utils import get_terminal_lines
+from paperspace.workspace import S3WorkspaceHandler
 
 experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, headers=client.default_headers)
 
@@ -168,3 +172,71 @@ def get_experiment_details(experiment_id, api=experiments_api):
             logger.error("Error parsing response data")
 
     log_response(response, details, "Unknown error while retrieving details of the experiment")
+
+
+class ExperimentLogsCommand(common.CommandBase):
+    last_line_number = 0
+    base_url = "/jobs/logs?experimentId={}&line={}&limit={}"
+    other_url = "/jobs/logs"
+
+    is_logs_complete = False
+
+    def execute(self, experiment_id, line, limit, follow):
+        self.last_line_number = line
+        table_title = "Experiment %s logs" % experiment_id
+        table_data = [("LINE", "MESSAGE")]
+        table = terminaltables.AsciiTable(table_data, title=table_title)
+
+        while not self.is_logs_complete:
+            print ("follow " + str(follow))
+            response = self._get_logs(experiment_id, self.last_line_number, limit)
+
+            print ("got data")
+            try:
+                data = response.json()
+                if not response.ok:
+                    self.logger.log_error_response(data)
+                    return
+            except (ValueError, KeyError) as e:
+                if response.status_code == 204:
+                    continue
+                self.logger.log("Error while parsing response data: {}".format(e))
+                return
+            else:
+                print ("parsed data")
+                self._log_logs_list(data, table, table_data)
+                print ("logged data")
+
+            print ("follow " + follow)
+            if not follow:
+                self.is_logs_complete = True
+
+    def _get_logs(self, experiment_id, line, limit):
+        url = self.base_url.format(experiment_id, line, limit)
+        params = {
+            'experimentId': experiment_id,
+            'line': line,
+            'limit': limit
+        };
+        return self.api.get(self.other_url, params=params)
+
+    def _log_logs_list(self, data, table, table_data):
+        if not data:
+            self.logger.log("No Logs found")
+        else:
+            table_str = self._make_table(data, table, table_data)
+            if len(table_str.splitlines()) > get_terminal_lines():
+                pydoc.pager(table_str)
+            else:
+                self.logger.log(table_str)
+
+    def _make_table(self, logs, table, table_data):
+        if logs[-1].get("message") == "PSEOF":
+            self.is_logs_complete = True
+        else:
+            self.last_line_number = logs[-1].get("line")
+
+        for log in logs:
+            table_data.append((style(fg="blue", text=str(log.get("jobId"))), style(fg="red", text=str(log.get("line"))), log.get("message")))
+
+        return table.table

paperspace/commands/jobs.py

@@ -72,17 +72,18 @@ def _get_table_data(self, jobs):
 
 class JobLogsCommand(common.CommandBase):
     last_line_number = 0
-    base_url = "/jobs/logs?jobId={}&line={}"
+    base_url = "/jobs/logs?jobId={}&line={}&limit={}"
 
     is_logs_complete = False
 
-    def execute(self, job_id):
+    def execute(self, job_id, line, limit, follow):
+        self.last_line_number = line
         table_title = "Job %s logs" % job_id
         table_data = [("LINE", "MESSAGE")]
         table = terminaltables.AsciiTable(table_data, title=table_title)
 
         while not self.is_logs_complete:
-            response = self._get_logs(job_id)
+            response = self._get_logs(job_id, self.last_line_number, limit)
 
             try:
                 data = response.json()
@@ -97,8 +98,11 @@ def execute(self, job_id):
             else:
                 self._log_logs_list(data, table, table_data)
 
-    def _get_logs(self, job_id):
-        url = self.base_url.format(job_id, self.last_line_number)
+            if not follow:
+                self.is_logs_complete = True
+
+    def _get_logs(self, job_id, line, limit):
+        url = self.base_url.format(job_id, line, limit)
         return self.api.get(url)
 
     def _log_logs_list(self, data, table, table_data):