Merge pull request #7 from AATools/0.3

Update to v.0.3

Author: nicko858

README.md

@@ -7,7 +7,7 @@ The collected metrics can be explored in Prometheus or Grafana.
 
 The metrics are collected using [mqsilist](https://www.ibm.com/support/knowledgecenter/en/SSMKHH_9.0.0/com.ibm.etools.mft.doc/an07250_.htm) command. So, you need to install `IBM Integration Bus`.
 
-Tested for IBM IB v9 and Python 2.6, 2.7, 3.6 on Linux.
+Tested for IBM IB v9 and v10 and Python 2.7, 3.6 on Linux.
 
 ## Collected metrics
 
@@ -48,6 +48,31 @@ nohup python3 iib_metrics_client.py &
 
 After that, you should set up your Prometheus server to collect metrics from Pushgateway (`http://<hostname>:9091/metrics`).
 
+You can specify `host` and `port` for pushgateway and Integration Bus version via command-line arguments.
+
+```bash
+python3 iib_metrics_client.py -h
+
+usage: iib_metrics_client.py [-h] [--pghost [pushgatewayHost]] [--pgport [pushgatewayPort]] [--iibver [iibVersion]]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --pghost [pushgatewayHost]
+                        pushgateway host
+  --pgport [pushgatewayPort]
+                        pushgateway port
+  --iibver [iibVersion]
+                        IIB version: 9 or 10
+```
+
+If argument is not set the default value is used.
+
+| Command-line argument | Description | Default value |
+|:---|:---|:---|
+| `pghost` | Pushgateway host | Hostname on which client is started.<br> Value define via `platform.node()`. |
+| `pgport` | Pushgateway port | `9091` |
+| `iibver` | IIB version | `9`<br> Valid value: **9** or **10**.<br> If argument is omitted or invalid value is passed, the client will try to determine version via environment variable `MQSI_VERSION_V`. If it can't determine the version using the environment variable, the default value will be used. |
+
 ## Grafana dashboard
 
 The Grafana dashboard visualizes collected metrics. This is an example of a dashboard. You can create your own dashboards to analyze metrics.

iib_metrics_client.py

@@ -1,10 +1,12 @@
 # -*- coding: utf-8 -*-
 """Python client for collecting IBM Integration Bus metrics and exporting to Prometheus pushgateway."""
 import sys
+import os
 import time
 import traceback
 import platform
 import requests
+import argparse
 from requests import ConnectionError
 from urllib3.exceptions import ResponseError
 from modules.iib_brokers import (
@@ -15,7 +17,9 @@
 from modules.iib_applications import format_applications
 from modules.iib_message_flows import format_message_flows
 from log.logger_client import set_logger
-from modules.iib_api import run_iib_command
+from modules.iib_api import (
+    run_iib_command,
+    get_platform_params_for_commands)
 
 
 logger = set_logger()
@@ -28,15 +32,39 @@ class PrometheusBadResponse(Exception):
 def static_content():
     """Client name and version."""
     name = "ib-metrics-pyclient"
-    version = "0.2"
+    version = "0.3"
     return '{0} v.{1}'.format(name, version)
 
 
-def put_metric_to_gateway(metric_data, job):
+def get_version_from_env():
+    """Get Integration Bus version from env variable."""
+    iib_version = os.getenv("MQSI_VERSION_V")
+    if iib_version:
+        logger.info("Integration Bus version is defined via MQSI_VERSION_V environment variable.")
+    else:
+        logger.info("Integration Bus default version is used.")
+        iib_version = "9"
+    return iib_version
+
+
+def parse_commandline_args():
+    """Parse command-line arguments."""
+    parser = argparse.ArgumentParser(prog='iib_metrics_client.py')
+    parser.add_argument('--pghost', metavar='pushgatewayHost', nargs='?', default=platform.node(), dest='pushgateway_host', help='pushgateway host')
+    parser.add_argument('--pgport', metavar='pushgatewayPort', nargs='?', default='9091', dest='pushgateway_port', help='pushgateway port')
+    parser.add_argument('--iibver', metavar='iibVersion', nargs='?', default=None, dest='iib_cmd_ver', help='IIB version: 9 or 10')
+    args = parser.parse_args()
+    if (args.iib_cmd_ver is None) or ((args.iib_cmd_ver != '9') and (args.iib_cmd_ver != '10')):
+        logger.info("Trying to determine Integration Bus version from environment variable MQSI_VERSION_V.")
+        iib_cmd_ver = get_version_from_env()
+    else:
+        iib_cmd_ver = args.iib_cmd_ver
+    return args.pushgateway_host, args.pushgateway_port, iib_cmd_ver
+
+
+def put_metric_to_gateway(metric_data, job, pushgateway_host, pushgateway_port):
     """Sends data to Prometheus pushgateway."""
-    hostname = platform.node()
-    port = 9091
-    src_url = "http://{0}:{1}".format(hostname, port)
+    src_url = "http://{0}:{1}".format(pushgateway_host, pushgateway_port)
     headers = {"Content-Type": "text/plain; version=0.0.4"}
     dest_url = "{0}/metrics/job/{1}".format(src_url, job)
     logger.info("Destination url: {0}".format(dest_url))
@@ -51,35 +79,53 @@ def put_metric_to_gateway(metric_data, job):
         raise PrometheusBadResponse("{0} is not available!".format(dest_url))
 
 
-def main():
+def get_iib_metrics(pushgateway_host, pushgateway_port, mqsilist_command, bip_codes_brokers, bip_codes_components):
     start_time = time.time()
     logger.info("Starting metrics collecting for Integration Bus!")
     try:
-        brokers_data = run_iib_command(task='get_brokers_status')
-        brokers = get_brokers_status(brokers_data=brokers_data)
+        brokers_data = run_iib_command(task=mqsilist_command)
+        brokers = get_brokers_status(brokers_data=brokers_data, bip_codes=bip_codes_brokers)
         for broker in brokers:
             broker_name, status, qm_name = broker
             broker_data = format_broker(
                 broker_name=broker_name,
                 status=status,
                 qm_name=qm_name)
-            if status == 'running.':
+            if status == 'running':
                 broker_row_data = run_iib_command(
                     task='get_broker_objects',
                     broker_name=broker_name)
-                exec_groups, applications, message_flows = get_broker_items(broker_row_data=broker_row_data)
-                exec_groups_data = format_exec_groups(exec_groups=exec_groups)
-                applications_data = format_applications(applications=applications, broker_name=broker_name)
-                message_flows_data = format_message_flows(message_flows=message_flows, broker_name=broker_name)
+                exec_groups, applications, message_flows = get_broker_items(
+                    broker_row_data=broker_row_data,
+                    bip_codes=bip_codes_components)
+                exec_groups_data = format_exec_groups(
+                    exec_groups=exec_groups,
+                    bip_codes=bip_codes_components)
+                applications_data = format_applications(
+                    applications=applications,
+                    broker_name=broker_name,
+                    bip_codes=bip_codes_components)
+                message_flows_data = format_message_flows(
+                    message_flows=message_flows,
+                    broker_name=broker_name,
+                    bip_codes=bip_codes_components)
                 metric_data = "{0}{1}{2}{3}".format(
                     broker_data,
                     exec_groups_data,
                     applications_data,
                     message_flows_data)
-                put_metric_to_gateway(metric_data=metric_data, job=broker_name)
+                put_metric_to_gateway(
+                    metric_data=metric_data,
+                    job=broker_name,
+                    pushgateway_host=pushgateway_host,
+                    pushgateway_port=pushgateway_port)
                 logger.info("All metrics pushed successfully!")
             else:
-                put_metric_to_gateway(metric_data=broker_data, job=broker_name)
+                put_metric_to_gateway(
+                    metric_data=broker_data,
+                    job=broker_name,
+                    pushgateway_host=pushgateway_host,
+                    pushgateway_port=pushgateway_port)
                 logger.warning("The status of broker is {0}\nOther metrics will not be collected!".format(status))
         logger.info("Script finished in - {0} seconds -".format(time.time() - start_time))
     except PrometheusBadResponse as error:
@@ -92,6 +138,14 @@ def main():
 
 if __name__ == "__main__":
     logger.info("Run {0}".format(static_content()))
+    pushgateway_host, pushgateway_port, iib_ver = parse_commandline_args()
+    logger.info("Integration Bus version: {0}".format(iib_ver))
+    mqsilist_command, bip_codes_brokers, bip_codes_components = get_platform_params_for_commands(iib_ver=iib_ver)
     while True:
-        main()
+        get_iib_metrics(
+            pushgateway_host=pushgateway_host,
+            pushgateway_port=pushgateway_port,
+            mqsilist_command=mqsilist_command,
+            bip_codes_brokers=bip_codes_brokers,
+            bip_codes_components=bip_codes_components)
         time.sleep(60)

modules/iib_api.py

@@ -7,6 +7,7 @@ def run_iib_command(**kwargs):
     """Calls predefined commands and returns their result."""
     command_mapping = {
         'get_brokers_status': 'mqsilist | grep Broker',
+        'get_integration_nodes_status': 'mqsilist | grep "Integration node"',
         'get_broker_objects': 'mqsilist {0} -r',
     }
     broker = str()
@@ -37,6 +38,63 @@ def execute_command(command):
 def get_status(status):
     """Returns a numeric status value."""
     status_map = {
-        'running.': 1,
-        'stopped.': 0}
+        'running': 1,
+        'stopped': 0}
     return status_map[status]
+
+
+def get_platform_params_for_commands(iib_ver):
+    """Returns parameters for internal functions depending on Integration Bus version."""
+    mqsilist_brokers = "get_brokers_status"
+    mqsilist_integration_nodes = "get_integration_nodes_status"
+    # See IBM diagnostic messages:
+    # https://www.ibm.com/support/knowledgecenter/en/SSMKHH_9.0.0/com.ibm.etools.mft.bipmsgs.doc/ay_bip1.htm
+    # Also you can use command: mqsiexplain <bip_code>
+    bip_codes_broker = {
+        # BIPCode: [broker_name_position, qm_name_position, status_position, trim_last_dot_in_qm_name]
+        'BIP1284I': [2, 6, 8, 'false'],
+        'BIP1285I': [2, 6, 8, 'false'],
+        'BIP1293I': [2, 14 ,7, 'true'],
+        'BIP1294I': [2, 14 ,7, 'true'],
+        'BIP1295I': [2, 17, 13, 'true'],
+        'BIP1296I': [2, 22, 4, 'true'],
+        'BIP1297I': [2, 14, 7, 'true'],
+        'BIP1298I': [2, 17, 4, 'true']
+    }
+    # https://www.ibm.com/support/knowledgecenter/en/SSMKHH_10.0.0/com.ibm.etools.mft.bipmsgs.doc/ay_bip1.htm
+    bip_codes_integration_nodes = {
+        # BIPCode: [broker_name_position, qm_name_position, status_position, trim_last_dot_in_qm_name]
+        'BIP1284I': [3, 8, 14, 'false'],
+        'BIP1285I': [3, 7, 9, 'false'],
+        'BIP1295I': [3, 19, 15, 'true'],
+        'BIP1296I': [3, 24 ,5, 'true'],
+        'BIP1298I': [3, 18, 5, 'true'],
+        'BIP1325I': [3, None, 9, 'false'],
+        'BIP1326I': [3, None, 5, 'false'],
+        'BIP1340I': [3, None, 5, 'false'],
+        'BIP1353I': [3, 8, 10, 'false'],
+        'BIP1366I': [3, 19, 15, 'true'],
+        'BIP1376I': [3, 19, 15, 'true'],
+        'BIP1377I': [3, 24, 5, 'true']
+    }
+    bip_codes_broker_components = {
+        # BIPCode: [component_type, positions: broker_name, egname, status]
+        'BIP1286I': ['exec_groups', 6, 3, 8],
+        'BIP1287I': ['exec_groups', 6, 3, 8],
+        # BIPCode: [component_type, positions: egname, appname, status]
+        'BIP1275I': ['applications', 6, 2, 8],
+        'BIP1276I': ['applications', 6, 2, 8],
+        # BIPCode: [component_type, positions: egname, appname, msgflowname, status]
+        'BIP1277I': ['message_flows', 7, 11, 3, 9],
+        'BIP1278I': ['message_flows', 7, 11, 3, 9]}
+    bip_codes_integration_nodes_components = {
+        'BIP1286I': ['exec_groups', 7, 3, 9],
+        'BIP1287I': ['exec_groups', 7, 3, 9],
+        'BIP1275I': ['applications', 6, 2, 8],
+        'BIP1276I': ['applications', 6, 2, 8],
+        'BIP1277I': ['message_flows', 7, 11, 3, 9],
+        'BIP1278I': ['message_flows', 7, 11, 3, 9]}
+    if iib_ver == "9":
+        return mqsilist_brokers, bip_codes_broker, bip_codes_broker_components
+    if iib_ver == "10":
+        return mqsilist_integration_nodes, bip_codes_integration_nodes, bip_codes_integration_nodes_components

modules/iib_applications.py

@@ -16,23 +16,28 @@ def get_metric_annotation():
     return annotations
 
 
-def format_applications(applications, broker_name):
+def format_applications(applications, broker_name, bip_codes):
     """Returns string with all metrics for all applications which ready to push to pushgateway."""
     metrics_annotation = get_metric_annotation()
     app_metric_data = str()
     for app in applications:
         app_list = app.split()
-        egname, app_name, status = app_list[6], app_list[2], app_list[8]
-        template_string = 'egname="{0}", brokername="{1}", appname="{2}"'.format(
-            egname.replace("'", ""),
-            broker_name,
-            app_name.replace("'", ""))
-        app_metric = '{0}{{{1}}} {2}\n'.format(
-            get_metric_name(metric_label='status'),
-            template_string,
-            get_status(status=status))
-        app_metric_data += app_metric
-    app_metric_data = '{0}{1}'.format(
-        metrics_annotation['status'],
-        app_metric_data)
+        bip_code = app_list[0].replace(':', '')
+        if bip_code in bip_codes.keys():
+            egname = app_list[bip_codes[bip_code][1]]
+            app_name = app_list[bip_codes[bip_code][2]]
+            status = app_list[bip_codes[bip_code][3]].replace(".", "")
+            template_string = 'egname="{0}", brokername="{1}", appname="{2}"'.format(
+                egname.replace("'", ""),
+                broker_name,
+                app_name.replace("'", ""))
+            app_metric = '{0}{{{1}}} {2}\n'.format(
+                get_metric_name(metric_label='status'),
+                template_string,
+                get_status(status=status))
+            app_metric_data += app_metric
+    if app_metric_data:
+        app_metric_data = '{0}{1}'.format(
+            metrics_annotation['status'],
+            app_metric_data)
     return app_metric_data

modules/iib_brokers.py

@@ -16,40 +16,42 @@ def get_metric_annotation():
     return annotations
 
 
-def get_brokers_status(brokers_data):
+def get_brokers_status(brokers_data, bip_codes):
     """Returns list with statuses for brokers."""
     output_list = brokers_data.split('\n')
     brokers = list()
     for record in filter(None, output_list):
         record_list = record.split()
-        broker_name = record_list[2].replace("'", "")
-        qm_name = record_list[6].replace("'", "")
-        status = record_list[8].replace("'", "")
-        brokers.append([broker_name, status, qm_name])
+        bip_code = record_list[0].replace(':', '')
+        if bip_code in bip_codes.keys():
+            qm_name = str()
+            broker_name = record_list[bip_codes[bip_code][0]].replace("'", "")
+            trim_last_dot = bip_codes[bip_code][3]
+            if bip_codes[bip_code][1] is not None:
+                qm_name = record_list[bip_codes[bip_code][1]].replace("'", "")
+                if trim_last_dot == 'true':
+                    qm_name = qm_name[:-1]
+            status = record_list[bip_codes[bip_code][2]].replace("'", "").replace(".", "")
+            brokers.append([broker_name, status, qm_name])
     return brokers
 
 
-def get_broker_items(broker_row_data):
+def get_broker_items(broker_row_data, bip_codes):
     """Returns lists with data for broker items: execution groups, applications, message flows."""
     output_list = broker_row_data.split('\n')
     exec_groups = list()
     applications = list()
     message_flows = list()
-    # See IBM diagnostic messages:
-    # https://www.ibm.com/support/knowledgecenter/en/SSMKHH_9.0.0/com.ibm.etools.mft.bipmsgs.doc/ay_bip1.htm
-    # Also you can use command: mqsiexplain <bip_code>
-    bip_codes = {
-     'BIP1286I': exec_groups,
-     'BIP1287I': exec_groups,
-     'BIP1275I': applications,
-     'BIP1276I': applications,
-     'BIP1277I': message_flows,
-     'BIP1278I': message_flows}
     for record in output_list:
         if record:
             bip_code = record.split()[0].replace(':', '')
             if bip_code in bip_codes.keys():
-                bip_codes[bip_code].append(record)
+                if  bip_codes[bip_code][0] == 'exec_groups':
+                    exec_groups.append(record)
+                if  bip_codes[bip_code][0] == 'applications':
+                    applications.append(record)
+                if  bip_codes[bip_code][0] == 'message_flows':
+                    message_flows.append(record)
     return exec_groups, applications, message_flows