Merge pull request #32 from stackql/feature/updates

v3.5.3

Author: jeffreyaven

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## v3.5.3 (2024-04-08)
+
+### Updates
+
+ * added `backend_storage_mode` and `backend_file_storage_location` constructor args for specifying a file based backend (not applicable in `server_mode`)
+
 ## v3.5.2 (2024-03-21)
 
 ### Updates

README.rst

@@ -161,6 +161,7 @@ Before testing, ensure you have all the required packages installed:
 ::
 
     pip install -r requirements.txt
+    pip install psycopg2-binary
 
 Once the dependencies are installed, you can run the tests using the provided script:
 
@@ -193,4 +194,4 @@ To publish the package to PyPI, run the following command:
 
 ::
 
-    twine upload dist/pystackql-3.5.2.tar.gz
+    twine upload dist/pystackql-3.5.3.tar.gz

docs/source/conf.py

@@ -20,13 +20,13 @@
 # -- Project information -----------------------------------------------------
 
 project = 'pystackql'
-copyright = '2024, StackQL Studios'
+copyright = '2021-2024, StackQL Studios'
 author = 'StackQL Studios'
 
 # The short X.Y version
 version = ''
 # The full version, including alpha/beta/rc tags
-release = '3.5.2'
+release = '3.5.3'
 
 
 # -- General configuration ---------------------------------------------------

pystackql/stackql.py

@@ -26,6 +26,12 @@ class StackQL:
 	:param server_port: The port of the StackQL server 
 		(`server_mode` only, defaults to `5466`)
 	:type server_port: int, optional
+	:param backend_storage_mode: Specifies backend storage mode, options are 'memory' and 'file'
+		(defaults to `'memory'`, this option is ignored in `server_mode`)
+	:type backend_storage_mode: str, optional
+	:param backend_file_storage_location: Specifies location for database file, only applicable when `backend_storage_mode` is 'file'
+		(defaults to `'{cwd}/stackql.db'`, this option is ignored in `server_mode`)
+	:type backend_file_storage_location: str, optional
 	:param output: Determines the format of the output, options are 'dict', 'pandas', and 'csv' 
 		(defaults to `'dict'`, `'csv'` is not supported in `server_mode`)
 	:type output: str, optional
@@ -210,7 +216,9 @@ def _run_query(self, query):
 	def __init__(self, 
 				 server_mode=False, 
 				 server_address='127.0.0.1', 
-				 server_port=5466, 
+				 server_port=5466,
+				 backend_storage_mode='memory',
+				 backend_file_storage_location='stackql.db', 
 				 download_dir=None, 
 				 app_root=None,
 				 execution_concurrency_limit=1,
@@ -284,7 +292,12 @@ def __init__(self,
 				self.params.append("csv")
 			else:
 				self.params.append("json")
-			
+
+			# backend storage settings
+			if backend_storage_mode == 'file':
+				self.params.append("--sqlBackend")
+				self.params.append(json.dumps({ "dsn": f"file:{backend_file_storage_location}" }))
+
 			# get or download the stackql binary
 			binary = _get_binary_name(this_os)
 
@@ -330,10 +343,8 @@ def __init__(self,
 			# if custom_registry is set, use it
 			if custom_registry is not None:
 				self.custom_registry = custom_registry
-				custom_reg_obj = { "url": custom_registry }
-				custom_reg_str = json.dumps(custom_reg_obj)
 				self.params.append("--registry")
-				self.params.append(custom_reg_str)
+				self.params.append(json.dumps({ "url": custom_registry }))
 
 			# csv output
 			if self.output == "csv":

setup.py

@@ -10,7 +10,7 @@
 
 setup(
     name='pystackql',
-    version='3.5.2',
+    version='3.5.3',
     description='A Python interface for StackQL',
     long_description=readme,
     author='Jeffrey Aven',

tests/pystackql_tests.py

@@ -41,6 +41,9 @@ def setUpModule():
     print("downloading aws provider for tests...")
     res = PyStackQLTestsBase.stackql.executeStmt(registry_pull_aws_query)
     print(res)
+    print("downloading awscc provider for tests...")
+    res = PyStackQLTestsBase.stackql.executeStmt(registry_pull_awscc_query)
+    print(res)    
     print("downloading google provider for tests...")
     res = PyStackQLTestsBase.stackql.executeStmt(registry_pull_google_query)
     print(res)
@@ -201,8 +204,8 @@ def test_12_execute_with_pandas_output(self):
         self.assertTrue(is_valid_dataframe, f"Result is not a valid DataFrame: {result}")
         # Check datatypes of the columns
         expected_dtypes = {
-            'instance_state': 'object',  # This should be 'object' for older Pandas versions
-            'num_instances': 'int64'
+            'year': 'int64',
+            'num_buckets': 'int64'
         }
         for col, expected_dtype in expected_dtypes.items():
             actual_dtype = result[col].dtype
@@ -280,8 +283,8 @@ def test_22_execute_server_mode_pandas_output(self):
         self.assertTrue(is_valid_dataframe, f"Result is not a valid DataFrame: {result}")
         # Check datatypes of the columns
         expected_dtypes = {
-            'instance_state': 'object',  # This should be 'object' for older Pandas versions
-            'num_instances': 'int64'
+            'year': 'int64',
+            'num_buckets': 'int64'
         }
         for col, expected_dtype in expected_dtypes.items():
             actual_dtype = result[col].dtype

tests/test_params.py

@@ -27,6 +27,7 @@ def get_custom_download_dir(platform_name):
 
 registry_pull_google_query = "REGISTRY PULL google"
 registry_pull_aws_query = "REGISTRY PULL aws"
+registry_pull_awscc_query = "REGISTRY PULL awscc"
 registry_pull_okta_query = "REGISTRY PULL okta"
 registry_pull_github_query = "REGISTRY PULL github"
 
@@ -43,21 +44,18 @@ def registry_pull_resp_pattern(provider):
 
 aws_query = f"""
 SELECT 
-split_part(instanceState, '\n', 3) as instance_state,
-count(*) as num_instances
-FROM aws.ec2.instances 
-WHERE region = '{os.environ['AWS_REGION']}'
-GROUP BY instance_state
+SPLIT_PART(CreationDate, '-', 1) as year, count(*) as num_buckets FROM aws.s3.buckets
+WHERE region = 'us-east-1'
+GROUP BY year
 """
 
 regions = os.environ.get('AWS_REGIONS').split(',')
 
 async_queries = [
     f"""
-    SELECT '{region}' as region, instanceType, COUNT(*) as num_instances
-    FROM aws.ec2.instances
+    SELECT region, COUNT(*) as num_functions
+    FROM awscc.lambda.functions
     WHERE region = '{region}'
-    GROUP BY instanceType
     """
     for region in regions
 ]