Merge branch 'WIP/specify_settings_filepath'

Author: Raphael67

.gitignore

@@ -1,6 +1,4 @@
 node_modules
-custom_schema.json
-schema.json
 dist
 lib
 coverage

.npmignore

@@ -3,8 +3,6 @@ src
 .gitignore
 tsconfig.json
 tslint.json
-custom_schema.json
-schema.json
 coverage
 test
-settings
+settings

README.md

@@ -1,6 +1,6 @@
 This is a tool to easily fill a SQL database.
 It is able to analyse a schema and generate a `settings/schema.jsonc` which will be used to generate accurate data. It does its best to handle foreign keys.
-You can provide a `settings/custom_schema.jsonc` to customize `settings/schema.jsonc` during the generation phase. This will allow you to override datatype for a column, force the use of a foreign key
+You can provide a `settings/schema_custom.jsonc` to customize `settings/schema.jsonc` during the generation phase. This will allow you to override datatype for a column, force the use of a foreign key
 or specify a list of values.
 
 ## functionalities
@@ -16,13 +16,13 @@ or specify a list of values.
 
 The first step is to analyse your database to generate a `settings/schema.jsonc` by providing database credentials:
 
-```
-npm install -g @corteks/mysql-data-generator
+The `schema` parameter allows you to specify a name for the output files and differentiate between multiple schemas and setups.
 
-mysqldatagen --db mysql://user:password@127.0.0.1:3306/database --analyse
+```
+npx @corteks/mysql-data-generator --db mysql://user:password@127.0.0.1:3306/database --analyse --schema schema
 ```
 
-If you want to customize the schema, modify the default `settings/custom_schema.jsonc` that has also be generated.
+If you want to customize the schema, modify the default `settings/schema_custom.jsonc` that has also be generated.
 
 ## 2. Data generation
 
@@ -42,7 +42,7 @@ For every tables listed in `settings/schema.jsonc`, the tool will:
 -   insert rows until it reaches the defined table limit
 -   columns in table are ordered accordingly to your custom schema so you can rely on other column value in the same row.
 
-Available options in `custom_schema.json`:
+Available options in `schema_custom.json`:
 
 -   `settings`: Global settings
     -   `disableTriggers: boolean` // disable triggers per table during process and recreate them afterward

src/database/mariadb-connector.ts

@@ -1,12 +1,12 @@
+import * as fs from 'fs-extra';
 import Knex from 'knex';
 import { getLogger } from 'log4js';
-import * as fs from 'fs-extra';
+import { Connection, MysqlError } from 'mysql';
 import * as path from 'path';
-import { Table, Column, Schema } from '../schema/schema.class';
-import { DatabaseConnector } from './database-connector-builder';
-import { Generators } from '../generation/generators/generators';
 import * as URI from 'uri-js';
-import { Connection, MysqlError } from 'mysql';
+import { Generators } from '../generation/generators/generators';
+import { Column, Schema, Table } from '../schema/schema.class';
+import { DatabaseConnector } from './database-connector-builder';
 
 export class MariaDBConnector implements DatabaseConnector {
     private dbConnection: Knex;
@@ -84,7 +84,7 @@ export class MariaDBConnector implements DatabaseConnector {
 
     async insert(table: string, rows: any[]): Promise<number> {
         if (rows.length === 0) return 0;
-        const query = await this.dbConnection(table)
+        const query = this.dbConnection(table)
             .insert(rows)
             .toQuery()
             .replace('insert into', 'insert ignore into');
@@ -114,7 +114,7 @@ export class MariaDBConnector implements DatabaseConnector {
             .filter((column: MySQLColumn) => {
                 return ['enum', 'set'].includes(column.DATA_TYPE || '');
             }).forEach((column: MySQLColumn) => {
-                column.NUMERIC_PRECISION = column.COLUMN_TYPE.match(/[enum,set]\((.*)\)$/)![1].split('\',\'').length;
+                column.NUMERIC_PRECISION = column.COLUMN_TYPE.match(/(enum|set)\((.*)\)$/)![1].split('\',\'').length;
             });
 
         table.columns = columns.map((mysqlColumn: MySQLColumn) => {
@@ -293,16 +293,15 @@ export class MariaDBConnector implements DatabaseConnector {
             .andWhere('t.TABLE_TYPE', 'BASE TABLE')
             .groupBy('t.TABLE_SCHEMA', 't.TABLE_NAME');
 
-        const tables = tableNames.map((row) => {
+        return tableNames.map((row) => {
             const table = new Table();
             table.name = row.name;
             return table;
         });
-        return tables;
     }
 
     async getColumnsInformation(table: Table) {
-        return await this.dbConnection.select()
+        return this.dbConnection.select()
             .from('information_schema.COLUMNS')
             .where({
                 'TABLE_SCHEMA': this.database,
@@ -329,8 +328,7 @@ export class MariaDBConnector implements DatabaseConnector {
             .having(this.dbConnection.raw('count(kcu2.CONSTRAINT_NAME) < 2'))
             .as('indexes');
 
-
-        const foreignKeys = await this.dbConnection.select([
+        return this.dbConnection.select([
             'kcu.column_name AS column',
             'kcu.referenced_table_name AS foreignTable',
             'kcu.referenced_column_name AS foreignColumn',
@@ -344,8 +342,6 @@ export class MariaDBConnector implements DatabaseConnector {
             })
             .where('kcu.table_name', table.name)
             .whereNotNull('kcu.referenced_column_name');
-
-        return foreignKeys;
     }
 
     async getValuesForForeignKeys(

src/main.ts

@@ -22,12 +22,15 @@ class Main extends CliMainClass {
     @CliParameter({ alias: 'db', demandOption: true, description: 'Database URI. Eg: mysql://user:password@127.0.0.1:3306/database' })
     private uri: string | undefined = undefined;
 
-    @CliParameter()
+    @CliParameter({ description: 'Extrat schema information and generate default settings' })
     private analyse: boolean = false;
 
-    @CliParameter()
+    @CliParameter({ description: 'Empty tables before filling them' })
     private reset: boolean = false;
 
+    @CliParameter({ description: 'Schema filename to use. Will be generated with --analyse' })
+    private schema: string = 'schema';
+
     private dbConnector: DatabaseConnector | undefined;
     private filler: Filler | undefined;
 
@@ -38,7 +41,7 @@ class Main extends CliMainClass {
         try {
             this.dbConnector = await dbConnectorBuilder.build();
         } catch (err) {
-            logger.error(err.message);
+            logger.error((err as Error).message);
             return 1;
         }
         if (!fs.pathExistsSync('settings')) {
@@ -50,12 +53,12 @@ class Main extends CliMainClass {
         try {
             if (this.analyse) {
                 return await this.generateSchemaFromDB();
-            };
+            }
 
             await this.generateData();
         } catch (ex) {
-            if (ex.code === 'ENOENT') {
-                logger.error('Unable to read from ./settings/schema.json. Please run with --analyse first.');
+            if ((ex as any).code === 'ENOENT') {
+                logger.error(`Unable to read from ./settings/${this.schema}.json. Please run with --analyse first.`);
             } else {
                 logger.error(ex);
             }
@@ -69,21 +72,23 @@ class Main extends CliMainClass {
     async generateSchemaFromDB() {
         if (!this.dbConnector) throw new Error('DB connection not ready');
         const schema = await this.dbConnector.getSchema();
-        fs.writeJSONSync(path.join('settings', 'schema.json'), schema.toJSON(), { spaces: 4 });
-        if (!fs.existsSync(path.join('settings', 'custom_schema.jsonc'))) {
+        fs.writeJSONSync(path.join('settings', `${this.schema}.json`), schema.toJSON(), { spaces: 4 });
+        if (!fs.existsSync(path.join('settings', `${this.schema}_custom.jsonc`))) {
             const customSchema = new CustomSchema();
-            fs.writeJSONSync(path.join('settings', 'custom_schema.jsonc'), customSchema, { spaces: 4 });
+            fs.writeJSONSync(path.join('settings', `${this.schema}_custom.jsonc`), customSchema, { spaces: 4 });
         }
         return 0;
     }
 
     async runScripts() {
         if (!this.dbConnector) throw new Error('DB connection not ready');
-        if (!fs.existsSync(path.join('settings', 'scripts'))) {
+        const scriptsFolder = path.join('settings', 'scripts');
+        if (!fs.existsSync(scriptsFolder)) {
+            fs.mkdirSync(scriptsFolder);
             logger.info('No scripts provided.');
             return;
         }
-        const scripts = fs.readdirSync(path.join('settings', 'scripts'));
+        const scripts = fs.readdirSync(scriptsFolder);
         if (scripts.length === 0) {
             logger.info('No scripts provided.');
             return;
@@ -101,18 +106,18 @@ class Main extends CliMainClass {
 
     async generateData() {
         if (!this.dbConnector) throw new Error('DB connection not ready');
-        const schema: Schema = await Schema.fromJSON(fs.readJSONSync(path.join('settings', 'schema.json')));
+        const schema: Schema = await Schema.fromJSON(fs.readJSONSync(path.join('settings', `${this.schema}.json`)));
         let customSchema: CustomSchema = new CustomSchema();
         try {
-            customSchema = JSONC.parse(fs.readFileSync(path.join('settings', 'custom_schema.jsonc')).toString());
+            customSchema = JSONC.parse(fs.readFileSync(path.join('settings', `${this.schema}_custom.jsonc`)).toString());
         } catch (ex) {
-            logger.warn('Unable to read ./settings/custom_schema.json, this will not take any customization into account.');
+            logger.warn(`Unable to read ./settings/${this.schema}_custom.json, this will not take any customization into account.`);
         }
         try {
             await this.runScripts();
         } catch (ex) {
             logger.error('An error occured while running scripts:');
-            logger.error(ex.message);
+            logger.error((ex as Error).message);
             return;
         }
         const customizedSchema = CustomizedSchema.create(schema, customSchema);
@@ -129,7 +134,7 @@ class Main extends CliMainClass {
         return (event: ProgressEvent) => {
             let diff = false;
             if (previousEvent.currentTable !== event.currentTable) {
-                console.log(colors.green(event.currentTable));
+                logger.info(colors.green(event.currentTable));
                 diff = true;
             }
             if (previousEvent.step !== event.step) {

src/schema/custom-schema.class.ts

@@ -21,9 +21,13 @@ export class CustomSettings {
     @IsArray()
     @ValidateNested({ each: true })
     tablesToFill: string[] = [];
+    /**
+     * A default value has been provided to maxLengthValue as this can drastically improve performances. This is a parameter which used to be
+     * defined a lot by user anyway. The value of 36 has been chosen to allow usage of UID in string generator.
+     */
     @IsNumber()
     @IsOptional()
-    maxLengthValue?: number;
+    maxLengthValue?: number = 36;
     values: { [key: string]: any[]; } = {};
     options: {
         generators: Generators[],

test/schema/customized-schema.test.ts

@@ -1,8 +1,8 @@
+import { Builder } from '../../src/builder';
+import { Generators } from '../../src/generation/generators/generators';
 import { CustomSchema } from '../../src/schema/custom-schema.class';
-import { Schema, Table, Column } from '../../src/schema/schema.class';
 import { CustomizedSchema } from '../../src/schema/customized-schema.class';
-import { Generators } from '../../src/generation/generators/generators';
-import { Builder } from '../../src/builder';
+import { Column, Schema, Table } from '../../src/schema/schema.class';
 
 describe('CustomizedSchema', () => {
     it('handle missing custom table', async () => {
@@ -94,6 +94,69 @@ describe('CustomizedSchema', () => {
         schema.tables = [table];
 
         const customSchema = new CustomSchema();
+        customSchema.tables = [{
+            name: 'table1',
+            maxLines: 100,
+            columns: [
+                {
+                    name: 'column1',
+                    max: 10,
+                    values: [],
+                },
+            ],
+        }];
+        const result = CustomizedSchema.create(schema, customSchema);
+        expect(result.tables[0].columns[0].max).toBe(10);
+    });
+    it('Column options takes in account default maxLengthValue', async () => {
+        const column = new Builder(Column)
+            .set('name', 'column1')
+            .set('generator', Generators.string)
+            .build();
+
+        const table = new Builder(Table)
+            .set('name', 'table1')
+            .set('columns', [
+                column,
+            ])
+            .build();
+
+        const schema = new Schema();
+        schema.tables = [table];
+
+        const customSchema = new CustomSchema();
+        customSchema.tables = [{
+            name: 'table1',
+            maxLines: 100,
+            columns: [
+                {
+                    name: 'column1',
+                    max: 100,
+                    values: [],
+                },
+            ],
+        }];
+        const result = CustomizedSchema.create(schema, customSchema);
+        expect(result.tables[0].columns[0].max).toBe(36);
+    });
+    it('Column options do not override maxLengthValue', async () => {
+        const column = new Builder(Column)
+            .set('name', 'column1')
+            .set('generator', Generators.string)
+            .build();
+
+        const table = new Builder(Table)
+            .set('name', 'table1')
+            .set('columns', [
+                column,
+            ])
+            .build();
+
+        const schema = new Schema();
+        schema.tables = [table];
+
+        const customSchema = new CustomSchema();
+        customSchema.settings.maxLengthValue = 42;
         customSchema.tables = [{
             name: 'table1',
             maxLines: 100,
@@ -106,7 +169,7 @@ describe('CustomizedSchema', () => {
             ],
         }];
         const result = CustomizedSchema.create(schema, customSchema);
-        expect(result.tables[0].columns[0].max).toBe(100);
+        expect(result.tables[0].columns[0].max).toBe(42);
     });
     it('reorder columns', async () => {
         const columns = [