Skip to content

Commit 96dc7f8

Browse files
committed
Merge branch 'WIP/faker_generator'
2 parents 77d28d2 + e7eb02a commit 96dc7f8

15 files changed

+107
-49
lines changed

.vscode/launch.json

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,20 @@
66
"configurations": [
77
{
88
"type": "node",
9-
"request": "attach",
9+
"request": "launch",
1010
"name": "attach to process",
11-
"port": 9229
11+
"args": [
12+
"${workspaceFolder}/src/main.ts"
13+
],
14+
"runtimeArgs": [
15+
"--nolazy",
16+
"-r",
17+
"ts-node/register",
18+
],
19+
"sourceMaps": true,
20+
"envFile": "${workspaceFolder}/.env",
21+
"cwd": "${workspaceRoot}",
22+
"console": "integratedTerminal",
1223
}
1324
]
1425
}
3.44 MB
Binary file not shown.
-9.14 MB
Binary file not shown.
10.9 MB
Binary file not shown.
10.9 MB
Binary file not shown.
-9.15 MB
Binary file not shown.

README.md

Lines changed: 39 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,38 @@
1+
# MySQL Data Generator
2+
13
This is a tool to easily fill a SQL database.
24
It is able to analyse a schema and generate a `settings/schema.jsonc` which will be used to generate accurate data. It does its best to handle foreign keys.
35
You can provide a `settings/schema_custom.jsonc` to customize `settings/schema.jsonc` during the generation phase. This will allow you to override datatype for a column, force the use of a foreign key
46
or specify a list of values.
57

68
## functionalities
79

8-
- analyse a table and generate a schema
9-
- allow for customization on data types, foreign keys, values, uniqueness etc.
10-
- handle foreign keys
11-
- define a number of rows to generate per table
12-
- specify a seed to always generate the same dataset
13-
- disable/enable triggers during process
10+
- analyse a table and generate a schema
11+
- allow for customization on data types, foreign keys, values, uniqueness etc.
12+
- handle foreign keys
13+
- define a number of rows to generate per table
14+
- specify a seed to always generate the same dataset
15+
- disable/enable triggers during process
1416

1517
## 1. Analysis
1618

1719
The first step is to analyse your database to generate a `settings/schema.jsonc` by providing database credentials:
1820

1921
The `schema` parameter allows you to specify a name for the output files and differentiate between multiple schemas and setups.
2022

21-
```
23+
```bash
2224
npx @corteks/mysql-data-generator --db mysql://user:password@127.0.0.1:3306/database --analyse --schema schema
2325
```
2426

27+
The `--schema` parameter allows you to generate mutliple configuration with different names.
28+
2529
If you want to customize the schema, modify the default `settings/schema_custom.jsonc` that has also be generated.
2630

2731
## 2. Data generation
2832

2933
Next step is to fill the database with randomly generated values:
3034

31-
```
35+
```bash
3236
mysqldatagen --db mysql://user:password@127.0.0.1:3306/database
3337
```
3438

@@ -37,34 +41,36 @@ As they will be run every time the generation is launched you have to take care
3741

3842
For every tables listed in `settings/schema.jsonc`, the tool will:
3943

40-
- get the values of foreign keys if needed
41-
- generate batches of 1000 rows
42-
- insert rows until it reaches the defined table limit
43-
- columns in table are ordered accordingly to your custom schema so you can rely on other column value in the same row.
44+
- get the values of foreign keys if needed
45+
- generate batches of 1000 rows
46+
- insert rows until it reaches the defined table limit
47+
- columns in table are ordered accordingly to your custom schema so you can rely on other column value in the same row.
4448

4549
Available options in `schema_custom.json`:
4650

47-
- `settings`: Global settings
48-
- `disableTriggers: boolean` // disable triggers per table during process and recreate them afterward
49-
- `engine: "MariaDB"` // only MariaDB is supported for the time being but it should also be compatible with MySQL.
50-
- `ignoredTables: string[]` // list of table name that should not be analysed nor filled
51-
- `options: Array<[key: string]: any[]>` // an array of column options to configure specific generators for the whole file `generator` is an array of string to allow multiple settings at once
52-
- `maxLengthValue: number?` // Hard limit of the maximum number of characters in `string` column type. This will override your custom column `max` value if it's bigger than `maxLengthValue`.
53-
- `seed: number` // The seed used by the random generator. This is optional. filling process.
54-
- `tablesToFill: string[]` // list of table name that should be analysed and filled. You can set this parameter or `ignoredTables` depending on the number of table to work with
55-
- `values: [key: string]: any[]` // an object of user defined array of values
56-
- `tables: Table[]` // list of tables handled by the tool
57-
- `Table.name: string` // table name
58-
- `Table.lines: number` // Deprecated in favor of maxLines
59-
- `Table.maxLines: number` // Maximum number of rows this table should contains
60-
- `Table.addLines: number` // Number of rows to be inserted on a single run. The number of lines resulting in the table will not exceed `Table.maxLines`
61-
- `Table.columns: Column[]` // list of columns handled by the tool
62-
- `Column.name: string` // column name
63-
- `Column.generator: bit | boolean | date | foreignKey | integer | real | time | string | values | function` // data type generator used for this column
64-
- `Column.[key: string]: any[]` // list of options for this column
65-
- `Column.foreignKey: { table: string, column: string, where: string }` // link to the table.column referenced by this foreign key. A custom clause can ba added to filter value from the foreign column
66-
- `Column.values: string | any[] | { [key: string]: number }`
51+
- `settings`: Global settings
52+
- `disableTriggers: boolean` // disable triggers per table during process and recreate them afterward
53+
- `engine: "MariaDB"` // only MariaDB is supported for the time being but it should also be compatible with MySQL.
54+
- `ignoredTables: string[]` // list of table name that should not be analysed nor filled
55+
- `options: Array<[key: string]: any[]>` // an array of column options to configure specific generators for the whole file `generator` is an array of string to allow multiple settings at once
56+
- `maxLengthValue: number?` // Hard limit of the maximum number of characters in `string` column type. This will override your custom column `max` value if it's bigger than `maxLengthValue`.
57+
- `seed: number` // The seed used by the random generator. This is optional. filling process.
58+
- `tablesToFill: string[]` // list of table name that should be analysed and filled. You can set this parameter or `ignoredTables` depending on the number of table to work with
59+
- `values: [key: string]: any[]` // an object of user defined array of values
60+
- `tables: Table[]` // list of tables handled by the tool
61+
- `Table.name: string` // table name
62+
- `Table.lines: number` // Deprecated in favor of maxLines
63+
- `Table.maxLines: number` // Maximum number of rows this table should contains
64+
- `Table.addLines: number` // Number of rows to be inserted on a single run. The number of lines resulting in the table will not exceed `Table.maxLines`
65+
- `Table.columns: Column[]` // list of columns handled by the tool
66+
- `Column.name: string` // column name
67+
- `Column.generator: bit | boolean | date | foreignKey | integer | real | time | string | values | function | faker` // data type generator used for this column
68+
- `Column.[key: string]: any[]` // list of options for this column
69+
- `Column.foreignKey: { table: string, column: string, where: string }` // link to the table.column referenced by this foreign key. A custom clause can ba added to filter value from the foreign column
70+
- `Column.values: string | any[] | { [key: string]: number }`
6771
// Name of the list of values to use for this column.
6872
// You can also directly specify an array of strings for values.
6973
// Or you can use an object to specify a ratio per value. Ratio will be a number between 0 and 1.
70-
- `Column.customFunction: (rowIndex: number, row: { [key: string]: string | number }` // a string representing a javascript custom function. It will receive the row index and the full row as arguments.
74+
- `Column.customFunction: (rowIndex: number, row: { [key: string]: string | number }` // a string representing a javascript custom function. It will receive the row index and the full row as arguments.
75+
- `Column.template: string` // a template string for `faker` generator. See [fakerjs](https://www.npmjs.com/package/@faker-js/faker) for more information.
76+
- `Column.locale: string` // locale used by the faker generator.

package.json

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
},
2626
"dependencies": {
2727
"@corteks/clify": "^0.2.1",
28+
"@faker-js/faker": "^6.3.1",
2829
"class-transformer": "^0.3.1",
2930
"class-validator": "^0.13.2",
3031
"cli-progress": "^3.8.2",
@@ -51,7 +52,7 @@
5152
"ts-jest": "^26.4.0",
5253
"ts-node": "^8.10.2",
5354
"tslint": "^6.1.3",
54-
"typescript": "^3.9.7",
55+
"typescript": "^4.6.4",
5556
"typescript-tslint-plugin": "^0.5.5"
5657
},
5758
"license": "GPL-3.0-or-later",
@@ -71,4 +72,4 @@
7172
"lib": "lib"
7273
},
7374
"packageManager": "yarn@3.2.0"
74-
}
75+
}
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import { faker } from '@faker-js/faker';
2+
import { AbstractGenerator } from './generators';
3+
4+
export class FakerGenerator extends AbstractGenerator<string> {
5+
public validate(): boolean {
6+
if (this.column.template === undefined)
7+
throw new Error(`faker template required for type faker: ${this.table.name}.${this.column.name}`);
8+
return true;
9+
}
10+
11+
public generate(rowIndex: number, row: { [key: string]: any; }): string {
12+
faker.locale = this.column.locale || 'en';
13+
return faker.fake(this.column.template!);
14+
}
15+
}

src/generation/generators/generators.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ export enum Generators {
1313
values = 'values',
1414
foreignKey = 'foreignKey',
1515
function = 'function',
16+
faker = 'faker',
1617
}
1718

1819
export abstract class AbstractGenerator<T>{
@@ -22,7 +23,7 @@ export abstract class AbstractGenerator<T>{
2223
protected column: CustomizedColumn,
2324
) { }
2425

25-
public static validate(table: CustomizedTable, column: CustomizedColumn): boolean { return true };
26+
public static validate(table: CustomizedTable, column: CustomizedColumn): boolean { return true; };
2627

2728
public async init(): Promise<AbstractGenerator<T>> { return this; }
2829

0 commit comments

Comments
 (0)