Skip to content

Commit 065038b

Browse files
mihajmihaj
authored
Bombardier test asserter and small updates
* logo update * generator operationId update * Bombardier test asserter, readme update * ci fix * readme update
2 parents e3d90e5 + 878b856 commit 065038b

File tree

11 files changed

+541
-15
lines changed

11 files changed

+541
-15
lines changed

‎.github/workflows/dotnet-core.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ jobs:
2727
- name: Install dependencies
2828
run: dotnet restore
2929
- name: Build
30-
run: dotnet build --configuration Release
30+
run: dotnet build --configuration Release --no-restore
3131
- name: Test
3232
run: dotnet test --no-restore --verbosity normal
3333
- name: Upload a Build Artifact

‎Directory.Build.props‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
<Project>
22
<PropertyGroup>
3-
<Version>0.3.2</Version>
3+
<Version>0.3.5</Version>
44
</PropertyGroup>
55
</Project>

‎README.md‎

Lines changed: 45 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
# QAToolKit.Engine.Bombardier
2-
![https://github.com/qatoolkit/qatoolkit-engine-bombardier-net/actions](https://github.com/qatoolkit/qatoolkit-engine-bombardier-net/workflows/.NET%20Core/badge.svg?branch=main)
3-
![https://github.com/qatoolkit/qatoolkit-engine-bombardier-net/security/code-scanning](https://github.com/qatoolkit/qatoolkit-engine-bombardier-net/workflows/CodeQL%20Analyze/badge.svg)
4-
![https://sonarcloud.io/dashboard?id=qatoolkit_qatoolkit-engine-bombardier-net](https://github.com/qatoolkit/qatoolkit-engine-bombardier-net/workflows/Sonarqube%20Analyze/badge.svg)
5-
![https://www.nuget.org/packages/QAToolKit.Engine.Bombardier/](https://img.shields.io/nuget/v/QAToolKit.Engine.Bombardier?label=QAToolKit.Engine.Bombardier)
2+
[![Build .NET Library](https://github.com/qatoolkit/qatoolkit-engine-bombardier-net/workflows/.NET%20Core/badge.svg?branch=main)](https://github.com/qatoolkit/qatoolkit-engine-bombardier-net/actions)
3+
[![CodeQL](https://github.com/qatoolkit/qatoolkit-engine-bombardier-net/workflows/CodeQL%20Analyze/badge.svg)](https://github.com/qatoolkit/qatoolkit-engine-bombardier-net/security/code-scanning)
4+
[![Sonarcloud Quality gate](https://github.com/qatoolkit/qatoolkit-engine-bombardier-net/workflows/Sonarqube%20Analyze/badge.svg)](https://sonarcloud.io/dashboard?id=qatoolkit_qatoolkit-engine-bombardier-net)
5+
[![NuGet package](https://img.shields.io/nuget/v/QAToolKit.Engine.Bombardier?label=QAToolKit.Engine.Bombardier)](https://www.nuget.org/packages/QAToolKit.Engine.Bombardier/)
66

77
## Description
88
`QAToolKit.Engine.Bombardier` is a .NET standard library, which takes `IEnumerable<HttpTestRequest>` object and runs load tests with tool called [Bombardier](https://github.com/codesenberg/bombardier).
@@ -29,6 +29,7 @@ var httpRequest = JsonConvert.DeserializeObject<IEnumerable<HttpRequest>>(conten
2929
var bombardierTestsGenerator = new BombardierTestsGenerator(httpRequest, options =>
3030
{
3131
options.BombardierConcurrentUsers = 1;
32+
3233
options.BombardierDuration = 1;
3334
options.BombardierTimeout = 30;
3435
options.BombardierUseHttp2 = true;
@@ -70,7 +71,45 @@ Results sample `bombardierResults`:
7071
]
7172
```
7273

73-
As you can see you get a lot of metrics from the tests.
74+
As you can see you get a lot of metrics from the tests. You can use xUnit to assert the numbers, or you can use `BombardierTestAsserter` like this:
75+
76+
```csharp
77+
var asserter = new BombardierTestAsserter(bombardierResults.FirstOrDefault());
78+
var assertResults = asserter
79+
.NumberOf1xxResponses((x) => x == 0)
80+
.NumberOf2xxResponses((x) => x >= 0)
81+
.NumberOf3xxResponses((x) => x == 0)
82+
.NumberOf4xxResponses((x) => x == 0)
83+
.NumberOf5xxResponses((x) => x == 0)
84+
.AverageLatency((x) => x >= 0)
85+
.AverageRequestsPerSecond((x) => x >= 0)
86+
.MaximumLatency((x) => x >= 0)
87+
.MaximumRequestsPerSecond((x) => x >= 0)
88+
.AssertAll();
89+
```
90+
91+
Asserter produces a list of `AssertResult`:
92+
93+
```csharp
94+
/// <summary>
95+
/// Assert result object
96+
/// </summary>
97+
public class AssertResult
98+
{
99+
/// <summary>
100+
/// Assert name
101+
/// </summary>
102+
public string Name { get; set; }
103+
/// <summary>
104+
/// Is assert true
105+
/// </summary>
106+
public bool IsTrue { get; set; }
107+
/// <summary>
108+
/// Assert message
109+
/// </summary>
110+
public string Message { get; set; }
111+
}
112+
```
74113

75114
## Description
76115

@@ -158,6 +197,7 @@ in the sample code above. Check the [QAToolKit.Source.Swagger](https://github.co
158197

159198
- **This library is an early alpha version**
160199
- Currently tested for GET, POST, PUT and DELETE HTTP methods. Need to extend support.
200+
- NTLM authentication is currently not possible because of the Bombardier limitations.
161201

162202
## License
163203

‎qatoolkit-64x64.png‎

-813 Bytes
Loading

‎src/QAToolKit.Engine.Bombardier.Test/BombardierTestAsserterTests.cs‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
using Microsoft.Extensions.Logging;
2+
using Newtonsoft.Json;
3+
using QAToolKit.Core.Models;
4+
using QAToolKit.Engine.Bombardier.Models;
5+
using System;
6+
using System.Collections.Generic;
7+
using System.IO;
8+
using System.Linq;
9+
using System.Threading.Tasks;
10+
using Xunit;
11+
using Xunit.Abstractions;
12+
13+
namespace QAToolKit.Engine.Bombardier.Test
14+
{
15+
public class BombardierTestAsserterTests
16+
{
17+
private readonly ILogger<BombardierTestAsserterTests> _logger;
18+
19+
public BombardierTestAsserterTests(ITestOutputHelper testOutputHelper)
20+
{
21+
var loggerFactory = new LoggerFactory();
22+
loggerFactory.AddProvider(new XunitLoggerProvider(testOutputHelper));
23+
_logger = loggerFactory.CreateLogger<BombardierTestAsserterTests>();
24+
}
25+
26+
[IgnoreOnGithubFact]
27+
public async Task BombardierGetTestWithOptionsTest_Successfull()
28+
{
29+
var content = File.ReadAllText("Assets/getPetById.json");
30+
var httpRequest = JsonConvert.DeserializeObject<IEnumerable<HttpRequest>>(content);
31+
32+
var bombardierTestsGenerator = new BombardierTestsGenerator(httpRequest, options =>
33+
{
34+
options.BombardierConcurrentUsers = 1;
35+
options.BombardierDuration = 1;
36+
options.BombardierTimeout = 30;
37+
options.BombardierUseHttp2 = true;
38+
});
39+
40+
var bombardierTests = await bombardierTestsGenerator.Generate();
41+
42+
//Run Bombardier Tests
43+
var bombardierTestsRunner = new BombardierTestsRunner(bombardierTests.ToList(), options =>
44+
{
45+
options.ObfuscateAuthenticationHeader = true;
46+
});
47+
var bombardierResults = await bombardierTestsRunner.Run();
48+
49+
_logger.LogInformation(JsonConvert.SerializeObject(bombardierResults, Formatting.Indented));
50+
51+
var asserter = new BombardierTestAsserter(bombardierResults.FirstOrDefault());
52+
var assertResults = asserter
53+
.NumberOf1xxResponses((x) => x == 0)
54+
.NumberOf2xxResponses((x) => x >= 0)
55+
.NumberOf3xxResponses((x) => x == 0)
56+
.NumberOf4xxResponses((x) => x == 0)
57+
.NumberOf5xxResponses((x) => x == 0)
58+
.AverageLatency((x) => x >= 0)
59+
.AverageRequestsPerSecond((x) => x >= 0)
60+
.MaximumLatency((x) => x >= 0)
61+
.MaximumRequestsPerSecond((x) => x >= 0)
62+
.AssertAll();
63+
64+
foreach (var result in assertResults)
65+
{
66+
Assert.True(result.IsTrue, result.Message);
67+
}
68+
}
69+
70+
[IgnoreOnGithubFact]
71+
public async Task BombardierPostTestWithOptionsTest_Successfull()
72+
{
73+
var content = File.ReadAllText("Assets/addPet.json");
74+
var httpRequest = JsonConvert.DeserializeObject<IList<HttpRequest>>(content);
75+
76+
var bombardierTestsGenerator = new BombardierTestsGenerator(httpRequest, options =>
77+
{
78+
options.BombardierConcurrentUsers = 1;
79+
options.BombardierDuration = 1;
80+
options.BombardierTimeout = 30;
81+
options.BombardierUseHttp2 = true;
82+
options.BombardierNumberOfTotalRequests = 1;
83+
options.AddReplacementValues(new Dictionary<string, object> {
84+
{"id",1241451},
85+
{ "name","MJ"}
86+
});
87+
});
88+
89+
var bombardierTests = await bombardierTestsGenerator.Generate();
90+
91+
//Run Bombardier Tests
92+
var bombardierTestsRunner = new BombardierTestsRunner(bombardierTests.ToList(), options =>
93+
{
94+
options.ObfuscateAuthenticationHeader = true;
95+
});
96+
var bombardierResults = await bombardierTestsRunner.Run();
97+
98+
var asserter = new BombardierTestAsserter(bombardierResults.FirstOrDefault());
99+
var assertResults = asserter
100+
.NumberOf1xxResponses((x) => x == 0)
101+
.NumberOf2xxResponses((x) => x >= 0)
102+
.NumberOf3xxResponses((x) => x == 0)
103+
.NumberOf4xxResponses((x) => x == 0)
104+
.NumberOf5xxResponses((x) => x == 0)
105+
.AverageLatency((x) => x >= 0)
106+
.AverageRequestsPerSecond((x) => x >= 0)
107+
.MaximumLatency((x) => x >= 0)
108+
.MaximumRequestsPerSecond((x) => x >= 0)
109+
.AssertAll();
110+
111+
foreach (var result in assertResults)
112+
{
113+
Assert.True(result.IsTrue, result.Message);
114+
}
115+
}
116+
117+
[IgnoreOnGithubFact]
118+
public async Task BombardierPostTestWithBodyAndOptionsTest_Successfull()
119+
{
120+
var content = File.ReadAllText("Assets/AddBike.json");
121+
var httpRequest = JsonConvert.DeserializeObject<IList<HttpRequest>>(content);
122+
123+
var bombardierTestsGenerator = new BombardierTestsGenerator(httpRequest, options =>
124+
{
125+
options.BombardierConcurrentUsers = 1;
126+
options.BombardierDuration = 1;
127+
options.BombardierTimeout = 30;
128+
options.BombardierUseHttp2 = true;
129+
options.AddReplacementValues(new Dictionary<string, object> {
130+
{"Bicycle",@"{""id"":66,""name"":""my bike"",""brand"":""cannondale"",""BicycleType"":1}"}
131+
});
132+
});
133+
134+
var bombardierTests = await bombardierTestsGenerator.Generate();
135+
136+
//Run Bombardier Tests
137+
var bombardierTestsRunner = new BombardierTestsRunner(bombardierTests.ToList(), options =>
138+
{
139+
options.ObfuscateAuthenticationHeader = true;
140+
});
141+
var bombardierResults = await bombardierTestsRunner.Run();
142+
143+
_logger.LogInformation(JsonConvert.SerializeObject(bombardierResults, Formatting.Indented));
144+
145+
Assert.NotNull(bombardierResults);
146+
Assert.Single(bombardierResults);
147+
Assert.Equal(@"-m POST https://qatoolkitapi.azurewebsites.net/api/bicycles?api-version=1 -c 1 -H ""Content-Type: application/json"" -b ""{""id"":66,""name"":""my bike"",""brand"":""cannondale"",""BicycleType"":1}"" --http2 --timeout=30s --duration=1s", bombardierResults.FirstOrDefault().Command);
148+
149+
var asserter = new BombardierTestAsserter(bombardierResults.FirstOrDefault());
150+
var assertResults = asserter
151+
.NumberOf1xxResponses((x) => x == 0)
152+
.NumberOf2xxResponses((x) => x >= 0)
153+
.NumberOf3xxResponses((x) => x == 0)
154+
.NumberOf4xxResponses((x) => x == 0)
155+
.NumberOf5xxResponses((x) => x == 0)
156+
.AverageLatency((x) => x >= 0)
157+
.AverageRequestsPerSecond((x) => x >= 0)
158+
.MaximumLatency((x) => x >= 0)
159+
.MaximumRequestsPerSecond((x) => x >= 0)
160+
.AssertAll();
161+
162+
foreach (var result in assertResults)
163+
{
164+
Assert.True(result.IsTrue, result.Message);
165+
}
166+
}
167+
168+
[Fact]
169+
public void BombardierAsserterNullObject_Successfull()
170+
{
171+
Assert.Throws<ArgumentNullException>(() => new BombardierTestAsserter(null));
172+
}
173+
174+
[Fact]
175+
public void BombardierAsserterNullResults_Successfull()
176+
{
177+
var asserter = new BombardierTestAsserter(new BombardierResult());
178+
var assertResults = asserter
179+
.NumberOf1xxResponses((x) => x == 0)
180+
.NumberOf2xxResponses((x) => x >= 0)
181+
.NumberOf3xxResponses((x) => x == 0)
182+
.NumberOf4xxResponses((x) => x == 0)
183+
.NumberOf5xxResponses((x) => x == 0)
184+
.AverageLatency((x) => x >= 0)
185+
.AverageRequestsPerSecond((x) => x >= 0)
186+
.MaximumLatency((x) => x >= 0)
187+
.MaximumRequestsPerSecond((x) => x >= 0)
188+
.AssertAll();
189+
190+
foreach (var result in assertResults)
191+
{
192+
Assert.True(result.IsTrue, result.Message);
193+
}
194+
}
195+
}
196+
}

‎src/QAToolKit.Engine.Bombardier/BombardierGeneratorOptions.cs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ public class BombardierGeneratorOptions
7575
/// <returns></returns>
7676
public BombardierGeneratorOptions AddReplacementValues(Dictionary<string, object> replacementValues)
7777
{
78-
ReplacementValues = replacementValues ?? throw new ArgumentException(nameof(replacementValues));
78+
ReplacementValues = replacementValues ?? throw new ArgumentException($"{nameof(replacementValues)} is null.");
7979
return this;
8080
}
8181

@@ -88,9 +88,9 @@ public BombardierGeneratorOptions AddReplacementValues(Dictionary<string, object
8888
public BombardierGeneratorOptions AddOAuth2Token(string token, AuthenticationType authenticationType)
8989
{
9090
if (string.IsNullOrEmpty(token))
91-
throw new ArgumentException(nameof(token));
91+
throw new ArgumentException($"{nameof(token)} is null.");
9292
if (authenticationType == null)
93-
throw new ArgumentException(nameof(authenticationType));
93+
throw new ArgumentException($"{nameof(authenticationType)} is null.");
9494

9595
AccessTokens.Add(authenticationType, token);
9696
return this;
@@ -104,7 +104,7 @@ public BombardierGeneratorOptions AddOAuth2Token(string token, AuthenticationTyp
104104
public BombardierGeneratorOptions AddApiKey(string apiKey)
105105
{
106106
if (string.IsNullOrEmpty(apiKey))
107-
throw new ArgumentException(nameof(apiKey));
107+
throw new ArgumentException($"{nameof(apiKey)} is null.");
108108

109109
ApiKey = apiKey;
110110
return this;
@@ -119,9 +119,9 @@ public BombardierGeneratorOptions AddApiKey(string apiKey)
119119
public BombardierGeneratorOptions AddBasicAuthentication(string userName, string password)
120120
{
121121
if (string.IsNullOrEmpty(userName))
122-
throw new ArgumentException(nameof(userName));
122+
throw new ArgumentException($"{nameof(userName)} is null.");
123123
if (string.IsNullOrEmpty(password))
124-
throw new ArgumentException(nameof(password));
124+
throw new ArgumentException($"{nameof(password)} is null.");
125125

126126
UserName = userName;
127127
Password = password;

0 commit comments

Comments
 (0)