Skip to content

Commit 501e5fa

Browse files
jonsimantovjonsimantov
authored
Deflake a few flaky tests (#516)
Refactor a few tests to use [`RunFlakyBlock`](https://github.com/firebase/firebase-cpp-sdk/blob/acac90f7939f30a48c24ba5aa8d69fffe1ef6927/testing/test_framework/src/firebase_test_framework.h#L204): - FirebaseInstallationsTest.TestGettingTokenTwiceMatches (copied logic from TestGettingIdTwiceMatches) - FirebaseInstallationsTest.TestDeleteGivesNewTokenNextTime (copied logic from TestDeleteGivesNewIdNextTime) - FirebaseMessagingTest.TestSendMessageToTopic Additionally, to make the RunFlakyBlock tests more readable, this PR adds some FLAKY_EXPECT_* macros for equality and boolean values, as well as FLAKY_WAIT_FOR_COMPLETION. Refactored a Storage test to use these.
1 parent eca6ed1 commit 501e5fa

File tree

4 files changed

+178
-130
lines changed

4 files changed

+178
-130
lines changed

‎installations/integration_test/src/integration_test.cc‎

Lines changed: 52 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -147,30 +147,15 @@ TEST_F(FirebaseInstallationsTest, TestGettingIdTwiceMatches) {
147147
if (!RunFlakyBlock(
148148
[](firebase::installations::Installations* installations) {
149149
firebase::Future<std::string> id = installations->GetId();
150-
WaitForCompletionAnyResult(id, "GetId");
151-
if (id.error() != 0) {
152-
LogError("GetId returned error %d: %s", id.error(),
153-
id.error_message());
154-
return false;
155-
}
156-
if (*id.result() == "") {
157-
LogError("GetId returned blank");
158-
return false;
159-
}
150+
FLAKY_WAIT_FOR_COMPLETION(id, "GetId");
151+
FLAKY_EXPECT_NE(*id.result(), ""); // ensure non-blank
160152
std::string first_id = *id.result();
161153
id = installations->GetId();
162-
WaitForCompletionAnyResult(id, "GetId 2");
163-
if (id.error() != 0) {
164-
LogError("GetId 2 returned error %d: %s", id.error(),
165-
id.error_message());
166-
return false;
167-
}
168-
if (*id.result() != first_id) {
169-
LogError(
170-
"GetId 2 returned non-matching ID: first(%s) vs second(%s)",
171-
first_id.c_str(), id.result()->c_str());
172-
return false;
173-
}
154+
FLAKY_WAIT_FOR_COMPLETION(id, "GetId 2");
155+
FLAKY_EXPECT_NE(*id.result(), ""); // ensure non-blank
156+
157+
// Ensure the second ID returned is the same as the first.
158+
FLAKY_EXPECT_EQ(*id.result(), first_id);
174159
return true;
175160
},
176161
installations_)) {
@@ -182,47 +167,24 @@ TEST_F(FirebaseInstallationsTest, TestDeleteGivesNewIdNextTime) {
182167
if (!RunFlakyBlock(
183168
[](firebase::installations::Installations* installations) {
184169
firebase::Future<std::string> id = installations->GetId();
185-
WaitForCompletionAnyResult(id, "GetId");
186-
if (id.error() != 0) {
187-
LogError("GetId returned error %d: %s", id.error(),
188-
id.error_message());
189-
return false;
190-
}
191-
if (*id.result() == "") {
192-
LogError("GetId returned blank");
193-
return false;
194-
}
170+
FLAKY_WAIT_FOR_COMPLETION(id, "GetId");
171+
FLAKY_EXPECT_NE(*id.result(), ""); // ensure non-blank
195172
std::string first_id = *id.result();
196173

197174
firebase::Future<void> del = installations->Delete();
198-
WaitForCompletionAnyResult(del, "Delete");
199-
if (del.error() != 0) {
200-
LogError("Delete returned error %d: %s", id.error(),
201-
id.error_message());
202-
return false;
203-
}
175+
FLAKY_WAIT_FOR_COMPLETION(del, "Delete");
204176

205177
// Ensure that we now get a different installations id.
206178
id = installations->GetId();
207-
WaitForCompletionAnyResult(id, "GetId 2");
208-
if (id.error() != 0) {
209-
LogError("GetId 2 returned error %d: %s", id.error(),
210-
id.error_message());
211-
return false;
212-
}
213-
if (*id.result() == "") {
214-
LogError("GetId 2 returned blank");
215-
return false;
216-
}
179+
FLAKY_WAIT_FOR_COMPLETION(id, "GetId 2");
180+
FLAKY_EXPECT_NE(*id.result(), ""); // ensure non-blank
217181
#if defined(__ANDROID__) || (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE)
218182
// Desktop is a stub and returns the same ID, but on mobile it
219183
// should return a new ID.
220-
if (*id.result() == first_id) {
221-
LogError("IDs match (should be different): %s", first_id.c_str());
222-
return false;
223-
}
184+
FLAKY_EXPECT_NE(*id.result(), first_id);
224185
#endif // defined(__ANDROID__) || (defined(TARGET_OS_IPHONE) &&
225186
// TARGET_OS_IPHONE)
187+
226188
return true;
227189
},
228190
installations_)) {
@@ -237,34 +199,52 @@ TEST_F(FirebaseInstallationsTest, TestCanGetToken) {
237199
}
238200

239201
TEST_F(FirebaseInstallationsTest, TestGettingTokenTwiceMatches) {
240-
firebase::Future<std::string> token = installations_->GetToken(false);
241-
WaitForCompletion(token, "GetToken");
242-
EXPECT_NE(*token.result(), "");
243-
std::string first_token = *token.result();
244-
token = installations_->GetToken(false);
245-
WaitForCompletion(token, "GetToken 2");
246-
EXPECT_EQ(*token.result(), first_token);
202+
if (!RunFlakyBlock(
203+
[](firebase::installations::Installations* installations) {
204+
firebase::Future<std::string> token =
205+
installations->GetToken(false);
206+
FLAKY_WAIT_FOR_COMPLETION(token, "GetToken");
207+
FLAKY_EXPECT_NE(*token.result(), ""); // ensure non-blank
208+
std::string first_token = *token.result();
209+
token = installations->GetToken(false);
210+
FLAKY_WAIT_FOR_COMPLETION(token, "GetToken 2");
211+
FLAKY_EXPECT_NE(*token.result(), ""); // ensure non-blank
212+
FLAKY_EXPECT_EQ(*token.result(), first_token);
213+
return true;
214+
},
215+
installations_)) {
216+
FAIL() << "Test failed, check error log for details.";
217+
}
247218
}
248219

249220
TEST_F(FirebaseInstallationsTest, TestDeleteGivesNewTokenNextTime) {
250-
firebase::Future<std::string> token = installations_->GetToken(false);
251-
WaitForCompletion(token, "GetToken");
252-
EXPECT_NE(*token.result(), "");
253-
std::string first_token = *token.result();
221+
if (!RunFlakyBlock(
222+
[](firebase::installations::Installations* installations) {
223+
firebase::Future<std::string> token =
224+
installations->GetToken(false);
225+
FLAKY_WAIT_FOR_COMPLETION(token, "GetToken");
226+
FLAKY_EXPECT_NE(*token.result(), ""); // ensure non-blank
227+
std::string first_token = *token.result();
254228

255-
firebase::Future<void> del = installations_->Delete();
256-
WaitForCompletion(del, "Delete");
229+
firebase::Future<void> del = installations->Delete();
230+
FLAKY_WAIT_FOR_COMPLETION(del, "Delete");
257231

258-
// Ensure that we now get a different installations token.
259-
token = installations_->GetToken(false);
260-
WaitForCompletion(token, "GetToken 2");
261-
EXPECT_NE(*token.result(), "");
232+
// Ensure that we now get a different installations token.
233+
token = installations->GetToken(false);
234+
FLAKY_WAIT_FOR_COMPLETION(token, "GetToken 2");
235+
FLAKY_EXPECT_NE(*token.result(), ""); // ensure non-blank
262236
#if defined(__ANDROID__) || (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE)
263-
// Desktop is a stub and returns the same token, but on mobile it should
264-
// return a new token.
265-
EXPECT_NE(*token.result(), first_token);
237+
// Desktop is a stub and returns the same token, but on mobile it
238+
// should return a new token.
239+
FLAKY_EXPECT_NE(*token.result(), first_token);
266240
#endif // defined(__ANDROID__) || (defined(TARGET_OS_IPHONE) &&
267241
// TARGET_OS_IPHONE)
242+
243+
return true;
244+
},
245+
installations_)) {
246+
FAIL() << "Test failed, check error log for details.";
247+
}
268248
}
269249

270250
TEST_F(FirebaseInstallationsTest, TestCanGetIdAndTokenTogether) {

‎messaging/integration_test/src/integration_test.cc‎

Lines changed: 56 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,9 @@ const int kTimeoutSeconds = 120;
5555
const char kTestingNotificationKey[] = "fcm_testing_notification";
5656

5757
using app_framework::LogDebug;
58+
using app_framework::LogError;
5859
using app_framework::LogInfo;
60+
using app_framework::LogWarning;
5961

6062
using app_framework::GetCurrentTimeInMicroseconds;
6163
using app_framework::PathForResource;
@@ -90,7 +92,6 @@ class FirebaseMessagingTest : public FirebaseTest {
9092
const char* notification_body,
9193
const std::map<std::string, std::string>& message_fields);
9294

93-
protected:
9495
// Get a unique message ID so we can confirm the correct message is being
9596
// received.
9697
std::string GetUniqueMessageId();
@@ -106,6 +107,7 @@ class FirebaseMessagingTest : public FirebaseTest {
106107
bool WaitForMessage(firebase::messaging::Message* message_out,
107108
int timeout = kTimeoutSeconds);
108109

110+
protected:
109111
static firebase::App* shared_app_;
110112
static firebase::messaging::PollableListener* shared_listener_;
111113
static std::string* shared_token_;
@@ -218,6 +220,12 @@ bool FirebaseMessagingTest::CreateTestMessage(
218220
// Don't send HTTP requests in stub mode.
219221
return false;
220222
}
223+
if (strcmp(kFcmServerKey, "REPLACE_WITH_YOUR_SERVER_KEY") == 0) {
224+
LogWarning(
225+
"Please put your Firebase Cloud Messaging server key in "
226+
"kFcmServerKey.");
227+
LogWarning("Without a server key, most of these tests will fail.");
228+
}
221229
std::map<std::string, std::string> headers;
222230
headers.insert(std::make_pair("Content-type", "application/json"));
223231
headers.insert(
@@ -504,37 +512,54 @@ TEST_F(FirebaseMessagingTest, TestSendMessageToTopic) {
504512

505513
EXPECT_TRUE(RequestPermission());
506514
EXPECT_TRUE(WaitForToken());
507-
std::string unique_id = GetUniqueMessageId();
508-
const char kNotificationTitle[] = "Topic Test";
509-
const char kNotificationBody[] = "Topic Test notification body";
510-
// Create a somewhat unique topic name using 2 digits near the end of
511-
// unique_id (but not the LAST 2 digits, due to timestamp resolution on some
512-
// platforms).
513-
std::string unique_id_tag =
514-
(unique_id.length() >= 7 ? unique_id.substr(unique_id.length() - 5, 2)
515-
: "00");
516-
std::string topic = "FCMTestTopic" + unique_id_tag;
517-
EXPECT_TRUE(WaitForCompletion(firebase::messaging::Subscribe(topic.c_str()),
518-
"Subscribe"));
519-
SendTestMessage(("/topics/" + topic).c_str(), kNotificationTitle,
520-
kNotificationBody,
521-
{{"message", "Hello, world!"}, {"unique_id", unique_id}});
522-
firebase::messaging::Message message;
523-
EXPECT_TRUE(WaitForMessage(&message));
524-
EXPECT_EQ(message.data["unique_id"], unique_id);
525-
if (message.notification) {
526-
EXPECT_EQ(message.notification->title, kNotificationTitle);
527-
EXPECT_EQ(message.notification->body, kNotificationBody);
528-
}
529-
530-
EXPECT_TRUE(WaitForCompletion(firebase::messaging::Unsubscribe(topic.c_str()),
531-
"Unsubscribe"));
532515

533-
// Ensure that we *don't* receive a message now.
534-
unique_id = GetUniqueMessageId();
535-
SendTestMessage(("/topics/" + topic).c_str(), "Topic Title 2", "Topic Body 2",
536-
{{"message", "Hello, world!"}, {"unique_id", unique_id}});
537-
EXPECT_FALSE(WaitForMessage(&message, 5));
516+
if (!RunFlakyBlock(
517+
[](FirebaseMessagingTest* this_) {
518+
std::string unique_id = this_->GetUniqueMessageId();
519+
const char kNotificationTitle[] = "Topic Test";
520+
const char kNotificationBody[] = "Topic Test notification body";
521+
// Create a somewhat unique topic name using 2 digits near the end
522+
// of unique_id (but not the LAST 2 digits, due to timestamp
523+
// resolution on some platforms).
524+
std::string unique_id_tag =
525+
(unique_id.length() >= 7
526+
? unique_id.substr(unique_id.length() - 5, 2)
527+
: "00");
528+
std::string topic = "FCMTestTopic" + unique_id_tag;
529+
firebase::Future<void> sub =
530+
firebase::messaging::Subscribe(topic.c_str());
531+
FLAKY_WAIT_FOR_COMPLETION(sub, "Subscribe");
532+
this_->SendTestMessage(
533+
("/topics/" + topic).c_str(), kNotificationTitle,
534+
kNotificationBody,
535+
{{"message", "Hello, world!"}, {"unique_id", unique_id}});
536+
firebase::messaging::Message message;
537+
FLAKY_EXPECT_TRUE(this_->WaitForMessage(&message));
538+
539+
FLAKY_EXPECT_EQ(message.data["unique_id"], unique_id);
540+
if (message.notification) {
541+
FLAKY_EXPECT_EQ(message.notification->title, kNotificationTitle);
542+
FLAKY_EXPECT_EQ(message.notification->body, kNotificationBody);
543+
}
544+
firebase::Future<void> unsub =
545+
firebase::messaging::Unsubscribe(topic.c_str());
546+
FLAKY_WAIT_FOR_COMPLETION(unsub, "Unsubscribe");
547+
548+
// Ensure that we *don't* receive a message now.
549+
unique_id = this_->GetUniqueMessageId();
550+
this_->SendTestMessage(
551+
("/topics/" + topic).c_str(), "Topic Title 2", "Topic Body 2",
552+
{{"message", "Hello, world!"}, {"unique_id", unique_id}});
553+
554+
// If this returns true, it means we received a message but
555+
// shouldn't have.
556+
FLAKY_EXPECT_FALSE(this_->WaitForMessage(&message, 5));
557+
558+
return true;
559+
},
560+
this)) {
561+
FAIL() << "Test failed, check error log for details.";
562+
}
538563
}
539564

540565
TEST_F(FirebaseMessagingTest, TestChangingListener) {

‎storage/integration_test/src/integration_test.cc‎

Lines changed: 7 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -711,10 +711,7 @@ TEST_F(FirebaseStorageTest, TestLargeFilePauseResumeAndDownloadCancel) {
711711

712712
// Ensure the Controller is valid now that we have associated it
713713
// with an operation.
714-
if (!controller.is_valid()) {
715-
LogError("Controller is invalid");
716-
return false;
717-
}
714+
FLAKY_EXPECT_TRUE(controller.is_valid());
718715

719716
while (controller.bytes_transferred() == 0) {
720717
#if FIREBASE_PLATFORM_DESKTOP
@@ -740,34 +737,17 @@ TEST_F(FirebaseStorageTest, TestLargeFilePauseResumeAndDownloadCancel) {
740737
// The StorageListener's OnPaused will call Resume().
741738

742739
LogDebug("Waiting for future.");
743-
WaitForCompletionAnyResult(future, "WriteLargeFile");
744-
if (future.error() != firebase::storage::kErrorNone) {
745-
LogError("PutBytes returned error %d: %s", future.error(),
746-
future.error_message());
747-
return false;
748-
}
740+
FLAKY_WAIT_FOR_COMPLETION(future, "WriteLargeFile");
749741
LogDebug("Upload complete.");
750742

751743
// Ensure the various callbacks were called.
752-
if (!listener.on_paused_was_called()) {
753-
LogError("Listener::OnPaused was not called");
754-
return false;
755-
}
756-
if (!listener.on_progress_was_called()) {
757-
LogError("Listener::OnProgress was not called");
758-
return false;
759-
}
760-
if (!listener.resume_succeeded()) {
761-
LogError("Resume failed");
762-
return false;
763-
}
744+
FLAKY_EXPECT_TRUE(listener.on_paused_was_called());
745+
FLAKY_EXPECT_TRUE(listener.on_progress_was_called());
746+
FLAKY_EXPECT_TRUE(listener.resume_succeeded());
764747

765748
auto metadata = future.result();
766-
if (metadata->size_bytes() != test_file_size) {
767-
LogError(
768-
"Metadata reports incorrect size, file failed to upload.");
769-
return false;
770-
}
749+
// If metadata reports incorrect size, file failed to upload.
750+
FLAKY_EXPECT_EQ(metadata->size_bytes(), test_file_size);
771751
return true;
772752
},
773753
&context, "PutBytes")) {

0 commit comments

Comments
 (0)