add tc, fix key, fix gha flow (#920)

Author: ben-c-at-moz

.github/workflows/check-beta-functional.yml

@@ -2,7 +2,7 @@ name: Check new beta version - Functional
 
 on:
   schedule:
-    - cron: "55 1/5 * * *"
+    - cron: "55 1/5 1 * *"
 env:
   LATEST: ""
 permissions:

.github/workflows/main.yml

@@ -57,29 +57,22 @@ jobs:
       - name: Set Environment (Manual)
         if: ${{ inputs.win_installer_link }}
         run: |
-          echo "MANUAL=true" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+          echo "MANUAL=true" | Out-File -FilePath .env -Encoding utf8 -Append -Force
           Write-Host "Running smoke tests on supplied executable"
       - name: Set Environment (Scheduled Beta)
         if: ${{ inputs.job_to_run == 'Test-Windows' && inputs.is_pull_request == false}}
         run: |
-          echo "TESTRAIL_REPORT=true" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+          echo "TESTRAIL_REPORT=true" | Out-File -FilePath .env -Encoding utf8 -Append -Force
           Write-Host "Running report for most recent Beta on Windows"
-          Write-Host "$env:TESTRAIL_REPORT"
       - name: Select Test Set
         if: ${{ inputs.test_set != null }}
         run: |
-          echo "STARFOX_MANIFEST=.\manifests\${{ inputs.test_set }}.yaml" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
-          Write-Host "$env:TESTRAIL_REPORT"
+          echo "STARFOX_MANIFEST=.\manifests\${{ inputs.test_set }}.yaml" | Out-File -FilePath .env -Encoding utf8 -Append -Force
         shell: pwsh
       - name: Set Functional Split
         if: ${{ inputs.test_set == 'functional' }}
         run: |
-          $env_contents = @"
-          TESTRAIL_FUNCTIONAL_SPLIT=1
-          "@
-          New-Item -Name .env -Value $env_contents -ItemType File -Force
-          Write-Host "$env:TESTRAIL_REPORT"
-          Write-Host "$env:STARFOX_MANIFEST"
+          echo "TESTRAIL_FUNCTIONAL_SPLIT=1" | Out-File -FilePath .env -Encoding utf8 -Append -Force
       - name: Install dependencies
         run: |
           mkdir -p artifacts;
@@ -107,10 +100,7 @@ jobs:
       - name: Download Executable
         if: ${{ inputs.win_installer_link }}
         run: |
-          $env_contents = @"
-          STARFOX_MANIFEST='.\manifests\incident.yaml'
-          "@
-          New-Item -Name .env -Value $env_contents -ItemType File -Force
+          echo "STARFOX_MANIFEST='.\manifests\incident.yaml'" | Out-File -FilePath .env -Encoding utf8 -Append -Force
           Invoke-WebRequest -Uri ${{ inputs.win_installer_link }} -OutFile "${{ github.workspace }}\setup.exe"
           New-Item -ItemType Directory -Path "C:\Program Files\Custom Firefox" -Force
         shell: pwsh
@@ -181,15 +171,15 @@ jobs:
           echo "Running report for most recent Beta on MacOS";
       - name: Select Test Set
         if: ${{ inputs.test_set != null }}
-        run: echo "STARFOX_MANIFEST=./manifests/${{ inputs.test_set }}.yaml" >> "$GITHUB_ENV"
+        run: echo "STARFOX_MANIFEST=./manifests/${{ inputs.test_set }}.yaml" >> .env
       - name: Set Functional Split
         if: ${{ inputs.test_set == 'functional' }}
-        run: echo "TESTRAIL_FUNCTIONAL_SPLIT=1" >> "$GITHUB_ENV"
+        run: echo "TESTRAIL_FUNCTIONAL_SPLIT=1" >> .env
       - name: Set Environment (Manual)
         if: ${{ inputs.mac_installer_link }}
         run: |
           echo "MANUAL='true'" >> "$GITHUB_ENV";
-          echo "STARFOX_MANIFEST=manifests/incident.yaml" >> "$GITHUB_ENV"
+          echo "STARFOX_MANIFEST=manifests/incident.yaml" >> .env
           echo "Running smoke tests on supplied executable";
       - name: Install dependencies
         run: |

choose_ci_set.py

@@ -116,9 +116,13 @@ def is_addable(test_name, result):
     test_name = test_name.replace(".py", "")
     if "::" in test_name:
         test, subtest = test_name.split("::")
+        if test not in pointer:
+            raise ValueError(f"{test} not in key.yaml")
         pointer = pointer[test]
         pointer = pointer[subtest]
     else:
+        if test_name not in pointer:
+            raise ValueError(f"{test_name} not in key.yaml")
         pointer = pointer[test_name]
 
     if pointer == "pass":

manifests/key.yaml

@@ -8,9 +8,11 @@ address_bar_and_search:
   test_addon_suggestion: unstable
   test_addressbar_search_engine_keywords: pass
   test_copied_url_contains_https: pass
+  test_ctrl_enter_completes_link_and_can_refresh: pass
   test_ctrl_enter_fixes_url: pass
   test_default_search_provider_change_awesome_bar: unstable
   test_default_search_provider_change_legacy_search_bar: deprecated
+  test_disable_websearch_from_awesome_bar: pass
   test_dont_show_search_suggestions_in_private_window: pass
   test_google_search_counts_us:
     linux: pass
@@ -20,13 +22,15 @@ address_bar_and_search:
   test_insertion_point_no_search_terms_display: pass
   test_no_suggestions_for_empty_query: pass
   test_non_sponsored_topsite_context_menu_option: pass
+  test_paste_and_go_opens_correct_url: pass
   test_refresh_firefox_dialog: pass
   test_sap_google_adclick: unstable
   test_seach_suggestions_can_be_disabled: pass
   test_search_code_google_non_us: pass
   test_search_code_google_us: pass
   test_search_engine_selector: pass
   test_search_mode_appears_after_input: pass
+  test_search_mode_exits_correctly: pass
   test_search_mode_persists_mixed_with_bing: pass
   test_search_modes_for_sites: unstable
   test_search_string_displayed_when_addressbar_unfocused: pass
@@ -36,10 +40,13 @@ address_bar_and_search:
   test_searchbar_on_engines_remove_restore: pass
   test_searchbar_results_shown_in_a_new_tab: pass
   test_searchmode_change_tab: pass
+  test_searchmode_cleared_on_engine_removal: pass
   test_server_not_found_error: pass
   test_server_not_found_error_pb: pass
   test_suggestion_engine_selection: pass
   test_tile_menu_options: pass
+  test_use_different_search_shortcut_while_already_in_search_mode: pass
+  test_verify_url_after_tab_switch_with_search_mode: pass
 audio_video:
   test_allow_audio_video_functionality:
     linux: unstable

taskcluster/fx_desktop_qa_automation_taskgraph/target_tasks.py

@@ -10,6 +10,15 @@ def filter(task, parameters):
 
     return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)]
 
+@register_target_task("new_beta_func")
+def target_tasks_beta_func(full_task_graph, parameters, graph_config):
+    """Select the set of tasks required for a Beta Functional + Reporting session."""
+
+    def filter(task, parameters):
+        return task.attributes.get("beta-func", False)
+
+    return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)]
+
 @register_target_task("new_devedition_qa")
 def target_tasks_devedition_qa(full_task_graph, parameters, graph_config):
     """Select the set of tasks required for a DevEdition Smoke + Reporting session."""

taskcluster/kinds/new-beta-func/kind.yml

@@ -0,0 +1,114 @@
+transforms:
+  - fx_desktop_qa_automation_taskgraph.transforms.new_beta_qa:transforms
+  - taskgraph.transforms.notify:transforms
+
+task-defaults:
+  label: "Test New Beta Version - Functional"
+  description: "Checks for new betas, runs functional tests, reports to TestRail and Slack"
+  worker-type: t-linux-wayland
+  worker:
+    taskcluster-proxy: true
+    max-run-time: 3600
+    artifacts:
+      - name: public/results
+        path: checkouts/vcs/artifacts
+        type: directory
+    env:
+      TESTRAIL_REPORT: "true"
+      STARFOX_MANIFEST: "./manifests/functional.yaml"
+      TESTRAIL_FUNCTIONAL_SPLIT: 1
+  routes:
+    - notify.slack-channel.C07AHPJ525V.on-resolved
+  scopes:
+    - secrets:get:project/mozilla/fx-desktop-qa-automation/level-3/*
+    - queue:route:notify.slack-channel.C07AHPJ525V # notify mobile-alerts-sandbox on failure
+    - notify:slack-channel:C07AHPJ525V
+
+tasks:
+  linux:
+    attributes:
+      beta-func: true
+    run-on-tasks-for: []
+    run:
+      using: run-task
+      cwd: "{checkout}"
+      command: |-
+        export PATH=$HOME/.local/bin:$PWD:$PATH
+        mkdir -p artifacts;
+        pip3 install 'pipenv==2023.11.15';
+        pip3 install 'ruff>=0.4.8,<0.5';
+        mv ./ci_pyproject.toml ./pyproject.toml;
+        pipenv install;
+        export FX_CHANNEL="beta";
+        ./collect_executables.sh;
+        export FX_EXECUTABLE=./firefox/firefox;
+        pipenv run python -c 'from modules import testrail_integration as tri; tri.tc_reportable()' || exit 0;
+        $FX_EXECUTABLE --version;
+        . ./keyring-unlock.sh
+        pipenv run python choose_ci_set.py;
+        pipenv run pytest --fx-executable $FX_EXECUTABLE $(cat selected_tests);
+        export FAILURE=${?};
+        mv ./ci_pyproject_headed.toml ./pyproject.toml;
+        export REPORTABLE=1;
+        pipenv run pytest --fx-executable $FX_EXECUTABLE $(cat selected_tests);
+        exit $((${?} | ${FAILURE}))
+    notify:
+      recipients:
+        - type: slack-channel
+          channel-id: C07AHPJ525V
+          status-type: on-defined
+      content:
+        slack:
+          blocks: [
+            {
+              "type": "header",
+              "text": {
+                "type": "plain_text",
+                "text": "firefox-desktop :firefox: ${task.metadata.name} Linux\n "
+              }
+            },
+            {
+              "type": "divider"
+            },
+            {
+              "type": "section",
+              "text": {
+                "type": "mrkdwn",
+                "text": "*Task*: <https://firefox-ci-tc.services.mozilla.com/tasks/${status.taskId}|Taskcluster>"
+              }
+            },
+            {
+              "type": "section",
+              "text": {
+                "type": "mrkdwn",
+                "text": "*Owner*: ${task.metadata.owner}"
+              }
+            },
+            {
+              "type": "section",
+              "text": {
+                "type": "mrkdwn",
+                "text": "*Commit*: <${task.metadata.source}>"
+              }
+            },
+            {
+              "type": "section",
+              "text": {
+                "type": "mrkdwn",
+                "text": "*Test Summary*: <https://firefoxci.taskcluster-artifacts.net/${status.taskId}/0/public/results/report.html?sort=result> :debug:"
+              }
+            },
+            {
+              "type": "divider"
+            },
+            {
+              "type": "context",
+              "elements": [
+                {
+                  "type": "mrkdwn",
+                  "text": ":testops-notify: created by Desktop QA Test Engineering"
+                }
+              ]
+            }
+          ]
+          text: "Cron {task[name]} with id $taskId has finished!"