Detox/E2E: Modify reporting according to sharding (#6663)

detox/e2e/config.js

@@ -2,6 +2,7 @@
 // See LICENSE.txt for license information.
 
 const platform = process.env.IOS === 'true' ? 'ios' : 'android';
+const shard = process.env.CI_NODE_INDEX ? process.env.CI_NODE_INDEX : '';
 
 module.exports = {
     setupFilesAfterEnv: ['./test/setup.ts'],
@@ -19,15 +20,21 @@ module.exports = {
         ['jest-junit', {
             suiteName: 'Mobile App E2E with Detox and Jest',
             outputDirectory: './artifacts',
-            outputName: `${platform}-junit.xml`,
+            outputName: `${platform}-junit${shard}.xml`,
             uniqueOutputName: false,
         }],
         ['jest-html-reporters', {
             pageTitle: 'Mobile App E2E with Detox and Jest',
             publicPath: './artifacts',
-            filename: `${platform}-report.html`,
+            filename: `${platform}-report${shard}.html`,
             expand: false,
         }],
+        ['jest-stare', {
+            reportHeadline: 'Mobile App E2E with Detox and Jest',
+            resultDir: './artifacts/jest-stare',
+            resultJson: `${platform}-data${shard}.json`,
+            resultHtml: `${platform}-main${shard}.html`,
+        }],
     ],
     verbose: true,
     moduleNameMapper: {

detox/e2e/test/autocomplete/create_channel.e2e.ts

@@ -61,7 +61,7 @@ describe('Autocomplete - Create Channel', () => {
         await expect(Autocomplete.sectionAtMentionList).toBeVisible();
     });
 
-    it('MM-T4904_2 - should render channel mention autocomplete in header input -- KNOWN ISSUE: MM-46650', async () => {
+    it('MM-T4904_2 - should render channel mention autocomplete in header input', async () => {
         // * Verify channel mention list is not displayed
         await expect(Autocomplete.sectionChannelMentionList).not.toBeVisible();
 

detox/package.json

@@ -26,6 +26,8 @@
     "jest-cli": "29.1.2",
     "jest-html-reporters": "3.0.11",
     "jest-junit": "14.0.1",
+    "jest-stare": "2.4.1",
+    "junit-report-merger": "4.0.0",
     "moment-timezone": "0.5.37",
     "recursive-readdir": "2.2.2",
     "sanitize-filename": "1.6.3",

detox/save_report.js

@@ -32,12 +32,18 @@
 
 const assert = require('assert');
 const os = require('os');
+const path = require('path');
 
 const fse = require('fs-extra');
+const {mergeFiles} = require('junit-report-merger');
 const shell = require('shelljs');
 
 const {saveArtifacts} = require('./utils/artifacts');
 const {ARTIFACTS_DIR} = require('./utils/constants');
+const {
+    generateJestStareHtmlReport,
+    mergeJestStareJsonFiles,
+} = require('./utils/jest_stare');
 const {
     convertXmlToJson,
     generateShortSummary,
@@ -88,9 +94,14 @@ const saveReport = async () => {
     };
     writeJsonToFile(environmentDetails, 'environment.json', ARTIFACTS_DIR);
 
-    // Read XML from a file
+    // Merge all XML reports into one single XML report
     const platform = process.env.IOS === 'true' ? 'ios' : 'android';
-    const xml = fse.readFileSync(`${ARTIFACTS_DIR}/${platform}-junit.xml`);
+    const combinedFilePath = `${ARTIFACTS_DIR}/${platform}-combined.xml`;
+    await mergeFiles(path.join(__dirname, combinedFilePath), [`${ARTIFACTS_DIR}/${platform}-junit*.xml`]);
+    console.log(`Merged, check ${combinedFilePath}`);
+
+    // Read XML from a file
+    const xml = fse.readFileSync(combinedFilePath);
     const {testsuites} = convertXmlToJson(xml);
 
     // Generate short summary, write to file and then send report via webhook
@@ -99,6 +110,12 @@ const saveReport = async () => {
     console.log(summary);
     writeJsonToFile(summary, 'summary.json', ARTIFACTS_DIR);
 
+    // Generate jest-stare report
+    const jestStareOutputDir = path.join(__dirname, `${ARTIFACTS_DIR}/jest-stare`);
+    const jestStareCombinedFilePath = `${jestStareOutputDir}/${platform}-combined.json`;
+    await mergeJestStareJsonFiles(jestStareCombinedFilePath, [`${ARTIFACTS_DIR}/jest-stare/${platform}-data*.json`]);
+    generateJestStareHtmlReport(jestStareOutputDir, `${platform}-report.html`, jestStareCombinedFilePath);
+
     const result = await saveArtifacts();
     if (result && result.success) {
         console.log('Successfully uploaded artifacts to S3:', result.reportLink);

detox/utils/artifacts.js

@@ -79,7 +79,7 @@ async function saveArtifacts() {
                     return reject(new Error(err));
                 }
 
-                const reportLink = `https://${DETOX_AWS_S3_BUCKET}.s3.amazonaws.com/${s3Folder}/${platform}-report.html`;
+                const reportLink = `https://${DETOX_AWS_S3_BUCKET}.s3.amazonaws.com/${s3Folder}/jest-stare/${platform}-report.html`;
                 resolve({success: true, reportLink});
             },
         );

detox/utils/jest_stare.js

@@ -0,0 +1,148 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+/* eslint-disable no-console */
+
+const fse = require('fs-extra');
+const glob = require('glob');
+const processor = require('jest-stare');
+
+function flatten(items) {
+    return items.reduce((acc, arr) => {
+        return [...acc, ...arr];
+    }, []);
+}
+
+function flatMap(fn) {
+    return (items) => {
+        return flatten(items.map(fn));
+    };
+}
+
+const collectSourceFiles = flatMap((pattern) => {
+    const files = glob.sync(pattern);
+    if (!files.length) {
+        throw new Error(`Pattern ${pattern} matched no report files`);
+    }
+    return files;
+});
+
+function collectReportFiles(files) {
+    return Promise.all(files.map((filename) => {
+        return fse.readJson(filename);
+    }));
+}
+
+function collectReportSuites(reports) {
+    let numFailedTestSuites = 0;
+    let numFailedTests = 0;
+    let numPassedTestSuites = 0;
+    let numPassedTests = 0;
+    let numPendingTestSuites = 0;
+    let numPendingTests = 0;
+    let numRuntimeErrorTestSuites = 0;
+    let numTodoTests = 0;
+    let numTotalTestSuites = 0;
+    let numTotalTests = 0;
+    let openHandles = [];
+    const snapshot = {
+        added: 0,
+        didUpdate: false,
+        failure: false,
+        filesAdded: 0,
+        filesRemoved: 0,
+        filesRemovedList: [],
+        filesUnmatched: 0,
+        filesUpdated: 0,
+        matched: 0,
+        total: 0,
+        unchecked: 0,
+        uncheckedKeysByFile: [],
+        unmatched: 0,
+        updated: 0,
+    };
+    let startTime = 0;
+    let success = true;
+    let testResults = [];
+    let wasInterrupted = false;
+    reports.forEach((report) => {
+        numFailedTestSuites += report.numFailedTestSuites;
+        numFailedTests += report.numFailedTests;
+        numPassedTestSuites += report.numPassedTestSuites;
+        numPassedTests += report.numPassedTests;
+        numPendingTestSuites += report.numPendingTestSuites;
+        numPendingTests += report.numPendingTests;
+        numRuntimeErrorTestSuites += report.numRuntimeErrorTestSuites;
+        numTodoTests += report.numTodoTests;
+        numTotalTestSuites += report.numTotalTestSuites;
+        numTotalTests += report.numTotalTests;
+        openHandles = openHandles.concat(report.openHandles);
+        snapshot.added += report.snapshot.added;
+        if (report.snapshot.didUpdate === true) {
+            snapshot.didUpdate = true;
+        }
+        if (report.snapshot.failure === true) {
+            snapshot.failure = true;
+        }
+        snapshot.filesAdded += report.snapshot.filesAdded;
+        snapshot.filesRemoved += report.snapshot.filesRemoved;
+        snapshot.filesRemovedList = snapshot.filesRemovedList.concat(report.snapshot.filesRemovedList);
+        snapshot.filesUnmatched += report.snapshot.filesUnmatched;
+        snapshot.filesUpdated += report.snapshot.filesUpdated;
+        snapshot.matched += report.snapshot.matched;
+        snapshot.total += report.snapshot.total;
+        snapshot.unchecked += report.snapshot.unchecked;
+        snapshot.uncheckedKeysByFile = snapshot.uncheckedKeysByFile.concat(report.snapshot.uncheckedKeysByFile);
+        snapshot.unmatched += report.snapshot.unmatched;
+        snapshot.updated += report.snapshot.updated;
+        if (startTime === 0) {
+            startTime = report.startTime;
+        } else if (report.startTime < startTime) {
+            startTime = report.startTime;
+        }
+        if (report.success === false) {
+            success = false;
+        }
+        testResults = testResults.concat(report.testResults);
+        if (report.wasInterrupted === true) {
+            wasInterrupted = true;
+        }
+    });
+
+    return {
+        numFailedTestSuites,
+        numFailedTests,
+        numPassedTestSuites,
+        numPassedTests,
+        numPendingTestSuites,
+        numPendingTests,
+        numRuntimeErrorTestSuites,
+        numTodoTests,
+        numTotalTestSuites,
+        numTotalTests,
+        openHandles,
+        snapshot,
+        startTime,
+        success,
+        testResults,
+        wasInterrupted,
+    };
+}
+
+function generateJestStareHtmlReport(outputDir, outputFile, inputFilePath) {
+    const suites = fse.readJsonSync(inputFilePath);
+    processor(suites, {log: false, resultDir: outputDir, resultHtml: outputFile, reportHeadline: 'Mobile App E2E with Detox and Jest'});
+}
+
+async function mergeJestStareJsonFiles(outputFilePath, inputFiles) {
+    const files = collectSourceFiles(inputFiles);
+    const reports = await collectReportFiles(files);
+    const suites = collectReportSuites(reports);
+    fse.writeJsonSync(outputFilePath, suites);
+    console.log('Successfully written:', outputFilePath);
+}
+
+module.exports = {
+    generateJestStareHtmlReport,
+    mergeJestStareJsonFiles,
+};

detox/utils/report.js

@@ -35,10 +35,16 @@ function getAllTests(testSuites) {
     const suites = [];
     const tests = [];
     let skipped = 0;
+    let failures = 0;
+    let errors = 0;
+    let duration = 0;
     let firstTimestamp;
     let incrementalDuration = 0;
     testSuites.testsuite.forEach((testSuite) => {
         skipped += parseInt(testSuite.skipped[0], 10);
+        failures += parseInt(testSuite.failures[0], 10);
+        errors += parseInt(testSuite.errors[0], 10);
+        duration += parseFloat(testSuite.time[0] * 1000);
         if (!firstTimestamp) {
             firstTimestamp = testSuite.timestamp[0];
         }
@@ -83,16 +89,16 @@ function getAllTests(testSuites) {
     });
     const startDate = new Date(firstTimestamp);
     const start = startDate.toISOString();
-    startDate.setTime(startDate.getTime() + parseFloat(testSuites.time[0] * 1000));
+    startDate.setTime(startDate.getTime() + duration);
     const end = startDate.toISOString();
 
     return {
         suites,
         tests,
         skipped,
-        failures: parseInt(testSuites.failures[0], 10),
-        errors: parseInt(testSuites.errors[0], 10),
-        duration: parseFloat(testSuites.time[0] * 1000),
+        failures,
+        errors,
+        duration,
         start,
         end,
     };

detox/utils/test_cases.js

@@ -114,9 +114,14 @@ async function createTestExecutions(allTests, testCycle) {
     const promises = [];
     Object.entries(testCases).forEach(([key, steps], index) => {
         const testScriptResults = steps.
-            sort((a, b) => a.title.localeCompare(b.title)).
+            sort((a, b) => {
+                const aKey = a.title.match(/(MM-T\d+_\d+)/)[0].split('_')[1];
+                const bKey = b.title.match(/(MM-T\d+_\d+)/)[0].split('_')[1];
+                return parseInt(aKey, 10) - parseInt(bKey, 10);
+            }).
             map((item) => {
                 return {
+                    title: item.title,
                     statusName: status[item.state],
                     actualEndDate: new Date(startTime + item.incrementalDuration).toISOString(),
                     actualResult: 'Detox automated test completed',