forked from Ivasoft/mattermost-mobile
Detox/E2E: Scripts to run tests and save report (#6560)
* Detox/E2E: Scripts to run tests and save report * Change TM4J to ZEPHYR * Change AWS to DETOX_AWS * Removed send report on type release; moved incrementalDuration to getAllTests * Apply change requests * Fix import order * Fixed save_report comments; Fixed IOS checks * Added TEST_CYCLE_LINK_PREFIX to save_report comments * Re-order variables
This commit is contained in:
Joseph Baylon
@@ -1,7 +1,7 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
const platform = process.env.IOS ? 'ios' : 'android';
|
||||
const platform = process.env.IOS === 'true' ? 'ios' : 'android';
|
||||
|
||||
module.exports = {
|
||||
setupFilesAfterEnv: ['./test/setup.ts'],
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
export const serverOneUrl = process.env.SITE_1_URL || (process.env.IOS ? 'http://127.0.0.1:8065' : 'http://10.0.2.2:8065');
|
||||
export const serverOneUrl = process.env.SITE_1_URL || (process.env.IOS === 'true' ? 'http://127.0.0.1:8065' : 'http://10.0.2.2:8065');
|
||||
export const siteOneUrl = process.env.SITE_1_URL || 'http://127.0.0.1:8065';
|
||||
export const serverTwoUrl = process.env.SITE_2_URL || 'https://mobile02.test.mattermost.cloud';
|
||||
export const siteTwoUrl = process.env.SITE_2_URL || 'https://mobile02.test.mattermost.cloud';
|
||||
|
||||
1307
detox/package-lock.json
generated
1307
detox/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -11,6 +11,7 @@
|
||||
"@types/jest": "28.1.6",
|
||||
"@types/tough-cookie": "4.0.2",
|
||||
"@types/uuid": "8.3.4",
|
||||
"aws-sdk": "2.1189.0",
|
||||
"axios": "0.27.2",
|
||||
"axios-cookiejar-support": "4.0.3",
|
||||
"babel-jest": "28.1.3",
|
||||
@@ -25,12 +26,15 @@
|
||||
"jest-html-reporters": "3.0.10",
|
||||
"jest-junit": "14.0.0",
|
||||
"moment-timezone": "0.5.34",
|
||||
"recursive-readdir": "2.2.2",
|
||||
"sanitize-filename": "1.6.3",
|
||||
"shelljs": "0.8.5",
|
||||
"tough-cookie": "4.0.0",
|
||||
"ts-jest": "28.0.7",
|
||||
"tslib": "2.4.0",
|
||||
"typescript": "4.7.4",
|
||||
"uuid": "8.3.2"
|
||||
"uuid": "8.3.2",
|
||||
"xml2js": "0.4.23"
|
||||
},
|
||||
"scripts": {
|
||||
"e2e:android-create-emulator": "./create_android_emulator.sh",
|
||||
|
||||
129
detox/save_report.js
Normal file
129
detox/save_report.js
Normal file
@@ -0,0 +1,129 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
/* eslint-disable no-console, no-process-env */
|
||||
|
||||
/*
|
||||
* This is used for saving artifacts to AWS S3, sending data to automation dashboard and
|
||||
* publishing quick summary to community channels.
|
||||
*
|
||||
* Usage: [ENV] node save_report.js
|
||||
*
|
||||
* Environment variables:
|
||||
* BRANCH=[branch] : Branch identifier from CI
|
||||
* BUILD_ID=[build_id] : Build identifier from CI
|
||||
* DEVICE_NAME=[device_name] : Name of the device used for testing
|
||||
* DEVICE_OS_NAME=[device_os_name] : OS of the device used for testing
|
||||
* HEADLESS=[boolean] : Headed by default (false) or headless (true)
|
||||
* IOS=[boolean] : Android by default (false) or iOS (true)
|
||||
*
|
||||
* For saving artifacts to AWS S3
|
||||
* - DETOX_AWS_S3_BUCKET, DETOX_AWS_ACCESS_KEY_ID and DETOX_AWS_SECRET_ACCESS_KEY
|
||||
* For saving test cases to Test Management
|
||||
* - ZEPHYR_ENABLE=true|false
|
||||
* - ZEPHYR_API_KEY=[api_key]
|
||||
* - JIRA_PROJECT_KEY=[project_key], e.g. "MM",
|
||||
* - ZEPHYR_FOLDER_ID=[folder_id], e.g. 847997
|
||||
* For sending hooks to Mattermost channels
|
||||
* - FULL_REPORT, WEBHOOK_URL and TEST_CYCLE_LINK_PREFIX
|
||||
* Test type
|
||||
* - TYPE=[type], e.g. "MASTER", "PR", "RELEASE", "GEKIDOU"
|
||||
*/
|
||||
|
||||
const assert = require('assert');
|
||||
const os = require('os');
|
||||
|
||||
const fse = require('fs-extra');
|
||||
const shell = require('shelljs');
|
||||
|
||||
const {saveArtifacts} = require('./utils/artifacts');
|
||||
const {ARTIFACTS_DIR} = require('./utils/constants');
|
||||
const {
|
||||
convertXmlToJson,
|
||||
generateShortSummary,
|
||||
generateTestReport,
|
||||
getAllTests,
|
||||
removeOldGeneratedReports,
|
||||
sendReport,
|
||||
readJsonFromFile,
|
||||
writeJsonToFile,
|
||||
} = require('./utils/report');
|
||||
const {createTestCycle, createTestExecutions} = require('./utils/test_cases');
|
||||
|
||||
require('dotenv').config();
|
||||
|
||||
const saveReport = async () => {
|
||||
const {
|
||||
DEVICE_NAME,
|
||||
DEVICE_OS_VERSION,
|
||||
FAILURE_MESSAGE,
|
||||
HEADLESS,
|
||||
IOS,
|
||||
TYPE,
|
||||
WEBHOOK_URL,
|
||||
ZEPHYR_ENABLE,
|
||||
ZEPHYR_CYCLE_KEY,
|
||||
} = process.env;
|
||||
|
||||
// Remove old generated reports
|
||||
removeOldGeneratedReports();
|
||||
|
||||
const detox_version = shell.exec('npm list detox').stdout.split('\n')[1].split('@')[1].trim();
|
||||
const headless = IOS === 'true' ? false : HEADLESS === 'true';
|
||||
const os_name = os.platform();
|
||||
const os_version = os.release();
|
||||
const node_version = process.version;
|
||||
const npm_version = shell.exec('npm --version').stdout.trim();
|
||||
|
||||
// Write environment details to file
|
||||
const environmentDetails = {
|
||||
detox_version,
|
||||
device_name: DEVICE_NAME,
|
||||
device_os_version: DEVICE_OS_VERSION,
|
||||
headless,
|
||||
os_name,
|
||||
os_version,
|
||||
node_version,
|
||||
npm_version,
|
||||
};
|
||||
writeJsonToFile(environmentDetails, 'environment.json', ARTIFACTS_DIR);
|
||||
|
||||
// Read XML from a file
|
||||
const platform = process.env.IOS === 'true' ? 'ios' : 'android';
|
||||
const xml = fse.readFileSync(`${ARTIFACTS_DIR}/${platform}-junit.xml`);
|
||||
const {testsuites} = convertXmlToJson(xml);
|
||||
|
||||
// Generate short summary, write to file and then send report via webhook
|
||||
const allTests = getAllTests(testsuites);
|
||||
const summary = generateShortSummary(allTests);
|
||||
console.log(summary);
|
||||
writeJsonToFile(summary, 'summary.json', ARTIFACTS_DIR);
|
||||
|
||||
const result = await saveArtifacts();
|
||||
if (result && result.success) {
|
||||
console.log('Successfully uploaded artifacts to S3:', result.reportLink);
|
||||
}
|
||||
|
||||
// Create or use an existing test cycle
|
||||
let testCycle = {};
|
||||
if (ZEPHYR_ENABLE === 'true') {
|
||||
const {start, end} = summary.stats;
|
||||
testCycle = ZEPHYR_CYCLE_KEY ? {key: ZEPHYR_CYCLE_KEY} : await createTestCycle(start, end);
|
||||
}
|
||||
|
||||
// Send test report to "QA: Mobile Test Automation Report" channel via webhook
|
||||
if (TYPE && TYPE !== 'NONE' && WEBHOOK_URL) {
|
||||
const environment = readJsonFromFile(`${ARTIFACTS_DIR}/environment.json`);
|
||||
const data = generateTestReport(summary, result && result.success, result && result.reportLink, environment, testCycle.key);
|
||||
await sendReport('summary report to Community channel', WEBHOOK_URL, data);
|
||||
}
|
||||
|
||||
// Save test cases to Test Management
|
||||
if (ZEPHYR_ENABLE === 'true') {
|
||||
await createTestExecutions(allTests, testCycle);
|
||||
}
|
||||
|
||||
assert(summary.stats.failures === 0, FAILURE_MESSAGE);
|
||||
};
|
||||
|
||||
saveReport();
|
||||
89
detox/utils/artifacts.js
Normal file
89
detox/utils/artifacts.js
Normal file
@@ -0,0 +1,89 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
/* eslint-disable no-console, consistent-return, no-process-env */
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const async = require('async');
|
||||
const AWS = require('aws-sdk');
|
||||
const mime = require('mime-types');
|
||||
const readdir = require('recursive-readdir');
|
||||
|
||||
const {ARTIFACTS_DIR} = require('./constants');
|
||||
|
||||
require('dotenv').config();
|
||||
|
||||
const {
|
||||
BRANCH,
|
||||
BUILD_ID,
|
||||
DETOX_AWS_S3_BUCKET,
|
||||
DETOX_AWS_ACCESS_KEY_ID,
|
||||
DETOX_AWS_SECRET_ACCESS_KEY,
|
||||
IOS,
|
||||
} = process.env;
|
||||
const platform = IOS === 'true' ? 'ios' : 'android';
|
||||
|
||||
const s3 = new AWS.S3({
|
||||
signatureVersion: 'v4',
|
||||
accessKeyId: DETOX_AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: DETOX_AWS_SECRET_ACCESS_KEY,
|
||||
});
|
||||
|
||||
function getFiles(dirPath) {
|
||||
return fs.existsSync(dirPath) ? readdir(dirPath) : [];
|
||||
}
|
||||
|
||||
async function saveArtifacts() {
|
||||
if (!DETOX_AWS_S3_BUCKET || !DETOX_AWS_ACCESS_KEY_ID || !DETOX_AWS_SECRET_ACCESS_KEY) {
|
||||
console.log('No AWS credentials found. Test artifacts not uploaded to S3.');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const s3Folder = `${BUILD_ID}-${BRANCH}`.replace(/\./g, '-');
|
||||
const uploadPath = path.resolve(__dirname, `../${ARTIFACTS_DIR}`);
|
||||
const filesToUpload = await getFiles(uploadPath);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
async.eachOfLimit(
|
||||
filesToUpload,
|
||||
10,
|
||||
async.asyncify(async (file) => {
|
||||
const Key = file.replace(uploadPath, s3Folder);
|
||||
const contentType = mime.lookup(file);
|
||||
const charset = mime.charset(contentType);
|
||||
|
||||
return new Promise((res, rej) => {
|
||||
s3.upload(
|
||||
{
|
||||
Key,
|
||||
Bucket: DETOX_AWS_S3_BUCKET,
|
||||
Body: fs.readFileSync(file),
|
||||
ContentType: `${contentType}${charset ? '; charset=' + charset : ''}`,
|
||||
},
|
||||
(err) => {
|
||||
if (err) {
|
||||
console.log('Failed to upload artifact:', file);
|
||||
return rej(new Error(err));
|
||||
}
|
||||
res({success: true});
|
||||
},
|
||||
);
|
||||
});
|
||||
}),
|
||||
(err) => {
|
||||
if (err) {
|
||||
console.log('Failed to upload artifacts');
|
||||
return reject(new Error(err));
|
||||
}
|
||||
|
||||
const reportLink = `https://${DETOX_AWS_S3_BUCKET}.s3.amazonaws.com/${s3Folder}/${platform}-report.html`;
|
||||
resolve({success: true, reportLink});
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {saveArtifacts};
|
||||
8
detox/utils/constants.js
Normal file
8
detox/utils/constants.js
Normal file
@@ -0,0 +1,8 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
const ARTIFACTS_DIR = 'artifacts';
|
||||
|
||||
module.exports = {
|
||||
ARTIFACTS_DIR,
|
||||
};
|
||||
375
detox/utils/report.js
Normal file
375
detox/utils/report.js
Normal file
@@ -0,0 +1,375 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
/* eslint-disable no-console, camelcase, no-process-env */
|
||||
|
||||
const axios = require('axios');
|
||||
const fse = require('fs-extra');
|
||||
const xml2js = require('xml2js');
|
||||
|
||||
const {ARTIFACTS_DIR} = require('./constants');
|
||||
|
||||
const MAX_FAILED_TITLES = 5;
|
||||
|
||||
function convertXmlToJson(xml) {
|
||||
const platform = process.env.IOS === 'true' ? 'ios' : 'android';
|
||||
const jsonFile = `${ARTIFACTS_DIR}/${platform}-junit.json`;
|
||||
|
||||
// Convert XML to JSON
|
||||
xml2js.parseString(xml, {mergeAttrs: true}, (err, result) => {
|
||||
if (err) {
|
||||
throw err;
|
||||
}
|
||||
|
||||
// Convert result to a JSON string
|
||||
const json = JSON.stringify(result, null, 4);
|
||||
|
||||
// Save JSON in a file
|
||||
fse.writeFileSync(jsonFile, json);
|
||||
});
|
||||
|
||||
return readJsonFromFile(jsonFile);
|
||||
}
|
||||
|
||||
function getAllTests(testSuites) {
|
||||
const suites = [];
|
||||
const tests = [];
|
||||
let skipped = 0;
|
||||
let firstTimestamp;
|
||||
let incrementalDuration = 0;
|
||||
testSuites.testsuite.forEach((testSuite) => {
|
||||
skipped += parseInt(testSuite.skipped[0], 10);
|
||||
if (!firstTimestamp) {
|
||||
firstTimestamp = testSuite.timestamp[0];
|
||||
}
|
||||
suites.push({
|
||||
name: testSuite.name[0],
|
||||
errors: parseInt(testSuite.errors[0], 10),
|
||||
failures: parseInt(testSuite.failures[0], 10),
|
||||
skipped: parseInt(testSuite.skipped[0], 10),
|
||||
timestamp: testSuite.timestamp[0],
|
||||
time: parseFloat(testSuite.time[0] * 1000),
|
||||
tests: testSuite.tests[0],
|
||||
});
|
||||
testSuite.testcase.filter((test) => !test.name[0].startsWith(' Test execution failure:')).forEach((test) => {
|
||||
const time = parseFloat(test.time[0] * 1000);
|
||||
incrementalDuration += time;
|
||||
let state = 'passed';
|
||||
let pass = 'true';
|
||||
let fail = 'false';
|
||||
let pending = 'false';
|
||||
if (test.failure) {
|
||||
state = 'failed';
|
||||
fail = 'true';
|
||||
pass = 'false';
|
||||
} else if (test.skipped) {
|
||||
state = 'skipped';
|
||||
pending = 'true';
|
||||
pass = 'false';
|
||||
}
|
||||
tests.push({
|
||||
classname: test.classname[0],
|
||||
name: test.name[0],
|
||||
time,
|
||||
failure: test.failure ? test.failure[0] : '',
|
||||
skipped: test.skipped ? test.skipped[0] : '',
|
||||
incrementalDuration,
|
||||
state,
|
||||
pass,
|
||||
fail,
|
||||
pending,
|
||||
});
|
||||
});
|
||||
});
|
||||
const startDate = new Date(firstTimestamp);
|
||||
const start = startDate.toISOString();
|
||||
startDate.setTime(startDate.getTime() + parseFloat(testSuites.time[0] * 1000));
|
||||
const end = startDate.toISOString();
|
||||
|
||||
return {
|
||||
suites,
|
||||
tests,
|
||||
skipped,
|
||||
failures: parseInt(testSuites.failures[0], 10),
|
||||
errors: parseInt(testSuites.errors[0], 10),
|
||||
duration: parseFloat(testSuites.time[0] * 1000),
|
||||
start,
|
||||
end,
|
||||
};
|
||||
}
|
||||
|
||||
function generateStats(allTests) {
|
||||
const suites = allTests.suites.length;
|
||||
const tests = allTests.tests.length;
|
||||
const skipped = allTests.skipped;
|
||||
const failures = allTests.failures;
|
||||
const errors = allTests.errors;
|
||||
const duration = allTests.duration;
|
||||
const start = allTests.start;
|
||||
const end = allTests.end;
|
||||
const passes = tests - (skipped + failures + errors);
|
||||
const passPercent = tests > 0 ? (passes / tests) * 100 : 0;
|
||||
|
||||
return {
|
||||
suites,
|
||||
tests,
|
||||
skipped,
|
||||
failures,
|
||||
errors,
|
||||
duration,
|
||||
start,
|
||||
end,
|
||||
passes,
|
||||
passPercent,
|
||||
};
|
||||
}
|
||||
|
||||
function generateStatsFieldValue(stats, failedFullTitles) {
|
||||
let statsFieldValue = `
|
||||
| Key | Value |
|
||||
|:---|:---|
|
||||
| Passing Rate | ${stats.passPercent.toFixed(2)}% |
|
||||
| Duration | ${(stats.duration / (60 * 1000)).toFixed(4)} mins |
|
||||
| Suites | ${stats.suites} |
|
||||
| Tests | ${stats.tests} |
|
||||
| :white_check_mark: Passed | ${stats.passes} |
|
||||
| :x: Failed | ${stats.failures} |
|
||||
| :fast_forward: Skipped | ${stats.skipped} |
|
||||
`;
|
||||
|
||||
// If present, add full title of failing tests.
|
||||
// Only show per maximum number of failed titles with the last item as "more..." if failing tests are more than that.
|
||||
let failedTests;
|
||||
if (failedFullTitles && failedFullTitles.length > 0) {
|
||||
const re = /[:'"\\]/gi;
|
||||
const failed = failedFullTitles;
|
||||
if (failed.length > MAX_FAILED_TITLES) {
|
||||
failedTests = failed.slice(0, MAX_FAILED_TITLES - 1).map((f) => `- ${f.replace(re, '')}`).join('\n');
|
||||
failedTests += '\n- more...';
|
||||
} else {
|
||||
failedTests = failed.map((f) => `- ${f.replace(re, '')}`).join('\n');
|
||||
}
|
||||
}
|
||||
|
||||
if (failedTests) {
|
||||
statsFieldValue += '###### Failed Tests:\n' + failedTests;
|
||||
}
|
||||
|
||||
return statsFieldValue;
|
||||
}
|
||||
|
||||
function generateShortSummary(allTests) {
|
||||
const failedFullTitles = allTests.tests.filter((t) => t.failure).map((t) => t.name);
|
||||
const stats = generateStats(allTests);
|
||||
const statsFieldValue = generateStatsFieldValue(stats, failedFullTitles);
|
||||
|
||||
return {
|
||||
stats,
|
||||
statsFieldValue,
|
||||
};
|
||||
}
|
||||
|
||||
function removeOldGeneratedReports() {
|
||||
const platform = process.env.IOS === 'true' ? 'ios' : 'android';
|
||||
[
|
||||
'environment.json',
|
||||
'summary.json',
|
||||
`${platform}-junit.json`,
|
||||
].forEach((file) => fse.removeSync(`${ARTIFACTS_DIR}/${file}`));
|
||||
}
|
||||
|
||||
function writeJsonToFile(jsonObject, filename, dir) {
|
||||
fse.writeJson(`${dir}/${filename}`, jsonObject).
|
||||
then(() => console.log('Successfully written:', filename)).
|
||||
catch((err) => console.error(err));
|
||||
}
|
||||
|
||||
function readJsonFromFile(file) {
|
||||
try {
|
||||
return fse.readJsonSync(file);
|
||||
} catch (err) {
|
||||
return {err};
|
||||
}
|
||||
}
|
||||
|
||||
const result = [
|
||||
{status: 'Passed', priority: 'none', cutOff: 100, color: '#43A047'},
|
||||
{status: 'Failed', priority: 'low', cutOff: 98, color: '#FFEB3B'},
|
||||
{status: 'Failed', priority: 'medium', cutOff: 95, color: '#FF9800'},
|
||||
{status: 'Failed', priority: 'high', cutOff: 0, color: '#F44336'},
|
||||
];
|
||||
|
||||
function generateTestReport(summary, isUploadedToS3, reportLink, environment, testCycleKey) {
|
||||
const {
|
||||
FULL_REPORT,
|
||||
IOS,
|
||||
TEST_CYCLE_LINK_PREFIX,
|
||||
} = process.env;
|
||||
const platform = IOS === 'true' ? 'iOS' : 'Android';
|
||||
const {statsFieldValue, stats} = summary;
|
||||
const {
|
||||
detox_version,
|
||||
device_name,
|
||||
device_os_version,
|
||||
headless,
|
||||
os_name,
|
||||
os_version,
|
||||
node_version,
|
||||
npm_version,
|
||||
} = environment;
|
||||
|
||||
let testResult;
|
||||
for (let i = 0; i < result.length; i++) {
|
||||
if (stats.passPercent >= result[i].cutOff) {
|
||||
testResult = result[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const title = generateTitle();
|
||||
const envValue = `detox@${detox_version} | node@${node_version} | npm@${npm_version} | ${device_name}@${device_os_version}${headless ? ' (headless)' : ''} | ${os_name}@${os_version}`;
|
||||
|
||||
if (FULL_REPORT === 'true') {
|
||||
let reportField;
|
||||
if (isUploadedToS3) {
|
||||
reportField = {
|
||||
short: false,
|
||||
title: `${platform} Test Report`,
|
||||
value: `[Link to the report](${reportLink})`,
|
||||
};
|
||||
}
|
||||
|
||||
let testCycleField;
|
||||
if (testCycleKey) {
|
||||
testCycleField = {
|
||||
short: false,
|
||||
title: `${platform} Test Execution`,
|
||||
value: `[Recorded test executions](${TEST_CYCLE_LINK_PREFIX}${testCycleKey})`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
username: 'Mobile Detox Test',
|
||||
icon_url: 'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
|
||||
attachments: [{
|
||||
color: testResult.color,
|
||||
author_name: 'Mobile End-to-end Testing',
|
||||
author_icon: 'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
|
||||
author_link: 'https://www.mattermost.com',
|
||||
title,
|
||||
fields: [
|
||||
{
|
||||
short: false,
|
||||
title: 'Environment',
|
||||
value: envValue,
|
||||
},
|
||||
reportField,
|
||||
testCycleField,
|
||||
{
|
||||
short: false,
|
||||
title: `Key metrics (required support: ${testResult.priority})`,
|
||||
value: statsFieldValue,
|
||||
},
|
||||
],
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
let quickSummary = `${stats.passPercent.toFixed(2)}% (${stats.passes}/${stats.tests}) in ${stats.suites} suites`;
|
||||
if (isUploadedToS3) {
|
||||
quickSummary = `[${quickSummary}](${reportLink})`;
|
||||
}
|
||||
|
||||
let testCycleLink = '';
|
||||
if (testCycleKey) {
|
||||
testCycleLink = testCycleKey ? `| [Recorded test executions](${TEST_CYCLE_LINK_PREFIX}${testCycleKey})` : '';
|
||||
}
|
||||
|
||||
return {
|
||||
username: 'Mobile Detox Test',
|
||||
icon_url: 'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
|
||||
attachments: [{
|
||||
color: testResult.color,
|
||||
author_name: 'Mobile End-to-end Testing',
|
||||
author_icon: 'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
|
||||
author_link: 'https://www.mattermost.com/',
|
||||
title,
|
||||
text: `${quickSummary} | ${(stats.duration / (60 * 1000)).toFixed(4)} mins ${testCycleLink}\n${envValue}`,
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
function generateTitle() {
|
||||
const {
|
||||
BRANCH,
|
||||
IOS,
|
||||
PULL_REQUEST,
|
||||
RELEASE_BUILD_NUMBER,
|
||||
RELEASE_DATE,
|
||||
RELEASE_VERSION,
|
||||
TYPE,
|
||||
} = process.env;
|
||||
|
||||
const platform = IOS === 'true' ? 'iOS' : 'Android';
|
||||
const lane = `${platform} Build`;
|
||||
const appExtension = IOS === 'true' ? 'ipa' : 'apk';
|
||||
const appFileName = TYPE === 'GEKIDOU' ? `Mattermost_Beta.${appExtension}` : `Mattermost.${appExtension}`;
|
||||
let buildLink = ` with [${lane}](https://pr-builds.mattermost.com/mattermost-mobile/${BRANCH}/${appFileName})`;
|
||||
if (RELEASE_VERSION && RELEASE_BUILD_NUMBER) {
|
||||
const releaseType = TYPE === 'GEKIDOU' ? 'mattermost-mobile-beta' : 'mattermost-mobile';
|
||||
buildLink = ` with [${RELEASE_VERSION}:${RELEASE_BUILD_NUMBER}](https://releases.mattermost.com/${releaseType}/${RELEASE_VERSION}/${RELEASE_BUILD_NUMBER}/${appFileName})`;
|
||||
}
|
||||
|
||||
let releaseDate = '';
|
||||
if (RELEASE_DATE) {
|
||||
releaseDate = ` for ${RELEASE_DATE}`;
|
||||
}
|
||||
|
||||
let title;
|
||||
|
||||
switch (TYPE) {
|
||||
case 'PR':
|
||||
title = `${platform} E2E for Pull Request Build: [${BRANCH}](${PULL_REQUEST})${buildLink}`;
|
||||
break;
|
||||
case 'RELEASE':
|
||||
title = `${platform} E2E for Release Build${buildLink}${releaseDate}`;
|
||||
break;
|
||||
case 'MASTER':
|
||||
title = `${platform} E2E for Master Nightly Build (Prod tests)${buildLink}`;
|
||||
break;
|
||||
case 'GEKIDOU':
|
||||
title = `${platform} E2E for Gekidou Nightly Build (Prod tests)${buildLink}`;
|
||||
break;
|
||||
default:
|
||||
title = `${platform} E2E for Build${buildLink}`;
|
||||
}
|
||||
|
||||
return title;
|
||||
}
|
||||
|
||||
async function sendReport(name, url, data) {
|
||||
const requestOptions = {method: 'POST', url, data};
|
||||
|
||||
try {
|
||||
const response = await axios(requestOptions);
|
||||
|
||||
if (response.data) {
|
||||
console.log(`Successfully sent ${name}.`);
|
||||
}
|
||||
return response;
|
||||
} catch (er) {
|
||||
console.log(`Something went wrong while sending ${name}.`, er);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
convertXmlToJson,
|
||||
generateShortSummary,
|
||||
generateTestReport,
|
||||
getAllTests,
|
||||
removeOldGeneratedReports,
|
||||
sendReport,
|
||||
readJsonFromFile,
|
||||
writeJsonToFile,
|
||||
};
|
||||
195
detox/utils/test_cases.js
Normal file
195
detox/utils/test_cases.js
Normal file
@@ -0,0 +1,195 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
/* eslint-disable no-console, no-process-env */
|
||||
|
||||
// See reference: https://support.smartbear.com/tm4j-cloud/api-docs/
|
||||
|
||||
const axios = require('axios');
|
||||
const chalk = require('chalk');
|
||||
|
||||
const status = {
|
||||
passed: 'Pass',
|
||||
failed: 'Fail',
|
||||
pending: 'Pending',
|
||||
skipped: 'Skip',
|
||||
};
|
||||
|
||||
function getStepStateResult(steps = []) {
|
||||
return steps.reduce((acc, item) => {
|
||||
if (acc[item.state]) {
|
||||
acc[item.state] += 1;
|
||||
} else {
|
||||
acc[item.state] = 1;
|
||||
}
|
||||
|
||||
return acc;
|
||||
}, {});
|
||||
}
|
||||
|
||||
function getStepStateSummary(steps = []) {
|
||||
const result = getStepStateResult(steps);
|
||||
|
||||
return Object.entries(result).map(([key, value]) => `${value} ${key}`).join(',');
|
||||
}
|
||||
|
||||
function getTM4JTestCases(allTests) {
|
||||
return allTests.tests.
|
||||
filter((item) => /(MM-T)\w+/g.test(item.name)). // eslint-disable-line wrap-regex
|
||||
map((item) => {
|
||||
return {
|
||||
title: item.name,
|
||||
duration: item.time,
|
||||
incrementalDuration: item.incrementalDuration,
|
||||
state: item.state,
|
||||
pass: item.pass,
|
||||
fail: item.fail,
|
||||
pending: item.pending,
|
||||
};
|
||||
}).
|
||||
reduce((acc, item) => {
|
||||
// Extract the key to exactly match with "MM-T[0-9]+"
|
||||
const key = item.title.match(/(MM-T\d+)/)[0];
|
||||
|
||||
if (acc[key]) {
|
||||
acc[key].push(item);
|
||||
} else {
|
||||
acc[key] = [item];
|
||||
}
|
||||
|
||||
return acc;
|
||||
}, {});
|
||||
}
|
||||
|
||||
function saveToEndpoint(url, data) {
|
||||
return axios({
|
||||
method: 'POST',
|
||||
url,
|
||||
headers: {
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
Authorization: process.env.ZEPHYR_API_KEY,
|
||||
},
|
||||
data,
|
||||
}).catch((error) => {
|
||||
console.log('Something went wrong:', error.response.data.message);
|
||||
return error.response.data;
|
||||
});
|
||||
}
|
||||
|
||||
async function createTestCycle(startDate, endDate) {
|
||||
const {
|
||||
BRANCH,
|
||||
BUILD_ID,
|
||||
JIRA_PROJECT_KEY,
|
||||
ZEPHYR_CYCLE_NAME,
|
||||
ZEPHYR_FOLDER_ID,
|
||||
} = process.env;
|
||||
|
||||
const testCycle = {
|
||||
projectKey: JIRA_PROJECT_KEY,
|
||||
name: ZEPHYR_CYCLE_NAME ? `${ZEPHYR_CYCLE_NAME} (${BUILD_ID}-${BRANCH})` : `${BUILD_ID}-${BRANCH}`,
|
||||
description: `Detox automated test with ${BRANCH}`,
|
||||
plannedStartDate: startDate,
|
||||
plannedEndDate: endDate,
|
||||
statusName: 'Done',
|
||||
folderId: ZEPHYR_FOLDER_ID,
|
||||
};
|
||||
|
||||
const response = await saveToEndpoint('https://api.zephyrscale.smartbear.com/v2/testcycles', testCycle);
|
||||
return response.data;
|
||||
}
|
||||
|
||||
async function createTestExecutions(allTests, testCycle) {
|
||||
const {
|
||||
IOS,
|
||||
JIRA_PROJECT_KEY,
|
||||
ZEPHYR_ENVIRONMENT_NAME,
|
||||
} = process.env;
|
||||
const platform = IOS === 'true' ? 'iOS' : 'Android';
|
||||
|
||||
const testCases = getTM4JTestCases(allTests);
|
||||
const startDate = new Date(allTests.start);
|
||||
const startTime = startDate.getTime();
|
||||
|
||||
const promises = [];
|
||||
Object.entries(testCases).forEach(([key, steps], index) => {
|
||||
const testScriptResults = steps.
|
||||
sort((a, b) => a.title.localeCompare(b.title)).
|
||||
map((item) => {
|
||||
return {
|
||||
statusName: status[item.state],
|
||||
actualEndDate: new Date(startTime + item.incrementalDuration).toISOString(),
|
||||
actualResult: 'Detox automated test completed',
|
||||
};
|
||||
});
|
||||
|
||||
const stateResult = getStepStateResult(steps);
|
||||
|
||||
const testExecution = {
|
||||
projectKey: JIRA_PROJECT_KEY,
|
||||
testCaseKey: key,
|
||||
testCycleKey: testCycle.key,
|
||||
statusName: stateResult.passed && stateResult.passed === steps.length ? 'Pass' : 'Fail',
|
||||
testScriptResults,
|
||||
environmentName: ZEPHYR_ENVIRONMENT_NAME || platform,
|
||||
actualEndDate: testScriptResults[testScriptResults.length - 1].actualEndDate,
|
||||
executionTime: steps.reduce((acc, prev) => {
|
||||
acc += prev.duration; // eslint-disable-line no-param-reassign
|
||||
return acc;
|
||||
}, 0),
|
||||
comment: `Detox automated test - ${getStepStateSummary(steps)}`,
|
||||
};
|
||||
|
||||
// Temporarily log to verify cases that were being saved.
|
||||
console.log(index, key); // eslint-disable-line no-console
|
||||
|
||||
promises.push(saveTestExecution(testExecution, index));
|
||||
});
|
||||
|
||||
await Promise.all(promises);
|
||||
console.log('Successfully saved test cases into the Test Management System');
|
||||
}
|
||||
|
||||
const saveTestCases = async (allTests) => {
|
||||
const {start, end} = allTests;
|
||||
|
||||
const testCycle = await createTestCycle(start, end);
|
||||
|
||||
await createTestExecutions(allTests, testCycle);
|
||||
};
|
||||
|
||||
const RETRY = [];
|
||||
|
||||
async function saveTestExecution(testExecution, index) {
|
||||
await axios({
|
||||
method: 'POST',
|
||||
url: 'https://api.zephyrscale.smartbear.com/v2/testexecutions',
|
||||
headers: {
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
Authorization: process.env.ZEPHYR_API_KEY,
|
||||
},
|
||||
data: testExecution,
|
||||
}).then(() => {
|
||||
console.log(chalk.green('Success:', index, testExecution.testCaseKey));
|
||||
}).catch((error) => {
|
||||
// Retry on 500 error code / internal server error
|
||||
if (!error.response || error.response.data.errorCode === 500) {
|
||||
if (RETRY[testExecution.testCaseKey]) {
|
||||
RETRY[testExecution.testCaseKey] += 1;
|
||||
} else {
|
||||
RETRY[testExecution.testCaseKey] = 1;
|
||||
}
|
||||
|
||||
saveTestExecution(testExecution, index);
|
||||
console.log(chalk.magenta('Retry:', index, testExecution.testCaseKey, `(${RETRY[testExecution.testCaseKey]}x)`));
|
||||
} else {
|
||||
console.log(chalk.red('Error:', index, testExecution.testCaseKey, error.response.data.message));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
createTestCycle,
|
||||
saveTestCases,
|
||||
createTestExecutions,
|
||||
};
|
||||
Reference in New Issue
Block a user