1*4882a593Smuzhiyun# 2*4882a593Smuzhiyun# SPDX-License-Identifier: MIT 3*4882a593Smuzhiyun# 4*4882a593Smuzhiyun 5*4882a593Smuzhiyunimport os 6*4882a593Smuzhiyunimport sys 7*4882a593Smuzhiyunbasepath = os.path.abspath(os.path.dirname(__file__) + '/../../../../../') 8*4882a593Smuzhiyunlib_path = basepath + '/scripts/lib' 9*4882a593Smuzhiyunsys.path = sys.path + [lib_path] 10*4882a593Smuzhiyunfrom resulttool.report import ResultsTextReport 11*4882a593Smuzhiyunfrom resulttool import regression as regression 12*4882a593Smuzhiyunfrom resulttool import resultutils as resultutils 13*4882a593Smuzhiyunfrom oeqa.selftest.case import OESelftestTestCase 14*4882a593Smuzhiyun 15*4882a593Smuzhiyunclass ResultToolTests(OESelftestTestCase): 16*4882a593Smuzhiyun base_results_data = {'base_result1': {'configuration': {"TEST_TYPE": "runtime", 17*4882a593Smuzhiyun "TESTSERIES": "series1", 18*4882a593Smuzhiyun "IMAGE_BASENAME": "image", 19*4882a593Smuzhiyun "IMAGE_PKGTYPE": "ipk", 20*4882a593Smuzhiyun "DISTRO": "mydistro", 21*4882a593Smuzhiyun "MACHINE": "qemux86"}, 22*4882a593Smuzhiyun 'result': {}}, 23*4882a593Smuzhiyun 'base_result2': {'configuration': {"TEST_TYPE": "runtime", 24*4882a593Smuzhiyun "TESTSERIES": "series1", 25*4882a593Smuzhiyun "IMAGE_BASENAME": "image", 26*4882a593Smuzhiyun "IMAGE_PKGTYPE": "ipk", 27*4882a593Smuzhiyun "DISTRO": "mydistro", 28*4882a593Smuzhiyun "MACHINE": "qemux86-64"}, 29*4882a593Smuzhiyun 'result': {}}} 30*4882a593Smuzhiyun target_results_data = {'target_result1': {'configuration': {"TEST_TYPE": "runtime", 31*4882a593Smuzhiyun "TESTSERIES": "series1", 32*4882a593Smuzhiyun "IMAGE_BASENAME": "image", 33*4882a593Smuzhiyun "IMAGE_PKGTYPE": "ipk", 34*4882a593Smuzhiyun "DISTRO": "mydistro", 35*4882a593Smuzhiyun "MACHINE": "qemux86"}, 36*4882a593Smuzhiyun 'result': {}}, 37*4882a593Smuzhiyun 'target_result2': {'configuration': {"TEST_TYPE": "runtime", 38*4882a593Smuzhiyun "TESTSERIES": "series1", 39*4882a593Smuzhiyun "IMAGE_BASENAME": "image", 40*4882a593Smuzhiyun "IMAGE_PKGTYPE": "ipk", 41*4882a593Smuzhiyun "DISTRO": "mydistro", 42*4882a593Smuzhiyun "MACHINE": "qemux86"}, 43*4882a593Smuzhiyun 'result': {}}, 44*4882a593Smuzhiyun 'target_result3': {'configuration': {"TEST_TYPE": "runtime", 45*4882a593Smuzhiyun "TESTSERIES": "series1", 46*4882a593Smuzhiyun "IMAGE_BASENAME": "image", 47*4882a593Smuzhiyun "IMAGE_PKGTYPE": "ipk", 48*4882a593Smuzhiyun "DISTRO": "mydistro", 49*4882a593Smuzhiyun "MACHINE": "qemux86-64"}, 50*4882a593Smuzhiyun 'result': {}}} 51*4882a593Smuzhiyun 52*4882a593Smuzhiyun def test_report_can_aggregate_test_result(self): 53*4882a593Smuzhiyun result_data = {'result': {'test1': {'status': 'PASSED'}, 54*4882a593Smuzhiyun 'test2': {'status': 'PASSED'}, 55*4882a593Smuzhiyun 'test3': {'status': 'FAILED'}, 56*4882a593Smuzhiyun 'test4': {'status': 'ERROR'}, 57*4882a593Smuzhiyun 'test5': {'status': 'SKIPPED'}}} 58*4882a593Smuzhiyun report = ResultsTextReport() 59*4882a593Smuzhiyun result_report = report.get_aggregated_test_result(None, result_data, 'DummyMachine') 60*4882a593Smuzhiyun self.assertTrue(result_report['passed'] == 2, msg="Passed count not correct:%s" % result_report['passed']) 61*4882a593Smuzhiyun self.assertTrue(result_report['failed'] == 2, msg="Failed count not correct:%s" % result_report['failed']) 62*4882a593Smuzhiyun self.assertTrue(result_report['skipped'] == 1, msg="Skipped count not correct:%s" % result_report['skipped']) 63*4882a593Smuzhiyun 64*4882a593Smuzhiyun def test_regression_can_get_regression_base_target_pair(self): 65*4882a593Smuzhiyun 66*4882a593Smuzhiyun results = {} 67*4882a593Smuzhiyun resultutils.append_resultsdata(results, ResultToolTests.base_results_data) 68*4882a593Smuzhiyun resultutils.append_resultsdata(results, ResultToolTests.target_results_data) 69*4882a593Smuzhiyun self.assertTrue('target_result1' in results['runtime/mydistro/qemux86/image'], msg="Pair not correct:%s" % results) 70*4882a593Smuzhiyun self.assertTrue('target_result3' in results['runtime/mydistro/qemux86-64/image'], msg="Pair not correct:%s" % results) 71*4882a593Smuzhiyun 72*4882a593Smuzhiyun def test_regression_can_get_regression_result(self): 73*4882a593Smuzhiyun base_result_data = {'result': {'test1': {'status': 'PASSED'}, 74*4882a593Smuzhiyun 'test2': {'status': 'PASSED'}, 75*4882a593Smuzhiyun 'test3': {'status': 'FAILED'}, 76*4882a593Smuzhiyun 'test4': {'status': 'ERROR'}, 77*4882a593Smuzhiyun 'test5': {'status': 'SKIPPED'}}} 78*4882a593Smuzhiyun target_result_data = {'result': {'test1': {'status': 'PASSED'}, 79*4882a593Smuzhiyun 'test2': {'status': 'FAILED'}, 80*4882a593Smuzhiyun 'test3': {'status': 'PASSED'}, 81*4882a593Smuzhiyun 'test4': {'status': 'ERROR'}, 82*4882a593Smuzhiyun 'test5': {'status': 'SKIPPED'}}} 83*4882a593Smuzhiyun result, text = regression.compare_result(self.logger, "BaseTestRunName", "TargetTestRunName", base_result_data, target_result_data) 84*4882a593Smuzhiyun self.assertTrue(result['test2']['base'] == 'PASSED', 85*4882a593Smuzhiyun msg="regression not correct:%s" % result['test2']['base']) 86*4882a593Smuzhiyun self.assertTrue(result['test2']['target'] == 'FAILED', 87*4882a593Smuzhiyun msg="regression not correct:%s" % result['test2']['target']) 88*4882a593Smuzhiyun self.assertTrue(result['test3']['base'] == 'FAILED', 89*4882a593Smuzhiyun msg="regression not correct:%s" % result['test3']['base']) 90*4882a593Smuzhiyun self.assertTrue(result['test3']['target'] == 'PASSED', 91*4882a593Smuzhiyun msg="regression not correct:%s" % result['test3']['target']) 92*4882a593Smuzhiyun 93*4882a593Smuzhiyun def test_merge_can_merged_results(self): 94*4882a593Smuzhiyun results = {} 95*4882a593Smuzhiyun resultutils.append_resultsdata(results, ResultToolTests.base_results_data, configmap=resultutils.flatten_map) 96*4882a593Smuzhiyun resultutils.append_resultsdata(results, ResultToolTests.target_results_data, configmap=resultutils.flatten_map) 97*4882a593Smuzhiyun self.assertEqual(len(results[''].keys()), 5, msg="Flattened results not correct %s" % str(results)) 98*4882a593Smuzhiyun 99