1From 27ed9962f5cb3afcc44d6c96c53277132a999712 Mon Sep 17 00:00:00 2001 2From: Alexander Kanavin <alex.kanavin@gmail.com> 3Date: Mon, 14 Jun 2021 19:57:30 +0200 4Subject: [PATCH 6/6] reproducible 5 6This patch fixes various things which make the build more reproducible. Some changes 7here only change intermediate artefacts but that means when you have two build trees 8giving differing results, the differences can be isolated more easily. The issues here 9usually become apparent with longer paths. 10 11This was all debugged with: 12TMPDIR = "${TOPDIR}/tmp" 13vs. 14TMPDIR = "${TOPDIR}/tmp-inital-mylongpath-mylongpath-mylongpath-mylongpath-mylongpath-mylongpath-mylongpath-mylongpath-mylongpath" 15 16The patch specifically: 17 18 * Sorts output in GNUmakefile 19 * Always generates indirect flags files used to avoid pathlength issues else the 20 compile commands suddenly change when using longer paths 21 * Sorts the AutoGenTimeStamp file contents 22 * Makes the TargetDescBlock objects from BuildEngine sortable to allow the makefile fix 23 * Fix ElfConvert within GenFw so that only the basename of the binary being converted 24 is used, else the output from "GenFw XXX.bin" differs from "GenFw /long/path/XXX.bin" 25 with sufficiently long paths 26 27Upstream-Status: Submitted [https://github.com/tianocore/edk2/pull/2176] 28Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> 29Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com> 30--- 31 BaseTools/Source/C/GenFw/Elf64Convert.c | 8 ++++--- 32 .../Source/Python/AutoGen/BuildEngine.py | 3 +++ 33 BaseTools/Source/Python/AutoGen/GenMake.py | 24 +++++++++---------- 34 .../Source/Python/AutoGen/ModuleAutoGen.py | 5 +++- 35 4 files changed, 24 insertions(+), 16 deletions(-) 36 37diff --git a/BaseTools/Source/C/GenFw/Elf64Convert.c b/BaseTools/Source/C/GenFw/Elf64Convert.c 38index d097db8632..a87ae6f3d0 100644 39--- a/BaseTools/Source/C/GenFw/Elf64Convert.c 40+++ b/BaseTools/Source/C/GenFw/Elf64Convert.c 41@@ -14,6 +14,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent 42 #ifndef __GNUC__ 43 #include <windows.h> 44 #include <io.h> 45+#else 46+#define _GNU_SOURCE 47 #endif 48 #include <assert.h> 49 #include <stdio.h> 50@@ -769,7 +771,7 @@ ScanSections64 ( 51 } 52 mCoffOffset = mDebugOffset + sizeof(EFI_IMAGE_DEBUG_DIRECTORY_ENTRY) + 53 sizeof(EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY) + 54- strlen(mInImageName) + 1; 55+ strlen(basename(mInImageName)) + 1; 56 57 mCoffOffset = CoffAlign(mCoffOffset); 58 if (SectionCount == 0) { 59@@ -1608,7 +1610,7 @@ WriteDebug64 ( 60 EFI_IMAGE_DEBUG_DIRECTORY_ENTRY *Dir; 61 EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY *Nb10; 62 63- Len = strlen(mInImageName) + 1; 64+ Len = strlen(basename(mInImageName)) + 1; 65 66 Dir = (EFI_IMAGE_DEBUG_DIRECTORY_ENTRY*)(mCoffFile + mDebugOffset); 67 Dir->Type = EFI_IMAGE_DEBUG_TYPE_CODEVIEW; 68@@ -1618,7 +1620,7 @@ WriteDebug64 ( 69 70 Nb10 = (EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY*)(Dir + 1); 71 Nb10->Signature = CODEVIEW_SIGNATURE_NB10; 72- strcpy ((char *)(Nb10 + 1), mInImageName); 73+ strcpy ((char *)(Nb10 + 1), basename(mInImageName)); 74 75 76 NtHdr = (EFI_IMAGE_OPTIONAL_HEADER_UNION *)(mCoffFile + mNtHdrOffset); 77diff --git a/BaseTools/Source/Python/AutoGen/BuildEngine.py b/BaseTools/Source/Python/AutoGen/BuildEngine.py 78index 722fead75a..8f1c236970 100644 79--- a/BaseTools/Source/Python/AutoGen/BuildEngine.py 80+++ b/BaseTools/Source/Python/AutoGen/BuildEngine.py 81@@ -70,6 +70,9 @@ class TargetDescBlock(object): 82 else: 83 return str(Other) == self.Target.Path 84 85+ def __lt__(self, other): 86+ return str(self) < str(other) 87+ 88 def AddInput(self, Input): 89 if Input not in self.Inputs: 90 self.Inputs.append(Input) 91diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py 92index 961b2ab1c3..23c1592025 100755 93--- a/BaseTools/Source/Python/AutoGen/GenMake.py 94+++ b/BaseTools/Source/Python/AutoGen/GenMake.py 95@@ -575,7 +575,7 @@ cleanlib: 96 os.remove(RespFileList) 97 98 # convert source files and binary files to build targets 99- self.ResultFileList = [str(T.Target) for T in MyAgo.CodaTargetList] 100+ self.ResultFileList = sorted([str(T.Target) for T in MyAgo.CodaTargetList]) 101 if len(self.ResultFileList) == 0 and len(MyAgo.SourceFileList) != 0: 102 EdkLogger.error("build", AUTOGEN_ERROR, "Nothing to build", 103 ExtraData="[%s]" % str(MyAgo)) 104@@ -726,7 +726,7 @@ cleanlib: 105 OutputFile = '' 106 DepsFileList = [] 107 108- for Cmd in self.GenFfsList: 109+ for Cmd in sorted(self.GenFfsList): 110 if Cmd[2]: 111 for CopyCmd in Cmd[2]: 112 Src, Dst = CopyCmd 113@@ -759,7 +759,7 @@ cleanlib: 114 self.BuildTargetList.append('\t%s' % CmdString) 115 116 self.ParseSecCmd(DepsFileList, Cmd[1]) 117- for SecOutputFile, SecDepsFile, SecCmd in self.FfsOutputFileList : 118+ for SecOutputFile, SecDepsFile, SecCmd in sorted(self.FfsOutputFileList): 119 self.BuildTargetList.append('%s : %s' % (self.ReplaceMacro(SecOutputFile), self.ReplaceMacro(SecDepsFile))) 120 self.BuildTargetList.append('\t%s' % self.ReplaceMacro(SecCmd)) 121 self.FfsOutputFileList = [] 122@@ -798,13 +798,13 @@ cleanlib: 123 124 def CommandExceedLimit(self): 125 FlagDict = { 126- 'CC' : { 'Macro' : '$(CC_FLAGS)', 'Value' : False}, 127- 'PP' : { 'Macro' : '$(PP_FLAGS)', 'Value' : False}, 128- 'APP' : { 'Macro' : '$(APP_FLAGS)', 'Value' : False}, 129- 'ASLPP' : { 'Macro' : '$(ASLPP_FLAGS)', 'Value' : False}, 130- 'VFRPP' : { 'Macro' : '$(VFRPP_FLAGS)', 'Value' : False}, 131- 'ASM' : { 'Macro' : '$(ASM_FLAGS)', 'Value' : False}, 132- 'ASLCC' : { 'Macro' : '$(ASLCC_FLAGS)', 'Value' : False}, 133+ 'CC' : { 'Macro' : '$(CC_FLAGS)', 'Value' : True}, 134+ 'PP' : { 'Macro' : '$(PP_FLAGS)', 'Value' : True}, 135+ 'APP' : { 'Macro' : '$(APP_FLAGS)', 'Value' : True}, 136+ 'ASLPP' : { 'Macro' : '$(ASLPP_FLAGS)', 'Value' : True}, 137+ 'VFRPP' : { 'Macro' : '$(VFRPP_FLAGS)', 'Value' : True}, 138+ 'ASM' : { 'Macro' : '$(ASM_FLAGS)', 'Value' : True}, 139+ 'ASLCC' : { 'Macro' : '$(ASLCC_FLAGS)', 'Value' : True}, 140 } 141 142 RespDict = {} 143@@ -1007,9 +1007,9 @@ cleanlib: 144 if not self.ObjTargetDict.get(T.Target.SubDir): 145 self.ObjTargetDict[T.Target.SubDir] = set() 146 self.ObjTargetDict[T.Target.SubDir].add(NewFile) 147- for Type in self._AutoGenObject.Targets: 148+ for Type in sorted(self._AutoGenObject.Targets): 149 resp_file_number = 0 150- for T in self._AutoGenObject.Targets[Type]: 151+ for T in sorted(self._AutoGenObject.Targets[Type]): 152 # Generate related macros if needed 153 if T.GenFileListMacro and T.FileListMacro not in self.FileListMacros: 154 self.FileListMacros[T.FileListMacro] = [] 155diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py 156index d70b0d7ae8..25dca9a6df 100755 157--- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py 158+++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py 159@@ -1484,6 +1484,9 @@ class ModuleAutoGen(AutoGen): 160 for File in Files: 161 if File.lower().endswith('.pdb'): 162 AsBuiltInfDict['binary_item'].append('DISPOSABLE|' + File) 163+ 164+ AsBuiltInfDict['binary_item'] = sorted(AsBuiltInfDict['binary_item']) 165+ 166 HeaderComments = self.Module.HeaderComments 167 StartPos = 0 168 for Index in range(len(HeaderComments)): 169@@ -1759,7 +1762,7 @@ class ModuleAutoGen(AutoGen): 170 if os.path.exists (self.TimeStampPath): 171 os.remove (self.TimeStampPath) 172 173- SaveFileOnChange(self.TimeStampPath, "\n".join(FileSet), False) 174+ SaveFileOnChange(self.TimeStampPath, "\n".join(sorted(FileSet)), False) 175 176 # Ignore generating makefile when it is a binary module 177 if self.IsBinaryModule: 178-- 1792.32.0 180 181