macOS x64 CI: fix dependency install and OpenMP runtime copy (use Homebrew libomp, adjust Helix payload)

build/ci/job-template.yml

@@ -66,13 +66,12 @@ jobs:
 
     steps:
     # Extra MacOS step required to install OS-specific dependencies
-    - ${{ if and(contains(parameters.pool.vmImage, 'macOS'), not(contains(parameters.name, 'cross')))  }}:
-      - script: export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=TRUE && brew install $(Build.SourcesDirectory)/build/libomp.rb --build-from-source --formula
+    - ${{ if contains(parameters.pool.vmImage, 'macOS')  }}:
+      - script: |
+          brew update
+          brew install libomp
+          brew link libomp --force
         displayName: Install MacOS build dependencies
-    # Extra Apple MacOS step required to install OS-specific dependencies
-    - ${{ if and(contains(parameters.pool.vmImage, 'macOS'), contains(parameters.name, 'cross'))  }}:
-      - script: brew update && brew install -f --overwrite [email protected] && brew install libomp && brew link libomp --force
-        displayName: Install MacOS ARM build dependencies
     - ${{ if and( eq(parameters.nightlyBuild, 'true'), eq(parameters.pool.vmImage, 'ubuntu-18.04')) }}:
       - bash: echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(nightlyBuildRunPath):$LD_LIBRARY_PATH"
         displayName: Set LD_LIBRARY_PATH for Ubuntu and CentOS to locate Native shared library in current running path

build/vsts-ci.yml

@@ -142,7 +142,10 @@ extends:
             PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets
             ArtifactName: pkgassets
         steps:
-        - script: export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 &&  rm '/usr/local/bin/2to3-3.11' && brew install $(Build.SourcesDirectory)/build/libomp.rb --build-from-source --formula
+        - script: |
+            brew update
+            brew install libomp
+            brew link libomp --force
           displayName: Install build dependencies
         # Only build native assets to avoid conflicts.
         - script: ./build.sh -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x64 /p:CopyPackageAssets=true

eng/helix.proj

@@ -111,6 +111,9 @@
 
       <HelixPreCommands Condition="$(HelixTargetQueues.ToLowerInvariant().Contains('osx'))">$(HelixPreCommands);export LD_LIBRARY_PATH=/opt/homebrew/opt/mono-libgdiplus/lib;ls /usr/lib;ls $HELIX_WORKITEM_ROOT;export KMP_DUPLICATE_LIB_OK=TRUE;otool -L $HELIX_WORKITEM_ROOT/runtimes/osx-x64/native/lib_lightgbm.dylib</HelixPreCommands>
 
+      <!-- macOS x64 only: add DYLD paths so the local libomp/libiomp5 in the work item is discoverable -->
+      <HelixPreCommands Condition="$(HelixTargetQueues.ToLowerInvariant().Contains('osx')) and '$(BuildArchitecture)'=='x64'">$(HelixPreCommands);export DYLD_LIBRARY_PATH=$HELIX_WORKITEM_ROOT:$DYLD_LIBRARY_PATH;export DYLD_FALLBACK_LIBRARY_PATH=$HELIX_WORKITEM_ROOT:$DYLD_FALLBACK_LIBRARY_PATH</HelixPreCommands>
+
       <HelixPreCommands Condition="$(HelixTargetQueues.ToLowerInvariant().Contains('armarch'))">$(HelixPreCommands);sudo apt update;sudo apt-get install libomp-dev libomp5 -y</HelixPreCommands>
 
       <HelixCorrelationPayloadPath Condition="$(IsPosixShell)">$HELIX_CORRELATION_PAYLOAD</HelixCorrelationPayloadPath>
@@ -119,11 +122,10 @@
       <RunTestScript Condition="'$(OS)' != 'Windows_NT'">runTests.sh</RunTestScript>
       <RunTestScript Condition="'$(OS)' == 'Windows_NT'">.\runTests.cmd</RunTestScript>
 
-      <MacFilesToCopy>/usr/local/opt/libomp/lib/libiomp5.dylib;/usr/local/opt/libomp/lib/libomp.dylib;</MacFilesToCopy>
-      <MacFilesToCopy Condition="$(HelixTargetQueues.ToLowerInvariant().Contains('arm64'))">/usr/local/opt/libomp/lib/libomp.dylib;</MacFilesToCopy>
+      <MacFilesToCopy>/usr/local/opt/libomp/lib/libomp.dylib;</MacFilesToCopy>
     </PropertyGroup>
 
-    <!-- Copy libiomp5.dylib and libomp.dylib to the publish folders for OSX-->
+    <!-- Copy OpenMP runtime(s) to the publish folders for OSX -->
     <Copy Condition="$(HelixTargetQueues.ToLowerInvariant().Contains('osx'))"
       SourceFiles = "$(MacFilesToCopy)"
       Retries="10"
@@ -132,6 +134,7 @@
       DestinationFolder="$(BUILD_SOURCESDIRECTORY)\artifacts\bin\%(ProjectsWithTargetFramework.Filename)\$(BuildConfig)\%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)">
     </Copy>
 
+    <!-- Fix up install names where we know about libomp.dylib usage -->
     <Exec Condition="Exists('$(BUILD_SOURCESDIRECTORY)\artifacts\bin\%(ProjectsWithTargetFramework.Filename)\$(BuildConfig)\%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)\libSymSgdNative.dylib') AND $(HelixTargetQueues.ToLowerInvariant().Contains('osx'))" 
     Command="install_name_tool -change &quot;/usr/local/opt/libomp/lib/libomp.dylib&quot; &quot;@loader_path/libomp.dylib&quot; $(BUILD_SOURCESDIRECTORY)/artifacts/bin/%(ProjectsWithTargetFramework.Filename)/$(BuildConfig)/%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)/libSymSgdNative.dylib" />
 
@@ -141,6 +144,10 @@
     <Exec Condition="Exists('$(BUILD_SOURCESDIRECTORY)\artifacts\bin\%(ProjectsWithTargetFramework.Filename)\$(BuildConfig)\%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)\runtimes\osx-x64\native\libonnxruntime.dylib') AND $(HelixTargetQueues.ToLowerInvariant().Contains('osx'))" 
     Command="install_name_tool -change &quot;/usr/local/opt/libomp/lib/libomp.dylib&quot; &quot;@loader_path/libomp.dylib&quot; $(BUILD_SOURCESDIRECTORY)/artifacts/bin/%(ProjectsWithTargetFramework.Filename)/$(BuildConfig)/%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)/runtimes/osx-x64/native/libonnxruntime.dylib" />
 
+    <!-- macOS x64 ONLY: if libiomp5.dylib isn't present in the publish folder, duplicate libomp.dylib as libiomp5.dylib -->
+    <Exec Condition="$(HelixTargetQueues.ToLowerInvariant().Contains('osx')) and '$(BuildArchitecture)'=='x64' and Exists('$(BUILD_SOURCESDIRECTORY)\artifacts\bin\%(ProjectsWithTargetFramework.Filename)\$(BuildConfig)\%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)\libomp.dylib') and !Exists('$(BUILD_SOURCESDIRECTORY)\artifacts\bin\%(ProjectsWithTargetFramework.Filename)\$(BuildConfig)\%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)\libiomp5.dylib')"
+      Command="cp &quot;$(BUILD_SOURCESDIRECTORY)/artifacts/bin/%(ProjectsWithTargetFramework.Filename)/$(BuildConfig)/%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)/libomp.dylib&quot; &quot;$(BUILD_SOURCESDIRECTORY)/artifacts/bin/%(ProjectsWithTargetFramework.Filename)/$(BuildConfig)/%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)/libiomp5.dylib&quot;" />
+
     <!-- Remove the native libraries for other OS to save on payload size -->
     <ItemGroup>
       <WindowsFiles Include="$(BUILD_SOURCESDIRECTORY)\artifacts\bin\%(ProjectsWithTargetFramework.Filename)\$(BuildConfig)\%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)\runtimes\win*\**\*.*"/>