Wasm libc rt archive (#5328)

* Add math builtins to wasm compiler-rt, move memset/memcpy/memmove to compiler-rt

* Move extra libc items into its own rt archive, wasm-libc-rt

* Extract files_in_path function

* Older comment was better

* Use files_in_path in more places

* Remove defaults

Author: jgravelle-google

emcc.py

@@ -1104,12 +1104,7 @@ def check(input_file):
       if shared.Settings.WASM_BACKEND:
         options.js_opts = None
         shared.Settings.BINARYEN = 1
-        # Static linking is tricky with LLVM, since e.g. memset might not be used from libc,
-        # but be used as an intrinsic, and codegen will generate a libc call from that intrinsic
-        # *after* static linking would have thought it is all in there. In asm.js this is not an
-        # issue as we do JS linking anyhow, and have asm.js-optimized versions of all the LLVM
-        # intrinsics. But for wasm, we need a better solution. For now, just pin stuff.
-        shared.Settings.EXPORTED_FUNCTIONS += ['_memcpy', '_memmove', '_memset']
+
         # to bootstrap struct_info, we need binaryen
         os.environ['EMCC_WASM_BACKEND_BINARYEN'] = '1'
 

emscripten.py

@@ -1956,17 +1956,20 @@ def create_backend_args_wasm(infile, temp_s, settings):
 
 
 def create_s2wasm_args(temp_s):
-  def compiler_rt_fail():
-    raise Exception('Expected wasm_compiler_rt.a to already be built')
-  compiler_rt_lib = shared.Cache.get('wasm_compiler_rt.a', compiler_rt_fail, 'a')
+  def wasm_rt_fail(archive_file):
+    def wrapped():
+      raise Exception('Expected {} to already be built'.format(archive_file))
+    return wrapped
+  compiler_rt_lib = shared.Cache.get('wasm_compiler_rt.a', wasm_rt_fail('wasm_compiler_rt.a'), 'a')
+  libc_rt_lib = shared.Cache.get('wasm_libc_rt.a', wasm_rt_fail('wasm_libc_rt.a'), 'a')
 
   s2wasm_path = os.path.join(shared.Settings.BINARYEN_ROOT, 'bin', 's2wasm')
 
   args = [s2wasm_path, temp_s, '--emscripten-glue']
   args += ['--global-base=%d' % shared.Settings.GLOBAL_BASE]
   args += ['--initial-memory=%d' % shared.Settings.TOTAL_MEMORY]
   args += ['--allow-memory-growth'] if shared.Settings.ALLOW_MEMORY_GROWTH else []
-  args += ['-l', compiler_rt_lib]
+  args += ['-l', compiler_rt_lib, '-l', libc_rt_lib]
   return args
 
 

tests/core/test_float_builtins.c

@@ -0,0 +1,17 @@
+#include <stdio.h>
+#include <math.h>
+
+int main() {
+    float f1 = 0.1234f;
+    float f2 = 0.5678f;
+    double d1 = 1.0101;
+    double d2 = 0.10101;
+    printf(
+        "%f\n%f\n%f\n%f\n",
+        __builtin_fmaxf(f1, f2),
+        __builtin_fminf(f1, f2),
+        __builtin_fmax(d1, d2),
+        __builtin_fmin(d1, d2)
+    );
+    return 0;
+}

tests/core/test_float_builtins.out

@@ -0,0 +1,4 @@
+0.567800
+0.123400
+1.010100
+0.101010

tests/test_core.py

@@ -1235,10 +1235,13 @@ def test_iswdigit(self):
   def test_polymorph(self):
       self.do_run_in_out_file_test('tests', 'core', 'test_polymorph')
 
-  @no_wasm_backend('no support for complex math division yet')
   def test_complex(self):
     self.do_run_in_out_file_test('tests', 'core', 'test_complex', force_c=True)
 
+  def test_float_builtins(self):
+    if not self.is_wasm_backend(): return self.skip('no __builtin_fmin support in JSBackend')
+    self.do_run_in_out_file_test('tests', 'core', 'test_float_builtins')
+
   @no_wasm_backend("wasm backend doesn't add Runtime.setDynamicTop and crashes")
   def test_segfault(self):
     Settings.SAFE_HEAP = 1

tools/system_libs.py

@@ -27,6 +27,10 @@ def run_commands(commands):
     # https://stackoverflow.com/questions/1408356/keyboard-interrupts-with-pythons-multiprocessing-pool, https://bugs.python.org/issue8296
     pool.map_async(call_process, commands, chunksize=1).get(maxint)
 
+def files_in_path(path_components, filenames):
+  srcdir = shared.path_from_root(*path_components)
+  return [os.path.join(srcdir, f) for f in filenames]
+
 def calculate(temp_files, in_temp, stdout_, stderr_, forced=[]):
   global stdout, stderr
   stdout = stdout_
@@ -124,16 +128,55 @@ def create_libc(libname):
 
   def create_pthreads(libname):
     # Add pthread files.
-    pthreads_files = [os.path.join('pthread', 'library_pthread.c')]
-    pthreads_files += [shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'thread', x) for x in ('pthread_attr_destroy.c', 'pthread_condattr_setpshared.c', 'pthread_mutex_lock.c', 'pthread_spin_destroy.c', 'pthread_attr_get.c', 'pthread_cond_broadcast.c', 'pthread_mutex_setprioceiling.c', 'pthread_spin_init.c', 'pthread_attr_init.c', 'pthread_cond_destroy.c', 'pthread_mutex_timedlock.c', 'pthread_spin_lock.c', 'pthread_attr_setdetachstate.c', 'pthread_cond_init.c', 'pthread_mutex_trylock.c', 'pthread_spin_trylock.c', 'pthread_attr_setguardsize.c', 'pthread_cond_signal.c', 'pthread_mutex_unlock.c', 'pthread_spin_unlock.c', 'pthread_attr_setinheritsched.c', 'pthread_cond_timedwait.c', 'pthread_once.c', 'sem_destroy.c', 'pthread_attr_setschedparam.c', 'pthread_cond_wait.c', 'pthread_rwlockattr_destroy.c', 'sem_getvalue.c', 'pthread_attr_setschedpolicy.c', 'pthread_equal.c', 'pthread_rwlockattr_init.c', 'sem_init.c', 'pthread_attr_setscope.c', 'pthread_getspecific.c', 'pthread_rwlockattr_setpshared.c', 'sem_open.c', 'pthread_attr_setstack.c', 'pthread_key_create.c', 'pthread_rwlock_destroy.c', 'sem_post.c', 'pthread_attr_setstacksize.c', 'pthread_mutexattr_destroy.c', 'pthread_rwlock_init.c', 'sem_timedwait.c', 'pthread_barrierattr_destroy.c', 'pthread_mutexattr_init.c', 'pthread_rwlock_rdlock.c', 'sem_trywait.c', 'pthread_barrierattr_init.c', 'pthread_mutexattr_setprotocol.c', 'pthread_rwlock_timedrdlock.c', 'sem_unlink.c', 'pthread_barrierattr_setpshared.c', 'pthread_mutexattr_setpshared.c', 'pthread_rwlock_timedwrlock.c', 'sem_wait.c', 'pthread_barrier_destroy.c', 'pthread_mutexattr_setrobust.c', 'pthread_rwlock_tryrdlock.c', '__timedwait.c', 'pthread_barrier_init.c', 'pthread_mutexattr_settype.c', 'pthread_rwlock_trywrlock.c', 'vmlock.c', 'pthread_barrier_wait.c', 'pthread_mutex_consistent.c', 'pthread_rwlock_unlock.c', '__wait.c', 'pthread_condattr_destroy.c', 'pthread_mutex_destroy.c', 'pthread_rwlock_wrlock.c', 'pthread_condattr_init.c', 'pthread_mutex_getprioceiling.c', 'pthread_setcanceltype.c', 'pthread_condattr_setclock.c', 'pthread_mutex_init.c', 'pthread_setspecific.c')]
+    pthreads_files = files_in_path(
+      path_components=['system', 'lib', 'libc', 'musl', 'src', 'thread'],
+      filenames=[
+        'pthread_attr_destroy.c', 'pthread_condattr_setpshared.c',
+        'pthread_mutex_lock.c', 'pthread_spin_destroy.c', 'pthread_attr_get.c',
+        'pthread_cond_broadcast.c', 'pthread_mutex_setprioceiling.c',
+        'pthread_spin_init.c', 'pthread_attr_init.c', 'pthread_cond_destroy.c',
+        'pthread_mutex_timedlock.c', 'pthread_spin_lock.c',
+        'pthread_attr_setdetachstate.c', 'pthread_cond_init.c',
+        'pthread_mutex_trylock.c', 'pthread_spin_trylock.c',
+        'pthread_attr_setguardsize.c', 'pthread_cond_signal.c',
+        'pthread_mutex_unlock.c', 'pthread_spin_unlock.c',
+        'pthread_attr_setinheritsched.c', 'pthread_cond_timedwait.c',
+        'pthread_once.c', 'sem_destroy.c', 'pthread_attr_setschedparam.c',
+        'pthread_cond_wait.c', 'pthread_rwlockattr_destroy.c', 'sem_getvalue.c',
+        'pthread_attr_setschedpolicy.c', 'pthread_equal.c', 'pthread_rwlockattr_init.c',
+        'sem_init.c', 'pthread_attr_setscope.c', 'pthread_getspecific.c',
+        'pthread_rwlockattr_setpshared.c', 'sem_open.c', 'pthread_attr_setstack.c',
+        'pthread_key_create.c', 'pthread_rwlock_destroy.c', 'sem_post.c',
+        'pthread_attr_setstacksize.c', 'pthread_mutexattr_destroy.c',
+        'pthread_rwlock_init.c', 'sem_timedwait.c', 'pthread_barrierattr_destroy.c',
+        'pthread_mutexattr_init.c', 'pthread_rwlock_rdlock.c', 'sem_trywait.c',
+        'pthread_barrierattr_init.c', 'pthread_mutexattr_setprotocol.c',
+        'pthread_rwlock_timedrdlock.c', 'sem_unlink.c',
+        'pthread_barrierattr_setpshared.c', 'pthread_mutexattr_setpshared.c',
+        'pthread_rwlock_timedwrlock.c', 'sem_wait.c', 'pthread_barrier_destroy.c',
+        'pthread_mutexattr_setrobust.c', 'pthread_rwlock_tryrdlock.c',
+        '__timedwait.c', 'pthread_barrier_init.c', 'pthread_mutexattr_settype.c',
+        'pthread_rwlock_trywrlock.c', 'vmlock.c', 'pthread_barrier_wait.c',
+        'pthread_mutex_consistent.c', 'pthread_rwlock_unlock.c', '__wait.c',
+        'pthread_condattr_destroy.c', 'pthread_mutex_destroy.c',
+        'pthread_rwlock_wrlock.c', 'pthread_condattr_init.c',
+        'pthread_mutex_getprioceiling.c', 'pthread_setcanceltype.c',
+        'pthread_condattr_setclock.c', 'pthread_mutex_init.c',
+        'pthread_setspecific.c'
+      ])
+    pthreads_files += [os.path.join('pthread', 'library_pthread.c')]
     return build_libc(libname, pthreads_files, ['-O2', '-s', 'USE_PTHREADS=1'])
 
   def create_wasm_libc(libname):
-    # in asm.js we just use Math.sin etc., which is good for code size. But wasm doesn't have such builtins, so
-    # we need to bundle in more code
-    # we also build in musl versions of things that we have hand-optimized asm.js for
-    files = ([shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'math', x) for x in ('cos.c', 'cosf.c', 'cosl.c', 'sin.c', 'sinf.c', 'sinl.c', 'tan.c', 'tanf.c', 'tanl.c', 'acos.c', 'acosf.c', 'acosl.c', 'asin.c', 'asinf.c', 'asinl.c', 'atan.c', 'atanf.c', 'atanl.c', 'atan2.c', 'atan2f.c', 'atan2l.c', 'exp.c', 'expf.c', 'expl.c', 'log.c', 'logf.c', 'logl.c', 'pow.c', 'powf.c', 'powl.c')] +
-             [shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'string', x) for x in ('memcpy.c', 'memset.c', 'memmove.c')])
+    # in asm.js we just use Math.sin etc., which is good for code size. But
+    # wasm doesn't have such builtins, so we need to bundle in more code
+    files = files_in_path(
+      path_components=['system', 'lib', 'libc', 'musl', 'src', 'math'],
+      filenames=['cos.c', 'cosf.c', 'cosl.c', 'sin.c', 'sinf.c', 'sinl.c',
+                 'tan.c', 'tanf.c', 'tanl.c', 'acos.c', 'acosf.c', 'acosl.c',
+                 'asin.c', 'asinf.c', 'asinl.c', 'atan.c', 'atanf.c', 'atanl.c',
+                 'atan2.c', 'atan2f.c', 'atan2l.c', 'exp.c', 'expf.c', 'expl.c',
+                 'log.c', 'logf.c', 'logl.c', 'pow.c', 'powf.c', 'powl.c'])
 
     return build_libc(libname, files, ['-O2'])
 
@@ -169,7 +212,11 @@ def create_libcxx(libname):
       'utility.cpp',
       'valarray.cpp'
     ]
-    return build_libcxx(os.path.join('system', 'lib', 'libcxx'), libname, libcxx_files, ['-DLIBCXX_BUILDING_LIBCXXABI=1', '-Oz', '-I' + shared.path_from_root('system', 'lib', 'libcxxabi', 'include')], has_noexcept_version=True)
+    libcxxabi_include = shared.path_from_root('system', 'lib', 'libcxxabi', 'include')
+    return build_libcxx(
+      os.path.join('system', 'lib', 'libcxx'), libname, libcxx_files,
+      ['-DLIBCXX_BUILDING_LIBCXXABI=1', '-Oz', '-I' + libcxxabi_include],
+      has_noexcept_version=True)
 
   # libcxxabi - just for dynamic_cast for now
   def create_libcxxabi(libname):
@@ -188,7 +235,10 @@ def create_libcxxabi(libname):
       'typeinfo.cpp',
       'private_typeinfo.cpp'
     ]
-    return build_libcxx(os.path.join('system', 'lib', 'libcxxabi', 'src'), libname, libcxxabi_files, ['-Oz', '-I' + shared.path_from_root('system', 'lib', 'libcxxabi', 'include')])
+    libcxxabi_include = shared.path_from_root('system', 'lib', 'libcxxabi', 'include')
+    return build_libcxx(
+      os.path.join('system', 'lib', 'libcxxabi', 'src'), libname, libcxxabi_files,
+      ['-Oz', '-I' + libcxxabi_include])
 
   # gl
   def create_gl(libname): # libname is ignored, this is just one .o file
@@ -197,9 +247,9 @@ def create_gl(libname): # libname is ignored, this is just one .o file
     return o
 
   def create_compiler_rt(libname):
-    srcdir = shared.path_from_root('system', 'lib', 'compiler-rt', 'lib', 'builtins')
-    filenames = ['divdc3.c', 'divsc3.c', 'muldc3.c', 'mulsc3.c']
-    files = (os.path.join(srcdir, f) for f in filenames)
+    files = files_in_path(
+      path_components=['system', 'lib', 'compiler-rt', 'lib', 'builtins'],
+      filenames=['divdc3.c', 'divsc3.c', 'muldc3.c', 'mulsc3.c'])
 
     o_s = []
     commands = []
@@ -237,20 +287,7 @@ def create_dlmalloc_split(libname):
     shared.Building.link([dlmalloc_o, split_malloc_o], lib)
     return lib
 
-  def create_wasm_compiler_rt(libname):
-    srcdir = shared.path_from_root('system', 'lib', 'compiler-rt', 'lib', 'builtins')
-    filenames = ['addtf3.c', 'ashlti3.c', 'ashrti3.c', 'atomic.c', 'comparetf2.c',
-                 'divtf3.c', 'divti3.c', 'udivmodti4.c',
-                 'extenddftf2.c', 'extendsftf2.c',
-                 'fixdfti.c', 'fixsfti.c', 'fixtfdi.c', 'fixtfsi.c', 'fixtfti.c',
-                 'fixunsdfti.c', 'fixunssfti.c', 'fixunstfdi.c', 'fixunstfsi.c', 'fixunstfti.c',
-                 'floatditf.c', 'floatsitf.c', 'floattidf.c', 'floattisf.c',
-                 'floatunditf.c', 'floatunsitf.c', 'floatuntidf.c', 'floatuntisf.c', 'lshrti3.c',
-                 'modti3.c', 'multf3.c', 'multi3.c', 'subtf3.c', 'udivti3.c', 'umodti3.c', 'ashrdi3.c',
-                 'ashldi3.c', 'fixdfdi.c', 'floatdidf.c', 'lshrdi3.c', 'moddi3.c',
-                 'trunctfdf2.c', 'trunctfsf2.c', 'umoddi3.c', 'fixunsdfdi.c', 'muldi3.c',
-                 'divdi3.c', 'divmoddi4.c', 'udivdi3.c', 'udivmoddi4.c']
-    files = (os.path.join(srcdir, f) for f in filenames)
+  def create_wasm_rt_lib(libname, files):
     o_s = []
     commands = []
     for src in files:
@@ -264,6 +301,36 @@ def create_wasm_compiler_rt(libname):
     run_commands([[shared.LLVM_AR, 'cr', '-format=gnu', lib] + o_s])
     return lib
 
+  def create_wasm_compiler_rt(libname):
+    files = files_in_path(
+      path_components=['system', 'lib', 'compiler-rt', 'lib', 'builtins'],
+      filenames=['addtf3.c', 'ashlti3.c', 'ashrti3.c', 'atomic.c', 'comparetf2.c',
+                 'divtf3.c', 'divti3.c', 'udivmodti4.c',
+                 'extenddftf2.c', 'extendsftf2.c',
+                 'fixdfti.c', 'fixsfti.c', 'fixtfdi.c', 'fixtfsi.c', 'fixtfti.c',
+                 'fixunsdfti.c', 'fixunssfti.c', 'fixunstfdi.c', 'fixunstfsi.c', 'fixunstfti.c',
+                 'floatditf.c', 'floatsitf.c', 'floattidf.c', 'floattisf.c',
+                 'floatunditf.c', 'floatunsitf.c', 'floatuntidf.c', 'floatuntisf.c', 'lshrti3.c',
+                 'modti3.c', 'multf3.c', 'multi3.c', 'subtf3.c', 'udivti3.c', 'umodti3.c', 'ashrdi3.c',
+                 'ashldi3.c', 'fixdfdi.c', 'floatdidf.c', 'lshrdi3.c', 'moddi3.c',
+                 'trunctfdf2.c', 'trunctfsf2.c', 'umoddi3.c', 'fixunsdfdi.c', 'muldi3.c',
+                 'divdi3.c', 'divmoddi4.c', 'udivdi3.c', 'udivmoddi4.c'])
+    return create_wasm_rt_lib(libname, files)
+
+  def create_wasm_libc_rt(libname):
+    # Static linking is tricky with LLVM, since e.g. memset might not be used from libc,
+    # but be used as an intrinsic, and codegen will generate a libc call from that intrinsic
+    # *after* static linking would have thought it is all in there. In asm.js this is not an
+    # issue as we do JS linking anyhow, and have asm.js-optimized versions of all the LLVM
+    # intrinsics. But for wasm, we need a better solution. For now, make another archive
+    # that gets included at the same time as compiler-rt.
+    math_files = files_in_path(
+      path_components=['system', 'lib', 'libc', 'musl', 'src', 'math'],
+      filenames=['fmaxf.c', 'fminf.c', 'fmax.c', 'fmin.c'])
+    string_files = files_in_path(
+      path_components=['system', 'lib', 'libc', 'musl', 'src', 'string'],
+      filenames=['memcpy.c', 'memset.c', 'memmove.c'])
+    return create_wasm_rt_lib(libname, math_files + string_files)
 
   # Setting this in the environment will avoid checking dependencies and make building big projects a little faster
   # 1 means include everything; otherwise it can be the name of a lib (libcxx, etc.)
@@ -411,7 +478,8 @@ def do_create():
   # Handle backend compiler_rt separately because it is not a bitcode system lib like the others.
   # Here, just ensure that it's in the cache.
   if shared.Settings.BINARYEN and shared.Settings.WASM_BACKEND:
-    crt_file = shared.Cache.get('wasm_compiler_rt.a', lambda: create_wasm_compiler_rt('wasm_compiler_rt.a'), extension='a')
+    shared.Cache.get('wasm_compiler_rt.a', lambda: create_wasm_compiler_rt('wasm_compiler_rt.a'), extension='a')
+    shared.Cache.get('wasm_libc_rt.a', lambda: create_wasm_libc_rt('wasm_libc_rt.a'), extension='a')
 
   for actual in ret:
     if os.path.basename(actual) == 'libcxxabi.bc':