MoonchildProductions#1251 - Part 17: All the libffi and libxul.so issues, resolved.

https://bugzilla.mozilla.org/show_bug.cgi?id=1185424
http://www.mindfruit.co.uk/2012/06/relocations-relocations.html

The libxul.so fix was implemented by Mozilla in Firefox 57 and personally recommended to me by an Oracle employee on the OpenIndiana mailing list. It can easily be made ifdef XP_SOLARIS, but it seems like the new way is considered a better solution overall by the original author of the code that had it use that null pointer hack to begin with.

I can't link where I found the fix for libffi because I came up with it myself while looking at the way sysv.S does things. Something clicked in my brain while reading that mindfruit link above, though, and gave me enough of a sense of what was going on to be able to fix libffi.

The libffi fix looks a bit hairy because of all the FDE_ENCODE statements, but if you examine the code closely, you should find that it does exactly what it did before on all platforms besides Solaris. I later discovered that people who originally ported Firefox to Solaris never figured this out during the Firefox 52 era and had to use GNU LD for linking libxul.so while using the Sun LD for the rest of the build to make it work. For whatever reason, it works for me now without the GNU LD trick.

4 files changed, 48 insertions, 11 deletions

diff --git a/config/external/ffi/moz.build b/config/external/ffi/moz.build
index e9ef07de3..3a5478967 100644
--- a/config/external/ffi/moz.build
+++ b/config/external/ffi/moz.build
@@ -35,9 +35,17 @@ else:
         'FFI_NO_RAW_API': True,
         'HAVE_AS_ASCII_PSEUDO_OP': True,
         'HAVE_AS_STRING_PSEUDO_OP': True,
-        'HAVE_AS_X86_64_UNWIND_SECTION_TYPE': True,
     })
 
+# This should NEVER be true on 32-bit x86 systems. It's called x86_64 unwind 
+# section type for a reason. By rights the way it was before should have broken
+# all 32-bit builds on x86.
+
+    if CONFIG['FFI_TARGET'] == 'X86':
+        DEFINES['HAVE_AS_X86_64_UNWIND_SECTION_TYPE'] = False
+    else:
+        DEFINES['HAVE_AS_X86_64_UNWIND_SECTION_TYPE'] = True
+
     if CONFIG['MOZ_DEBUG']:
         DEFINES['FFI_DEBUG'] = True
         if not CONFIG['MOZ_NO_DEBUG_RTL']:
@@ -49,13 +57,20 @@ else:
     if CONFIG['OS_TARGET'] not in ('WINNT', 'Darwin'):
         DEFINES['HAVE_HIDDEN_VISIBILITY_ATTRIBUTE'] = True
 
-    if CONFIG['INTEL_ARCHITECTURE']:
+# Solaris uses datarel encoding for x86. This causes a lot of really stupid 
+# problems, like the vast majority of x86 assembler not being considered PIC 
+# on Solaris.
+
+    if CONFIG['INTEL_ARCHITECTURE'] and CONFIG['OS_TARGET'] != 'SunOS':
         DEFINES['HAVE_AS_X86_PCREL'] = True
 
     # Don't bother setting EH_FRAME_FLAGS on Windows.
     # Quoted defines confuse msvcc.sh, and the value isn't used there.
     if CONFIG['OS_TARGET'] != 'WINNT':
-        if CONFIG['FFI_TARGET'] == 'ARM':
+        # Solaris seems to require EH_FRAME to be writable even on x86.
+        # It works fine most of the time and there's no rule against it, 
+        # but it causes a lot of weird problems.
+        if CONFIG['FFI_TARGET'] == 'ARM' or CONFIG['OS_ARCH'] == 'SunOS':
             DEFINES['EH_FRAME_FLAGS'] = '"aw"'
         else:
             DEFINES['EH_FRAME_FLAGS'] = '"a"'
diff --git a/js/src/ctypes/libffi/src/x86/win32.S b/js/src/ctypes/libffi/src/x86/win32.S
index daf0e799c..4f702e8b1 100644
--- a/js/src/ctypes/libffi/src/x86/win32.S
+++ b/js/src/ctypes/libffi/src/x86/win32.S
@@ -1158,8 +1158,24 @@ L_ffi_closure_SYSV_inner$stub:
 	.byte	0x7c	/* .sleb128 -4; CIE Data Alignment Factor */
 	.byte	0x8	/* CIE RA Column */
 #ifdef __PIC__
-	.byte	0x1	/* .uleb128 0x1; Augmentation size */
-	.byte	0x1b	/* FDE Encoding (pcrel sdata4) */
+# if defined __sun__ && defined __svr4__
+/* 32-bit Solaris 2/x86 uses datarel encoding for PIC.  GNU ld before 2.22
+   doesn't correctly sort .eh_frame_hdr with mixed encodings, so match this.  */
+#  define FDE_ENCODING		0x30	/* datarel */
+#  define FDE_ENCODE(X)		X@GOTOFF
+# else
+#  define FDE_ENCODING		0x1b	/* pcrel sdata4 */
+#  if defined HAVE_AS_X86_PCREL
+#   define FDE_ENCODE(X)	X-.
+#  else
+#   define FDE_ENCODE(X)	X@rel
+#  endif
+# endif
+#else
+# define FDE_ENCODING		0	/* absolute */
+# define FDE_ENCODE(X)		X
+.byte	0x1	/* .uleb128 0x1; Augmentation size */
+.byte   FDE_ENCODING
 #endif
 	.byte	0xc	/* DW_CFA_def_cfa CFA = r4 + 4 = 4(%esp) */
 	.byte	0x4	/* .uleb128 0x4 */
@@ -1176,7 +1192,7 @@ L_ffi_closure_SYSV_inner$stub:
 #if defined __PIC__ && defined HAVE_AS_X86_PCREL
 	.long	.LFB1-.	/* FDE initial location */
 #else
-	.long	.LFB1
+	.long	FDE_ENCODE(.LFB1)
 #endif
 	.long	.LFE1-.LFB1	/* FDE address range */
 #ifdef __PIC__
@@ -1207,7 +1223,7 @@ L_ffi_closure_SYSV_inner$stub:
 #if defined __PIC__ && defined HAVE_AS_X86_PCREL
 	.long	.LFB3-.	/* FDE initial location */
 #else
-	.long	.LFB3
+	.long	FDE_ENCODE(.LFB3)
 #endif
 	.long	.LFE3-.LFB3	/* FDE address range */
 #ifdef __PIC__
@@ -1240,7 +1256,7 @@ L_ffi_closure_SYSV_inner$stub:
 #if defined __PIC__ && defined HAVE_AS_X86_PCREL
 	.long	.LFB4-.	/* FDE initial location */
 #else
-	.long	.LFB4
+	.long	FDE_ENCODE(.LFB4)
 #endif
 	.long	.LFE4-.LFB4	/* FDE address range */
 #ifdef __PIC__
@@ -1278,7 +1294,7 @@ L_ffi_closure_SYSV_inner$stub:
 #if defined __PIC__ && defined HAVE_AS_X86_PCREL
 	.long	.LFB5-.	/* FDE initial location */
 #else
-	.long	.LFB5
+	.long	FDE_ENCODE(.LFB5)
 #endif
 	.long	.LFE5-.LFB5	/* FDE address range */
 #ifdef __PIC__
diff --git a/toolkit/library/StaticXULComponentsEnd/StaticXULComponentsEnd.cpp b/toolkit/library/StaticXULComponentsEnd/StaticXULComponentsEnd.cpp
index 28fd9d484..8b5b1f4cb 100644
--- a/toolkit/library/StaticXULComponentsEnd/StaticXULComponentsEnd.cpp
+++ b/toolkit/library/StaticXULComponentsEnd/StaticXULComponentsEnd.cpp
@@ -10,4 +10,7 @@
 #  undef NSMODULE_SECTION
 #  define NSMODULE_SECTION __declspec(allocate(".kPStaticModules$Z"), dllexport)
 #endif
-NSMODULE_DEFN(end_kPStaticModules) = nullptr;
+/* This could be null, but this needs a dummy value to ensure it actually ends
+ * up in the same section as other NSMODULE_DEFNs, instead of being moved to a
+ * separate readonly section. */
+NSMODULE_DEFN(end_kPStaticModules) = (mozilla::Module*)&NSMODULE_NAME(end_kPStaticModules);
diff --git a/toolkit/library/StaticXULComponentsStart.cpp b/toolkit/library/StaticXULComponentsStart.cpp
index 1738aa810..d2e9a8828 100644
--- a/toolkit/library/StaticXULComponentsStart.cpp
+++ b/toolkit/library/StaticXULComponentsStart.cpp
@@ -1,3 +1,6 @@
 #include "mozilla/Module.h"
 
-NSMODULE_DEFN(start_kPStaticModules) = nullptr;
+/* This could be null, but this needs a dummy value to ensure it actually ends
+ * up in the same section as other NSMODULE_DEFNs, instead of being moved to a
+ * separate readonly section. */
+NSMODULE_DEFN(start_kPStaticModules) = (mozilla::Module*)&NSMODULE_NAME(start_kPStaticModules);