[Patch] skipping import libraries for performance reasons - direct auto-import of dll's

Ralf Habacker kde-cygwin@mail.kde.org
Mon, 25 Nov 2002 13:46:50 +0100

This is a multi-part message in MIME format.

Content-Type: text/plain;
Content-Transfer-Encoding: 7bit

Hi all,

the recent ld contains support importing of symbols located in other dll's
through import libraries.

Especially for big libraries using import libraries has some big disadvantages:

1. ld need much time and very much memory to process them  - This is because
each symbol is located in a seperate objectfile and ld opens a bfd for each.

2. The size of import libraries could be very huge (for example 10 MB for
qt-3) - This is because each symbol is located in a seperate objectfile.

current ld releases are able to skip import libraries by grabbing the needed
informations directly from a dll. The only issue which was open until now, is
that ld has no auto-import support for linking directly to a dll, but this is be
fixed with the appended patch.

Using dll's directly instead of import libraries has several advantages:

1. linking goes much faster (for qt-3 about 50%: from 2 minutes down to one

2. ld uses much less memory (for qt-3 about 70%: from 500 MB to 150 MB)

3. ld works more like the linux version. There are only static archives and
shared libraries which could be linked directly without the indirection of using
import libraries. This simplifies for example libtool handling.

How does it works:

See the example path layout. There are two dll's xxx.dll and yyy.dll in an
applications bin dir. In the lib dir there are the import library for xxx.dll
and a link to the dll cygyyy.dll.


	[cyg|lib|]yyy.dll	(may be the dll  or a symbolic link to ../bin/cygyyy.dll)

Now the link line (you can see that using the dll directly is the same as using
the import libraries, there is no difference)

gcc -Wl,-verbose  -o a.exe -L../lib/ -lxxx -lyyy ...

The ld log below shows, that for xxx the import library is used and for yyy the
dll is directly used. This means, the only task of using the dll instead of the
import library is to replace the import library with a symbolic link to the dll.

attempt to open /usr/lib/crt0.o succeeded
attempt to open client.o succeeded
attempt to open ../lib/libxxx.dll.a succeeded !!!
attempt to open ../lib/libyyy.dll.a failed
attempt to open ../lib/yyy.dll.a failed
attempt to open ../lib/libyyy.a failed
attempt to open ../lib/cygyyy.dll failed
attempt to open ../lib/libyyy.dll failed
attempt to open ../lib/yyy.dll succeeded !!!
Info: resolving _var by linking to __imp__var (auto-import)
Info: resolving _foo by linking to __imp__foo (auto-import)
Info: resolving _func_ptr by linking to __imp__func_ptr (auto-import)

I have tested this patch with recent qt-2/3 and kde2 sources without any
problems. A sample test case is appended too.

Please give this patch a try and report problems to me o0r to this list.

Thank you

Ralf Habacker

KDE on cygwin http://cygwin.kde.org

Content-Type: application/octet-stream;
Content-Transfer-Encoding: quoted-printable
Content-Disposition: attachment;

Index: deffile.h=0A=
RCS file: /cvs/src/src/ld/deffile.h,v=0A=
retrieving revision 1.5=0A=
diff -u -3 -p -B -u -b -B -r1.5 deffile.h=0A=
--- deffile.h	13 Mar 2001 06:14:27 -0000	1.5=0A=
+++ deffile.h	20 Nov 2002 21:23:45 -0000=0A=
@@ -52,6 +52,7 @@ typedef struct def_file_import {=0A=
   def_file_module *module;	/* always set */=0A=
   char *name;			/* may be NULL; either this or ordinal will be set */=0A=
   int ordinal;			/* may be -1 */=0A=
+  int data;					/* =3D 1 if data */ =0A=
 } def_file_import;=0A=
 typedef struct def_file {=0A=
Index: pe-dll.c=0A=
RCS file: /cvs/src/src/ld/pe-dll.c,v=0A=
retrieving revision 1.48=0A=
diff -u -3 -p -B -u -b -B -r1.48 pe-dll.c=0A=
--- pe-dll.c	14 Nov 2002 18:03:16 -0000	1.48=0A=
+++ pe-dll.c	20 Nov 2002 21:24:20 -0000=0A=
@@ -2418,7 +2418,7 @@ pe_process_import_defs (output_bfd, link=0A=
 		exp.hint =3D exp.ordinal >=3D 0 ? exp.ordinal : 0;=0A=
 		exp.flag_private =3D 0;=0A=
 		exp.flag_constant =3D 0;=0A=
-		exp.flag_data =3D 0;=0A=
+    exp.flag_data =3D pe_def_file->imports[i].data;=0A=
 		exp.flag_noname =3D exp.name ? 0 : 1;=0A=
 		one =3D make_one (&exp, output_bfd);=0A=
 		add_bfd_to_link (one, one->filename, link_info);=0A=
@@ -2490,11 +2490,13 @@ pe_implied_import_dll (filename)=0A=
   bfd *dll;=0A=
   unsigned long pe_header_offset, opthdr_ofs, num_entries, i;=0A=
-  unsigned long export_rva, export_size, nsections, secptr, expptr;=0A=
+  unsigned long export_rva, export_size, nsections, secptr, =
   unsigned char *expdata, *erva;=0A=
   unsigned long name_rvas, ordinals, nexp, ordbase;=0A=
   const char *dll_name;=0A=
+  unsigned long data_start, data_end, bss_start, bss_end;=0A=
   /* No, I can't use bfd here.  kernel32.dll puts its export table in=0A=
      the middle of the .rdata section.  */=0A=
   dll =3D bfd_openr (filename, pe_details->target_name);=0A=
@@ -2511,11 +2513,8 @@ pe_implied_import_dll (filename)=0A=
       return false;=0A=
-  dll_name =3D filename;=0A=
-  for (i =3D 0; filename[i]; i++)=0A=
-    if (filename[i] =3D=3D '/' || filename[i] =3D=3D '\\' || =
filename[i] =3D=3D ':')=0A=
-      dll_name =3D filename + i + 1;=0A=
+  /* get pe-header, optional header and numbers  =0A=
+     of export entries */=0A=
   pe_header_offset =3D pe_get32 (dll, 0x3c);=0A=
   opthdr_ofs =3D pe_header_offset + 4 + 20;=0A=
   num_entries =3D pe_get32 (dll, opthdr_ofs + 92);=0A=
@@ -2530,6 +2529,7 @@ pe_implied_import_dll (filename)=0A=
 	    pe_get16 (dll, pe_header_offset + 4 + 16));=0A=
   expptr =3D 0;=0A=
+  /* get the rva and size of the export section */ =0A=
   for (i =3D 0; i < nsections; i++)=0A=
       char sname[8];=0A=
@@ -2550,6 +2550,36 @@ pe_implied_import_dll (filename)=0A=
+  /* scan sections and store the base and =0A=
+     size of the data and bss segments in =0A=
+     data/base_start/end */ =0A=
+  for (i =3D 0; i < nsections; i++) {=0A=
+      unsigned long secptr1 =3D secptr + 40 * i;=0A=
+      unsigned long vsize =3D pe_get32 (dll, secptr1 + 8);=0A=
+      unsigned long vaddr =3D pe_get32 (dll, secptr1 + 12);=0A=
+      unsigned long flags =3D pe_get32 (dll, secptr1 + 36);=0A=
+      char sec_name[9];=0A=
+      sec_name[8] =3D '\0';=0A=
+      bfd_seek (dll, (file_ptr) secptr1 + 0, SEEK_SET);=0A=
+      bfd_bread (sec_name, (bfd_size_type) 8, dll);=0A=
+      if (strcmp(sec_name,".data") =3D=3D 0) {=0A=
+        data_start =3D vaddr;=0A=
+        data_end =3D vaddr + vsize;=0A=
+        if (pe_dll_extra_pe_debug)=0A=
+          printf("%s %s: 0x%08x-0x%08x =
+      }=0A=
+      else if (strcmp(sec_name,".bss") =3D=3D 0) {=0A=
+        bss_start =3D vaddr;=0A=
+        bss_end =3D vaddr + vsize;=0A=
+        if (pe_dll_extra_pe_debug)=0A=
+          printf("%s %s: 0x%08x-0x%08x =
+      }=0A=
+  }=0A=
   expdata =3D (unsigned char *) xmalloc (export_size);=0A=
   bfd_seek (dll, (file_ptr) expptr, SEEK_SET);=0A=
   bfd_bread (expdata, (bfd_size_type) export_size, dll);=0A=
@@ -2562,14 +2592,39 @@ pe_implied_import_dll (filename)=0A=
   name_rvas =3D pe_as32 (expdata + 32);=0A=
   ordinals =3D pe_as32 (expdata + 36);=0A=
   ordbase =3D pe_as32 (expdata + 16);=0A=
+  exp_funcbase =3D pe_as32 (expdata + 28);=0A=
+  /* use internal dll name instead of filename =0A=
+     to enable symbolic dll linking */ =0A=
+  dll_name =3D pe_as32 (expdata + 12) + erva ;=0A=
+	/* iterate through the list of symbols */ =0A=
   for (i =3D 0; i < nexp; i++)=0A=
+    	/* pointer to the names vector */ =0A=
       unsigned long name_rva =3D pe_as32 (erva + name_rvas + i * 4);=0A=
       def_file_import *imp;=0A=
+    	/* pointer to the function address vector */ =0A=
+      unsigned long func_rva =3D pe_as32 (erva + exp_funcbase + i * 4);=0A=
+      int is_data =3D 0;=0A=
+        // skip unwanted symbols, which are exported in buggy =
auto-import releases=0A=
+        if (strstr(erva + name_rva,"_nm_") =3D=3D 0) {=0A=
+					/* is_data is true if the address is in the data or bss segment */ =0A=
+          is_data =3D (func_rva >=3D data_start && func_rva < data_end )=0A=
+                  || (func_rva >=3D bss_start && func_rva < bss_end);=0A=
       imp =3D def_file_add_import (pe_def_file, erva + name_rva, =
 				 i, 0);=0A=
+        	/* mark symbole type */=0A=
+          imp->data =3D is_data;=0A=
+          if (pe_dll_extra_pe_debug)=0A=
+            printf("%s dll-name: %s sym: %s addr: 0x%x =
%s\n",__FUNCTION__, dll_name, erva + name_rva, func_rva, is_data ? =
"(data)" : "");=0A=
+        }=0A=
   return true;=0A=
Index: emultempl/pe.em=0A=
RCS file: /cvs/src/src/ld/emultempl/pe.em,v=0A=
retrieving revision 1.69=0A=
diff -u -3 -p -B -u -b -B -r1.69 pe.em=0A=
--- emultempl/pe.em	14 Nov 2002 18:03:16 -0000	1.69=0A=
+++ emultempl/pe.em	20 Nov 2002 21:24:38 -0000=0A=
@@ -1009,9 +1009,10 @@ gld_${EMULATION_NAME}_after_open ()=0A=
   if (pe_enable_stdcall_fixup) /* -1=3Dwarn or 1=3Ddisable */=0A=
     pe_fixup_stdcalls ();=0A=
+  pe_process_import_defs(output_bfd, &link_info);=0A=
   pe_find_data_imports ();=0A=
-  pe_process_import_defs(output_bfd, &link_info);=0A=
   if (link_info.shared)=0A=
     pe_dll_build_sections (output_bfd, &link_info);=0A=

Content-Type: application/octet-stream;
Content-Transfer-Encoding: base64
Content-Disposition: attachment;