dpkg-shlibdeps: optimize by caching symbols files and objdump objects

* scripts/Dpkg/Shlibs/Objdump.pm (parse): Split into add_object()
and parse().
* scripts/Dpkg/Shlibs/SymbolFile.pm (merge_object_from_symfile):
New function to reintegrate symbols from previous symfiles.
* scripts/dpkg-shlibdeps.pl: Optimizes it by caching parsed
symbols files and objdump objects. This way neither of the
libraries or symbols files are parsed more than once.

This patch significantly improves performance of dpkg-shlibdeps bringing
it near to performance levels of versions prior to 1.14.8 without loosing
any of new functionally at all. Memory requirements are reduced too.

This patch SHOULD NOT change the end result of dpkg-shlibdeps. If it
does, it is a bug.

Improved-by: Raphael Hertzog <hertzog@debian.org>

ChangeLog

@@ -1,3 +1,14 @@
+2008-12-08  Modestas Vainius  <modestas@vainius.eu>
+	    Raphael Hertzog  <hertzog@debian.org>
+
+	* scripts/Dpkg/Shlibs/Objdump.pm (parse): Split into add_object()
+	and parse().
+	* scripts/Dpkg/Shlibs/SymbolFile.pm (merge_object_from_symfile):
+	New function to reintegrate symbols from previous symfiles.
+	* scripts/dpkg-shlibdeps.pl: Optimizes it by caching parsed
+	symbols files and objdump objects. This way neither of the
+	libraries or symbols files are parsed more than once.
+
 2008-12-05  Guillem Jover  <guillem@debian.org>
 
 	* lib/mlib.c: Remove <sys/wait.h> include.

debian/changelog

@@ -86,6 +86,9 @@ dpkg (1.15.0) UNRELEASED; urgency=low
   * Drop some unneeded lintian overrides.
   * Fix a chmod call in dpkg-source to not fail when POSIXLY_CORRECT is set.
     Closes: #506028
+  * Optimize dpkg-shlibdeps by caching parsed symbols files and
+    objdump objects. Thanks to Modestas Vainius <modestas@vainius.eu> for the
+    patch. Closes: #503954
 
   [ Pierre Habouzit ]
   * Add a --query option to update-alternatives. Closes: #336091, #441904

scripts/Dpkg/Shlibs/Objdump.pm

@@ -30,10 +30,8 @@ sub new {
     return $self;
 }
 
-sub parse {
-    my ($self, $file) = @_;
-    my $obj = Dpkg::Shlibs::Objdump::Object->new($file);
-
+sub add_object {
+    my ($self, $obj) = @_;
     my $id = $obj->get_id;
     if ($id) {
 	$self->{objects}{$id} = $obj;
@@ -41,6 +39,12 @@ sub parse {
     return $id;
 }
 
+sub parse {
+    my ($self, $file) = @_;
+    my $obj = Dpkg::Shlibs::Objdump::Object->new($file);
+
+    return $self->add_object($obj);
+}
 
 sub locate_symbol {
     my ($self, $name) = @_;
@@ -55,12 +59,17 @@ sub locate_symbol {
 
 sub get_object {
     my ($self, $objid) = @_;
-    if (exists $self->{objects}{$objid}) {
+    if ($self->has_object($objid)) {
 	return $self->{objects}{$objid};
     }
     return undef;
 }
 
+sub has_object {
+    my ($self, $objid) = @_;
+    return exists $self->{objects}{$objid};
+}
+
 {
     my %format; # Cache of result
     sub get_format {

scripts/Dpkg/Shlibs/SymbolFile.pm

@@ -175,6 +175,18 @@ sub load {
     delete $seen->{$file};
 }
 
+
+# Beware: we reuse the data structure of the provided symfile so make
+# sure to not modify them after having called this function
+sub merge_object_from_symfile {
+    my ($self, $src, $objid) = @_;
+    if (not $self->has_object($objid)) {
+        $self->{objects}{$objid} = $src->{objects}{$objid};
+    } else {
+        warning(_g("Tried to merge the same object (%s) twice in a symfile."), $objid);
+    }
+}
+
 sub save {
     my ($self, $file, $with_deprecated) = @_;
     $file = $self->{file} unless defined($file);

scripts/dpkg-shlibdeps.pl

@@ -131,6 +131,11 @@ my %global_soname_notfound;
 my %global_soname_used;
 my %global_soname_needed;
 
+# Symfile and objdump caches
+my %symfile_cache;
+my %objdump_cache;
+my %symfile_has_soname_cache;
+
 my $cur_field;
 foreach my $file (keys %exec) {
     $cur_field = $exec{$file};
@@ -193,17 +198,21 @@ foreach my $file (keys %exec) {
 
 	# Load symbols/shlibs files from packages providing libraries
 	foreach my $pkg (@{$file2pkg->{$lib}}) {
-	    my $dpkg_symfile;
+	    my $symfile_path;
 	    if ($packagetype eq "deb") {
 		# Use fine-grained dependencies only on real deb
-		$dpkg_symfile = find_symbols_file($pkg, $soname, $lib);
-		if (defined $dpkg_symfile) {
-		    # Load symbol information
-		    print "Using symbols file $dpkg_symfile for $soname\n" if $debug;
-		    $symfile->load($dpkg_symfile);
-		}
-	    }
-	    if (defined($dpkg_symfile) && $symfile->has_object($soname)) {
+		$symfile_path = find_symbols_file($pkg, $soname, $lib);
+            }
+            if (defined($symfile_path)) {
+                # Load symbol information
+                print "Using symbols file $symfile_path for $soname\n" if $debug;
+                unless (exists $symfile_cache{$symfile_path}) {
+                    $symfile_cache{$symfile_path} =
+                        Dpkg::Shlibs::SymbolFile->new($symfile_path);
+                }
+                $symfile->merge_object_from_symfile($symfile_cache{$symfile_path}, $soname);
+            }
+	    if (defined($symfile_path) && $symfile->has_object($soname)) {
 		# Initialize dependencies with the smallest minimal version
                 # of all symbols (unversioned dependency is not ok as the
                 # library might not have always been available in the
@@ -217,13 +226,17 @@ foreach my $file (keys %exec) {
 		}
 	    } else {
 		# No symbol file found, fall back to standard shlibs
-		my $id = $dumplibs_wo_symfile->parse($lib);
+                print "Using shlibs+objdump for $soname (file $lib)\n" if $debug;
+                unless (exists $objdump_cache{$lib}) {
+                    $objdump_cache{$lib} = Dpkg::Shlibs::Objdump::Object->new($lib);
+                }
+                my $libobj = $objdump_cache{$lib};
+                my $id = $dumplibs_wo_symfile->add_object($libobj);
 		if (($id ne $soname) and ($id ne $lib)) {
 		    warning(_g("%s has an unexpected SONAME (%s)"), $lib, $id);
 		    $alt_soname{$id} = $soname;
 		}
 		push @soname_wo_symfile, $soname;
-		my $libobj = $dumplibs_wo_symfile->get_object($id);
 		# Only try to generate a dependency for libraries with a SONAME
 		if ($libobj->is_public_library() and not
 		    add_shlibs_dep($soname, $pkg, $lib)) {
@@ -672,6 +685,11 @@ sub find_symbols_file {
 
 sub symfile_has_soname {
     my ($file, $soname) = @_;
+
+    if (exists $symfile_has_soname_cache{$file}{$soname}) {
+        return $symfile_has_soname_cache{$file}{$soname};
+    }
+
     open(SYM_FILE, "<", $file) ||
         syserr(_g("cannot open file %s"), $file);
     my $result = 0;
@@ -682,6 +700,7 @@ sub symfile_has_soname {
 	}
     }
     close(SYM_FILE);
+    $symfile_has_soname_cache{$file}{$soname} = $result;
     return $result;
 }