Allow inclusion of binary files outside of the debian subdirectory

* scripts/Dpkg/ErrorHandling.pm (errormsg): New function displaying
an error message to stderr without dying.
* scripts/Dpkg/Source/Patch.pm (add_diff_file): Add a hook in the
handling of modified binary files. Don't fail unconditionnaly when a
binary file is detected. Return a value indicating if the diff
call succeeded.
(add_diff_directory): Don't display some warnings (that apply only to
files whose changes can be represented in diff) for modified
binary files.
(_fail_with_msg, _fail_not_same_type): Use new errormsg() function.
* scripts/Dpkg/Source/Package/V2_0.pm (do_extract): Extract the debian
tarball "in-place" so that it can contain other files besides the debian
sub-directory.
(do_build): Handle modified binary files. They are added to the debian
tarball if --include-binaries is used, or if they are whitelisted in
debian/source/include-binaries. The usage of --include-binaries will
modify debian/source/include-binaries if needed.
* man/dpkg-source.1: Update manual page accordingly.

man/dpkg-source.1

@@ -323,8 +323,11 @@ tarballs (\fB.orig-\fP\fIcomponent\fP\fB.tar.\fP\fIext\fP).
 The main original tarball is extracted first, then all additional original
 tarballs are extracted in subdirectories named after the \fIcomponent\fP
 part of their filename (any pre-existing directory is replaced). The
-debian tarball is extracted in a \fBdebian\fP subdirectory (and replaces
-any pre-existing \fBdebian\fP directory).
+debian tarball is extracted on top of the source directory after prior
+removal of any pre-existing \fBdebian\fP directory). Note that the
+debian tarball must contain a \fBdebian\fP sub-directory but it
+can also contain binary files outside of that directory (see
+\fB\-\-include\-binaries\fP option).
 .PP
 All patches in \fBdebian/patches\fP matching the perl regular expression
 \fB[\\w\-]+\fP are then applied in alphabetical order. The timestamp of
@@ -339,8 +342,14 @@ debian directory is copied over in the temporary directory, and all
 patches except \fBzz_debian-diff-auto\fP are applied.
 The temporary directory is compared to the source package directory
 and the diff (if non-empty) is stored in
-\fBdebian/patches/zz_debian-diff-auto\fP. The updated debian directory is
-then used to regenerate the debian tarball.
+\fBdebian/patches/zz_debian-diff-auto\fP. Any change on a binary file
+is not representable in a diff and will thus lead to a failure unless
+the maintainer deliberately decided to include that modified binary
+file in the debian tarball (by listing it in
+\fBdebian/source/include-binaries\fP).
+
+The updated debian directory and the list of modified binaries is then
+used to regenerate the debian tarball.
 
 The automatically generated diff doesn't include changes on VCS specific
 files as well as many temporary files (see default value associated to
@@ -364,12 +373,17 @@ in the 1.0 format. It supports all compression methods and
 will ignore by default any VCS specific files and directories
 as well as many temporary files (see default value associated to
 \fB-I\fP option in the \fB\-\-help\fP output).
+.TP
+.B \-\-include\-binaries
+Add all modified binaries in the debian tarball. Also add them to
+\fBdebian/source/include-binaries\fP: they will be added by default
+in subsequent builds and this option is thus no more needed.
 .
 .SS Format: 3.0 (quilt)
 This is a variant of the 2.0 format. The differences concern the
 management of the patches. This format uses an explicit list of
 patches contained in \fBdebian/patches/debian.series\fP or
-\fBdebian/patches/series\fP. The patches can remove files.
+\fBdebian/patches/series\fP.
 .PP
 .B Extracting
 .PP

scripts/Dpkg/ErrorHandling.pm

@@ -4,9 +4,9 @@ use Dpkg;
 use Dpkg::Gettext;
 
 use base qw(Exporter);
-our @EXPORT_OK = qw(warning warnerror error failure unknown syserr internerr
-                    subprocerr usageerr syntaxerr report info
-		    $warnable_error $quiet_warnings);
+our @EXPORT_OK = qw(warning warnerror error errormsg failure unknown
+                    syserr internerr subprocerr usageerr syntaxerr report
+                    info $warnable_error $quiet_warnings);
 
 our $warnable_error = 1;
 our $quiet_warnings = 0;
@@ -54,6 +54,11 @@ sub error($;@)
     die report(_g("error"), @_);
 }
 
+sub errormsg($;@)
+{
+    print STDERR report(_g("error"), @_);
+}
+
 sub internerr($;@)
 {
     die report(_g("internal error"), @_);

scripts/Dpkg/Source/Package/V2_0.pm

@@ -23,7 +23,7 @@ use base 'Dpkg::Source::Package';
 
 use Dpkg;
 use Dpkg::Gettext;
-use Dpkg::ErrorHandling qw(error syserr warning usageerr subprocerr info);
+use Dpkg::ErrorHandling qw(error errormsg syserr warning usageerr subprocerr info);
 use Dpkg::Compression;
 use Dpkg::Source::Archive;
 use Dpkg::Source::Patch;
@@ -44,6 +44,8 @@ sub init_options {
         unless exists $self->{'options'}{'include_removal'};
     $self->{'options'}{'include_timestamp'} = 0
         unless exists $self->{'options'}{'include_timestamp'};
+    $self->{'options'}{'include_binaries'} = 0
+        unless exists $self->{'options'}{'include_binaries'};
 }
 
 sub parse_cmdline_option {
@@ -54,6 +56,9 @@ sub parse_cmdline_option {
     } elsif ($opt =~ /^--include-timestamp$/) {
         $self->{'options'}{'include_timestamp'} = 1;
         return 1;
+    } elsif ($opt =~ /^--include-binaries$/) {
+        $self->{'options'}{'include_binaries'} = 1;
+        return 1;
     }
     return 0;
 }
@@ -96,7 +101,7 @@ sub do_extract {
     # Extract main tarball
     info(_g("unpacking %s"), $tarfile);
     my $tar = Dpkg::Source::Archive->new(filename => "$dscdir$tarfile");
-    $tar->extract($newdirectory);
+    $tar->extract($newdirectory, no_fixperms => 1);
 
     # Extract additional orig tarballs
     foreach my $subdir (keys %origtar) {
@@ -107,14 +112,14 @@ sub do_extract {
             erasedir("$newdirectory/$subdir");
         }
         $tar = Dpkg::Source::Archive->new(filename => "$dscdir$file");
-        $tar->extract("$newdirectory/$subdir");
+        $tar->extract("$newdirectory/$subdir", no_fixperms => 1);
     }
 
     # Extract debian tarball after removing the debian directory
     info(_g("unpacking %s"), $debianfile);
     erasedir("$newdirectory/debian");
     $tar = Dpkg::Source::Archive->new(filename => "$dscdir$debianfile");
-    $tar->extract("$newdirectory/debian");
+    $tar->extract($newdirectory, in_place => 1);
 
     # Apply patches (in a separate method as it might be overriden)
     $self->apply_patches($newdirectory);
@@ -176,6 +181,7 @@ sub do_build {
     my ($self, $dir) = @_;
     my @argv = @{$self->{'options'}{'ARGV'}};
     my @tar_ignore = map { "--exclude=$_" } @{$self->{'options'}{'tar_ignore'}};
+    my $include_binaries = $self->{'options'}{'include_binaries'};
 
     my ($dirname, $updir) = fileparse($dir);
     if (scalar(@argv)) {
@@ -232,6 +238,36 @@ sub do_build {
     # Apply all patches except the last automatic one
     $self->apply_patches($tmp, 1);
 
+    # Prepare handling of binary files
+    my %auth_bin_files;
+    my $incbin_file = File::Spec->catfile($dir, "debian", "source", "include-binaries");
+    if (-f $incbin_file) {
+        open(INC, "<", $incbin_file) || syserr(_g("can't read %s"), $incbin_file);
+        while(defined($_ = <INC>)) {
+            chomp; s/^\s*//; s/\s*$//;
+            next if /^#/;
+            $auth_bin_files{$_} = 1;
+        }
+        close(INC);
+    }
+    my @binary_files;
+    my $handle_binary = sub {
+        my ($self, $old, $new) = @_;
+        my $relfn = File::Spec->abs2rel($new, $dir);
+        # Include binaries if they are whitelisted or if
+        # --include-binaries has been given
+        if ($include_binaries or $auth_bin_files{$relfn}) {
+            push @binary_files, $relfn;
+        } else {
+            errormsg(_g("cannot represent change to %s: %s"), $new,
+                     _g("binary file contents changed"));
+            errormsg(_g("add %s in debian/source/include-binaries if you want" .
+                     " to store the modified binary in the debian tarball"),
+                     $relfn);
+            $self->register_error();
+        }
+    };
+
     # Create a patch
     my ($difffh, $tmpdiff) = tempfile("$basenamerev.diff.XXXXXX",
                                       DIR => $updir, UNLINK => 0);
@@ -240,11 +276,12 @@ sub do_build {
                                         compression => "none");
     $diff->create();
     $diff->add_diff_directory($tmp, $dir, basedirname => $basedirname,
-            %{$self->{'diff_options'}});
+            %{$self->{'diff_options'}}, handle_binary_func => $handle_binary);
     error(_g("unrepresentable changes to source")) if not $diff->finish();
     # The previous auto-patch must be removed, it has not been used and it
     # will be recreated if it's still needed
-    my $autopatch = "$dir/debian/patches/" . $self->get_autopatch_name();
+    my $autopatch = File::Spec->catfile($dir, "debian", "patches",
+                                        $self->get_autopatch_name());
     if (-e $autopatch) {
         unlink($autopatch) || syserr(_g("cannot remove %s"), $autopatch);
     }
@@ -263,12 +300,27 @@ sub do_build {
     erasedir($tmp);
     pop @Dpkg::Exit::handlers;
 
+    # Update debian/source/include-binaries if needed
+    if (scalar(@binary_files) and $include_binaries) {
+        mkpath(File::Spec->catdir($dir, "debian", "source"));
+        open(INC, ">>", $incbin_file) || syserr(_g("can't write %s"), $incbin_file);
+        foreach my $binary (@binary_files) {
+            unless ($auth_bin_files{$binary}) {
+                print INC "$binary\n";
+                info(_g("adding %s to %s"), $binary, "debian/source/include-binaries");
+            }
+        }
+        close(INC);
+    }
     # Create the debian.tar
     my $debianfile = "$basenamerev.debian.tar." . $self->{'options'}{'comp_ext'};
     info(_g("building %s in %s"), $sourcepackage, $debianfile);
     $tar = Dpkg::Source::Archive->new(filename => $debianfile);
     $tar->create(options => \@tar_ignore, 'chdir' => $dir);
     $tar->add_directory("debian");
+    foreach my $binary (@binary_files) {
+        $tar->add_file($binary);
+    }
     $tar->finish();
 
     $self->add_file($debianfile);

scripts/Dpkg/Source/Patch.pm

@@ -25,7 +25,7 @@ use Dpkg::Source::Compressor;
 use Dpkg::Compression;
 use Dpkg::Gettext;
 use Dpkg::IPC;
-use Dpkg::ErrorHandling qw(error syserr warning subprocerr);
+use Dpkg::ErrorHandling qw(error errormsg syserr warning subprocerr);
 
 use POSIX;
 use File::Find;
@@ -59,6 +59,10 @@ sub create {
 sub add_diff_file {
     my ($self, $old, $new, %opts) = @_;
     $opts{"include_timestamp"} = 0 unless exists $opts{"include_timestamp"};
+    my $handle_binary = $opts{"handle_binary_func"} || sub {
+        my ($self, $old, $new) = @_;
+        $self->_fail_with_msg($new, _g("binary file contents changed"));
+    };
     # Default diff options
     my @options;
     if ($opts{"options"}) {
@@ -94,9 +98,11 @@ sub add_diff_file {
     );
     # Check diff and write it in patch file
     my $difflinefound = 0;
+    my $binary = 0;
     while (<$diffgen>) {
         if (m/^binary/i) {
-            $self->_fail_with_msg($new,_g("binary file contents changed"));
+            $binary = 1;
+            &$handle_binary($self, $old, $new);
             last;
         } elsif (m/^[-+\@ ]/) {
             $difflinefound++;
@@ -115,11 +121,12 @@ sub add_diff_file {
                cmdline => "diff -u @options -- $old $new");
     # Verify diff process ended successfully
     # Exit code of diff: 0 => no difference, 1 => diff ok, 2 => error
+    # Ignore error if binary content detected
     my $exit = WEXITSTATUS($?);
-    unless (WIFEXITED($?) && ($exit == 0 || $exit == 1)) {
+    unless (WIFEXITED($?) && ($exit == 0 || $exit == 1 || $binary)) {
         subprocerr(_g("diff on %s"), $new);
     }
-    return $exit;
+    return ($exit == 0 || $exit == 1);
 }
 
 sub add_diff_directory {
@@ -163,6 +170,17 @@ sub add_diff_directory {
                 $! == ENOENT ||
                     syserr(_g("cannot stat file %s"), "$old/$fn");
                 $old_file = '/dev/null';
+            } elsif (not -f _) {
+                $self->_fail_not_same_type("$old/$fn", "$new/$fn");
+                return;
+            }
+
+            my $success = $self->add_diff_file($old_file, "$new/$fn",
+                label_old => "$basedir.orig/$fn",
+                label_new => "$basedir/$fn",
+                %opts);
+
+            if ($success and ($old_file eq "/dev/null")) {
                 if (not $size) {
                     warning(_g("newly created empty file '%s' will not " .
                                "be represented in diff"), $fn);
@@ -177,15 +195,7 @@ sub add_diff_directory {
                                    "be represented in diff"), $mode, $fn);
                     }
                 }
-            } elsif (not -f _) {
-                $self->_fail_not_same_type("$old/$fn", "$new/$fn");
-                return;
             }
-
-            $self->add_diff_file($old_file, "$new/$fn",
-                label_old => "$basedir.orig/$fn",
-                label_new => "$basedir/$fn",
-                %opts);
         } elsif (-p _) {
             unless (-p "$old/$fn") {
                 $self->_fail_not_same_type("$old/$fn", "$new/$fn");
@@ -240,21 +250,23 @@ sub finish {
     return not $self->{'errors'};
 }
 
+sub register_error {
+    my ($self) = @_;
+    $self->{'errors'}++;
+}
 sub _fail_with_msg {
     my ($self, $file, $msg) = @_;
-    printf(STDERR _g("%s: cannot represent change to %s: %s")."\n",
-                  $progname, $file, $msg);
-    $self->{'errors'}++;
+    errormsg(_g("cannot represent change to %s: %s"), $file, $msg);
+    $self->register_error();
 }
 sub _fail_not_same_type {
     my ($self, $old, $new) = @_;
     my $old_type = get_type($old);
     my $new_type = get_type($new);
-    printf(STDERR _g("%s: cannot represent change to %s:\n".
-                     "%s:  new version is %s\n".
-                     "%s:  old version is %s\n"),
-                  $progname, $new, $progname, $old_type, $progname, $new_type);
-    $self->{'errors'}++;
+    errormsg(_g("cannot represent change to %s:"), $new);
+    errormsg(_g("  new version is %s"), $old_type);
+    errormsg(_g("  old version is %s"), $new_type);
+    $self->register_error();
 }
 
 # check diff for sanity, find directories to create as a side effect