dpkg: Allow shared files between Multi-Arch: same packages

If all the already installed packages are synchronized, and the version
to be installed is the same, then we check that the shared file has the
same contents as what's already installed. Otherwise we blindly allow
the updated package to replace the file.

For conffiles, instead of checking against the on-disk version, we
check instead against either:
- the MD5 hash stored for any of the other configured instances of
  the package.
- the MD5 hash of the already present .dpkg-new file for the
  unpacked instances of the packages.

During removal, we only remove the file if it's not shared with other
packages of the same set.

During upgrade, we don't remove the file from other packages containing
the same file if the package is part of the same set. We do this even
for packages which are not “Multi-Arch: same” since there's only one
.list controlg file for the package set and we should not modify it
since it has already been written.

Based-on-patch-by: Raphaël Hertzog <hertzog@debian.org>
Patch-sponsored-by: Linaro Limited

Signed-off-by: Guillem Jover <guillem@debian.org>

src/archives.c

@@ -5,6 +5,8 @@
  * Copyright © 1994,1995 Ian Jackson <ian@chiark.greenend.org.uk>
  * Copyright © 2000 Wichert Akkerman <wakkerma@debian.org>
  * Copyright © 2007-2012 Guillem Jover <guillem@debian.org>
+ * Copyright © 2011 Linaro Limited
+ * Copyright © 2011 Raphaël Hertzog <hertzog@debian.org>
  *
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -168,6 +170,47 @@ filesavespackage(struct fileinlist *file,
   return true;
 }
 
+static void
+md5hash_prev_conffile(struct pkginfo *pkg, char *oldhash, const char *oldname,
+                      struct filenamenode *namenode)
+{
+  struct pkginfo *otherpkg;
+  struct conffile *conff;
+
+  debug(dbg_conffdetail, "tarobject looking for shared conffile %s",
+        namenode->name);
+
+  for (otherpkg = &pkg->set->pkg; otherpkg; otherpkg = otherpkg->arch_next) {
+    if (otherpkg == pkg)
+      continue;
+    /* The hash in the Conffiles is only meaningful if the package
+     * configuration has been at least tried. */
+    if (otherpkg->status <= stat_unpacked)
+      continue;
+    for (conff = otherpkg->installed.conffiles; conff; conff = conff->next) {
+      if (strcmp(conff->name, namenode->name) == 0)
+        break;
+    }
+    if (conff) {
+      debug(dbg_conffdetail,
+            "tarobject found shared conffile, from pkg %s (%s); hash=%s",
+            pkg_name(otherpkg, pnaw_always),
+            statusinfos[otherpkg->status].name, oldhash);
+      strcpy(oldhash, conff->hash);
+      break;
+    }
+  }
+
+  /* If no package was found with a valid Conffiles field, we make the
+   * risky assumption that the hash of the current .dpkg-new file is
+   * the one of the previously unpacked package. */
+  if (otherpkg == NULL) {
+    md5hash(pkg, oldhash, oldname);
+    debug(dbg_conffdetail,
+          "tarobject found shared conffile, from disk; hash=%s", oldhash);
+  }
+}
+
 void cu_pathname(int argc, void **argv) {
   ensure_pathname_nonexisting((char*)(argv[0]));
 }
@@ -344,6 +387,31 @@ tarobject_extract(struct tarcontext *tc, struct tar_entry *te,
   }
 }
 
+static void
+tarobject_hash(struct tarcontext *tc, struct tar_entry *te,
+               struct filenamenode *namenode)
+{
+  if (te->type == tar_filetype_file) {
+    char fnamebuf[256];
+    char *newhash;
+
+    newhash = nfmalloc(MD5HASHLEN + 1);
+    fd_md5(tc->backendpipe, newhash, te->size,
+           _("backend dpkg-deb during `%.255s'"),
+           path_quote_filename(fnamebuf, te->name, 256));
+    tarobject_skip_padding(tc, te);
+
+    namenode->newhash = newhash;
+    debug(dbg_eachfiledetail, "tarobject file hash=%s", namenode->newhash);
+  } else if (te->type == tar_filetype_hardlink) {
+    struct filenamenode *linknode;
+
+    linknode = findnamenode(te->linkname, 0);
+    namenode->newhash = linknode->newhash;
+    debug(dbg_eachfiledetail, "tarobject hardlink hash=%s", namenode->newhash);
+  }
+}
+
 static void
 tarobject_set_mtime(struct tar_entry *te, const char *path)
 {
@@ -421,6 +489,73 @@ tarobject_set_se_context(const char *matchpath, const char *path, mode_t mode)
 #endif /* WITH_SELINUX */
 }
 
+static void
+tarobject_matches(struct tarcontext *tc,
+                  const char *fn_old, struct stat *stab, char *oldhash,
+                  const char *fn_new, struct tar_entry *te,
+                  struct filenamenode *namenode)
+{
+  char *linkname;
+  ssize_t linksize;
+
+  debug(dbg_eachfiledetail, "tarobject matches on-disk object?");
+
+  switch (te->type) {
+  case tar_filetype_dir:
+    /* Nothing to check for a new directory. */
+    return;
+  case tar_filetype_symlink:
+    /* Symlinks to existing dirs have already been dealt with, only
+     * reamin real symlinks where we can compare the target. */
+    if (!S_ISLNK(stab->st_mode))
+      break;
+    linkname = m_malloc(stab->st_size + 1);
+    linksize = readlink(fn_old, linkname, stab->st_size + 1);
+    if (linksize < 0)
+      ohshite(_("unable to read link `%.255s'"), fn_old);
+    else if (linksize != stab->st_size)
+      ohshit(_("symbolic link '%.250s' size has changed from %jd to %zd"),
+             fn_old, stab->st_size, linksize);
+    linkname[linksize] = '\0';
+    if (strcmp(linkname, te->linkname) == 0) {
+      free(linkname);
+      return;
+    } else {
+      free(linkname);
+    }
+    break;
+  case tar_filetype_chardev:
+    if (S_ISCHR(stab->st_mode) && stab->st_rdev == te->dev)
+      return;
+    break;
+  case tar_filetype_blockdev:
+    if (S_ISBLK(stab->st_mode) && stab->st_rdev == te->dev)
+      return;
+    break;
+  case tar_filetype_fifo:
+    if (S_ISFIFO(stab->st_mode))
+      return;
+    break;
+  case tar_filetype_hardlink:
+    /* Fall through. */
+  case tar_filetype_file:
+    /* Only check metadata for non-conffiles. */
+    if (!(namenode->flags & fnnf_new_conff) &&
+        !(S_ISREG(stab->st_mode) && te->size == stab->st_size))
+      break;
+    if (strcmp(oldhash, namenode->newhash) == 0)
+      return;
+    break;
+  default:
+    internerr("unknown tar type '%d', but already checked", te->type);
+  }
+
+  forcibleerr(fc_overwrite,
+              _("trying to overwrite shared '%.250s', which is different "
+                "from other instances of package %.250s"),
+              namenode->name, pkg_name(tc->pkg, pnaw_nonambig));
+}
+
 void setupfnamevbs(const char *filename) {
   varbuf_trunc(&fnamevb, fnameidlu);
   varbuf_add_str(&fnamevb, filename);
@@ -550,6 +685,8 @@ tarobject(void *ctx, struct tar_entry *ti)
   struct conffile *conff;
   struct tarcontext *tc = ctx;
   bool existingdir, keepexisting;
+  bool refcounting;
+  char oldhash[MD5HASHLEN + 1];
   int statr;
   ssize_t r;
   struct stat stab, stabtmp;
@@ -675,6 +812,7 @@ tarobject(void *ctx, struct tar_entry *ti)
   }
 
   keepexisting = false;
+  refcounting = false;
   if (!existingdir) {
     struct filepackages_iterator *iter;
 
@@ -685,6 +823,21 @@ tarobject(void *ctx, struct tar_entry *ti)
       debug(dbg_eachfile, "tarobject ... found in %s",
             pkg_name(otherpkg, pnaw_always));
 
+      /* A pkgset can share files between its instances. Overwriting
+       * is allowed when they are not getting in sync, otherwise the
+       * file content must match the installed file. */
+      if (otherpkg->set == tc->pkg->set &&
+          otherpkg->installed.multiarch == multiarch_same &&
+          tc->pkg->available.multiarch == multiarch_same) {
+        if (statr == 0 && tc->pkgset_getting_in_sync)
+          refcounting = true;
+        debug(dbg_eachfiledetail, "tarobject ... shared with %s %s (syncing=%d)",
+              pkg_name(otherpkg, pnaw_always),
+              versiondescribe(&otherpkg->installed.version, vdew_nonambig),
+              tc->pkgset_getting_in_sync);
+        continue;
+      }
+
       if (nifd->namenode->divert && nifd->namenode->divert->useinstead) {
         /* Right, so we may be diverting this file. This makes the conflict
          * OK iff one of us is the diverting package (we don't need to
@@ -800,23 +953,51 @@ tarobject(void *ctx, struct tar_entry *ti)
   if (existingdir)
     return 0;
 
-  /* Now, at this stage we want to make sure neither of .dpkg-new and
-   * .dpkg-tmp are hanging around. */
-  ensure_pathname_nonexisting(fnamenewvb.buf);
-  ensure_pathname_nonexisting(fnametmpvb.buf);
+  /* Compute the hash of the previous object, before we might replace it
+   * with the new version on forced overwrites. */
+  if (refcounting) {
+    if (nifd->namenode->flags & fnnf_new_conff) {
+      md5hash_prev_conffile(tc->pkg, oldhash, fnamenewvb.buf, nifd->namenode);
+    } else if (S_ISREG(stab.st_mode)) {
+      md5hash(tc->pkg, oldhash, fnamevb.buf);
+    } else {
+      strcpy(oldhash, EMPTYHASHFLAG);
+    }
+  }
 
-  /* Now we start to do things that we need to be able to undo
-   * if something goes wrong. Watch out for the CLEANUP comments to
-   * keep an eye on what's installed on the disk at each point. */
-  push_cleanup(cu_installnew, ~ehflag_normaltidy, NULL, 0, 1, nifd->namenode);
+  if (refcounting && !fc_overwrite) {
+    /* If we are not forced to overwrite the path and are refcounting,
+     * just compute the hash w/o extracting the object. */
+    tarobject_hash(tc, ti, nifd->namenode);
+  } else {
+    /* Now, at this stage we want to make sure neither of .dpkg-new and
+     * .dpkg-tmp are hanging around. */
+    ensure_pathname_nonexisting(fnamenewvb.buf);
+    ensure_pathname_nonexisting(fnametmpvb.buf);
 
-  /*
-   * CLEANUP: Now we either have the old file on the disk, or not, in
-   * its original filename.
-   */
+    /* Now we start to do things that we need to be able to undo
+     * if something goes wrong. Watch out for the CLEANUP comments to
+     * keep an eye on what's installed on the disk at each point. */
+    push_cleanup(cu_installnew, ~ehflag_normaltidy, NULL, 0, 1, nifd->namenode);
+
+    /*
+     * CLEANUP: Now we either have the old file on the disk, or not, in
+     * its original filename.
+     */
+
+    /* Extract whatever it is as .dpkg-new ... */
+    tarobject_extract(tc, ti, fnamenewvb.buf, st, nifd->namenode);
+  }
+
+  /* For shared files, check now if the object matches. */
+  if (refcounting)
+    tarobject_matches(tc, fnamevb.buf, &stab, oldhash,
+                          fnamenewvb.buf, ti, nifd->namenode);
+
+  /* If we didn't extract anything, there's nothing else to do. */
+  if (refcounting && !fc_overwrite)
+    return 0;
 
-  /* Extract whatever it is as .dpkg-new ... */
-  tarobject_extract(tc, ti, fnamenewvb.buf, st, nifd->namenode);
   tarobject_set_perms(ti, fnamenewvb.buf, st);
   tarobject_set_mtime(ti, fnamenewvb.buf);
   tarobject_set_se_context(fnamevb.buf, fnamenewvb.buf, st->mode);

src/processarc.c

@@ -1338,6 +1338,10 @@ void process_archive(const char *filename) {
        * and we don't bother with it here, clearly. */
       if (otherpkg == pkg || !otherpkg->clientdata->fileslistvalid)
         continue;
+      /* A pkgset can share files between instances, so there's no point
+       * in rewriting the file that's already in place. */
+      if (otherpkg->set == pkg->set)
+        continue;
       if (otherpkg->set == divpkgset) {
         debug(dbg_eachfiledetail, "process_archive ... diverted, skipping");
         continue;

src/remove.c

@@ -210,6 +210,33 @@ removal_bulk_remove_file(const char *filename, const char *filetype)
   debug(dbg_scripts, "removal_bulk info unlinked %s", filename);
 }
 
+static bool
+removal_bulk_file_is_shared(struct pkginfo *pkg, struct filenamenode *namenode)
+{
+  struct filepackages_iterator *iter;
+  struct pkginfo *otherpkg;
+  bool shared = false;
+
+  if (pkgset_installed_instances(pkg->set) <= 1)
+    return false;
+
+  iter = filepackages_iter_new(namenode);
+  while ((otherpkg = filepackages_iter_next(iter))) {
+    if (otherpkg == pkg)
+      continue;
+    if (otherpkg->set != pkg->set)
+      continue;
+
+    debug(dbg_eachfiledetail, "removal_bulk file shared with %s, skipping",
+          pkg_name(otherpkg, pnaw_always));
+    shared = true;
+    break;
+  }
+  filepackages_iter_free(iter);
+
+  return shared;
+}
+
 static void
 removal_bulk_remove_files(struct pkginfo *pkg)
 {
@@ -228,22 +255,36 @@ removal_bulk_remove_files(struct pkginfo *pkg)
     leftover = NULL;
     while ((namenode= reversefilelist_next(&rlistit))) {
       struct filenamenode *usenode;
+      bool is_dir;
 
       debug(dbg_eachfile, "removal_bulk `%s' flags=%o",
             namenode->name, namenode->flags);
-      if (namenode->flags & fnnf_old_conff) {
-        push_leftover(&leftover,namenode);
-        continue;
-      }
 
       usenode = namenodetouse(namenode, pkg, &pkg->installed);
-      trig_file_activate(usenode, pkg);
 
       varbuf_reset(&fnvb);
       varbuf_add_str(&fnvb, instdir);
       varbuf_add_str(&fnvb, usenode->name);
+      varbuf_end_str(&fnvb);
       before= fnvb.used;
 
+      is_dir = stat(fnvb.buf, &stab) == 0 && S_ISDIR(stab.st_mode);
+
+      /* A pkgset can share files between its instances that we
+       * don't want to remove, we just want to forget them. This
+       * applies to shared conffiles too. */
+      if (!is_dir && removal_bulk_file_is_shared(pkg, namenode))
+        continue;
+
+      /* Non-shared conffiles are kept. */
+      if (namenode->flags & fnnf_old_conff) {
+        push_leftover(&leftover, namenode);
+        continue;
+      }
+
+      trig_file_activate(usenode, pkg);
+
+      varbuf_trunc(&fnvb, before);
       varbuf_add_str(&fnvb, DPKGTEMPEXT);
       varbuf_end_str(&fnvb);
       debug(dbg_eachfiledetail, "removal_bulk cleaning temp `%s'", fnvb.buf);
@@ -258,7 +299,7 @@ removal_bulk_remove_files(struct pkginfo *pkg)
 
       varbuf_trunc(&fnvb, before);
       varbuf_end_str(&fnvb);
-      if (!stat(fnvb.buf,&stab) && S_ISDIR(stab.st_mode)) {
+      if (is_dir) {
         debug(dbg_eachfiledetail, "removal_bulk is a directory");
         /* Only delete a directory or a link to one if we're the only
          * package which uses it. Other files should only be listed