dpkg-split: Do not slurp into memory the whole package parts

Use buffered I/O when reassembling split packages. This will make
possible to handle parts > 2 GiB on 32 bit systems.

dpkg-split/join.c

@@ -24,6 +24,7 @@
 #include <assert.h>
 #include <limits.h>
 #include <string.h>
+#include <fcntl.h>
 #include <unistd.h>
 #include <stdlib.h>
 #include <stdio.h>
@@ -31,52 +32,40 @@
 #include <dpkg/i18n.h>
 #include <dpkg/dpkg.h>
 #include <dpkg/dpkg-db.h>
+#include <dpkg/buffer.h>
 #include <dpkg/myopt.h>
 
 #include "dpkg-split.h"
 
 void reassemble(struct partinfo **partlist, const char *outputfile) {
-  FILE *output, *input;
-  void *buffer;
+  int fd_out, fd_in;
   unsigned int i;
-  size_t nr,buffersize;
 
   printf(P_("Putting package %s together from %d part: ",
             "Putting package %s together from %d parts: ",
             partlist[0]->maxpartn),
          partlist[0]->package,partlist[0]->maxpartn);
 
-  buffersize= partlist[0]->maxpartlen;
-  for (i=0; i<partlist[0]->maxpartn; i++)
-    if (partlist[0]->headerlen > buffersize) buffersize= partlist[0]->headerlen;
-  buffer= m_malloc(partlist[0]->maxpartlen);
-  output= fopen(outputfile,"w");
-  if (!output) ohshite(_("unable to open output file `%.250s'"),outputfile);
+  fd_out = creat(outputfile, 0644);
+  if (fd_out < 0)
+    ohshite(_("unable to open output file `%.250s'"), outputfile);
   for (i=0; i<partlist[0]->maxpartn; i++) {
     struct partinfo *pi = partlist[i];
 
-    input= fopen(pi->filename,"r");
-    if (!input) ohshite(_("unable to (re)open input part file `%.250s'"),pi->filename);
-    assert(pi->headerlen <= buffersize);
-    nr= fread(buffer,1,pi->headerlen,input);
-    if (nr != pi->headerlen) rerreof(input,pi->filename);
-    assert(pi->thispartlen <= buffersize);
+    fd_in = open(pi->filename, O_RDONLY);
+    if (fd_in < 0)
+      ohshite(_("unable to (re)open input part file `%.250s'"), pi->filename);
+    fd_null_copy(fd_in, pi->headerlen, _("skipping split package header"));
+    fd_fd_copy(fd_in, fd_out, pi->thispartlen, _("split package part"));
+    close(fd_in);
+
     printf("%d ",i+1);
-    nr= fread(buffer,1,pi->thispartlen,input);
-    if (nr != pi->thispartlen) rerreof(input,pi->filename);
-    if (pi->thispartlen & 1)
-      if (getc(input) == EOF) rerreof(input,pi->filename);
-    if (ferror(input)) rerr(pi->filename);
-    fclose(input);
-    nr= fwrite(buffer,1,pi->thispartlen,output);
-    if (nr != pi->thispartlen) werr(outputfile);
   }
-  if (fflush(output))
-    ohshite(_("unable to flush file '%s'"), outputfile);
-  if (fsync(fileno(output)))
+  if (fsync(fd_out))
     ohshite(_("unable to sync file '%s'"), outputfile);
-  if (fclose(output)) werr(outputfile);
-  free(buffer);
+  if (close(fd_out))
+    ohshite(_("unable to close file '%s'"), outputfile);
+
   printf(_("done\n"));
 }
 

dpkg-split/queue.c

@@ -36,6 +36,7 @@
 #include <dpkg/dpkg.h>
 #include <dpkg/dpkg-db.h>
 #include <dpkg/dir.h>
+#include <dpkg/buffer.h>
 #include <dpkg/myopt.h>
 
 #include "dpkg-split.h"
@@ -161,32 +162,29 @@ void do_auto(const char *const *argv) {
   for (j=refi->maxpartn-1; j>=0 && partlist[j]; j--);
 
   if (j>=0) {
-    void *buffer;
-    long nr;
+    int fd_src, fd_dst;
     int ap;
     char *p, *q;
 
-    part= fopen(partfile,"r");
-    if (!part) ohshite(_("unable to reopen part file `%.250s'"),partfile);
-    buffer= nfmalloc(refi->filesize);
-    nr= fread(buffer,1,refi->filesize,part);
-    if (nr != refi->filesize) rerreof(part,partfile);
-    if (getc(part) != EOF) ohshit(_("part file `%.250s' has trailing garbage"),partfile);
-    if (ferror(part)) rerr(partfile);
-    fclose(part);
-
     m_asprintf(&p, "%st.%lx", opt_depotdir, (long)getpid());
     m_asprintf(&q, "%s%s.%lx.%x.%x", opt_depotdir, refi->md5sum,
                refi->maxpartlen, refi->thispartn, refi->maxpartn);
-    part= fopen(p,"w");
-    if (!part) ohshite(_("unable to open new depot file `%.250s'"),p);
-    nr= fwrite(buffer,1,refi->filesize,part);
-    if (nr != refi->filesize) werr(p);
-    if (fflush(part))
-      ohshite(_("unable to flush file '%s'"), p);
-    if (fsync(fileno(part)))
+
+    fd_src = open(partfile, O_RDONLY);
+    if (fd_src < 0)
+      ohshite(_("unable to reopen part file `%.250s'"), partfile);
+    fd_dst = creat(p, 0644);
+    if (fd_dst)
+      ohshite(_("unable to open new depot file `%.250s'"), p);
+
+    fd_fd_copy(fd_src, fd_dst, refi->filesize, _("extracing split part"));
+
+    if (fsync(fd_dst))
       ohshite(_("unable to sync file '%s'"), p);
-    if (fclose(part)) werr(p);
+    if (close(fd_dst))
+      ohshite(_("unable to close file '%s'"), p);
+    close(fd_src);
+
     if (rename(p,q)) ohshite(_("unable to rename new depot file `%.250s' to `%.250s'"),p,q);
     free(q);
     free(p);