Support large files in the complete toolset. Indexes of this

size are pretty unlikely for now, but we need it for deb
packages which could become bigger than 4GB now (LP: #815895)

apt-inst/contrib/arfile.h

@@ -54,7 +54,7 @@ struct ARArchive::Member
    unsigned long UID;
    unsigned long GID;
    unsigned long Mode;
-   unsigned long Size;
+   unsigned long long Size;
    
    // Location of the data.
    unsigned long Start;

apt-pkg/acquire-worker.cc

@@ -258,9 +258,9 @@ bool pkgAcquire::Worker::RunMessages()
 	    
 	    CurrentItem = Itm;
 	    CurrentSize = 0;
-	    TotalSize = atoi(LookupTag(Message,"Size","0").c_str());
-	    ResumePoint = atoi(LookupTag(Message,"Resume-Point","0").c_str());
-	    Itm->Owner->Start(Message,atoi(LookupTag(Message,"Size","0").c_str()));
+	    TotalSize = strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10);
+	    ResumePoint = strtoull(LookupTag(Message,"Resume-Point","0").c_str(), NULL, 10);
+	    Itm->Owner->Start(Message,strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10));
 
 	    // Display update before completion
 	    if (Log != 0 && Log->MorePulses == true)
@@ -289,7 +289,7 @@ bool pkgAcquire::Worker::RunMessages()
 	       Log->Pulse(Owner->GetOwner());
 	    
 	    OwnerQ->ItemDone(Itm);
-	    unsigned long long const ServerSize = atoll(LookupTag(Message,"Size","0").c_str());
+	    unsigned long long const ServerSize = strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10);
 	    if (TotalSize != 0 && ServerSize != TotalSize)
 	       _error->Warning("Size of file %s is not what the server reported %s %llu",
 			       Owner->DestFile.c_str(), LookupTag(Message,"Size","0").c_str(),TotalSize);

apt-pkg/contrib/crc-16.cc

@@ -65,7 +65,7 @@ static unsigned short const crc16_table[256] =
 /* Recompute the FCS with one more character appended. */
 #define CalcFCS(fcs, c) (((fcs) >> 8) ^ crc16_table[((fcs) ^ (c)) & 0xff])
 unsigned short AddCRC16(unsigned short fcs, void const *Buf,
-			unsigned long len) 
+			unsigned long long len)
 {
    unsigned char const *buf = (unsigned char const *)Buf;
    while (len--)

apt-pkg/contrib/crc-16.h

@@ -12,6 +12,6 @@
 
 #define INIT_FCS  0xffff
 unsigned short AddCRC16(unsigned short fcs, void const *buf,
-			unsigned long len);
+			unsigned long long len);
 
 #endif

apt-pkg/contrib/fileutl.cc

@@ -133,10 +133,10 @@ bool CopyFile(FileFd &From,FileFd &To)
    
    // Buffered copy between fds
    SPtrArray<unsigned char> Buf = new unsigned char[64000];
-   unsigned long Size = From.Size();
+   unsigned long long Size = From.Size();
    while (Size != 0)
    {
-      unsigned long ToRead = Size;
+      unsigned long long ToRead = Size;
       if (Size > 64000)
 	 ToRead = 64000;
       
@@ -800,7 +800,7 @@ FileFd::~FileFd()
 // ---------------------------------------------------------------------
 /* We are carefull to handle interruption by a signal while reading 
    gracefully. */
-bool FileFd::Read(void *To,unsigned long Size,unsigned long *Actual)
+bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual)
 {
    int Res;
    errno = 0;
@@ -839,13 +839,13 @@ bool FileFd::Read(void *To,unsigned long Size,unsigned long *Actual)
    }
    
    Flags |= Fail;
-   return _error->Error(_("read, still have %lu to read but none left"),Size);
+   return _error->Error(_("read, still have %llu to read but none left"), Size);
 }
 									/*}}}*/
 // FileFd::Write - Write to the file					/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-bool FileFd::Write(const void *From,unsigned long Size)
+bool FileFd::Write(const void *From,unsigned long long Size)
 {
    int Res;
    errno = 0;
@@ -872,13 +872,13 @@ bool FileFd::Write(const void *From,unsigned long Size)
       return true;
    
    Flags |= Fail;
-   return _error->Error(_("write, still have %lu to write but couldn't"),Size);
+   return _error->Error(_("write, still have %llu to write but couldn't"), Size);
 }
 									/*}}}*/
 // FileFd::Seek - Seek in the file					/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-bool FileFd::Seek(unsigned long To)
+bool FileFd::Seek(unsigned long long To)
 {
    int res;
    if (gz)
@@ -888,7 +888,7 @@ bool FileFd::Seek(unsigned long To)
    if (res != (signed)To)
    {
       Flags |= Fail;
-      return _error->Error("Unable to seek to %lu",To);
+      return _error->Error("Unable to seek to %llu", To);
    }
    
    return true;
@@ -897,7 +897,7 @@ bool FileFd::Seek(unsigned long To)
 // FileFd::Skip - Seek in the file					/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-bool FileFd::Skip(unsigned long Over)
+bool FileFd::Skip(unsigned long long Over)
 {
    int res;
    if (gz)
@@ -907,7 +907,7 @@ bool FileFd::Skip(unsigned long Over)
    if (res < 0)
    {
       Flags |= Fail;
-      return _error->Error("Unable to seek ahead %lu",Over);
+      return _error->Error("Unable to seek ahead %llu",Over);
    }
    
    return true;
@@ -916,7 +916,7 @@ bool FileFd::Skip(unsigned long Over)
 // FileFd::Truncate - Truncate the file 				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-bool FileFd::Truncate(unsigned long To)
+bool FileFd::Truncate(unsigned long long To)
 {
    if (gz)
    {
@@ -926,7 +926,7 @@ bool FileFd::Truncate(unsigned long To)
    if (ftruncate(iFd,To) != 0)
    {
       Flags |= Fail;
-      return _error->Error("Unable to truncate to %lu",To);
+      return _error->Error("Unable to truncate to %llu",To);
    }
    
    return true;
@@ -935,7 +935,7 @@ bool FileFd::Truncate(unsigned long To)
 // FileFd::Tell - Current seek position					/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-unsigned long FileFd::Tell()
+unsigned long long FileFd::Tell()
 {
    off_t Res;
    if (gz)
@@ -950,7 +950,7 @@ unsigned long FileFd::Tell()
 // FileFd::FileSize - Return the size of the file			/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-unsigned long FileFd::FileSize()
+unsigned long long FileFd::FileSize()
 {
    struct stat Buf;
 
@@ -962,9 +962,9 @@ unsigned long FileFd::FileSize()
 // FileFd::Size - Return the size of the content in the file		/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-unsigned long FileFd::Size()
+unsigned long long FileFd::Size()
 {
-   unsigned long size = FileSize();
+   unsigned long long size = FileSize();
 
    // only check gzsize if we are actually a gzip file, just checking for
    // "gz" is not sufficient as uncompressed files will be opened with
@@ -974,6 +974,7 @@ unsigned long FileFd::Size()
        /* unfortunately zlib.h doesn't provide a gzsize(), so we have to do
 	* this ourselves; the original (uncompressed) file size is the last 32
 	* bits of the file */
+       // FIXME: Size for gz-files is limited by 32bit… no largefile support
        off_t orig_pos = lseek(iFd, 0, SEEK_CUR);
        if (lseek(iFd, -4, SEEK_END) < 0)
 	   return _error->Errno("lseek","Unable to seek to end of gzipped file");

apt-pkg/contrib/fileutl.h

@@ -49,21 +49,36 @@ class FileFd
    enum OpenMode {ReadOnly,WriteEmpty,WriteExists,WriteAny,WriteTemp,ReadOnlyGzip,
                   WriteAtomic};
    
-   inline bool Read(void *To,unsigned long Size,bool AllowEof)
+   inline bool Read(void *To,unsigned long long Size,bool AllowEof)
    {
-      unsigned long Jnk;
+      unsigned long long Jnk;
       if (AllowEof)
 	 return Read(To,Size,&Jnk);
       return Read(To,Size);
    }   
-   bool Read(void *To,unsigned long Size,unsigned long *Actual = 0);
-   bool Write(const void *From,unsigned long Size);
-   bool Seek(unsigned long To);
-   bool Skip(unsigned long To);
-   bool Truncate(unsigned long To);
-   unsigned long Tell();
-   unsigned long Size();
-   unsigned long FileSize();
+   bool Read(void *To,unsigned long long Size,unsigned long long *Actual = 0);
+   bool Write(const void *From,unsigned long long Size);
+   bool Seek(unsigned long long To);
+   bool Skip(unsigned long long To);
+   bool Truncate(unsigned long long To);
+   unsigned long long Tell();
+   unsigned long long Size();
+   unsigned long long FileSize();
+
+   /* You want to use 'unsigned long long' if you are talking about a file
+      to be able to support large files (>2 or >4 GB) properly.
+      This shouldn't happen all to often for the indexes, but deb's might be…
+      And as the auto-conversation converts a 'unsigned long *' to a 'bool'
+      instead of 'unsigned long long *' we need to provide this explicitely -
+      otherwise applications magically start to fail… */
+   __deprecated bool Read(void *To,unsigned long long Size,unsigned long *Actual)
+   {
+	unsigned long long R;
+	bool const T = Read(To, Size, &R);
+	*Actual = R;
+	return T;
+   }
+
    bool Open(string FileName,OpenMode Mode,unsigned long Perms = 0666);
    bool OpenDescriptor(int Fd, OpenMode Mode, bool AutoClose=false);
    bool Close();

apt-pkg/contrib/hashes.cc

@@ -108,18 +108,18 @@ string HashString::toStr() const
 // Hashes::AddFD - Add the contents of the FD				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-bool Hashes::AddFD(int const Fd,unsigned long Size, bool const addMD5,
+bool Hashes::AddFD(int const Fd,unsigned long long Size, bool const addMD5,
 		   bool const addSHA1, bool const addSHA256, bool const addSHA512)
 {
    unsigned char Buf[64*64];
-   int Res = 0;
+   ssize_t Res = 0;
    int ToEOF = (Size == 0);
    while (Size != 0 || ToEOF)
    {
-      unsigned n = sizeof(Buf);
-      if (!ToEOF) n = min(Size,(unsigned long)n);
+      unsigned long long n = sizeof(Buf);
+      if (!ToEOF) n = min(Size, n);
       Res = read(Fd,Buf,n);
-      if (Res < 0 || (!ToEOF && (unsigned) Res != n)) // error, or short read
+      if (Res < 0 || (!ToEOF && Res != (ssize_t) n)) // error, or short read
 	 return false;
       if (ToEOF && Res == 0) // EOF
 	 break;

apt-pkg/contrib/hashes.h

@@ -62,14 +62,14 @@ class Hashes
    SHA256Summation SHA256;
    SHA512Summation SHA512;
    
-   inline bool Add(const unsigned char *Data,unsigned long Size)
+   inline bool Add(const unsigned char *Data,unsigned long long Size)
    {
       return MD5.Add(Data,Size) && SHA1.Add(Data,Size) && SHA256.Add(Data,Size) && SHA512.Add(Data,Size);
    };
    inline bool Add(const char *Data) {return Add((unsigned char *)Data,strlen(Data));};
-   inline bool AddFD(int const Fd,unsigned long Size = 0)
+   inline bool AddFD(int const Fd,unsigned long long Size = 0)
    { return AddFD(Fd, Size, true, true, true, true); };
-   bool AddFD(int const Fd, unsigned long Size, bool const addMD5,
+   bool AddFD(int const Fd, unsigned long long Size, bool const addMD5,
 	      bool const addSHA1, bool const addSHA256, bool const addSHA512);
    inline bool Add(const unsigned char *Beg,const unsigned char *End) 
                   {return Add(Beg,End-Beg);};

apt-pkg/contrib/hashsum.cc

@@ -7,16 +7,16 @@
 // Summation::AddFD - Add content of file into the checksum		/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-bool SummationImplementation::AddFD(int const Fd, unsigned long Size) {
+bool SummationImplementation::AddFD(int const Fd, unsigned long long Size) {
    unsigned char Buf[64 * 64];
-   int Res = 0;
+   ssize_t Res = 0;
    int ToEOF = (Size == 0);
    while (Size != 0 || ToEOF)
    {
-      unsigned n = sizeof(Buf);
-      if (!ToEOF) n = min(Size,(unsigned long)n);
+      unsigned long long n = sizeof(Buf);
+      if (!ToEOF) n = min(Size, n);
       Res = read(Fd, Buf, n);
-      if (Res < 0 || (!ToEOF && (unsigned) Res != n)) // error, or short read
+      if (Res < 0 || (!ToEOF && Res != (ssize_t) n)) // error, or short read
 	 return false;
       if (ToEOF && Res == 0) // EOF
 	 break;

apt-pkg/contrib/hashsum_template.h

@@ -87,8 +87,8 @@ class HashSumValue
 class SummationImplementation
 {
    public:
-   virtual bool Add(const unsigned char *inbuf, unsigned long inlen) = 0;
-   inline bool Add(const char *inbuf, unsigned long const inlen)
+   virtual bool Add(const unsigned char *inbuf, unsigned long long inlen) = 0;
+   inline bool Add(const char *inbuf, unsigned long long const inlen)
    { return Add((unsigned char *)inbuf, inlen); };
 
    inline bool Add(const unsigned char *Data)
@@ -101,7 +101,7 @@ class SummationImplementation
    inline bool Add(const char *Beg, const char *End)
    { return Add((const unsigned char *)Beg, End - Beg); };
 
-   bool AddFD(int Fd, unsigned long Size = 0);
+   bool AddFD(int Fd, unsigned long long Size = 0);
 };
 
 #endif

apt-pkg/contrib/md5.cc

@@ -187,7 +187,7 @@ MD5Summation::MD5Summation()
 // MD5Summation::Add - 'Add' a data set to the hash			/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-bool MD5Summation::Add(const unsigned char *data,unsigned long len)
+bool MD5Summation::Add(const unsigned char *data,unsigned long long len)
 {
    if (Done == true)
       return false;

apt-pkg/contrib/md5.h

@@ -45,7 +45,7 @@ class MD5Summation : public SummationImplementation
 
    public:
 
-   bool Add(const unsigned char *inbuf, unsigned long inlen);
+   bool Add(const unsigned char *inbuf, unsigned long long inlen);
    using SummationImplementation::Add;
 
    MD5SumValue Result();

apt-pkg/contrib/mmap.cc

@@ -95,7 +95,7 @@ bool MMap::Map(FileFd &Fd)
 	    return false;
       }
       else
-	 return _error->Errno("mmap",_("Couldn't make mmap of %lu bytes"),
+	 return _error->Errno("mmap",_("Couldn't make mmap of %llu bytes"),
 	                      iSize);
      }
 
@@ -166,7 +166,7 @@ bool MMap::Sync(unsigned long Start,unsigned long Stop)
       return true;
    
 #ifdef _POSIX_SYNCHRONIZED_IO
-   unsigned long PSize = sysconf(_SC_PAGESIZE);
+   unsigned long long PSize = sysconf(_SC_PAGESIZE);
    if ((Flags & ReadOnly) != ReadOnly)
    {
       if (SyncToFd != 0)
@@ -177,7 +177,7 @@ bool MMap::Sync(unsigned long Start,unsigned long Stop)
       }
       else
       {
-	 if (msync((char *)Base+(int)(Start/PSize)*PSize,Stop - Start,MS_SYNC) < 0)
+	 if (msync((char *)Base+(unsigned long long)(Start/PSize)*PSize,Stop - Start,MS_SYNC) < 0)
 	    return _error->Errno("msync", _("Unable to synchronize mmap"));
       }
    }
@@ -197,7 +197,7 @@ DynamicMMap::DynamicMMap(FileFd &F,unsigned long Flags,unsigned long const &Work
    if (_error->PendingError() == true)
       return;
    
-   unsigned long EndOfFile = Fd->Size();
+   unsigned long long EndOfFile = Fd->Size();
    if (EndOfFile > WorkSpace)
       WorkSpace = EndOfFile;
    else if(WorkSpace > 0)
@@ -285,7 +285,7 @@ DynamicMMap::~DynamicMMap()
       return;
    }
    
-   unsigned long EndOfFile = iSize;
+   unsigned long long EndOfFile = iSize;
    iSize = WorkSpace;
    Close(false);
    if(ftruncate(Fd->Fd(),EndOfFile) < 0)
@@ -295,9 +295,9 @@ DynamicMMap::~DynamicMMap()
 // DynamicMMap::RawAllocate - Allocate a raw chunk of unaligned space	/*{{{*/
 // ---------------------------------------------------------------------
 /* This allocates a block of memory aligned to the given size */
-unsigned long DynamicMMap::RawAllocate(unsigned long Size,unsigned long Aln)
+unsigned long DynamicMMap::RawAllocate(unsigned long long Size,unsigned long Aln)
 {
-   unsigned long Result = iSize;
+   unsigned long long Result = iSize;
    if (Aln != 0)
       Result += Aln - (iSize%Aln);
 
@@ -412,7 +412,7 @@ bool DynamicMMap::Grow() {
 	if (GrowFactor <= 0)
 		return _error->Error(_("Unable to increase size of the MMap as automatic growing is disabled by user."));
 
-	unsigned long const newSize = WorkSpace + GrowFactor;
+	unsigned long long const newSize = WorkSpace + GrowFactor;
 
 	if(Fd != 0) {
 		Fd->Seek(newSize - 1);

apt-pkg/contrib/mmap.h

@@ -41,7 +41,7 @@ class MMap
    protected:
    
    unsigned long Flags;
-   unsigned long iSize;
+   unsigned long long iSize;
    void *Base;
 
    // In case mmap can not be used, we keep a dup of the file
@@ -60,8 +60,8 @@ class MMap
    // Simple accessors
    inline operator void *() {return Base;};
    inline void *Data() {return Base;}; 
-   inline unsigned long Size() {return iSize;};
-   inline void AddSize(unsigned long const size) {iSize += size;};
+   inline unsigned long long Size() {return iSize;};
+   inline void AddSize(unsigned long long const size) {iSize += size;};
    inline bool validData() const { return Base != (void *)-1 && Base != 0; };
    
    // File manipulators
@@ -99,7 +99,7 @@ class DynamicMMap : public MMap
    public:
 
    // Allocation
-   unsigned long RawAllocate(unsigned long Size,unsigned long Aln = 0);
+   unsigned long RawAllocate(unsigned long long Size,unsigned long Aln = 0);
    unsigned long Allocate(unsigned long ItemSize);
    unsigned long WriteString(const char *String,unsigned long Len = (unsigned long)-1);
    inline unsigned long WriteString(const string &S) {return WriteString(S.c_str(),S.length());};

apt-pkg/contrib/progress.cc

@@ -37,7 +37,7 @@ OpProgress::OpProgress() : Current(0), Total(0), Size(0), SubTotal(1),
 /* Current is the Base Overall progress in units of Total. Cur is the sub
    progress in units of SubTotal. Size is a scaling factor that says what
    percent of Total SubTotal is. */
-void OpProgress::Progress(unsigned long Cur)
+void OpProgress::Progress(unsigned long long Cur)
 {
    if (Total == 0 || Size == 0 || SubTotal == 0)
       Percent = 0;
@@ -49,8 +49,8 @@ void OpProgress::Progress(unsigned long Cur)
 // OpProgress::OverallProgress - Set the overall progress		/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-void OpProgress::OverallProgress(unsigned long Current, unsigned long Total,
-	  			 unsigned long Size,const string &Op)
+void OpProgress::OverallProgress(unsigned long long Current, unsigned long long Total,
+	  			 unsigned long long Size,const string &Op)
 {
    this->Current = Current;
    this->Total = Total;
@@ -67,7 +67,7 @@ void OpProgress::OverallProgress(unsigned long Current, unsigned long Total,
 // OpProgress::SubProgress - Set the sub progress state			/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-void OpProgress::SubProgress(unsigned long SubTotal,const string &Op,
+void OpProgress::SubProgress(unsigned long long SubTotal,const string &Op,
 			     float const Percent)
 {
    this->SubTotal = SubTotal;

apt-pkg/contrib/progress.h

@@ -30,10 +30,10 @@ using std::string;
 class Configuration;
 class OpProgress
 {
-   unsigned long Current;
-   unsigned long Total;
-   unsigned long Size;
-   unsigned long SubTotal;
+   unsigned long long Current;
+   unsigned long long Total;
+   unsigned long long Size;
+   unsigned long long SubTotal;
    float LastPercent;
    
    // Change reduction code
@@ -54,10 +54,10 @@ class OpProgress
    
    public:
    
-   void Progress(unsigned long Current);
-   void SubProgress(unsigned long SubTotal, const string &Op = "", float const Percent = -1);
-   void OverallProgress(unsigned long Current,unsigned long Total,
-			unsigned long Size,const string &Op);
+   void Progress(unsigned long long Current);
+   void SubProgress(unsigned long long SubTotal, const string &Op = "", float const Percent = -1);
+   void OverallProgress(unsigned long long Current,unsigned long long Total,
+			unsigned long long Size,const string &Op);
    virtual void Done() {};
    
    OpProgress();

apt-pkg/contrib/sha1.cc

@@ -243,7 +243,7 @@ SHA1SumValue SHA1Summation::Result()
 // SHA1Summation::Add - Adds content of buffer into the checksum        /*{{{*/
 // ---------------------------------------------------------------------
 /* May not be called after Result() is called */
-bool SHA1Summation::Add(const unsigned char *data,unsigned long len)
+bool SHA1Summation::Add(const unsigned char *data,unsigned long long len)
 {
    if (Done)
       return false;

apt-pkg/contrib/sha1.h

@@ -34,7 +34,7 @@ class SHA1Summation : public SummationImplementation
    bool Done;
    
    public:
-   bool Add(const unsigned char *inbuf, unsigned long inlen);
+   bool Add(const unsigned char *inbuf, unsigned long long inlen);
    using SummationImplementation::Add;
 
    SHA1SumValue Result();

apt-pkg/contrib/sha2.h

@@ -30,7 +30,7 @@ class SHA2SummationBase : public SummationImplementation
  protected:
    bool Done;
  public:
-   bool Add(const unsigned char *inbuf, unsigned long len) = 0;
+   bool Add(const unsigned char *inbuf, unsigned long long len) = 0;
 
    void Result();
 };
@@ -41,7 +41,7 @@ class SHA256Summation : public SHA2SummationBase
    unsigned char Sum[32];
 
    public:
-   bool Add(const unsigned char *inbuf, unsigned long len)
+   bool Add(const unsigned char *inbuf, unsigned long long len)
    {
       if (Done) 
          return false;
@@ -73,7 +73,7 @@ class SHA512Summation : public SHA2SummationBase
    unsigned char Sum[64];
 
    public:
-   bool Add(const unsigned char *inbuf, unsigned long len)
+   bool Add(const unsigned char *inbuf, unsigned long long len)
    {
       if (Done) 
          return false;

apt-pkg/contrib/strutl.cc

@@ -970,6 +970,34 @@ bool StrToNum(const char *Str,unsigned long &Res,unsigned Len,unsigned Base)
    return true;
 }
 									/*}}}*/
+// StrToNum - Convert a fixed length string to a number			/*{{{*/
+// ---------------------------------------------------------------------
+/* This is used in decoding the crazy fixed length string headers in 
+   tar and ar files. */
+bool StrToNum(const char *Str,unsigned long long &Res,unsigned Len,unsigned Base)
+{
+   char S[30];
+   if (Len >= sizeof(S))
+      return false;
+   memcpy(S,Str,Len);
+   S[Len] = 0;
+   
+   // All spaces is a zero
+   Res = 0;
+   unsigned I;
+   for (I = 0; S[I] == ' '; I++);
+   if (S[I] == 0)
+      return true;
+   
+   char *End;
+   Res = strtoull(S,&End,Base);
+   if (End == S)
+      return false;
+   
+   return true;
+}
+									/*}}}*/
+
 // Base256ToNum - Convert a fixed length binary to a number             /*{{{*/
 // ---------------------------------------------------------------------
 /* This is used in decoding the 256bit encoded fixed length fields in

apt-pkg/contrib/strutl.h

@@ -52,6 +52,7 @@ string LookupTag(const string &Message,const char *Tag,const char *Default = 0);
 int StringToBool(const string &Text,int Default = -1);
 bool ReadMessages(int Fd, vector<string> &List);
 bool StrToNum(const char *Str,unsigned long &Res,unsigned Len,unsigned Base = 0);
+bool StrToNum(const char *Str,unsigned long long &Res,unsigned Len,unsigned Base = 0);
 bool Base256ToNum(const char *Str,unsigned long &Res,unsigned int Len);
 bool Hex2Num(const string &Str,unsigned char *Num,unsigned int Length);
 bool TokSplitString(char Tok,char *Input,char **List,

apt-pkg/depcache.cc

@@ -171,14 +171,14 @@ bool pkgDepCache::readStateFile(OpProgress *Prog)			/*{{{*/
    string const state = _config->FindFile("Dir::State::extended_states");
    if(RealFileExists(state)) {
       state_file.Open(state, FileFd::ReadOnly);
-      int const file_size = state_file.Size();
+      off_t const file_size = state_file.Size();
       if(Prog != NULL)
 	 Prog->OverallProgress(0, file_size, 1, 
 			       _("Reading state information"));
 
       pkgTagFile tagfile(&state_file);
       pkgTagSection section;
-      int amt = 0;
+      off_t amt = 0;
       bool const debug_autoremove = _config->FindB("Debug::pkgAutoRemove",false);
       while(tagfile.Step(section)) {
 	 string const pkgname = section.FindS("Package");

apt-pkg/indexcopy.cc

@@ -29,6 +29,7 @@
 #include <sys/types.h>
 #include <fcntl.h>
 #include <stdio.h>
+#include <stdlib.h>
 
 #include "indexcopy.h"
 #include <apti18n.h>
@@ -55,7 +56,7 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List,
    bool Debug = _config->FindB("Debug::aptcdrom",false);
    
    // Prepare the progress indicator
-   unsigned long TotalSize = 0;
+   off_t TotalSize = 0;
    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
    {
       struct stat Buf;
@@ -66,14 +67,14 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List,
       TotalSize += Buf.st_size;
    }	
 
-   unsigned long CurrentSize = 0;
+   off_t CurrentSize = 0;
    unsigned int NotFound = 0;
    unsigned int WrongSize = 0;
    unsigned int Packages = 0;
    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
    {      
       string OrigPath = string(*I,CDROM.length());
-      unsigned long FileSize = 0;
+      off_t FileSize = 0;
       
       // Open the package file
       FileFd Pkg;
@@ -166,7 +167,7 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List,
 	 if(Progress)
 	    Progress->Progress(Parser.Offset());
 	 string File;
-	 unsigned long Size;
+	 unsigned long long Size;
 	 if (GetFile(File,Size) == false)
 	 {
 	    fclose(TargetFl);
@@ -221,7 +222,7 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List,
 	    }	    
 	    			    	    
 	    // Size match
-	    if ((unsigned)Buf.st_size != Size)
+	    if ((unsigned long long)Buf.st_size != Size)
 	    {
 	       if (Debug == true)
 		  clog << "Wrong Size: " << File << endl;
@@ -455,7 +456,7 @@ bool IndexCopy::GrabFirst(string Path,string &To,unsigned int Depth)
 // PackageCopy::GetFile - Get the file information from the section	/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-bool PackageCopy::GetFile(string &File,unsigned long &Size)
+bool PackageCopy::GetFile(string &File,unsigned long long &Size)
 {
    File = Section->FindS("Filename");
    Size = Section->FindI("Size");
@@ -481,7 +482,7 @@ bool PackageCopy::RewriteEntry(FILE *Target,string File)
 // SourceCopy::GetFile - Get the file information from the section	/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-bool SourceCopy::GetFile(string &File,unsigned long &Size)
+bool SourceCopy::GetFile(string &File,unsigned long long &Size)
 {
    string Files = Section->FindS("Files");
    if (Files.empty() == true)
@@ -504,7 +505,7 @@ bool SourceCopy::GetFile(string &File,unsigned long &Size)
       return _error->Error("Error parsing file record");
    
    // Parse the size and append the directory
-   Size = atoi(sSize.c_str());
+   Size = strtoull(sSize.c_str(), NULL, 10);
    File = Base + File;
    return true;
 }
@@ -787,7 +788,7 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name,	/*{{{*/
    bool Debug = _config->FindB("Debug::aptcdrom",false);
    
    // Prepare the progress indicator
-   unsigned long TotalSize = 0;
+   off_t TotalSize = 0;
    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
    {
       struct stat Buf;
@@ -798,14 +799,14 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name,	/*{{{*/
       TotalSize += Buf.st_size;
    }	
 
-   unsigned long CurrentSize = 0;
+   off_t CurrentSize = 0;
    unsigned int NotFound = 0;
    unsigned int WrongSize = 0;
    unsigned int Packages = 0;
    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
    {      
       string OrigPath = string(*I,CDROM.length());
-      unsigned long FileSize = 0;
+      off_t FileSize = 0;
       
       // Open the package file
       FileFd Pkg;

apt-pkg/indexcopy.h

@@ -37,7 +37,7 @@ class IndexCopy								/*{{{*/
    bool ReconstructChop(unsigned long &Chop,string Dir,string File);
    void ConvertToSourceList(string CD,string &Path);
    bool GrabFirst(string Path,string &To,unsigned int Depth);
-   virtual bool GetFile(string &Filename,unsigned long &Size) = 0;
+   virtual bool GetFile(string &Filename,unsigned long long &Size) = 0;
    virtual bool RewriteEntry(FILE *Target,string File) = 0;
    virtual const char *GetFileName() = 0;
    virtual const char *Type() = 0;
@@ -53,7 +53,7 @@ class PackageCopy : public IndexCopy					/*{{{*/
 {
    protected:
    
-   virtual bool GetFile(string &Filename,unsigned long &Size);
+   virtual bool GetFile(string &Filename,unsigned long long &Size);
    virtual bool RewriteEntry(FILE *Target,string File);
    virtual const char *GetFileName() {return "Packages";};
    virtual const char *Type() {return "Package";};
@@ -64,7 +64,7 @@ class SourceCopy : public IndexCopy					/*{{{*/
 {
    protected:
    
-   virtual bool GetFile(string &Filename,unsigned long &Size);
+   virtual bool GetFile(string &Filename,unsigned long long &Size);
    virtual bool RewriteEntry(FILE *Target,string File);
    virtual const char *GetFileName() {return "Sources";};
    virtual const char *Type() {return "Source";};

apt-pkg/indexrecords.cc

@@ -80,7 +80,7 @@ bool indexRecords::Load(const string Filename)				/*{{{*/
 
       string Name;
       string Hash;
-      size_t Size;
+      unsigned long long Size;
       while (Start < End)
       {
 	 if (!parseSumData(Start, End, Name, Hash, Size))
@@ -147,7 +147,7 @@ vector<string> indexRecords::MetaKeys()					/*{{{*/
 }
 									/*}}}*/
 bool indexRecords::parseSumData(const char *&Start, const char *End,	/*{{{*/
-				   string &Name, string &Hash, size_t &Size)
+				   string &Name, string &Hash, unsigned long long &Size)
 {
    Name = "";
    Hash = "";
@@ -184,7 +184,7 @@ bool indexRecords::parseSumData(const char *&Start, const char *End,	/*{{{*/
    if (EntryEnd == End)
       return false;
    
-   Size = strtol (Start, NULL, 10);
+   Size = strtoull (Start, NULL, 10);
       
    /* Skip over intermediate blanks */
    Start = EntryEnd;

apt-pkg/indexrecords.h

@@ -17,7 +17,7 @@
 class indexRecords
 {
    bool parseSumData(const char *&Start, const char *End, string &Name,
-		     string &Hash, size_t &Size);
+		     string &Hash, unsigned long long &Size);
    public:
    struct checkSum;
    string ErrorText;
@@ -53,7 +53,7 @@ struct indexRecords::checkSum
 {
    string MetaKeyFilename;
    HashString Hash;
-   size_t Size;      
+   unsigned long long Size;
 };
 
 #endif

apt-pkg/pkgrecords.h

@@ -19,7 +19,6 @@
 
 
 #include <apt-pkg/pkgcache.h>
-#include <apt-pkg/fileutl.h>
 #include <vector>
 
 class pkgRecords							/*{{{*/

apt-pkg/tagfile.cc

@@ -29,7 +29,7 @@ using std::string;
 class pkgTagFilePrivate
 {
 public:
-   pkgTagFilePrivate(FileFd *pFd, unsigned long Size) : Fd(*pFd), Size(Size)
+   pkgTagFilePrivate(FileFd *pFd, unsigned long long Size) : Fd(*pFd), Size(Size)
    {
    }
    FileFd &Fd;
@@ -37,14 +37,14 @@ public:
    char *Start;
    char *End;
    bool Done;
-   unsigned long iOffset;
-   unsigned long Size;
+   unsigned long long iOffset;
+   unsigned long long Size;
 };
 
 // TagFile::pkgTagFile - Constructor					/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-pkgTagFile::pkgTagFile(FileFd *pFd,unsigned long Size)
+pkgTagFile::pkgTagFile(FileFd *pFd,unsigned long long Size)
 {
    d = new pkgTagFilePrivate(pFd, Size);
 
@@ -86,7 +86,7 @@ unsigned long pkgTagFile::Offset()
 bool pkgTagFile::Resize()
 {
    char *tmp;
-   unsigned long EndSize = d->End - d->Start;
+   unsigned long long EndSize = d->End - d->Start;
 
    // fail is the buffer grows too big
    if(d->Size > 1024*1024+1)
@@ -138,8 +138,8 @@ bool pkgTagFile::Step(pkgTagSection &Tag)
    then fills the rest from the file */
 bool pkgTagFile::Fill()
 {
-   unsigned long EndSize = d->End - d->Start;
-   unsigned long Actual = 0;
+   unsigned long long EndSize = d->End - d->Start;
+   unsigned long long Actual = 0;
    
    memmove(d->Buffer,d->Start,EndSize);
    d->Start = d->Buffer;
@@ -180,12 +180,12 @@ bool pkgTagFile::Fill()
 // ---------------------------------------------------------------------
 /* This jumps to a pre-recorded file location and reads the record
    that is there */
-bool pkgTagFile::Jump(pkgTagSection &Tag,unsigned long Offset)
+bool pkgTagFile::Jump(pkgTagSection &Tag,unsigned long long Offset)
 {
    // We are within a buffer space of the next hit..
    if (Offset >= d->iOffset && d->iOffset + (d->End - d->Start) > Offset)
    {
-      unsigned long Dist = Offset - d->iOffset;
+      unsigned long long Dist = Offset - d->iOffset;
       d->Start += Dist;
       d->iOffset += Dist;
       return Step(Tag);

apt-pkg/tagfile.h

@@ -94,9 +94,9 @@ class pkgTagFile
 
    bool Step(pkgTagSection &Section);
    unsigned long Offset();
-   bool Jump(pkgTagSection &Tag,unsigned long Offset);
+   bool Jump(pkgTagSection &Tag,unsigned long long Offset);
 
-   pkgTagFile(FileFd *F,unsigned long Size = 32*1024);
+   pkgTagFile(FileFd *F,unsigned long long Size = 32*1024);
    virtual ~pkgTagFile();
 };
 

buildlib/config.h.in

@@ -33,6 +33,9 @@
 /* If there is no socklen_t, define this for the netdb shim */
 #undef NEED_SOCKLEN_T_DEFINE
 
+/* Define to the size of the filesize containing structures */
+#undef _FILE_OFFSET_BITS
+
 /* Define the arch name string */
 #undef COMMON_ARCH
 

cmdline/acqprogress.cc

@@ -163,7 +163,7 @@ bool AcqTextStatus::Pulse(pkgAcquire *Owner)
       ScreenWidth = sizeof(Buffer)-1;
 
    // Put in the percent done
-   sprintf(S,"%ld%%",long(double((CurrentBytes + CurrentItems)*100.0)/double(TotalBytes+TotalItems)));
+   sprintf(S,"%.0f%%",((CurrentBytes + CurrentItems)*100.0)/(TotalBytes+TotalItems));
 
    bool Shown = false;
    for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0;
@@ -214,11 +214,11 @@ bool AcqTextStatus::Pulse(pkgAcquire *Owner)
       if (I->TotalSize > 0 && I->CurrentItem->Owner->Complete == false)
       {
 	 if (Mode == Short)
-	    snprintf(S,End-S," %lu%%",
-		     long(double(I->CurrentSize*100.0)/double(I->TotalSize)));
+	    snprintf(S,End-S," %.0f%%",
+		     (I->CurrentSize*100.0)/I->TotalSize);
 	 else
-	    snprintf(S,End-S,"/%sB %lu%%",SizeToStr(I->TotalSize).c_str(),
-		     long(double(I->CurrentSize*100.0)/double(I->TotalSize)));
+	    snprintf(S,End-S,"/%sB %.0f%%",SizeToStr(I->TotalSize).c_str(),
+		     (I->CurrentSize*100.0)/I->TotalSize);
       }      
       S += strlen(S);
       snprintf(S,End-S,"]");
@@ -238,7 +238,7 @@ bool AcqTextStatus::Pulse(pkgAcquire *Owner)
    if (CurrentCPS != 0)
    {      
       char Tmp[300];
-      unsigned long ETA = (unsigned long)((TotalBytes - CurrentBytes)/CurrentCPS);
+      unsigned long long ETA = (TotalBytes - CurrentBytes)/CurrentCPS;
       sprintf(Tmp," %sB/s %s",SizeToStr(CurrentCPS).c_str(),TimeToStr(ETA).c_str());
       unsigned int Len = strlen(Buffer);
       unsigned int LenT = strlen(Tmp);

cmdline/apt-get.cc

@@ -27,9 +27,6 @@
 // Include Files							/*{{{*/
 #include <config.h>
 
-#define _LARGEFILE_SOURCE
-#define _LARGEFILE64_SOURCE
-
 #include <apt-pkg/aptconfiguration.h>
 #include <apt-pkg/error.h>
 #include <apt-pkg/cmndline.h>
@@ -68,9 +65,6 @@
 #include <sstream>
 
 #include <apti18n.h>
-
-#define statfs statfs64
-#define statvfs statvfs64
 									/*}}}*/
 
 #define RAMFS_MAGIC     0x858458f6

configure.in

@@ -31,6 +31,10 @@ AC_CHECK_TOOL_PREFIX
 AC_PROG_CC
 AC_ISC_POSIX
 
+dnl check for large file support and enable it if possible
+dnl do this early as other stuff might depend on it
+AC_SYS_LARGEFILE
+
 dnl Check for other programs
 AC_PROG_CXX
 AC_PROG_CPP
@@ -151,7 +155,7 @@ if test "$cross_compiling" = "yes" -a "x$archline" = "x"; then
   AC_MSG_ERROR(When cross compiling, architecture must be present in sizetable)
 fi
 AC_C_BIGENDIAN
-   
+
 dnl We do not need this if we have inttypes!
 HAVE_C9X=yes
 if test x"$apt_cv_c9x_ints" = x"no"; then

debian/changelog

@@ -1,3 +1,12 @@
+apt (0.8.16~exp3+nmu1) experimental; urgency=low
+
+  [ David Kalnischkies ]
+  * Support large files in the complete toolset. Indexes of this
+    size are pretty unlikely for now, but we need it for deb
+    packages which could become bigger than 4GB now (LP: #815895)
+
+ -- David Kalnischkies <kalnischkies@gmail.com>  Tue, 13 Sep 2011 17:44:00 +0200
+
 apt (0.8.16~exp3) experimental; urgency=low
 
   [ David Kalnischkies ]

ftparchive/apt-ftparchive.cc

@@ -206,7 +206,7 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
    Packages.Output = 0;      // Just in case
    
    // Finish compressing
-   unsigned long Size;
+   unsigned long long Size;
    if (Comp.Finalize(Size) == false)
    {
       c0out << endl;
@@ -292,7 +292,7 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
    Sources.Output = 0;      // Just in case
    
    // Finish compressing
-   unsigned long Size;
+   unsigned long long Size;
    if (Comp.Finalize(Size) == false)
    {
       c0out << endl;
@@ -363,11 +363,11 @@ bool PackageMap::GenContents(Configuration &Setup,
       if (_error->PendingError() == true)
 	 return false;
       
-      unsigned long Size = Head.Size();
+      unsigned long long Size = Head.Size();
       unsigned char Buf[4096];
       while (Size != 0)
       {
-	 unsigned long ToRead = Size;
+	 unsigned long long ToRead = Size;
 	 if (Size > sizeof(Buf))
 	    ToRead = sizeof(Buf);
 	 
@@ -401,7 +401,7 @@ bool PackageMap::GenContents(Configuration &Setup,
    Contents.Finish();
    
    // Finish compressing
-   unsigned long Size;
+   unsigned long long Size;
    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
    {
       c0out << endl;

ftparchive/cachedb.h

@@ -81,7 +81,7 @@ class CacheDB
    {
       uint32_t Flags;
       uint32_t mtime;          
-      uint32_t FileSize;
+      uint64_t FileSize;
       uint8_t  MD5[16];
       uint8_t  SHA1[20];
       uint8_t  SHA256[32];
@@ -114,7 +114,7 @@ class CacheDB
       double SHA512Bytes;
       unsigned long Packages;
       unsigned long Misses;  
-      unsigned long DeLinkBytes;
+      unsigned long long DeLinkBytes;
       
       inline void Add(const Stats &S) {
 	 Bytes += S.Bytes; 
@@ -133,7 +133,7 @@ class CacheDB
    inline bool DBFailed() {return Dbp != 0 && DBLoaded == false;};
    inline bool Loaded() {return DBLoaded == true;};
    
-   inline off_t GetFileSize(void) {return CurStat.FileSize;}
+   inline unsigned long long GetFileSize(void) {return CurStat.FileSize;}
    
    bool SetFile(string const &FileName,struct stat St,FileFd *Fd);
    bool GetFileInfo(string const &FileName, bool const &DoControl, bool const &DoContents, bool const &GenContentsOnly,

ftparchive/contents.cc

@@ -349,7 +349,7 @@ bool ContentsExtract::DoItem(Item &Itm,int &Fd)
 // ContentsExtract::TakeContents - Load the contents data		/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-bool ContentsExtract::TakeContents(const void *NewData,unsigned long Length)
+bool ContentsExtract::TakeContents(const void *NewData,unsigned long long Length)
 {
    if (Length == 0)
    {

ftparchive/contents.h

@@ -71,15 +71,15 @@ class ContentsExtract : public pkgDirStream
 
    // The Data Block
    char *Data;
-   unsigned long MaxSize;
-   unsigned long CurSize;
+   unsigned long long MaxSize;
+   unsigned long long CurSize;
    void AddData(const char *Text);
    
    bool Read(debDebFile &Deb);
    
    virtual bool DoItem(Item &Itm,int &Fd);      
    void Reset() {CurSize = 0;};
-   bool TakeContents(const void *Data,unsigned long Length);
+   bool TakeContents(const void *Data,unsigned long long Length);
    void Add(GenContents &Contents,string const &Package);
    
    ContentsExtract() : Data(0), MaxSize(0), CurSize(0) {};

ftparchive/multicompress.cc

@@ -215,7 +215,7 @@ bool MultiCompress::Die()
 // MultiCompress::Finalize - Finish up writing				/*{{{*/
 // ---------------------------------------------------------------------
 /* This is only necessary for statistics reporting. */
-bool MultiCompress::Finalize(unsigned long &OutSize)
+bool MultiCompress::Finalize(unsigned long long &OutSize)
 {
    OutSize = 0;
    if (Input == 0 || Die() == false)
@@ -383,7 +383,7 @@ bool MultiCompress::Child(int const &FD)
       stash a hash of the data to use later. */
    SetNonBlock(FD,false);
    unsigned char Buffer[32*1024];
-   unsigned long FileSize = 0;
+   unsigned long long FileSize = 0;
    MD5Summation MD5;
    while (1)
    {
@@ -445,7 +445,7 @@ bool MultiCompress::Child(int const &FD)
             
       // Compute the hash
       MD5Summation OldMD5;
-      unsigned long NewFileSize = 0;
+      unsigned long long NewFileSize = 0;
       while (1)
       {
 	 int Res = read(CompFd,Buffer,sizeof(Buffer));

ftparchive/multicompress.h

@@ -54,7 +54,7 @@ class MultiCompress
    FILE *Input;
    unsigned long UpdateMTime;
    
-   bool Finalize(unsigned long &OutSize);
+   bool Finalize(unsigned long long &OutSize);
    bool OpenOld(int &Fd,pid_t &Proc);
    bool CloseOld(int Fd,pid_t Proc);
    static bool GetStat(string const &Output,string const &Compress,struct stat &St);

ftparchive/override.cc

@@ -34,7 +34,7 @@ bool Override::ReadOverride(string const &File,bool const &Source)
       return _error->Errno("fopen",_("Unable to open %s"),File.c_str());
    
    char Line[500];
-   unsigned long Counter = 0;
+   unsigned long long Counter = 0;
    while (fgets(Line,sizeof(Line),F) != 0)
    {
       Counter++;
@@ -57,7 +57,7 @@ bool Override::ReadOverride(string const &File,bool const &Source)
       for (; isspace(*End) == 0 && *End != 0; End++);
       if (*End == 0)
       {
-	 _error->Warning(_("Malformed override %s line %lu #1"),File.c_str(),
+	 _error->Warning(_("Malformed override %s line %llu #1"),File.c_str(),
 			 Counter);
 	 continue;
       }      
@@ -71,7 +71,7 @@ bool Override::ReadOverride(string const &File,bool const &Source)
 	 for (; isspace(*End) == 0 && *End != 0; End++);
 	 if (*End == 0)
 	 {
-	    _error->Warning(_("Malformed override %s line %lu #2"),File.c_str(),
+	    _error->Warning(_("Malformed override %s line %llu #2"),File.c_str(),
 			    Counter);
 	    continue;
 	 }
@@ -85,7 +85,7 @@ bool Override::ReadOverride(string const &File,bool const &Source)
       for (; isspace(*End) == 0 && *End != 0; End++);
       if (*End == 0)
       {
-	 _error->Warning(_("Malformed override %s line %lu #3"),File.c_str(),
+	 _error->Warning(_("Malformed override %s line %llu #3"),File.c_str(),
 			 Counter);
 	 continue;
       }      
@@ -142,7 +142,7 @@ bool Override::ReadExtraOverride(string const &File,bool const &Source)
       return _error->Errno("fopen",_("Unable to open %s"),File.c_str());
   
    char Line[500];
-   unsigned long Counter = 0;
+   unsigned long long Counter = 0;
    while (fgets(Line,sizeof(Line),F) != 0)
    {
       Counter++;
@@ -163,7 +163,7 @@ bool Override::ReadExtraOverride(string const &File,bool const &Source)
       for (; isspace(*End) == 0 && *End != 0; End++);
       if (*End == 0)
       {
-	 _error->Warning(_("Malformed override %s line %lu #1"),File.c_str(),
+	 _error->Warning(_("Malformed override %s line %llu #1"),File.c_str(),
 			 Counter);
 	 continue;
       }      
@@ -175,7 +175,7 @@ bool Override::ReadExtraOverride(string const &File,bool const &Source)
       for (; isspace(*End) == 0 && *End != 0; End++);
       if (*End == 0)
       {
-	 _error->Warning(_("Malformed override %s line %lu #2"),File.c_str(),
+	 _error->Warning(_("Malformed override %s line %llu #2"),File.c_str(),
 			 Counter);
 	 continue;
       }
@@ -188,7 +188,7 @@ bool Override::ReadExtraOverride(string const &File,bool const &Source)
       for (; isspace(*(End-1)) && End > Value; End--);
       if (End == Value)
       {
-	 _error->Warning(_("Malformed override %s line %lu #3"),File.c_str(),
+	 _error->Warning(_("Malformed override %s line %llu #3"),File.c_str(),
 			 Counter);
 	 continue;
       }      

ftparchive/writer.cc

@@ -248,8 +248,8 @@ bool FTWScanner::LoadFileList(string const &Dir, string const &File)
 // ---------------------------------------------------------------------
 /* */
 bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
-			unsigned long &DeLinkBytes,
-			off_t const &FileSize)
+			unsigned long long &DeLinkBytes,
+			unsigned long long const &FileSize)
 {
    // See if this isn't an internaly prefix'd file name.
    if (InternalPrefix.empty() == false &&
@@ -379,7 +379,7 @@ bool PackagesWriter::DoPackage(string FileName)
       return false;
    }
 
-   off_t FileSize = Db.GetFileSize();
+   unsigned long long FileSize = Db.GetFileSize();
    if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
       return false;
    
@@ -415,7 +415,7 @@ bool PackagesWriter::DoPackage(string FileName)
    }
 
    char Size[40];
-   sprintf(Size,"%lu", (unsigned long) FileSize);
+   sprintf(Size,"%llu", (unsigned long long) FileSize);
    
    // Strip the DirStrip prefix from the FileName and add the PathPrefix
    string NewFileName;
@@ -612,7 +612,7 @@ bool SourcesWriter::DoPackage(string FileName)
    if (St.st_size > 128*1024)
       return _error->Error("DSC file '%s' is too large!",FileName.c_str());
          
-   if (BufSize < (unsigned)St.st_size+1)
+   if (BufSize < (unsigned long long)St.st_size+1)
    {
       BufSize = St.st_size+1;
       Buffer = (char *)realloc(Buffer,St.st_size+1);
@@ -1067,7 +1067,7 @@ void ReleaseWriter::Finish()
       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
 	  I != CheckSums.end(); ++I)
       {
-	 fprintf(Output, " %s %16ld %s\n",
+	 fprintf(Output, " %s %16llu %s\n",
 		 (*I).second.MD5.c_str(),
 		 (*I).second.size,
 		 (*I).first.c_str());
@@ -1079,7 +1079,7 @@ void ReleaseWriter::Finish()
       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
 	  I != CheckSums.end(); ++I)
       {
-	 fprintf(Output, " %s %16ld %s\n",
+	 fprintf(Output, " %s %16llu %s\n",
 		 (*I).second.SHA1.c_str(),
 		 (*I).second.size,
 		 (*I).first.c_str());
@@ -1091,7 +1091,7 @@ void ReleaseWriter::Finish()
       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
 	  I != CheckSums.end(); ++I)
       {
-	 fprintf(Output, " %s %16ld %s\n",
+	 fprintf(Output, " %s %16llu %s\n",
 		 (*I).second.SHA256.c_str(),
 		 (*I).second.size,
 		 (*I).first.c_str());
@@ -1103,7 +1103,7 @@ void ReleaseWriter::Finish()
        I != CheckSums.end();
        ++I)
    {
-      fprintf(Output, " %s %32ld %s\n",
+      fprintf(Output, " %s %16llu %s\n",
               (*I).second.SHA512.c_str(),
               (*I).second.size,
               (*I).first.c_str());

ftparchive/writer.h

@@ -48,7 +48,7 @@ class FTWScanner
    static int ScannerFile(const char *File, bool const &ReadLink);
 
    bool Delink(string &FileName,const char *OriginalPath,
-	       unsigned long &Bytes,off_t const &FileSize);
+	       unsigned long long &Bytes,unsigned long long const &FileSize);
 
    inline void NewLine(unsigned const &Priority)
    {
@@ -159,7 +159,7 @@ class SourcesWriter : public FTWScanner
    Override BOver;
    Override SOver;
    char *Buffer;
-   unsigned long BufSize;
+   unsigned long long BufSize;
    
    public:
 
@@ -198,7 +198,7 @@ protected:
       string SHA256;
       string SHA512;
       // Limited by FileFd::Size()
-      unsigned long size;
+      unsigned long long size;
       ~CheckSum() {};
    };
    map<string,struct CheckSum> CheckSums;

methods/ftp.cc

@@ -629,7 +629,7 @@ bool FTPConn::ExtGoPasv()
 // FTPConn::Size - Return the size of a file				/*{{{*/
 // ---------------------------------------------------------------------
 /* Grab the file size from the server, 0 means no size or empty file */
-bool FTPConn::Size(const char *Path,unsigned long &Size)
+bool FTPConn::Size(const char *Path,unsigned long long &Size)
 {
    // Query the size
    unsigned int Tag;
@@ -639,7 +639,7 @@ bool FTPConn::Size(const char *Path,unsigned long &Size)
       return false;
    
    char *End;
-   Size = strtol(Msg.c_str(),&End,10);
+   Size = strtoull(Msg.c_str(),&End,10);
    if (Tag >= 400 || End == Msg.c_str())
       Size = 0;
    return true;
@@ -841,7 +841,7 @@ bool FTPConn::Finalize()
 // ---------------------------------------------------------------------
 /* This opens a data connection, sends REST and RETR and then
    transfers the file over. */
-bool FTPConn::Get(const char *Path,FileFd &To,unsigned long Resume,
+bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume,
 		  Hashes &Hash,bool &Missing)
 {
    Missing = false;
@@ -1004,7 +1004,7 @@ bool FtpMethod::Fetch(FetchItem *Itm)
    
    // Get the files information
    Status(_("Query"));
-   unsigned long Size;
+   unsigned long long Size;
    if (Server->Size(File,Size) == false ||
        Server->ModTime(File,FailTime) == false)
    {
@@ -1026,7 +1026,7 @@ bool FtpMethod::Fetch(FetchItem *Itm)
    struct stat Buf;
    if (stat(Itm->DestFile.c_str(),&Buf) == 0)
    {
-      if (Size == (unsigned)Buf.st_size && FailTime == Buf.st_mtime)
+      if (Size == (unsigned long long)Buf.st_size && FailTime == Buf.st_mtime)
       {
 	 Res.Size = Buf.st_size;
 	 Res.LastModified = Buf.st_mtime;
@@ -1036,7 +1036,7 @@ bool FtpMethod::Fetch(FetchItem *Itm)
       }
       
       // Resume?
-      if (FailTime == Buf.st_mtime && Size > (unsigned)Buf.st_size)
+      if (FailTime == Buf.st_mtime && Size > (unsigned long long)Buf.st_size)
 	 Res.ResumePoint = Buf.st_size;
    }
    

methods/ftp.h

@@ -53,9 +53,9 @@ class FTPConn
    bool ExtGoPasv();
    
    // Query
-   bool Size(const char *Path,unsigned long &Size);
+   bool Size(const char *Path,unsigned long long &Size);
    bool ModTime(const char *Path, time_t &Time);
-   bool Get(const char *Path,FileFd &To,unsigned long Resume,
+   bool Get(const char *Path,FileFd &To,unsigned long long Resume,
 	    Hashes &MD5,bool &Missing);
    
    FTPConn(URI Srv);

methods/gzip.cc

@@ -64,7 +64,7 @@ bool GzipMethod::Fetch(FetchItem *Itm)
    while (1) 
    {
       unsigned char Buffer[4*1024];
-      unsigned long Count;
+      unsigned long long Count = 0;
       
       if (!From.Read(Buffer,sizeof(Buffer),&Count))
       {

methods/http.cc

@@ -65,15 +65,15 @@ bool AllowRedirect = false;
 bool Debug = false;
 URI Proxy;
 
-unsigned long CircleBuf::BwReadLimit=0;
-unsigned long CircleBuf::BwTickReadData=0;
+unsigned long long CircleBuf::BwReadLimit=0;
+unsigned long long CircleBuf::BwTickReadData=0;
 struct timeval CircleBuf::BwReadTick={0,0};
 const unsigned int CircleBuf::BW_HZ=10;
  
 // CircleBuf::CircleBuf - Circular input buffer				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0)
+CircleBuf::CircleBuf(unsigned long long Size) : Size(Size), Hash(0)
 {
    Buf = new unsigned char[Size];
    Reset();
@@ -89,7 +89,7 @@ void CircleBuf::Reset()
    InP = 0;
    OutP = 0;
    StrPos = 0;
-   MaxGet = (unsigned int)-1;
+   MaxGet = (unsigned long long)-1;
    OutQueue = string();
    if (Hash != 0)
    {
@@ -104,7 +104,7 @@ void CircleBuf::Reset()
    is non-blocking.. */
 bool CircleBuf::Read(int Fd)
 {
-   unsigned long BwReadMax;
+   unsigned long long BwReadMax;
 
    while (1)
    {
@@ -119,7 +119,7 @@ bool CircleBuf::Read(int Fd)
 	 struct timeval now;
 	 gettimeofday(&now,0);
 
-	 unsigned long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
+	 unsigned long long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
 	    now.tv_usec-CircleBuf::BwReadTick.tv_usec;
 	 if(d > 1000000/BW_HZ) {
 	    CircleBuf::BwReadTick = now;
@@ -133,7 +133,7 @@ bool CircleBuf::Read(int Fd)
       }
 
       // Write the buffer segment
-      int Res;
+      ssize_t Res;
       if(CircleBuf::BwReadLimit) {
 	 Res = read(Fd,Buf + (InP%Size), 
 		    BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
@@ -182,7 +182,7 @@ void CircleBuf::FillOut()
 	 return;
       
       // Write the buffer segment
-      unsigned long Sz = LeftRead();
+      unsigned long long Sz = LeftRead();
       if (OutQueue.length() - StrPos < Sz)
 	 Sz = OutQueue.length() - StrPos;
       memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
@@ -216,7 +216,7 @@ bool CircleBuf::Write(int Fd)
 	 return true;
       
       // Write the buffer segment
-      int Res;
+      ssize_t Res;
       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 
       if (Res == 0)
@@ -242,7 +242,7 @@ bool CircleBuf::Write(int Fd)
 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 {
    // We cheat and assume it is unneeded to have more than one buffer load
-   for (unsigned long I = OutP; I < InP; I++)
+   for (unsigned long long I = OutP; I < InP; I++)
    {      
       if (Buf[I%Size] != '\n')
 	 continue;
@@ -260,7 +260,7 @@ bool CircleBuf::WriteTillEl(string &Data,bool Single)
       Data = "";
       while (OutP < I)
       {
-	 unsigned long Sz = LeftWrite();
+	 unsigned long long Sz = LeftWrite();
 	 if (Sz == 0)
 	    return false;
 	 if (I - OutP < Sz)
@@ -455,7 +455,7 @@ bool ServerState::RunData()
 	    return false;
 	 	 
 	 // See if we are done
-	 unsigned long Len = strtol(Data.c_str(),0,16);
+	 unsigned long long Len = strtoull(Data.c_str(),0,16);
 	 if (Len == 0)
 	 {
 	    In.Limit(-1);
@@ -598,7 +598,7 @@ bool ServerState::HeaderLine(string Line)
       if (StartPos != 0)
 	 return true;
       
-      if (sscanf(Val.c_str(),"%lu",&Size) != 1)
+      if (sscanf(Val.c_str(),"%llu",&Size) != 1)
 	 return _error->Error(_("The HTTP server sent an invalid Content-Length header"));
       return true;
    }
@@ -613,9 +613,9 @@ bool ServerState::HeaderLine(string Line)
    {
       HaveContent = true;
       
-      if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
+      if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
 	 return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
-      if ((unsigned)StartPos > Size)
+      if ((unsigned long long)StartPos > Size)
 	 return _error->Error(_("This HTTP server has broken range support"));
       return true;
    }
@@ -718,7 +718,7 @@ void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
    {
       // In this case we send an if-range query with a range header
-      sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
+      sprintf(Buf,"Range: bytes=%lli-\r\nIf-Range: %s\r\n",(long long)SBuf.st_size - 1,
 	      TimeRFC1123(SBuf.st_mtime).c_str());
       Req += Buf;
    }

methods/http.h

@@ -23,29 +23,29 @@ class HttpMethod;
 class CircleBuf
 {
    unsigned char *Buf;
-   unsigned long Size;
-   unsigned long InP;
-   unsigned long OutP;
+   unsigned long long Size;
+   unsigned long long InP;
+   unsigned long long OutP;
    string OutQueue;
-   unsigned long StrPos;
-   unsigned long MaxGet;
+   unsigned long long StrPos;
+   unsigned long long MaxGet;
    struct timeval Start;
    
-   static unsigned long BwReadLimit;
-   static unsigned long BwTickReadData;
+   static unsigned long long BwReadLimit;
+   static unsigned long long BwTickReadData;
    static struct timeval BwReadTick;
    static const unsigned int BW_HZ;
 
-   unsigned long LeftRead()
+   unsigned long long LeftRead()
    {
-      unsigned long Sz = Size - (InP - OutP);
+      unsigned long long Sz = Size - (InP - OutP);
       if (Sz > Size - (InP%Size))
 	 Sz = Size - (InP%Size);
       return Sz;
    }
-   unsigned long LeftWrite()
+   unsigned long long LeftWrite()
    {
-      unsigned long Sz = InP - OutP;
+      unsigned long long Sz = InP - OutP;
       if (InP > MaxGet)
 	 Sz = MaxGet - OutP;
       if (Sz > Size - (OutP%Size))
@@ -67,7 +67,7 @@ class CircleBuf
    bool WriteTillEl(string &Data,bool Single = false);
    
    // Control the write limit
-   void Limit(long Max) {if (Max == -1) MaxGet = 0-1; else MaxGet = OutP + Max;}   
+   void Limit(long long Max) {if (Max == -1) MaxGet = 0-1; else MaxGet = OutP + Max;}   
    bool IsLimit() {return MaxGet == OutP;};
    void Print() {cout << MaxGet << ',' << OutP << endl;};
 
@@ -79,7 +79,7 @@ class CircleBuf
    void Reset();
    void Stats();
 
-   CircleBuf(unsigned long Size);
+   CircleBuf(unsigned long long Size);
    ~CircleBuf() {delete [] Buf; delete Hash;};
 };
 
@@ -92,8 +92,8 @@ struct ServerState
    char Code[MAXLEN];
    
    // These are some statistics from the last parsed header lines
-   unsigned long Size;
-   signed long StartPos;
+   unsigned long long Size;
+   signed long long StartPos;
    time_t Date;
    bool HaveContent;
    enum {Chunked,Stream,Closes} Encoding;

methods/https.cc

@@ -52,7 +52,7 @@ HttpsMethod::progress_callback(void *clientp, double dltotal, double dlnow,
 {
    HttpsMethod *me = (HttpsMethod *)clientp;
    if(dltotal > 0 && me->Res.Size == 0) {
-      me->Res.Size = (unsigned long)dltotal;
+      me->Res.Size = (unsigned long long)dltotal;
       me->URIStart(me->Res);
    }
    return 0;

methods/rred.cc

@@ -256,7 +256,7 @@ RredMethod::State RredMethod::patchMMap(FileFd &Patch, FileFd &From,		/*{{{*/
 #ifdef _POSIX_MAPPED_FILES
 	MMap ed_cmds(MMap::ReadOnly);
 	if (Patch.gzFd() != NULL) {
-		unsigned long mapSize = Patch.Size();
+		unsigned long long mapSize = Patch.Size();
 		DynamicMMap* dyn = new DynamicMMap(0, mapSize, 0);
 		if (dyn->validData() == false) {
 			delete dyn;

methods/rsh.cc

@@ -251,7 +251,7 @@ bool RSHConn::WriteMsg(string &Text,bool Sync,const char *Fmt,...)
 // ---------------------------------------------------------------------
 /* Right now for successfull transfer the file size must be known in 
    advance. */
-bool RSHConn::Size(const char *Path,unsigned long &Size)
+bool RSHConn::Size(const char *Path,unsigned long long &Size)
 {
    // Query the size
    string Msg;
@@ -263,7 +263,7 @@ bool RSHConn::Size(const char *Path,unsigned long &Size)
    // FIXME: Sense if the bad reply is due to a File Not Found. 
    
    char *End;
-   Size = strtoul(Msg.c_str(),&End,10);
+   Size = strtoull(Msg.c_str(),&End,10);
    if (End == Msg.c_str())
       return _error->Error(_("File not found"));
    return true;
@@ -288,8 +288,8 @@ bool RSHConn::ModTime(const char *Path, time_t &Time)
 // RSHConn::Get - Get a file						/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-bool RSHConn::Get(const char *Path,FileFd &To,unsigned long Resume,
-                  Hashes &Hash,bool &Missing, unsigned long Size)
+bool RSHConn::Get(const char *Path,FileFd &To,unsigned long long Resume,
+                  Hashes &Hash,bool &Missing, unsigned long long Size)
 {
    Missing = false;
 
@@ -314,7 +314,7 @@ bool RSHConn::Get(const char *Path,FileFd &To,unsigned long Resume,
       return false;
 
    // Copy loop
-   unsigned int MyLen = Resume;
+   unsigned long long MyLen = Resume;
    unsigned char Buffer[4096];
    while (MyLen < Size)
    {
@@ -428,7 +428,7 @@ bool RSHMethod::Fetch(FetchItem *Itm)
    Status(_("Connecting to %s"), Get.Host.c_str());
 
    // Get the files information
-   unsigned long Size;
+   unsigned long long Size;
    if (Server->Size(File,Size) == false ||
        Server->ModTime(File,FailTime) == false)
    {
@@ -449,7 +449,7 @@ bool RSHMethod::Fetch(FetchItem *Itm)
    // See if the file exists
    struct stat Buf;
    if (stat(Itm->DestFile.c_str(),&Buf) == 0) {
-      if (Size == (unsigned)Buf.st_size && FailTime == Buf.st_mtime) {
+      if (Size == (unsigned long long)Buf.st_size && FailTime == Buf.st_mtime) {
 	 Res.Size = Buf.st_size;
 	 Res.LastModified = Buf.st_mtime;
 	 Res.ResumePoint = Buf.st_size;
@@ -458,7 +458,7 @@ bool RSHMethod::Fetch(FetchItem *Itm)
       }
 
       // Resume?
-      if (FailTime == Buf.st_mtime && Size > (unsigned)Buf.st_size)
+      if (FailTime == Buf.st_mtime && Size > (unsigned long long)Buf.st_size)
 	 Res.ResumePoint = Buf.st_size;
    }
 

methods/rsh.h

@@ -41,10 +41,10 @@ class RSHConn
    void Close();
 
    // Query
-   bool Size(const char *Path,unsigned long &Size);
+   bool Size(const char *Path,unsigned long long &Size);
    bool ModTime(const char *Path, time_t &Time);
-   bool Get(const char *Path,FileFd &To,unsigned long Resume,
-            Hashes &Hash,bool &Missing, unsigned long Size);
+   bool Get(const char *Path,FileFd &To,unsigned long long Resume,
+            Hashes &Hash,bool &Missing, unsigned long long Size);
 
    RSHConn(URI Srv);
    ~RSHConn();