FileFd: (native) LZ4 support

Implement native support for LZ4 compression, using the official
lz4 library.

apt-pkg/aptconfiguration.cc

@@ -365,6 +365,12 @@ const Configuration::getCompressors(bool const Cached) {
 	setDefaultConfigurationForCompressors();
 
 	compressors.push_back(Compressor(".", "", "", NULL, NULL, 0));
+	if (_config->Exists("Dir::Bin::lz4") == false || FileExists(_config->FindFile("Dir::Bin::lz4")) == true)
+		compressors.push_back(Compressor("lz4",".lz4","lz4","-1","-d",50));
+#ifdef HAVE_LZ4
+	else
+		compressors.push_back(Compressor("lz4",".lz4","false", NULL, NULL, 50));
+#endif
 	if (_config->Exists("Dir::Bin::gzip") == false || FileExists(_config->FindFile("Dir::Bin::gzip")) == true)
 		compressors.push_back(Compressor("gzip",".gz","gzip","-6n","-d",100));
 #ifdef HAVE_ZLIB

apt-pkg/contrib/fileutl.cc

@@ -63,6 +63,9 @@
 #ifdef HAVE_LZMA
 	#include <lzma.h>
 #endif
+#ifdef HAVE_LZ4
+	#include <lz4frame.h>
+#endif
 #include <endian.h>
 #include <stdint.h>
 
@@ -1494,6 +1497,158 @@ public:
    explicit Bz2FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), bz2(nullptr) {}
    virtual ~Bz2FileFdPrivate() { InternalClose(""); }
 #endif
+};
+									/*}}}*/
+class APT_HIDDEN Lz4FileFdPrivate: public FileFdPrivate {				/*{{{*/
+   static constexpr unsigned long long BLK_SIZE = 64 * 1024;
+   static constexpr unsigned long long LZ4_HEADER_SIZE = 19;
+   static constexpr unsigned long long LZ4_FOOTER_SIZE = 4;
+#ifdef HAVE_LZ4
+   LZ4F_decompressionContext_t dctx;
+   LZ4F_compressionContext_t cctx;
+   LZ4F_errorCode_t res;
+   FileFd backend;
+   simple_buffer lz4_buffer;
+   // Count of bytes that the decompressor expects to read next, or buffer size.
+   size_t next_to_load = BLK_SIZE;
+public:
+   virtual bool InternalOpen(int const iFd, unsigned int const Mode) override
+   {
+      if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite)
+	 return _error->Error("lz4 only supports write or read mode");
+
+      if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) {
+	 res = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION);
+	 lz4_buffer.reset(LZ4F_compressBound(BLK_SIZE, nullptr)
+			  + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE);
+      } else {
+	 res = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
+	 lz4_buffer.reset(64 * 1024);
+      }
+
+      filefd->Flags |= FileFd::Compressed;
+
+      if (LZ4F_isError(res))
+	 return false;
+
+      unsigned int flags = (Mode & (FileFd::WriteOnly|FileFd::ReadOnly));
+      if (backend.OpenDescriptor(iFd, flags) == false)
+	 return false;
+
+      // Write the file header
+      if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly)
+      {
+	 res = LZ4F_compressBegin(cctx, lz4_buffer.buffer, lz4_buffer.buffersize_max, nullptr);
+	 if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false)
+	    return false;
+      }
+
+      return true;
+   }
+   virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) override
+   {
+      /* Keep reading as long as the compressor still wants to read */
+      while (next_to_load) {
+	 // Fill compressed buffer;
+	 if (lz4_buffer.empty()) {
+	    unsigned long long read;
+	    /* Reset - if LZ4 decompressor wants to read more, allocate more */
+	    lz4_buffer.reset(next_to_load);
+	    if (backend.Read(lz4_buffer.getend(), lz4_buffer.free(), &read) == false)
+	       return -1;
+	    lz4_buffer.bufferend += read;
+
+	    /* Expected EOF */
+	    if (read == 0) {
+	       res = -1;
+	       return filefd->FileFdError("LZ4F: %s %s",
+					  filefd->FileName.c_str(),
+					  _("Unexpected end of file")), -1;
+	    }
+	 }
+	 // Drain compressed buffer as far as possible.
+	 size_t in = lz4_buffer.size();
+	 size_t out = Size;
+
+	 res = LZ4F_decompress(dctx, To, &out, lz4_buffer.get(), &in, nullptr);
+	 if (LZ4F_isError(res))
+	       return -1;
+
+	 next_to_load = res;
+	 lz4_buffer.bufferstart += in;
+
+	 if (out != 0)
+	    return out;
+      }
+
+      return 0;
+   }
+   virtual bool InternalReadError() override
+   {
+      char const * const errmsg = LZ4F_getErrorName(res);
+
+      return filefd->FileFdError("LZ4F: %s %s (%zu: %s)", filefd->FileName.c_str(), _("Read error"), res, errmsg);
+   }
+   virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) override
+   {
+      unsigned long long const towrite = std::min(BLK_SIZE, Size);
+
+      res = LZ4F_compressUpdate(cctx,
+				lz4_buffer.buffer, lz4_buffer.buffersize_max,
+				From, towrite, nullptr);
+
+      if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false)
+	 return -1;
+
+      return towrite;
+   }
+   virtual bool InternalWriteError() override
+   {
+      char const * const errmsg = LZ4F_getErrorName(res);
+
+      return filefd->FileFdError("LZ4F: %s %s (%zu: %s)", filefd->FileName.c_str(), _("Write error"), res, errmsg);
+   }
+   virtual bool InternalStream() const override { return true; }
+
+   virtual bool InternalFlush() override
+   {
+      return backend.Flush();
+   }
+
+   virtual bool InternalClose(std::string const &) override
+   {
+      /* Reset variables */
+      res = 0;
+      next_to_load = BLK_SIZE;
+
+      if (cctx != nullptr)
+      {
+	 res = LZ4F_compressEnd(cctx, lz4_buffer.buffer, lz4_buffer.buffersize_max, nullptr);
+	 if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false)
+	    return false;
+	 if (!backend.Flush())
+	    return false;
+	 if (!backend.Close())
+	    return false;
+
+	 res = LZ4F_freeCompressionContext(cctx);
+	 cctx = nullptr;
+      }
+
+      if (dctx != nullptr)
+      {
+	 res = LZ4F_freeDecompressionContext(dctx);
+	 dctx = nullptr;
+      }
+
+      return LZ4F_isError(res) == false;
+   }
+
+   explicit Lz4FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), dctx(nullptr), cctx(nullptr) {}
+   virtual ~Lz4FileFdPrivate() {
+      InternalClose("");
+   }
+#endif
 };
 									/*}}}*/
 class APT_HIDDEN LzmaFileFdPrivate: public FileFdPrivate {				/*{{{*/
@@ -1968,6 +2123,7 @@ bool FileFd::Open(string FileName,unsigned int const Mode,CompressMode Compress,
       case Bzip2: name = "bzip2"; break;
       case Lzma: name = "lzma"; break;
       case Xz: name = "xz"; break;
+      case Lz4: name = "lz4"; break;
       case Auto:
       case Extension:
 	 // Unreachable
@@ -2084,6 +2240,7 @@ bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, CompressMode Compre
    case Bzip2: name = "bzip2"; break;
    case Lzma: name = "lzma"; break;
    case Xz: name = "xz"; break;
+   case Lz4: name = "lz4"; break;
    case Auto:
    case Extension:
       if (AutoClose == true && Fd != -1)
@@ -2145,6 +2302,9 @@ bool FileFd::OpenInternDescriptor(unsigned int const Mode, APT::Configuration::C
       APT_COMPRESS_INIT("xz", LzmaFileFdPrivate);
       APT_COMPRESS_INIT("lzma", LzmaFileFdPrivate);
 #endif
+#ifdef HAVE_LZ4
+      APT_COMPRESS_INIT("lz4", Lz4FileFdPrivate);
+#endif
 #undef APT_COMPRESS_INIT
       else if (compressor.Name == "." || compressor.Binary.empty() == true)
 	 d = new DirectFileFdPrivate(this);

apt-pkg/contrib/fileutl.h

@@ -45,6 +45,7 @@ class FileFd
    friend class GzipFileFdPrivate;
    friend class Bz2FileFdPrivate;
    friend class LzmaFileFdPrivate;
+   friend class Lz4FileFdPrivate;
    friend class DirectFileFdPrivate;
    friend class PipedFileFdPrivate;
    protected:
@@ -75,7 +76,7 @@ class FileFd
 	ReadOnlyGzip,
 	WriteAtomic = ReadWrite | Create | Atomic
    };
-   enum CompressMode { Auto = 'A', None = 'N', Extension = 'E', Gzip = 'G', Bzip2 = 'B', Lzma = 'L', Xz = 'X' };
+   enum CompressMode { Auto = 'A', None = 'N', Extension = 'E', Gzip = 'G', Bzip2 = 'B', Lzma = 'L', Xz = 'X', Lz4='4' };
    
    inline bool Read(void *To,unsigned long long Size,bool AllowEof)
    {

apt-pkg/makefile

@@ -25,6 +25,9 @@ endif
 ifeq ($(HAVE_LZMA),yes)
 SLIBS+= -llzma
 endif
+ifeq ($(HAVE_LZ4),yes)
+SLIBS+= -llz4
+endif
 APT_DOMAIN:=libapt-pkg$(LIBAPTPKG_MAJOR)
 
 SOURCE = $(wildcard *.cc */*.cc)

buildlib/config.h.in

@@ -14,6 +14,9 @@
 /* Define if we have the lzma library for lzma/xz */
 #undef HAVE_LZMA
 
+/* Define if we have the lz4 library for lz4 */
+#undef HAVE_LZ4
+
 /* These two are used by the statvfs shim for glibc2.0 and bsd */
 /* Define if we have sys/vfs.h */
 #undef HAVE_VFS_H

buildlib/environment.mak.in

@@ -69,6 +69,7 @@ HAVE_STATVFS = @HAVE_STATVFS@
 HAVE_ZLIB = @HAVE_ZLIB@
 HAVE_BZ2 = @HAVE_BZ2@
 HAVE_LZMA = @HAVE_LZMA@
+HAVE_LZ4 = @HAVE_LZ4@
 NEED_SOCKLEN_T_DEFINE = @NEED_SOCKLEN_T_DEFINE@
 
 # Shared library things

configure.ac

@@ -108,6 +108,13 @@ if test "x$HAVE_ZLIB" = "xyes"; then
 	AC_DEFINE(HAVE_ZLIB)
 fi
 
+HAVE_LZ4=no
+AC_CHECK_LIB(lz4, LZ4F_createCompressionContext,[AC_CHECK_HEADER(lz4frame.h, [HAVE_LZ4=yes], [])], [])
+AC_SUBST(HAVE_LZ4)
+if test "x$HAVE_LZ4" = "xyes"; then
+	AC_DEFINE(HAVE_LZ4)
+fi
+
 HAVE_BZ2=no
 AC_CHECK_LIB(bz2, BZ2_bzopen,[AC_CHECK_HEADER(bzlib.h, [HAVE_BZ2=yes], [])], [])
 AC_SUBST(HAVE_BZ2)

debian/control

@@ -7,7 +7,7 @@ Uploaders: Michael Vogt <mvo@debian.org>,
 Standards-Version: 3.9.6
 Build-Depends: dpkg-dev (>= 1.15.8), debhelper (>= 8.1.3~), libdb-dev,
  gettext (>= 0.12), libcurl4-gnutls-dev (>= 7.19.4~),
- zlib1g-dev, libbz2-dev, liblzma-dev,
+ zlib1g-dev, libbz2-dev, liblzma-dev, liblz4-dev,
  xsltproc, docbook-xsl, docbook-xml, po4a (>= 0.34-2),
  autotools-dev, autoconf, automake, libgtest-dev <!nocheck>,
  g++ (>= 4:5.2)