Merge remote-tracking branch 'mvo/feature/source-deb822' into debian/experimental-no-abi-break

apt-pkg/sourcelist.cc

@@ -17,6 +17,7 @@
 #include <apt-pkg/configuration.h>
 #include <apt-pkg/metaindex.h>
 #include <apt-pkg/indexfile.h>
+#include <apt-pkg/tagfile.h>
 
 #include <fstream>
 
@@ -159,7 +160,6 @@ bool pkgSourceList::Type::ParseLine(vector<metaIndex *> &List,
    return true;
 }
 									/*}}}*/
-
 // SourceList::pkgSourceList - Constructors				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -181,7 +181,6 @@ pkgSourceList::~pkgSourceList()
       delete *I;
 }
 									/*}}}*/
-									/*}}}*/
 // SourceList::ReadMainList - Read the main source list from etc	/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -216,7 +215,6 @@ bool pkgSourceList::ReadMainList()
    return Res;
 }
 									/*}}}*/
-// CNC:2003-03-03 - Needed to preserve backwards compatibility.
 // SourceList::Reset - Clear the sourcelist contents			/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -227,7 +225,6 @@ void pkgSourceList::Reset()
    SrcList.erase(SrcList.begin(),SrcList.end());
 }
 									/*}}}*/
-// CNC:2003-03-03 - Function moved to ReadAppend() and Reset().
 // SourceList::Read - Parse the sourcelist file				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -241,17 +238,29 @@ bool pkgSourceList::Read(string File)
 // ---------------------------------------------------------------------
 /* */
 bool pkgSourceList::ReadAppend(string File)
+{
+   if (_config->FindB("APT::Sources::Use-Deb822", true) == true)
+   {
+      int lines_parsed =ParseFileDeb822(File);
+      if (lines_parsed < 0)
+         return false;
+      else if (lines_parsed > 0)
+         return true;
+      // no lines parsed  ... fall through and use old style parser
+   }
+   return ParseFileOldStyle(File);
+}
+
+// SourceList::ReadFileOldStyle - Read Traditional style sources.list 	/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool pkgSourceList::ParseFileOldStyle(string File)
 {
    // Open the stream for reading
    ifstream F(File.c_str(),ios::in /*| ios::nocreate*/);
    if (!F != 0)
       return _error->Errno("ifstream::ifstream",_("Opening %s"),File.c_str());
-   
-#if 0 // Now Reset() does this.
-   for (const_iterator I = SrcList.begin(); I != SrcList.end(); I++)
-      delete *I;
-   SrcList.erase(SrcList.begin(),SrcList.end());
-#endif
+
    // CNC:2003-12-10 - 300 is too short.
    char Buffer[1024];
 
@@ -298,6 +307,78 @@ bool pkgSourceList::ReadAppend(string File)
    return true;
 }
 									/*}}}*/
+// SourceList::ParseFileDeb822 - Parse deb822 style sources.list 	/*{{{*/
+// ---------------------------------------------------------------------
+/* Returns: the number of stanzas parsed*/
+int pkgSourceList::ParseFileDeb822(string File)
+{
+   pkgTagSection Tags;
+   map<string, string> Options;
+   unsigned int i=0;
+
+   // see if we can read the file
+   _error->PushToStack();
+   FileFd Fd(File, FileFd::ReadOnly);
+   pkgTagFile Sources(&Fd);
+   if (_error->PendingError() == true)
+   {
+      _error->RevertToStack();
+      return 0;
+   }
+   _error->MergeWithStack();
+   
+   // read step by step
+   while (Sources.Step(Tags) == true)
+   {
+      if(!Tags.Exists("Type")) 
+         continue;
+
+      string const type = Tags.FindS("Type");
+      Type *Parse = Type::GetType(type.c_str());
+      if (Parse == 0)
+      {
+         _error->Error(_("Type '%s' is not known on stanza %u in source list %s"),type.c_str(),i,Fd.Name().c_str());
+         // true means we do not retry with old-style sources.list
+         return -1;
+      }
+         
+      string URI = Tags.FindS("URL");
+      if (!Parse->FixupURI(URI))
+      {
+         _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str());
+         // means we do not retry with old-style sources.list
+         return -1;
+      }
+
+      string Dist = Tags.FindS("Dist");
+      Dist = SubstVar(Dist,"$(ARCH)",_config->Find("APT::Architecture"));
+
+      // check if there are any options we support
+      const char* option_str[] = { 
+         "arch", "arch+", "arch-", "trusted",
+      };
+      for (unsigned int j=0; j < sizeof(option_str)/sizeof(char*); j++)
+         if (Tags.Exists(option_str[j]))
+            Options[option_str[j]] = Tags.FindS(option_str[j]);
+
+      // now create one item per section
+      string const Section = Tags.FindS("Section");
+      std::vector<std::string> list;
+      if (Section.find(","))
+         list = StringSplit(Section, ",");
+      else
+         list = StringSplit(Section, " ");
+      for (std::vector<std::string>::const_iterator I = list.begin();
+           I != list.end(); I++)
+         Parse->CreateItem(SrcList, URI, Dist, (*I), Options);
+
+      i++;
+   }
+
+   // we are done, return the number of stanzas read
+   return i;
+}
+									/*}}}*/
 // SourceList::FindIndex - Get the index associated with a file		/*{{{*/
 // ---------------------------------------------------------------------
 /* */

apt-pkg/sourcelist.h

@@ -75,7 +75,10 @@ class pkgSourceList
    protected:
 
    std::vector<metaIndex *> SrcList;
-   
+
+   int ParseFileDeb822(std::string File);
+   bool ParseFileOldStyle(std::string File);
+
    public:
 
    bool ReadMainList();

test/integration/framework

@@ -948,13 +948,22 @@ testempty() {
 	test -z "$($* 2>&1)" && msgpass || msgfail
 }
 
-testequal() {
+testequalwithmsg() {
+        local MSG="$1"
+        shift
 	local COMPAREFILE=$(mktemp)
 	addtrap "rm $COMPAREFILE;"
 	echo "$1" > $COMPAREFILE
 	shift
-	msgtest "Test for equality of" "$*"
+	msgtest "$MSG"
 	$* 2>&1 | checkdiff $COMPAREFILE - && msgpass || msgfail
+}        
+
+testequal() { 
+       local EXPECTED="$1"
+       shift
+       local MSG="Test for equality of $*"
+       testequalwithmsg "$MSG" "$EXPECTED" $*
 }
 
 testequalor2() {

test/integration/test-apt-sources-deb822

@@ -0,0 +1,80 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+
+setupenvironment
+configarchitecture "i386"
+
+SOURCES="rootdir/etc/apt/sources.list"
+
+echo "deb http://ftp.debian.org/debian stable main" > $SOURCES
+testequalwithmsg "Old style sources.list works" "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 :
+'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 :
+'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 "  aptget update --print-uris 
+
+
+BASE="# some comment
+# that contains a : as well
+#Type: meep
+
+Type: deb
+URL: http://ftp.debian.org/debian
+Dist: stable
+Section: main
+Comment: Some random string
+ that can be very long"
+
+# simple case
+echo "$BASE"  > $SOURCES
+
+testequalwithmsg "Simple deb822 sources.list works" "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 :
+'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 :
+'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 "  aptget update --print-uris 
+
+
+# two sections (we support both "," and " " as seperator)
+echo "$BASE" | sed s/main/"main,contrib"/ > $SOURCES
+
+testequalwithmsg "Two sections deb822 sources.list work" "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 :
+'http://ftp.debian.org/debian/dists/stable/contrib/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_contrib_binary-i386_Packages 0 :
+'http://ftp.debian.org/debian/dists/stable/contrib/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_contrib_i18n_Translation-en 0 :
+'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 :
+'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 "  aptget update --print-uris 
+
+
+# Two entries
+echo "$BASE" > $SOURCES
+echo "" >> $SOURCES
+echo "$BASE" | sed  s/stable/unstable/  >> $SOURCES
+
+testequalwithmsg "Multiple entries in deb822 sources.list work" "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 :
+'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 :
+'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 
+'http://ftp.debian.org/debian/dists/unstable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_unstable_main_binary-i386_Packages 0 :
+'http://ftp.debian.org/debian/dists/unstable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_unstable_main_i18n_Translation-en 0 :
+'http://ftp.debian.org/debian/dists/unstable/InRelease' ftp.debian.org_debian_dists_unstable_InRelease 0 "  aptget update --print-uris 
+
+
+# ARCH option
+echo "$BASE" > $SOURCES
+echo "Arch: amd64,armel" >> $SOURCES
+
+testequalwithmsg "Arch: option in deb822 sources.list works" "'http://ftp.debian.org/debian/dists/stable/main/binary-amd64/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-amd64_Packages 0 :
+'http://ftp.debian.org/debian/dists/stable/main/binary-armel/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-armel_Packages 0 :
+'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 :
+'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 "  aptget update --print-uris
+
+# invalid sources.list file
+echo "deb http://ftp.debian.org" > $SOURCES
+
+testequalwithmsg "Invalid sources.list file gives proper error" "E: Malformed line 1 in source list $TMPWORKINGDIRECTORY/rootdir/etc/apt/sources.list (dist)
+E: The list of sources could not be read."  aptget update --print-uris
+
+echo "Type: deb
+Dist: stable
+" > $SOURCES
+
+testequalwithmsg "Invalid deb822 sources.list file gives proper error" "E: Malformed stanza 0 in source list $TMPWORKINGDIRECTORY/rootdir/etc/apt/sources.list (URI parse)
+E: The list of sources could not be read."  aptget update --print-uris

test/libapt/makefile

@@ -111,3 +111,9 @@ SLIBS = -lapt-pkg
 SOURCE = tagfile_test.cc
 include $(PROGRAM_H)
 
+# test sourcelist
+PROGRAM = SourceList${BASENAME}
+SLIBS = -lapt-pkg
+SOURCE = sourcelist_test.cc
+include $(PROGRAM_H)
+

test/libapt/sourcelist_test.cc

@@ -0,0 +1,52 @@
+#include <apt-pkg/sourcelist.h>
+#include <apt-pkg/tagfile.h>
+
+#include "assert.h"
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+char *tempfile = NULL;
+int tempfile_fd = -1;
+
+void remove_tmpfile(void)
+{
+   if (tempfile_fd > 0)
+      close(tempfile_fd);
+   if (tempfile != NULL) {
+      unlink(tempfile);
+      free(tempfile);
+   }
+}
+
+int main(int argc, char *argv[])
+{
+   const char contents[] = ""
+      "Type: deb\n"
+      "URL: http://ftp.debian.org/debian\n"
+      "Dist: stable\n"
+      "Section: main\n"
+      "Comment: Some random string\n"
+      " that can be very long\n"
+      "\n"
+      "Type: deb\n"
+      "URL: http://ftp.debian.org/debian\n"
+      "Dist: unstable\n"
+      "Section: main non-free\n"
+      ;
+
+   FileFd fd;
+   tempfile = strdup("apt-test.XXXXXXXX");
+   tempfile_fd = mkstemp(tempfile);
+
+   /* (Re-)Open (as FileFd), write and seek to start of the temp file */
+   equals(fd.OpenDescriptor(tempfile_fd, FileFd::ReadWrite), true);
+   equals(fd.Write(contents, strlen(contents)), true);
+   equals(fd.Seek(0), true);
+
+   pkgSourceList sources(tempfile);
+   equals(sources.size(), 2);
+
+   /* clean up handled by atexit handler, so just return here */
+   return 0;
+}