parse and retrieve multiple Descriptions in one record

It seems unlikely for now that proper archives will carry multiple
Description-* stanzas in the Packages (or Translation-*) file, but
sometimes apt eats its own output as shown by the usage of the CD team
and it would be interesting to let apt output multiple translations
e.g. in 'apt-cache show'.

apt-pkg/deb/deblistparser.cc

@@ -194,35 +194,31 @@ bool debListParser::NewVersion(pkgCache::VerIterator &Ver)
 /* This is to return the string describing the package in debian
    form. If this returns the blank string then the entry is assumed to
    only describe package properties */
-string debListParser::Description()
+string debListParser::Description(std::string const &lang)
 {
-   string const lang = DescriptionLanguage();
    if (lang.empty())
       return Section.FindS("Description");
    else
       return Section.FindS(string("Description-").append(lang).c_str());
 }
-                                                                        /*}}}*/
-// ListParser::DescriptionLanguage - Return the description lang string	/*{{{*/
-// ---------------------------------------------------------------------
-/* This is to return the string describing the language of
-   description. If this returns the blank string then the entry is
-   assumed to describe original description. */
-string debListParser::DescriptionLanguage()
+									/*}}}*/
+// ListParser::AvailableDescriptionLanguages				/*{{{*/
+std::vector<std::string> debListParser::AvailableDescriptionLanguages()
 {
-   if (Section.FindS("Description").empty() == false)
-      return "";
-
-   std::vector<string> const lang = APT::Configuration::getLanguages(true);
-   for (std::vector<string>::const_iterator l = lang.begin();
-	l != lang.end(); ++l)
-      if (Section.FindS(string("Description-").append(*l).c_str()).empty() == false)
-	 return *l;
-
-   return "";
+   std::vector<std::string> const understood = APT::Configuration::getLanguages();
+   std::vector<std::string> avail;
+   if (Section.Exists("Description") == true)
+      avail.push_back("");
+   for (std::vector<std::string>::const_iterator lang = understood.begin(); lang != understood.end(); ++lang)
+   {
+      std::string const tagname = "Description-" + *lang;
+      if (Section.Exists(tagname.c_str()) == true)
+	 avail.push_back(*lang);
+   }
+   return avail;
 }
-                                                                        /*}}}*/
-// ListParser::Description - Return the description_md5 MD5SumValue	/*{{{*/
+									/*}}}*/
+// ListParser::Description_md5 - Return the description_md5 MD5SumValue	/*{{{*/
 // ---------------------------------------------------------------------
 /* This is to return the md5 string to allow the check if it is the right
    description. If no Description-md5 is found in the section it will be
@@ -233,7 +229,7 @@ MD5SumValue debListParser::Description_md5()
    string const value = Section.FindS("Description-md5");
    if (value.empty() == true)
    {
-      std::string const desc = Description() + "\n";
+      std::string const desc = Description("") + "\n";
       if (desc == "\n")
 	 return MD5SumValue();
 

apt-pkg/deb/deblistparser.h

@@ -68,8 +68,8 @@ class debListParser : public pkgCacheGenerator::ListParser
    virtual bool ArchitectureAll();
    virtual std::string Version();
    virtual bool NewVersion(pkgCache::VerIterator &Ver);
-   virtual std::string Description();
-   virtual std::string DescriptionLanguage();
+   virtual std::string Description(std::string const &lang);
+   virtual std::vector<std::string> AvailableDescriptionLanguages();
    virtual MD5SumValue Description_md5();
    virtual unsigned short VersionHash();
 #if (APT_PKG_MAJOR >= 4 && APT_PKG_MINOR >= 13)

apt-pkg/deb/debrecords.cc

@@ -106,10 +106,12 @@ string debRecordParser::RecordField(const char *fieldName)
 // RecordParser::ShortDesc - Return a 1 line description		/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-string debRecordParser::ShortDesc()
+string debRecordParser::ShortDesc(std::string const &lang)
 {
-   string Res = LongDesc();
-   string::size_type Pos = Res.find('\n');
+   string const Res = LongDesc(lang);
+   if (Res.empty() == true)
+      return "";
+   string::size_type const Pos = Res.find('\n');
    if (Pos == string::npos)
       return Res;
    return string(Res,0,Pos);
@@ -118,26 +120,44 @@ string debRecordParser::ShortDesc()
 // RecordParser::LongDesc - Return a longer description			/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-string debRecordParser::LongDesc()
+string debRecordParser::LongDesc(std::string const &lang)
 {
-  string orig, dest;
+   string orig;
+   if (lang.empty() == true)
+   {
+      std::vector<string> const lang = APT::Configuration::getLanguages();
+      for (std::vector<string>::const_iterator l = lang.begin();
+	    l != lang.end(); ++l)
+      {
+	 std::string const tagname = "Description-" + *l;
+	 orig = Section.FindS(tagname.c_str());
+	 if (orig.empty() == false)
+	    break;
+	 else if (*l == "en")
+	 {
+	    orig = Section.FindS("Description");
+	    if (orig.empty() == false)
+	       break;
+	 }
+      }
+      if (orig.empty() == true)
+	 orig = Section.FindS("Description");
+   }
+   else
+   {
+      std::string const tagname = "Description-" + lang;
+      orig = Section.FindS(tagname.c_str());
+      if (orig.empty() == true && lang == "en")
+	 orig = Section.FindS("Description");
+   }
 
-  if (!Section.FindS("Description").empty())
-     orig = Section.FindS("Description").c_str();
-  else
-  {
-     std::vector<string> const lang = APT::Configuration::getLanguages();
-     for (std::vector<string>::const_iterator l = lang.begin();
-	  orig.empty() && l != lang.end(); ++l)
-	orig = Section.FindS(string("Description-").append(*l).c_str());
-  }
+   char const * const codeset = nl_langinfo(CODESET);
+   if (strcmp(codeset,"UTF-8") != 0) {
+      string dest;
+      UTF8ToCodeset(codeset, orig, &dest);
+      return dest;
+   }
 
-  char const * const codeset = nl_langinfo(CODESET);
-  if (strcmp(codeset,"UTF-8") != 0) {
-     UTF8ToCodeset(codeset, orig, &dest);
-     orig = dest;
-   }    
-  
    return orig;
 }
 									/*}}}*/

apt-pkg/deb/debrecords.h

@@ -49,8 +49,8 @@ class debRecordParser : public pkgRecords::Parser
 
    // These are some general stats about the package
    virtual std::string Maintainer();
-   virtual std::string ShortDesc();
-   virtual std::string LongDesc();
+   virtual std::string ShortDesc(std::string const &lang);
+   virtual std::string LongDesc(std::string const &lang);
    virtual std::string Name();
    virtual std::string Homepage();
 

apt-pkg/pkgcachegen.cc

@@ -302,10 +302,9 @@ bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator
 
    // Find the right version to write the description
    MD5SumValue CurMd5 = List.Description_md5();
-   if (CurMd5.Value().empty() == true || List.Description().empty() == true)
+   if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
       return true;
-   std::string CurLang = List.DescriptionLanguage();
-
+   std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
    {
       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
@@ -314,31 +313,16 @@ bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator
       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 	 continue;
 
-      // don't add a new description if we have one for the given
-      // md5 && language
-      if (IsDuplicateDescription(VerDesc, CurMd5, CurLang) == true)
-	 continue;
-
-      pkgCache::DescIterator Desc;
-      Dynamic<pkgCache::DescIterator> DynDesc(Desc);
-
-      map_ptrloc const descindex = NewDescription(Desc, CurLang, CurMd5, VerDesc->md5sum);
-      if (unlikely(descindex == 0 && _error->PendingError()))
-	 return _error->Error(_("Error occurred while processing %s (%s%d)"),
-			      Pkg.Name(), "NewDescription", 1);
-
-      Desc->ParentPkg = Pkg.Index();
-
-      // we add at the end, so that the start is constant as we need
-      // that to be able to efficiently share these lists
-      VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
-      for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
-      map_ptrloc * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
-      *LastNextDesc = descindex;
+      map_ptrloc md5idx = VerDesc->md5sum;
+      for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
+      {
+	 // don't add a new description if we have one for the given
+	 // md5 && language
+	 if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
+	    continue;
 
-      if (NewFileDesc(Desc,List) == false)
-	 return _error->Error(_("Error occurred while processing %s (%s%d)"),
-			      Pkg.Name(), "NewFileDesc", 1);
+	 AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
+      }
 
       // we can stop here as all "same" versions will share the description
       break;
@@ -486,11 +470,10 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
       return true;
    }
 
-   /* Record the Description (it is not translated) */
+   /* Record the Description(s) based on their master md5sum */
    MD5SumValue CurMd5 = List.Description_md5();
-   if (CurMd5.Value().empty() == true || List.Description().empty() == true)
+   if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
       return true;
-   std::string CurLang = List.DescriptionLanguage();
 
    /* Before we add a new description we first search in the group for
       a version with a description of the same MD5 - if so we reuse this
@@ -501,28 +484,44 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
       for (pkgCache::VerIterator V = P.VersionList();
 	   V.end() == false; ++V)
       {
-	 if (IsDuplicateDescription(V.DescriptionList(), CurMd5, "") == false)
+	 if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 	    continue;
 	 Ver->DescriptionList = V->DescriptionList;
-	 return true;
       }
    }
 
-   // We haven't found reusable descriptions, so add the first description
-   pkgCache::DescIterator Desc = Ver.DescriptionList();
+   // We haven't found reusable descriptions, so add the first description(s)
+   map_ptrloc md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
+   std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
+   for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
+      if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
+	 return false;
+   return true;
+}
+									/*}}}*/
+bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_ptrloc &md5idx) /*{{{*/
+{
+   pkgCache::DescIterator Desc;
    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 
-   map_ptrloc const descindex = NewDescription(Desc, CurLang, CurMd5, 0);
+   map_ptrloc const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
    if (unlikely(descindex == 0 && _error->PendingError()))
       return _error->Error(_("Error occurred while processing %s (%s%d)"),
-			   Pkg.Name(), "NewDescription", 2);
+	    Ver.ParentPkg().Name(), "NewDescription", 1);
+
+   md5idx = Desc->md5sum;
+   Desc->ParentPkg = Ver.ParentPkg().Index();
 
-   Desc->ParentPkg = Pkg.Index();
-   Ver->DescriptionList = descindex;
+   // we add at the end, so that the start is constant as we need
+   // that to be able to efficiently share these lists
+   pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
+   for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
+   map_ptrloc * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
+   *LastNextDesc = descindex;
 
    if (NewFileDesc(Desc,List) == false)
       return _error->Error(_("Error occurred while processing %s (%s%d)"),
-			   Pkg.Name(), "NewFileDesc", 2);
+	    Ver.ParentPkg().Name(), "NewFileDesc", 1);
 
    return true;
 }

apt-pkg/pkgcachegen.h

@@ -125,6 +125,9 @@ class pkgCacheGenerator							/*{{{*/
    APT_HIDDEN bool AddImplicitDepends(pkgCache::GrpIterator &G, pkgCache::PkgIterator &P,
 			   pkgCache::VerIterator &V);
    APT_HIDDEN bool AddImplicitDepends(pkgCache::VerIterator &V, pkgCache::PkgIterator &D);
+
+   APT_HIDDEN bool AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver,
+	 std::string const &lang, MD5SumValue const &CurMd5, map_ptrloc &md5idx);
 };
 									/*}}}*/
 // This is the abstract package list parser class.			/*{{{*/
@@ -160,8 +163,8 @@ class pkgCacheGenerator::ListParser
    virtual bool ArchitectureAll() = 0;
    virtual std::string Version() = 0;
    virtual bool NewVersion(pkgCache::VerIterator &Ver) = 0;
-   virtual std::string Description() = 0;
-   virtual std::string DescriptionLanguage() = 0;
+   virtual std::string Description(std::string const &lang) = 0;
+   virtual std::vector<std::string> AvailableDescriptionLanguages() = 0;
    virtual MD5SumValue Description_md5() = 0;
    virtual unsigned short VersionHash() = 0;
    /** compare currently parsed version with given version

apt-pkg/pkgrecords.h

@@ -75,8 +75,22 @@ class pkgRecords::Parser						/*{{{*/
 
    // These are some general stats about the package
    virtual std::string Maintainer() {return std::string();};
-   virtual std::string ShortDesc() {return std::string();};
-   virtual std::string LongDesc() {return std::string();};
+   /** return short description in language from record.
+    *
+    * @see #LongDesc
+    */
+   virtual std::string ShortDesc(std::string const &/*lang*/) {return std::string();};
+   /** return long description in language from record.
+    *
+    * If \b lang is empty the "best" available language will be
+    * returned as determined by the APT::Languages configuration.
+    * If a (requested) language can't be found in this record an empty
+    * string will be returned.
+    */
+   virtual std::string LongDesc(std::string const &/*lang*/) {return std::string();};
+   std::string ShortDesc() {return ShortDesc("");};
+   std::string LongDesc() {return LongDesc("");};
+
    virtual std::string Name() {return std::string();};
    virtual std::string Homepage() {return std::string();}
 

test/integration/test-bug-712435-missing-descriptions

@@ -87,13 +87,10 @@ $DESCRIPTION
 Description-md5: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
 " aptcache show apt-normal
 
-# displaying the translated Description would be equally valid,
-# but we assume only one description is in a Packages file and
-# so we prefer "Description" over "Description-*" currently.
 for variant in 'below' 'middle' 'top'; do
 	testequal "Package: apt-both-$variant
 $PACKAGESTANZA
-$DESCRIPTION
+$TRANSDESCRIPTION
 Description-md5: bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
 " aptcache show apt-both-$variant
 done
@@ -122,7 +119,7 @@ X-Some-Flag: yes
 
 testequal "Package: apt-intermixed2
 $PACKAGESTANZA
-$DESCRIPTION
+$TRANSDESCRIPTION
 Description-md5: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
 X-Some-Flag: yes
 X-Foo-Flag: Something with a Description
@@ -131,7 +128,7 @@ X-Bar-Flag: no
 
 testequal "Package: apt-intermixed3
 $PACKAGESTANZA
-$DESCRIPTION
+$TRANSDESCRIPTION
 Description-md5: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
 X-Some-Flag: yes
 X-Foo-Flag: Something with a Description