sync TFRewrite*Order arrays with dpkg and dak

dpkg and dak know various field names and order them in their output,
while we have yet another order and have to play catch up with them as
we are sitting between chairs here and neither order is ideal for us,
too.

A little testcase is from now on supposed to help ensureing that we do
not derivate to far away from which fields dpkg knows and orders.

apt-pkg/tagfile-order.c

@@ -0,0 +1,109 @@
+/* In this file is the order defined in which e.g. apt-ftparchive will write stanzas in.
+   Other commands might or might not use this. 'apt-cache show' e.g. does NOT!
+
+   The order we chose here is inspired by both dpkg and dak.
+   The testcase test/integration/test-apt-tagfile-fields-order intends to ensure that
+   this file isn't lacking (too far) behind dpkg over time. */
+
+static const char *iTFRewritePackageOrder[] = {
+   "Package",
+   "Package-Type",
+   "Architecture",
+   "Subarchitecture", // Used only by d-i
+   "Version",
+   "Revision",         // Obsolete (warning in dpkg)
+   "Package-Revision", // Obsolete (warning in dpkg)
+   "Package_Revision", // Obsolete (warning in dpkg)
+   "Kernel-Version", // Used only by d-i
+   "Built-Using",
+   "Built-For-Profiles",
+   "Multi-Arch",
+   "Status",
+   "Priority",
+   "Class", // dpkg nickname for Priority
+   "Essential",
+   "Installer-Menu-Item", // Used only by d-i
+   "Section",
+   "Source",
+   "Origin",
+   "Maintainer",
+   "Original-Maintainer", // unknown in dpkg order
+   "Bugs",
+   "Config-Version", // Internal of dpkg
+   "Conffiles",
+   "Triggers-Awaited",
+   "Triggers-Pending",
+   "Installed-Size",
+   "Provides",
+   "Pre-Depends",
+   "Depends",
+   "Recommends",
+   "Recommended", // dpkg nickname for Recommends
+   "Suggests",
+   "Optional", // dpkg nickname for Suggests
+   "Conflicts",
+   "Breaks",
+   "Replaces",
+   "Enhances",
+   "Filename",
+   "MSDOS-Filename", // Obsolete (used by dselect)
+   "Size",
+   "MD5sum",
+   "SHA1",
+   "SHA256",
+   "SHA512",
+   "Homepage",
+   "Description",
+   "Tag",
+   "Task",
+   0
+};
+static const char *iTFRewriteSourceOrder[] = {
+   "Package",
+   "Source", // dsc file, renamed to Package in Sources
+   "Format",
+   "Binary",
+   "Architecture",
+   "Version",
+   "Priority",
+   "Class", // dpkg nickname for Priority
+   "Section",
+   "Origin",
+   "Maintainer",
+   "Original-Maintainer", // unknown in dpkg order
+   "Uploaders",
+   "Dm-Upload-Allowed", // Obsolete (ignored by dak)
+   "Standards-Version",
+   "Build-Depends",
+   "Build-Depends-Arch",
+   "Build-Depends-Indep",
+   "Build-Conflicts",
+   "Build-Conflicts-Arch",
+   "Build-Conflicts-Indep",
+   "Testsuite",
+   "Homepage",
+   "Vcs-Browser",
+   "Vcs-Browse", // dak only (nickname?)
+   "Vcs-Arch",
+   "Vcs-Bzr",
+   "Vcs-Cvs",
+   "Vcs-Darcs",
+   "Vcs-Git",
+   "Vcs-Hg",
+   "Vcs-Mtn",
+   "Vcs-Svn",
+   "Directory",
+   "Package-List",
+   "Files",
+   "Checksums-Md5",
+   "Checksums-Sha1",
+   "Checksums-Sha256",
+   "Checksums-Sha512",
+   0
+};
+
+/* Two levels of initialization are used because gcc will set the symbol
+   size of an array to the length of the array, causing dynamic relinking
+   errors. Doing this makes the symbol size constant */
+const char **TFRewritePackageOrder = iTFRewritePackageOrder;
+const char **TFRewriteSourceOrder = iTFRewriteSourceOrder;

apt-pkg/tagfile.cc

@@ -623,71 +623,14 @@ APT_PURE unsigned int pkgTagSection::Count() const {			/*{{{*/
    return d->Tags.size() - 1;
 }
 									/*}}}*/
+
+#include "tagfile-order.c"
+
 // TFRewrite - Rewrite a control record					/*{{{*/
 // ---------------------------------------------------------------------
 /* This writes the control record to stdout rewriting it as necessary. The
    override map item specificies the rewriting rules to follow. This also
    takes the time to sort the feild list. */
-
-/* The order of this list is taken from dpkg source lib/parse.c the fieldinfos
-   array. */
-static const char *iTFRewritePackageOrder[] = {
-                          "Package",
-                          "Essential",
-                          "Status",
-                          "Priority",
-                          "Section",
-                          "Installed-Size",
-                          "Maintainer",
-                          "Original-Maintainer",
-                          "Architecture",
-                          "Source",
-                          "Version",
-                           "Revision",         // Obsolete
-                           "Config-Version",   // Obsolete
-                          "Replaces",
-                          "Provides",
-                          "Depends",
-                          "Pre-Depends",
-                          "Recommends",
-                          "Suggests",
-                          "Conflicts",
-                          "Breaks",
-                          "Conffiles",
-                          "Filename",
-                          "Size",
-                          "MD5sum",
-                          "SHA1",
-                          "SHA256",
-                          "SHA512",
-                           "MSDOS-Filename",   // Obsolete
-                          "Description",
-                          0};
-static const char *iTFRewriteSourceOrder[] = {"Package",
-                                      "Source",
-                                      "Binary",
-                                      "Version",
-                                      "Priority",
-                                      "Section",
-                                      "Maintainer",
-				      "Original-Maintainer",
-                                      "Build-Depends",
-                                      "Build-Depends-Indep",
-                                      "Build-Conflicts",
-                                      "Build-Conflicts-Indep",
-                                      "Architecture",
-                                      "Standards-Version",
-                                      "Format",
-                                      "Directory",
-                                      "Files",
-                                      0};   
-
-/* Two levels of initialization are used because gcc will set the symbol
-   size of an array to the length of the array, causing dynamic relinking 
-   errors. Doing this makes the symbol size constant */
-const char **TFRewritePackageOrder = iTFRewritePackageOrder;
-const char **TFRewriteSourceOrder = iTFRewriteSourceOrder;
-   
 bool TFRewrite(FILE *Output,pkgTagSection const &Tags,const char *Order[],
 	       TFRewriteData *Rewrite)
 {

debian/tests/run-tests

@@ -11,6 +11,7 @@ make -C test/interactive-helper/
 # run tests against the installed apt, use "env -i" to ensure
 # the host env does not pollute our environment
 env -i \
+APT_INTEGRATION_TESTS_SOURCE_DIR=$(pwd) \
 APT_INTEGRATION_TESTS_WEBSERVER_BIN_DIR=$(pwd)/build/bin \
 APT_INTEGRATION_TESTS_METHODS_DIR=/usr/lib/apt/methods \
 APT_INTEGRATION_TESTS_LIBEXEC_DIR=/usr/lib/apt/ \

test/integration/framework

@@ -189,6 +189,7 @@ setupenvironment() {
 
 	TESTDIRECTORY=$(readlink -f $(dirname $0))
         # allow overriding the default BUILDDIR location
+	SOURCEDIRECTORY=${APT_INTEGRATION_TESTS_SOURCE_DIR:-"${TESTDIRECTORY}/../../"}
 	BUILDDIRECTORY=${APT_INTEGRATION_TESTS_BUILD_DIR:-"${TESTDIRECTORY}/../../build/bin"}
 	LIBRARYPATH=${APT_INTEGRATION_TESTS_LIBRARY_PATH:-"${BUILDDIRECTORY}"}
         METHODSDIR=${APT_INTEGRATION_TESTS_METHODS_DIR:-"${BUILDDIRECTORY}/methods"}

test/integration/test-apt-ftparchive-cachedb

@@ -3,12 +3,12 @@ set -e
 
 ensure_correct_packages_file() {
     testequal "Package: foo
+Architecture: i386
+Version: 1
 Priority: optional
 Section: others
-$(dpkg-deb -I ./aptarchive/pool/main/foo_1_i386.deb | grep 'Installed-Size:' | sed 's#^ ##')
 Maintainer: Joe Sixpack <joe@example.org>
-Architecture: i386
-Version: 1
+$(dpkg-deb -I ./aptarchive/pool/main/foo_1_i386.deb | grep 'Installed-Size:' | sed 's#^ ##')
 Filename: pool/main/foo_1_i386.deb" head -n8 ./aptarchive/dists/test/main/binary-i386/Packages
 }
 

test/integration/test-apt-ftparchive-cachedb-lp1274466

@@ -27,12 +27,12 @@ testfailure grep 7da58ff901a40ecf42a730dc33198b182e9ba9ec98799fc2c2b6fabeeee40cc
 
 # regression test for corruption with previous generation of cachedb
 testsuccessequal "Package: foo
+Architecture: i386
+Version: 1
 Priority: optional
 Section: others
-Installed-Size: 29
 Maintainer: Joe Sixpack <joe@example.org>
-Architecture: i386
-Version: 1
+Installed-Size: 29
 Filename: ./foo_1_i386.deb
 Size: 1270
 MD5sum: 85d0e908c1a897700e2c5dea72d7e3c0

test/integration/test-apt-ftparchive-src-cachedb

@@ -3,16 +3,16 @@ set -e
 
 assert_correct_sources_file() {
     testsuccessequal "Package: bar
+Format: 3.0 (native)
+Binary: bar
 Architecture: all
 Version: 1.0
-Binary: bar
-Format: 3.0 (native)
 Directory: pool/main
+Package-List:
+ bar deb admin extra
 Files:
  7b57dd065e51de5905288a5104d4bef5 406 bar_1.0.dsc
  d41d8cd98f00b204e9800998ecf8427e 0 bar_1.0.tar.gz
-Package-List:
- bar deb admin extra
 Checksums-Sha1:
  17a40b76715f393ab7fd6485c9392a02f1adf903 406 bar_1.0.dsc
  da39a3ee5e6b4b0d3255bfef95601890afd80709 0 bar_1.0.tar.gz
@@ -24,16 +24,16 @@ Checksums-Sha512:
  cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e 0 bar_1.0.tar.gz
 
 Package: foo
+Format: 3.0 (native)
+Binary: foo
 Architecture: all
 Version: 1.0
-Binary: foo
-Format: 3.0 (native)
 Directory: pool/main
+Package-List: 
+ foo deb admin extra
 Files:
  d144826e6f02831c1933e910c92cd7e0 171 foo_1.0.dsc
  d41d8cd98f00b204e9800998ecf8427e 0 foo_1.0.tar.gz
-Package-List: 
- foo deb admin extra
 Checksums-Sha1:
  979306aa3ccff3d61bba062bb6977e2493c6f907 171 foo_1.0.dsc
  da39a3ee5e6b4b0d3255bfef95601890afd80709 0 foo_1.0.tar.gz
@@ -43,7 +43,7 @@ Checksums-Sha256:
 Checksums-Sha512:
  3da0240fd764657c2f3661b4d750578a9a99b0580591b133756379d48117ebda87a5ed2467f513200d6e7eaf51422cbe91c15720eef7fb4bba2cc8ff81ebc547 171 foo_1.0.dsc
  cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e 0 foo_1.0.tar.gz
-" aptsortpkgs ./aptarchive/dists/test/main/source/Sources
+" aptsortpkgs ./aptarchive/dists/test/main/source/Sources -o APT::SortPkgs::Source=true
 }
 
 create_source_files() {

test/integration/test-apt-tagfile-fields-order

@@ -0,0 +1,82 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+setupenvironment
+
+dpkg_field_ordered_list() {
+	local FIELDS="$(perl -e "
+use Dpkg::Control;
+use Dpkg::Control::Fields;
+foreach \$f (field_ordered_list(${1})) {
+	print \"\$f\\n\";
+}" | sort -u)"
+	if [ -z "$FIELDS" ]; then
+		msgfail 'Could not get fields via libdpkg-perl'
+	fi
+	echo "$FIELDS"
+}
+
+comparelsts() {
+	local DIFFOUTPUT="$(diff -u apt.lst dpkg.lst || true)"
+	if echo "$DIFFOUTPUT" | grep -q '^+[^+]'; then
+		echo
+		echo "$DIFFOUTPUT" | grep '^[+-][^+-]'
+		msgfail
+	else
+		msgpass
+	fi
+}
+
+msgtest 'Check that apt knows all fields dpkg orders in' 'Packages'
+dpkg_field_ordered_list 'CTRL_INDEX_PKG' > dpkg.lst
+sed -ne 's#^   "\(.*\)",.*$#\1#p' ${SOURCEDIRECTORY}/apt-pkg/tagfile-order.c | sed -n '/^Package$/,/^Package$/ p' | head -n -1 | sort -u > apt.lst
+comparelsts
+
+msgtest 'Check that apt knows all fields dpkg orders in' 'status'
+dpkg_field_ordered_list 'CTRL_FILE_STATUS' > dpkg.lst
+comparelsts
+
+msgtest 'Check that apt knows all fields dpkg orders in' 'DEBIAN/control'
+dpkg_field_ordered_list 'CTRL_PKG_DEB' > dpkg.lst
+comparelsts
+
+msgtest 'Check that apt knows all fields dpkg orders in' 'Sources'
+dpkg_field_ordered_list 'CTRL_INDEX_SRC' > dpkg.lst
+echo 'Package' > apt.tmp
+sed -ne 's#^   "\(.*\)",.*$#\1#p' ${SOURCEDIRECTORY}/apt-pkg/tagfile-order.c | sed '/^Package$/,/^Package$/ d' >> apt.tmp
+sort -u apt.tmp > apt.lst
+comparelsts
+
+msgtest 'Check that apt knows all fields dpkg orders in' 'dsc'
+dpkg_field_ordered_list 'CTRL_PKG_SRC' > dpkg.lst
+comparelsts
+
+# HACK, but there is no good way to acquire sources in tests and/or to remember to run this regular manually
+if [ "$USER" = 'david' ]; then
+	msgtest 'Check if we have somewhere the sources of' 'dpkg'
+	DPKGSOURCE="$(locate dpkg/lib/dpkg/parse.c  | head -n 1 || true)"
+	if [ -z "$DPKGSOURCE" ]; then
+		msgskip 'Not found'
+	else
+		msgpass
+		msgtest 'Check that apt knows about all fields' 'dpkg parses'
+		sed -n 's#^.*FIELD("\(.*\)").*$#\1#p' "${DPKGSOURCE}" | sort -u > dpkg.lst
+		sed -ne 's#^   "\(.*\)",.*$#\1#p' ${SOURCEDIRECTORY}/apt-pkg/tagfile-order.c | sed -n '/^Package$/,/^Package$/ p' | head -n -1 | sort -u > apt.lst
+		comparelsts
+	fi
+
+	msgtest 'Check if we have somewhere the sources of' 'dak'
+	DAKSOURCE="$(locate dak/setup/core-init.d/080_metadatakeys | head -n 1 || true)"
+	if [ -z "$DAKSOURCE" ]; then
+		msgskip 'Not found'
+	else
+		msgpass
+		msgtest 'Check that apt knows about all fields' 'dak knows'
+		# dak mixes both, so we can only check with the mixed one as well
+		sed -ne "s#^.* VALUES ('\(.*\)', \(.*\)).*\$#\1 \2#p" "${DAKSOURCE}" | cut -d ' ' -f 1 | sort -u > dpkg.lst
+		sed -ne 's#^   "\(.*\)",.*$#\1#p' ${SOURCEDIRECTORY}/apt-pkg/tagfile-order.c | sort -u > apt.lst
+		comparelsts
+	fi
+fi