* Add back some parts of the cron job (100syncarchive_*)

* Use these files from bin/parse-*
* Support different archives

diff --git a/bin/daily b/bin/daily
new file mode 100755 (executable)
index 0000000..0d5afb4
--- /dev/null
+++ b/bin/daily
@@ -0,0 +1,45 @@
+#! /bin/bash
+
+#   Copyright (c) 2004  Martin Schulze <joey@debian.org>
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; either version 2 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program; if not, write to the Free Software
+#   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+# Should run after the mirror pulse.  The mirror tends to finish by
+# the ctime of rsync.log.* in ~archvsync.
+
+topdir=/org/packages.debian.org
+files=$topdir/files
+logs=$files/logs
+log=$logs/cron.log
+
+test -d $files || mkdir -p $files
+test -d $logs || mkdir -p $logs
+
+if [ -s $log ]
+then
+    savelog -c 14 $log > /dev/null
+fi
+
+date > $log
+
+if lockfile -600 -r5 "${files}/cron.lock" >> $log 2>&1
+then
+    run-parts --verbose $topdir/cron.d >> $log 2>&1
+else
+    echo "couldn't aquire lock file in due time" >> $log
+fi
+rm -f "${files}/cron.lock"
+
+date >> $log

diff --git a/bin/parse-packages b/bin/parse-packages
index 8b132abee525dab852b3c4af3d233ed52b3426b3..7a67e691a9dd0a9e95abc61dcbd8a7ed2a034e0e 100755 (executable)
--- a/bin/parse-packages
+++ b/bin/parse-packages
@@ -35,18 +35,20 @@ my @descriptions = ("we count lines one-based\000");
 my %packages_descriptions = ();
 my %descriptions_packages = ();
 
+my @archives = ( 'us', 'non-US', 'security', 'volatile', 'backports' );
 my @suites = ('oldstable', 'stable', 'testing', 'unstable', 'experimental');
 
 $/ = "";
 
-for my $suite (@suites) {
+for my $archive (@archives) {
+    for my $suite (@suites) {
 
-       print "Reading $suite...\n";
+       print "Reading $archive/$suite...\n";
        my %packages_all_db;
        tie %packages_all_db, "DB_File", "packages_all_$suite.db.new",
                O_RDWR|O_CREAT, 0666, $DB_BTREE
                or die "Error creating DB: $!";
-       open PKG, "zcat /org/ftp.debian.org/ftp/dists/$suite/$what/{,debian-installer/}binary-*/Packages.gz|";
+       open PKG, "zcat /org/packages.debian.org/archive/$archive/$suite/$what/{,debian-installer/}binary-*/Packages.gz|";
        while (<PKG>) {
                next if /^\s*$/;
                my $data = "";
@@ -86,19 +88,22 @@ for my $suite (@suites) {
                my $sdescr = $descr;
                $sdescr =~ s/\n.*//s;
                my $section = 'main';
-               my $subsection = $data{section};
+               my $subsection = $data{section} || '-';
                if ($data{section} && ($data{section} =~ m=/=o)) {
                    ($section, $subsection) = split m=/=o, $data{section}, 2;
                }
                $data{'section'} = $section;
                $data{'subsection'} = $subsection;
-               $packages_small{$data{'package'}} .= "$suite $data{'architecture'} ".
+               $data{'priority'} ||= '-';
+               $packages_small{$data{'package'}} .= "$archive $suite $data{'architecture'} ".
                        "$section $subsection $data{'priority'} $data{'version'} $sdescr\000";
                $sources_packages{$src} .=
-                       "$suite $data{'package'} $data{'version'} $data{'architecture'}\000";
+                       "$archive $suite $data{'package'} $data{'version'} $data{'architecture'}\000";
+               $data{archive} = $archive;
                while (my ($key, $value) = each (%data)) {
                        next if $key eq 'package' or $key eq 'version' or $key eq 'architecture'
                                or $key eq 'maintainer';
+                       print STDERR "WARN: $key ($suite/$archive/$data{package}/$data{architecture}\n" unless defined $value;
                        $data .= "$key: $value\n";
                }
                $packages_all_db{"$data{'package'} $data{'architecture'} $data{'version'}"}
@@ -106,6 +111,7 @@ for my $suite (@suites) {
        }
 
        untie %packages_all_db;
+    }
 }
 
 print "Writing databases...\n";

diff --git a/bin/parse-sources b/bin/parse-sources
index 63eda09c28878304aa56ec937a66b4bf99007c26..82716b3c35216d54728cf03fa423377ef546786b 100755 (executable)
--- a/bin/parse-sources
+++ b/bin/parse-sources
@@ -30,18 +30,20 @@ my %sources_small = ();
 my %source_names = ();
 my %source_postfixes = ();
 
+my @archives = ( 'us', 'non-US', 'security', 'volatile', 'backports' );
 my @suites = ('oldstable', 'stable', 'testing', 'unstable', 'experimental');
 
 $/ = "";
 
-for my $suite (@suites) {
+for my $archive (@archives) {
+    for my $suite (@suites) {
 
-       print "Reading $suite...\n";
+       print "Reading $archive/$suite...\n";
        my %sources_all_db;
        tie %sources_all_db, "DB_File", "sources_all_$suite.db.new",
                O_RDWR|O_CREAT, 0666, $DB_BTREE
                or die "Error creating DB: $!";
-       open PKG, "zcat /org/ftp.debian.org/ftp/dists/$suite/$what/source/Sources.gz|";
+       open PKG, "zcat /org/packages.debian.org/archive/$archive/$suite/$what/source/Sources.gz|";
        while (<PKG>) {
                next if /^\s*$/;
                my $data = "";
@@ -55,22 +57,24 @@ for my $suite (@suites) {
                        $key =~ tr [A-Z] [a-z];
                        $data{$key} = $value;
                }
+               $data .= "Archive: $archive\n";
                $sources_all_db{"$data{'package'} $data{'version'}"}
                        = $data;
 
                $source_names{$data{'package'}} = 1;
 
                my $section = 'main';
-               my $subsection = $data{section};
+               my $subsection = $data{section} || '-';
                if ($data{section} && ($data{section} =~ m=/=o)) {
                    ($section, $subsection) = split m=/=o, $data{section}, 2;
                }
                $data{'priority'} = "-" if not exists($data{'priority'});
                $sources_small{$data{'package'}} .=
-                       "$suite $section $subsection $data{'priority'} $data{'version'}\000";
+                       "$archive $suite $section $subsection $data{'priority'} $data{'version'}\000";
        }
 
        untie %sources_all_db;
+    }
 }
 
 print "Writing databases...\n";

diff --git a/cgi-bin/search_packages.pl b/cgi-bin/search_packages.pl
index 25b0e132ea0874a862b3dee434442982b55155a8..48fabb4c35f3ebac80cdecff2f5e25d2ab7ed241 100755 (executable)
--- a/cgi-bin/search_packages.pl
+++ b/cgi-bin/search_packages.pl
@@ -48,19 +48,16 @@ $Packages::CGI::debug = $debug;
 # read the configuration
 our $config_read_time ||= 0;
 our $db_read_time ||= 0;
-our $topdir;
-our $ROOT;
-our @SUITES;
-our @SECTIONS;
-our @ARCHITECTURES;
+our ( $topdir, $ROOT, @SUITES, @SECTIONS, @ARCHIVES, @ARCHITECTURES );
 
 # FIXME: move to own module
 my $modtime = (stat( "../config.sh" ))[9];
 if ($modtime > $config_read_time) {
     if (!open (C, '<', "../config.sh")) {
-       error( "Internal Error: Cannot open configuration file." );
+       error( "Internal: Cannot open configuration file." );
     }
     while (<C>) {
+       next if /^\s*\#/o;
        chomp;
        $topdir = $1 if /^\s*topdir="?([^\"]*)"?\s*$/o;
        $ROOT = $1 if /^\s*root="?([^\"]*)"?\s*$/o;
@@ -71,6 +68,7 @@ if ($modtime > $config_read_time) {
        $Packages::HTML::CONTACT_MAIL = $1 if /^\s*contact="?([^\"]*)"?\s*$/o;
        @SUITES = split(/\s+/, $1) if /^\s*suites="?([^\"]*)"?\s*$/o;
        @SECTIONS = split(/\s+/, $1) if /^\s*sections="?([^\"]*)"?\s*$/o;
+       @ARCHIVES = split(/\s+/, $1) if /^\s*archives="?([^\"]*)"?\s*$/o;
        @ARCHITECTURES = split(/\s+/, $1) if /^\s*architectures="?([^\"]*)"?\s*$/o;
     }
     close (C);
@@ -85,6 +83,7 @@ if (my $path = $input->param('path')) {
 
     my %SUITES = map { $_ => 1 } @SUITES;
     my %SECTIONS = map { $_ => 1 } @SECTIONS;
+    my %ARCHIVES = map { $_ => 1 } @ARCHIVES;
     my %ARCHITECTURES = map { $_ => 1 } @ARCHITECTURES;
 
     foreach (@components) {
@@ -92,41 +91,46 @@ if (my $path = $input->param('path')) {
            $input->param('suite', $_);
        } elsif ($SECTIONS{$_}) {
            $input->param('section', $_);
-       }elsif ($ARCHITECTURES{$_}) {
+       } elsif ($ARCHIVES{$_}) {
+           $input->param('archive', $_);
+       } elsif ($ARCHITECTURES{$_}) {
            $input->param('arch', $_);
        }
     }
 }
 
 my ( $format, $keyword, $case, $subword, $exact, $searchon,
-     @suites, @sections, @archs );
-
-my %params_def = ( keywords => { default => undef,
-                                match => '^\s*([-+\@\w\/.:]+)\s*$',
-                                var => \$keyword },
-                  suite => { default => 'stable', match => '^(\w+)$',
-                             alias => 'version', array => ',',
-                             var => \@suites,
-                             replace => { all => \@SUITES } },
-                  case => { default => 'insensitive', match => '^(\w+)$',
-                            var => \$case },
-                  official => { default => 0, match => '^(\w+)$' },
-                  subword => { default => 0, match => '^(\w+)$',
-                               var => \$subword },
-                  exact => { default => undef, match => '^(\w+)$',
-                             var => \$exact },
-                  searchon => { default => 'all', match => '^(\w+)$',
-                                var => \$searchon },
-                  section => { default => 'all', match => '^([\w-]+)$',
-                               alias => 'release', array => ',',
-                               var => \@sections,
-                               replace => { all => \@SECTIONS } },
-                  arch => { default => 'any', match => '^(\w+)$',
-                            array => ',', var => \@archs, replace =>
-                            { any => \@ARCHITECTURES } },
-                  format => { default => 'html', match => '^(\w+)$',
-                               var => \$format },
-                  );
+     @suites, @sections, @archives, @archs );
+
+# my %params_def = ( keywords => { default => undef,
+#                               match => '^\s*([-+\@\w\/.:]+)\s*$',
+#                               var => \$keyword },
+#                 suite => { default => 'stable', match => '^([\w-]+)$',
+#                            alias => 'version', array => ',',
+#                            var => \@suites,
+#                            replace => { all => \@SUITES } },
+#                 archive => { default => 'all', match => '^([\w-]+)$',
+#                              array => ',', var => \@archives,
+#                              replace => { all => \@ARCHIVES } },
+#                 case => { default => 'insensitive', match => '^(\w+)$',
+#                           var => \$case },
+#                 official => { default => 0, match => '^(\w+)$' },
+#                 subword => { default => 0, match => '^(\w+)$',
+#                              var => \$subword },
+#                 exact => { default => undef, match => '^(\w+)$',
+#                            var => \$exact },
+#                 searchon => { default => 'all', match => '^(\w+)$',
+#                               var => \$searchon },
+#                 section => { default => 'all', match => '^([\w-]+)$',
+#                              alias => 'release', array => ',',
+#                              var => \@sections,
+#                              replace => { all => \@SECTIONS } },
+#                 arch => { default => 'any', match => '^(\w+)$',
+#                           array => ',', var => \@archs, replace =>
+#                           { any => \@ARCHITECTURES } },
+#                 format => { default => 'html', match => '^(\w+)$',
+#                                var => \$format },
+#                 );
 my %opts;
 my %params = Packages::Search::parse_params( $input, \%params_def, \%opts );
 
@@ -146,6 +150,7 @@ my $case_bool = ( $case !~ /insensitive/ );
 $exact = !$subword unless defined $exact;
 $opts{h_suites} = { map { $_ => 1 } @suites };
 $opts{h_sections} = { map { $_ => 1 } @sections };
+$opts{h_archives} = { map { $_ => 1 } @archives };
 $opts{h_archs} = { map { $_ => 1 } @archs };
 
 # for URL construction
@@ -299,20 +304,20 @@ print_errors();
 print_hints();
 print_debug();
 if (@results) {
-    my (%pkgs, %sect, %part, %desc, %binaries);
+    my (%pkgs, %subsect, %sect, %desc, %binaries);
 
     unless ($opts{searchon} eq 'sourcenames') {
        foreach (@results) {
-           my ($pkg_t, $suite, $arch, $section, $subsection,
+           my ($pkg_t, $archive, $suite, $arch, $section, $subsection,
                $priority, $version, $desc) = @$_;
        
            my ($pkg) = $pkg_t =~ m/^(.+)/; # untaint
-           $pkgs{$pkg}{$suite}{$version}{$arch} = 1;
-           $sect{$pkg}{$suite}{$version} = $subsection;
-           $part{$pkg}{$suite}{$version} = $section
+           $pkgs{$pkg}{$suite}{$archive}{$version}{$arch} = 1;
+           $subsect{$pkg}{$suite}{$archive}{$version} = $subsection;
+           $sect{$pkg}{$suite}{$archive}{$version} = $section
                unless $section eq 'main';
            
-           $desc{$pkg}{$suite}{$version} = $desc;
+           $desc{$pkg}{$suite}{$archive}{$version} = $desc;
        }
 
        if ($opts{format} eq 'html') {
@@ -325,21 +330,23 @@ if (@results) {
                printf "<h3>Package %s</h3>\n", $pkg;
                print "<ul>\n";
                foreach my $suite (@SUITES) {
-                   if (exists $pkgs{$pkg}{$suite}) {
-                       my @versions = version_sort keys %{$pkgs{$pkg}{$suite}};
-                       my $part_str = "";
-                       if ($part{$pkg}{$suite}{$versions[0]}) {
-                           $part_str = "[<span style=\"color:red\">$part{$pkg}{$suite}{$versions[0]}</span>]";
+                   foreach my $archive (@ARCHIVES) {
+                       if (exists $pkgs{$pkg}{$suite}{$archive}) {
+                           my @versions = version_sort keys %{$pkgs{$pkg}{$suite}{$archive}};
+                           my $origin_str = "";
+                           if ($sect{$pkg}{$suite}{$archive}{$versions[0]}) {
+                               $origin_str .= " [<span style=\"color:red\">$sect{$pkg}{$suite}{$versions[0]}</span>]";
+                           }
+                           printf "<li><a href=\"$ROOT/%s/%s\">%s</a> (%s): %s   %s\n",
+                           $suite.(($archive ne 'us')?"/$archive":''), $pkg, $suite.(($archive ne 'us')?"/$archive":''), $subsect{$pkg}{$suite}{$archive}{$versions[0]},
+                           $desc{$pkg}{$suite}{$archive}{$versions[0]}, $origin_str;
+                           
+                           foreach my $v (@versions) {
+                               printf "<br>%s: %s\n",
+                               $v, join (" ", (sort keys %{$pkgs{$pkg}{$suite}{$archive}{$v}}) );
+                           }
+                           print "</li>\n";
                        }
-                       printf "<li><a href=\"$ROOT/%s/%s\">%s</a> (%s): %s   %s\n",
-                       $suite, $pkg, $suite, $sect{$pkg}{$suite}{$versions[0]},
-                       $desc{$pkg}{$suite}{$versions[0]}, $part_str;
-                       
-                       foreach my $v (@versions) {
-                           printf "<br>%s: %s\n",
-                           $v, join (" ", (sort keys %{$pkgs{$pkg}{$suite}{$v}}) );
-                       }
-                       print "</li>\n";
                    }
                }
                print "</ul>\n";
@@ -347,15 +354,15 @@ if (@results) {
        }
     } else {
        foreach (@results) {
-           my ($pkg, $suite, $section, $subsection, $priority,
+           my ($pkg, $archive, $suite, $section, $subsection, $priority,
                $version) = @$_;
        
-           $pkgs{$pkg}{$suite} = $version;
-           $sect{$pkg}{$suite}{source} = $subsection;
-           $part{$pkg}{$suite}{source} = $section
+           $pkgs{$pkg}{$suite}{$archive} = $version;
+           $subsect{$pkg}{$suite}{$archive}{source} = $subsection;
+           $sect{$pkg}{$suite}{$archive}{source} = $section
                unless $section eq 'main';
 
-           $binaries{$pkg}{$suite} = find_binaries( $pkg, $suite, \%src2bin );
+           $binaries{$pkg}{$suite}{$archive} = find_binaries( $pkg, $archive, $suite, \%src2bin );
        }
 
        if ($opts{format} eq 'html') {
@@ -368,24 +375,26 @@ if (@results) {
                printf "<h3>Source package %s</h3>\n", $pkg;
                print "<ul>\n";
                foreach my $suite (@SUITES) {
-                   if (exists $pkgs{$pkg}{$suite}) {
-                       my $part_str = "";
-                       if ($part{$pkg}{$suite}{source}) {
-                           $part_str = "[<span style=\"color:red\">$part{$pkg}{$suite}{source}</span>]";
-                       }
-                       printf( "<li><a href=\"$ROOT/%s/source/%s\">%s</a> (%s): %s   %s",
-                               $suite, $pkg, $suite, $sect{$pkg}{$suite}{source},
-                               $pkgs{$pkg}{$suite}, $part_str );
-                       
-                       print "<br>Binary packages: ";
-                       my @bp_links;
-                       foreach my $bp (@{$binaries{$pkg}{$suite}}) {
-                           my $bp_link = sprintf( "<a href=\"$ROOT/%s/%s\">%s</a>",
-                                                  $suite, uri_escape( $bp ),  $bp );
-                           push @bp_links, $bp_link;
+                   foreach my $archive (@ARCHIVES) {
+                       if (exists $pkgs{$pkg}{$suite}{$archive}) {
+                           my $origin_str = "";
+                           if ($sect{$pkg}{$suite}{$archive}{source}) {
+                               $origin_str .= " [<span style=\"color:red\">$sect{$pkg}{$suite}{$archive}{source}</span>]";
+                           }
+                           printf( "<li><a href=\"$ROOT/%s/source/%s\">%s</a> (%s): %s   %s",
+                                   $suite.(($archive ne 'us')?"/$archive":''), $pkg, $suite.(($archive ne 'us')?"/$archive":''), $subsect{$pkg}{$suite}{$archive}{source},
+                                   $pkgs{$pkg}{$suite}{$archive}, $origin_str );
+                           
+                           print "<br>Binary packages: ";
+                           my @bp_links;
+                           foreach my $bp (@{$binaries{$pkg}{$suite}{$archive}}) {
+                               my $bp_link = sprintf( "<a href=\"$ROOT/%s/%s\">%s</a>",
+                                                      $suite.(($archive ne 'us')?"/$archive":''), uri_escape( $bp ),  $bp );
+                               push @bp_links, $bp_link;
+                           }
+                           print join( ", ", @bp_links );
+                           print "</li>\n";
                        }
-                       print join( ", ", @bp_links );
-                       print "</li>\n";
                    }
                }
                print "</ul>\n";

diff --git a/config.sh b/config.sh
index 6059fa70a8fd6668c5c411beb7381aadef1e0fd2..85f35acf2fa16e56e49a17178c1533168311f3ca 100644 (file)
--- a/config.sh
+++ b/config.sh
@@ -27,6 +27,7 @@ ftpsite=http://ftp.debian.org/debian
 nonus_ftpsite=http://ftp.uk.debian.org/debian-non-US
 security_ftpsite=http://security.debian.org/debian-security
 volatile_ftpsite=http://volatile.debian.net/debian-volatile
+backports_ftpsite=http://backports.org/debian
 amd64_ftpsite=http://amd64.debian.net/debian
 kfreebsd_ftpsite=http://kfreebsd-gnu.debian.net/debian
 
@@ -41,8 +42,11 @@ home="http://www.debian.org"
 #
 polangs="de fi nl fr uk"
 ddtplangs="de cs da eo es fi fr hu it ja nl pl pt_BR pt_PT ru sk sv_SE uk"
+archives="us non-US security volatile backports"
 sections="main contrib non-free"
-suites="oldstable stable testing unstable"
+parts="$sections"
+suites="oldstable stable testing unstable experimental"
+dists="$suites"
 architectures="alpha amd64 arm hppa hurd-i386 i386 ia64 kfreebsd-i386 m68k mips mipsel powerpc s390 sparc"
 arch_oldstable="alpha arm hppa i386 ia64 m68k mips mipsel powerpc s390 sparc"
 arch_stable="${arch_oldstable} amd64"

diff --git a/cron.d/100syncarchive b/cron.d/100syncarchive
new file mode 100755 (executable)
index 0000000..ee495ba
--- /dev/null
+++ b/cron.d/100syncarchive
@@ -0,0 +1,123 @@
+#! /bin/bash
+
+. `dirname $0`/../config.sh
+
+test -d ${archivedir} || mkdir -p ${archivedir}
+cd ${archivedir}
+
+# Main archive
+#
+if [ -z "${localdir}" ]
+then
+    echo using remote mirror
+    for dist in $dists
+    do
+       foo=\$arch_${dist//-/_}
+       for arch in `eval echo $foo`
+       do
+           test ! "$arch" = "amd64" || continue
+           test ! "$arch" = "kfreebsd-i386" || continue
+           for part in ${parts}
+           do
+               echo retrieve Packages $dist/$part/$arch
+               test -d us/${dist}/${part}/binary-${arch} || mkdir -p us/${dist}/${part}/binary-${arch}
+               (cd us/${dist}/${part}/binary-${arch} &&
+                   wget -q -N ${ftpsite}/dists/${dist}/${part}/binary-${arch}/Packages.gz)
+           done
+           #FIXME: no Contents files for p-u
+           test ! "$dist" = "experimental" || continue
+           echo retrieve Contents $dist/$arch
+           (cd us/${dist} &&
+               wget -q -N ${ftpsite}/dists/${dist}/Contents-${arch}.gz)
+       done
+       for part in ${parts}
+       do
+           echo retrieve Sources $dist/$part
+           test -d us/${dist}/${part}/source || mkdir -p us/${dist}/${part}/source
+            (cd us/${dist}/${part}/source &&
+                wget -q -N ${ftpsite}/dists/${dist}/${part}/source/Sources.gz)
+       done
+    done
+else
+    echo using local mirror
+    for dist in $dists
+    do
+       foo=\$arch_${dist//-/_}
+       for arch in `eval echo $foo`
+       do
+           test ! "$arch" = "amd64" || continue
+           test ! "$arch" = "kfreebsd-i386" || continue
+           for part in ${parts}
+           do
+               echo retrieve Packages $dist/$part/$arch
+               test -d us/${dist}/${part}/binary-${arch} || mkdir -p us/${dist}/${part}/binary-${arch}
+               rsync -t ${localdir}/dists/${dist}/${part}/binary-${arch}/Packages.gz \
+                   us/${dist}/${part}/binary-${arch}/Packages.gz
+           done
+           #FIXME: no Contents files for p-u
+           test ! "$dist" = "experimental" || continue
+           echo retrieve Contents $dist/$arch
+           rsync -t ${localdir}/dists/${dist}/Contents-${arch}.gz \
+               us/${dist}/Contents-${arch}.gz
+       done
+       for part in ${parts}
+        do
+           echo retrieve Sources $dist/$part
+            test -d us/${dist}/${part}/source || mkdir -p us/${dist}/${part}/source
+            rsync -t ${localdir}/dists/${dist}/${part}/source/Sources.gz \
+               us/${dist}/${part}/source/Sources.gz
+       done
+    done
+fi
+
+# Non-US archive
+#
+for dist in oldstable
+do
+    foo=\$arch_${dist//-/_}
+    for arch in `eval echo $foo`
+    do
+       for part in ${parts}
+       do
+           echo retrieve Packages $dist/$part/non-US/$arch
+           test -d non-US/${dist}/${part}/binary-${arch} || mkdir -p non-US/${dist}/${part}/binary-${arch}
+           (cd non-US/${dist}/${part}/binary-${arch} &&
+               wget -q -N ${nonus_ftpsite}/dists/${dist}/non-US/${part}/binary-${arch}/Packages.gz)
+       done
+       #FIXME: no Contents files for p-u
+       echo retrieve Contents $dist/non-US/$arch
+       (cd non-US/${dist} &&
+           wget -q -N ${nonus_ftpsite}/dists/${dist}/non-US/Contents-${arch}.gz)
+    done
+    for part in ${parts}
+    do
+        echo retrieve Sources $dist/$part/non-US
+       test -d non-US/${dist}/${part}/source || mkdir -p non-US/${dist}/${part}/source
+       (cd non-US/${dist}/${part}/source &&
+           wget -q -N ${nonus_ftpsite}/dists/${dist}/non-US/${part}/source/Sources.gz)
+    done
+done
+
+# security archive
+#
+for dist in oldstable stable testing
+do
+    foo=\$arch_${dist//-/_}
+    for arch in `eval echo $foo`
+    do
+      for part in ${parts}
+      do
+       echo retrieve Packages $dist/$part/security/$arch
+       test -d security/${dist}/${part}/binary-${arch} || mkdir -p security/${dist}/${part}/binary-${arch}
+       (cd  security/${dist}/${part}/binary-${arch} &&
+           wget -q -N ${security_ftpsite}/dists/${dist}/updates/${part}/binary-${arch}/Packages.gz)
+      done
+    done
+    for part in ${parts}
+    do
+        echo retrieve Sources $dist/$part/security
+        test -d security/${dist}/${part}/source || mkdir -p security/${dist}/${part}/source
+        (cd security/${dist}/${part}/source &&
+            wget -q -N ${security_ftpsite}/dists/${dist}/updates/${part}/source/Sources.gz)
+    done
+done

diff --git a/cron.d/100syncarchive_amd64 b/cron.d/100syncarchive_amd64
new file mode 100755 (executable)
index 0000000..45754a4
--- /dev/null
+++ b/cron.d/100syncarchive_amd64
@@ -0,0 +1,25 @@
+#! /bin/bash
+
+. `dirname $0`/../config.sh
+
+test -d ${archivedir} || mkdir -p ${archivedir}
+cd ${archivedir}
+
+# Main archive
+#
+arch=amd64
+
+for dist in stable testing unstable
+  do
+  for part in ${parts} main/debian-installer
+    do
+    echo retrieve Packages $dist/$part/$arch
+    test -d us/${dist}/${part}/binary-${arch} || mkdir -p us/${dist}/${part}/binary-${arch}
+    (cd us/${dist}/${part}/binary-${arch} &&
+       wget -q -N ${amd64_ftpsite}/dists/${dist}/${part}/binary-${arch}/Packages.gz)
+  done
+  echo retrieve Contents $dist/$arch
+  (cd us/${dist} &&
+      wget -q -N ${amd64_ftpsite}/dists/${dist}/Contents-${arch}.gz)
+done
+

diff --git a/cron.d/100syncarchive_backports b/cron.d/100syncarchive_backports
new file mode 100755 (executable)
index 0000000..78ca56a
--- /dev/null
+++ b/cron.d/100syncarchive_backports
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+. `dirname $0`/../config.sh
+
+test -d ${archivedir} || mkdir -p ${archivedir}
+cd ${archivedir}
+
+# backports archive
+#
+for dist in stable
+do
+    foo=\$arch_${dist//-/_}
+    for arch in `eval echo $foo`
+    do
+      for part in ${parts}
+      do
+       echo retrieve Packages $dist/$part/backports/$arch
+       test -d backports/${dist}/${part}/binary-${arch} || mkdir -p backports/${dist}/${part}/binary-${arch}
+       (cd backports/${dist}/${part}/binary-${arch} &&
+           wget -q -N ${backports_ftpsite}/dists/sarge-backports/${part}/binary-${arch}/Packages.gz)
+      done
+      echo retrieve Contents $dist/$arch
+      (cd backports/${dist} &&
+         wget -q -N ${backports_ftpsite}/dists/sarge-backports/Contents-${arch}.gz)
+    done
+    for part in ${parts}
+    do
+        echo retrieve Sources $dist/$part/backports
+        test -d backports/${dist}/${part}/source || mkdir -p backports/${dist}/${part}/source
+        (cd backports/${dist}/${part}/source &&
+            wget -q -N ${backports_ftpsite}/dists/sarge-backports/${part}/source/Sources.gz)
+    done
+done

diff --git a/cron.d/100syncarchive_installer b/cron.d/100syncarchive_installer
new file mode 100755 (executable)
index 0000000..3784ec1
--- /dev/null
+++ b/cron.d/100syncarchive_installer
@@ -0,0 +1,53 @@
+#! /bin/bash
+
+. `dirname $0`/../config.sh
+
+test -d ${archivedir} || mkdir -p ${archivedir}
+cd ${archivedir}
+
+# Main archive
+#
+if [ -z "${localdir}" ]
+then
+    echo using remote mirror
+    for dist in ${dists}
+    do
+
+      if [ $dist = oldstable ]; then
+         continue
+      fi
+
+       foo=\$arch_${dist//-/_}
+       for arch in `eval echo $foo`
+       do
+           for part in "main/debian-installer"
+           do
+               echo retrieve Packages $dist/$part/$arch
+               test -d us/${dist}/${part}/binary-${arch} || mkdir -p us/${dist}/${part}/binary-${arch}
+               (cd us/${dist}/${part}/binary-${arch} &&
+                   wget -q -N ${ftpsite}/dists/${dist}/${part}/binary-${arch}/Packages.gz)
+           done
+       done
+    done
+else
+    echo using local mirror
+    for dist in ${dists}
+    do
+
+      if [ $dist = oldstable ]; then
+         continue
+      fi
+
+       foo=\$arch_${dist//-/_}
+       for arch in `eval echo $foo`
+       do
+           for part in "main/debian-installer"
+           do
+               echo retrieve Sources $dist/$part/$arch
+               test -d us/${dist}/${part}/binary-${arch} || mkdir -p us/${dist}/${part}/binary-${arch}
+               rsync -t ${localdir}/dists/${dist}/${part}/binary-${arch}/Packages.gz \
+                   us/${dist}/${part}/binary-${arch}/Packages.gz
+           done
+       done
+    done
+fi

diff --git a/cron.d/100syncarchive_kfreebsd b/cron.d/100syncarchive_kfreebsd
new file mode 100755 (executable)
index 0000000..9700f69
--- /dev/null
+++ b/cron.d/100syncarchive_kfreebsd
@@ -0,0 +1,25 @@
+#! /bin/bash
+
+. `dirname $0`/../config.sh
+
+test -d ${archivedir} || mkdir -p ${archivedir}
+cd ${archivedir}
+
+# Main archive
+#
+arch=kfreebsd-i386
+
+for dist in unstable experimental
+  do
+  for part in ${parts} main/debian-installer
+    do
+    echo retrieve Packages $dist/$part/$arch
+    test -d us/${dist}/${part}/binary-${arch} || mkdir -p us/${dist}/${part}/binary-${arch}
+    (cd us/${dist}/${part}/binary-${arch} &&
+       wget -q -N ${kfreebsd_ftpsite}/dists/${dist}/${part}/binary-${arch}/Packages.gz)
+  done
+  echo retrieve Contents $dist/$arch
+  (cd us/${dist} &&
+      wget -q -N ${kfreebsd_ftpsite}/dists/${dist}/Contents-${arch}.gz)
+done
+

diff --git a/cron.d/100syncarchive_volatile b/cron.d/100syncarchive_volatile
new file mode 100755 (executable)
index 0000000..ed7a030
--- /dev/null
+++ b/cron.d/100syncarchive_volatile
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+. `dirname $0`/../config.sh
+
+test -d ${archivedir} || mkdir -p ${archivedir}
+cd ${archivedir}
+
+# volatile archive
+#
+for dist in stable
+do
+    foo=\$arch_${dist//-/_}
+    for arch in `eval echo $foo`
+    do
+      for part in ${parts}
+      do
+       echo retrieve Packages $dist/$part/volatile/$arch
+       test -d volatile/${dist}/${part}/binary-${arch} || mkdir -p volatile/${dist}/${part}/binary-${arch}
+       (cd  volatile/${dist}/${part}/binary-${arch} &&
+           wget -q -N ${volatile_ftpsite}/dists/${dist}/volatile/${part}/binary-${arch}/Packages.gz)
+      done
+    done
+    for part in ${parts}
+    do
+        echo retrieve Sources $dist/$part/volatile
+        test -d volatile/${dist}/${part}/source || mkdir -p volatile/${dist}/${part}/source
+        (cd volatile/${dist}/${part}/source &&
+            wget -q -N ${volatile_ftpsite}/dists/${dist}/volatile/${part}/source/Sources.gz)
+    done
+done

diff --git a/lib/Packages/Search.pm b/lib/Packages/Search.pm
index ae8cee3c577edf729db8d0db3152147c59693309..31fe9d78d4821bf0c71a1a5b9309dd0070c070f8 100644 (file)
--- a/lib/Packages/Search.pm
+++ b/lib/Packages/Search.pm
@@ -342,11 +342,11 @@ sub read_entry {
     my ($hash, $key, $results, $opts) = @_;
     my $result = $hash->{$key} || '';
     foreach (split /\000/, $result) {
-       my @data = split ( /\s/, $_, 7 );
+       my @data = split ( /\s/, $_, 8 );
        debug( "Considering entry ".join( ':', @data), 2);
-       if ($opts->{h_suites}{$data[0]}
-           && ($opts->{h_archs}{$data[1]} || $data[1] eq 'all')
-           && $opts->{h_sections}{$data[2]}) {
+       if ($opts->{h_archives}{$data[0]} && $opts->{h_suites}{$data[1]}
+           && ($opts->{h_archs}{$data[2]} || $data[2] eq 'all')
+           && $opts->{h_sections}{$data[3]}) {
            debug( "Using entry ".join( ':', @data), 2);
            push @$results, [ $key, @data ];
        }
@@ -356,9 +356,11 @@ sub read_src_entry {
     my ($hash, $key, $results, $opts) = @_;
     my $result = $hash->{$key} || '';
     foreach (split /\000/, $result) {
-       my @data = split ( /\s/, $_, 5 );
+       my @data = split ( /\s/, $_, 6 );
        debug( "Considering entry ".join( ':', @data), 2);
-       if ($opts->{h_suites}{$data[0]} && $opts->{h_sections}{$data[1]}) {
+       if ($opts->{h_archives}{$data[0]}
+           && $opts->{h_suites}{$data[1]}
+           && $opts->{h_sections}{$data[2]}) {
            debug( "Using entry ".join( ':', @data), 2);
            push @$results, [ $key, @data ];
        }
@@ -446,15 +448,16 @@ sub do_fulltext_search {
 }
 
 sub find_binaries {
-    my ($pkg, $suite, $src2bin) = @_;
+    my ($pkg, $archive, $suite, $src2bin) = @_;
 
     my $bins = $src2bin->{$pkg} || '';
     my %bins;
     foreach (split /\000/o, $bins) {
-       my @data = split /\s/, $_, 4;
+       my @data = split /\s/, $_, 5;
 
-       if ($data[0] eq $suite) {
-           $bins{$data[1]}++;
+       if (($data[0] eq $archive)
+           && ($data[1] eq $suite)) {
+           $bins{$data[2]}++;
        }
     }