From f7dae3f81707c8cab707df20ba2742a0087baf33 Mon Sep 17 00:00:00 2001 From: Frank Lichtenheld Date: Sat, 4 Feb 2006 00:14:50 +0000 Subject: [PATCH] * Add back some parts of the cron job (100syncarchive_*) * Use these files from bin/parse-* * Support different archives --- bin/daily | 45 +++++++++ bin/parse-packages | 18 ++-- bin/parse-sources | 14 ++- cgi-bin/search_packages.pl | 163 +++++++++++++++++--------------- config.sh | 6 +- cron.d/100syncarchive | 123 ++++++++++++++++++++++++ cron.d/100syncarchive_amd64 | 25 +++++ cron.d/100syncarchive_backports | 33 +++++++ cron.d/100syncarchive_installer | 53 +++++++++++ cron.d/100syncarchive_kfreebsd | 25 +++++ cron.d/100syncarchive_volatile | 30 ++++++ lib/Packages/Search.pm | 23 +++-- 12 files changed, 459 insertions(+), 99 deletions(-) create mode 100755 bin/daily create mode 100755 cron.d/100syncarchive create mode 100755 cron.d/100syncarchive_amd64 create mode 100755 cron.d/100syncarchive_backports create mode 100755 cron.d/100syncarchive_installer create mode 100755 cron.d/100syncarchive_kfreebsd create mode 100755 cron.d/100syncarchive_volatile diff --git a/bin/daily b/bin/daily new file mode 100755 index 0000000..0d5afb4 --- /dev/null +++ b/bin/daily @@ -0,0 +1,45 @@ +#! /bin/bash + +# Copyright (c) 2004 Martin Schulze +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +# Should run after the mirror pulse. The mirror tends to finish by +# the ctime of rsync.log.* in ~archvsync. + +topdir=/org/packages.debian.org +files=$topdir/files +logs=$files/logs +log=$logs/cron.log + +test -d $files || mkdir -p $files +test -d $logs || mkdir -p $logs + +if [ -s $log ] +then + savelog -c 14 $log > /dev/null +fi + +date > $log + +if lockfile -600 -r5 "${files}/cron.lock" >> $log 2>&1 +then + run-parts --verbose $topdir/cron.d >> $log 2>&1 +else + echo "couldn't aquire lock file in due time" >> $log +fi +rm -f "${files}/cron.lock" + +date >> $log diff --git a/bin/parse-packages b/bin/parse-packages index 8b132ab..7a67e69 100755 --- a/bin/parse-packages +++ b/bin/parse-packages @@ -35,18 +35,20 @@ my @descriptions = ("we count lines one-based\000"); my %packages_descriptions = (); my %descriptions_packages = (); +my @archives = ( 'us', 'non-US', 'security', 'volatile', 'backports' ); my @suites = ('oldstable', 'stable', 'testing', 'unstable', 'experimental'); $/ = ""; -for my $suite (@suites) { +for my $archive (@archives) { + for my $suite (@suites) { - print "Reading $suite...\n"; + print "Reading $archive/$suite...\n"; my %packages_all_db; tie %packages_all_db, "DB_File", "packages_all_$suite.db.new", O_RDWR|O_CREAT, 0666, $DB_BTREE or die "Error creating DB: $!"; - open PKG, "zcat /org/ftp.debian.org/ftp/dists/$suite/$what/{,debian-installer/}binary-*/Packages.gz|"; + open PKG, "zcat /org/packages.debian.org/archive/$archive/$suite/$what/{,debian-installer/}binary-*/Packages.gz|"; while () { next if /^\s*$/; my $data = ""; @@ -86,19 +88,22 @@ for my $suite (@suites) { my $sdescr = $descr; $sdescr =~ s/\n.*//s; my $section = 'main'; - my $subsection = $data{section}; + my $subsection = $data{section} || '-'; if ($data{section} && ($data{section} =~ m=/=o)) { ($section, $subsection) = split m=/=o, $data{section}, 2; } $data{'section'} = $section; $data{'subsection'} = $subsection; - $packages_small{$data{'package'}} .= "$suite $data{'architecture'} ". + $data{'priority'} ||= '-'; + $packages_small{$data{'package'}} .= "$archive $suite $data{'architecture'} ". "$section $subsection $data{'priority'} $data{'version'} $sdescr\000"; $sources_packages{$src} .= - "$suite $data{'package'} $data{'version'} $data{'architecture'}\000"; + "$archive $suite $data{'package'} $data{'version'} $data{'architecture'}\000"; + $data{archive} = $archive; while (my ($key, $value) = each (%data)) { next if $key eq 'package' or $key eq 'version' or $key eq 'architecture' or $key eq 'maintainer'; + print STDERR "WARN: $key ($suite/$archive/$data{package}/$data{architecture}\n" unless defined $value; $data .= "$key: $value\n"; } $packages_all_db{"$data{'package'} $data{'architecture'} $data{'version'}"} @@ -106,6 +111,7 @@ for my $suite (@suites) { } untie %packages_all_db; + } } print "Writing databases...\n"; diff --git a/bin/parse-sources b/bin/parse-sources index 63eda09..82716b3 100755 --- a/bin/parse-sources +++ b/bin/parse-sources @@ -30,18 +30,20 @@ my %sources_small = (); my %source_names = (); my %source_postfixes = (); +my @archives = ( 'us', 'non-US', 'security', 'volatile', 'backports' ); my @suites = ('oldstable', 'stable', 'testing', 'unstable', 'experimental'); $/ = ""; -for my $suite (@suites) { +for my $archive (@archives) { + for my $suite (@suites) { - print "Reading $suite...\n"; + print "Reading $archive/$suite...\n"; my %sources_all_db; tie %sources_all_db, "DB_File", "sources_all_$suite.db.new", O_RDWR|O_CREAT, 0666, $DB_BTREE or die "Error creating DB: $!"; - open PKG, "zcat /org/ftp.debian.org/ftp/dists/$suite/$what/source/Sources.gz|"; + open PKG, "zcat /org/packages.debian.org/archive/$archive/$suite/$what/source/Sources.gz|"; while () { next if /^\s*$/; my $data = ""; @@ -55,22 +57,24 @@ for my $suite (@suites) { $key =~ tr [A-Z] [a-z]; $data{$key} = $value; } + $data .= "Archive: $archive\n"; $sources_all_db{"$data{'package'} $data{'version'}"} = $data; $source_names{$data{'package'}} = 1; my $section = 'main'; - my $subsection = $data{section}; + my $subsection = $data{section} || '-'; if ($data{section} && ($data{section} =~ m=/=o)) { ($section, $subsection) = split m=/=o, $data{section}, 2; } $data{'priority'} = "-" if not exists($data{'priority'}); $sources_small{$data{'package'}} .= - "$suite $section $subsection $data{'priority'} $data{'version'}\000"; + "$archive $suite $section $subsection $data{'priority'} $data{'version'}\000"; } untie %sources_all_db; + } } print "Writing databases...\n"; diff --git a/cgi-bin/search_packages.pl b/cgi-bin/search_packages.pl index 25b0e13..48fabb4 100755 --- a/cgi-bin/search_packages.pl +++ b/cgi-bin/search_packages.pl @@ -48,19 +48,16 @@ $Packages::CGI::debug = $debug; # read the configuration our $config_read_time ||= 0; our $db_read_time ||= 0; -our $topdir; -our $ROOT; -our @SUITES; -our @SECTIONS; -our @ARCHITECTURES; +our ( $topdir, $ROOT, @SUITES, @SECTIONS, @ARCHIVES, @ARCHITECTURES ); # FIXME: move to own module my $modtime = (stat( "../config.sh" ))[9]; if ($modtime > $config_read_time) { if (!open (C, '<', "../config.sh")) { - error( "Internal Error: Cannot open configuration file." ); + error( "Internal: Cannot open configuration file." ); } while () { + next if /^\s*\#/o; chomp; $topdir = $1 if /^\s*topdir="?([^\"]*)"?\s*$/o; $ROOT = $1 if /^\s*root="?([^\"]*)"?\s*$/o; @@ -71,6 +68,7 @@ if ($modtime > $config_read_time) { $Packages::HTML::CONTACT_MAIL = $1 if /^\s*contact="?([^\"]*)"?\s*$/o; @SUITES = split(/\s+/, $1) if /^\s*suites="?([^\"]*)"?\s*$/o; @SECTIONS = split(/\s+/, $1) if /^\s*sections="?([^\"]*)"?\s*$/o; + @ARCHIVES = split(/\s+/, $1) if /^\s*archives="?([^\"]*)"?\s*$/o; @ARCHITECTURES = split(/\s+/, $1) if /^\s*architectures="?([^\"]*)"?\s*$/o; } close (C); @@ -85,6 +83,7 @@ if (my $path = $input->param('path')) { my %SUITES = map { $_ => 1 } @SUITES; my %SECTIONS = map { $_ => 1 } @SECTIONS; + my %ARCHIVES = map { $_ => 1 } @ARCHIVES; my %ARCHITECTURES = map { $_ => 1 } @ARCHITECTURES; foreach (@components) { @@ -92,41 +91,46 @@ if (my $path = $input->param('path')) { $input->param('suite', $_); } elsif ($SECTIONS{$_}) { $input->param('section', $_); - }elsif ($ARCHITECTURES{$_}) { + } elsif ($ARCHIVES{$_}) { + $input->param('archive', $_); + } elsif ($ARCHITECTURES{$_}) { $input->param('arch', $_); } } } my ( $format, $keyword, $case, $subword, $exact, $searchon, - @suites, @sections, @archs ); - -my %params_def = ( keywords => { default => undef, - match => '^\s*([-+\@\w\/.:]+)\s*$', - var => \$keyword }, - suite => { default => 'stable', match => '^(\w+)$', - alias => 'version', array => ',', - var => \@suites, - replace => { all => \@SUITES } }, - case => { default => 'insensitive', match => '^(\w+)$', - var => \$case }, - official => { default => 0, match => '^(\w+)$' }, - subword => { default => 0, match => '^(\w+)$', - var => \$subword }, - exact => { default => undef, match => '^(\w+)$', - var => \$exact }, - searchon => { default => 'all', match => '^(\w+)$', - var => \$searchon }, - section => { default => 'all', match => '^([\w-]+)$', - alias => 'release', array => ',', - var => \@sections, - replace => { all => \@SECTIONS } }, - arch => { default => 'any', match => '^(\w+)$', - array => ',', var => \@archs, replace => - { any => \@ARCHITECTURES } }, - format => { default => 'html', match => '^(\w+)$', - var => \$format }, - ); + @suites, @sections, @archives, @archs ); + +# my %params_def = ( keywords => { default => undef, +# match => '^\s*([-+\@\w\/.:]+)\s*$', +# var => \$keyword }, +# suite => { default => 'stable', match => '^([\w-]+)$', +# alias => 'version', array => ',', +# var => \@suites, +# replace => { all => \@SUITES } }, +# archive => { default => 'all', match => '^([\w-]+)$', +# array => ',', var => \@archives, +# replace => { all => \@ARCHIVES } }, +# case => { default => 'insensitive', match => '^(\w+)$', +# var => \$case }, +# official => { default => 0, match => '^(\w+)$' }, +# subword => { default => 0, match => '^(\w+)$', +# var => \$subword }, +# exact => { default => undef, match => '^(\w+)$', +# var => \$exact }, +# searchon => { default => 'all', match => '^(\w+)$', +# var => \$searchon }, +# section => { default => 'all', match => '^([\w-]+)$', +# alias => 'release', array => ',', +# var => \@sections, +# replace => { all => \@SECTIONS } }, +# arch => { default => 'any', match => '^(\w+)$', +# array => ',', var => \@archs, replace => +# { any => \@ARCHITECTURES } }, +# format => { default => 'html', match => '^(\w+)$', +# var => \$format }, +# ); my %opts; my %params = Packages::Search::parse_params( $input, \%params_def, \%opts ); @@ -146,6 +150,7 @@ my $case_bool = ( $case !~ /insensitive/ ); $exact = !$subword unless defined $exact; $opts{h_suites} = { map { $_ => 1 } @suites }; $opts{h_sections} = { map { $_ => 1 } @sections }; +$opts{h_archives} = { map { $_ => 1 } @archives }; $opts{h_archs} = { map { $_ => 1 } @archs }; # for URL construction @@ -299,20 +304,20 @@ print_errors(); print_hints(); print_debug(); if (@results) { - my (%pkgs, %sect, %part, %desc, %binaries); + my (%pkgs, %subsect, %sect, %desc, %binaries); unless ($opts{searchon} eq 'sourcenames') { foreach (@results) { - my ($pkg_t, $suite, $arch, $section, $subsection, + my ($pkg_t, $archive, $suite, $arch, $section, $subsection, $priority, $version, $desc) = @$_; my ($pkg) = $pkg_t =~ m/^(.+)/; # untaint - $pkgs{$pkg}{$suite}{$version}{$arch} = 1; - $sect{$pkg}{$suite}{$version} = $subsection; - $part{$pkg}{$suite}{$version} = $section + $pkgs{$pkg}{$suite}{$archive}{$version}{$arch} = 1; + $subsect{$pkg}{$suite}{$archive}{$version} = $subsection; + $sect{$pkg}{$suite}{$archive}{$version} = $section unless $section eq 'main'; - $desc{$pkg}{$suite}{$version} = $desc; + $desc{$pkg}{$suite}{$archive}{$version} = $desc; } if ($opts{format} eq 'html') { @@ -325,21 +330,23 @@ if (@results) { printf "

Package %s

\n", $pkg; print "
    \n"; foreach my $suite (@SUITES) { - if (exists $pkgs{$pkg}{$suite}) { - my @versions = version_sort keys %{$pkgs{$pkg}{$suite}}; - my $part_str = ""; - if ($part{$pkg}{$suite}{$versions[0]}) { - $part_str = "[$part{$pkg}{$suite}{$versions[0]}]"; + foreach my $archive (@ARCHIVES) { + if (exists $pkgs{$pkg}{$suite}{$archive}) { + my @versions = version_sort keys %{$pkgs{$pkg}{$suite}{$archive}}; + my $origin_str = ""; + if ($sect{$pkg}{$suite}{$archive}{$versions[0]}) { + $origin_str .= " [$sect{$pkg}{$suite}{$versions[0]}]"; + } + printf "
  • %s (%s): %s %s\n", + $suite.(($archive ne 'us')?"/$archive":''), $pkg, $suite.(($archive ne 'us')?"/$archive":''), $subsect{$pkg}{$suite}{$archive}{$versions[0]}, + $desc{$pkg}{$suite}{$archive}{$versions[0]}, $origin_str; + + foreach my $v (@versions) { + printf "
    %s: %s\n", + $v, join (" ", (sort keys %{$pkgs{$pkg}{$suite}{$archive}{$v}}) ); + } + print "
    • \n"; } - printf "
  • %s (%s): %s %s\n", - $suite, $pkg, $suite, $sect{$pkg}{$suite}{$versions[0]}, - $desc{$pkg}{$suite}{$versions[0]}, $part_str; - - foreach my $v (@versions) { - printf "
    %s: %s\n", - $v, join (" ", (sort keys %{$pkgs{$pkg}{$suite}{$v}}) ); - } - print "
    • \n"; } } print "
\n"; @@ -347,15 +354,15 @@ if (@results) { } } else { foreach (@results) { - my ($pkg, $suite, $section, $subsection, $priority, + my ($pkg, $archive, $suite, $section, $subsection, $priority, $version) = @$_; - $pkgs{$pkg}{$suite} = $version; - $sect{$pkg}{$suite}{source} = $subsection; - $part{$pkg}{$suite}{source} = $section + $pkgs{$pkg}{$suite}{$archive} = $version; + $subsect{$pkg}{$suite}{$archive}{source} = $subsection; + $sect{$pkg}{$suite}{$archive}{source} = $section unless $section eq 'main'; - $binaries{$pkg}{$suite} = find_binaries( $pkg, $suite, \%src2bin ); + $binaries{$pkg}{$suite}{$archive} = find_binaries( $pkg, $archive, $suite, \%src2bin ); } if ($opts{format} eq 'html') { @@ -368,24 +375,26 @@ if (@results) { printf "

Source package %s

\n", $pkg; print "
    \n"; foreach my $suite (@SUITES) { - if (exists $pkgs{$pkg}{$suite}) { - my $part_str = ""; - if ($part{$pkg}{$suite}{source}) { - $part_str = "[$part{$pkg}{$suite}{source}]"; - } - printf( "
  • %s (%s): %s %s", - $suite, $pkg, $suite, $sect{$pkg}{$suite}{source}, - $pkgs{$pkg}{$suite}, $part_str ); - - print "
    Binary packages: "; - my @bp_links; - foreach my $bp (@{$binaries{$pkg}{$suite}}) { - my $bp_link = sprintf( "%s", - $suite, uri_escape( $bp ), $bp ); - push @bp_links, $bp_link; + foreach my $archive (@ARCHIVES) { + if (exists $pkgs{$pkg}{$suite}{$archive}) { + my $origin_str = ""; + if ($sect{$pkg}{$suite}{$archive}{source}) { + $origin_str .= " [$sect{$pkg}{$suite}{$archive}{source}]"; + } + printf( "
  • %s (%s): %s %s", + $suite.(($archive ne 'us')?"/$archive":''), $pkg, $suite.(($archive ne 'us')?"/$archive":''), $subsect{$pkg}{$suite}{$archive}{source}, + $pkgs{$pkg}{$suite}{$archive}, $origin_str ); + + print "
    Binary packages: "; + my @bp_links; + foreach my $bp (@{$binaries{$pkg}{$suite}{$archive}}) { + my $bp_link = sprintf( "%s", + $suite.(($archive ne 'us')?"/$archive":''), uri_escape( $bp ), $bp ); + push @bp_links, $bp_link; + } + print join( ", ", @bp_links ); + print "
    • \n"; } - print join( ", ", @bp_links ); - print "\n"; } } print "
\n"; diff --git a/config.sh b/config.sh index 6059fa7..85f35ac 100644 --- a/config.sh +++ b/config.sh @@ -27,6 +27,7 @@ ftpsite=http://ftp.debian.org/debian nonus_ftpsite=http://ftp.uk.debian.org/debian-non-US security_ftpsite=http://security.debian.org/debian-security volatile_ftpsite=http://volatile.debian.net/debian-volatile +backports_ftpsite=http://backports.org/debian amd64_ftpsite=http://amd64.debian.net/debian kfreebsd_ftpsite=http://kfreebsd-gnu.debian.net/debian @@ -41,8 +42,11 @@ home="http://www.debian.org" # polangs="de fi nl fr uk" ddtplangs="de cs da eo es fi fr hu it ja nl pl pt_BR pt_PT ru sk sv_SE uk" +archives="us non-US security volatile backports" sections="main contrib non-free" -suites="oldstable stable testing unstable" +parts="$sections" +suites="oldstable stable testing unstable experimental" +dists="$suites" architectures="alpha amd64 arm hppa hurd-i386 i386 ia64 kfreebsd-i386 m68k mips mipsel powerpc s390 sparc" arch_oldstable="alpha arm hppa i386 ia64 m68k mips mipsel powerpc s390 sparc" arch_stable="${arch_oldstable} amd64" diff --git a/cron.d/100syncarchive b/cron.d/100syncarchive new file mode 100755 index 0000000..ee495ba --- /dev/null +++ b/cron.d/100syncarchive @@ -0,0 +1,123 @@ +#! /bin/bash + +. `dirname $0`/../config.sh + +test -d ${archivedir} || mkdir -p ${archivedir} +cd ${archivedir} + +# Main archive +# +if [ -z "${localdir}" ] +then + echo using remote mirror + for dist in $dists + do + foo=\$arch_${dist//-/_} + for arch in `eval echo $foo` + do + test ! "$arch" = "amd64" || continue + test ! "$arch" = "kfreebsd-i386" || continue + for part in ${parts} + do + echo retrieve Packages $dist/$part/$arch + test -d us/${dist}/${part}/binary-${arch} || mkdir -p us/${dist}/${part}/binary-${arch} + (cd us/${dist}/${part}/binary-${arch} && + wget -q -N ${ftpsite}/dists/${dist}/${part}/binary-${arch}/Packages.gz) + done + #FIXME: no Contents files for p-u + test ! "$dist" = "experimental" || continue + echo retrieve Contents $dist/$arch + (cd us/${dist} && + wget -q -N ${ftpsite}/dists/${dist}/Contents-${arch}.gz) + done + for part in ${parts} + do + echo retrieve Sources $dist/$part + test -d us/${dist}/${part}/source || mkdir -p us/${dist}/${part}/source + (cd us/${dist}/${part}/source && + wget -q -N ${ftpsite}/dists/${dist}/${part}/source/Sources.gz) + done + done +else + echo using local mirror + for dist in $dists + do + foo=\$arch_${dist//-/_} + for arch in `eval echo $foo` + do + test ! "$arch" = "amd64" || continue + test ! "$arch" = "kfreebsd-i386" || continue + for part in ${parts} + do + echo retrieve Packages $dist/$part/$arch + test -d us/${dist}/${part}/binary-${arch} || mkdir -p us/${dist}/${part}/binary-${arch} + rsync -t ${localdir}/dists/${dist}/${part}/binary-${arch}/Packages.gz \ + us/${dist}/${part}/binary-${arch}/Packages.gz + done + #FIXME: no Contents files for p-u + test ! "$dist" = "experimental" || continue + echo retrieve Contents $dist/$arch + rsync -t ${localdir}/dists/${dist}/Contents-${arch}.gz \ + us/${dist}/Contents-${arch}.gz + done + for part in ${parts} + do + echo retrieve Sources $dist/$part + test -d us/${dist}/${part}/source || mkdir -p us/${dist}/${part}/source + rsync -t ${localdir}/dists/${dist}/${part}/source/Sources.gz \ + us/${dist}/${part}/source/Sources.gz + done + done +fi + +# Non-US archive +# +for dist in oldstable +do + foo=\$arch_${dist//-/_} + for arch in `eval echo $foo` + do + for part in ${parts} + do + echo retrieve Packages $dist/$part/non-US/$arch + test -d non-US/${dist}/${part}/binary-${arch} || mkdir -p non-US/${dist}/${part}/binary-${arch} + (cd non-US/${dist}/${part}/binary-${arch} && + wget -q -N ${nonus_ftpsite}/dists/${dist}/non-US/${part}/binary-${arch}/Packages.gz) + done + #FIXME: no Contents files for p-u + echo retrieve Contents $dist/non-US/$arch + (cd non-US/${dist} && + wget -q -N ${nonus_ftpsite}/dists/${dist}/non-US/Contents-${arch}.gz) + done + for part in ${parts} + do + echo retrieve Sources $dist/$part/non-US + test -d non-US/${dist}/${part}/source || mkdir -p non-US/${dist}/${part}/source + (cd non-US/${dist}/${part}/source && + wget -q -N ${nonus_ftpsite}/dists/${dist}/non-US/${part}/source/Sources.gz) + done +done + +# security archive +# +for dist in oldstable stable testing +do + foo=\$arch_${dist//-/_} + for arch in `eval echo $foo` + do + for part in ${parts} + do + echo retrieve Packages $dist/$part/security/$arch + test -d security/${dist}/${part}/binary-${arch} || mkdir -p security/${dist}/${part}/binary-${arch} + (cd security/${dist}/${part}/binary-${arch} && + wget -q -N ${security_ftpsite}/dists/${dist}/updates/${part}/binary-${arch}/Packages.gz) + done + done + for part in ${parts} + do + echo retrieve Sources $dist/$part/security + test -d security/${dist}/${part}/source || mkdir -p security/${dist}/${part}/source + (cd security/${dist}/${part}/source && + wget -q -N ${security_ftpsite}/dists/${dist}/updates/${part}/source/Sources.gz) + done +done diff --git a/cron.d/100syncarchive_amd64 b/cron.d/100syncarchive_amd64 new file mode 100755 index 0000000..45754a4 --- /dev/null +++ b/cron.d/100syncarchive_amd64 @@ -0,0 +1,25 @@ +#! /bin/bash + +. `dirname $0`/../config.sh + +test -d ${archivedir} || mkdir -p ${archivedir} +cd ${archivedir} + +# Main archive +# +arch=amd64 + +for dist in stable testing unstable + do + for part in ${parts} main/debian-installer + do + echo retrieve Packages $dist/$part/$arch + test -d us/${dist}/${part}/binary-${arch} || mkdir -p us/${dist}/${part}/binary-${arch} + (cd us/${dist}/${part}/binary-${arch} && + wget -q -N ${amd64_ftpsite}/dists/${dist}/${part}/binary-${arch}/Packages.gz) + done + echo retrieve Contents $dist/$arch + (cd us/${dist} && + wget -q -N ${amd64_ftpsite}/dists/${dist}/Contents-${arch}.gz) +done + diff --git a/cron.d/100syncarchive_backports b/cron.d/100syncarchive_backports new file mode 100755 index 0000000..78ca56a --- /dev/null +++ b/cron.d/100syncarchive_backports @@ -0,0 +1,33 @@ +#!/bin/bash + +. `dirname $0`/../config.sh + +test -d ${archivedir} || mkdir -p ${archivedir} +cd ${archivedir} + +# backports archive +# +for dist in stable +do + foo=\$arch_${dist//-/_} + for arch in `eval echo $foo` + do + for part in ${parts} + do + echo retrieve Packages $dist/$part/backports/$arch + test -d backports/${dist}/${part}/binary-${arch} || mkdir -p backports/${dist}/${part}/binary-${arch} + (cd backports/${dist}/${part}/binary-${arch} && + wget -q -N ${backports_ftpsite}/dists/sarge-backports/${part}/binary-${arch}/Packages.gz) + done + echo retrieve Contents $dist/$arch + (cd backports/${dist} && + wget -q -N ${backports_ftpsite}/dists/sarge-backports/Contents-${arch}.gz) + done + for part in ${parts} + do + echo retrieve Sources $dist/$part/backports + test -d backports/${dist}/${part}/source || mkdir -p backports/${dist}/${part}/source + (cd backports/${dist}/${part}/source && + wget -q -N ${backports_ftpsite}/dists/sarge-backports/${part}/source/Sources.gz) + done +done diff --git a/cron.d/100syncarchive_installer b/cron.d/100syncarchive_installer new file mode 100755 index 0000000..3784ec1 --- /dev/null +++ b/cron.d/100syncarchive_installer @@ -0,0 +1,53 @@ +#! /bin/bash + +. `dirname $0`/../config.sh + +test -d ${archivedir} || mkdir -p ${archivedir} +cd ${archivedir} + +# Main archive +# +if [ -z "${localdir}" ] +then + echo using remote mirror + for dist in ${dists} + do + + if [ $dist = oldstable ]; then + continue + fi + + foo=\$arch_${dist//-/_} + for arch in `eval echo $foo` + do + for part in "main/debian-installer" + do + echo retrieve Packages $dist/$part/$arch + test -d us/${dist}/${part}/binary-${arch} || mkdir -p us/${dist}/${part}/binary-${arch} + (cd us/${dist}/${part}/binary-${arch} && + wget -q -N ${ftpsite}/dists/${dist}/${part}/binary-${arch}/Packages.gz) + done + done + done +else + echo using local mirror + for dist in ${dists} + do + + if [ $dist = oldstable ]; then + continue + fi + + foo=\$arch_${dist//-/_} + for arch in `eval echo $foo` + do + for part in "main/debian-installer" + do + echo retrieve Sources $dist/$part/$arch + test -d us/${dist}/${part}/binary-${arch} || mkdir -p us/${dist}/${part}/binary-${arch} + rsync -t ${localdir}/dists/${dist}/${part}/binary-${arch}/Packages.gz \ + us/${dist}/${part}/binary-${arch}/Packages.gz + done + done + done +fi diff --git a/cron.d/100syncarchive_kfreebsd b/cron.d/100syncarchive_kfreebsd new file mode 100755 index 0000000..9700f69 --- /dev/null +++ b/cron.d/100syncarchive_kfreebsd @@ -0,0 +1,25 @@ +#! /bin/bash + +. `dirname $0`/../config.sh + +test -d ${archivedir} || mkdir -p ${archivedir} +cd ${archivedir} + +# Main archive +# +arch=kfreebsd-i386 + +for dist in unstable experimental + do + for part in ${parts} main/debian-installer + do + echo retrieve Packages $dist/$part/$arch + test -d us/${dist}/${part}/binary-${arch} || mkdir -p us/${dist}/${part}/binary-${arch} + (cd us/${dist}/${part}/binary-${arch} && + wget -q -N ${kfreebsd_ftpsite}/dists/${dist}/${part}/binary-${arch}/Packages.gz) + done + echo retrieve Contents $dist/$arch + (cd us/${dist} && + wget -q -N ${kfreebsd_ftpsite}/dists/${dist}/Contents-${arch}.gz) +done + diff --git a/cron.d/100syncarchive_volatile b/cron.d/100syncarchive_volatile new file mode 100755 index 0000000..ed7a030 --- /dev/null +++ b/cron.d/100syncarchive_volatile @@ -0,0 +1,30 @@ +#!/bin/bash + +. `dirname $0`/../config.sh + +test -d ${archivedir} || mkdir -p ${archivedir} +cd ${archivedir} + +# volatile archive +# +for dist in stable +do + foo=\$arch_${dist//-/_} + for arch in `eval echo $foo` + do + for part in ${parts} + do + echo retrieve Packages $dist/$part/volatile/$arch + test -d volatile/${dist}/${part}/binary-${arch} || mkdir -p volatile/${dist}/${part}/binary-${arch} + (cd volatile/${dist}/${part}/binary-${arch} && + wget -q -N ${volatile_ftpsite}/dists/${dist}/volatile/${part}/binary-${arch}/Packages.gz) + done + done + for part in ${parts} + do + echo retrieve Sources $dist/$part/volatile + test -d volatile/${dist}/${part}/source || mkdir -p volatile/${dist}/${part}/source + (cd volatile/${dist}/${part}/source && + wget -q -N ${volatile_ftpsite}/dists/${dist}/volatile/${part}/source/Sources.gz) + done +done diff --git a/lib/Packages/Search.pm b/lib/Packages/Search.pm index ae8cee3..31fe9d7 100644 --- a/lib/Packages/Search.pm +++ b/lib/Packages/Search.pm @@ -342,11 +342,11 @@ sub read_entry { my ($hash, $key, $results, $opts) = @_; my $result = $hash->{$key} || ''; foreach (split /\000/, $result) { - my @data = split ( /\s/, $_, 7 ); + my @data = split ( /\s/, $_, 8 ); debug( "Considering entry ".join( ':', @data), 2); - if ($opts->{h_suites}{$data[0]} - && ($opts->{h_archs}{$data[1]} || $data[1] eq 'all') - && $opts->{h_sections}{$data[2]}) { + if ($opts->{h_archives}{$data[0]} && $opts->{h_suites}{$data[1]} + && ($opts->{h_archs}{$data[2]} || $data[2] eq 'all') + && $opts->{h_sections}{$data[3]}) { debug( "Using entry ".join( ':', @data), 2); push @$results, [ $key, @data ]; } @@ -356,9 +356,11 @@ sub read_src_entry { my ($hash, $key, $results, $opts) = @_; my $result = $hash->{$key} || ''; foreach (split /\000/, $result) { - my @data = split ( /\s/, $_, 5 ); + my @data = split ( /\s/, $_, 6 ); debug( "Considering entry ".join( ':', @data), 2); - if ($opts->{h_suites}{$data[0]} && $opts->{h_sections}{$data[1]}) { + if ($opts->{h_archives}{$data[0]} + && $opts->{h_suites}{$data[1]} + && $opts->{h_sections}{$data[2]}) { debug( "Using entry ".join( ':', @data), 2); push @$results, [ $key, @data ]; } @@ -446,15 +448,16 @@ sub do_fulltext_search { } sub find_binaries { - my ($pkg, $suite, $src2bin) = @_; + my ($pkg, $archive, $suite, $src2bin) = @_; my $bins = $src2bin->{$pkg} || ''; my %bins; foreach (split /\000/o, $bins) { - my @data = split /\s/, $_, 4; + my @data = split /\s/, $_, 5; - if ($data[0] eq $suite) { - $bins{$data[1]}++; + if (($data[0] eq $archive) + && ($data[1] eq $suite)) { + $bins{$data[2]}++; } } -- 2.39.2