From: Frank Lichtenheld Date: Wed, 12 Sep 2007 20:51:18 +0000 (+0200) Subject: Merge branch 'master' of ssh://source/srv/source.djpig.de/git/packages into ubuntu... X-Git-Url: https://git.deb.at/w?a=commitdiff_plain;h=2ba8772e72ef20f2486e86194c5862212a99ccbf;hp=643c0d669ad783d14919ee7b7644ee4f05e7c0bf;p=deb%2Fpackages.git Merge branch 'master' of ssh://source/srv/source.djpig.de/git/packages into ubuntu-master Conflicts: conf/apache.conf.sed.in config.sh.sed.in lib/Packages/Search.pm --- diff --git a/BACKEND b/BACKEND index 11f1409..7c87b1c 100644 --- a/BACKEND +++ b/BACKEND @@ -1,5 +1,3 @@ -$Id$ - Most package data is available via sleepycat databases. Those have been specifically crafted for efficient lookup, and reasonable times to generate them, so that both the daily cronjobs and the actual pagerequests take only a diff --git a/URLS b/URLS index 359730d..418e033 100644 --- a/URLS +++ b/URLS @@ -1,5 +1,3 @@ -$Id$ - Proposal for URL layout $ROOT/search <- search handling diff --git a/bin/build-maintainerdb b/bin/build-maintainerdb index 1978854..b64be00 100755 --- a/bin/build-maintainerdb +++ b/bin/build-maintainerdb @@ -2,6 +2,7 @@ # build-maintainerdb - convert several Packages files to maintainer database # Copyright (c) 1998,9,2001,3,4,6 Martin Schulze +# Copyright (C) 2007 Joerg Jaspert # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,9 +16,7 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -# $Id$ +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # Todo: # . Read maintainer changes from overrides file(s), need to rub @@ -25,6 +24,11 @@ use strict; use warnings; +use DB_File; + +use lib '../lib'; + +use Deb::Versions; # read the configuration if (!open (C, "../config.sh")) { @@ -38,14 +42,61 @@ while () { close (C); my $maildomain = "packages.debian.org"; +my $ptsdomain = "packages.qa.debian.org"; -my $maintainerfile = "$topdir/archive/Maintainers"; + +my $maintainerfile = "$topdir/../ftp.root/debian/indices/Maintainers"; my $maintainerdb = "$topdir/conf/maintainer"; my $overridefile = "$topdir/conf/maintainerdb.override"; my @postcall = ( "/usr/sbin/postmap", $maintainerdb ); my $opt_verbose = 0; +my $dbdir = "$topdir/files/db"; my %maint; +my %binsrc; + +# Lets take pdo database input files for the source mapping +sub read_bin_src_mapping +{ + tie my %src_packages, 'DB_File', "$dbdir/sources_small.db", + O_RDONLY, 0666, $DB_BTREE + or die "couldn't tie DB $dbdir/sources_small.db: $!"; + tie my %src2bin, 'DB_File', "$dbdir/sources_packages.db", + O_RDONLY, 0666, $DB_BTREE + or die "couldn't open $dbdir/sources_packages.db: $!"; + + my %bin; + while (my ($source, $data) = each %src_packages) { + my @src = map { [ split(/\s+/) ] } split( /\0/, $data ); + my $newest = (sort( { version_cmp($b->[-1],$a->[-1]) } @src))[0]; + my ($section, $version) = @{$newest}[2,5]; + + my $bin_data = $src2bin{$source}; + unless ($bin_data) { + warn "no binary data found for $source\n"; + # source packages without binaries have most + # likely been succeded + next; + } + + my @bin = map { [ split(/\s+/) ] } split( /\0/, $bin_data ); + my %seen; + foreach (@bin) { + my ($pkg, $ver) = @{$_}[2,3]; + next if $seen{"$pkg/$ver"}++; # weed out multiple arches/suites faster + if ($bin{$pkg} + && ($bin{$pkg}{source} ne $source) + && (version_cmp($bin{$pkg}{version},$ver) > 0)) { + next; + } else { + $bin{$pkg} = { version => $ver, source => $source, + target => "$section" }; + $binsrc{$pkg} = $source; + } + } + $binsrc{$source} = $source; + } +} sub package_maintainer { @@ -95,20 +146,23 @@ sub read_maintainer } printf " %s: %s\n", $package, $maint{$package} if ($opt_verbose > 1); - my $pkgshort = ""; - if ($package =~ /(.*[^\d\.]+)([\d\.]*\d)$/) { - $pkgshort = $1; - $maint{$pkgshort} = $maint{$package} if (! exists $maint{$pkgshort}); - printf " %s: %s\n", $pkgshort, $maint{$package} if ($opt_verbose > 1); - } - if ($maint{$package} =~ /([^\@]+)\@(master\.)?debian\.org/) { - my $addrsave = $maint{$package} if ($opt_verbose > 1); - $maint{$package} = package_maintainer ($package, $1, $maint{$package}); - printf " Changed to %s\n", $maint{$package} if ($opt_verbose > 1 && ($addrsave ne $maint{$package})); - if (length ($pkgshort) > 0) { - $maint{$pkgshort} = package_maintainer ($pkgshort, $1, $maint{$pkgshort}); - } - } + +# Short what? Whats this supposed to do, except creating new (and 99% broken) something -> email mappings? +# my $pkgshort = ""; +# if ($package =~ /(.*[^\d\.]+)([\d\.]*\d)$/) { +# $pkgshort = $1; +# print "Short what? $pkgshort (long is $package)\n"; +# $maint{$pkgshort} = $maint{$package} if (! exists $maint{$pkgshort}); +# printf " %s: %s\n", $pkgshort, $maint{$package} if ($opt_verbose > 1); +# } +# if ($maint{$package} =~ /([^\@]+)\@(master\.)?debian\.org/) { +# my $addrsave = $maint{$package} if ($opt_verbose > 1); +# $maint{$package} = package_maintainer ($package, $1, $maint{$package}); +# printf " Changed to %s\n", $maint{$package} if ($opt_verbose > 1 && ($addrsave ne $maint{$package})); +# if (length ($pkgshort) > 0) { +# $maint{$pkgshort} = package_maintainer ($pkgshort, $1, $maint{$pkgshort}); +# } +# } } else { printf "Skipping double $package\n" if ($opt_verbose); printf "LINE: $_" if ($opt_verbose > 2); @@ -117,15 +171,27 @@ sub read_maintainer close (F); } + + sub write_maintainer { my $file = shift; printf "Writing to %s.new\n", $file if ($opt_verbose > 0); open (CONF, ">$file.new") || die "Can't open $file.new, $!"; + my $forward; foreach my $package (sort(keys(%maint))) { - printf "%s -> %s\n", $package, $maint{$package} if ($opt_verbose > 1); - printf CONF "%s@%s\t%s\n", $package, $maildomain, $maint{$package}; +# It is possible that we do not know a source package -> in that case fall back to old behaviour +# and only mail the maintainer. Can happen when pdo doesnt know the suite the source is in, like +# it doesnt know oldstable-proposed-updates at date of writing this code. + $forward = "$maint{$package}"; + if ($binsrc{$package}) { + $forward .= ", $binsrc{$package}_contact\@$ptsdomain"; + } + printf "%s -> %s\n", $package, $forward if ($opt_verbose); + printf CONF "%s@%s\t%s\n", $package, $maildomain, $forward; +# printf "%s -> %s and pts: %s\n", $package, $maint{$package}, $binsrc{$package} if ($opt_verbose); +# printf CONF "%s@%s\t%s, %s_contact@%s\n", $package, $maildomain, $maint{$package}, $binsrc{$package}, $ptsdomain; } close (CONF); printf "Renaming to %s\n", $file if ($opt_verbose > 0); @@ -162,6 +228,8 @@ while ($#ARGV > -1) { shift; } +&read_bin_src_mapping(); + &read_maintainer ($overridefile); &read_maintainer ($maintainerfile); diff --git a/bin/create_index_pages b/bin/create_index_pages index 2c62033..30962ea 100755 --- a/bin/create_index_pages +++ b/bin/create_index_pages @@ -44,8 +44,6 @@ tie my %src2bin, 'DB_File', "$DBDIR/sources_packages.db", my $sections = retrieve "$DBDIR/sections.info"; my $subsections = retrieve "$DBDIR/subsections.info"; -# work around problems with non-US security updates -$subsections->{oldstable}{us}{'non-US'}++; my $priorities = retrieve "$DBDIR/priorities.info"; #use Data::Dumper; diff --git a/bin/daily b/bin/daily deleted file mode 100755 index 0a67587..0000000 --- a/bin/daily +++ /dev/null @@ -1,53 +0,0 @@ -#! /bin/bash - -# Copyright (c) 2004 Martin Schulze -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -# Should run after the mirror pulse. The mirror tends to finish by -# the ctime of rsync.log.* in ~archvsync. - -source config.sh -files=$topdir/files -logs=$files/logs -log=$logs/cron.log - -test -d $files || mkdir -p $files -test -d $logs || mkdir -p $logs - -if [ -s $log ] -then - savelog -c 14 $log > /dev/null -fi - -date > $log - -if lockfile -600 -r5 "${files}/cron.lock" >> $log 2>&1 -then - run-parts --verbose $topdir/cron.d >> $log 2>&1 - rm -f "${files}/cron.lock" -else - echo "couldn't aquire cron lock file in due time" >> $log -fi -date >> $log - -if lockfile -600 -r5 "${files}/changelogs.lock" >> $log 2>&1 -then - $topdir/bin/extract_changelogs >> $log 2>&1 - rm -f "${files}/changelogs.lock" -else - echo "couldn't aquire changelogs lock file in due time" >> $log -fi -date >> $log diff --git a/bin/daily.sed.in b/bin/daily.sed.in new file mode 100755 index 0000000..28c8333 --- /dev/null +++ b/bin/daily.sed.in @@ -0,0 +1,55 @@ +#! /bin/bash + +# Copyright (c) 2004 Martin Schulze +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# Should run after the mirror pulse. The mirror tends to finish by +# the ctime of rsync.log.* in ~archvsync. + +source %TOPDIR%/config.sh +files=$topdir/files +logs=$files/logs +log=$logs/cron.log + +test -d $files || mkdir -p $files +test -d $logs || mkdir -p $logs + +if [ -s $log ] +then + savelog -c 14 $log > /dev/null +fi + +exec >"$log" 2>&1 + +date + +if lockfile -300 -r2 "${files}/cron.lock" +then + run-parts --verbose $topdir/cron.d + rm -f "${files}/cron.lock" +else + echo "couldn't aquire cron lock file in due time" +fi +date + +if lockfile -300 -r2 "${files}/changelogs.lock" +then + $topdir/bin/extract_changelogs + rm -f "${files}/changelogs.lock" +else + echo "couldn't aquire changelogs lock file in due time" +fi +date diff --git a/bin/extract_changelogs b/bin/extract_changelogs index 08be087..7d6f3e4 100755 --- a/bin/extract_changelogs +++ b/bin/extract_changelogs @@ -26,21 +26,29 @@ fi ( date +successful=1 for part in $parts; do time "${bindir}/extract_files" -v -d "${localdir}/pool/$part/" \ -t "${htmldir}/changelogs/pool/$part/" \ -c "$configdir" -w "$tmpdir" \ --dumpfile "${filesdir}/changelogs.$part.dump" \ - --cachefile "${filesdir}/changelogs.cache" + --cachefile "${filesdir}/changelogs.cache" --debug + if [ $? -ne 0 ]; then + echo program exited with non-zero exitcode: $? >&2 + successful=0 + fi done -find "${htmldir}/changelogs/" -name log -cmin +7200 \ - | while read logfile; do - dir=$(dirname "$logfile") +if [ "$successful" -eq 1 ]; then + find "${htmldir}/changelogs/" -name log -cmin +14440 \ + | while read logfile; do + dir=$(dirname "$logfile") echo deleting $dir rm -r "$dir" rmdir --ignore-fail-on-non-empty $(dirname "$dir") -done - + done +else + echo clean-up skipped because at least one of the extraction runs failed >&2 +fi date ) > $log 2>&1 diff --git a/bin/extract_files b/bin/extract_files index 2d4dbdc..3bfc559 100755 --- a/bin/extract_files +++ b/bin/extract_files @@ -1,7 +1,7 @@ #!/usr/bin/perl # # Script to extract files from Debian packages -# Copyright 2004 Frank Lichtenheld +# Copyright 2004-2007 Frank Lichtenheld # # based on a shell script which was # Copyright 2003 Noel Köthe @@ -19,7 +19,7 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. use strict; use warnings; @@ -72,7 +72,15 @@ my %opthash = ( 'help' => \$help, ); -my (%src_packages, %bin_packages, %cache, %stats); +my (%src_packages, %bin_packages, %cache); + +my %stats = ( + src_pkgs => 0, + src_cache => 0, + already_extracted => 0, + bin_pkgs => 0, + bin_cache => 0, + ); Getopt::Long::config('no_getopt_compat', 'no_auto_abbrev'); @@ -126,26 +134,20 @@ sub add_log { # PACKAGE HANDLING (UNPACKING/CLEANUP) sub unpack_srcpkg { - my ( $dscname, $log ) = @_; + my ( $pkgname, $dscname, $log ) = @_; chdir( $TEMPDIR ) or do_error( "couldn't change working directory to $TEMPDIR" ); - add_log( $log, "dpkg-source -sn -x $dscname" ); - - my $out = qx/dpkg-source -sn -x "$dscname" 2>&1/; - my ($dir) = ($out =~ /(\S+)$/mo); + add_log( $log, "dpkg-source -sn -x $dscname $pkgname+source" ); - add_log( $log, "dpkg-source output: ", $out ); - -#Bug#246802 -# system("dpkg-source", "-x", $dscname ) == 0 -# or do { -# do_warning( "couldn't unpack $dscname" ); -# add_log( $log, "couldn't unpack $dscname" ); -# return; -# }; + system("dpkg-source", "-sn", "-x", $dscname, "$pkgname+source" ) == 0 + or do { + do_warning( "couldn't unpack $dscname: $!" ); + add_log( $log, "couldn't unpack $dscname: $!" ); + return; + }; - return $dir; + return "$pkgname+source"; } sub unpack_binpkg { @@ -401,7 +403,7 @@ sub extract_files { return; } - if (my $source_dir = unpack_srcpkg( $pkg_data->{dsc}, \$log )) { + if (my $source_dir = unpack_srcpkg( $pkg_data->{src_name}, $pkg_data->{dsc}, \$log )) { $source_dir = "$TEMPDIR/$source_dir"; @@ -529,7 +531,7 @@ sub read_dsc { dsc => $dscname, }; - unless( $pkg_data->{src_name} && $pkg_data->{src_version} + unless( $pkg_data->{src_name} && defined($pkg_data->{src_version}) && $pkg_data->{dsc} ) { use Data::Dumper; do_error( "something fishy happened.\n", Dumper( $pkg_data ) ); diff --git a/bin/newpkg_info b/bin/newpkg_info index e2f36ae..37b7290 100755 --- a/bin/newpkg_info +++ b/bin/newpkg_info @@ -62,11 +62,15 @@ for (my $age = 0; $age < 7; $age++) { $changes{$_} = -1; } - my %archives = map { $_ => 1 } qw( us security non-US ); + my %archives = map { $_ => 1 } qw( us security ); foreach (sort keys %changes) { - my $entry = read_entry_simple( \%packages, $_, \%archives, $suite) - or die "Fatal Error: Can't find entry for package $_\n"; - shift @$entry; # remove virtual pkg info + my $entry = []; + if ($changes{$_} == 1) { + $entry = read_entry_simple( \%packages, $_, \%archives, $suite); + die "Fatal Error: Can't find entry for package $_\n" + unless @$entry; + shift @$entry; # remove virtual pkg info + } print CHANGES join(" ", $_, $age, @$entry)."\n"; print "Wrote entry: ".join(" ", $_, $age, @$entry)."\n"; } diff --git a/bin/parse-contents b/bin/parse-contents index 767aad4..36c90b1 100755 --- a/bin/parse-contents +++ b/bin/parse-contents @@ -2,8 +2,6 @@ # Convert Contents.gz files into Sleepycat db files for efficient usage of # data # -# $Id$ -# # Copyright (C) 2006 Jeroen van Wolffelaar # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,7 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. use strict; use lib './lib'; diff --git a/bin/parse-debtags-voc b/bin/parse-debtags-voc index 4caf0b0..85649a6 100755 --- a/bin/parse-debtags-voc +++ b/bin/parse-debtags-voc @@ -1,8 +1,6 @@ #!/usr/bin/perl -w # Convert Debtags vocabulary.gz files into Sleepycat db files # -# $Id: parse-packages 227 2006-11-12 20:24:48Z djpig $ -# # Copyright (C) 2006 Frank Lichtenheld # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,7 +14,7 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. use strict; use warnings; diff --git a/bin/parse-packages b/bin/parse-packages index b0883a2..6e27c3a 100755 --- a/bin/parse-packages +++ b/bin/parse-packages @@ -2,9 +2,9 @@ # Convert Packages.gz files into Sleepycat db files for efficient usage of # data # -# $Id$ -# # Copyright (C) 2006 Jeroen van Wolffelaar +# Copyright (C) 2006-2007 Frank Lichtenheld +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. use strict; use warnings; @@ -149,8 +149,6 @@ for my $suite (@SUITES) { my $subsection = $data{section} || '-'; if ($data{section} && ($data{section} =~ m=/=o)) { ($section, $subsection) = split m=/=o, $data{section}, 2; - ($subsection, $section) = split m=/=o, $data{section}, 2 - if $section eq 'non-US'; } $data{'section'} = $section; $data{'subsection'} = $subsection; @@ -269,6 +267,7 @@ print "Index $#descriptions descriptions\n"; for (my $i=1; $i<= $#descriptions; $i++) { # strip away additional data my ($only_desc, $pkg, $tags) = split /\000/o, $descriptions[$i], 3; + my $orig_desc = $only_desc; # WARNING: This needs to correspond with what happens in # Packages/Search.pm:do_xapian_search $only_desc =~ s#[^\w/+]+# #og; @@ -316,7 +315,7 @@ for (my $i=1; $i<= $#descriptions; $i++) { }; die $@ if $@; - $descriptions_db{$i} = $only_desc; + $descriptions_db{$i} = $orig_desc; } untie %descriptions_db; $xapian_db->flush; diff --git a/bin/parse-sources b/bin/parse-sources index 8ec35db..da49d6a 100755 --- a/bin/parse-sources +++ b/bin/parse-sources @@ -2,8 +2,6 @@ # Convert Sources.gz files into Sleepycat db files for efficient usage of # data # -# $Id$ -# # Copyright (C) 2006 Jeroen van Wolffelaar # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,7 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. use strict; use lib './lib'; @@ -77,8 +75,6 @@ for my $archive (@ARCHIVES) { my $subsection = $data{section} || '-'; if ($data{section} && ($data{section} =~ m=/=o)) { ($section, $subsection) = split m=/=o, $data{section}, 2; - ($subsection, $section) = split m=/=o, $data{section}, 2 - if $section eq 'non-US'; } $data{'section'} = $section; $data{'subsection'} = $subsection; diff --git a/bin/parse-translations b/bin/parse-translations index fa9855d..0958204 100755 --- a/bin/parse-translations +++ b/bin/parse-translations @@ -2,9 +2,8 @@ # Convert Translation.gz files into Sleepycat db files for efficient usage of # data # -# $Id$ -# # Copyright (C) 2006 Jeroen van Wolffelaar +# Copyright (C) 2007 Frank Lichtenheld # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or @@ -17,7 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. use strict; use warnings; diff --git a/bin/setup-site b/bin/setup-site index c277ecb..a0a7839 100755 --- a/bin/setup-site +++ b/bin/setup-site @@ -16,6 +16,7 @@ do wdir=$(dirname "$file") newfile="$wdir/"$(basename "$file" .sed.in) echo -e "processing\n\t$file\n->\t$newfile" - sed -e "s#%SITE%#$site#g;s#%TOPDIR%#$topdir#g" "$file" > "$newfile" + cp -p "$file" "$newfile" + sed -i -e "s#%SITE%#$site#g;s#%TOPDIR%#$topdir#g" "$newfile" diff -u "$file" "$newfile" done diff --git a/cgi-bin/index.html b/cgi-bin/index.html index 16e0ddd..e4084a3 100644 --- a/cgi-bin/index.html +++ b/cgi-bin/index.html @@ -30,7 +30,6 @@ Section: - diff --git a/conf/apache.conf.sed.in b/conf/apache.conf.sed.in index 6384605..bbb24d9 100644 --- a/conf/apache.conf.sed.in +++ b/conf/apache.conf.sed.in @@ -74,6 +74,9 @@ # RSS stuff AddType application/rss+xml .rdf .xml + + RemoveType .gz + AddEncoding x-gzip .gz @@ -88,6 +91,7 @@ # PerlModule ModPerl::Registry # PerlTaintCheck On # PerlRequire %TOPDIR%/bin/mod_perl-startup +# PerlSwitches -T # SetHandler perl-script # mod-perl etch @@ -127,6 +131,14 @@ RewriteRule ^/([^/+]*)([+])([^/]*)$ "/$1%%{%}2B$3" [N] RewriteRule ^/changelog:(.+)$ http://%SITE%/changelogs/${changelog-url:$1} [R,L,NE] RewriteRule ^/src:([^/]+)$ /source/$1 [R,L,NE] + RewriteRule ^/file:(.+)$ http://%SITE%/search?searchon=contents&keywords=$1 [R,L,NE] + + RewriteCond %{QUERY_STRING} ="" + RewriteCond %TOPDIR%/www%{REQUEST_FILENAME} -f [OR] + RewriteCond %TOPDIR%/www%{REQUEST_FILENAME}.en.html -f [OR] + RewriteCond %TOPDIR%/www%{REQUEST_FILENAME} -l [OR] + RewriteCond %TOPDIR%/www%{REQUEST_FILENAME} -d + RewriteRule . - [L] RewriteRule ^/(.+)$ /cgi-bin/dispatcher.pl/$1 [L,PT] diff --git a/config.sh.sed.in b/config.sh.sed.in index 155e8b8..1be02c8 100644 --- a/config.sh.sed.in +++ b/config.sh.sed.in @@ -20,26 +20,26 @@ cachedir=${topdir}/cache # unset this if %SITE% moves somewhere where the packages files # cannot be obtained locally # -localdir=/org/ftp.debian.org/debian +#localdir=/org/ftp.debian.org/debian # path to private ftp directory #ftproot=/org/ftp.root ftpsite=http://archive.ubuntu.com/ubuntu security_ftpsite=$ftpsite +ddtp_ftpsite=http://ddtp.debian.net/debian root="" # Architectures # polangs="de fi nl fr uk" -ddtplangs="de cs da eo es fi fr hu it ja nl pl pt_BR pt_PT ru sk sv_SE uk" +ddtplangs="ca cs da de eo es fi fr go hu it ja km_KH ko nl pl pt pt_BR ru sk sv uk zh zh_CN zh_TW" archives="us security" sections="main multiverse restricted universe" parts="${sections}" suites="warty hoary hoary-backports breezy breezy-backports dapper dapper-backports edgy edgy-backports feisty" priorities="required important standard optional extra" -dists="${suites}" architectures="i386 amd64 powerpc" arch_warty="i386 amd64 powerpc" arch_warty_updates="${arch_warty}" diff --git a/cron.d/010git b/cron.d/010git index 7a18d6d..fd91aab 100755 --- a/cron.d/010git +++ b/cron.d/010git @@ -15,7 +15,7 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. . `dirname $0`/../config.sh diff --git a/cron.d/050checkinst b/cron.d/050checkinst index 7f054cb..fff0088 100755 --- a/cron.d/050checkinst +++ b/cron.d/050checkinst @@ -11,8 +11,9 @@ required="$required libtext-iconv-perl libhtml-template-perl" required="$required libclass-accessor-perl" required="$required liburi-perl libxml-rss-perl" required="$required libtemplate-perl libi18n-acceptlanguage-perl" +required="$required liblingua-stem-perl" -locales="de_DE.UTF-8 en_US.UTF-8 nl_NL fr_FR.UTF-8" +locales="de_DE.UTF-8 en_US.UTF-8 nl_NL fr_FR.UTF-8 uk_UA.UTF-8" packages=`tempfile` locs=`tempfile` diff --git a/cron.d/100syncarchive b/cron.d/100syncarchive index 928fefc..5cb5af4 100755 --- a/cron.d/100syncarchive +++ b/cron.d/100syncarchive @@ -51,9 +51,8 @@ else do echo "retrieve Releases(.gpg) $dist" test -d us/${dist} || mkdir -p us/${dist} - (cd us/${dist} && - rsync -t ${localdir}/dists/${dist}/Release && - rsync -t ${localdir}/dists/${dist}/Release.gpg ) + rsync -t ${localdir}/dists/${dist}/Release us/${dist}/Release + rsync -t ${localdir}/dists/${dist}/Release.gpg us/${dist}/Release.gpg foo=\$arch_${dist//-/_} for arch in `eval echo $foo` @@ -86,7 +85,7 @@ fi # security archive # -for dist in oldstable stable testing +for dist in sarge etch lenny do echo "retrieve Release(.gpg) $dist" test -d security/${dist} || mkdir -p security/${dist} diff --git a/cron.d/100syncarchive_gnuab b/cron.d/100syncarchive_gnuab index 1295c63..b42d165 100755 --- a/cron.d/100syncarchive_gnuab +++ b/cron.d/100syncarchive_gnuab @@ -9,7 +9,7 @@ cd ${archivedir} # architectures="armel kfreebsd-i386 kfreebsd-amd64" -for dist in unstable experimental +for dist in sid experimental do echo "retrieve Release(.gpg) $dist" test -d gnuab/${dist} || mkdir -p gnuab/${dist} diff --git a/cron.d/120synctrans b/cron.d/120synctrans index add4c35..115b59f 100755 --- a/cron.d/120synctrans +++ b/cron.d/120synctrans @@ -5,10 +5,14 @@ test -d ${archivedir} || mkdir -p ${archivedir} cd ${archivedir} -if [ -z "${localdir}" ] +if [ -z "${localdir}" ] || [ -n "$ddtp_ftpsite" ] then - echo using remote mirror - for dist in unstable + if [ -z "$ddtp_ftpsite" ] + then + ddtp_ftpsite="$ftpsite" + fi + echo using remote mirror $ddtp_ftpsite + for dist in sid do for part in main do @@ -17,13 +21,13 @@ then do echo retrieve translated Descs $dist/$part/$lang (cd us/${dist}/${part}/i18n && - wget -q -N ${ftpsite}/dists/${dist}/${part}/i18n/Translations-$lang.gz) + wget -q -N ${ddtp_ftpsite}/dists/${dist}/${part}/i18n/Translation-$lang.gz) done done done else echo using local mirror - for dist in unstable + for dist in sid do for part in main do diff --git a/cron.d/200process_archive b/cron.d/200process_archive index 6fa394e..b8f6a6d 100755 --- a/cron.d/200process_archive +++ b/cron.d/200process_archive @@ -9,7 +9,7 @@ date date ./bin/parse-sources date -./bin/parse-contents -date ./bin/parse-translations date +./bin/parse-contents +date diff --git a/cron.d/600prepare_newpkg b/cron.d/600prepare_newpkg index 083d72e..bddf65b 100755 --- a/cron.d/600prepare_newpkg +++ b/cron.d/600prepare_newpkg @@ -5,11 +5,11 @@ packagesdir=${filesdir}/packages test -d ${packagesdir} || mkdir -p ${packagesdir} -LANG=C cat ${filesdir}/db/package_names_unstable.txt \ - | sort | uniq > ${packagesdir}/package_names_unstable.$(date -I) +LANG=C cat ${filesdir}/db/package_names_sid.txt \ + | sort | uniq > ${packagesdir}/package_names_sid.$(date -I) LANG=C cat ${filesdir}/db/source_names_unstable.txt \ - | sort | uniq > ${packagesdir}/source_names_unstable.$(date -I) + | sort | uniq > ${packagesdir}/source_names_sid.$(date -I) cd $topdir -./bin/newpkg_info unstable +./bin/newpkg_info sid diff --git a/lib/Deb/Versions.pm b/lib/Deb/Versions.pm index 366af76..d8d8282 100644 --- a/lib/Deb/Versions.pm +++ b/lib/Deb/Versions.pm @@ -1,6 +1,5 @@ # # Deb::Versions -# $Id$ # # Copyright 2003, 2004 Frank Lichtenheld # @@ -16,7 +15,7 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # =head1 NAME @@ -142,9 +141,9 @@ sub _lcmp { for ( my $i = 0; $i <= length( $v1 ); $i++ ) { my ( $n1, $n2 ) = ( ord( substr( $v1, $i, 1 ) ), ord( substr( $v2, $i, 1 ) ) ); - $n1 += 256 if $n1 < 65; # letters sort earlier than non-letters + $n1 += 256 if $n1 && $n1 < 65; # letters sort earlier than non-letters $n1 = -1 if $n1 == 126; # '~' sorts earlier than everything else - $n2 += 256 if $n2 < 65; + $n2 += 256 if $n2 && $n2 < 65; $n2 = -1 if $n2 == 126; if ( my $r = ($n1 <=> $n2) ) { return $r; @@ -154,9 +153,9 @@ sub _lcmp { } our @SUITES_SORT = qw( woody oldstable sarge stable stable-proposed-updates - etch etch-m68k testing testing-proposed-updates sid unstable - experimental warty hoary hoary-backports breezy - breezy-backports dapper ); + etch etch-m68k testing testing-proposed-updates lenny + sid unstable experimental + warty hoary breezy breezy dapper edgy feisty gutsy ); our @ARCHIVE_SORT = qw( non-US security updates volatile backports ); our @PRIORITY_SORT = qw( required important standard optional extra ); my $i = 1000; diff --git a/lib/Packages/Dispatcher.pm b/lib/Packages/Dispatcher.pm index 3dcab38..a360ddf 100755 --- a/lib/Packages/Dispatcher.pm +++ b/lib/Packages/Dispatcher.pm @@ -15,7 +15,7 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. package Packages::Dispatcher; @@ -25,6 +25,7 @@ use warnings; use CGI; use POSIX; use File::Basename; +use URI; use URI::Escape; use HTML::Entities; use Template; @@ -65,6 +66,13 @@ sub do_dispatch { delete $ENV{'LC_ALL'}; delete $ENV{'LC_MESSAGES'}; + my %SUITES_ALIAS = ( oldstable => 'sarge', + stable => 'etch', + testing => 'lenny', + unstable => 'sid', + '3.1' => 'sarge', + '4.0' => 'etch' ); + # Read in all the variables set by the form my $input; if ($ARGV[0] && ($ARGV[0] eq 'php')) { @@ -168,10 +176,6 @@ sub do_dispatch { } my %SUITES = map { $_ => 1 } @SUITES; - my %SUITES_ALIAS = ( sarge => 'oldstable', - etch => 'stable', - lenny => 'testing', - sid => 'unstable', ); my %SECTIONS = map { $_ => 1 } @SECTIONS; my %ARCHIVES = map { $_ => 1 } @ARCHIVES; my %ARCHITECTURES = map { $_ => 1 } (@ARCHITECTURES, 'all', 'any'); @@ -202,8 +206,6 @@ sub do_dispatch { set_param_once( $input, \%params_set, 'archive', $_); } elsif (!$need_pkg && $sections_descs{$_}) { set_param_once( $input, \%params_set, 'subsection', $_); - } elsif (!$need_pkg && ($_ eq 'non-us')) { # non-US hack - set_param_once( $input, \%params_set, 'subsection', 'non-US'); } elsif (!$need_pkg && ($_ eq 'source')) { set_param_once( $input, \%params_set, 'source', 1); } elsif ($ARCHITECTURES{$_}) { @@ -223,8 +225,9 @@ sub do_dispatch { } # else if (@components == 1) if (@components) { - $input->param( 'keywords', $components[0] ); - $input->param( 'package', $components[0] ); + my $c = uri_unescape($components[0]); + $input->param( 'keywords', $c ); + $input->param( 'package', $c ); } } @@ -240,7 +243,8 @@ sub do_dispatch { suite => { default => 'default', match => '^([\w-]+)$', array => ',', var => \@suites, replace => { all => \@SUITES, - default => \@SUITES } }, + default => \@SUITES, + %SUITES_ALIAS } }, archive => { default => ($what_to_do eq 'search') ? 'all' : 'default', match => '^([\w-]+)$', @@ -266,7 +270,7 @@ sub do_dispatch { array => ',', var => \@archs, replace => { any => \@ARCHITECTURES } }, format => { default => 'html', match => '^([\w.]+)$', }, - mode => { default => undef, match => '^(\w+)$', }, + mode => { default => '', match => '^(\w+)$', }, sort_by => { default => 'file', match => '^(\w+)$', }, ); my %opts; @@ -311,7 +315,9 @@ Packages::CGI::init_url( $input, \%params, \%opts ); debug => ( DEBUG ? $opts{debug} : 0 ) }, ( $CACHEDIR ? { COMPILE_DIR => $CACHEDIR } : {} ) ); - unless (-e "$TEMPLATEDIR/$opts{format}/${what_to_do}.tmpl") { + #FIXME: ugly hack + unless (($what_to_do eq 'allpackages' and $opts{format} =~ /^(html|txt\.gz)/) + || -e "$TEMPLATEDIR/$opts{format}/${what_to_do}.tmpl") { fatal_error( "requested format not available for this document", "406 requested format not available"); } @@ -327,6 +333,11 @@ Packages::CGI::init_url( $input, \%params, \%opts ); $page_content{make_search_url} = sub { return &Packages::CGI::make_search_url(@_) }; $page_content{make_url} = sub { return &Packages::CGI::make_url(@_) }; + $page_content{extract_host} = sub { my $uri = URI->new($_[0]); + my $host = $uri->host; + $host .= ':'.$uri->port if $uri->port != $uri->default_port; + return $host; + }; # needed to work around the limitations of the the FILTER syntax $page_content{html_encode} = sub { return HTML::Entities::encode_entities(@_,'<>&"') }; $page_content{uri_escape} = sub { return URI::Escape::uri_escape(@_) }; diff --git a/lib/Packages/DoDownload.pm b/lib/Packages/DoDownload.pm index 7c59272..bc7dbab 100644 --- a/lib/Packages/DoDownload.pm +++ b/lib/Packages/DoDownload.pm @@ -85,9 +85,11 @@ sub do_download { $page_content->{archive} = $archive; $page_content->{suite} = $suite; $page_content->{pkg} = $pkg; - $page_content->{pkgsize} = sprintf( '%.1f', floor(($data{size}/102.4)+0.5)/10 ); - $page_content->{pkgsize_unit} = _g( 'kByte' ); - if ($page_content->{pkgsize} > 1024) { + my $pkgsize = floor(($data{size}/102.4)+0.5)/10; + if ($pkgsize < 1024) { + $page_content->{pkgsize} = sprintf( '%.1f', $pkgsize ); + $page_content->{pkgsize_unit} = _g( 'kByte' ); + } else { $page_content->{pkgsize} = sprintf( '%.1f', floor(($data{size}/(102.4*102.4))+0.5)/100 ); $page_content->{pkgsize_unit} = _g( 'MByte' ); } diff --git a/lib/Packages/DoFilelist.pm b/lib/Packages/DoFilelist.pm index c806190..f76efb9 100644 --- a/lib/Packages/DoFilelist.pm +++ b/lib/Packages/DoFilelist.pm @@ -51,7 +51,7 @@ sub do_filelist { fatal_error( _g( "No such package in this suite on this architecture." ) ); } else { my @files = unpack "L/(CC/a)", $contents{$pkg}; - my $file; + my $file = ''; $page_content->{files} = []; for (my $i=0; $i{priority}}; # we don't have translated index pages for subsections yet $opts->{lang} = 'en' if @{$opts->{subsection}} or $file eq 'allpackages'; - $path .= "$file.$opts->{lang}.$opts->{format}"; + + #FIXME: ugly hack + if ($opts->{lang} ne 'en' and !-f "$wwwdir/$path$file.$opts->{lang}.$opts->{format}") { + $opts->{lang} = 'en'; + } + $path .= "$file.$opts->{lang}.$opts->{format}"; unless (@Packages::CGI::fatal_errors) { my $buffer; @@ -55,6 +60,9 @@ sub send_file { $headers{'-charset'} = get_charset( $opts->{lang} ); $headers{'-type'} = get_mime( $opts->{format}, 'text/plain' ); $headers{'-content-encoding'} = $encoding{$opts->{format}} if exists $encoding{$opts->{format}}; + my ($size,$mtime) = (stat("$wwwdir/$path"))[7,9]; + $headers{'-content-length'} = $size; + $headers{'-last-modified'} = gmtime($mtime); print header( %headers ); binmode INDEX; diff --git a/lib/Packages/DoNewPkg.pm b/lib/Packages/DoNewPkg.pm index c9e0ca2..60e6876 100644 --- a/lib/Packages/DoNewPkg.pm +++ b/lib/Packages/DoNewPkg.pm @@ -30,7 +30,7 @@ sub do_newpkg { my $sort_func = sub { $_[0][0] cmp $_[1][0] }; $sort_func = sub { $_[0][1] <=> $_[1][1] or $_[0][0] cmp $_[1][0] } - if $opts->{mode} eq 'byage'; + if $opts->{mode} eq 'byage'; my $suite = $opts->{suite}[0]; my $one_archive = @{$opts->{archive}} == 1 ? @@ -46,6 +46,7 @@ sub do_newpkg { chomp; my @data = split /\s/, $_, 10; + next unless $data[2]; #removed packages next unless $data[3] eq $suite; next if $one_archive and $data[2] ne $one_archive; next if $one_section and $data[5] ne $one_section; diff --git a/lib/Packages/DoSearch.pm b/lib/Packages/DoSearch.pm index 5f91d24..c1b6dfd 100644 --- a/lib/Packages/DoSearch.pm +++ b/lib/Packages/DoSearch.pm @@ -123,7 +123,7 @@ sub do_search { $version) = @$_; my $real_archive = ''; - if ($archive =~ /^(security|non-US)$/) { + if ($archive eq 'security') { $real_archive = $archive; $archive = 'us'; } diff --git a/lib/Packages/DoSearchContents.pm b/lib/Packages/DoSearchContents.pm index 29b7977..7054bb9 100644 --- a/lib/Packages/DoSearchContents.pm +++ b/lib/Packages/DoSearchContents.pm @@ -14,7 +14,7 @@ our @EXPORT = qw( do_search_contents ); use Deb::Versions; use Packages::I18N::Locale; use Packages::Search qw( :all ); -use Packages::CGI; +use Packages::CGI qw( :DEFAULT error ); use Packages::DB; use Packages::Config qw( $DBDIR @SUITES @ARCHIVES @ARCHITECTURES $ROOT ); @@ -34,7 +34,7 @@ sub do_search_contents { #FIXME: that's extremely hacky atm if ($params->{values}{suite}{no_replace}[0] eq 'default') { $params->{values}{suite}{no_replace} = - $params->{values}{suite}{final} = $opts->{suite} = [ 'stable' ]; + $params->{values}{suite}{final} = $opts->{suite} = [ 'etch' ]; } if (@{$opts->{suite}} > 1) { @@ -82,6 +82,7 @@ sub do_search_contents { if @keywords; my $kw = reverse $first_kw; + $kw =~ s{/+$}{}; # exact filename searching follows trivially: $kw = "$kw/" if $mode eq 'exactfilename'; diff --git a/lib/Packages/DoShow.pm b/lib/Packages/DoShow.pm index 1d21462..30158c5 100644 --- a/lib/Packages/DoShow.pm +++ b/lib/Packages/DoShow.pm @@ -120,9 +120,13 @@ sub do_show { my $std = timediff($st1, $st0); debug( "Data search and merging took ".timestr($std) ) if DEBUG; + my @similar = find_similar( $pkg, "$DBDIR/xapian/", + \%did2pkg ); + $contents{similar} = \@similar; + my $did = $page->get_newest( 'description' ); my $desc_md5 = $page->get_newest( 'description-md5' ); - my @complete_tags = split(/, /, $page->get_newest( 'tag' )); + my @complete_tags = split(/, /, $page->get_newest( 'tag' )||'' ); my @tags; foreach (@complete_tags) { my ($facet, $tag) = split( /::/, $_, 2); @@ -302,8 +306,7 @@ sub do_show { foreach( @$source_files ) { my ($src_file_md5, $src_file_size, $src_file_name) = split /\s+/, $_; - (my $server = lc $archive) =~ s/-//go; # non-US hack - $server = $FTP_SITES{$server} + my $server = $FTP_SITES{lc $archive} || $FTP_SITES{us}; my $path = "/$source_dir/$src_file_name"; @@ -357,9 +360,7 @@ sub moreinfo { foreach( @$files ) { my ($src_file_md5, $src_file_size, $src_file_name) = split /\s/o, $_; my ($server, $path); - # non-US hack - ($server = lc $page->get_newest('archive')) =~ s/-//go; - $server = $env->{$server}||$env->{us}; + $server = $env->{lc $page->get_newest('archive')}||$env->{us}; $path = "/$src_dir/$src_file_name"; push @downloads, { name => $src_file_name, server => $server, path => $path }; } @@ -372,7 +373,6 @@ sub moreinfo { (my $src_basename = $source_version) =~ s,^\d+:,,; # strip epoche $src_basename = "${source}_$src_basename"; $src_dir =~ s,pool/updates,pool,o; - $src_dir =~ s,pool/non-US,pool,o; $contents->{files}{changelog}{path} = "$src_dir/$src_basename/changelog"; $contents->{files}{copyright}{path} = "$src_dir/$src_basename/".( $is_source ? 'copyright' : "$name.copyright" ); diff --git a/lib/Packages/I18N/LanguageNames.pm b/lib/Packages/I18N/LanguageNames.pm index d414e47..8798f49 100644 --- a/lib/Packages/I18N/LanguageNames.pm +++ b/lib/Packages/I18N/LanguageNames.pm @@ -1,5 +1,3 @@ -# $Id$ -# # Taken from the webwml CVS tree (english/templates/language_names.wml) package Packages::I18N::LanguageNames; diff --git a/lib/Packages/I18N/Languages.pm b/lib/Packages/I18N/Languages.pm index 9f30453..f13ac2f 100644 --- a/lib/Packages/I18N/Languages.pm +++ b/lib/Packages/I18N/Languages.pm @@ -1,5 +1,3 @@ -# $Id$ -# # Taken from the webwml CVS tree (english/templates/languages.wml) package Packages::I18N::Languages; diff --git a/lib/Packages/Page.pm b/lib/Packages/Page.pm index 96ee156..946a5a4 100644 --- a/lib/Packages/Page.pm +++ b/lib/Packages/Page.pm @@ -95,7 +95,7 @@ sub is_virtual { } our @TAKE_NEWEST = qw( description description-md5 essential priority section subsection tag - archive source source-version url ); + archive source source-version url homepage ); our @STORE_ALL = qw( version source source-version installed-size size filename md5sum sha1 sha256 task origin bugs suite archive section ); diff --git a/lib/Packages/Search.pm b/lib/Packages/Search.pm index ad43390..3a20498 100644 --- a/lib/Packages/Search.pm +++ b/lib/Packages/Search.pm @@ -1,8 +1,8 @@ # # Packages::Search # -# Copyright (C) 2004-2006 Frank Lichtenheld -# +# Copyright (C) 2004-2007 Frank Lichtenheld +# # The code is based on the old search_packages.pl script that # was: # @@ -23,7 +23,7 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # =head1 NAME @@ -58,6 +58,7 @@ our @ISA = qw( Exporter ); our @EXPORT_OK = qw( read_entry read_entry_all read_entry_simple read_src_entry read_src_entry_all find_binaries do_names_search do_fulltext_search do_xapian_search + find_similar ); our %EXPORT_TAGS = ( all => [ @EXPORT_OK ] ); @@ -81,7 +82,7 @@ sub read_entry_all { } } - foreach (split /\000/o, $result) { + foreach (split(/\000/o, $result||'')) { my @data = split ( /\s/o, $_, 8 ); debug( "Considering entry ".join( ':', @data), 2) if DEBUG; if ($opts->{h_suites}{$data[1]} @@ -199,7 +200,7 @@ sub do_names_search { } sub do_xapian_search { - my ($keywords, $db, $did2pkg, $packages, $read_entry, $opts, + my ($keywords, $dbpath, $did2pkg, $packages, $read_entry, $opts, $results, $non_results) = @_; # NOTE: this needs to correspond with parse-packages! @@ -211,7 +212,7 @@ sub do_xapian_search { my $stemmer = Lingua::Stem->new(); my $stemmed_keywords = $stemmer->stem( @tmp ); - my $db = Search::Xapian::Database->new( $db ); + my $db = Search::Xapian::Database->new( $dbpath ); my $enq = $db->enquire( OP_OR, @$keywords, @$stemmed_keywords ); debug( "Xapian Query was: ".$enq->get_query()->get_description(), 1) if DEBUG; my @matches = $enq->matches(0, 999); @@ -241,6 +242,50 @@ sub do_xapian_search { } } +sub find_similar { + my ($pkg, $dbpath, $did2pkg) = @_; + + my $db = Search::Xapian::Database->new( $dbpath ); + my $enq = $db->enquire( "P$pkg" ); + debug( "Xapian Query was: ".$enq->get_query()->get_description(), 1) if DEBUG; + my $first_match = ($enq->matches(0,1))[0]->get_document(); + + my @terms; + my $term_it = $first_match->termlist_begin(); + my $term_end = $first_match->termlist_end(); + + for (; $term_it ne $term_end; $term_it++) { + debug( "TERM: ".$term_it->get_termname(), 3); + push @terms, $term_it->get_termname(); + } + + my $rel_enq = $db->enquire( OP_OR, @terms ); + debug( "Xapian Query was: ".$rel_enq->get_query()->get_description(), 1) if DEBUG; + my @rel_pkg = $rel_enq->matches(2,20); + +# use Data::Dumper; +# debug(Dumper(\@rel_pkg),1); + + my (@order, %tmp_results); + foreach my $match ( @rel_pkg ) { + my $id = $match->get_docid(); + my $result = $did2pkg->{$id}; + + foreach (split /\000/o, $result) { + my @data = split /\s/, $_, 3; + debug ("Considering $data[0], arch = $data[2], relevance=".$match->get_percent(), 3) if DEBUG; + next if $data[0] eq $pkg; + unless ($tmp_results{$data[0]}++) { + push @order, $data[0]; + } + } + } + undef $db; + + debug ("ORDER: @order", 2) if DEBUG; + return @order[0..10]; +} + sub find_binaries { my ($pkg, $archive, $suite, $src2bin) = @_; diff --git a/lib/Packages/Sections.pm b/lib/Packages/Sections.pm index 32eb2b0..2e4fd11 100644 --- a/lib/Packages/Sections.pm +++ b/lib/Packages/Sections.pm @@ -51,8 +51,6 @@ our %sections_descs = ( N_("Daemons and clients to connect your Debian GNU/Linux system to the world.") ], news => [ N_("Newsgroups"), N_("Software to access Usenet, to set up news servers, etc.") ], - 'non-US' => [ N_("Software restricted in the U.S."), - N_("These packages probably may not be used in or distributed from the U.S. due to software patents. You should check the regulations in your country before using this software.") ], oldlibs => [ N_("Old Libraries"), N_("Old versions of libraries, kept for backward compatibility with old applications.") ], otherosfs => [ N_("Other OS\'s and file systems"), diff --git a/static/packages.css.sed.in b/static/packages.css.sed.in index 1b2defe..9e09dd8 100644 --- a/static/packages.css.sed.in +++ b/static/packages.css.sed.in @@ -1,3 +1,11 @@ +@media handheld, print, projection, screen, tv { + .nonvisual { display: none; } +} + +@media braille, embossed, speech, tty { + .visual { display: none; } +} + #hpacketsearch { display: block; padding-top: 1px; @@ -76,18 +84,22 @@ h1 { list-style-type: none; } +#pdeps table ul { + margin: 0; +} + #pdeps ul.uldep, #pdeps ul.uladep { list-style-type: disc; list-style-image: url(http://%SITE%/Pics/dep.gif); } #pdeps ul.ulrec, #pdeps ul.ulidep { - list-style-type: disc; + list-style-type: square; list-style-image: url(http://%SITE%/Pics/rec.gif); } #pdeps ul.ulsug { - list-style-type: disc; + list-style-type: circle; list-style-image: url(http://%SITE%/Pics/sug.gif); } @@ -173,7 +185,7 @@ table#pdeplegend, #pdownload table { font-family: monospace; } #pdownload p { - display: inline; + margin-left: 1em; } #pdownloadnotes { diff --git a/templates/config/archive_layout.tmpl b/templates/config/archive_layout.tmpl index 1eacb16..6e501c7 100644 --- a/templates/config/archive_layout.tmpl +++ b/templates/config/archive_layout.tmpl @@ -2,4 +2,11 @@ main_section = 'main' main_archive = 'us' standard_priority = 'optional' + + suite_aliases = { + sarge => 'oldstable', + etch => 'stable', + lenny => 'testing', + sid => 'unstable', + } %] diff --git a/templates/config/mirrors.tmpl b/templates/config/mirrors.tmpl index 4dbb61a..d80342d 100644 --- a/templates/config/mirrors.tmpl +++ b/templates/config/mirrors.tmpl @@ -18,12 +18,9 @@ mirrors = { #only amd64,i386 "ftp.debian.org/debian", "ftp.egr.msu.edu/debian", "mirrors.kernel.org/debian", - "debian.crosslink.net/debian", #not amd64 "ftp-mirror.internap.com/pub/debian", #only amd64,i386 "ftp.cerias.purdue.edu/pub/os/debian", - "ftp.lug.udel.edu/debian", "debian.lcs.mit.edu/debian", - "debian.teleglobe.net", "debian.oregonstate.edu/debian", ], europa => [ @@ -172,53 +169,6 @@ mirrors = { "ftp.citkit.ru/pub/HURD/debian", ], }, - - nonus => { - mirror_list => 'http://www.debian.org/mirror/list-non-US', - north_america => [ -# "ftp.ca.debian.org/debian-non-US", - "debian.yorku.ca/debian/non-US", - "mirror.direct.ca/linux/debian-non-US", - ], - europa => [ - "non-us.debian.org/debian-non-US", - "ftp.de.debian.org/debian-non-US", - "ftp.at.debian.org/debian-non-US", - "ftp.bg.debian.org/debian-non-US", - "ftp.cz.debian.org/debian-non-US", - "ftp.fi.debian.org/debian-non-US", - "ftp.fr.debian.org/debian-non-US", - "ftp.hr.debian.org/debian-non-US", - "ftp.hu.debian.org/debian-non-US", - "ftp.ie.debian.org/debian-non-US", - "ftp.is.debian.org/debian-non-US", - "ftp.it.debian.org/debian-non-US", - "ftp.nl.debian.org/debian-non-US", - "ftp.no.debian.org/debian-non-US", - "ftp.pl.debian.org/debian/non-US", - "ftp.si.debian.org/debian-non-US", - "ftp.es.debian.org/debian-non-US", - "ftp.se.debian.org/debian-non-US", - "ftp.tr.debian.org/debian-non-US", - "ftp.uk.debian.org/debian/non-US", - ], - australia => [ - "ftp.au.debian.org/debian-non-US", - "ftp.wa.au.debian.org/debian-non-US", - "ftp.nz.debian.org/debian-non-US", - ], - asia => [ - "ftp.jp.debian.org/debian-non-US", -# "ftp.kr.debian.org/debian-non-US", - "linux.csie.nctu.edu.tw/debian-non-US", - "debian.linux.org.tw/debian-non-US", - "linux.cdpa.nsysu.edu.tw/debian-non-US", - ], - south_america => [ - "ftp.br.debian.org/debian-non-US", - "ftp.cl.debian.org/debian-non-US", - ], - }, } ports_url = 'http://www.debian.org/ports/' diff --git a/templates/html/filelist.tmpl b/templates/html/filelist.tmpl index be74cc3..dbf9fea 100644 --- a/templates/html/filelist.tmpl +++ b/templates/html/filelist.tmpl @@ -15,3 +15,5 @@ [% file %] [% '' IF loop.last -%] [% END %] + +[%- PROCESS 'html/foot.tmpl' -%] diff --git a/templates/html/foot.tmpl b/templates/html/foot.tmpl index 4930698..64eb550 100644 --- a/templates/html/foot.tmpl +++ b/templates/html/foot.tmpl @@ -11,9 +11,9 @@ Total page evaluation took [% benchmark %]

This page is also available in the following languages:

-[% FOREACH langs %] -[% selfname %] -[%- IF transliteration %] ([% transliteration %])[% END %] +[% FOREACH l IN langs %] +[% l.selfname %] +[%- IF l.transliteration %] ([% l.transliteration %])[% END %] [% END %]

How to set the default document language

diff --git a/templates/html/head.tmpl b/templates/html/head.tmpl index 00894ce..1dbd8a8 100644 --- a/templates/html/head.tmpl +++ b/templates/html/head.tmpl @@ -53,7 +53,7 @@ package names -package contents all options diff --git a/templates/html/index.tmpl b/templates/html/index.tmpl index 11cab5c..043274a 100644 --- a/templates/html/index.tmpl +++ b/templates/html/index.tmpl @@ -31,7 +31,9 @@ [% END %] [% IF providers.size %]
[% name %]
-
virtual package provided by [% FOREACH provider IN providers %][% provider %][% END %]
+
virtual package provided by + [% FOREACH provider IN providers %][% provider %] + [%- ', ' UNLESS loop.last %][% END %]
[% IF versions.length %]
[% name %] ([% versions %])[% PROCESS markers %]
[% desc | html %]
diff --git a/templates/html/newpkg.tmpl b/templates/html/newpkg.tmpl index 016dcb5..0f36b4c 100644 --- a/templates/html/newpkg.tmpl +++ b/templates/html/newpkg.tmpl @@ -6,7 +6,12 @@ -%]

New Packages in "[% suite %]"

-

The following packages were added to suite [% suite %][% " (section $section)" IF section %] in the [% organisation %] archive during the last 7 days.

+

The following packages were added to suite [% suite %][% " (section $section)" IF section %] in the [% organisation %] archive during the last 7 days. You can also display this list +[%- IF opts.mode == 'byage' %] + sorted by name +[%- ELSE %] + sorted by age +[%- END %].

This information is also available as an RSS feed [RSS 1.0 Feed]

diff --git a/templates/html/search.tmpl b/templates/html/search.tmpl index 9eafd1e..faf582c 100644 --- a/templates/html/search.tmpl +++ b/templates/html/search.tmpl @@ -65,7 +65,7 @@ Found [% results %] matching packages.

[% END %] [% IF too_many_hits %] -[% IF opts.search != "names" %] +[% IF opts.searchon != "names" %]

Note that this only shows the best matches, sorted by relevance. If the first few packages don't match what you searched for, try using more keywords or alternative keywords.

@@ -85,8 +85,10 @@ Please consider using a longer keyword or more keywords.

[% FOREACH packages %]

Package [% pkg %]

    - [% FOREACH s IN suites %] -
  • [% s.suite %][% ' (' _ s.subsection _ ')' IF s.subsection %]: [% s.desc %] [%- IF s.section != main_section %] [[% s.section %]][% END %] + [% FOREACH s IN suites; + suite = s.suite %] +
  • [% suite %] + [%- ' (' _ suite_aliases.$suite _ ')' IF suite_aliases.$suite %][% ' (' _ s.subsection _ ')' IF s.subsection %]: [% s.desc %] [%- IF s.section != main_section %] [[% s.section %]][% END %] [% FOREACH s.versions %]
    [% version %] [%- IF archive != main_archive %] [[% archive %]][% END %]: [% architectures.join(' ') %] [% END %] diff --git a/templates/html/show.tmpl b/templates/html/show.tmpl index c4b1e90..1d9f655 100644 --- a/templates/html/show.tmpl +++ b/templates/html/show.tmpl @@ -1,6 +1,17 @@ [% PROCESS 'config/archive_layout.tmpl' %] [% PROCESS 'config/mirrors.tmpl' %] -[%- nav_arr = [ { prefix=>'Distribution:', title=>'Overview over this suite', url=>make_url('/','','source',''), name=>suite }, ]; +[%- known_vcs = [ [ 'cvs', 'CVS', 'cvs' ], + [ 'svn', 'Subversion', 'subversion' ], + [ 'bzr', 'bzr', 'bzr' ], + [ 'darcs', 'Darcs', 'darcs' ], + [ 'arch', 'arch', 'tla' ], + [ 'git', 'Git', 'git-core' ], + [ 'hg', 'Hg', 'mercurial' ], ]; + vcs_link = page.get_newest("vcs-browser"); +-%] +[%- suite_name = suite; + SET suite_name = suite_name _ ' (' _ suite_aliases.$suite _ ')' IF suite_aliases.$suite; + nav_arr = [ { prefix=>'Distribution:', title=>'Overview over this suite', url=>make_url('/','','source',''), name=>suite_name }, ]; nav_arr.push( { title => 'Source packages', url=>make_url('/'), name=>'Source' } ) IF is_source; nav_arr.push( { prefix=>'Section:', title=>'All packages in this section', url=>make_url("$subsection/"), name=>subsection } ); nav_arr.push( { prefix=>'Package:', name=>pkg } ); -%] @@ -45,7 +56,6 @@

    Package: [% pkg %] ([% versions.short %]) [% END %] [%- PROCESS marker text=archive IF archive && archive != main_archive %] -[%- PROCESS marker text=subsection IF subsection == 'non-US' and archive != 'non-US' %] [%- PROCESS marker text=section IF section && section != main_section %] [%- PROCESS marker text='essential' IF page.get_newest('essential') == 'yes' %]

    @@ -62,31 +72,24 @@
  • Developer Information (PTS)
    • [%- END %] -[% IF files %] +[% IF files && (!archive or archive == main_archive or archive == 'security') %]
  • Debian Changelog
  • Copyright File
    • [% END %] -
-[% vcs_link = page.get_newest("vcs-browser") %] -[% #FIXME: should this only be used for the PTS? - known_vcs = [ [ 'cvs', 'CVS', 'cvs' ], - [ 'svn', 'Subversion', 'subversion' ], - [ 'bzr', 'bzr', 'bzr' ], - [ 'darcs', 'Darcs', 'darcs' ], - [ 'arch', 'arch', 'tla' ], - [ 'git', 'Git', 'git-core' ], - [ 'hg', 'Hg', 'mercurial' ], ]; - FOREACH vcs IN known_vcs; +[% IF vcs_link %] +
  • Debian Source Repository +[%- FOREACH vcs IN known_vcs; vcs_id = vcs.0; vcs_name = vcs.1; vcs_pkg = vcs.2; vcs_info = page.get_newest("vcs-$vcs_id"); SET vcs_info = page.get_newest("x-vcs-$vcs_id") UNLESS vcs_info; - IF vcs_info; %] -

    Debian Package Repository (VCS: -[% vcs_name %]):
    [% vcs_info %]

    -[% IF vcs_link %] -

    Debian Package Repository (Browsable):
    [% vcs_link %]

    -[% END; END; END %] + IF vcs_info %] + ([% vcs_name %]) +[% END %] +
    • +[% END; END %] + + [% IF src %]

    Download Source Package [% src.pkg %]: @@ -102,25 +105,33 @@ [% IF maintainers.size == 1 -%]

    Maintainer:

    - [%- FOREACH maintainers -%] - [% name | html %] - (QA Page) - [%- END -%] [%- ELSE -%]

    Maintainers:

    - [%- FOREACH maintainers -%] - [%- '
      ' IF loop.first -%] +[%- END %] +[%- FOREACH maintainers -%] + [%- '' IF loop.last -%] - [%- END -%] -[%- END %] + [%- '
    ' IF loop.last -%] +[%- END -%] [% url = page.get_newest('url'); + SET url = page.get_newest('homepage') IF page.get_newest('homepage'); IF url %]

    External Resources:

    -

    Homepage: [% url | html %]

    + +[% END %] + +[% FOREACH sim IN similar %] + [% IF loop.first %] +

    Similar packages:

    + ' IF loop.last %] [% END %] @@ -206,14 +217,14 @@ Do not install it on a normal Debian system.

    Other Packages Related to [% pkg %]

    - +
    [% IF is_source %] - - + + [% ELSE %] - - - + + + [% END %]
    [adep]= build-depends[idep]= build-depends-indep
    • build-depends
    • build-depends-indep
    		[dep]= depends[rec]= recommends[sug]= suggests
    • depends
    • recommends
    • suggests
    [%- END %] @@ -223,7 +234,7 @@ Do not install it on a normal Debian system.

    [% '
  • ' UNLESS is_old_pkgs %] [% FOREACH alternatives %] [% '
    ' IF loop.first %] -
    [% IF loop.first %][[% id %]][% ELSE %]or [% END %] +
    [% IF loop.first %][% id %]:[% ELSE %]or [% END %] [% IF suite %][% name %][% ELSE %][% name %][% END %] [% ' (' _ version _ ')' IF version %] [% ' [' _ arch_str _ ']' IF arch_str %]
    @@ -309,10 +320,24 @@ Do not install it on a normal Debian system.

    [% IF loop.last -%] +[% FOREACH vcs IN known_vcs; + vcs_id = vcs.0; vcs_name = vcs.1; vcs_pkg = vcs.2; + vcs_info = page.get_newest("vcs-$vcs_id"); + SET vcs_info = page.get_newest("x-vcs-$vcs_id") UNLESS vcs_info; + IF vcs_info; %] +
    +
    Debian Package Source Repository (VCS: [% vcs_name %])
    +
    [% vcs_info %]
    +[% IF vcs_link %] +
    Debian Package Source Repository (Browsable)
    [% vcs_link %]
    +[% END %] +
    +[% END; END %] +
    • [%- END %] [% END %] - +[%# %] [%- PROCESS 'html/foot.tmpl' -%] diff --git a/templates/txt/index.tmpl b/templates/txt/index.tmpl index ab8c473..12e249c 100644 --- a/templates/txt/index.tmpl +++ b/templates/txt/index.tmpl @@ -6,12 +6,12 @@ Copyright (C) [% copyright.years %] [% copyright.name %]; See for the license terms. [% FOREACH packages -%] - [%- BLOCK marker %] [[% text %][% END -%] + [%- BLOCK marker %] [[% text %]][% END -%] [%- BLOCK markers -%] [%- PROCESS marker text=section IF section != main_section -%] [%- PROCESS marker text=archive IF archive != main_archive -%] [%- END -%] - [%- IF providers %] + [%- IF providers.size %] [% name %] virtual package provided by [% providers.join(', ') %] [%- END -%] [%- IF versions %]