X-Git-Url: https://git.deb.at/?a=blobdiff_plain;f=cgi-bin%2Fsearch_packages.pl;h=a7184dc7d5050a4c6a61365536672fdd6f343d27;hb=1d735d4df7e110481faa67897ba4c35936998fc0;hp=da41da04cd7950306e85a179c301b8b563f5262e;hpb=cd5cf062990ad4fc82df48ff28e54d1f6db9cf34;p=deb%2Fpackages.git diff --git a/cgi-bin/search_packages.pl b/cgi-bin/search_packages.pl index da41da0..a7184dc 100755 --- a/cgi-bin/search_packages.pl +++ b/cgi-bin/search_packages.pl @@ -12,6 +12,7 @@ # see http://www.fsf.org/copyleft/gpl.html for a copy of the license use strict; +use lib '../lib'; use CGI qw( -oldstyle_urls ); use CGI::Carp qw( fatalsToBrowser ); use POSIX; @@ -20,25 +21,15 @@ use HTML::Entities; use DB_File; use Benchmark; -use lib "../lib"; - use Deb::Versions; +use Packages::Config qw( $DBDIR $ROOT $SEARCH_CGI $SEARCH_PAGE + @SUITES @SECTIONS @ARCHIVES @ARCHITECTURES ); +use Packages::CGI; +use Packages::DB; use Packages::Search qw( :all ); use Packages::HTML (); -my $thisscript = $Packages::HTML::SEARCH_CGI; -my $HOME = "http://www.debian.org"; -my $ROOT = ""; -my $SEARCHPAGE = "http://packages.debian.org/"; -my @SUITES = qw( oldstable stable testing unstable experimental ); -my @SECTIONS = qw( main contrib non-free ); -my @ARCHIVES = qw( us security installer ); -my @ARCHITECTURES = qw( alpha amd64 arm hppa hurd-i386 i386 ia64 - kfreebsd-i386 mips mipsel powerpc s390 sparc ); -my %SUITES = map { $_ => 1 } @SUITES; -my %SECTIONS = map { $_ => 1 } @SECTIONS; -my %ARCHIVES = map { $_ => 1 } @ARCHIVES; -my %ARCHITECTURES = map { $_ => 1 } @ARCHITECTURES; +&Packages::CGI::reset; $ENV{PATH} = "/bin:/usr/bin"; @@ -51,20 +42,24 @@ if ($ARGV[0] && ($ARGV[0] eq 'php')) { } my $pet0 = new Benchmark; +my $tet0 = new Benchmark; # use this to disable debugging in production mode completly my $debug_allowed = 1; my $debug = $debug_allowed && $input->param("debug"); -$debug = 0 if not defined($debug); -#$Packages::Search::debug = 1 if $debug > 1; +$debug = 0 if !defined($debug) || $debug !~ /^\d+$/o; +$Packages::CGI::debug = $debug; -# If you want, just print out a list of all of the variables and exit. -#print $input->header if $debug; -# print $input->dump; -# exit; +&Packages::Config::init( '../' ); +&Packages::DB::init(); if (my $path = $input->param('path')) { my @components = map { lc $_ } split /\//, $path; + my %SUITES = map { $_ => 1 } @SUITES; + my %SECTIONS = map { $_ => 1 } @SECTIONS; + my %ARCHIVES = map { $_ => 1 } @ARCHIVES; + my %ARCHITECTURES = map { $_ => 1 } @ARCHITECTURES; + foreach (@components) { if ($SUITES{$_}) { $input->param('suite', $_); @@ -72,26 +67,30 @@ if (my $path = $input->param('path')) { $input->param('section', $_); } elsif ($ARCHIVES{$_}) { $input->param('archive', $_); - }elsif ($ARCHITECTURES{$_}) { + } elsif ($ARCHITECTURES{$_}) { $input->param('arch', $_); + } elsif ($_ eq 'source') { + $input->param('searchon','sourcenames'); } } } my ( $format, $keyword, $case, $subword, $exact, $searchon, - @suites, @sections, @archs ); + @suites, @sections, @archives, @archs ); my %params_def = ( keywords => { default => undef, match => '^\s*([-+\@\w\/.:]+)\s*$', var => \$keyword }, - suite => { default => 'stable', match => '^(\w+)$', + suite => { default => 'stable', match => '^([\w-]+)$', alias => 'version', array => ',', var => \@suites, replace => { all => \@SUITES } }, + archive => { default => 'all', match => '^([\w-]+)$', + array => ',', var => \@archives, + replace => { all => \@ARCHIVES } }, case => { default => 'insensitive', match => '^(\w+)$', var => \$case }, -# official => { default => 0, match => '^(\w+)$' }, -# use_cache => { default => 1, match => '^(\w+)$' }, + official => { default => 0, match => '^(\w+)$' }, subword => { default => 0, match => '^(\w+)$', var => \$subword }, exact => { default => undef, match => '^(\w+)$', @@ -105,9 +104,6 @@ my %params_def = ( keywords => { default => undef, arch => { default => 'any', match => '^(\w+)$', array => ',', var => \@archs, replace => { any => \@ARCHITECTURES } }, - archive => { default => 'all', match => '^(\w+)$', - array => ',', replace => - { all => \@ARCHIVES } }, format => { default => 'html', match => '^(\w+)$', var => \$format }, ); @@ -116,68 +112,21 @@ my %params = Packages::Search::parse_params( $input, \%params_def, \%opts ); #XXX: Don't use alternative output formats yet $format = 'html'; - if ($format eq 'html') { - print $input->header; -} elsif ($format eq 'xml') { -# print $input->header( -type=>'application/rdf+xml' ); - print $input->header( -type=>'text/plain' ); -} - -my (@errors, @debug, @msgs, @hints); -sub error { - push @errors, $_[0]; -} -sub hint { - push @hints, $_[0]; -} -sub debug { - my $lvl = $_[1] || 0; - push(@debug, $_[0]) if $debug > $lvl; -} -sub msg { - push @msgs, $_[0]; -} -sub print_errors { - return unless @errors; - print '
'; - foreach (@errors) { - print "

$_

"; - } - print '
'; -} -sub print_debug { - return unless $debug && @debug; - print '
'; - print '

Debugging:

';
-    foreach (@debug) {
-	print "$_\n";
-    }
-    print '
'; - -} -sub print_hints { - return unless @hints; - print '
'; - foreach (@hints) { - print "

$_

"; - } - print '
'; -} -sub print_msgs { - foreach (@msgs) { - print "

$_

"; - } + print $input->header( -charset => 'utf-8' ); } if ($params{errors}{keywords}) { - error( "Error: keyword not valid or missing" ); + fatal_error( "keyword not valid or missing" ); +} elsif (length($keyword) < 2) { + fatal_error( "keyword too short (keywords need to have at least two characters)" ); } my $case_bool = ( $case !~ /insensitive/ ); $exact = !$subword unless defined $exact; $opts{h_suites} = { map { $_ => 1 } @suites }; $opts{h_sections} = { map { $_ => 1 } @sections }; +$opts{h_archives} = { map { $_ => 1 } @archives }; $opts{h_archs} = { map { $_ => 1 } @archs }; # for URL construction @@ -186,7 +135,7 @@ my $sections_param = join ',', @{$params{values}{section}{no_replace}}; my $archs_param = join ',', @{$params{values}{arch}{no_replace}}; # for output -my $keyword_enc = encode_entities $keyword; +my $keyword_enc = encode_entities $keyword || ''; my $searchon_enc = encode_entities $searchon; my $suites_enc = encode_entities join ', ', @{$params{values}{suite}{no_replace}}; my $sections_enc = encode_entities join ', ', @{$params{values}{section}{no_replace}}; @@ -195,198 +144,31 @@ my $pet1 = new Benchmark; my $petd = timediff($pet1, $pet0); debug( "Parameter evaluation took ".timestr($petd) ); -# read the configuration -my $topdir; -if (!open (C, "../config.sh")) { - error( "Internal Error: Cannot open configuration file." ); -} -while () { - $topdir = $1 if /^\s*topdir="?(.*)"?\s*$/; - $ROOT = $1 if /^\s*root="?(.*)"?\s*$/; -} -close (C); - -my $DBDIR = $topdir . "/files/db"; -my $search_on_sources = 0; - my $st0 = new Benchmark; my @results; -my $too_many_hits; -if ($searchon eq 'sourcenames') { - $search_on_sources = 1; -} - -sub print_header { - print Packages::HTML::header( title => 'Package Search Results' , - lang => 'en', - title_tag => 'Debian Package Search Results', - print_title_above => 1, - print_search_field => 'packages', - search_field_values => { - keywords => $keyword_enc, - searchon => $searchon, - arch => $archs_enc, - suite => $suites_enc, - section => $sections_enc, - subword => $subword, - exact => $exact, - case => $case, - }, - ); -} -sub read_entry { - my ($hash, $key, $results, $opts) = @_; - my $result = $hash->{$key} || ''; - foreach (split /\000/, $result) { - my @data = split ( /\s/, $_, 7 ); - debug( "Considering entry ".join( ':', @data), 2); - if ($opts->{h_suites}{$data[0]} - && ($opts->{h_archs}{$data[1]} || $data[1] eq 'all') - && $opts->{h_sections}{$data[2]}) { - debug( "Using entry ".join( ':', @data), 2); - push @$results, [ $key, @data ]; - } - } -} -sub read_src_entry { - my ($hash, $key, $results, $opts) = @_; - my $result = $hash->{$key} || ''; - foreach (split /\000/, $result) { - my @data = split ( /\s/, $_, 5 ); - debug( "Considering entry ".join( ':', @data), 2); - if ($opts->{h_suites}{$data[0]} && $opts->{h_sections}{$data[1]}) { - debug( "Using entry ".join( ':', @data), 2); - push @$results, [ $key, @data ]; - } - } -} -sub do_names_search { - my ($keyword, $file, $postfix_file, $read_entry, $opts) = @_; - my @results; - - $keyword = lc $keyword unless $opts->{case_bool}; - - my $obj = tie my %packages, 'DB_File', "$DBDIR/$file", O_RDONLY, 0666, $DB_BTREE - or die "couldn't tie DB $DBDIR/$file: $!"; - - if ($opts->{exact}) { - &$read_entry( \%packages, $keyword, \@results, $opts ); - } else { - my ($key, $prefixes) = ($keyword, ''); - my %pkgs; - my $p_obj = tie my %pref, 'DB_File', "$DBDIR/$postfix_file", O_RDONLY, 0666, $DB_BTREE - or die "couldn't tie postfix db $DBDIR/$postfix_file: $!"; - $p_obj->seq( $key, $prefixes, R_CURSOR ); - while (index($key, $keyword) >= 0) { - if ($prefixes =~ /^\001(\d+)/o) { - $too_many_hits += $1; - } else { - foreach (split /\000/o, $prefixes) { - $_ = '' if $_ eq '^'; - debug( "add word $_$key", 2); - $pkgs{$_.$key}++; - } - } - last if $p_obj->seq( $key, $prefixes, R_NEXT ) != 0; - last if $too_many_hits or keys %pkgs >= 100; - } - - my $no_results = keys %pkgs; - if ($too_many_hits || ($no_results >= 100)) { - $too_many_hits += $no_results; - %pkgs = ( $keyword => 1 ); - } - foreach my $pkg (sort keys %pkgs) { - &$read_entry( \%packages, $pkg, \@results, $opts ); - } - } - return \@results; -} -sub do_fulltext_search { - my ($keword, $file, $mapping, $lookup, $read_entry, $opts) = @_; - my @results; - - my @lines; - my $regex; - if ($opts->{case_bool}) { - if ($opts->{exact}) { - $regex = qr/\b\Q$keyword\E\b/o; - } else { - $regex = qr/\Q$keyword\E/o; - } +unless (@Packages::CGI::fatal_errors) { + + if ($searchon eq 'names') { + push @results, @{ do_names_search( $keyword, \%packages, + $p_obj, + \&read_entry, \%opts ) }; + } elsif ($searchon eq 'sourcenames') { + push @results, @{ do_names_search( $keyword, \%sources, + $sp_obj, + \&read_src_entry, \%opts ) }; + } elsif ($searchon eq 'contents') { + require "./search_contents.pl"; + &contents($input); } else { - if ($opts->{exact}) { - $regex = qr/\b\Q$keyword\E\b/io; - } else { - $regex = qr/\Q$keyword\E/io; - } + push @results, @{ do_names_search( $keyword, \%packages, + $p_obj, + \&read_entry, \%opts ) }; + push @results, @{ do_fulltext_search( $keyword, "$DBDIR/descriptions.txt", + \%did2pkg, + \%packages, + \&read_entry, \%opts ) }; } - - open DESC, '<', "$DBDIR/$file" - or die "couldn't open $DBDIR/$file: $!"; - while () { - $_ =~ $regex or next; - debug( "Matched line $.", 2); - push @lines, $.; - } - close DESC; - - tie my %packages, 'DB_File', "$DBDIR/$lookup", O_RDONLY, 0666, $DB_BTREE - or die "couldn't tie DB $DBDIR/$lookup: $!"; - tie my %did2pkg, 'DB_File', "$DBDIR/$mapping", O_RDONLY, 0666, $DB_BTREE - or die "couldn't tie DB $DBDIR/$mapping: $!"; - - my %tmp_results; - foreach my $l (@lines) { - my $result = $did2pkg{$l}; - foreach (split /\000/o, $result) { - my @data = split /\s/, $_, 3; - next unless $opts->{h_archs}{$data[2]}; - $tmp_results{$data[0]}++; - } - } - foreach my $pkg (keys %tmp_results) { - &$read_entry( \%packages, $pkg, \@results, $opts ); - } - return \@results; -} - -sub find_binaries { - my ($pkg, $suite) = @_; - - tie my %src2bin, 'DB_File', "$DBDIR/sources_packages.db", O_RDONLY, 0666, $DB_BTREE - or die "couldn't open $DBDIR/sources_packages.db: $!"; - - my $bins = $src2bin{$pkg} || ''; - my %bins; - foreach (split /\000/o, $bins) { - my @data = split /\s/, $_, 4; - - if ($data[0] eq $suite) { - $bins{$data[1]}++; - } - } - - return [ keys %bins ]; -} - -if ($searchon eq 'names') { - push @results, @{ do_names_search( $keyword, 'packages_small.db', - 'package_postfixes.db', - \&read_entry, \%opts ) }; -} elsif ($searchon eq 'sourcenames') { - push @results, @{ do_names_search( $keyword, 'sources_small.db', - 'source_postfixes.db', - \&read_src_entry, \%opts ) }; -} else { - push @results, @{ do_names_search( $keyword, 'packages_small.db', - 'package_postfixes.db', - \&read_entry, \%opts ) }; - push @results, @{ do_fulltext_search( $keyword, 'descriptions.txt', - 'descriptions_packages.db', - 'packages_small.db', - \&read_entry, \%opts ) }; } my $st1 = new Benchmark; @@ -401,7 +183,7 @@ if ($format eq 'html') { my $arch_wording = $archs_enc eq 'any' ? "all architectures" : "architecture(s) $archs_enc"; if (($searchon eq "names") || ($searchon eq 'sourcenames')) { - my $source_wording = $search_on_sources ? "source " : ""; + my $source_wording = ( $searchon eq 'sourcenames' ) ? "source " : ""; my $exact_wording = $exact ? "named" : "that names contain"; msg( "You have searched for ${source_wording}packages $exact_wording $keyword_enc in $suite_wording, $section_wording, and $arch_wording." ); } else { @@ -410,11 +192,11 @@ if ($format eq 'html') { } } -if ($too_many_hits) { - error( "Your search was too wide so we will only display exact matches. At least $too_many_hits results have been omitted and will not be displayed. Please consider using a longer keyword or more keywords." ); +if ($Packages::Search::too_many_hits) { + error( "Your search was too wide so we will only display exact matches. At least $Packages::Search::too_many_hits results have been omitted and will not be displayed. Please consider using a longer keyword or more keywords." ); } -if (!@results) { +if (!@Packages::CGI::fatal_errors && !@results) { if ($format eq 'html') { my $keyword_esc = uri_escape( $keyword ); my $printed = 0; @@ -425,11 +207,12 @@ if (!@results) { error( "Can't find that package." ); } else { error( "Can't find that package, at least not in that suite ". - ( $search_on_sources ? "" : " and on that architecture" ) ) + ( ( $searchon eq 'sourcenames' ) ? "" : " and on that architecture" ) ) } if ($exact) { - hint( "You have searched only for exact matches of the package name. You can try to search for package names that contain your search string." ); + $printed++; + hint( "You have searched only for exact matches of the package name. You can try to search for package names that contain your search string." ); } } else { if (($suites_enc eq 'all') @@ -441,188 +224,195 @@ if (!@results) { } unless ($subword) { - hint( "You have searched only for words exactly matching your keywords. You can try to search allowing subword matching." ); + $printed++; + hint( "You have searched only for words exactly matching your keywords. You can try to search allowing subword matching." ); } } - hint( ( @hints ? "Or you" : "You" )." can try a different search on the Packages search page." ); + hint( ( $printed ? "Or you" : "You" )." can try a different search on the Packages search page." ); } } -print_header; -print_msgs; -print_errors; -print_hints; -print_debug; - -my (%pkgs, %sect, %part, %desc, %binaries); - -unless ($search_on_sources) { - foreach (@results) { - my ($pkg_t, $suite, $arch, $section, $subsection, - $priority, $version, $desc) = @$_; - - my ($package) = $pkg_t =~ m/^(.+)/; # untaint - $pkgs{$package}{$suite}{$version}{$arch} = 1; - $sect{$package}{$suite}{$version} = $subsection; - $part{$package}{$suite}{$version} = $section unless $section eq 'main'; - - $desc{$package}{$suite}{$version} = $desc; - } - - if ($format eq 'html') { - my ($start, $end) = multipageheader( scalar keys %pkgs ); - my $count = 0; +print Packages::HTML::header( title => 'Package Search Results' , + lang => 'en', + title_tag => 'Debian Package Search Results', + print_title_above => 1, + print_search_field => 'packages', + search_field_values => { + keywords => $keyword_enc, + searchon => $searchon, + arch => $archs_enc, + suite => $suites_enc, + section => $sections_enc, + subword => $subword, + exact => $exact, + case => $case, + debug => $debug, + }, + ); +print_msgs(); +print_errors(); +print_hints(); +print_debug(); +if (@results) { + my (%pkgs, %subsect, %sect, %archives, %desc, %binaries, %provided_by); + + unless ($opts{searchon} eq 'sourcenames') { + foreach (@results) { + my ($pkg_t, $archive, $suite, $arch, $section, $subsection, + $priority, $version, $desc) = @$_; - foreach my $pkg (sort keys %pkgs) { - $count++; - next if $count < $start or $count > $end; - printf "

Package %s

\n", $pkg; - print "\n"; } - } -} else { - foreach (@results) { - my ($package, $suite, $section, $subsection, $priority, - $version) = @$_; - - $pkgs{$package}{$suite} = $version; - $sect{$package}{$suite}{source} = $subsection; - $part{$package}{$suite}{source} = $section unless $section eq 'main'; - - $binaries{$package}{$suite} = find_binaries( $package, $suite ); - } - if ($format eq 'html') { - my ($start, $end) = multipageheader( scalar keys %pkgs ); - my $count = 0; +my @pkgs = sort(keys %pkgs, keys %provided_by); + if ($opts{format} eq 'html') { + #my ($start, $end) = multipageheader( $input, scalar @pkgs, \%opts ); + print "

Found ".(scalar @pkgs)." matching packages,"; + #my $count = 0; - foreach my $pkg (sort keys %pkgs) { - $count++; - next if ($count < $start) or ($count > $end); - printf "

Source package %s

\n", $pkg; - print "\n"; } - } -} - -if ($format eq 'html') { - &printindexline( scalar keys %pkgs ); - &printfooter; -} - -exit; - -sub printindexline { - my $no_results = shift; - - my $index_line; - if ($no_results > $opts{number}) { - - $index_line = prevlink($input,\%params)." | ". - indexline( $input, \%params, $no_results)." | ". - nextlink($input,\%params, $no_results); - - print "

$index_line

"; - } -} - -sub multipageheader { - my $no_results = shift; - - my ($start, $end); - if ($opts{number} =~ /^all$/i) { - $start = 1; - $end = $no_results; - $opts{number} = $no_results; - } else { - $start = Packages::Search::start( \%params ); - $end = Packages::Search::end( \%params ); - if ($end > $no_results) { $end = $no_results; } - } - - print "

Found $no_results matching packages,"; - if ($end == $start) { - print " displaying package $end.

"; } else { - print " displaying packages $start to $end.

"; - } - - printindexline( $no_results ); - - if ($no_results > 100) { - print "

Results per page: "; - my @resperpagelinks; - for (50, 100, 200) { - if ($opts{number} == $_) { - push @resperpagelinks, $_; - } else { - push @resperpagelinks, resperpagelink($input,\%params,$_); + foreach (@results) { + my ($pkg, $archive, $suite, $section, $subsection, $priority, + $version) = @$_; + + my $real_archive = ''; + if ($archive =~ /^(security|non-US)$/) { + $real_archive = $archive; + $archive = 'us'; } + if (($real_archive eq $archive) && + $pkgs{$pkg}{$suite}{$archive} && + (version_cmp( $pkgs{$pkg}{$suite}{$archive}, $version ) >= 0)) { + next; + } + $pkgs{$pkg}{$suite}{$archive} = $version; + $subsect{$pkg}{$suite}{$archive}{source} = $subsection; + $sect{$pkg}{$suite}{$archive}{source} = $section + unless $section eq 'main'; + $archives{$pkg}{$suite}{$archive}{source} = $real_archive + if $real_archive; + + $binaries{$pkg}{$suite}{$archive} = find_binaries( $pkg, $archive, $suite, \%src2bin ); } - if ($params{values}{number}{final} =~ /^all$/i) { - push @resperpagelinks, "all"; - } else { - push @resperpagelinks, resperpagelink($input, \%params,"all"); + + if ($opts{format} eq 'html') { + #my ($start, $end) = multipageheader( $input, scalar keys %pkgs, \%opts ); + print "

Found ".(scalar keys %pkgs)." matching packages,"; + #my $count = 0; + + foreach my $pkg (sort keys %pkgs) { + #$count++; + #next if ($count < $start) or ($count > $end); + printf "

Source package %s

\n", $pkg; + print "\n"; + } } - print join( " | ", @resperpagelinks )."

"; } - return ( $start, $end ); + #printindexline( $input, scalar keys %pkgs, \%opts ); } - -sub printfooter { -print < - -
-

Packages search page

- - -END - -my $pete = new Benchmark; -my $petd = timediff($pete, $pet0); -print "Total page evaluation took ".timestr($petd)."
" +#print_results(\@results, \%opts) if @results;; +my $tet1 = new Benchmark; +my $tetd = timediff($tet1, $tet0); +print "Total page evaluation took ".timestr($tetd)."
" if $debug_allowed; -print $input->end_html; -} + +my $trailer = Packages::HTML::trailer( $ROOT ); +$trailer =~ s/LAST_MODIFIED_DATE/gmtime()/e; #FIXME +print $trailer; # vim: ts=8 sw=4