Packages::Search: Implement find_similar

[deb/packages.git] / lib / Packages / Search.pm

diff --git a/lib/Packages/Search.pm b/lib/Packages/Search.pm
index fdd3a3a4beeb5f730b76a9fbd980f509b42f6b83..021f0fd46196fad9847ce57616e3ebd47da5f802 100644 (file)
--- a/lib/Packages/Search.pm
+++ b/lib/Packages/Search.pm
@@ -58,6 +58,7 @@ our @ISA = qw( Exporter );
 our @EXPORT_OK = qw( read_entry read_entry_all read_entry_simple
                     read_src_entry read_src_entry_all find_binaries
                     do_names_search do_fulltext_search do_xapian_search
+                    find_similar
                     );
 our %EXPORT_TAGS = ( all => [ @EXPORT_OK ] );
 
@@ -196,100 +197,92 @@ sub do_names_search {
        &$read_entry( $packages, $pkg, $results, $non_results, $opts );
     }
 }
-sub do_fulltext_search {
-    my ($keywords, $file, $did2pkg, $packages, $read_entry, $opts,
+
+sub do_xapian_search {
+    my ($keywords, $db, $did2pkg, $packages, $read_entry, $opts,
        $results, $non_results) = @_;
 
 # NOTE: this needs to correspond with parse-packages!
     my @tmp;
     foreach my $keyword (@$keywords) {
-       $keyword =~ tr [A-Z] [a-z];
-       if ($opts->{exact}) {
-           $keyword = " $keyword ";
-       }
-       $keyword =~ s/[(),.-]+//og;
-       $keyword =~ s;[^a-z0-9_/+]+; ;og;
+       $keyword =~ s;[^\w/+]+; ;og;
        push @tmp, $keyword;
     }
-    my $first_keyword = shift @tmp;
-    @$keywords = @tmp;
-
-    my $numres = 0;
-    my %tmp_results;
-    # fgrep is seriously faster than using perl
-    open DESC, '-|', 'fgrep', '-n', '--', $first_keyword, $file
-       or die "couldn't open $file: $!";
-  LINE:
-    while (<DESC>) {
-       foreach my $k (@$keywords) {
-           next LINE unless /\Q$k\E/;
-       }
-       /^(\d+)/;
-       my $nr = $1;
-       debug( "Matched line $_", 2) if DEBUG;
-       my $result = $did2pkg->{$nr};
+    my $stemmer = Lingua::Stem->new();
+    my $stemmed_keywords = $stemmer->stem( @tmp );
+
+    my $db = Search::Xapian::Database->new( $db );
+    my $enq = $db->enquire( OP_OR, @$keywords, @$stemmed_keywords );
+    debug( "Xapian Query was: ".$enq->get_query()->get_description(), 1) if DEBUG;
+    my @matches = $enq->matches(0, 999);
+
+    my (@order, %tmp_results);
+    foreach my $match ( @matches ) {
+       my $id = $match->get_docid();
+       my $result = $did2pkg->{$id};
+
        foreach (split /\000/o, $result) {
            my @data = split /\s/, $_, 3;
-#          debug ("Considering $data[0], arch = $data[2]", 3) if DEBUG;
+           debug ("Considering $data[0], arch = $data[2], relevance=".$match->get_percent(), 3) if DEBUG;
 #          next unless $data[2] eq 'all' || $opts->{h_archs}{$data[2]};
 #          debug ("Ok", 3) if DEBUG;
-           $numres++ unless $tmp_results{$data[0]}++;
+           unless ($tmp_results{$data[0]}++) {
+               push @order, $data[0];
+           }
        }
-       last if $numres > 100;
+       last if @order > 100;
     }
-    close DESC;
-    $too_many_hits++ if $numres > 100;
+    undef $db;
+    $too_many_hits++ if @order > 100;
 
-    my @results;
-    foreach my $pkg (keys %tmp_results) {
+    debug ("ORDER: @order", 2) if DEBUG;
+    foreach my $pkg (@order) {
        &$read_entry( $packages, $pkg, $results, $non_results, $opts );
     }
- }
-
-sub do_xapian_search {
-    my ($keywords, $db, $did2pkg, $packages, $read_entry, $opts,
-       $results, $non_results) = @_;
+}
 
-# NOTE: this needs to correspond with parse-packages!
-    my @tmp;
-    foreach my $keyword (@$keywords) {
-       $keyword =~ tr [A-Z] [a-z];
-       if ($opts->{exact}) {
-           $keyword = " $keyword ";
-       }
-       $keyword =~ s/[(),.-]+//og;
-       $keyword =~ s;[^a-z0-9_/+]+; ;og;
-       push @tmp, $keyword;
-    }
-    my $stemmer = Lingua::Stem->new();
-    $keywords = $stemmer->stem( @tmp );
+sub find_similar {
+    my ($pkg, $db, $did2pkg) = @_;
 
     my $db = Search::Xapian::Database->new( $db );
-    my $enq = $db->enquire( OP_AND, @$keywords );
+    my $enq = $db->enquire( "P$pkg" );
     debug( "Xapian Query was: ".$enq->get_query()->get_description(), 1) if DEBUG;
-    my @matches = $enq->matches(0, 100);
+    my $first_match = ($enq->matches(0,1))[0]->get_document();
 
-    my $numres = 0;
-    my %tmp_results;
-    foreach my $match ( @matches ) {
+    my @terms;
+    my $term_it = $first_match->termlist_begin();
+    my $term_end = $first_match->termlist_end();
+
+    for ($term_it; $term_it ne $term_end; $term_it++) {
+       debug( "TERM: ".$term_it->get_termname(), 3);
+       push @terms, $term_it->get_termname();
+    }
+
+    my $rel_enq = $db->enquire( OP_OR, @terms );
+    debug( "Xapian Query was: ".$rel_enq->get_query()->get_description(), 1) if DEBUG;
+    my @rel_pkg = $rel_enq->matches(2,20);
+
+    use Data::Dumper;
+    debug(Dumper(\@rel_pkg),1);
+
+    my (@order, %tmp_results);
+    foreach my $match ( @rel_pkg ) {
        my $id = $match->get_docid();
        my $result = $did2pkg->{$id};
 
        foreach (split /\000/o, $result) {
            my @data = split /\s/, $_, 3;
-#          debug ("Considering $data[0], arch = $data[2]", 3) if DEBUG;
-#          next unless $data[2] eq 'all' || $opts->{h_archs}{$data[2]};
-#          debug ("Ok", 3) if DEBUG;
-           $numres++ unless $tmp_results{$data[0]}++;
+           debug ("Considering $data[0], arch = $data[2], relevance=".$match->get_percent(), 3) if DEBUG;
+           next if $data[0] eq $pkg;
+           unless ($tmp_results{$data[0]}++) {
+               push @order, $data[0];
+           }
        }
-       last if $numres > 100;
     }
     undef $db;
-    $too_many_hits++ if $numres > 100;
 
-    foreach my $pkg (keys %tmp_results) {
-       &$read_entry( $packages, $pkg, $results, $non_results, $opts );
-    }
+    debug ("ORDER: @order", 2) if DEBUG;
+    return @order[0..10];
 }
 
 sub find_binaries {