Fulltext search: Greatly improve by using a more fuzzy approach

Most of this done on suggestion of Enrico Zini <enrico@enricozini.org>

Using OP_OR instead of OP_AND as default can actually lead to better
matches because the ones found with OP_AND often don't actually are
the most relevant ones. This is especially true when using more than
two keywords.

Accordingly sort by relevance on the result page.

Improve indexing:
 + Add both the unstemmed and the stemmed description to index
   this will increase the relevance of exact matches. Only the
   latter is done with positional information
 + Really index debtags and the package name

diff --git a/bin/parse-packages b/bin/parse-packages
index 64a00cdf125d897e2d039f1746dffc94ab43f15b..b0883a279da39798715b9b9a0a67eb73527ecd45 100755 (executable)
--- a/bin/parse-packages
+++ b/bin/parse-packages
@@ -267,32 +267,50 @@ tie %descriptions_db, "DB_File", "$DBDIR/descriptions.db.new",
        or die "Error creating DB: $!";
 print "Index $#descriptions descriptions\n";
 for (my $i=1; $i<= $#descriptions; $i++) {
-       my $plain_description = $descriptions[$i];
        # strip away additional data
-       my ($only_desc) = split /\000/o, $plain_description, 2;
+       my ($only_desc, $pkg, $tags) = split /\000/o, $descriptions[$i], 3;
 # WARNING: This needs to correspond with what happens in
-# Packages/Search.pm:do_fulltext_search
-       $plain_description =~ tr [A-Z] [a-z];
-       # ensure one space on both ends
-       $plain_description = " $plain_description ";
-       $plain_description =~ s/[(),.-]+//og;
-       $plain_description =~ s#[^a-z0-9_/+]+# #og;
+# Packages/Search.pm:do_xapian_search
+       $only_desc =~ s#[^\w/+]+# #og;
 
        #XAPIAN
        eval {
-           my @words = split /\s+/, $plain_description;
-           my $stem_words = $stemmer->stem( \@words );
+           my @words = split /\s+/, $only_desc;
+           unshift @words, $pkg;
+
            my $doc = Search::Xapian::Document->new()
                or die "can't create doc object for $i: $!\n";
            if ($doc->set_data($i)){
                warn "can't set_data in doc object for $i: $!\n";
            }
-           for my $j (0 .. (@$stem_words-1)) {
-               next if $stem_words->[$j] =~ /^\s*$/o;
-               if ($doc->add_posting($stem_words->[$j], $j)) {
-                   warn "can't add word $stem_words->[$j] $j: $!\n";
+
+           # package with prefix
+           if ($doc->add_term("P$pkg")) {
+               warn "can't add term P$pkg: $!\n";
+           }
+           # description, unstemmed with positional info
+           for my $j (0 .. (@words-1)) {
+               next if $words[$j] =~ /^\s*$/o;
+               if ($doc->add_posting($words[$j], $j)) {
+                   warn "can't add posting $words[$j] at $j: $!\n";
                }
            }
+           # description, stemmed
+           my $stem_words = $stemmer->stem( \@words );
+           foreach my $w (@$stem_words) {
+               next if $w =~ /^\s*$/o;
+               if ($doc->add_term($w)) {
+                   warn "can't add term $w: $!\n";
+               }
+           }
+           if ($tags) {
+               foreach my $t (split /, /, $tags) {
+                   if ($doc->add_term($t)) {
+                       warn "can't add term $t: $!\n";
+                   }
+               }
+           }
+
            $xapian_db->add_document($doc)
                or warn "failed to add document: $i\n";
        };

diff --git a/lib/Packages/DoSearch.pm b/lib/Packages/DoSearch.pm
index 5b0b4bb5e0c31a8c25e29c57dce2fdc6c6de7554..d395a310334b787775aa078162bb739b928a8d1c 100644 (file)
--- a/lib/Packages/DoSearch.pm
+++ b/lib/Packages/DoSearch.pm
@@ -79,6 +79,14 @@ sub do_search {
     if (@results) {
        my (%pkgs, %subsect, %sect, %archives, %desc, %binaries, %provided_by);
 
+       my %sort_by_relevance;
+       for (1 ... scalar @results) {
+#          debug("$results[$_][0] => $_", 4) if DEBUG;
+           $sort_by_relevance{$results[$_-1][0]} = $_;
+       }
+#      use Data::Dumper;
+#      debug( "sort_by_relevance=".Dumper(\%sort_by_relevance), 4);
+
        unless ($opts->{source}) {
            foreach (@results) {
                my ($pkg_t, $archive, $suite, $arch, $section, $subsection,
@@ -98,7 +106,12 @@ sub do_search {
            }
 
            my %uniq_pkgs = map { $_ => 1 } (keys %pkgs, keys %provided_by);
-           my @pkgs = sort keys %uniq_pkgs;
+           my @pkgs;
+           if ($searchon eq 'names') {
+               @pkgs = sort keys %uniq_pkgs;
+           } else {
+               @pkgs = sort { $sort_by_relevance{$a} <=> $sort_by_relevance{$b} } keys %uniq_pkgs;
+           }
            process_packages( $page_content, 'packages', \%pkgs, \@pkgs, $opts, \@keywords,
                              \&process_package, \%provided_by,
                              \%archives, \%sect, \%subsect,

diff --git a/lib/Packages/Search.pm b/lib/Packages/Search.pm
index 671e3401c1f75fe7eadef8f82182e7a1cb2660ae..66b09441ba1ea72d84de6578e200a488971bf099 100644 (file)
--- a/lib/Packages/Search.pm
+++ b/lib/Packages/Search.pm
@@ -204,41 +204,38 @@ sub do_xapian_search {
 # NOTE: this needs to correspond with parse-packages!
     my @tmp;
     foreach my $keyword (@$keywords) {
-       $keyword =~ tr [A-Z] [a-z];
-       if ($opts->{exact}) {
-           $keyword = " $keyword ";
-       }
-       $keyword =~ s/[(),.-]+//og;
-       $keyword =~ s;[^a-z0-9_/+]+; ;og;
+       $keyword =~ s;[^\w/+]+; ;og;
        push @tmp, $keyword;
     }
     my $stemmer = Lingua::Stem->new();
-    $keywords = $stemmer->stem( @tmp );
+    my $stemmed_keywords = $stemmer->stem( @tmp );
 
     my $db = Search::Xapian::Database->new( $db );
-    my $enq = $db->enquire( OP_AND, @$keywords );
+    my $enq = $db->enquire( OP_OR, @$keywords, @$stemmed_keywords );
     debug( "Xapian Query was: ".$enq->get_query()->get_description(), 1) if DEBUG;
-    my @matches = $enq->matches(0, 100);
+    my @matches = $enq->matches(0, 999);
 
-    my $numres = 0;
-    my %tmp_results;
+    my (@order, %tmp_results);
     foreach my $match ( @matches ) {
        my $id = $match->get_docid();
        my $result = $did2pkg->{$id};
 
        foreach (split /\000/o, $result) {
            my @data = split /\s/, $_, 3;
-#          debug ("Considering $data[0], arch = $data[2]", 3) if DEBUG;
+           debug ("Considering $data[0], arch = $data[2], relevance=".$match->get_percent(), 3) if DEBUG;
 #          next unless $data[2] eq 'all' || $opts->{h_archs}{$data[2]};
 #          debug ("Ok", 3) if DEBUG;
-           $numres++ unless $tmp_results{$data[0]}++;
+           unless ($tmp_results{$data[0]}++) {
+               push @order, $data[0];
+           }
        }
-       last if $numres > 100;
+       last if @order > 100;
     }
     undef $db;
-    $too_many_hits++ if $numres > 100;
+    $too_many_hits++ if @order > 100;
 
-    foreach my $pkg (keys %tmp_results) {
+    debug ("ORDER: @order", 2) if DEBUG;
+    foreach my $pkg (@order) {
        &$read_entry( $packages, $pkg, $results, $non_results, $opts );
     }
 }