X-Git-Url: https://git.deb.at/?p=deb%2Fpackages.git;a=blobdiff_plain;f=bin%2Fparse-packages;h=6e27c3a2bae3802ccac3ba26b31df412980d194d;hp=189ea0267ea78e946f1b3def780ea1561b1c149d;hb=d833a5eed5296d1a24c35e8f0802f8291037198d;hpb=e178006b4cca636a5aa78b526e35753895b885b3 diff --git a/bin/parse-packages b/bin/parse-packages index 189ea02..6e27c3a 100755 --- a/bin/parse-packages +++ b/bin/parse-packages @@ -2,9 +2,9 @@ # Convert Packages.gz files into Sleepycat db files for efficient usage of # data # -# $Id$ -# # Copyright (C) 2006 Jeroen van Wolffelaar +# Copyright (C) 2006-2007 Frank Lichtenheld +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. use strict; use warnings; @@ -32,7 +32,10 @@ my $MAX_PACKAGE_POSTFIXES = 100; use DB_File; use Storable; use File::Path; +use Digest::MD5; use Deb::Versions; +use Lingua::Stem v0.82; +use Search::Xapian; use Packages::Config qw( $TOPDIR $DBDIR @ARCHIVES @SUITES ); &Packages::Config::init( './' ); my %packages_small = (); @@ -52,6 +55,9 @@ my %priorities = (); $/ = ""; -d $DBDIR || mkpath( $DBDIR ); +-d "$DBDIR/xapian.new" && rmtree("$DBDIR/xapian.new"); +-d "$DBDIR/xapian.old" && rmtree("$DBDIR/xapian.old"); +mkpath( "$DBDIR/xapian.new" ); for my $suite (@SUITES) { my %package_names_suite = (); @@ -81,8 +87,9 @@ for my $suite (@SUITES) { } # Skip double package next if exists($packages_all_db{"$data{'package'} $data{'architecture'} $data{'version'}"}); - # Skip arch:all for amd64&kfreebsd, too often broken - next if ($archive eq 'amd64' or $archive eq 'kfreebsd') + # Skip arch:all for amd64 & gnuab, any non-redundancy is + # usually a bug anyway + next if ($archive eq 'amd64' or $archive eq 'gnuab') and $data{architecture} eq 'all'; if ($data{'provides'}) { @@ -97,10 +104,33 @@ for my $suite (@SUITES) { my $src_version = ''; if ($data{'source'}) { $src = $data{'source'}; - $src =~ s/\s+.*//; # strip version info + $src =~ s/\s+.*//o; # strip version info } $data{'source'} = $src; - my $descr = $data{'description'}; + + # expand tags like devel::{lang:c,lang:c++} + if ($data{'tag'} && $data{'tag'} =~ /\{/) { + my @complete_tags = split(/, /, $data{'tag'}); + my @tags; + foreach (@complete_tags) { + my ($facet, $tag) = split( /::/, $_, 2); + if ($tag =~ s/^\{(.+)\}$/$1/) { + foreach (split( /,/, $tag )) { + push @tags, "${facet}::$_"; + } + } else { + push @tags, "${facet}::$tag"; + } + } + my $old = $data{tag}; + $data{'tag'} = join ", ", @tags; + } + + # we add some additional data here + my $descr = "$data{'description'}\000$data{'package'}\000" + .($data{'tag'}||''); + my $sdescr = $data{'description'}; + $sdescr =~ s/\n.*//os; my $did = undef; if (exists($descriptions{$descr})) { $did = $descriptions{$descr}; @@ -109,19 +139,16 @@ for my $suite (@SUITES) { $descriptions[$did] = $descr; $descriptions{$descr} = $did; } + $data{'description-md5'} = Digest::MD5::md5_hex($data{'description'}, "\n"); $data{'description'} = $did; $packages_descriptions{"$data{'package'} $data{'version'} $data{'architecture'}"} = $did; $descriptions_packages{$did} .= "$data{'package'} $data{'version'} $data{'architecture'}\000"; - my $sdescr = $descr; - $sdescr =~ s/\n.*//s; my $section = 'main'; my $subsection = $data{section} || '-'; if ($data{section} && ($data{section} =~ m=/=o)) { ($section, $subsection) = split m=/=o, $data{section}, 2; - ($subsection, $section) = split m=/=o, $data{section}, 2 - if $section eq 'non-US'; } $data{'section'} = $section; $data{'subsection'} = $subsection; @@ -178,7 +205,7 @@ while (my ($pkg, $v) = each(%packages_small)) { $res3 .= $v3; } } - + if (exists $virtual_packages{$pkg}) { while (my ($suite, $v2) = each %{$virtual_packages{$pkg}}) { $res1 .= "$suite\01".(join ' ', keys %$v2)."\01"; @@ -223,25 +250,76 @@ while (my ($k, $v) = each(%descriptions_packages)) { } untie %descriptions_packages_db; +my $stemmer = Lingua::Stem->new(); +$stemmer->stem_caching({ -level => 2 }); +my $xapian_db; +eval { + $xapian_db = Search::Xapian::WritableDatabase->new("$DBDIR/xapian.new", + Search::Xapian::DB_CREATE_OR_OPEN) + or die "can't create write-able db object: $!\n"; +}; +die $@ if $@; my %descriptions_db; tie %descriptions_db, "DB_File", "$DBDIR/descriptions.db.new", O_RDWR|O_CREAT, 0666, $DB_BTREE or die "Error creating DB: $!"; -open DESCR, ">", "$DBDIR/descriptions.txt" or die "Error creating descriptions textfile"; +print "Index $#descriptions descriptions\n"; for (my $i=1; $i<= $#descriptions; $i++) { - my $plain_description = $descriptions[$i]; + # strip away additional data + my ($only_desc, $pkg, $tags) = split /\000/o, $descriptions[$i], 3; + my $orig_desc = $only_desc; # WARNING: This needs to correspond with what happens in -# Packages/Search.pm:do_fulltext_search - $plain_description =~ tr [A-Z] [a-z]; - # ensure one space on both ends - $plain_description = " $plain_description "; - $plain_description =~ s/[(),.-]+//og; - $plain_description =~ s#[^a-z0-9_/+]+# #og; - print DESCR "$plain_description\n"; - $descriptions_db{$i} = $descriptions[$i]; +# Packages/Search.pm:do_xapian_search + $only_desc =~ s#[^\w/+]+# #og; + + #XAPIAN + eval { + my @words = split /\s+/, $only_desc; + unshift @words, $pkg; + + my $doc = Search::Xapian::Document->new() + or die "can't create doc object for $i: $!\n"; + if ($doc->set_data($i)){ + warn "can't set_data in doc object for $i: $!\n"; + } + + # package with prefix + if ($doc->add_term("P$pkg")) { + warn "can't add term P$pkg: $!\n"; + } + # description, unstemmed with positional info + for my $j (0 .. (@words-1)) { + next if $words[$j] =~ /^\s*$/o; + if ($doc->add_posting($words[$j], $j)) { + warn "can't add posting $words[$j] at $j: $!\n"; + } + } + # description, stemmed + my $stem_words = $stemmer->stem( \@words ); + foreach my $w (@$stem_words) { + next if $w =~ /^\s*$/o; + if ($doc->add_term($w)) { + warn "can't add term $w: $!\n"; + } + } + if ($tags) { + foreach my $t (split /, /, $tags) { + if ($doc->add_term($t)) { + warn "can't add term $t: $!\n"; + } + } + } + + $xapian_db->add_document($doc) + or warn "failed to add document: $i\n"; + }; + die $@ if $@; + + $descriptions_db{$i} = $orig_desc; } -close DESCR; untie %descriptions_db; +$xapian_db->flush; +undef $xapian_db; # package names stuff: for my $pkg (keys %package_names) { @@ -280,10 +358,12 @@ for my $suite (@SUITES) { rename("$DBDIR/package_names_$suite.txt.new", "$DBDIR/package_names_$suite.txt"); } +rename("$DBDIR/xapian", "$DBDIR/xapian.old"); +rename("$DBDIR/xapian.new","$DBDIR/xapian"); +rmtree("$DBDIR/xapian.old"); rename("$DBDIR/packages_descriptions.db.new", "$DBDIR/packages_descriptions.db"); rename("$DBDIR/descriptions_packages.db.new", "$DBDIR/descriptions_packages.db"); -rename("$DBDIR/descriptions.txt.new", "$DBDIR/descriptions.txt"); rename("$DBDIR/descriptions.db.new", "$DBDIR/descriptions.db"); rename("$DBDIR/package_postfixes.db.new", "$DBDIR/package_postfixes.db");