From: Jeroen van Wolffelaar Date: Wed, 8 Feb 2006 20:06:00 +0000 (+0000) Subject: Generate contents indices for all suites and archs X-Git-Tag: switch-to-templates~151 X-Git-Url: https://git.deb.at/w?a=commitdiff_plain;h=22fe3171e7730755a48a3b854a8b7643ff2ea0c2;p=deb%2Fpackages.git Generate contents indices for all suites and archs --- diff --git a/bin/parse-contents b/bin/parse-contents index ceb0cb0..b3caaef 100755 --- a/bin/parse-contents +++ b/bin/parse-contents @@ -22,33 +22,43 @@ use strict; use lib './lib'; -my $what = $ARGV[0] ? "head -100000|" : ""; +my $what = $ARGV[0] ? "head -10000|" : ""; use DB_File; use Storable; -use Packages::Config qw( $TOPDIR $DBDIR @ARCHIVES @SUITES ); +use Packages::Config qw( $TOPDIR $DBDIR @ARCHIVES @SUITES @ARCHITECTURES ); &Packages::Config::init( './' ); -my %packages_contents = (); -my %packages_contents_nr = (); -my %packages_contents_lastword = (); -my %file_reverse = (); +my %filenames = (); -my @archives =( 'us'); #@ARCHIVES -my @suites = ('stable');#@SUITES +my @archives =( 'us'); #@ARCHIVES # NOT-IMPLEMENTED-YET +my @suites = @SUITES; +my @archs = @ARCHITECTURES; -for my $archive (@archives) { for my $suite (@suites) { +for my $archive (@archives) { for my $suite (@suites) { for my $arch (@archs) { - print "Reading $archive/$suite/i386...\n"; - open CONT, "zcat /org/ftp.debian.org/ftp/dists/stable/Contents-i386.gz|$what"; - while (1) {$_ = ;last if /^FILE/mo;} + my $filename = "$TOPDIR/archive/$archive/$suite/Contents-$arch.gz"; + my $db = "$DBDIR/packages_contents_${suite}_${arch}.db"; + next unless -f $filename; + my $ftime = (stat $filename)[9]; + my $dbtime = (stat $db)[9]; + next unless $ftime > $dbtime; + print "Reading $archive/$suite/$arch...\n"; + + my %packages_contents = (); + my %packages_contents_nr = (); + my %packages_contents_lastword = (); + my %contents_packages_reverse = (); + + open CONT, "zcat $filename|$what"; + while () {last if /^FILE/mo;} while () { my $data = ""; my %data = (); chomp; print "Doing line $.\n" if $. % 10000 == 0; - /^(\S+)\s+(\S+)/; + /^(.+)\s+(\S+)$/; my ($file, $value) = ($1, $2); - $value =~ s#[^,/]+/##g; + $value =~ s#[^,/]+/##og; my @packages = split /,/, $value; for (@packages) { $packages_contents_nr{$_}++; @@ -62,37 +72,44 @@ for my $archive (@archives) { for my $suite (@suites) { } # Searches are case-insensitive $file =~ tr [A-Z] [a-z]; + my $filename = $file; + $filename =~ s,.*/,,; + $filenames{$filename} = 1; + + $contents_packages_reverse{reverse $file} = join "\0", @packages; + } + my %contents_packages_reverse_db; + tie %contents_packages_reverse_db, "DB_File", "$DBDIR/contents_packages_reverse_${suite}_${arch}.db.new", + O_RDWR|O_CREAT, 0666, $DB_BTREE + or die "Error creating DB: $!"; + while (my ($x, $y) = each(%contents_packages_reverse)) { + $contents_packages_reverse_db{$x} = $y; + } + untie %contents_packages_reverse_db; - $file_reverse{reverse $file} = join "\0", @packages; + my %packages_contents_db; + tie %packages_contents_db, "DB_File", "$DBDIR/packages_contents_${suite}_${arch}.db.new", + O_RDWR|O_CREAT, 0666, $DB_BTREE + or die "Error creating DB: $!"; + while (my ($k, $v) = each(%packages_contents)) { + $packages_contents_db{$k} = (pack "L", $packages_contents_nr{$k}) + . $v; } -}} + untie %packages_contents_db; +}}} print "Writing databases...\n"; -my %packages_contents_db; -tie %packages_contents_db, "DB_File", "$DBDIR/packages_contents.db.new", - O_RDWR|O_CREAT, 0666, $DB_BTREE - or die "Error creating DB: $!"; -while (my ($k, $v) = each(%packages_contents)) { - $packages_contents_db{$k} = (pack "L", $packages_contents_nr{$k}) - . $v; -} -untie %packages_contents_db; -my %file_reverse_db; -tie %file_reverse_db, "DB_File", "$DBDIR/file_reverse.db.new", - O_RDWR|O_CREAT, 0666, $DB_BTREE - or die "Error creating DB: $!"; -while (my ($x, $y) = each(%file_reverse)) { -# $v =~ s/.$//s; -# my $nr = $v; -# $nr =~ s/[^\000]//g; -# $nr = length($nr) + 1; # < number of hits -# if ($nr > $MAX_file_reverse) { -# $v = "\001" . $nr; -# } - $file_reverse_db{$x} = $y; +# FIXME: missing filenames due to optimising above. Need to store filenames +# per-suite/arch, but merge them in the end for better cached searching +open FILENAMES, "> $DBDIR/filenames.txt.new"; +for (keys %filenames) { + print FILENAMES "$_\n"; } -untie %file_reverse_db; +close FILENAMES; -rename("$DBDIR/packages_contents.db.new", "$DBDIR/packages_contents.db"); -rename("$DBDIR/file_reverse.db.new", "$DBDIR/file_reverse.db"); +rename("$DBDIR/filenames.txt.new", "$DBDIR/filenames.txt"); +for my $archive (@archives) { for my $suite (@suites) { for my $arch (@archs) { + rename("$DBDIR/packages_contents_${suite}_${arch}.db.new", "$DBDIR/packages_contents_${suite}_${arch}.db"); + rename("$DBDIR/contents_packages_reverse_${suite}_${arch}.db.new", "$DBDIR/contents_packages_reverse_${suite}_${arch}.db"); +}}}