2 # Convert Contents.gz files into Sleepycat db files for efficient usage of
7 # Copyright (C) 2006 Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
8 # This program is free software; you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation; either version 2 of the License, or
11 # (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
18 # You should have received a copy of the GNU General Public License
19 # along with this program; if not, write to the Free Software
20 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25 my $what = $ARGV[0] ? "head -10000|" : "";
29 use Packages::Config qw( $TOPDIR $DBDIR @ARCHIVES @SUITES @ARCHITECTURES );
30 &Packages::Config::init( './' );
33 my @archives =( 'us'); #@ARCHIVES # NOT-IMPLEMENTED-YET
35 my @archs = @ARCHITECTURES;
37 for my $archive (@archives) { for my $suite (@suites) { for my $arch (@archs) {
39 my $filename = "$TOPDIR/archive/$archive/$suite/Contents-$arch.gz";
40 my $db = "$DBDIR/packages_contents_${suite}_${arch}.db";
41 next unless -f $filename;
42 my $ftime = (stat $filename)[9];
43 my $dbtime = (stat $db)[9];
44 next unless $ftime > $dbtime;
45 print "Reading $archive/$suite/$arch...\n";
47 my %packages_contents = ();
48 my %packages_contents_nr = ();
49 my %packages_contents_lastword = ();
50 my %contents_packages_reverse = ();
52 open CONT, "zcat $filename|$what";
53 while (<CONT>) {last if /^FILE/mo;}
58 print "Doing line $.\n" if $. % 10000 == 0;
60 my ($file, $value) = ($1, $2);
61 $value =~ s#[^,/]+/##og;
62 my @packages = split /,/, $value;
64 $packages_contents_nr{$_}++;
65 my $lw = $packages_contents_lastword{$_} || "\0";
67 while (substr($file,$i,1) eq substr($lw,$i++,1)) {}
70 $packages_contents{$_} .= pack "CC/a*", ($i, substr($file, $i));
71 $packages_contents_lastword{$_} = "$file\0";
73 # Searches are case-insensitive
74 $file =~ tr [A-Z] [a-z];
77 $filenames{$filename} = 1;
79 $contents_packages_reverse{reverse $file} = join "\0", @packages;
81 my %contents_packages_reverse_db;
82 tie %contents_packages_reverse_db, "DB_File", "$DBDIR/contents_packages_reverse_${suite}_${arch}.db.new",
83 O_RDWR|O_CREAT, 0666, $DB_BTREE
84 or die "Error creating DB: $!";
85 while (my ($x, $y) = each(%contents_packages_reverse)) {
86 $contents_packages_reverse_db{$x} = $y;
88 untie %contents_packages_reverse_db;
90 my %packages_contents_db;
91 tie %packages_contents_db, "DB_File", "$DBDIR/packages_contents_${suite}_${arch}.db.new",
92 O_RDWR|O_CREAT, 0666, $DB_BTREE
93 or die "Error creating DB: $!";
94 while (my ($k, $v) = each(%packages_contents)) {
95 $packages_contents_db{$k} = (pack "L", $packages_contents_nr{$k})
98 untie %packages_contents_db;
101 print "Writing databases...\n";
103 # FIXME: missing filenames due to optimising above. Need to store filenames
104 # per-suite/arch, but merge them in the end for better cached searching
105 open FILENAMES, "> $DBDIR/filenames.txt.new";
106 for (keys %filenames) {
107 print FILENAMES "$_\n";
111 rename("$DBDIR/filenames.txt.new", "$DBDIR/filenames.txt");
112 for my $archive (@archives) { for my $suite (@suites) { for my $arch (@archs) {
113 rename("$DBDIR/packages_contents_${suite}_${arch}.db.new", "$DBDIR/packages_contents_${suite}_${arch}.db");
114 rename("$DBDIR/contents_packages_reverse_${suite}_${arch}.db.new", "$DBDIR/contents_packages_reverse_${suite}_${arch}.db");