2 # Convert Contents.gz files into Sleepycat db files for efficient usage of
7 # Copyright (C) 2006 Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
8 # This program is free software; you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation; either version 2 of the License, or
11 # (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
18 # You should have received a copy of the GNU General Public License
19 # along with this program; if not, write to the Free Software
20 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25 # Important, we want sorting and such to happen like in the C locale: binary,
26 # without any fancy collation. FIXME: is this actually adequate?
29 my $what = $ARGV[0] ? "head -10000|" : "";
31 # More RAM vs more disk I/O tradeoff parameters, does not change
32 # functionality. True will always use more RAM at the benefit of less
33 # temporary files, and is adviced when possible
34 my $SORT_REVERSE_CONCURRENTLY = 1;
38 use Packages::Config qw( $TOPDIR $DBDIR @ARCHIVES @SUITES @ARCHITECTURES );
39 &Packages::Config::init( './' );
41 my @archives =( 'us'); #@ARCHIVES # NOT-IMPLEMENTED-YET
43 my @archs = @ARCHITECTURES;
45 $DBDIR .= "/contents";
47 for my $archive (@archives) { for my $suite (@suites) {
49 for my $arch (@archs) {
51 my $filename = "$TOPDIR/archive/$archive/$suite/Contents-$arch.gz";
52 my $filelist_db = "$DBDIR/filelists_${suite}_${arch}.db";
53 next unless -f $filename;
54 my $ftime = (stat $filename)[10]; # Note: ctime, because mtime is set back via rsync
55 my $dbtime = (stat $filelist_db)[9];
56 next if defined $dbtime and $dbtime > $ftime;
57 print "Reading $archive/$suite/$arch...\n";
59 my %packages_contents = ();
60 my %packages_contents_nr = ();
61 my %packages_contents_lastword = ();
64 $extra = "|sort" if $SORT_REVERSE_CONCURRENTLY;
66 open REVERSED, "$extra>$DBDIR/reverse.tmp"
67 or die "Failed to open output reverse file";
69 open CONT, "zcat $filename|$what";
70 while (<CONT>) {last if /^FILE/mo;}
75 print "Doing line ".($./1000)."k (out of approx 1.5M)\n" if $. % 250000 == 0;
77 my ($file, $value) = ($1, $2);
78 $value =~ s#[^,/]+/##og;
79 my @packages = split /,/, $value;
81 $packages_contents_nr{$_}++;
82 my $lw = $packages_contents_lastword{$_} || "\0";
84 while (substr($file,$i,1) eq substr($lw,$i++,1)) {}
87 $packages_contents{$_} .= pack "CC/a*", ($i, substr($file, $i));
88 $packages_contents_lastword{$_} = "$file\0";
90 # Searches are case-insensitive
91 $file =~ tr [A-Z] [a-z];
93 print REVERSED (reverse $file)."\0".(join ":$arch\0", @packages).":$arch\n";
98 print "Sorting reverse list if needed\n";
99 system("cd $DBDIR && sort reverse.tmp > reverse.sorted && mv reverse.{sorted,tmp}") == 0
100 or die "Failed to sort reverse"
101 unless $SORT_REVERSE_CONCURRENTLY;
103 print "Writing filelist db\n";
104 tie my %packages_contents_db, "DB_File", "$filelist_db.new",
105 O_RDWR|O_CREAT, 0666, $DB_BTREE
106 or die "Error creating DB: $!";
107 while (my ($k, $v) = each(%packages_contents)) {
108 $packages_contents_db{$k} = (pack "L", $packages_contents_nr{$k})
111 untie %packages_contents_db;
113 rename("$DBDIR/reverse.tmp", "$DBDIR/reverse_${suite}_${arch}.txt");
115 rename("$filelist_db.new", $filelist_db);
116 system("ln -sf $filelist_db $DBDIR/filelists_${suite}_all.db") == 0
122 my $suite_mtime = (stat "$DBDIR/reverse_$suite.db")[9];
123 for my $file (glob "$DBDIR/reverse_${suite}_*.txt") {
124 $go = 1 if not defined $suite_mtime
125 or $suite_mtime < (stat $file)[9];
129 print "Merging reverse path lists for ${suite}...\n";
131 open MERGED, "sort -m $DBDIR/reverse_${suite}_*.txt |"
132 or die "Failed to open merged list";
133 open FILENAMES, "> $DBDIR/filenames_$suite.txt.new"
134 or die "Failed to open filenames list";
135 tie my %reverse_path_db, "DB_File", "$DBDIR/reverse_${suite}.db.new",
136 O_RDWR|O_CREAT, 0666, $DB_BTREE
137 or die "Error creating DB: $!";
143 print "Doing line ".($./1000000)."M (out of approx. 16M)\n" if $. % 1000000 == 0;
145 my @line = split /\0/o, $_;
146 my $revpath = shift @line;
147 if ($revpath ne $lastpath) {
148 # Wrap: Do useful stuff with this ($lastpath, @matches)
149 $reverse_path_db{$lastpath} = join "\0", @matches if $lastpath ne "";
150 $lastpath =~ s,/.*,,o;
151 if ($lastfile ne $lastpath) {
152 $lastfile = $lastpath;
153 print FILENAMES (reverse $lastfile)."\n";
156 $lastpath = $revpath;
162 # Note: do useful stuff here too, for out last entry. Maybe prevent this by
163 # adding a fake ultimate entry?
164 $reverse_path_db{$lastpath} = join "\0", @matches;
166 untie %reverse_path_db;
170 rename "$DBDIR/filenames_$suite.txt.new", "$DBDIR/filenames_$suite.txt";
171 rename "$DBDIR/reverse_$suite.db.new", "$DBDIR/reverse_$suite.db";