X-Git-Url: https://git.deb.at/?p=deb%2Fpackages.git;a=blobdiff_plain;f=bin%2Fparse-packages;h=ab5fbcef5b12e357fbfcce47cea2093e14356e8c;hp=aad66889f510bb97bb5825ad80b9a5bd697969a5;hb=5ca21f1ba07a6b559395bec8d6bc1e528eb238d1;hpb=13a8ba9e1656ff1af0bcde63100907a5acb13315 diff --git a/bin/parse-packages b/bin/parse-packages index aad6688..ab5fbce 100755 --- a/bin/parse-packages +++ b/bin/parse-packages @@ -20,12 +20,15 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA use strict; +use lib './lib'; my $what = $ARGV[0] ? "non-free" : "*"; # max. distinct results for a given package postfix my $MAX_PACKAGE_POSTFIXES = 100; use DB_File; +use Packages::Config qw( $TOPDIR $DBDIR @ARCHIVES @SUITES ); +&Packages::Config::init( './' ); my %packages_small = (); my %package_names = (); my %package_postfixes = (); @@ -35,43 +38,42 @@ my @descriptions = ("we count lines one-based\000"); my %packages_descriptions = (); my %descriptions_packages = (); -my @suites = ('oldstable', 'stable', 'testing', 'unstable', 'experimental'); - $/ = ""; -for my $suite (@suites) { +for my $archive (@ARCHIVES) { + for my $suite (@SUITES) { - print "Reading $suite...\n"; + print "Reading $archive/$suite...\n"; my %packages_all_db; - tie %packages_all_db, "DB_File", "packages_all_$suite.db.new", + tie %packages_all_db, "DB_File", "$DBDIR/packages_all_$suite.db.new", O_RDWR|O_CREAT, 0666, $DB_BTREE or die "Error creating DB: $!"; - open PKG, "zcat /org/ftp.debian.org/ftp/dists/$suite/$what/{,debian-installer/}binary-*/Packages.gz|"; + open PKG, "zcat $TOPDIR/archive/$archive/$suite/$what/{,debian-installer/}binary-*/Packages.gz|"; while () { next if /^\s*$/; my $data = ""; my %data = (); chomp; s/\n /\377/g; - #s/\376\377\s*\376\377/\376\377/og; while (/^(\S+):\s*(.*)\s*$/mg) { my ($key, $value) = ($1, $2); $value =~ s/\377/\n /g; - $data .= "$key: $value\n"; $key =~ tr [A-Z] [a-z]; $data{$key} = $value; } # Skip double package next if exists($packages_all_db{"$data{'package'} $data{'architecture'} $data{'version'}"}); - $packages_all_db{"$data{'package'} $data{'architecture'} $data{'version'}"} - = $data; $package_names{$data{'package'}} = 1; my $src = $data{'package'}; + my $src_version = ''; if ($data{'source'}) { $src = $data{'source'}; - $src =~ s/ .*//; # strip version info + $src_version = $1 + if $src =~ s/\s+\(\s*=\s*(.*)\).*//; # strip version info } + $data{'source'} = $src; + $data{'source-version'} = $src_version; my $descr = $data{'description'}; my $did = undef; if (exists($descriptions{$descr})) { @@ -81,6 +83,7 @@ for my $suite (@suites) { $descriptions[$did] = $descr; $descriptions{$descr} = $did; } + $data{'description'} = $did; $packages_descriptions{"$data{'package'} $data{'version'} $data{'architecture'}"} = $did; $descriptions_packages{$did} .= "$data{'package'} $data{'version'} $data{'architecture'}\000"; @@ -88,22 +91,35 @@ for my $suite (@suites) { my $sdescr = $descr; $sdescr =~ s/\n.*//s; my $section = 'main'; - my $subsection = $data{section}; + my $subsection = $data{section} || '-'; if ($data{section} && ($data{section} =~ m=/=o)) { ($section, $subsection) = split m=/=o, $data{section}, 2; } - $packages_small{$data{'package'}} .= "$suite $data{'architecture'} ". + $data{'section'} = $section; + $data{'subsection'} = $subsection; + $data{'priority'} ||= '-'; + $packages_small{$data{'package'}} .= "$archive $suite $data{'architecture'} ". "$section $subsection $data{'priority'} $data{'version'} $sdescr\000"; $sources_packages{$src} .= - "$suite $data{'package'} $data{'version'} $data{'architecture'}\000"; + "$archive $suite $data{'package'} $data{'version'} $data{'architecture'}\000"; + $data{archive} = $archive; + while (my ($key, $value) = each (%data)) { + next if $key eq 'package' or $key eq 'version' or $key eq 'architecture' + or $key eq 'maintainer'; + print STDERR "WARN: $key ($suite/$archive/$data{package}/$data{architecture}\n" unless defined $value; + $data .= "$key: $value\n"; + } + $packages_all_db{"$data{'package'} $data{'architecture'} $data{'version'}"} + = $data; } untie %packages_all_db; + } } print "Writing databases...\n"; my %packages_small_db; -tie %packages_small_db, "DB_File", "packages_small.db.new", +tie %packages_small_db, "DB_File", "$DBDIR/packages_small.db.new", O_RDWR|O_CREAT, 0666, $DB_BTREE or die "Error creating DB: $!"; while (my ($k, $v) = each(%packages_small)) { @@ -113,7 +129,7 @@ while (my ($k, $v) = each(%packages_small)) { untie %packages_small_db; my %sources_packages_db; -tie %sources_packages_db, "DB_File", "sources_packages.db.new", +tie %sources_packages_db, "DB_File", "$DBDIR/sources_packages.db.new", O_RDWR|O_CREAT, 0666, $DB_BTREE or die "Error creating DB: $!"; while (my ($k, $v) = each(%sources_packages)) { @@ -123,7 +139,7 @@ while (my ($k, $v) = each(%sources_packages)) { untie %sources_packages_db; my %packages_descriptions_db; -tie %packages_descriptions_db, "DB_File", "packages_descriptions.db.new", +tie %packages_descriptions_db, "DB_File", "$DBDIR/packages_descriptions.db.new", O_RDWR|O_CREAT, 0666, $DB_BTREE or die "Error creating DB: $!"; while (my ($k, $v) = each(%packages_descriptions)) { @@ -132,7 +148,7 @@ while (my ($k, $v) = each(%packages_descriptions)) { untie %packages_descriptions_db; my %descriptions_packages_db; -tie %descriptions_packages_db, "DB_File", "descriptions_packages.db.new", +tie %descriptions_packages_db, "DB_File", "$DBDIR/descriptions_packages.db.new", O_RDWR|O_CREAT, 0666, $DB_BTREE or die "Error creating DB: $!"; while (my ($k, $v) = each(%descriptions_packages)) { @@ -142,14 +158,15 @@ while (my ($k, $v) = each(%descriptions_packages)) { untie %descriptions_packages_db; my %descriptions_db; -tie %descriptions_db, "DB_File", "descriptions.db.new", +tie %descriptions_db, "DB_File", "$DBDIR/descriptions.db.new", O_RDWR|O_CREAT, 0666, $DB_BTREE or die "Error creating DB: $!"; -open DESCR, "> descriptions.txt" or die "Error creating descriptions textfile"; +open DESCR, ">", "$DBDIR/descriptions.txt" or die "Error creating descriptions textfile"; for (my $i=1; $i<= $#descriptions; $i++) { - my $null_d = $descriptions[$i]; - $null_d =~ s/\n/\000/g; - print DESCR "$null_d\n"; + my $plain_description = $descriptions[$i]; + $plain_description =~ s/\n .\n/ /og; + $plain_description =~ s/[\n \t]+/ /og; + print DESCR "$plain_description\n"; $descriptions_db{$i} = $descriptions[$i]; } close DESCR; @@ -160,11 +177,12 @@ for my $pkg (keys %package_names) { for (my $i=0;$i