X-Git-Url: https://git.deb.at/w?a=blobdiff_plain;ds=sidebyside;f=bin%2Fcreate_index_pages;h=3e9fc529f63ff74fe27dcc2590370e1e3c0c297c;hb=6c89d8070adba05c8b39f529fba4fef517db4b80;hp=d76bebab3f58e4cf04b65174a6c26e7ec3353f02;hpb=3129ca0e45e5c4e9243c695a98cdae1d3e732c81;p=deb%2Fpackages.git diff --git a/bin/create_index_pages b/bin/create_index_pages index d76beba..3e9fc52 100755 --- a/bin/create_index_pages +++ b/bin/create_index_pages @@ -9,12 +9,12 @@ use DB_File; use Storable; use HTML::Entities; use URI::Escape; -use Locale::gettext; use Compress::Zlib; use lib './lib'; -use Packages::Config qw( $TOPDIR $DBDIR @ARCHIVES @SUITES @LANGUAGES $LOCALES); +use Packages::CommonCode qw(:all); +use Packages::Config qw( $TOPDIR $DBDIR @ARCHIVES @SUITES @LANGUAGES ); use Packages::Template; use Packages::I18N::Locale; use Packages::Page; @@ -22,15 +22,7 @@ use Packages::SrcPage; use Packages::Sections; &Packages::Config::init( './' ); -delete $ENV{'LANGUAGE'}; -delete $ENV{'LANG'}; -delete $ENV{'LC_ALL'}; -delete $ENV{'LC_MESSAGES'}; -bindtextdomain ( 'pdo', $LOCALES ); -bindtextdomain ( 'templates', $LOCALES ); -bindtextdomain ( 'sections', $LOCALES ); -bindtextdomain ( 'langs', $LOCALES ); -textdomain( 'pdo' ); +use constant DEBUG => 0; my $wwwdir = "$TOPDIR/www"; @@ -43,70 +35,206 @@ tie my %src_packages, 'DB_File', "$DBDIR/sources_small.db", tie my %src2bin, 'DB_File', "$DBDIR/sources_packages.db", O_RDONLY, 0666, $DB_BTREE or die "couldn't open $DBDIR/sources_packages.db: $!"; +tie my %desctrans, 'DB_File', "$DBDIR/descriptions_translated.db", + O_RDONLY, 0666, $DB_BTREE + or die "couldn't tie DB $DBDIR/descriptions_translated.db: $!"; my $sections = retrieve "$DBDIR/sections.info"; my $subsections = retrieve "$DBDIR/subsections.info"; my $priorities = retrieve "$DBDIR/priorities.info"; -#use Data::Dumper; +use Data::Dumper; #print STDERR Dumper($sections, $subsections, $priorities); -my (%pages); +my @PACKAGES = sort keys %packages; +my @SRC_PACKAGES = sort keys %src_packages; + +print "Found ".scalar(@PACKAGES)." packages\n"; +print "Found ".scalar(@SRC_PACKAGES)." source packages\n"; my $template = new Packages::Template( "$TOPDIR/templates", 'html'); my $txt_template = new Packages::Template( "$TOPDIR/templates", 'txt'); -print "write suite index files ...\n"; -foreach my $s (@SUITES) { - my $key = $s; - mkpath ( "$wwwdir/$key" ); - mkpath ( "$wwwdir/source/$key" ); +my $charset = 'UTF-8'; +my (%cat, %lang_vars, $prov_string, %s, %fh); +foreach my $lang (@LANGUAGES) { + $cat{$lang} = Packages::I18N::Locale->get_handle($lang) + or die "get_handle failed for $lang"; + $lang_vars{$lang} = { po_lang => $lang, ddtp_lang => $lang, + charset => $charset, + cat => $cat{$lang}, used_langs => \@LANGUAGES }; + $s{begin}{$lang} = '['.uc($lang).':'; + $s{end}{$lang} = ':'.uc($lang).']'; + $prov_string .= $s{begin}{$lang}.$cat{$lang}->g('virtual package provided by').$s{end}{$lang}; +} + +sub open_file { + my ($key, $vars, $file) = @_; + + $file ||= 'index'; + + print "opening $key\n"; + mkdirp ( "$wwwdir/$key" ); + open($fh{$key}, '>', + "$wwwdir/$key/$file.slices.new") + or die "Cannot open file $wwwdir/$key/$file.slices.new: $!"; + foreach my $lang (@LANGUAGES) { - my $locale = get_locale( $lang ); - my $charset = get_charset( $lang ); - setlocale ( LC_ALL, $locale ) or do { - warn "couldn't set locale ($lang/$locale)\n"; - next; - }; - print "writing $key/index (lang=$lang)...\n"; - - my %content = ( subsections => [], suite => $s, - lang => $lang, charset => $charset, - used_langs => \@LANGUAGES, suites => \@SUITES ); + print {$fh{$key}} "$s{begin}{$lang}\n"; + $template->page( 'index_head', + { %{$lang_vars{$lang}}, + %$vars }, + $fh{$key}); + print {$fh{$key}} "\n$s{end}{$lang}\n"; + } +} + +sub close_file { + my ($key, $vars, $file) = @_; + + $file ||= 'index'; + + print "closing $key\n"; + + foreach my $lang (@LANGUAGES) { + print {$fh{$key}} "\n$s{begin}{$lang}\n"; + $template->page( 'index_foot', + { %{$lang_vars{$lang}}, + %$vars }, + $fh{$key}); + print {$fh{$key}} "\n$s{end}{$lang}\n"; + } + close($fh{$key}) + or die "Cannot close file $wwwdir/$key/$file.slices.new: $!"; + + activate("$wwwdir/$key/$file.slices"); +} + + +sub open_txt_file { + my ($key, $vars, $file) = @_; + + $file ||= 'allpackages'; + my $lang = 'en'; + + print "opening $key (txt,lang=$lang)\n"; + mkdirp ( "$wwwdir/$key" ); + $fh{"$key/$lang/txt"} = gzopen("$wwwdir/$key/$file.$lang.txt.gz.new", 'wb9') + or die "Cannot open file $wwwdir/$key/$file.$lang.txt.gz.new: $!"; + + my $gztxt = $txt_template->page( 'index_head', + { %{$lang_vars{$lang}}, + %$vars }); + $fh{"$key/$lang/txt"}->gzwrite($gztxt); +} + +sub close_txt_file { + my ($key, $vars, $file) = @_; + + $file ||= 'allpackages'; + my $lang = 'en'; + + print "closing $key (txt,lang=$lang)\n"; + my $gztxt = $txt_template->page( 'index_foot', + { %{$lang_vars{$lang}}, + %$vars }); + $fh{"$key/$lang/txt"}->gzwrite($gztxt); + ($fh{"$key/$lang/txt"}->gzclose == Z_OK) or + warn("can't close text index file $wwwdir/$key/$file.$lang.txt.gz.new: " + . $fh{"$key/$lang/txt"}->gzerror); + activate("$wwwdir/$key/$file.$lang.txt.gz"); +} + + +foreach my $source (("", "source/")) { + print "write headers ...\n"; + foreach my $s (@SUITES) { + mkdirp ( "$wwwdir/$source$s" ); + my %common_vars = ( suite => $s, + is_source => $source ); + + open_file("$source$s", \%common_vars, 'allpackages'); + open_txt_file("$source$s", \%common_vars, 'allpackages'); + + foreach my $sec (keys %{$sections->{$s}}) { + open_file("$source$s/$sec", + { %common_vars, + category => { id => N_('Section'), + name => $sec }}); + } foreach my $ssec ((keys %{$subsections->{$s}}, 'virtual')) { next if $ssec eq '-'; - if ($sections_descs{$ssec}) { - push @{$content{subsections}}, { - id => $ssec, - name => dgettext( 'sections', $sections_descs{$ssec}[0] ), - desc => dgettext( 'sections', $sections_descs{$ssec}[1] ), - }; - } + open_file("$source$s/$ssec", + { %common_vars, + category => { id => N_('Subsection'), + name => $ssec }}); } + foreach my $prio (keys %{$priorities->{$s}}) { + next if $prio eq '-'; + open_file("$source$s/$prio", + { %common_vars, + category => { id => N_('Priority'), + name => $prio }}); + } + } - $template->page( 'suite_index', \%content, - "$wwwdir/$key/index.$lang.html.new"); - rename( "$wwwdir/$key/index.$lang.html.new", - "$wwwdir/$key/index.$lang.html" ); - - $content{source} = 'source'; - $template->page( 'suite_index', \%content, - "$wwwdir/source/$key/index.$lang.html.new"); - rename( "$wwwdir/source/$key/index.$lang.html.new", - "$wwwdir/source/$key/index.$lang.html" ); + if ($source) { + process_source_packages(); + } else { + process_packages(); + } + print "write footers ...\n"; + foreach my $s (@SUITES) { + my %common_vars = ( suite => $s, + is_source => $source ); + my $page_base = "$source$s/"; + close_file("$source$s", { %common_vars, + page_name => "${page_base}allpackages" }, + 'allpackages'); + close_txt_file("$source$s", { %common_vars, + page_name => "{$page_base}allpackages" }, + 'allpackages'); + + foreach my $sec (keys %{$sections->{$s}}) { + close_file("$source$s/$sec", + { %common_vars, + page_name => "$page_base$sec/", + category => { id => N_('Section'), + name => $sec }}); + } + foreach my $ssec ((keys %{$subsections->{$s}}, 'virtual')) { + next if $ssec eq '-'; + close_file("$source$s/$ssec", + { %common_vars, + page_name => "$page_base$ssec/", + category => { id => N_('Subsection'), + name => $ssec }}); + } + foreach my $prio (keys %{$priorities->{$s}}) { + next if $prio eq '-'; + close_file("$source$s/$prio", + { %common_vars, + page_name => "$page_base$prio/", + category => { id => N_('Priority'), + name => $prio }}); + } } } -setlocale( LC_ALL, 'C' ) or die "couldn't reset locale"; -print "collecting package info ...\n"; -my %allpkgs; -while (my ($pkg, $data) = each %packages) { +sub process_packages { + +print "processing package info ...\n"; +my $count = 0; +foreach my $pkg (@PACKAGES) { + warn "pkg=$pkg\n" if DEBUG; + print "$count\n" unless ++$count % 1000; + my (%pkg,%virt); - my ($virt, $p_data) = split /\000/o, $data, 2; - %virt = split /\01/o, $virt; + my ($virt, $p_data) = split /\000/o, $packages{$pkg}, 2; + %virt = split /\01/o, $virt; foreach (split /\000/o, $p_data||'') { - my @data = split ( /\s/o, $_, 8 ); + my @data = split ( /\s/o, $_, 9 ); $pkg{$data[1]} ||= new Packages::Page( $pkg ); $pkg{$data[1]}->merge_package( { package => $pkg, archive => $data[0], @@ -116,7 +244,8 @@ while (my ($pkg, $data) = each %packages) { subsection => $data[4], priority => $data[5], version => $data[6], - description => $data[7] } ); + 'description-md5' => $data[7], + description => $data[8] } ); } foreach (keys %virt) { next if $_ eq '-'; @@ -124,33 +253,115 @@ while (my ($pkg, $data) = each %packages) { $pkg{$_}->add_provided_by([split /\s+/, $virt{$_}]); } - while (my ($key, $entry) = each %pkg) { - $allpkgs{$key} ||= []; + while (my ($suite, $entry) = each %pkg) { + warn "\tsuite=$suite\n" if DEBUG; my %p = ( name => $pkg, providers => [], versions => '' ); if (my $provided_by = $entry->{provided_by}) { $p{providers} = $provided_by; - } - $p{subsection} = $p{section} = $p{archive} = $p{desc} = $p{priority} = ''; - unless ($entry->is_virtual) { + } + $p{subsection} = $p{section} = $p{archive} = + $p{desc} = $p{priority} = ''; + unless ($entry->is_virtual) { (undef, $p{versions}) = $entry->get_version_string; $p{subsection} = $entry->get_newest( 'subsection' ); $p{section} = $entry->get_newest( 'section' ); $p{archive} = $entry->get_newest( 'archive' ); $p{desc} = $entry->get_newest( 'description' ); + my $desc_md5 = $entry->get_newest( 'description-md5' ); + my $trans_desc = $desctrans{$desc_md5}; + if ($trans_desc) { + my %sdescs; + my %trans_desc = split /\000|\001/, $trans_desc; + while (my ($l, $d) = each %trans_desc) { + # filter out non-po languages + next unless exists $lang_vars{$l}; + + $d =~ s/\n.*//os; + $sdescs{$l} = $d; + } + $p{trans_desc} = \%sdescs if %sdescs; + } $p{priority} = $entry->get_newest( 'priority' ); } - push @{$allpkgs{$key}}, \%p; + + my $html = my $txt = ""; + my $id = " id='$p{name}'"; + if ($p{versions}) { + warn "\tversions=$p{versions}\n" if DEBUG; + + $html .= "\n