bin/create_index_pages

   1 #!/usr/bin/perl
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX;
   7 use File::Path;
   8 use DB_File;
   9 use Storable;
  10 use HTML::Entities;
  11 use URI::Escape;
  12 use Locale::gettext;
  13 use Compress::Zlib;
  14
  15 use lib './lib';
  16
  17 use Packages::Config qw( $TOPDIR $DBDIR @ARCHIVES @SUITES @LANGUAGES $LOCALES);
  18 use Packages::Template;
  19 use Packages::I18N::Locale;
  20 use Packages::Page;
  21 use Packages::SrcPage;
  22 use Packages::Sections;
  23 &Packages::Config::init( './' );
  24
  25 delete $ENV{'LANGUAGE'};
  26 delete $ENV{'LANG'};
  27 delete $ENV{'LC_ALL'};
  28 delete $ENV{'LC_MESSAGES'};
  29 bindtextdomain ( 'pdo', $LOCALES );
  30 bindtextdomain ( 'templates', $LOCALES );
  31 bindtextdomain ( 'sections', $LOCALES );
  32 bindtextdomain ( 'langs', $LOCALES );
  33 textdomain( 'pdo' );
  34
  35 my $wwwdir = "$TOPDIR/www";
  36
  37 tie my %packages, 'DB_File', "$DBDIR/packages_small.db",
  38     O_RDONLY, 0666, $DB_BTREE
  39     or die "couldn't tie DB $DBDIR/packages_small.db: $!";
  40 tie my %src_packages, 'DB_File', "$DBDIR/sources_small.db",
  41     O_RDONLY, 0666, $DB_BTREE
  42     or die "couldn't tie DB $DBDIR/sources_small.db: $!";
  43 tie my %src2bin, 'DB_File', "$DBDIR/sources_packages.db",
  44     O_RDONLY, 0666, $DB_BTREE
  45     or die "couldn't open $DBDIR/sources_packages.db: $!";
  46
  47 my $sections = retrieve "$DBDIR/sections.info";
  48 my $subsections = retrieve "$DBDIR/subsections.info";
  49 my $priorities = retrieve "$DBDIR/priorities.info";
  50
  51 #use Data::Dumper;
  52 #print STDERR Dumper($sections, $subsections, $priorities);
  53
  54 my (%pages);
  55
  56 my $template = new Packages::Template( "$TOPDIR/templates", 'html');
  57 my $txt_template = new Packages::Template( "$TOPDIR/templates", 'txt');
  58
  59 print "write suite index files ...\n";
  60 foreach my $s (@SUITES) {
  61     my $key = $s;
  62     mkpath ( "$wwwdir/$key" );
  63     mkpath ( "$wwwdir/source/$key" );
  64     foreach my $lang (@LANGUAGES) {
  65         my $locale = get_locale( $lang );
  66         my $charset = get_charset( $lang );
  67         setlocale ( LC_ALL, $locale ) or do {
  68             warn "couldn't set locale ($lang/$locale)\n";
  69             next;
  70         };
  71         print "writing $key/index (lang=$lang)...\n";
  72
  73         my %content = ( subsections => [], suite => $s,
  74                         lang => $lang, charset => $charset,
  75                         used_langs => \@LANGUAGES, suites => \@SUITES );
  76         foreach my $ssec ((keys %{$subsections->{$s}}, 'virtual')) {
  77             next if $ssec eq '-';
  78             if ($sections_descs{$ssec}) {
  79                 push @{$content{subsections}}, {
  80                     id => $ssec,
  81                     name => dgettext( 'sections', $sections_descs{$ssec}[0] ),
  82                     desc => dgettext( 'sections', $sections_descs{$ssec}[1] ),
  83                 };
  84             }
  85         }
  86
  87         $template->page( 'suite_index', \%content,
  88                          "$wwwdir/$key/index.$lang.html.new");
  89         rename( "$wwwdir/$key/index.$lang.html.new",
  90                 "$wwwdir/$key/index.$lang.html" );
  91
  92         $content{source} = 'source';
  93         $template->page( 'suite_index', \%content,
  94                          "$wwwdir/source/$key/index.$lang.html.new");
  95         rename( "$wwwdir/source/$key/index.$lang.html.new",
  96                 "$wwwdir/source/$key/index.$lang.html" );
  97
  98     }
  99 }
 100 setlocale( LC_ALL, 'C' ) or die "couldn't reset locale";
 101
 102 print "collecting package info ...\n";
 103 my %allpkgs;
 104 while (my ($pkg, $data) = each %packages) {
 105     my (%pkg,%virt);
 106     my ($virt, $p_data) = split /\000/o, $data, 2;
 107     %virt = split /\01/o, $virt;
 108     foreach (split /\000/o, $p_data||'') {
 109         my @data = split ( /\s/o, $_, 8 );
 110         $pkg{$data[1]} ||= new Packages::Page( $pkg );
 111         $pkg{$data[1]}->merge_package( { package => $pkg,
 112                                          archive => $data[0],
 113                                          suite => $data[1],
 114                                          architecture => $data[2],
 115                                          section => $data[3],
 116                                          subsection => $data[4],
 117                                          priority => $data[5],
 118                                          version => $data[6],
 119                                          description => $data[7] } );
 120     }
 121     foreach (keys %virt) {
 122         next if $_ eq '-';
 123         $pkg{$_} ||= new Packages::Page( $pkg );
 124         $pkg{$_}->add_provided_by([split /\s+/, $virt{$_}]);
 125     }
 126
 127     while (my ($key, $entry) = each %pkg) {
 128         $allpkgs{$key} ||= [];
 129
 130         my %p = ( name => $pkg, providers => [], versions => '' );
 131         if (my $provided_by = $entry->{provided_by}) {
 132             $p{providers} = $provided_by;
 133         }
 134         $p{subsection} = $p{section} = $p{archive} = $p{desc} = $p{priority} = '';
 135         unless ($entry->is_virtual) {
 136             (undef, $p{versions}) = $entry->get_version_string;
 137             $p{subsection} = $entry->get_newest( 'subsection' );
 138             $p{section} = $entry->get_newest( 'section' );
 139             $p{archive} = $entry->get_newest( 'archive' );
 140             $p{desc} = $entry->get_newest( 'description' );
 141             $p{priority} = $entry->get_newest( 'priority' );
 142         }
 143         push @{$allpkgs{$key}}, \%p;
 144     }
 145 }
 146
 147 write_files(\%allpkgs);
 148
 149 print "collecting source package info ...\n";
 150 my %allsrcpkgs;
 151 while (my ($pkg, $data) = each %src_packages) {
 152     my %pkg;
 153     foreach (split /\000/o, $data||'') {
 154         my @data = split ( /\s/o, $_ );
 155         $pkg{$data[1]} ||= new Packages::SrcPage( $pkg );
 156         $pkg{$data[1]}->merge_package( { package => $pkg,
 157                                          archive => $data[0],
 158                                          suite => $data[1],
 159                                          section => $data[2],
 160                                          subsection => $data[3],
 161                                          priority => $data[4],
 162                                          version => $data[5],
 163                                          } );
 164     }
 165
 166     while (my ($key, $entry) = each %pkg) {
 167         $allsrcpkgs{$key} ||= [];
 168
 169         my %p = ( name => $pkg, providers => [], versions => '' );
 170         $p{versions} = $entry->{version};
 171         $p{subsection} = $entry->get_newest( 'subsection' );
 172         $p{section} = $entry->get_newest( 'section' );
 173         $p{archive} = $entry->get_newest( 'archive' );
 174         $p{priority} = $entry->get_newest( 'priority' );
 175
 176         $p{desc} = '';
 177         $p{binaries} = [];
 178 #       my $binaries = find_binaries( $pkg, $p{archive}, $p{suite}, \%src2bin );
 179 #       if ($binaries && @$binaries) {
 180 #           pkg_list( \%packages, $opts, $binaries, 'en', $contents{binaries} );
 181 #       }
 182
 183         push @{$allsrcpkgs{$key}}, \%p;
 184     }
 185 }
 186
 187 write_files(\%allsrcpkgs, 1);
 188
 189 sub write_files {
 190     my ($pkgs, $source) = @_;
 191
 192     $source = $source ? 'source/' : '';
 193     print "writing files ...\n";
 194     foreach my $s (@SUITES) {
 195         my $key = $s;
 196         mkpath ( "$wwwdir/$source$key" );
 197         foreach my $lang (@LANGUAGES) {
 198             my $locale = get_locale( $lang );
 199             my $charset = get_charset( $lang );
 200             setlocale ( LC_ALL, $locale ) or do {
 201                 warn "couldn't set locale ($lang/$locale)\n";
 202                 next;
 203             };
 204
 205             my %lang_vars = ( lang => $lang, charset => $charset,
 206                               used_langs => \@LANGUAGES );
 207             print "writing $source$s/allpackages (lang=$lang)...\n";
 208             $template->page( 'index', { %lang_vars, packages => $pkgs->{$key},
 209                                         suite => $s, is_source => $source  },
 210                              "$wwwdir/$source$key/allpackages.$lang.html.new" );
 211             print "writing $source$s/allpackages (txt,lang=$lang)...\n";
 212             my $gzfh = gzopen("$wwwdir/$source$key/allpackages.$lang.txt.gz.new",
 213                               'wb9')
 214                 or die "can't open text index file for output: $!";
 215             my $gztxt;
 216             $gztxt = $txt_template->page( 'index', { %lang_vars, packages => $pkgs->{$key},
 217                                                      suite => $s, is_source => $source  },
 218                                           );
 219             $gzfh->gzwrite($gztxt);
 220             ($gzfh->gzclose == Z_OK) or
 221                 warn "can't close text index file $wwwdir/$source$key/allpackages.$lang.txt.gz.new: ".$gzfh->gzerror;
 222
 223             rename( "$wwwdir/$source$key/allpackages.$lang.html.new",
 224                     "$wwwdir/$source$key/allpackages.$lang.html" );
 225             rename( "$wwwdir/$source$key/allpackages.$lang.txt.gz.new",
 226                     "$wwwdir/$source$key/allpackages.$lang.txt.gz" );
 227
 228             foreach my $sec (keys %{$sections->{$s}}) {
 229                 mkpath ( "$wwwdir/$source$key/$sec" );
 230
 231                 print "writing $source$s/$sec/index (lang=$lang)...\n";
 232                 $template->page( 'index', { packages => [ grep { $_->{section} eq $sec } @{$pkgs->{$key}} ],
 233                                             %lang_vars, suite => $s, is_source => $source,
 234                                             category => { id => _g('Section'), name => $sec } },
 235                                  "$wwwdir/$source$key/$sec/index.$lang.html.new" );
 236                 rename( "$wwwdir/$source$key/$sec/index.$lang.html.new",
 237                         "$wwwdir/$source$key/$sec/index.$lang.html" );
 238             }
 239             foreach my $ssec ((keys %{$subsections->{$s}}, 'virtual')) {
 240                 next if $ssec eq '-';
 241                 mkpath ( "$wwwdir/$source$key/$ssec" );
 242
 243                 print "writing $source$s/$ssec/index (lang=$lang)...\n";
 244                 $template->page( 'index', { packages => [ grep { $_->{subsection} eq $ssec } @{$pkgs->{$key}} ],
 245                                             %lang_vars, suite => $s, is_source => $source,
 246                                             category => { id => _g('Subsection'), name => $ssec } },
 247                                  "$wwwdir/$source$key/$ssec/index.$lang.html.new" );
 248                 rename( "$wwwdir/$source$key/$ssec/index.$lang.html.new",
 249                         "$wwwdir/$source$key/$ssec/index.$lang.html" );
 250             }
 251             foreach my $prio (keys %{$priorities->{$s}}) {
 252                 next if $prio eq '-';
 253                 mkpath ( "$wwwdir/$source$key/$prio" );
 254
 255                 print "writing $source$s/$prio/index (lang=$lang)...\n";
 256                 $template->page( 'index', { packages => [ grep { $_->{priority} eq $prio } @{$pkgs->{$key}} ],
 257                                             %lang_vars, suite => $s, is_source => $source,
 258                                             category => { id => _g('Priority'), name => $prio } },
 259                                  "$wwwdir/$source$key/$prio/index.$lang.html.new" );
 260                 rename( "$wwwdir/$source$key/$prio/index.$lang.html.new",
 261                         "$wwwdir/$source$key/$prio/index.$lang.html" );
 262             }
 263         }
 264     }
 265 }