bin/create_index_pages

   1 #!/usr/bin/perl
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX;
   7 use File::Path;
   8 use DB_File;
   9 use Storable;
  10 use HTML::Entities;
  11
  12 use lib './lib';
  13
  14 use Packages::Config qw( $TOPDIR $DBDIR @ARCHIVES @SUITES );
  15 use Packages::HTML;
  16 use Packages::Page;
  17 &Packages::Config::init( './' );
  18 sub gettext { return $_[0]; }
  19 sub dgettext { return $_[1]; }
  20
  21 my $wwwdir = "$TOPDIR/www";
  22
  23 tie my %packages, 'DB_File', "$DBDIR/packages_small.db",
  24     O_RDONLY, 0666, $DB_BTREE
  25     or die "couldn't tie DB $DBDIR/packages_small.db: $!";
  26
  27 my $sections = retrieve "$DBDIR/sections.info";
  28 my $subsections = retrieve "$DBDIR/subsections.info";
  29 # work around problems with non-US security updates
  30 $subsections->{oldstable}{us}{'non-US'}++;
  31 my $priorities = retrieve "$DBDIR/priorities.info";
  32
  33 #use Data::Dumper;
  34 #print STDERR Dumper($sections, $subsections, $priorities);
  35 my %sections_descs = (
  36                       'admin'           => [ "Administration Utilities",
  37                                              "Utilities to administer system resources, manage user accounts, etc." ],
  38                       'base'            => [ "Base Utilities",
  39                                              "Basic needed utilities of every Debian system." ],
  40                       'comm'            => [ "Communication Programs",
  41                                              "Software to use your modem in the old fashioned style." ],
  42                       'devel'           => [ "Development",
  43                                              "Development utilities, compilers, development environments, libraries, etc." ],
  44                       'doc'             => [ "Documentation",
  45                                              "FAQs, HOWTOs and other documents trying to explain everything related to Debian, and software needed to browse documentation (man, info, etc)." ],
  46                       'editors' => [ "Editors",
  47                                      "Software to edit files. Programming environments." ],
  48                       'electronics'     => [ "Electronics",
  49                                              "Electronics utilities." ],
  50                       'embedded'        => [ "Embedded software",
  51                                              "Software suitable for use in embedded applications." ],
  52                       'games'           => [ "Games",
  53                                              "Programs to spend a nice time with after all this setting up." ],
  54                       'gnome'           => [ "GNOME",
  55                                              "The GNOME desktop environment, a powerful, easy to use set of integrated applications." ],
  56                       'graphics'        => [ "Graphics",
  57                                              "Editors, viewers, converters... Everything to become an artist." ],
  58                       'hamradio'        => [ "Ham Radio",
  59                                              "Software for ham radio." ],
  60                       'interpreters'    => [ "Interpreters",
  61                                              "All kind of interpreters for interpreted languages. Macro processors." ],
  62                       'kde'             => [ "KDE",
  63                                              "The K Desktop Environment, a powerful, easy to use set of integrated applications." ],
  64                       'libs'            => [ "Libraries",
  65                                              "Libraries to make other programs work. They provide special features to developers." ],
  66                       'libdevel'        => [ "Library development",
  67                                              "Libraries necessary for developers to write programs that use them." ],
  68                       'mail'            => [ "Mail",
  69                                              "Programs to route, read, and compose E-mail messages." ],
  70                       'math'            => [ "Mathematics",
  71                                              "Math software." ],
  72                       'misc'            => [ "Miscellaneous",
  73                                              "Miscellaneous utilities that didn\'t fit well anywhere else." ],
  74                       'net'             => [ "Network",
  75                                              "Daemons and clients to connect your Debian GNU/Linux system to the world." ],
  76                       'news'            => [ "Newsgroups",
  77                                              "Software to access Usenet, to set up news servers, etc." ],
  78                       'non-US'  => [ "Software restricted in the U.S.",
  79                                      "These packages probably may not be used in or distributed from the U.S. due to software patents. You should check the regulations in your country before using this software." ],
  80                       'oldlibs' => [ "Old Libraries",
  81                                      "Old versions of libraries, kept for backward compatibility with old applications." ],
  82                       'otherosfs'       => [ "Other OS\'s and file systems",
  83                                              "Software to run programs compiled for other operating system, and to use their filesystems." ],
  84                       'perl'            => [ "Perl",
  85                                              "Everything about Perl, an interpreted scripting language." ],
  86                       'python'  => [ "Python",
  87                                      "Everything about Python, an interpreted, interactive object oriented language." ],
  88                       'science' => [ "Science",
  89                                      "Basic tools for scientific work" ],
  90                       'shells'  => [ "Shells",
  91                                      "Command shells. Friendly user interfaces for beginners." ],
  92                       'sound'           => [ "Sound",
  93                                              "Utilities to deal with sound: mixers, players, recorders, CD players, etc." ],
  94                       'tex'             => [ "TeX",
  95                                              "The famous typesetting software and related programs." ],
  96                       'text'            => [ "Text Processing",
  97                                              "Utilities to format and print text documents." ],
  98                       'utils'           => [ "Utilities",
  99                                              "Utilities for file/disk manipulation, backup and archive tools, system monitoring, input systems, etc." ],
 100                       'virtual' => [ "Virtual packages",
 101                                      "Virtual packages." ],
 102                       'web'             => [ "Web Software",
 103                                              "Web servers, browsers, proxies, download tools etc." ],
 104                       'x11'             => [ "X Window System software",
 105                                              "X servers, libraries, fonts, window managers, terminal emulators and many related applications." ],
 106                       'debian-installer' => [ "debian-installer udeb packages",
 107                                               "Special packages for building customized debian-installer variants. Do not install them on a normal system!" ],
 108                       );
 109
 110
 111 my (%pages);
 112
 113 print "opening files ...\n";
 114 foreach my $s (@SUITES) {
 115     foreach my $a (@ARCHIVES) {
 116         next if $a eq 'security';
 117         next if $a eq 'non-US';
 118         my $key = ($a eq 'us') ? $s : "$s/$a";
 119         mkpath ( "$wwwdir/$key" );
 120         open $pages{$key}{index}{fh}, '>', "$wwwdir/$key/index.en.html.new"
 121             or die "can't open index file for output: $!";
 122         open $pages{$key}{fh}, '>', "$wwwdir/$key/allpackages.en.html.new"
 123             or die "can't open index file for output: $!";
 124
 125         my $title = sprintf( gettext ( "Software Packages in \"%s\"" ),
 126                              $s );
 127         my $index_title = sprintf( gettext ( "List of sections in \"%s\"" ),
 128                                    $s );
 129         print {$pages{$key}{fh}} header( title => $title,
 130                                          title_keywords => "debian, $s",
 131                                          desc => encode_entities( $title, '"' ),
 132                                          lang => 'en' ),
 133         title( $title ), '<dl>';
 134         print {$pages{$key}{index}{fh}} header( title => $index_title,
 135                                                 title_keywords => "debian, $s",
 136                                                 desc => encode_entities( $index_title, '"' ),
 137                                                 lang => 'en' ),
 138         title( $index_title ), '<div id="lefthalfcol"><dl>';
 139
 140         foreach my $sec (keys %{$sections->{$s}{$a}}) {
 141             mkpath ( "$wwwdir/$key/$sec" );
 142             open $pages{$key}{$sec}{fh}, '>', "$wwwdir/$key/$sec/index.en.html.new"
 143                 or die "can't open index file for output: $!";
 144             $title = sprintf( gettext ( "Software Packages in \"%s\", section %s" ),
 145                               $s, $sec );
 146             print {$pages{$key}{$sec}{fh}} header( title => $title,
 147                                                    title_keywords => "debian, $s, $sec",
 148                                                    desc => encode_entities( $title, '"' ),
 149                                                    lang => 'en' ),
 150             title( $title ), '<dl>';
 151         }
 152         my $i = 0; my $num_sections = keys %{$subsections->{$s}{$a}};
 153         foreach my $ssec ((keys %{$subsections->{$s}{$a}}, 'virtual')) {
 154             next if $ssec eq '-';
 155             mkpath ( "$wwwdir/$key/$ssec" );
 156             open $pages{$key}{$ssec}{fh}, '>', "$wwwdir/$key/$ssec/index.en.html.new"
 157                 or die "can't open index file for output: $!";
 158             $title = sprintf( gettext ( "Software Packages in \"%s\", subsection %s" ),
 159                               $s, $ssec );
 160             print {$pages{$key}{$ssec}{fh}} header( title => $title,
 161                                                     title_keywords => "debian, $s, $ssec",
 162                                                     desc => encode_entities( $title, '"' ),
 163                                                     lang => 'en' ),
 164             title( $title ), '<dl>';
 165
 166             if ($sections_descs{$ssec}) {
 167                 print {$pages{$key}{index}{fh}} "<dt><a href=\"$ssec/\">".dgettext( 'sections', $sections_descs{$ssec}[0] )."</a></dt><dd>".dgettext( 'sections', $sections_descs{$ssec}[1] )."</dd>\n";
 168                 $i++;
 169                 if ($i eq ceil($num_sections/2)) {
 170                     print {$pages{$key}{index}{fh}} "</dl>\n</div> <!-- end lefthalfcol -->\n<div id=\"righthalfcol\">\n<dl>\n";
 171                 }
 172             }
 173         }
 174         foreach my $prio (keys %{$priorities->{$s}{$a}}) {
 175             next if $prio eq '-';
 176             mkpath ( "$wwwdir/$key/$prio" );
 177             open $pages{$key}{$prio}{fh}, '>', "$wwwdir/$key/$prio/index.en.html.new"
 178                 or die "can't open index file for output: $!";
 179             $title = sprintf( gettext ( "Software Packages in \"%s\", priority %s" ),
 180                               $s, $prio );
 181             print {$pages{$key}{$prio}{fh}} header( title => $title,
 182                                                     title_keywords => "debian, $s, $prio",
 183                                                     desc => encode_entities( $title, '"' ),
 184                                                     lang => 'en' ),
 185             title( $title ), '<dl>';
 186         }
 187     }
 188 }
 189
 190 print "writing package info ...\n";
 191 while (my ($pkg, $data) = each %packages) {
 192     my %pkg;
 193     foreach (split /\000/o, $data) {
 194         my @data = split ( /\s/o, $_, 8 );
 195         my $key = $data[1];
 196         if ($data[0] !~ /^(?:us|security|non-US)$/o) {
 197             $key = "$data[1]/$data[0]";
 198         }
 199         $pkg{$key} ||= new Packages::Page( $pkg );
 200         if ($data[2] ne 'virtual') {
 201             $pkg{$key}->merge_package( { package => $pkg,
 202                                          archive => $data[0],
 203                                          suite => $data[1],
 204                                          architecture => $data[2],
 205                                          section => $data[3],
 206                                          subsection => $data[4],
 207                                          priority => $data[5],
 208                                          version => $data[6],
 209                                          description => $data[7] } );
 210         } else {
 211             $pkg{$key}->add_provided_by([split /\s+/, $data[7]]);
 212         }
 213     }
 214     while (my ($key, $entry) = each %pkg) {
 215         if (my $provided_by = $entry->{provided_by}) {
 216             my $str = "<dt><a href=\"$pkg\">$pkg</a> ".
 217                 "</dt>\n     <dd>virtual package provided by ".
 218                 join( ', ',map { "<a href=\"../$_\">$_</a>" } @$provided_by)."</dd>\n";
 219             my $txt_str = "$pkg\tvirtual package provided by ".join(', ', @$provided_by)."\n";
 220             print {$pages{$key}{virtual}{fh}} $str
 221                 or die "couldn't write to output file: $!";
 222         }
 223         next if $entry->is_virtual;
 224         my (undef, $v_str) = $entry->get_version_string;
 225         my $subsection = $entry->get_newest( 'subsection' );
 226         my $section = $entry->get_newest( 'section' );
 227         my $archive = $entry->get_newest( 'archive' );
 228         my $short_desc_txt = $entry->get_newest( 'description' );
 229         my $short_desc = encode_entities( $short_desc_txt, "<>&\"" );
 230         my $priority = $entry->get_newest( 'priority' );
 231
 232         my $str = "<dt><a href=\"$pkg\">$pkg</a> ($v_str) ";
 233         my $txt_str = "$pkg ($v_str)";
 234         if ($section ne 'main') {
 235             $str .= marker( $section );
 236             $txt_str .= " [$section]";
 237         }
 238         if ($archive ne 'us') {
 239             $str .= marker( $archive );
 240             $txt_str .= " [$archive]";
 241         }
 242         $str .= "</dt>\n     <dd>$short_desc</dd>\n";
 243         $txt_str .= " $short_desc_txt\n";
 244         print {$pages{$key}{fh}} $str
 245             or die "couldn't write to output file: $!";
 246         print {$pages{$key}{$section}{fh}} $str
 247             or die "couldn't write to output file: $!";
 248         if ($subsection ne '-') {
 249             print {$pages{$key}{$subsection}{fh}} $str
 250                 or die "couldn't write to output file: $!";
 251         }
 252         if ($priority ne '-') {
 253             print {$pages{$key}{$priority}{fh}} $str
 254                 or die "couldn't write to output file: $!";
 255         }
 256     }
 257 }
 258
 259 print "closing files ...\n";
 260 foreach my $s (@SUITES) {
 261     foreach my $a (@ARCHIVES) {
 262         next if $a eq 'security';
 263         next if $a eq 'non-US';
 264         my $key = ($a eq 'us') ? $s : "$s/$a";
 265         my $root = ($a eq 'us') ? '' : '../';
 266         print {$pages{$key}{index}{fh}} '</dl></div>',
 267         "<p class=\"psmallcenter\"><a href=\"allpackages\" title=\"".gettext( "List of all packages" )."\">".
 268         gettext( "All packages" ) ."</a><br>(<a href=\"allpackages.en.txt.gz\">".
 269         gettext( "compact compressed textlist" )."</a>)</p>\n";
 270         print {$pages{$key}{index}{fh}} trailer( "$root../" );
 271         close $pages{$key}{index}{fh} or
 272             warn "can't open index file for output $wwwdir/$key/index.en.html.new: $!";
 273         rename( "$wwwdir/$key/index.en.html.new",
 274                 "$wwwdir/$key/index.en.html" );
 275         print {$pages{$key}{fh}} '</dl>', trailer( "$root../" );
 276         close $pages{$key}{fh} or
 277             warn "can't close index file $wwwdir/$key/allpackages.en.html.new: $!";
 278         rename( "$wwwdir/$key/allpackages.en.html.new",
 279                 "$wwwdir/$key/allpackages.en.html" );
 280         foreach my $sec (keys %{$sections->{$s}{$a}}) {
 281             print {$pages{$key}{$sec}{fh}} '</dl>', trailer( "$root../../" );
 282             close $pages{$key}{$sec}{fh} or
 283                 warn "can't close index file $wwwdir/$key/$sec/index.en.html.new: $!";
 284             rename( "$wwwdir/$key/$sec/index.en.html.new",
 285                     "$wwwdir/$key/$sec/index.en.html" );
 286         }
 287         foreach my $ssec ((keys %{$subsections->{$s}{$a}}, 'virtual')) {
 288             next if $ssec eq '-';
 289             print {$pages{$key}{$ssec}{fh}} '</dl>', trailer( "$root../../" );
 290             close $pages{$key}{$ssec}{fh} or
 291                 warn "can't close index file $wwwdir/$key/$ssec/index.en.html.new: $!";
 292             rename( "$wwwdir/$key/$ssec/index.en.html.new",
 293                     "$wwwdir/$key/$ssec/index.en.html" );
 294         }
 295         foreach my $prio (keys %{$priorities->{$s}{$a}}) {
 296             next if $prio eq '-';
 297             print {$pages{$key}{$prio}{fh}} '</dl>', trailer( "$root../../" );
 298             close $pages{$key}{$prio}{fh} or
 299                 warn "can't close index file $wwwdir/$key/$prio/index.en.html.new: $!";
 300             rename( "$wwwdir/$key/$prio/index.en.html.new",
 301                     "$wwwdir/$key/$prio/index.en.html" );
 302         }
 303     }
 304 }