From: Frank Lichtenheld Date: Sat, 16 Jan 2010 19:29:07 +0000 (+0000) Subject: Merge commit 'alioth/master' into debian-master X-Git-Url: https://git.deb.at/?p=deb%2Fpackages.git;a=commitdiff_plain;h=59da095beebcee4ec6d78531709d709db391017c;hp=88495a473fb1aef31bfcf623d2f2f9cb722165ad Merge commit 'alioth/master' into debian-master --- diff --git a/.gitignore b/.gitignore index 87f52c2..5204c11 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,7 @@ archive config.sh cache tmp +mirror *~ *.ttc +www-passwords diff --git a/README.Mirror b/README.Mirror new file mode 100644 index 0000000..479f459 --- /dev/null +++ b/README.Mirror @@ -0,0 +1,29 @@ + +To only operate a mirror of packages.d.o, the following packages +need to be installed: + + liblocale-maketext-lexicon-perl + libi18n-acceptlanguage-perl + libnumber-format-perl + libcompress-zlib-perl + libhtml-parser-perl + libmldbm-perl + libtext-iconv-perl + libclass-accessor-perl + liburi-perl + libtemplate-perl + liblingua-stem-perl + libsearch-xapian-perl (bpo) + + apache2-mpm-worker + libapache2-mod-perl2 + +The following apache modules need to be active + + rewrite + expires + perl + setenvif + alias + negotiation + diff --git a/bin/build-maintainerdb b/bin/build-maintainerdb new file mode 100755 index 0000000..91f3932 --- /dev/null +++ b/bin/build-maintainerdb @@ -0,0 +1,256 @@ +#! /usr/bin/perl + +# build-maintainerdb - convert several Packages files to maintainer database +# Copyright (c) 1998,9,2001,3,4,6 Martin Schulze +# Copyright (C) 2007 Joerg Jaspert +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# Todo: +# . Read maintainer changes from overrides file(s), need to rub +# out all existing entries + +use strict; +use warnings; +use DB_File; + +use lib '../lib'; + +use Deb::Versions; + +# read the configuration +if (!open (C, "../config.sh")) { + print "\nInternal Error: Cannot open configuration file.\n\n"; + exit 0; +} +my $topdir = "/org/packages.debian.org"; +while () { + $topdir = $1 if (/^\s*topdir="?(.*)"?\s*$/); +} +close (C); + +#my $maildomain = "packages.debian.org"; +my $ptsdomain = "packages.qa.debian.org"; + + +my $maintainerfile = "$topdir/../mirrors/ftp.debian.org/indices/Maintainers"; +my $maintainerdb = "$topdir/conf/maintainer"; +my $overridefile = "$topdir/conf/maintainerdb.override"; +my $rblfile = "$topdir/conf/rbllist"; +my $rhsblfile = "$topdir/mail/rhsbllist"; +my $calloutfile = "$topdir/mail/callout_users"; +my @postcall = ( "/usr/sbin/postmap", $maintainerdb ); +my $opt_verbose = 0; +my $dbdir = "$topdir/files/db"; + +my %maint; +my %binsrc; + +# Lets take pdo database input files for the source mapping +sub read_bin_src_mapping +{ + tie my %src_packages, 'DB_File', "$dbdir/sources_small.db", + O_RDONLY, 0666, $DB_BTREE + or die "couldn't tie DB $dbdir/sources_small.db: $!"; + tie my %src2bin, 'DB_File', "$dbdir/sources_packages.db", + O_RDONLY, 0666, $DB_BTREE + or die "couldn't open $dbdir/sources_packages.db: $!"; + + my %bin; + while (my ($source, $data) = each %src_packages) { + my @src = map { [ split(/\s+/) ] } split( /\0/, $data ); + my $newest = (sort( { version_cmp($b->[-1],$a->[-1]) } @src))[0]; + my ($section, $version) = @{$newest}[2,5]; + + my $bin_data = $src2bin{$source}; + unless ($bin_data) { + warn "no binary data found for $source\n"; + # source packages without binaries have most + # likely been succeded + next; + } + + my @bin = map { [ split(/\s+/) ] } split( /\0/, $bin_data ); + my %seen; + foreach (@bin) { + my ($pkg, $ver) = @{$_}[2,3]; + next if $seen{"$pkg/$ver"}++; # weed out multiple arches/suites faster + if ($bin{$pkg} + && ($bin{$pkg}{source} ne $source) + && (version_cmp($bin{$pkg}{version},$ver) > 0)) { + next; + } else { + $bin{$pkg} = { version => $ver, source => $source, + target => "$section" }; + $binsrc{$pkg} = $source; + } + } + $binsrc{$source} = $source; + } +} + +sub package_maintainer +{ + my $pkg = shift; + my $login = shift; + my $addr = shift; + my $home = "/debian/home/$login"; + + if (-d $home) { + if (-f "$home/.forward-$pkg") { + return "$login-$pkg\@master.debian.org"; + } + } + return $addr; +} + +sub read_maintainer +{ + my $file = shift; + my $package; + my $maintainer; + my $maint; + + open (F, "$file") || die "Can't open $file, $!"; + printf "Reading %s\n", $file if ($opt_verbose); + while () { + next if (/^#/); + next if (/^$/); + /(\S+)\s+(.*)/; + $package=$1; $maint=$2; + if (! exists $maint{$package}) { + printf " EVAL (%s, \"%s\")\n", $package, $maint if ($opt_verbose > 2); + + if (exists $maint{$package}) { + $maint{$package} .= " "; + printf " EXPAND (%s)\n", $package if ($opt_verbose > 2); + } + if ($maint =~ /.*<(.*)>/) { + $maint{$package} .= $1; + printf " ADD (%s, %s)\n", $package, $1 if ($opt_verbose > 2); + } elsif ($maint =~ /\s*(\S+)\s+\(.*\)/) { + $maint{$package} .= $1; + printf " ADD (%s, %s)\n", $package, $1 if ($opt_verbose > 2); + } else { + $maint{$package} .= $maint; + printf " ADD (%s, %s)\n", $package, $maint if ($opt_verbose > 2); + } + + printf " %s: %s\n", $package, $maint{$package} if ($opt_verbose > 1); + +# Short what? Whats this supposed to do, except creating new (and 99% broken) something -> email mappings? +# my $pkgshort = ""; +# if ($package =~ /(.*[^\d\.]+)([\d\.]*\d)$/) { +# $pkgshort = $1; +# print "Short what? $pkgshort (long is $package)\n"; +# $maint{$pkgshort} = $maint{$package} if (! exists $maint{$pkgshort}); +# printf " %s: %s\n", $pkgshort, $maint{$package} if ($opt_verbose > 1); +# } +# if ($maint{$package} =~ /([^\@]+)\@(master\.)?debian\.org/) { +# my $addrsave = $maint{$package} if ($opt_verbose > 1); +# $maint{$package} = package_maintainer ($package, $1, $maint{$package}); +# printf " Changed to %s\n", $maint{$package} if ($opt_verbose > 1 && ($addrsave ne $maint{$package})); +# if (length ($pkgshort) > 0) { +# $maint{$pkgshort} = package_maintainer ($pkgshort, $1, $maint{$pkgshort}); +# } +# } + } else { + printf "Skipping double $package\n" if ($opt_verbose); + printf "LINE: $_" if ($opt_verbose > 2); + } + } + close (F); +} + + + +sub write_maintainer +{ + my $file = shift; + + printf "Writing to %s.new\n", $file if ($opt_verbose > 0); + open (CONF, ">$file.new") || die "Can't open $file.new, $!"; + open (WRITE, "| cdbmake $file.cdb $file.cdb.tmp") || die "Can't talk to cdbmake, $!"; + open (RBL, ">$rblfile.new") || die "Can't open $file.new, $!"; + open (RHSBL, ">$rhsblfile.new") || die "Can't open $file.new, $!"; + open (CALLOUT, ">$calloutfile.new") || die "Can't open $file.new, $!"; + my $forward; + foreach my $package (sort(keys(%maint))) { +# It is possible that we do not know a source package -> in that case fall back to old behaviour +# and only mail the maintainer. Can happen when pdo doesnt know the suite the source is in, like +# it doesnt know oldstable-proposed-updates at date of writing this code. + $forward = "$maint{$package}"; + if ($binsrc{$package}) { + $forward .= ", $binsrc{$package}_contact\@$ptsdomain"; + } + printf "%s -> %s\n", $package, $forward if ($opt_verbose); + printf CONF "%s:\t%s\n", $package, $forward; + printf WRITE "+%d,%d:%s->%s\n", length($package), length($forward), $package, $forward; + printf RBL "%s : zen.spamhaus.org : cbl.abuseat.org\n", $package; + printf RHSBL "%s : bogusmx.rfc-ignorant.org/\$sender_address_domain : dsn.rfc-ignorant.org/\$sender_address_domain\n", $package; + printf CALLOUT "%s\n", $package; +# printf "%s -> %s and pts: %s\n", $package, $maint{$package}, $binsrc{$package} if ($opt_verbose); +# printf CONF "%s@%s\t%s, %s_contact@%s\n", $package, $maildomain, $maint{$package}, $binsrc{$package}, $ptsdomain; + } + print WRITE "\n"; + close (CONF); + close (WRITE); + close (RBL); + close (RHSBL); + close (CALLOUT); + printf "Renaming the new files\n" if ($opt_verbose > 0); + system "mv -f $file.new $file"; + system "mv -f $rblfile.new $rblfile"; + system "mv -f $rhsblfile.new $rhsblfile"; + system "mv -f $calloutfile.new $calloutfile"; +# printf "Executing @postcall\n" if ($opt_verbose > 0); +# system @postcall; +} + +sub help +{ + print "build-maintainerdb - Build the maintainer db for packages.debian.org.\n"; + print "-d debug, changes output file to ./maintainerdb\n"; + print "-h This help\n"; + print "-v Increase verbosity by 1\n"; + print "-vv Increase verbosity by 2\n"; + print "-vvv Increase verbosity by 3\n"; +} + +# +# Main program +# +while ($#ARGV > -1) { + if ($ARGV[0] eq "-v") { + $opt_verbose++; + } elsif ($ARGV[0] eq "-vv") { + $opt_verbose+= 2; + } elsif ($ARGV[0] eq "-vvv") { + $opt_verbose+= 3; + } elsif ($ARGV[0] eq "-h") { + help(); + } elsif ($ARGV[0] eq "-d") { + $maintainerdb = "./maintainerdb"; + } + shift; +} + +&read_bin_src_mapping(); + +&read_maintainer ($overridefile); +&read_maintainer ($maintainerfile); + +&write_maintainer ($maintainerdb); + diff --git a/bin/create_mirror b/bin/create_mirror new file mode 100755 index 0000000..a6d8220 --- /dev/null +++ b/bin/create_mirror @@ -0,0 +1,59 @@ +#!/bin/bash + +set -e + +topdir="$1" +[ -n "$topdir" -a -d "$topdir" ] || exit 1 +mirror="$topdir/mirror" + +mkdir -p "$mirror" + +STAMP=$(date "+%Y%m%d%H%M") +new="$mirror/$STAMP" +mkdir $new + +if lockfile -! -l 3600 -r2 -300 "$mirror/mirror.lock" +then + echo "couldn't aquire mirror.lock in due time" + exit 1; +fi + +mkdir -p "$new/cache" +chmod 777 "$new/cache" + +cd "$new" +ln -f "$topdir/config.sh" +ln -f "$topdir/README.Mirror" + +mkdir -p "$new/bin" +cd "$new/bin" +ln -f "$topdir/bin/mod_perl-startup" + +mkdir -p "$new/conf" +cd "$new/conf" +ln -f "$topdir/conf/apache.conf" + +mkdir -p "$new/cgi-bin" +cd "$new/cgi-bin" +ln -f "$topdir/cgi-bin/dispatcher.pl" + +for d in lib files templates po www +do + mkdir -p "$new/$d" + rsync -avH --link-dest "$topdir/$d" \ + --exclude '*.new' --exclude '*.sed.in' \ + --exclude '*.slices' --exclude '*~' \ + --exclude '*.lock' --exclude '*.ttc' --exclude 'logs/' \ + --exclude 'packages/*/' --exclude 'changelogs.*.dump' \ + --exclude 'changelogs.cache' --exclude 'changelogs/**/log' \ + --exclude "/auto" --exclude "/Search" --exclude "/perl" \ + --delete-excluded --delete-after \ + "$topdir/$d/" "$new/$d/" +done +date -u > "$new/www/Pics/.trace" + +ln -sfT $new $mirror/current +find $mirror -mindepth 1 -maxdepth 1 -not -name "${STAMP}" -type d -print0 \ + | xargs --no-run-if-empty -0 rm -rf + +rm -f "$mirror/mirror.lock" diff --git a/bin/daily.sed.in b/bin/daily.sed.in index 28c8333..fe220b5 100755 --- a/bin/daily.sed.in +++ b/bin/daily.sed.in @@ -53,3 +53,7 @@ else echo "couldn't aquire changelogs lock file in due time" fi date + +$topdir/bin/create_mirror $topdir +sudo -H -u archvsync /home/archvsync/bin/pushpdo & +date diff --git a/bin/extract_files b/bin/extract_files index 269078c..79b9946 100755 --- a/bin/extract_files +++ b/bin/extract_files @@ -564,6 +564,10 @@ sub read_deb { } } $cache{$debname} ||= qx/dpkg-deb --info "$debname" control/; + unless ( $cache{$debname} ) { + do_warning( "extracting control information of file $debname failed" ); + return; + } my $control = $cache{$debname}; unless ( $raw_data = $parser->parse_mem( $control, diff --git a/conf/.gitignore b/conf/.gitignore index 135b043..4ecbfe7 100644 --- a/conf/.gitignore +++ b/conf/.gitignore @@ -1,4 +1,6 @@ apache.conf ttreerc maintainer +maintainer.cdb crontab +rbllist diff --git a/conf/apache.conf.sed.in b/conf/apache.conf.sed.in index 91833a6..eac75bc 100644 --- a/conf/apache.conf.sed.in +++ b/conf/apache.conf.sed.in @@ -65,10 +65,11 @@ AddEncoding x-gzip .gz - + ServerAdmin webmaster@debian.org DocumentRoot %TOPDIR%/www/ ServerName %SITE% + ServerAlias packages-piatti.debian.org packages-powell.debian.org packages-rore.debian.org ErrorLog /var/log/apache2/%SITE%-error.log CustomLog /var/log/apache2/%SITE%-access.log combined @@ -110,8 +111,8 @@ RewriteRule ^/favicon.ico$ - [L] RewriteRule ^/robots.txt$ - [L] -# RewriteRule ^/$ http://www.debian.org/distrib/packages - RewriteRule ^/$ /index [L] + RewriteRule ^/$ http://www.debian.org/distrib/packages +# RewriteRule ^/$ /index [L] RewriteRule ^/([^/+]*)([+])([^/]*)$ "/$1%%{%}2B$3" [N] RewriteRule ^/changelog:(.+)$ /changelogs/${changelog-url:$1} [R,L,NE] RewriteRule ^/src:([^/]+)$ /search?searchon=sourcenames&keywords=$1 [R,L,NE] diff --git a/conf/maintainerdb.override b/conf/maintainerdb.override new file mode 100644 index 0000000..0f6f11d --- /dev/null +++ b/conf/maintainerdb.override @@ -0,0 +1,8 @@ +packages.debian.org Martin Schulze +nonus.debian.org Archive Maintenance +nonus NonUS Archive +non-us NonUS Archive +bugs Bug Tracking System +egcs debian-gcc@lists.debian.org +gcc debian-gcc@lists.debian.org +glibc debian-glibc@lists.debian.org diff --git a/config.sh.sed.in b/config.sh.sed.in index f5a511d..77ab212 100644 --- a/config.sh.sed.in +++ b/config.sh.sed.in @@ -19,7 +19,7 @@ cachedir=${topdir}/cache # unset this if %SITE% moves somewhere where the packages files # cannot be obtained locally # -#localdir=/org/ftp.debian.org/debian +localdir=/srv/mirrors/ftp.debian.org # path to private ftp directory #ftproot=/org/ftp.root diff --git a/cron.d/100syncarchive_maintainers b/cron.d/100syncarchive_maintainers new file mode 100755 index 0000000..9055606 --- /dev/null +++ b/cron.d/100syncarchive_maintainers @@ -0,0 +1,21 @@ +#! /bin/bash + +. `dirname $0`/../config.sh + +test -d ${archivedir} || mkdir -p ${archivedir} +cd ${archivedir} + +# scp -q auric:/org/ftp.debian.org/ftp/indices/Maintainers . + +if [ -f /org/ftp.root/debian/indices/Maintainers ] +then + if [ ! -s Maintainers -a -L Maintainers ] + then + ln -s /org/ftp.root/debian/indices/Maintainers . + fi +else + if [ ! -L Maintainers ] + then + $wget_cmd -O Maintainers $ftpsite/indices/Maintainers + fi +fi diff --git a/cron.d/300maintainerdb b/cron.d/300maintainerdb new file mode 100755 index 0000000..898a4c0 --- /dev/null +++ b/cron.d/300maintainerdb @@ -0,0 +1,6 @@ +#! /bin/bash + +. `dirname $0`/../config.sh + +cd ${bindir} +${bindir}/build-maintainerdb diff --git a/debian/control b/debian/control index 3af54e0..5ca566a 100644 --- a/debian/control +++ b/debian/control @@ -1,5 +1,5 @@ Source: packages -Build-Depends: git-core, rsync, dpkg-dev, procmail, gettext, +Build-Depends: rsync, dpkg-dev, procmail, gettext, liblocale-maketext-lexicon-perl, libi18n-acceptlanguage-perl, libnumber-format-perl, libcompress-zlib-perl, libhtml-parser-perl, libio-stringy-perl, libmldbm-perl, libtext-iconv-perl, libhtml-template-perl, diff --git a/mail/.gitignore b/mail/.gitignore new file mode 100644 index 0000000..6600526 --- /dev/null +++ b/mail/.gitignore @@ -0,0 +1,3 @@ +callout_users +grey_users +rhsbllist diff --git a/mail/aliases b/mail/aliases new file mode 100644 index 0000000..ace74b6 --- /dev/null +++ b/mail/aliases @@ -0,0 +1,14 @@ +root: joey@debian.org +admin: joey@debian.org +sysop: joey@debian.org +debian: debian@debian.org +postmaster: joey@debian.org +MAILER_DAEMON: joey@debian.org +usenet: joey@debian.org +news: joey@debian.org +newsmaster: joey@debian.org +joey: joey@debian.org + +submit: submit@bugs.debian.org + +base: base-files@packages.debian.org, base-passwd@packages.debian.org diff --git a/mail/dot.forward-default b/mail/dot.forward-default new file mode 100644 index 0000000..76b8f82 --- /dev/null +++ b/mail/dot.forward-default @@ -0,0 +1,13 @@ +# Exim Filter +# /org/packages.debian.org/mail/.forward-default + +# This filter looks up the package name in the maintainer database, if it is +# not found then a bounce message is generated. Although the docs say you +# can put spaces in the $lookup stuff you can't, don't try it. +if ${lookup{${local_part}}lsearch{/org/packages.debian.org/files/maintainerdb}{$value}{unknown}} is "unknown" +then + save /dev/null +else + seen deliver ${lookup{${local_part}}partial-lsearch{/org/packages.debian.org/files/maintainerdb}{$value}} +endif + diff --git a/setup-site.conf b/setup-site.conf index edaf9b6..c50889e 100644 --- a/setup-site.conf +++ b/setup-site.conf @@ -1,2 +1,2 @@ -topdir=/srv/packages.debian.org -site=packages.debian.net +topdir=/org/packages.debian.org +site=packages.debian.org diff --git a/static/about/index.tmpl b/static/about/index.tmpl index 78eb26a..5d25ca0 100644 --- a/static/about/index.tmpl +++ b/static/about/index.tmpl @@ -93,6 +93,10 @@ Copyright (C) 2006 Jeroen van Wolffelaar <jeroen@wolffelaar.nl> Copyright (C) 2006, 2007 Frank Lichtenheld <frank@lichtenheld.de> +

Sponsors

+ +

See our separate Sponsors page

+ [% PROCESS 'html/foot.tmpl' langs.size = 0 footer.doNotDisplayCopyright = 1 %] diff --git a/static/about/sponsors.tmpl b/static/about/sponsors.tmpl new file mode 100644 index 0000000..312a62f --- /dev/null +++ b/static/about/sponsors.tmpl @@ -0,0 +1,36 @@ +[% PROCESS 'html/head.tmpl' + title_tag = "$organisation Packages - Sponsors" + page_title = "$organisation Packages - Sponsors" + keywords = "$organisation, Sponsoring, Hardware, Hosting" +%] + +

Sponsors of packages.debian.org

+ +

Providing this popular service wouldn't be possible +without the help of the following companies and organisations:

+ + + + + + + + + + + + + + + + + + + + + + +
Machine Purpose Sponsor Hardware Sponsor Hosting
powell.debian.orgMain Web/Data Server, Mail Handling1&1 Internet AG
piatti.debian.orgMirrorHewlett-PackardUniversity of Helsinki - Department of Computer Science
rore.debian.orgMirrorHewlett-PackardMIT Computer Science & Artificial Intelligence Lab
+ +[% PROCESS 'html/foot.tmpl' + langs.size = 0 %] diff --git a/static/packages-site.css b/static/packages-site.css index 420506e..7254f37 100644 --- a/static/packages-site.css +++ b/static/packages-site.css @@ -2,3 +2,16 @@ .lenny-volatile, .lenny-backports { font-size: smaller; } + +table#sponsorslist { + text-align: center; + border: 1px solid black; + padding: 1em; + border-collapse: collapse; + empty-cells: show; +} + +#sponsorslist th, #sponsorslist td { + border: 1px solid black; + padding: .5em; +} diff --git a/templates/config.tmpl b/templates/config.tmpl index 85c52f7..26d6e85 100644 --- a/templates/config.tmpl +++ b/templates/config.tmpl @@ -6,7 +6,7 @@ organisation = 'Debian' project_homepage = 'http://www.debian.org/' packages_homepage = '/' - packages_homepage_abs = 'http://packages.debian.net/' + packages_homepage_abs = 'http://packages.debian.org/' old_releases = 'http://archive.debian.net/' searchformurl = packages_homepage search_url = '/search' @@ -19,7 +19,7 @@ security_mirror = 'security.debian.org' security_mirror_url = security_mirror _ '/debian-security' security_suite_suffix = '/updates' - changelogs_url = 'http://packages.debian.org/changelogs/' + changelogs_url = '/changelogs/' policy_url = 'http://www.debian.org/doc/debian-policy/' cn_help_url = project_homepage _ 'intro/cn' patch_tracking_url = 'http://patch-tracker.debian.org/package' @@ -52,19 +52,28 @@ trademarknotes = g('%s is a trademark of %s', organisation, trademark.url, trademark.name) sponsors = [ { - url => 'http://example.invalid/', - name => 'Example Sponsor, Inc.', + url => 'http://1und1.de/', + name => '1&1 Internet AG', }, { - url => 'http://example2.invalid/', - name => 'Example2 Sponsor, Inc.', + url => 'http://www.hp.com/', + name => 'Hewlett-Packard', + }, + { + url => 'http://cs.helsinki.fi/index.en.html', + name => 'University of Helsinki - Department of Computer Science', + }, + { + url => 'http://www.csail.mit.edu/', + name => 'MIT Computer Science & Artificial Intelligence Lab', }, ] -%] [%- # possible values for importance: high, normal, low - sitewidemsg = { importance => "high", - txt => g('Please note that this is an experimental version of %s. Errors and obsolete information should be expected', 'packages.debian.org', 'packages.debian.org') } +# sitewidemsg = { importance => "high", +# txt => g('Please note that this is an experimental version of %s. Errors and obsolete information should be expected', 'packages.debian.org', 'packages.debian.org') } +# txt => "Site maintainance in progress, some temporary problems might occour." } -%] [%# @translators: . = decimal_point , = thousands_sep, see Number::Format %] [%- USE num = Number.Format( decimal_point => g('.'),