parse-translations: new script to parse the Translation files

diff --git a/bin/parse-translations b/bin/parse-translations
new file mode 100755 (executable)
index 0000000..436e435
--- /dev/null
+++ b/bin/parse-translations
@@ -0,0 +1,93 @@
+#!/usr/bin/perl -w
+# Convert Translation.gz files into Sleepycat db files for efficient usage of
+# data
+#
+# $Id$
+#
+# Copyright (C) 2006  Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+use strict;
+use warnings;
+use lib './lib';
+
+$| = 1;
+
+# max. distinct results for a given package postfix
+my $MAX_PACKAGE_POSTFIXES = 100;
+
+use DB_File;
+use Storable;
+use File::Path;
+use Digest::MD5;
+use Deb::Versions;
+use Lingua::Stem v0.82;
+use Search::Xapian;
+use Packages::Config qw( $TOPDIR $DBDIR @DDTP_LANGUAGES );
+&Packages::Config::init( './' );
+my %descriptions = ();
+
+$/ = "";
+
+-d $DBDIR || mkpath( $DBDIR );
+
+foreach my $lang (@DDTP_LANGUAGES) {
+    print "Reading Translations for $lang...";
+    open PKG, "zcat $TOPDIR/archive/*/*/*/i18n/Translation-$lang.gz|";
+    my $count = 0;
+    while (<PKG>) {
+       next if /^\s*$/;
+       my $data = "";
+       my %data = ();
+       chomp;
+       s/\n /\377/g;
+       while (/^(\S+):\s*(.*)\s*$/mg) {
+           my ($key, $value) = ($1, $2);
+           $value =~ s/\377/\n /g;
+           $key =~ tr [A-Z] [a-z];
+           $data{$key} = $value;
+       }
+       # Skip double descriptions
+       next if exists($descriptions{$data{"description-md5"}}{$lang});
+       # some weirdnesses in the files
+       next unless defined $data{"description-".lc($lang)};
+       $descriptions{$data{"description-md5"}}{$lang} = $data{"description-".lc($lang)};
+       $count++;
+    }
+    print "($count)\n";
+}
+close PKG;
+
+print "Writing database (".scalar(keys %descriptions)." unique descriptions)...\n";
+my %descriptions_db;
+tie %descriptions_db, "DB_File", "$DBDIR/descriptions_translated.db.new",
+       O_RDWR|O_CREAT, 0666, $DB_BTREE
+       or die "Error creating DB: $!";
+while (my ($md5, $v) = each(%descriptions)) {
+    my $str = "";
+    while (my ($lang, $desc) = each %$v) {
+       unless ($lang && $desc) {
+           warn "MD5: $md5 LANG: $lang DESC: $desc\n";
+           exit;
+       }
+       $str .= "$lang\001$desc\000";
+    }
+
+    $descriptions_db{$md5} = $str;
+}
+untie %descriptions_db;
+
+rename("$DBDIR/descriptions_translated.db.new",
+       "$DBDIR/descriptions_translated.db");

diff --git a/cron.d/200process_archive b/cron.d/200process_archive
index c3d6b58bcc1b3bc845a7b8c7a5d57ff06fb8b979..6fa394e61f861c5dcd27d89de0ce87910bb058c8 100755 (executable)
--- a/cron.d/200process_archive
+++ b/cron.d/200process_archive
@@ -11,3 +11,5 @@ date
 date
 ./bin/parse-contents
 date
+./bin/parse-translations
+date

diff --git a/lib/Packages/Config.pm b/lib/Packages/Config.pm
index 8252a1b132271d8f1c49159d4a0c7ec914c15140..96f8b679ccc8c76ec5446abd1f51ee5096a5427b 100644 (file)
--- a/lib/Packages/Config.pm
+++ b/lib/Packages/Config.pm
@@ -9,11 +9,11 @@ use Packages::CGI qw( :DEFAULT error );
 our @ISA = qw( Exporter );
 
 our ( $TOPDIR, $DBDIR, $TEMPLATEDIR, $CACHEDIR, $ROOT,
-      @LANGUAGES, $LOCALES,
+      @LANGUAGES, @DDTP_LANGUAGES, $LOCALES,
       @SUITES, @SECTIONS, @ARCHIVES, @ARCHITECTURES,
       @PRIORITIES, %FTP_SITES );
 our @EXPORT_OK = qw( $TOPDIR $DBDIR $TEMPLATEDIR $CACHEDIR $ROOT
-                    @LANGUAGES $LOCALES
+                    @LANGUAGES @DDTP_LANGUAGES $LOCALES
                     @SUITES @SECTIONS @ARCHIVES @ARCHITECTURES
                     @PRIORITIES %FTP_SITES  );
 our %EXPORT_TAGS = ( all => [ @EXPORT_OK ] );
@@ -39,6 +39,7 @@ sub init {
            $FTP_SITES{us} = $1 if /^\s*ftpsite="?([^\"]*)"?\s*$/o;
            $FTP_SITES{$1} = $2 if /^\s*(\w+)_ftpsite="?([^\"]*)"?\s*$/o;
            @LANGUAGES = split(/\s+/, $1) if /^\s*polangs="?([^\"]*)"?\s*$/o;
+           @DDTP_LANGUAGES = split(/\s+/, $1) if /^\s*ddtplangs="?([^\"]*)"?\s*$/o;
            @SUITES = split(/\s+/, $1) if /^\s*suites="?([^\"]*)"?\s*$/o;
            @SECTIONS = split(/\s+/, $1) if /^\s*sections="?([^\"]*)"?\s*$/o;
            @ARCHIVES = split(/\s+/, $1) if /^\s*archives="?([^\"]*)"?\s*$/o;