extract_changelogs: Merge cron job changes from old site

Merge the new daily cron job scripts from old site that
splits the extract_changelogs from the rest of the
cron job. This avoids delaying the rest of the cronjob for
the changelog extraction. Using two different lock files
also makes the whole site update more robust.

diff --git a/bin/daily b/bin/daily
index f9e8853f77e64f1511c73c170464190a8268e464..0a67587bc36d49135f9ff2c73848ac1464077b95 100755 (executable)
--- a/bin/daily
+++ b/bin/daily
@@ -39,7 +39,15 @@ then
     run-parts --verbose $topdir/cron.d >> $log 2>&1
     rm -f "${files}/cron.lock"
 else
-    echo "couldn't aquire lock file in due time" >> $log
+    echo "couldn't aquire cron lock file in due time" >> $log
 fi
+date >> $log
 
+if lockfile -600 -r5 "${files}/changelogs.lock" >> $log 2>&1
+then
+    $topdir/bin/extract_changelogs >> $log 2>&1
+    rm -f "${files}/changelogs.lock"
+else
+    echo "couldn't aquire changelogs lock file in due time" >> $log
+fi
 date >> $log

diff --git a/bin/extract_changelogs b/bin/extract_changelogs
new file mode 100755 (executable)
index 0000000..08be087
--- /dev/null
+++ b/bin/extract_changelogs
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+. `dirname $0`/../config.sh
+ 
+if test -z "${localdir}"; then
+    echo skipping due to missing local archive
+    exit 1
+fi
+
+#set -e
+
+NOCPY_TEMPLATE=$topdir/etc/copyright_error
+NOCPY_BIN_TEMPLATE=$topdir/etc/copyright_error_bin
+
+logs="$topdir/files/logs"
+log="${logs}/changelogs.log"
+
+test -d "$logs" || mkdir -p "$logs"
+test -d "$tmpdir" || mkdir -p "$tmpdir"
+
+if [ -s "$log" ]
+then
+    savelog -c 14 "$log" > /dev/null
+fi
+
+(
+date
+
+for part in $parts; do
+    time "${bindir}/extract_files" -v -d "${localdir}/pool/$part/" \
+       -t "${htmldir}/changelogs/pool/$part/" \
+       -c "$configdir" -w "$tmpdir" \
+       --dumpfile "${filesdir}/changelogs.$part.dump" \
+       --cachefile "${filesdir}/changelogs.cache"
+done
+
+find "${htmldir}/changelogs/" -name log -cmin +7200 \
+ | while read logfile; do
+       dir=$(dirname "$logfile")
+       echo deleting $dir
+       rm -r "$dir"
+       rmdir --ignore-fail-on-non-empty $(dirname "$dir")
+done
+
+date
+) > $log 2>&1

diff --git a/config.sh.sed.in b/config.sh.sed.in
index 1868bce8cd62d7f6dd3af6f60a26f6e9a0e137ec..fffd884f500d91ec241bbea2be9fb94e12924cf8 100644 (file)
--- a/config.sh.sed.in
+++ b/config.sh.sed.in
@@ -20,7 +20,7 @@ cachedir=${topdir}/cache
 # unset this if %SITE% moves somewhere where the packages files
 # cannot be obtained locally
 #
-#localdir=/org/ftp.debian.org/ftp
+localdir=/org/ftp.debian.org/
 
 # path to private ftp directory
 #ftproot=/org/ftp.root

diff --git a/cron.d/300extract_changelogs b/cron.d/300extract_changelogs
deleted file mode 100755 (executable)
index d969781..0000000
--- a/cron.d/300extract_changelogs
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/bin/bash
-
-exit 0
-
-. `dirname $0`/../config.sh
- 
-if test -z "${localdir}"; then
-    echo skipping due to missing local archive
-    exit 1
-fi
-
-#set -e
-
-NOCPY_TEMPLATE=$topdir/etc/copyright_error
-NOCPY_BIN_TEMPLATE=$topdir/etc/copyright_error_bin
-
-logs="$topdir/files/logs"
-log="${logs}/changelogs.log"
-
-test -d "$logs" || mkdir -p "$logs"
-test -d "$tmpdir" || mkdir -p "$tmpdir"
-
-if [ -s "$log" ]
-then
-    savelog -c 14 "$log" > /dev/null
-fi
-
-(
-date
-
-for part in $parts; do
-    time "${bindir}/extract_files" -v -d "${localdir}/pool/$part/" \
-       -t "${htmldir}/changelogs/pool/$part/" \
-       -c "$configdir" -w "$tmpdir" \
-       --dumpfile "${filesdir}/changelogs.$part.dump" \
-       --cachefile "${filesdir}/changelogs.cache"
-done
-
-find "${htmldir}/changelogs/" -name log -cmin +7200 \
- | while read logfile; do
-       dir=$(dirname "$logfile")
-       echo deleting $dir
-       rm -r "$dir"
-       rmdir --ignore-fail-on-non-empty $(dirname "$dir")
-done
-
-date
-) > $log 2>&1