Update create-filelist and wrapper from upstream quick mirror.

Merge all the various copies of the wrapper into one copy.
Setup arguments as needed for each call.

2 files changed, 116 insertions, 38 deletions

diff --git a/files/scripts/create-filelist b/files/scripts/create-filelist
index adf8104c6..eeba9d089 100755
--- a/files/scripts/create-filelist
+++ b/files/scripts/create-filelist
@@ -63,7 +63,7 @@ def parseopts():
     p.add_argument('-C', '--checksum-file', action='append', dest='checksum_files',
                    help='Include checksums of all instances of the specified file.')
     p.add_argument('-s', '--skip', action='store_true',
-                   help='Skip the --filelist file in the top directory')
+                   help='Skip the file lists in the top directory')
     p.add_argument('-S', '--skip-file', action='append', dest='skip_files',
                    help='Skip the specified file in the top directory.')
 
@@ -87,6 +87,8 @@ def parseopts():
     if opts.skip:
         if not opts.timelist.name == '<stdout>':
             opts.skip_files += [opts.timelist.name]
+        if not opts.filelist.name == '<stdout>':
+            opts.skip_files += [opts.filelist.name]
 
     return opts
 
@@ -114,14 +116,15 @@ def main():
         # opts.timelist.write('{0}\t{1}\t{2}\n'.format(modtime, ftype, entry.path[2:]))
         print('{0}\t{1}\t{2}\t{3}'.format(modtime, ftype, size, entry.path[2:]), file=opts.timelist)
 
-    if not checksums:
-        sys.exit(0)
-
     print('\n[Checksums SHA1]', file=opts.timelist)
 
+    # It's OK if the checksum section is empty, but we should include it anyway
+    # as the client expects it.
     for f in sorted(checksums):
         print('{0}\t{1}'.format(sha1(f), f), file=opts.timelist)
 
+    print('\n[End]', file=opts.timelist)
+
 
 if __name__ == '__main__':
         main()
diff --git a/files/scripts/update-fullfiletimelist b/files/scripts/update-fullfiletimelist
index 44de5f212..016ca8ed6 100755
--- a/files/scripts/update-fullfiletimelist
+++ b/files/scripts/update-fullfiletimelist
@@ -1,29 +1,100 @@
 #!/bin/bash
+
+# Note: this is only an example of how you'd call create-filelist.  Edit to fit
+# your requirements.  Note that you must supply a valid path for the lockfile,
+# and it must be outside of your repository unless you want that lockfile to
+# show up in your file lists.
+
 # Takes a list of module names.  Generates file lists for all of them and them
 # moves them into place at once.  If you are creating hardlinks between rsync
 # modules, it is required that you update the file lists of both mirrors at the
 # same time.  Otherwise the clients may make separate copies of the files.
 
-mods=$*
+# The directory where all of the modules live
+# Or pass it with -t
+TOPD=/srv/mirror/pub
 
-if [[ -z $mods ]]; then
-    echo "usage: $0 <module> [<module> ...]"
-    exit 2
-fi
+# The modules to process.  Or pass them on the command line.
+MODS=()
 
+# Path to the create-filelist program.
+# Or specify it with -p.
 CREATE=/usr/local/bin/create-filelist
 
-# A single lockfile for everything we're modifying
-LOCKFILE=/srv/.lock.create-filelist
-
-# The directory where all of the modules live
-TOPD=/srv/pub
-
 # These strings will be eval'ed later with $mod replaced by its value in
 # context.
 FILELIST=fullfilelist
 TIMELIST='fullfiletimelist-$mod'
 
+usage () {
+    echo
+    echo "Usage: $0 [-l lockfile] [-p creator path] [-t top directory] module [module ...]"
+    echo
+    echo "    -l: Path to the lock file"
+    echo "    -p: Path to the create-filelist program"
+    echo "    -t: Path to directory containing modules"
+    echo
+    echo "At least one module to process must be provided."
+    echo "All paths must be absolute."
+}
+
+while [[ $# > 0 ]]; do
+    opt=$1
+    case $opt in
+        -l)
+            LOCKFILE=$(realpath $2)
+            shift
+            ;;
+        -p)
+            CREATE=$(realpath $2)
+            shift
+            ;;
+        -t)
+            TOPD=$(realpath $2)
+            shift
+            ;;
+        -*)
+            (>&2 echo "Unknown option $opt."; usage)
+            exit 1
+            ;;
+        *)  # Remaining args are modules
+            MODS+=($opt)
+            ;;
+    esac
+    shift
+done
+
+if [[ -z $LOCKFILE ]]; then
+    (>&2 echo "Must specify LOCKFILE, either by editing the source or via the -l option."; usage)
+    exit 2
+fi
+if [[ ! -d $(dirname $LOCKFILE) ]]; then
+    (>&2 echo "Given directory $(dirname $LOCKFILE) does not exist."; usage)
+    exit 2
+fi
+if [[ ! -f $CREATE ]]; then
+    (>&2 echo "Specified executable $CREATE does not exist."; usage)
+    exit 2
+fi
+
+if [[ ! -d $TOPD ]]; then
+    (>&2 echo "Provided directory $TOPD does not exist."; usage)
+    exit 2
+fi
+
+if [[ ${#MODS[@]} -eq 0 ]]; then
+    (>&2 echo "No modules specified"; usage)
+    exit 2
+fi
+
+tmpd=$(mktemp -d -t create-filelist.XXXXXXXXXX)
+if [[ $? -ne 0 ]]; then
+    (>&2 echo "Creating temporary directory failed?")
+    exit 1
+fi
+trap "rm -rf $tmpd" EXIT
+cd $tmpd
+
 (
     # We want to wait forever until we can do what we're asked
     flock -x 9
@@ -33,42 +104,46 @@ TIMELIST='fullfiletimelist-$mod'
     # flock -w 120 9 || exit 1  - Waits 120 seconds and then gives up
     # Don't change the '9', unless you change the last line of this script.
 
-    tmpd=$(mktemp -d -t create-filelist.XXXXXXXXXX)
-    trap "rm -rf $tmpd" EXIT
-    cd $tmpd
-
-    # Create all of the filelists
-    for mod in $mods; do
-        $CREATE -c -s -d $TOPD/$mod -f fl-$mod -t tl-$mod
-    done
-
-    # Now diff the file lists and delete the ones which didn't change
-    for mod in $mods; do
+    for mod in $MODS; do
         currentfl=$TOPD/$mod/${FILELIST/'$mod'/$mod}
         currenttl=$TOPD/$mod/${TIMELIST/'$mod'/$mod}
+        flname=$(basename $currentfl)
+        tlname=$(basename $currenttl)
+
+        $CREATE -c -s -d $TOPD/$mod -f $flname -t $tlname
 
         # If a file list exsts and doesn't differ from what we just generated,
         # delete the latter.
-        if [[ -f $currentfl ]] && diff -q fl-$mod $currentfl > /dev/null; then
-            rm -f $fl
+        if [[ -f $currentfl ]] && diff -q $currentfl $flname > /dev/null; then
+            rm -f $flname
         fi
-        if [[ -f $currenttl ]] && diff -q tl-$mod $currenttl > /dev/null; then
-            rm -f $fl
+        if [[ -f $currenttl ]] && diff -q $currenttl $tlname > /dev/null; then
+            rm -f $tlname
         fi
     done
 
-    # And finally, move all of the files which need updating into place
-    for mod in $mods; do
+    # Now we have the new file lists but in a temporary directory which
+    # probably isn't on the same filesystem.  Copy them to temporary files in
+    # the right place.
+    for mod in $MODS; do
         currentfl=$TOPD/$mod/${FILELIST/'$mod'/$mod}
         currenttl=$TOPD/$mod/${TIMELIST/'$mod'/$mod}
+        flname=$(basename $currentfl)
+        fldir=$(dirname $currentfl)
+        tlname=$(basename $currenttl)
+        tldir=$(dirname $currenttl)
 
-        if [[ -f fl-$mod ]]; then
-            chmod 644 fl-$mod
-            cp -p fl-$mod $currentfl
+        if [[ -f $flname ]]; then
+            tmpf=$(mktemp -p $fldir $flname.XXXXXXXXXX)
+            cp -p $flname $tmpf
+            chmod 644 $tmpf
+            mv $tmpf $currentfl
         fi
-        if [[ -f tl-$mod ]]; then
-            chmod 644 tl-$mod
-            cp -p tl-$mod $currenttl
+        if [[ -f $tlname ]]; then
+            tmpf=$(mktemp -p $tldir $tlname.XXXXXXXXXX)
+            cp -p $tlname $tmpf
+            chmod 644 $tmpf
+            mv $tmpf $currenttl
         fi
     done