From 45d8ea3f89db0b10f580703ffa4354acf2ee4368 Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Fri, 18 Nov 2016 16:34:38 -0800 Subject: Generate filtered file lists for fedfind to use This adds `filterlist` files alongside the `fullfilelist` and `fullfiletimelist` files. These are much, much shorter lists which skip the entries for packages, ARM device tree boot files and directories. They are intended for consumption by fedfind, so it can stop using rync scraping to discover the image files it looks for. To enable this, we update to a newer version of `create-filelist` from upstream `quick-fedora-mirror` and make `update-fullfiletimelist` create the filterlist files as well. We also delete a couple of old copies of `create-filelist`; nirik made the two roles that use it share a common copy a few months back, but missed deleting the copy each role had in its `files` directory. --- files/scripts/create-filelist | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'files/scripts/create-filelist') diff --git a/files/scripts/create-filelist b/files/scripts/create-filelist index eeba9d089..8fc336736 100755 --- a/files/scripts/create-filelist +++ b/files/scripts/create-filelist @@ -57,7 +57,9 @@ def recursedir(path='.', skip=[], alwaysskip=['.~tmp~']): def parseopts(): null = open(os.devnull, 'w') p = argparse.ArgumentParser( - description='Generate a list of files and times, suitable for consumption by quick-fedora-mirror.') + description='Generate a list of files and times, suitable for consumption by quick-fedora-mirror, ' + 'and a much smaller list with packages, Device Tree boot files, HTML files and ' + 'directories filtered out, for consumption by fedfind.') p.add_argument('-c', '--checksum', action='store_true', help='Include checksums of all repomd.xml files in the file list.') p.add_argument('-C', '--checksum-file', action='append', dest='checksum_files', @@ -73,6 +75,8 @@ def parseopts(): help='Filename of the file list with times (default: stdout).') p.add_argument('-f', '--filelist', type=argparse.FileType('w'), default=null, help='Filename of the file list without times (default: no plain file list is generated).') + p.add_argument('-F', '--filterlist', type=argparse.FileType('w'), default=null, + help='Filename of the filtered file list for fedfind (default: not generated).') opts = p.parse_args() @@ -107,6 +111,10 @@ def main(): for entry in recursedir(skip=opts.skip_files): # opts.filelist.write(entry.path + '\n') print(entry.path, file=opts.filelist) + # write to filtered list if appropriate + skips = ('.rpm', '.drpm', '.dtb', '.html') + if not any(entry.path.endswith(skip) for skip in skips) and not (entry.is_dir()): + print(entry.path, file=opts.filterlist) if entry.name in opts.checksum_files: checksums[entry.path[2:]] = True info = entry.stat(follow_symlinks=False) -- cgit