diff options
| -rw-r--r-- | comps.py | 5 | ||||
| -rw-r--r-- | harddrive.py | 6 | ||||
| -rw-r--r-- | image.py | 3 | ||||
| -rw-r--r-- | isys/isys.py | 4 | ||||
| -rw-r--r-- | todo.py | 2 | ||||
| -rw-r--r-- | urlinstall.py | 19 | ||||
| -rw-r--r-- | utils/Makefile | 4 | ||||
| -rw-r--r-- | utils/genhdlist.c | 194 | ||||
| -rw-r--r-- | utils/hash.c | 201 | ||||
| -rw-r--r-- | utils/hash.h | 28 |
10 files changed, 449 insertions, 17 deletions
@@ -118,6 +118,11 @@ class HeaderList: def list(self): return self.packages.values() + def mergeFullHeaders(self, file): + fd = os.open(file, os.O_RDONLY) + rpm.mergeHeaderListFromFD(self.hdlist, fd, 1000004) + os.close(fd) + def __init__(self, hdlist, compatPackages = None, noscore = 0): self.hdlist = hdlist self.packages = {} diff --git a/harddrive.py b/harddrive.py index 3e6851c31..500ddc5bb 100644 --- a/harddrive.py +++ b/harddrive.py @@ -69,6 +69,9 @@ class OldHardDriveInstallMethod(InstallMethod): self.umountMedia() return HeaderList(hl) + def mergeFullHeaders(self, hdlist): + # since we read headers from the disk, we don't need to do this + def systemMounted(self, fstab, mntPoint, selected): self.mountMedia() @@ -172,6 +175,9 @@ class HardDriveInstallMethod(InstallMethod): return hl + def mergeFullHeaders(self, hdlist): + # since we read headers from the disk, we don't need to do this + def systemMounted(self, fstab, mntPoint, selected): self.mountMedia(1) @@ -22,6 +22,9 @@ class ImageInstallMethod(InstallMethod): def readHeaders(self): return HeaderListFromFile(self.tree + "/RedHat/base/hdlist") + def mergeFullHeaders(self, hdlist): + hdlist.mergeFullHeaders(self.tree + "/RedHat/base/hdlist2") + def writeCleanupPath(self, f): isys.makeDevInode("loop0", "/tmp/loop0") f.write("umount /mnt/runtime\n") diff --git a/isys/isys.py b/isys/isys.py index a7a491c1e..7e801073b 100644 --- a/isys/isys.py +++ b/isys/isys.py @@ -357,7 +357,7 @@ def fbinfo(): return _isys.fbinfo() def ideCdRwList(): - if not os.access("/proc/sys/dev/cdrom/info", os.R_OK): return None + if not os.access("/proc/sys/dev/cdrom/info", os.R_OK): return [] f = open("/proc/sys/dev/cdrom/info", "r") lines = f.readlines() @@ -372,7 +372,7 @@ def ideCdRwList(): if (line and line[0] == "drive name"): line = string.split(line[1]) # no CDROM drives - if not line: return + if not line: return [] for device in line: if device[0:2] == 'sr': @@ -1650,6 +1650,8 @@ class ToDo: self.fstab.mountFilesystems (self.instPath) + self.method.mergeFullHeaders(self.hdList) + if self.upgrade: # An old mtab can cause confusion (esp if loop devices are # in it) diff --git a/urlinstall.py b/urlinstall.py index a417ea628..a38d88d92 100644 --- a/urlinstall.py +++ b/urlinstall.py @@ -35,16 +35,21 @@ class UrlInstallMethod(InstallMethod): tmppath = root + p break - file = tmppath + h[FILENAME] + # h doubles as a filename -- gross + if type("/") == type(h): + fullPath = self.baseUrl + "/" + h + else: + fullPath = self.baseUrl + "/RedHat/RPMS/" + h[FILENAME], + + file = tmppath + os.path.basename(fullPath) connected = 0 while not connected: try: - urllib.urlretrieve(self.baseUrl + "/RedHat/RPMS/" + h[FILENAME], - file) + urllib.urlretrieve(fullPath, file) except IOError, (errnum, msg): log("IOError %s occured getting %s: %s", - errnum, self.baseUrl + "/RedHat/RPMS/" + h[FILENAME], str(msg)) + errnum, fullPath, str(msg)) time.sleep(5) else: connected = 1 @@ -55,7 +60,6 @@ class UrlInstallMethod(InstallMethod): os.remove(fullName) def readHeaders(self): - connected = 0 while not connected: try: @@ -86,6 +90,11 @@ class UrlInstallMethod(InstallMethod): return HeaderList(hl) + def mergeFullHeaders(self, hdlist): + fn = self.getFilename("RedHat/base/hdlist2", None) + hdlist.mergeFullHeaders(fn) + os.unlink(fn) + def __init__(self, url): InstallMethod.__init__(self) diff --git a/utils/Makefile b/utils/Makefile index 677ab31c7..c81cd8f4e 100644 --- a/utils/Makefile +++ b/utils/Makefile @@ -25,8 +25,8 @@ moddeps: moddeps.o ../isys/modutils/util/libutil.a \ ../isys/modutils/obj/libobj.a -lrpm -lrpmio -lbz2 -lz -lpopt -genhdlist: genhdlist.c - $(CC) $(LDFLAGS) $(CFLAGS) -o genhdlist genhdlist.c -lrpm -lrpmio \ +genhdlist: genhdlist.c hash.c + $(CC) $(LDFLAGS) $(CFLAGS) -o genhdlist genhdlist.c hash.c -lrpm -lrpmio \ -lbz2 -lz -lpopt -ldb-3.1 -ldb1 depends: diff --git a/utils/genhdlist.c b/utils/genhdlist.c index f7ec22428..898ce167d 100644 --- a/utils/genhdlist.c +++ b/utils/genhdlist.c @@ -11,14 +11,19 @@ #include <stdio.h> #include <stdlib.h> #include <string.h> +#include <rpm/misc.h> #include <sys/stat.h> #include <sys/types.h> +#include <time.h> #include <unistd.h> +#include "hash.h" + #define FILENAME_TAG 1000000 #define FILESIZE_TAG 1000001 #define CDNUM_TAG 1000002 #define ORDER_TAG 1000003 +#define MATCHER_TAG 1000004 struct onePackageInfo { char * name; @@ -36,10 +41,18 @@ int pkgInfoCmp(const void * a, const void * b) { return strcmp(one->arch, two->arch); } +int tagsInList2[] = { + RPMTAG_FILESIZES, RPMTAG_FILEMODES, RPMTAG_FILEMD5S, RPMTAG_FILELINKTOS, + RPMTAG_FILEFLAGS, RPMTAG_FILELANGS, + RPMTAG_DIRNAMES, RPMTAG_DIRINDEXES, RPMTAG_BASENAMES, + -1 +}; + struct onePackageInfo * pkgList; int pkgListItems = 0; int pkgListAlloced = 0; char ** depOrder = NULL; +hashTable requireTable; /* mmmm... linear search */ int getOrder (char * fn) @@ -64,12 +77,82 @@ int getOrder (char * fn) return -1; } -int onePass(FD_t outfd, const char * dirName, int cdNum) { +int onePrePass(const char * dirName) { + struct dirent * ent; + DIR * dir; + char * subdir = alloca(strlen(dirName) + 20); + FD_t fd; + int rc; + Header h; + int isSource; + char ** requires; + int c; + + sprintf(subdir, "%s/RedHat/RPMS", dirName); + + dir = opendir(subdir); + if (!dir) { + fprintf(stderr,"error opening directory %s: %s\n", subdir, + strerror(errno)); + return 1; + } + + chdir(subdir); + + errno = 0; + ent = readdir(dir); + if (errno) { + perror("readdir"); + return 1; + } + + while (ent) { + int i = strlen (ent->d_name); + + if (i > 4 && strcasecmp (&ent->d_name [i - 4], ".rpm") == 0) { + fd = Fopen(ent->d_name, "r"); + + if (!fd) { + perror("open"); + exit(1); + } + + rc = rpmReadPackageHeader(fd, &h, &isSource, NULL, NULL); + + if (!rc) { + if (headerGetEntry(h, RPMTAG_REQUIRENAME, NULL, + (void **) &requires, &c)) { + while (c--) + if (*requires[c] == '/') + htAddToTable(requireTable, requires[c], "."); + } + + headerFree(h); + /* XXX free requires */ + } + + Fclose(fd); + } + + errno = 0; + ent = readdir(dir); + if (errno) { + perror("readdir"); + return 1; + } + } + + closedir(dir); + + return 0; +} + +int onePass(FD_t outfd, FD_t out2fd, const char * dirName, int cdNum) { FD_t fd; struct dirent * ent; char * subdir = alloca(strlen(dirName) + 20); int errno; - Header h, nh; + Header h, nh, h2; int isSource, rc; int_32 size; DIR * dir; @@ -77,7 +160,11 @@ int onePass(FD_t outfd, const char * dirName, int cdNum) { int_32 * fileSizes; int fileCount; int order = -1; - + char ** newFileList, ** fileNames; + uint_32 * newFileFlags, * fileFlags; + int newFileListCount; + int marker = time(NULL); /* good enough counter; we'll increment it */ + sprintf(subdir, "%s/RedHat/RPMS", dirName); dir = opendir(subdir); @@ -207,11 +294,79 @@ int onePass(FD_t outfd, const char * dirName, int cdNum) { &order, 1); } + expandFilelist(h); + newFileList = NULL; + if (headerGetEntry(h, RPMTAG_OLDFILENAMES, NULL, + (void **) &fileNames, &fileCount)) { + headerGetEntry(h, RPMTAG_FILEFLAGS, NULL, + (void **) &fileFlags, NULL); + + newFileList = malloc(sizeof(*newFileList) * fileCount); + newFileFlags = malloc(sizeof(*newFileList) * fileCount); + newFileListCount = 0; + + for (i = 0; i < fileCount; i++) { + if (htInTable(requireTable, fileNames[i], ".")) { + newFileList[newFileListCount] = strdup(fileNames[i]); + newFileFlags[newFileListCount] = fileFlags[i]; + newFileListCount++; + } + } + + if (!newFileListCount) { + free(newFileList); + free(newFileFlags); + newFileList = NULL; + } + + /* XXX free fileNames */ + } + compressFilelist(h); + + h2 = headerNew(); + for (i = 0; tagsInList2[i] > -1; i++) { + int_32 type, c; + void * p; + + if (headerGetEntry(h, tagsInList2[i], &type, &p, &c)) { + headerAddEntry(h2, tagsInList2[i], type, p, c); + headerRemoveEntry(h, tagsInList2[i]); + } + + /* XXX need to headerFreeData */ + } + + if (newFileList) { + headerAddEntry(h, RPMTAG_OLDFILENAMES, + RPM_STRING_ARRAY_TYPE, + newFileList, newFileListCount); + headerAddEntry(h, RPMTAG_FILEFLAGS, + RPM_INT32_TYPE, + &newFileFlags, newFileListCount); + + compressFilelist(h); + while (newFileListCount--) { + printf("f: %s\n", newFileList[newFileListCount]); + free(newFileList[newFileListCount]); + } + free(newFileList); + free(newFileFlags); + newFileList = NULL; + } + + headerAddEntry(h, MATCHER_TAG, RPM_INT32_TYPE, &marker, 1); + headerAddEntry(h2, MATCHER_TAG, RPM_INT32_TYPE, &marker, 1); + marker++; + nh = headerCopy (h); headerWrite(outfd, nh, HEADER_MAGIC_YES); + headerWrite(out2fd, h2, HEADER_MAGIC_YES); + headerFree(h); headerFree(nh); + headerFree(h2); } + Fclose(fd); } @@ -235,13 +390,14 @@ static void usage(void) { int main(int argc, const char ** argv) { char buf[300]; - FD_t outfd; + FD_t outfd, out2fd; int cdNum = -1; const char ** args; int doNumber = 0; int rc; int i; char * hdListFile = NULL; + char * hdListFile2 = NULL; char * depOrderFile = NULL; poptContext optCon; struct poptOption options[] = { @@ -302,6 +458,8 @@ int main(int argc, const char ** argv) { numpkgs++; } } + + requireTable = htNewTable(1000); if (!hdListFile) { strcpy(buf, args[0]); @@ -310,10 +468,21 @@ int main(int argc, const char ** argv) { } unlink(hdListFile); + + hdListFile2 = malloc(strlen(hdListFile) + 2); + sprintf(hdListFile2, "%s2", hdListFile); outfd = Fopen(hdListFile, "w"); if (!outfd) { - fprintf(stderr,"error creating file %s: %s\n", buf, strerror(errno)); + fprintf(stderr,"error creating file %s: %s\n", hdListFile, + strerror(errno)); + return 1; + } + + out2fd = Fopen(hdListFile2, "w"); + if (!out2fd) { + fprintf(stderr,"error creating file %s: %s\n", hdListFile2, + strerror(errno)); return 1; } @@ -325,14 +494,23 @@ int main(int argc, const char ** argv) { /* exit(1); */ /* } */ - while (args[0]) { - if (onePass(outfd, args[0], cdNum)) + i = 0; + while (args[i]) { + if (onePrePass(args[i])) + return 1; + i++; + } + + i = 0; + while (args[i]) { + if (onePass(outfd, out2fd, args[i], cdNum)) return 1; if (doNumber) cdNum++; - args++; + i++; } Fclose(outfd); + Fclose(out2fd); poptFreeContext(optCon); diff --git a/utils/hash.c b/utils/hash.c new file mode 100644 index 000000000..2bc6e0bf2 --- /dev/null +++ b/utils/hash.c @@ -0,0 +1,201 @@ +#include <stdlib.h> +#include <unistd.h> +#include <stdio.h> +#include <string.h> + +#include "hash.h" + +#define CHUNK 1 + +struct filePath { + char * dir; + char * base; +} ; + +struct bucket { + struct filePath * data; + int allocated; + int firstFree; /* as in data[firstFree] */ +}; + +struct hash_table { + int size; + int entries; + int overHead; + struct bucket *bucket; +}; + +struct hash_table *htNewTable(int size) +{ + struct hash_table *res; + int i = 0; + + res = malloc(sizeof(struct hash_table)); + res->bucket = malloc(sizeof(struct bucket) * size); + res->size = size; + res->entries = 0; + res->overHead = sizeof(struct bucket) * size + CHUNK * sizeof(char *); + + while (i < size) { + res->bucket[i].data = malloc(CHUNK * sizeof(*res->bucket[i].data)); + res->bucket[i].allocated = CHUNK; + res->bucket[i].firstFree = 0; + i++; + } + + return res; +} + +void htFreeHashTable(struct hash_table *ht) +{ + struct bucket * b; + int item; + + b = ht->bucket; + while (ht->size--) { + for (item = 0; item < b->firstFree; item++) { + free(b->data[item].dir); + free(b->data[item].base); + } + free(b->data); + b++; + } + free(ht->bucket); + free(ht); +} + +void htHashStats(struct hash_table *t) +{ + int i = 0; + int empty = 0; + + while (i < t->size) { + if (t->bucket[i].firstFree != 0) { + /*printf("Bucket %d used %d\n", i, t->bucket[i].firstFree);*/ + } else { + empty++; + } + i++; + } + + printf("Total Buckets : %d\n", t->size); + printf("Empty Buckets : %d\n", empty); + printf("Total Entries : %d\n", t->entries); + printf("Total Overhead: %d\n", t->overHead); + printf("Avergage Depth: %f\n", (double)t->entries / (double)t->size); +} + +static unsigned int htHashStrings(const char *s, const char *t) +{ + unsigned int res = 0; + + while (*s) + res = ((res<<1) + (int)(*(s++))); + while (*t) + res = ((res<<1) + (int)(*(t++))); + + return res; +} + +/* returns bucket # containing item, or -1 */ +static int in_table_aux(struct hash_table *t, int hash, const char * dir, + const char * base) +{ + int x; + + x = 0; + while (x < t->bucket[hash].firstFree) { + if (! strcmp(t->bucket[hash].data[x].dir, dir) && + ! strcmp(t->bucket[hash].data[x].base, base)) { + return x; + } + x++; + } + + return -1; +} + +int htInTable(struct hash_table *t, const char * dir, const char * base) +{ + int hash; + + hash = htHashStrings(dir, base) % t->size; + + if (in_table_aux(t, hash, dir, base) == -1) + return 0; + return 1; +} + +void htAddToTable(struct hash_table *t, const char * dir, const char * base) +{ + static int hash = 1; + + if (!dir || !base) + return; + + hash = htHashStrings(dir, base) % t->size; + if (in_table_aux(t, hash, dir, base) != -1) + return; + + if (t->bucket[hash].firstFree == t->bucket[hash].allocated) { + t->bucket[hash].allocated += CHUNK; + t->bucket[hash].data = + realloc(t->bucket[hash].data, + t->bucket[hash].allocated * sizeof(*(t->bucket->data))); + /*printf("Bucket %d grew to %d\n", hash, t->bucket[hash].allocated);*/ + t->overHead += sizeof(char *) * CHUNK; + } + /*printf("In bucket %d, item %d\n", hash, t->bucket[hash].firstFree);*/ + t->bucket[hash].data[t->bucket[hash].firstFree].dir = strdup(dir); + t->bucket[hash].data[t->bucket[hash].firstFree++].base = strdup(base); + t->entries++; +} + +void htRemoveFromTable(struct hash_table *t, const char * dir, + const char * base) { + int hash; + int item; + int last; + + hash = htHashStrings(dir, base) % t->size; + if ((item = in_table_aux(t, hash, dir, base)) == -1) { + return; + } + + free(t->bucket[hash].data[item].dir); + free(t->bucket[hash].data[item].base); + + last = --t->bucket[hash].firstFree; + t->bucket[hash].data[item] = t->bucket[hash].data[last]; +} + +int htNumEntries(struct hash_table *t) { + return t->entries; +} + +void htIterStart(htIterator * iter) { + iter->bucket = 0; + iter->item = -1; +} + +int htIterGetNext(struct hash_table * t, htIterator * iter, + const char ** dir, const char ** base) { + iter->item++; + + while (iter->bucket < t->size) { + if (iter->item < t->bucket[iter->bucket].firstFree) { + *dir = t->bucket[iter->bucket].data[iter->item].dir; + *base = t->bucket[iter->bucket].data[iter->item].base; + + return 1; + } + + iter->item++; + if (iter->item >= t->bucket[iter->bucket].firstFree) { + iter->bucket++; + iter->item = 0; + } + } + + return 0; +} diff --git a/utils/hash.h b/utils/hash.h new file mode 100644 index 000000000..954ad3bf4 --- /dev/null +++ b/utils/hash.h @@ -0,0 +1,28 @@ +#ifndef H_HASH +#define H_HASH + +struct hash_table; +typedef struct hash_table * hashTable; + +struct ht_iterator { + int bucket; + int item; +}; + +typedef struct ht_iterator htIterator; + +struct hash_table *htNewTable(int size); +void htFreeHashTable(struct hash_table *ht); +int htInTable(struct hash_table *t, const char * dir, const char * base); +void htAddToTable(struct hash_table *t, const char * dir, const char * base); +void htPrintHashStats(struct hash_table *t); +int htNumEntries(struct hash_table *t); +void htRemoveFromTable(struct hash_table *t, const char * dir, + const char * base); + +/* these use static storage */ +void htIterStart(htIterator * iter); +int htIterGetNext(struct hash_table * t, htIterator * iter, + const char ** dir, const char ** base); + +#endif |