78 lines
1.9 KiB
Diff
78 lines
1.9 KiB
Diff
From 72c64c76262fef08cc09b25b3ec3ed80ad6136e7 Mon Sep 17 00:00:00 2001
|
|
From: Juan RP <xtraeme@gmail.com>
|
|
Date: Wed, 17 Sep 2014 06:50:46 +0200
|
|
Subject: [PATCH] xbps_file_hash: reduce memory footprint and process file in
|
|
small chunks.
|
|
|
|
With xbps-rindex(8) -c/-r being multithreaded, the memory usage was
|
|
insanely high in the build server, with peaks of almost 4GB (this depends
|
|
on the file size being processed and number of threads).
|
|
|
|
Process the file in small chunks instead to compute the SHA256 hash.
|
|
---
|
|
NEWS | 4 ++++
|
|
lib/util_hash.c | 33 +++++++++++++++------------------
|
|
2 files changed, 19 insertions(+), 18 deletions(-)
|
|
|
|
diff --git a/lib/util_hash.c b/lib/util_hash.c
|
|
index 4663ea1..990e31a 100644
|
|
--- lib/util_hash.c
|
|
+++ lib/util_hash.c
|
|
@@ -61,17 +61,16 @@ char *
|
|
xbps_file_hash(const char *file)
|
|
{
|
|
struct stat st;
|
|
+ SHA256_CTX ctx;
|
|
char hash[SHA256_DIGEST_LENGTH * 2 + 1];
|
|
- unsigned char *buf = NULL, digest[SHA256_DIGEST_LENGTH];
|
|
+ unsigned char digest[SHA256_DIGEST_LENGTH];
|
|
+ ssize_t ret;
|
|
+ unsigned char buf[256];
|
|
int fd;
|
|
|
|
- assert(file != NULL);
|
|
-
|
|
- if ((fd = open(file, O_RDONLY|O_CLOEXEC)) == -1) {
|
|
- free(buf);
|
|
+ if ((fd = open(file, O_RDONLY)) == -1)
|
|
return NULL;
|
|
- }
|
|
- memset(&st, 0, sizeof(st));
|
|
+
|
|
if (fstat(fd, &st) == -1) {
|
|
(void)close(fd);
|
|
return NULL;
|
|
@@ -81,20 +80,18 @@ xbps_file_hash(const char *file)
|
|
return NULL;
|
|
}
|
|
|
|
- buf = malloc(st.st_size);
|
|
- assert(buf);
|
|
+ SHA256_Init(&ctx);
|
|
+ while ((ret = read(fd, buf, sizeof(buf))) > 0)
|
|
+ SHA256_Update(&ctx, buf, ret);
|
|
|
|
- if (read(fd, buf, st.st_size) != st.st_size) {
|
|
- free(buf);
|
|
- (void)close(fd);
|
|
- return NULL;
|
|
- }
|
|
- (void)close(fd);
|
|
- if (SHA256(buf, st.st_size, digest) == NULL) {
|
|
- free(buf);
|
|
+ if (ret == -1) {
|
|
+ /* read error */
|
|
+ (void)close(fd);
|
|
return NULL;
|
|
}
|
|
- free(buf);
|
|
+
|
|
+ SHA256_Final(digest, &ctx);
|
|
+ (void)close(fd);
|
|
digest2string(digest, hash, SHA256_DIGEST_LENGTH);
|
|
|
|
return strdup(hash);
|
|
--
|
|
2.1.0
|
|
|