diff options
| author | Christopher Baines <mail@cbaines.net> | 2023-09-11 15:42:13 +0100 |
|---|---|---|
| committer | Christopher Baines <mail@cbaines.net> | 2023-09-12 13:11:00 +0100 |
| commit | 4cd601a887d4a2c21d929c02d87d33b9bedcf9f1 (patch) | |
| tree | a2946284b38c39f041bb876e6272586f9fb063a3 | |
| parent | 8f59645a21c185cf17c0297ede804cb079bdb0aa (diff) | |
| download | nar-herder-4cd601a887d4a2c21d929c02d87d33b9bedcf9f1.tar nar-herder-4cd601a887d4a2c21d929c02d87d33b9bedcf9f1.tar.gz | |
Use fold-nar-files for check-storage
To avoid calling get-nar-files, as this doesn't scale well.
| -rw-r--r-- | nar-herder/database.scm | 19 | ||||
| -rw-r--r-- | nar-herder/storage.scm | 33 |
2 files changed, 35 insertions, 17 deletions
diff --git a/nar-herder/database.scm b/nar-herder/database.scm index 2bac86f..7a550ec 100644 --- a/nar-herder/database.scm +++ b/nar-herder/database.scm @@ -57,6 +57,7 @@ database-fold-all-narinfo-files database-map-all-narinfo-files + database-count-narinfo-files database-insert-cached-narinfo-file database-select-cached-narinfo-file-by-hash @@ -1117,6 +1118,24 @@ FROM narinfo_files" result)) '())) +(define (database-count-narinfo-files database) + (call-with-worker-thread + (database-reader-thread-channel database) + (lambda (db) + (let ((statement + (sqlite-prepare + db + " +SELECT COUNT(*) FROM narinfo_files" + #:cache? #t))) + + (let ((result + (vector-ref (sqlite-step statement) + 0))) + (sqlite-reset statement) + + result))))) + (define (database-insert-cached-narinfo-file database narinfo-id size diff --git a/nar-herder/storage.scm b/nar-herder/storage.scm index d5fae15..c53b106 100644 --- a/nar-herder/storage.scm +++ b/nar-herder/storage.scm @@ -212,7 +212,8 @@ (set! stored-files-count (1+ stored-files-count)) (set! not-stored-files-count (1+ not-stored-files-count))) - (if (or (and stored? nar-stored?) + (if (or (eq? stored? 'both) + (and stored? nar-stored?) (and (not stored?) (not nar-stored?))) (proc nar result) @@ -264,20 +265,19 @@ #:label-values '((stored . "false"))))) (define (check-storage database storage-root metrics-registry) - (define files - (get-nar-files database storage-root metrics-registry - #:stored? #t)) - (define files-count - (length files)) + (database-count-narinfo-files database)) (call-with-progress-reporter (progress-reporter/bar files-count (simple-format #f "checking ~A files" files-count) (current-error-port)) (lambda (report) - (fold - (lambda (file result) + (fold-nar-files + database + storage-root + metrics-registry + (lambda (file _) (let* ((full-filename (string-append storage-root (uri-decode (assq-ref file 'url)))) @@ -286,15 +286,14 @@ (database-size (assq-ref file 'size))) (report) - (if (not (= file-size database-size)) - (begin - (newline) - (log-msg 'WARN "file " full-filename - " has inconsistent size (database: " - database-size ", file: " file-size ")")) - #f))) - '() - files)))) + (unless (= file-size database-size) + (newline) + (log-msg 'WARN "file " full-filename + " has inconsistent size (database: " + database-size ", file: " file-size ")")) + #f)) + #f + #:stored? 'both)))) (define (at-most max-length lst) "If LST is shorter than MAX-LENGTH, return it and the empty list; otherwise |