From 46e23f37ba42ccb744da6526c634653276b461b1 Mon Sep 17 00:00:00 2001 From: Henrik Lissner Date: Sat, 10 Sep 2022 01:01:28 +0200 Subject: [PATCH] fix: memory leak & freezes on native-comp+pgtk builds b7f84bd introduced a nasty regression that caused an infinite loop and runaway memory usage on some pgtk+native-comp builds of Emacs when it attempted to perform deferred native compilation of your packages. This would make Emacs unusable and, if left alone, could even crash your system. The only Emacs builds I'm certain are affected are derived from flatwhatson/emacs (like emacs-pgtk-native-comp on Guix and Arch Linux in particular). 28.1 stable and master (on emacs-mirror/emacs@e13509468b7c) are unaffected. It appears that some, earlier pgtk builds stack idle timers differently. I'm not entirely sure how, because it doesn't manifest in more recent builds of Emacs, and I'm already burnt out on debugging this, but here's how Doom encountered it: Doom has an incremental package loader; it loads packages, piecemeal, after Emacs has been idle for 2s, then again every 0.75s until it finishes or the user sends input (then it waits another 2s before starting again). However, if at any time the iloader detected that native-compilation is in progress, it waits 2s before trying again (repeat, until native-comp is done). But here's the catch, given the following: (run-with-idle-timer 2 nil (lambda () (run-with-idle-timer 1 nil (lambda () (message "hi"))))) I had assumed "hi" would be emitted after 3 seconds (once idle), but instead it is emitted after 2. Like this, Doom's iloader would elapse one idle timer directly into another, ad infinitum, until Emacs was forcibly killed. By switching to run-at-time and employing my own rudimentary idle timer, I avoid this issue. Also, the iloader no longer needs to be considerate of native-comp, because the latter does its own rate-limiting controlled by native-comp-async-jobs-number. Amend: b7f84bdd0105 --- lisp/doom-start.el | 60 ++++++++++++++++++++++++---------------------- lisp/doom.el | 21 ++++++---------- 2 files changed, 39 insertions(+), 42 deletions(-) diff --git a/lisp/doom-start.el b/lisp/doom-start.el index cde2d7cde..69cc9a449 100644 --- a/lisp/doom-start.el +++ b/lisp/doom-start.el @@ -84,7 +84,7 @@ If you want to disable incremental loading altogether, either remove `doom-incremental-first-idle-timer' to nil. Incremental loading does not occur in daemon sessions (they are loaded immediately at startup).") -(defvar doom-incremental-first-idle-timer (if (featurep 'native-compile) 3.0 2.0) +(defvar doom-incremental-first-idle-timer 2.0 "How long (in idle seconds) until incremental loading starts. Set this to nil to disable incremental loading.") @@ -100,36 +100,40 @@ Set this to nil to disable incremental loading.") If NOW is non-nil, load PACKAGES incrementally, in `doom-incremental-idle-timer' intervals." - (if (not now) - (appendq! doom-incremental-packages packages) - (if (and packages (bound-and-true-p comp-files-queue)) - (run-with-idle-timer doom-incremental-idle-timer - nil #'doom-load-packages-incrementally - packages t) + (let ((gc-cons-threshold most-positive-fixnum)) + (if (not now) + (cl-callf append doom-incremental-packages packages) (while packages - (let* ((gc-cons-threshold most-positive-fixnum) - (req (pop packages))) - (unless (featurep req) - (doom-log "Incrementally loading %s" req) + (let ((req (pop packages)) + idle-time) + (if (featurep req) + (doom-log "[ILoader] Already loaded: %s (%d left)" req (length packages)) (condition-case-unless-debug e - (or (while-no-input - ;; If `default-directory' is a directory that doesn't exist - ;; or is unreadable, Emacs throws up file-missing errors, so - ;; we set it to a directory we know exists and is readable. - (let ((default-directory doom-emacs-dir) - (inhibit-message t) - file-name-handler-alist) - (require req nil t)) - t) - (push req packages)) + (and + (or (null (setq idle-time (current-idle-time))) + (< (float-time idle-time) doom-incremental-first-idle-timer) + (not + (while-no-input + (doom-log "[ILoader] Loading: %s (%d left)" req (length packages)) + ;; If `default-directory' doesn't exist or is + ;; unreadable, Emacs throws file errors. + (let ((default-directory doom-emacs-dir) + (inhibit-message t) + (file-name-handler-alist + (list (rassq 'jka-compr-handler file-name-handler-alist)))) + (require req nil t) + t)))) + (push req packages)) (error - (message "Failed to load %S package incrementally, because: %s" - req e))) - (if (not packages) - (doom-log "Finished incremental loading") - (run-with-idle-timer doom-incremental-idle-timer - nil #'doom-load-packages-incrementally - packages t) + (message "Error: failed to incrementally load %S because: %s" req e) + (setq packages nil))) + (if (null packages) + (doom-log "[ILoader] Finished!") + (run-at-time (if idle-time + doom-incremental-idle-timer + doom-incremental-first-idle-timer) + nil #'doom-load-packages-incrementally + packages t) (setq packages nil)))))))) (defun doom-load-packages-incrementally-h () diff --git a/lisp/doom.el b/lisp/doom.el index 1ae0752e0..3e72b49a8 100644 --- a/lisp/doom.el +++ b/lisp/doom.el @@ -321,23 +321,16 @@ users).") ;; ;;; Native Compilation support (http://akrl.sdf.org/gccemacs.html) -(when (featurep 'native-compile) - ;; Enable deferred compilation and disable ahead-of-time compilation, so we - ;; don't bog down the install process with an excruciatingly long compile - ;; times. It will mean more CPU time at runtime, but given its asynchronosity, - ;; this is acceptable. - (setq native-comp-deferred-compilation t - straight-disable-native-compile t) - - ;; Suppress compiler warnings, to avoid inundating users will popups. They - ;; don't cause breakage, so it's not worth dedicating screen estate to them. - (setq native-comp-async-report-warnings-errors init-file-debug - native-comp-warning-on-missing-source init-file-debug) - +(when (boundp 'native-comp-eln-load-path) ;; Don't store eln files in ~/.emacs.d/eln-cache (where they can easily be ;; deleted by 'doom upgrade'). ;; REVIEW Use `startup-redirect-eln-cache' when 28 support is dropped - (add-to-list 'native-comp-eln-load-path (expand-file-name "eln/" doom-cache-dir))) + (add-to-list 'native-comp-eln-load-path (expand-file-name "eln/" doom-cache-dir)) + + ;; UX: Suppress compiler warnings and don't inundate users with their popups. + ;; They are rarely more than warnings, so are safe to ignore. + (setq native-comp-async-report-warnings-errors init-file-debug + native-comp-warning-on-missing-source init-file-debug)) ;;