Re: Graphics preview in parallel

Pavel Sanda <[email protected]> Sat, 11 Jul 2026 04:03:09 +0200
Newsgroups gmane.editors.lyx.devel
Message-ID <[email protected]>
On Fri, Jul 10, 2026 at 11:38:40AM +0200, Scott Kostyshak wrote:
> > I have deal for you. We keep the number of threads on the limit I proposed
> > but I will add parameter [threads_count] to the lfun you haven't seen yet,
> > so if you bind this lfun with your personal lucky number you will get
> > the full doc image load on a single stroke.
> > What you think?
> 
> Sounds good! Can we (I can make a note to check it out after your commit
> if you want) add a special argument that would mean all cores? Maybe we
> can call it "all" ?

'all' is for including childern, but 'max' is your friend, see the commit
e0dd8db6ade.

For our convenience I also attach patch for 2.5.x if you want to run it
in your local tree, see attached patch -- I do not plan to backport.

Please try how it works and let me know.

> > The only thing to keep in mind is that if you put too many threads and have too
> > little memory you can enter swap hell.
> 
> Are you saying that the "new" desktop they just gave me at my university
> is not ideal (screenshot attached)?

Haha, I see that memory became really expensive nowadays.

Pavel

-- 
lyx-devel mailing list
[email protected]
https://lists.lyx.org/mailman/listinfo/lyx-devel
buffer-load-graphics.25x.patch (text/x-diff, 9.4 KB)
diff --git a/src/Buffer.cpp b/src/Buffer.cpp
index 2ea6125e06..7b41623caf 100644
--- a/src/Buffer.cpp
+++ b/src/Buffer.cpp
@@ -70,6 +70,7 @@
 #include "xml.h"
 
 #include "insets/InsetBranch.h"
+#include "insets/InsetGraphics.h"
 #include "insets/InsetInclude.h"
 #include "insets/InsetText.h"
 
@@ -93,6 +94,7 @@
 #include "support/FileName.h"
 #include "support/FileNameList.h"
 #include "support/filetools.h"
+#include "support/ForkedCalls.h"
 #include "support/gettext.h"
 #include "support/gzstream.h"
 #include "support/Lexer.h"
@@ -2887,6 +2889,10 @@ bool Buffer::getStatus(FuncRequest const & cmd, FuncStatus & flag) const
 		enable = (d->preview_file_).exists() && !(d->preview_file_).isFileEmpty();
 		break;
 
+	case LFUN_BUFFER_LOAD_GRAPHICS:
+		enable = true;
+		break;
+
 	case LFUN_CHANGES_TRACK:
 		flag.setEnabled(true);
 		flag.setOnOff(params().track_changes);
@@ -3162,6 +3168,33 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
 			dr.setMessage(_("Error viewing the output file."));
 		break;
 
+	case LFUN_BUFFER_LOAD_GRAPHICS: {
+		ListOfBuffers bufs;
+		for (string const & arg :
+		     getVectorFromString(to_utf8(func.argument()), " ")) {
+			if (arg == "all")	//include all childern
+				bufs = getDescendants();
+			else if (arg == "max")	//use all available cores
+				ForkedCallQueue::setMaxRunning(0);
+			else
+				ForkedCallQueue::setMaxRunning(convert<int>(arg));
+		}
+		bufs.push_front(this);
+		int counted = 0;
+		for (Buffer * b : bufs) {
+			InsetIterator it = begin(b->inset());
+			InsetIterator const itend = end(b->inset());
+			for (; it != itend; ++it) {
+				if (it->lyxCode() == GRAPHICS_CODE) {
+					static_cast<InsetGraphics const &>(*it).preload();
+					++counted;
+				}
+			}
+		}
+		//dr.setMessage(bformat(_("Queued %1$d image(s) for preview."), counted));
+		break;
+	}
+
 	case LFUN_CHANGES_TRACK:
 		if (params().save_transient_properties)
 			undo().recordUndoBufferParams();
diff --git a/src/FuncCode.h b/src/FuncCode.h
index b63d21abe1..d5418fd42d 100644
--- a/src/FuncCode.h
+++ b/src/FuncCode.h
@@ -516,7 +516,8 @@ enum FuncCode
 	LFUN_ERRORS_SHOW,               // spitz 20241231,
 	LFUN_ERROR_NEXT,                // spitz 20200101,
 	LFUN_BUFFER_UPDATE_EXTERNAL,    // spitz 20250406,
-	/// 405
+	LFUN_BUFFER_LOAD_GRAPHICS,      // ps 20260623
+	/// 406
 	LFUN_LASTACTION                 // end of the table
 };
 
diff --git a/src/LyXAction.cpp b/src/LyXAction.cpp
index fcf554e4d2..03f67f8ad3 100644
--- a/src/LyXAction.cpp
+++ b/src/LyXAction.cpp
@@ -904,6 +904,21 @@ void LyXAction::init()
  */
 		{ LFUN_BUFFER_VIEW_CACHE, "buffer-view-cache", ReadOnly, Buffer },
 
+/*!
+ * \var lyx::FuncCode lyx::LFUN_BUFFER_LOAD_GRAPHICS
+ * \li Action: Triggers image preview for every image in the current buffer,
+               instead of waiting for each image to be scrolled into view.
+ * \li Syntax: buffer-load-graphics [all] [<THREADS>|max]
+ * \li Params: all: also process all child documents; by default only the
+               current buffer is processed.
+               <THREADS>: fix number os threads for conversion, or 'max' to use 
+	       all cores available on the system.\n
+	       The setting persists until changed again.
+ * \li Origin: ps, 23 Jun 2026
+ * \endvar
+ */
+		{ LFUN_BUFFER_LOAD_GRAPHICS, "buffer-load-graphics", ReadOnly, Buffer },
+
 /*!
  * \var lyx::FuncCode lyx::LFUN_BUFFER_WRITE
  * \li Action: Saves the current buffer.
diff --git a/src/insets/InsetGraphics.cpp b/src/insets/InsetGraphics.cpp
index dacde1b179..11de00ef00 100644
--- a/src/insets/InsetGraphics.cpp
+++ b/src/insets/InsetGraphics.cpp
@@ -1158,6 +1158,13 @@ InsetGraphicsParams const & InsetGraphics::params() const
 }
 
 
+void InsetGraphics::preload() const
+{
+	if (graphic_)
+		graphic_->preload();
+}
+
+
 void InsetGraphics::editGraphics(InsetGraphicsParams const & p) const
 {
 	theFormats().edit(buffer(), p.filename,
diff --git a/src/insets/InsetGraphics.h b/src/insets/InsetGraphics.h
index 3fd986bf7b..99e593306c 100644
--- a/src/insets/InsetGraphics.h
+++ b/src/insets/InsetGraphics.h
@@ -91,6 +91,8 @@ public:
 	docstring layoutName() const override { return from_ascii("Graphics"); }
 	/// Get the inset parameters, used by the GUIndependent dialog.
 	InsetGraphicsParams const & params() const;
+	/// Force preview generation now (skip the wait-for-paint).
+	void preload() const;
 
 	///
 	int topOffset(BufferView const *) const override { return 0; }
diff --git a/src/insets/RenderGraphic.cpp b/src/insets/RenderGraphic.cpp
index 67d44e1bb4..5f88372953 100644
--- a/src/insets/RenderGraphic.cpp
+++ b/src/insets/RenderGraphic.cpp
@@ -51,6 +51,7 @@ RenderBase * RenderGraphic::clone(Inset const * inset) const
 	return new RenderGraphic(*this, inset);
 }
 
+
 void RenderGraphic::reload() const
 {
 	loader_.reload();
@@ -132,6 +133,17 @@ bool readyToDisplay(graphics::Loader const & loader)
 } // namespace
 
 
+void RenderGraphic::preload() const
+{
+	if (!displayGraphic(params_))
+		return;
+	// ErrorConverting is included so we retry failed conversions.
+	if (loader_.status() == graphics::WaitingToLoad
+	    || loader_.status() == graphics::ErrorConverting)
+		loader_.startLoading();
+}
+
+
 void RenderGraphic::metrics(MetricsInfo & mi, Dimension & dim) const
 {
 	if (displayGraphic(params_)) {
diff --git a/src/insets/RenderGraphic.h b/src/insets/RenderGraphic.h
index fc631a0c7a..2b175008a3 100644
--- a/src/insets/RenderGraphic.h
+++ b/src/insets/RenderGraphic.h
@@ -35,6 +35,8 @@ public:
 
 	/// Refresh the info about which file to display and how to display it.
 	void update(graphics::Params const & params);
+	/// Trigger conversion, no-op if the image is already loading, loaded.
+	void preload() const;
 	/// Reloads the image if necessary
 	void reload() const;
 
diff --git a/src/support/ForkedCalls.cpp b/src/support/ForkedCalls.cpp
index 6113080a3b..2c96280f20 100644
--- a/src/support/ForkedCalls.cpp
+++ b/src/support/ForkedCalls.cpp
@@ -20,11 +20,13 @@
 #include "support/lyxlib.h"
 #include "support/Timeout.h"
 
+#include <algorithm>
 #include <cerrno>
 #include <cstring>
 #include <list>
 #include <queue>
 #include <sstream>
+#include <thread>
 #include <utility>
 #include <vector>
 
@@ -441,18 +443,23 @@ typedef pair<string, ForkedCall::sigPtr> Process;
 /// in-progress queue
 static queue<Process> callQueue_;
 
-/// flag whether queue is running
-static bool running_ = false;
+/// Maximum number of forked children allowed to run concurrently.
+/// Graphics conversion is the only client of this queue.
+/// Default: min(4, #cores), serial if the hardware cannot be queried.
+/// Adjustable at runtime via setMaxRunning().
+static int max_running_ =
+	max(1, min(4, int(std::thread::hardware_concurrency())));
+
+/// Number of children currently running.
+static int running_count_ = 0;
 
 ///
-void startCaller();
-///
-void stopCaller();
+void callNext();
 ///
 void callback(pid_t, int);
 
-/** Add a process to the queue. Processes are forked sequentially
- *  only one is running at a time.
+/** Add a process to the queue. Up to max_running_ processes are
+ *  forked in parallel; the remainder wait for a slot to free up.
  *  Connect to the returned signal and you'll be informed when
  *  the process has ended.
  */
@@ -461,8 +468,10 @@ ForkedCall::sigPtr add(string const & process)
 	ForkedCall::sigPtr ptr;
 	ptr.reset(new ForkedCall::sig);
 	callQueue_.push(Process(process, ptr));
-	if (!running_)
-		startCaller();
+	if (running_count_ == 0)
+		LYXERR(Debug::GRAPHICS, "ForkedCallQueue: waking up");
+	while (running_count_ < max_running_ && !callQueue_.empty())
+		callNext();
 	return ptr;
 }
 
@@ -475,6 +484,7 @@ void callNext()
 	callQueue_.pop();
 	// Bind our chain caller
 	pro.second->connect(callback);
+	++running_count_;
 	ForkedCall call;
 	//If we fail to fork the process, then emit the signal
 	//to tell the outside world that it failed.
@@ -485,31 +495,33 @@ void callNext()
 
 void callback(pid_t, int)
 {
-	if (callQueue_.empty())
-		stopCaller();
-	else
+	--running_count_;
+	while (running_count_ < max_running_ && !callQueue_.empty())
 		callNext();
+	if (running_count_ == 0 && callQueue_.empty())
+		LYXERR(Debug::GRAPHICS, "ForkedCallQueue: I'm going to sleep");
 }
 
 
-void startCaller()
-{
-	LYXERR(Debug::GRAPHICS, "ForkedCallQueue: waking up");
-	running_ = true ;
-	callNext();
-}
-
-
-void stopCaller()
+void setMaxRunning(int n)
 {
-	running_ = false ;
-	LYXERR(Debug::GRAPHICS, "ForkedCallQueue: I'm going to sleep");
+	if (n < 1) {	//All cores available on the system.
+		int const hw = int(std::thread::hardware_concurrency());
+		if (hw < 1)	// Keep the current setting if the hardware cannot be queried.
+			return;
+		n = hw;
+	}
+	max_running_ = n;
+	LYXERR(Debug::GRAPHICS, "ForkedCallQueue: max parallel processes set to " << n);
+	// If the cap was raised while jobs were waiting, top up the pool.
+	while (running_count_ < max_running_ && !callQueue_.empty())
+		callNext();
 }
 
 
 bool running()
 {
-	return running_;
+	return running_count_ > 0;
 }
 
 } // namespace ForkedCallQueue
diff --git a/src/support/ForkedCalls.h b/src/support/ForkedCalls.h
index 84aeac9679..6c41ca9d9b 100644
--- a/src/support/ForkedCalls.h
+++ b/src/support/ForkedCalls.h
@@ -196,6 +196,9 @@ private:
 namespace ForkedCallQueue {
 
 ForkedCall::sigPtr add(std::string const & process);
+/// Set the maximum number of processes forked in parallel;
+/// n < 1 means all cores available on the system.
+void setMaxRunning(int n);
 /// Query whether the queue is running a forked process now.
 bool running();