add initial file watching support to trigger content reloading

not all of the paths that we'd probably like are being watched yet

Cargo.lock

@@ -301,6 +301,31 @@ version = "0.10.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341"
 
+[[package]]
+name = "async-trait"
+version = "0.1.68"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.18",
+]
+
+[[package]]
+name = "async-watcher"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f592305b7e0a9375714169a15870cd1c75763ca58a40e3d9a9d55d7d969f128"
+dependencies = [
+ "async-trait",
+ "notify",
+ "serde",
+ "thiserror",
+ "tokio",
+ "tracing",
+]
+
 [[package]]
 name = "atom_syndication"
 version = "0.12.1"
@@ -501,6 +526,25 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "crossbeam-channel"
+version = "0.5.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
+dependencies = [
+ "cfg-if",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "crypto-common"
 version = "0.1.6"
@@ -636,6 +680,18 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1"
 
+[[package]]
+name = "filetime"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5cbc844cecaee9d4443931972e1289c8ff485cb4cc2767cb03ca139ed6885153"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall 0.2.16",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "flate2"
 version = "1.0.26"
@@ -661,6 +717,15 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "fsevent-sys"
+version = "4.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "futures-core"
 version = "0.3.28"
@@ -908,6 +973,26 @@ dependencies = [
  "hashbrown 0.14.0",
 ]
 
+[[package]]
+name = "inotify"
+version = "0.9.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff"
+dependencies = [
+ "bitflags",
+ "inotify-sys",
+ "libc",
+]
+
+[[package]]
+name = "inotify-sys"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "instant"
 version = "0.1.12"
@@ -956,6 +1041,26 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "kqueue"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c8fc60ba15bf51257aa9807a48a61013db043fcf3a78cb0d916e8e396dcad98"
+dependencies = [
+ "kqueue-sys",
+ "libc",
+]
+
+[[package]]
+name = "kqueue-sys"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8367585489f01bc55dd27404dcf56b95e6da061a256a666ab23be9ba96a2e587"
+dependencies = [
+ "bitflags",
+ "libc",
+]
+
 [[package]]
 name = "language-tags"
 version = "0.3.2"
@@ -1102,7 +1207,7 @@ dependencies = [
  "libc",
  "log",
  "wasi 0.11.0+wasi-snapshot-preview1",
- "windows-sys",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -1111,6 +1216,24 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c96aba5aa877601bb3f6dd6a63a969e1f82e60646e81e71b14496995e9853c91"
 
+[[package]]
+name = "notify"
+version = "5.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "729f63e1ca555a43fe3efa4f3efdf4801c479da85b432242a7b726f353c88486"
+dependencies = [
+ "bitflags",
+ "crossbeam-channel",
+ "filetime",
+ "fsevent-sys",
+ "inotify",
+ "kqueue",
+ "libc",
+ "mio",
+ "walkdir",
+ "windows-sys 0.45.0",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.15"
@@ -1212,7 +1335,7 @@ dependencies = [
  "libc",
  "redox_syscall 0.3.5",
  "smallvec",
- "windows-targets",
+ "windows-targets 0.48.0",
 ]
 
 [[package]]
@@ -1237,9 +1360,11 @@ dependencies = [
  "actix-files",
  "actix-web",
  "anyhow",
+ "async-watcher",
  "chrono",
  "itertools",
  "log",
+ "notify",
  "pulldown-cmark",
  "rss",
  "serde",
@@ -1248,6 +1373,7 @@ dependencies = [
  "syntect",
  "tera",
  "thiserror",
+ "tokio",
  "url",
 ]
 
@@ -1895,11 +2021,24 @@ dependencies = [
  "bytes",
  "libc",
  "mio",
+ "num_cpus",
  "parking_lot 0.12.1",
  "pin-project-lite",
  "signal-hook-registry",
  "socket2",
- "windows-sys",
+ "tokio-macros",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "tokio-macros"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.18",
 ]
 
 [[package]]
@@ -1925,9 +2064,21 @@ dependencies = [
  "cfg-if",
  "log",
  "pin-project-lite",
+ "tracing-attributes",
  "tracing-core",
 ]
 
+[[package]]
+name = "tracing-attributes"
+version = "0.1.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.18",
+]
+
 [[package]]
 name = "tracing-core"
 version = "0.1.31"
@@ -2180,7 +2331,16 @@ version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f"
 dependencies = [
- "windows-targets",
+ "windows-targets 0.48.0",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.45.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
+dependencies = [
+ "windows-targets 0.42.2",
 ]
 
 [[package]]
@@ -2189,7 +2349,22 @@ version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
 dependencies = [
- "windows-targets",
+ "windows-targets 0.48.0",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
+dependencies = [
+ "windows_aarch64_gnullvm 0.42.2",
+ "windows_aarch64_msvc 0.42.2",
+ "windows_i686_gnu 0.42.2",
+ "windows_i686_msvc 0.42.2",
+ "windows_x86_64_gnu 0.42.2",
+ "windows_x86_64_gnullvm 0.42.2",
+ "windows_x86_64_msvc 0.42.2",
 ]
 
 [[package]]
@@ -2198,51 +2373,93 @@ version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5"
 dependencies = [
- "windows_aarch64_gnullvm",
+ "windows_aarch64_gnullvm 0.48.0",
- "windows_aarch64_msvc",
+ "windows_aarch64_msvc 0.48.0",
- "windows_i686_gnu",
+ "windows_i686_gnu 0.48.0",
- "windows_i686_msvc",
+ "windows_i686_msvc 0.48.0",
- "windows_x86_64_gnu",
+ "windows_x86_64_gnu 0.48.0",
- "windows_x86_64_gnullvm",
+ "windows_x86_64_gnullvm 0.48.0",
- "windows_x86_64_msvc",
+ "windows_x86_64_msvc 0.48.0",
 ]
 
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
+
 [[package]]
 name = "windows_aarch64_gnullvm"
 version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
 
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
+
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
 
+[[package]]
+name = "windows_i686_gnu"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
+
 [[package]]
 name = "windows_i686_gnu"
 version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
 
+[[package]]
+name = "windows_i686_msvc"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
+
 [[package]]
 name = "windows_i686_msvc"
 version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
 
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
+
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
 
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
+
 [[package]]
 name = "windows_x86_64_gnullvm"
 version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
 
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
+
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.48.0"

Cargo.toml

@@ -8,9 +8,11 @@ edition = "2021"
 actix-web = "4.3.1"
 actix-files = "0.6.2"
 anyhow = "1.0.71"
+async-watcher = "0.1.1"
 chrono = "0.4.26"
 itertools = "0.11.0"
 log = "0.4.19"
+notify = "5.1.0"
 pulldown-cmark = "0.9.3"
 rss = "2.0.4"
 serde = { version = "1.0.164", features = ["derive"]}
@@ -19,4 +21,5 @@ simple-log = "1.6.0"
 syntect = "5.0.0"
 tera = "1.19.0"
 thiserror = "1.0.40"
+tokio = "1"
 url = "2.4.0"

src/main.rs

@@ -1,14 +1,16 @@
+use std::env;
+use std::path::{Path, PathBuf};
+
 use actix_files::Files;
 use actix_web::web::Redirect;
 use actix_web::{web, App, Either, HttpRequest, HttpResponse, HttpServer, Responder};
 use anyhow::Context;
-use std::env;
-use std::path::{Path, PathBuf};
 
 mod config;
 mod markdown;
 mod site;
 mod util;
+mod watcher;
 
 fn not_found() -> HttpResponse {
 	HttpResponse::NotFound().body("not found")
@@ -48,6 +50,56 @@ async fn site_content(req: HttpRequest, data: web::Data<site::SiteService>) -> E
 	}
 }
 
+fn spawn_watcher(
+	watch_paths: Vec<PathBuf>,
+	pages_config_path: PathBuf,
+	posts_config_path: PathBuf,
+	data: web::Data<site::SiteService>,
+) -> tokio::task::JoinHandle<()> {
+	log::info!("Spawning filesystem watcher for paths {:?}", watch_paths);
+	tokio::spawn(async move {
+		watcher::debounce_watch(&watch_paths, move |event| {
+			match event {
+				Ok(_) => {
+					// right now we don't actually care which file was modified. just always rebuild the whole thing.
+					// this is also why using a debounced watch is important and probably should use a somewhat long
+					// debounce time in practice, just in case someone is doing a lengthy file upload to a remote
+					// server or something of that nature which takes more than 1-2 seconds.
+					// TODO: maybe try to selectively rebuild only what was changed? meh.
+					log::warn!(
+						"Modification to file(s) in watched paths detected, beginning re-generation of SiteContent"
+					);
+
+					log::info!("Reloading content configs");
+					let (pages_config, posts_config) =
+						match config::load_content(&pages_config_path, &posts_config_path, &data.server_config) {
+							Ok(configs) => configs,
+							Err(err) => {
+								log::error!("Error reloading content configs: {:?}", err);
+								return;
+							}
+						};
+
+					log::info!("Re-generating SiteContent");
+					if let Err(err) = data.refresh_content(pages_config, posts_config) {
+						log::error!("Error re-generating SiteContent: {:?}", err);
+						return;
+					}
+
+					log::info!("Finished re-generating SiteContent");
+				}
+				Err(errors) => {
+					for error in errors {
+						log::error!("debounce_watch event handler error: {:?}", error);
+					}
+				}
+			}
+		})
+		.await
+		.unwrap()
+	})
+}
+
 #[actix_web::main]
 async fn main() -> anyhow::Result<()> {
 	simple_log::new(
@@ -97,11 +149,15 @@ async fn main() -> anyhow::Result<()> {
 			.context("Constructing SiteService instance")?;
 		let data = web::Data::new(site_service);
 
+		let watch_paths = vec![pages_config_path.clone(), posts_config_path.clone()];
+		let watcher_handle = spawn_watcher(watch_paths, pages_config_path, posts_config_path, data.clone());
+
 		log::info!(
-			"Starting HTTP server for site, listening on {}:{} ...",
+			"Spawning HTTP server for site, listening on {}:{} ...",
 			server_config.bind_addr,
 			server_config.bind_port
 		);
+
 		HttpServer::new(move || {
 			App::new() //
 				.app_data(data.clone())
@@ -116,6 +172,12 @@ async fn main() -> anyhow::Result<()> {
 		.with_context(|| format!("Binding HTTP server on {}:{}", server_config.bind_addr, server_config.bind_port))?
 		.run()
 		.await
-		.map_err(anyhow::Error::from)
+		.map_err(anyhow::Error::from)?;
+
+		log::info!("Aborting filesystem watcher");
+		watcher_handle.abort();
+
+		log::info!("Finished!");
+		Ok(())
 	}
 }

src/site.rs

@@ -1,4 +1,5 @@
 use std::collections::HashMap;
+use std::ops::Deref;
 use std::path::PathBuf;
 use std::sync::RwLock;
 
@@ -276,11 +277,30 @@ impl SiteContent {
 	}
 }
 
+pub struct RefreshWrapper<T> {
+	pub data: T,
+}
+
+impl<T> RefreshWrapper<T> {
+	pub fn new(data: T) -> Self {
+		RefreshWrapper { data }
+	}
+}
+
+impl<T> Deref for RefreshWrapper<T> {
+	type Target = T;
+
+	#[inline]
+	fn deref(&self) -> &Self::Target {
+		&self.data
+	}
+}
+
 pub struct SiteService {
 	pub server_config: config::Server,
 	pub content_renderer: ContentRenderer,
 	pub template_renderer: tera::Tera,
-	pub content: RwLock<SiteContent>,
+	pub content: RwLock<RefreshWrapper<SiteContent>>,
 }
 
 impl SiteService {
@@ -290,7 +310,7 @@ impl SiteService {
 		posts_config: config::Posts,
 	) -> Result<Self, SiteError> {
 		let content_renderer = ContentRenderer::new(&server_config)?;
-		let content = SiteContent::new(pages_config, posts_config, &content_renderer)?;
+		let content = RefreshWrapper::new(SiteContent::new(pages_config, posts_config, &content_renderer)?);
 		let mut templates_path = PathBuf::from(&server_config.templates_path);
 		templates_path.push("**/*");
 		log::debug!("Using templates path: {:?}", templates_path);
@@ -307,6 +327,15 @@ impl SiteService {
 		})
 	}
 
+	pub fn refresh_content(&self, pages_config: config::Pages, posts_config: config::Posts) -> Result<(), SiteError> {
+		let mut existing_content = self.content.write().expect("SiteContent write lock failed"); // TODO: better error handling
+		log::debug!("Obtained write lock on SiteContent instance");
+		let content = SiteContent::new(pages_config, posts_config, &self.content_renderer)?;
+		log::debug!("New SiteContent instance built successfully");
+		existing_content.data = content;
+		Ok(())
+	}
+
 	pub fn serve_latest_post(&self) -> HttpResponse {
 		let content = self.content.read().expect("SiteContent read lock failed"); // TODO: better error handling
 		let post = content.get_latest_post();

src/watcher.rs

@@ -0,0 +1,55 @@
+use std::path::Path;
+use std::time::Duration;
+
+use async_watcher::{notify::RecursiveMode, AsyncDebouncer, DebouncedEvent};
+use tokio::sync::mpsc::channel;
+
+#[derive(Debug, thiserror::Error)]
+pub enum WatcherError {
+	#[error("Async Debounce Watcher error")]
+	AsyncWatcherError(#[from] async_watcher::error::Error),
+
+	#[error("Notify error")]
+	NotifyError(#[from] notify::Error),
+}
+
+const CHANNEL_BUFFER_SIZE: usize = 100;
+const DEBOUNCE_TIME: Duration = Duration::from_secs(1);
+
+pub type DeboundedEventResult = Result<Vec<DebouncedEvent>, Vec<notify::Error>>;
+
+pub trait WatchEventHandler: Send + 'static {
+	fn handle_event(&mut self, event: DeboundedEventResult);
+}
+
+impl<F> WatchEventHandler for F
+where
+	F: FnMut(DeboundedEventResult) + Send + 'static,
+{
+	fn handle_event(&mut self, event: DeboundedEventResult) {
+		(self)(event);
+	}
+}
+
+pub async fn debounce_watch<P, T>(paths: &[P], mut handler: T) -> Result<(), WatcherError>
+where
+	P: AsRef<Path>,
+	T: WatchEventHandler,
+{
+	log::debug!("Setting up debounced watcher");
+
+	let (tx, mut rx) = channel(CHANNEL_BUFFER_SIZE);
+
+	let mut debouncer = AsyncDebouncer::new(DEBOUNCE_TIME, Some(DEBOUNCE_TIME), tx).await?;
+
+	for path in paths.iter() {
+		log::debug!("Watching path {:?}", path.as_ref());
+		debouncer.watcher().watch(path.as_ref(), RecursiveMode::Recursive)?;
+	}
+
+	while let Some(event) = rx.recv().await {
+		handler.handle_event(event)
+	}
+
+	Ok(())
+}