feat(server): init for file walker

local_server/.gitignore

@@ -6,4 +6,5 @@ pkg/
 wasm-pack.log
 .idea
 !bin/*.rs
-*.db
+*.db
+testdocs

local_server/Cargo.toml

@@ -10,6 +10,7 @@ actix-web = "4"
 
 # core
 flume = "0.11.0"
+ignore = "=0.4.20"
 
 anyhow = "1.0.75"
 

local_server/src/doc_split/file_walker.rs

@@ -0,0 +1,31 @@
+use std::fs::canonicalize;
+use std::path::{Path, PathBuf};
+
+use tracing::{debug, warn};
+
+pub struct FileWalker {
+    file_list: Vec<PathBuf>,
+}
+
+impl FileWalker {
+    pub fn index_directory(dir: impl AsRef<Path>) -> Vec<PathBuf> {
+        let walker = ignore::WalkBuilder::new(&dir)
+            .standard_filters(true)
+            .hidden(false)
+            .build();
+
+        let file_list = walker
+            .filter_map(|de| match de {
+                Ok(de) => Some(de),
+                Err(err) => {
+                    warn!(%err, "access failure; skipping");
+                    None
+                }
+            })
+            .filter(|de| !de.path().strip_prefix(&dir).unwrap().starts_with(".git"))
+            .filter_map(|de| canonicalize(de.into_path()).ok())
+            .collect();
+
+        file_list
+    }
+}

local_server/src/doc_split/mod.rs

@@ -0,0 +1,22 @@
+pub mod file_walker;
+
+use std::path::PathBuf;
+fn doc_splitter(filename: &PathBuf) {
+    println!("doc_splitter: {}", filename.display());
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::doc_split::file_walker::FileWalker;
+    use super::*;
+
+    #[test]
+    fn test_doc_splitter() {
+        let testdir = PathBuf::from("testdocs");
+        let files = FileWalker::index_directory(testdir);
+
+        for file in files {
+            doc_splitter(&file);
+        }
+    }
+}

local_server/src/main.rs

@@ -10,6 +10,7 @@ pub mod scraper;
 mod document_handler;
 pub mod app_state;
 pub mod infra;
+pub mod doc_split;
 
 #[actix_web::main]
 async fn main() -> std::io::Result<()> {