From f7a3a38385a56841494e8c0cb9f67406090d8121 Mon Sep 17 00:00:00 2001 From: "Carpenter, Adam (CORP)" Date: Fri, 26 Nov 2021 11:58:06 -0500 Subject: feat: scaffold feature switch route extension hierarchy --- Cargo.lock | 2 +- angelsharkd/Cargo.toml | 9 +++++++-- angelsharkd/src/main.rs | 3 +++ angelsharkd/src/routes/mod.rs | 2 ++ libangelshark/Cargo.toml | 2 +- 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a3b5bd6..da97be8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,7 +24,7 @@ dependencies = [ [[package]] name = "angelsharkd" -version = "0.1.2" +version = "0.2.0" dependencies = [ "anyhow", "env_logger", diff --git a/angelsharkd/Cargo.toml b/angelsharkd/Cargo.toml index f820528..9ca459e 100644 --- a/angelsharkd/Cargo.toml +++ b/angelsharkd/Cargo.toml @@ -1,10 +1,15 @@ [package] name = "angelsharkd" -version = "0.1.2" +version = "0.2.0" edition = "2021" authors = ["Adam T. Carpenter "] description = "A HTTP interface into one or more Communication Managers" +[features] +extensions = [] +simple_search = ["extensions"] +simple_deprov = ["extensions"] + [dependencies.libangelshark] path = "../libangelshark" @@ -20,7 +25,7 @@ default-features = false version = "0.4" [dependencies.env_logger] -version = "0.8" +version = "0.9" [dependencies.serde] version = "1" diff --git a/angelsharkd/src/main.rs b/angelsharkd/src/main.rs index 8a8a53e..fb075ab 100644 --- a/angelsharkd/src/main.rs +++ b/angelsharkd/src/main.rs @@ -41,6 +41,9 @@ async fn main() -> Result<()> { }) .with(warp::log("angelsharkd")); + #[cfg(feature = "extensions")] + let routes = routes.or(routes::extensions::filter(&config)); + // Create server with shutdown signal. let (addr, server) = warp::serve(routes).bind_with_graceful_shutdown(config.bind_addr, async { signal::ctrl_c() diff --git a/angelsharkd/src/routes/mod.rs b/angelsharkd/src/routes/mod.rs index 40d024d..52373d6 100644 --- a/angelsharkd/src/routes/mod.rs +++ b/angelsharkd/src/routes/mod.rs @@ -13,6 +13,8 @@ use warp::{ }; mod dtos; +#[cfg(feature = "extensions")] +pub mod extensions; /// GET / -> Name and version # of app. pub fn index() -> impl Filter + Clone { diff --git a/libangelshark/Cargo.toml b/libangelshark/Cargo.toml index 6b945d0..8c3b77b 100644 --- a/libangelshark/Cargo.toml +++ b/libangelshark/Cargo.toml @@ -16,6 +16,6 @@ version = "0.9" features = ["vendored-openssl"] [dependencies.cached] -version = "0.25" +version = "0.26" default-features = false features = ["proc_macro"] -- cgit v1.2.3 From 03af9488f7d7fea69c779b1d9263f34841dfad37 Mon Sep 17 00:00:00 2001 From: "Carpenter, Adam (CORP)" Date: Fri, 26 Nov 2021 14:31:44 -0500 Subject: chore: bump package versions --- Cargo.lock | 50 +++++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index da97be8..5a272a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -50,6 +50,16 @@ dependencies = [ "event-listener", ] +[[package]] +name = "async-rwlock" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261803dcc39ba9e72760ba6e16d0199b1eef9fc44e81bffabbebb9f5aea3906c" +dependencies = [ + "async-mutex", + "event-listener", +] + [[package]] name = "atty" version = "0.2.14" @@ -108,24 +118,24 @@ checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" [[package]] name = "cached" -version = "0.25.0" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b99e696f7b2696ed5eae0d462a9eeafaea111d99e39b2c8ceb418afe1013bcfc" +checksum = "c2bc2fd249a24a9cdd4276f3a3e0461713271ab63b0e9e656e200e8e21c8c927" dependencies = [ "async-mutex", + "async-rwlock", "cached_proc_macro", "cached_proc_macro_types", - "hashbrown 0.9.1", + "hashbrown", "once_cell", ] [[package]] name = "cached_proc_macro" -version = "0.6.1" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25a685ba39b57a91a53d149dcbef854f50fbe204d1ff6081ea0bec3529a0c456" +checksum = "ac3531903b39df48a378a7ed515baee7c1fff32488489c7d0725eb1749b22a91" dependencies = [ - "async-mutex", "cached_proc_macro_types", "darling", "quote", @@ -253,9 +263,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.10.2" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d706e75d87e35569db781a9b5e2416cff1236a47ed380831f959382ccd5f858" +checksum = "757c0ded2af11d8e739c4daea1ac623dd1624b06c844cf3f5a39f1bdbd99bb12" dependencies = [ "darling_core", "darling_macro", @@ -263,9 +273,9 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.10.2" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c960ae2da4de88a91b2d920c2a7233b400bc33cb28453a2987822d8392519b" +checksum = "2c34d8efb62d0c2d7f60ece80f75e5c63c1588ba68032740494b0b9a996466e3" dependencies = [ "fnv", "ident_case", @@ -277,9 +287,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.10.2" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72" +checksum = "ade7bff147130fe5e6d39f089c6bd49ec0250f35d70b2eebf72afdfc919f15cc" dependencies = [ "darling_core", "quote", @@ -303,9 +313,9 @@ checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" [[package]] name = "env_logger" -version = "0.8.4" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3" +checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3" dependencies = [ "atty", "humantime", @@ -427,12 +437,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hashbrown" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" - [[package]] name = "hashbrown" version = "0.11.2" @@ -550,7 +554,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" dependencies = [ "autocfg", - "hashbrown 0.11.2", + "hashbrown", ] [[package]] @@ -997,9 +1001,9 @@ dependencies = [ [[package]] name = "strsim" -version = "0.9.3" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -- cgit v1.2.3 From 75ef35829dc57d5b6f04a0c59160fe7584324793 Mon Sep 17 00:00:00 2001 From: "Carpenter, Adam (CORP)" Date: Fri, 26 Nov 2021 14:32:11 -0500 Subject: feat: basic extension loader module and two scaffolded, unimplemented extensions --- angelsharkd/src/routes/extensions/mod.rs | 26 ++++++++++++++++++++++ angelsharkd/src/routes/extensions/simple_deprov.rs | 5 +++++ angelsharkd/src/routes/extensions/simple_search.rs | 14 ++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 angelsharkd/src/routes/extensions/mod.rs create mode 100644 angelsharkd/src/routes/extensions/simple_deprov.rs create mode 100644 angelsharkd/src/routes/extensions/simple_search.rs diff --git a/angelsharkd/src/routes/extensions/mod.rs b/angelsharkd/src/routes/extensions/mod.rs new file mode 100644 index 0000000..5dac1cf --- /dev/null +++ b/angelsharkd/src/routes/extensions/mod.rs @@ -0,0 +1,26 @@ +use crate::config::Config; +use warp::{path, Filter, Rejection, Reply}; + +#[cfg(feature = "simple_deprov")] +mod simple_deprov; +#[cfg(feature = "simple_search")] +mod simple_search; + +pub fn filter(config: &Config) -> impl Filter + Clone { + // Note: this next line deals with the common edge case of having no + // extensions loaded with feature flags. It ensures that the the type + // checking is right when the return `.and()` is called below. + let filters = default().or(default()); + + #[cfg(feature = "simple_search")] + let filters = filters.or(simple_search::filter()); + + #[cfg(feature = "simple_deprov")] + let filters = filters.or(simple_deprov::filter()); + + path("extensions").and(filters) +} + +fn default() -> impl Filter + Clone { + warp::path::end().map(|| "Angelshark extension route index. Enable extensions with feature switches and access them at `/extensions/`.") +} diff --git a/angelsharkd/src/routes/extensions/simple_deprov.rs b/angelsharkd/src/routes/extensions/simple_deprov.rs new file mode 100644 index 0000000..2b5ad40 --- /dev/null +++ b/angelsharkd/src/routes/extensions/simple_deprov.rs @@ -0,0 +1,5 @@ +use warp::{Filter, Rejection, Reply}; + +pub fn filter() -> impl Filter + Clone { + warp::path("deprov").map(|| -> &str { todo!() }) // TODO: +} diff --git a/angelsharkd/src/routes/extensions/simple_search.rs b/angelsharkd/src/routes/extensions/simple_search.rs new file mode 100644 index 0000000..454dba3 --- /dev/null +++ b/angelsharkd/src/routes/extensions/simple_search.rs @@ -0,0 +1,14 @@ +use crate::config::Config; +use std::convert::Infallible; +use warp::{path, post, Filter, Rejection, Reply}; + +const CMD_LIST_EXT: &str = "list extension-type"; +const CMD_LIST_STAT: &str = "list station"; + +pub fn filter() -> impl Filter + Clone { + path("search").and(post()).and_then(handle_simple_search) +} + +async fn handle_simple_search() -> Result { + Ok("Search!") +} -- cgit v1.2.3 From 5b0786905d400a8414dd0b0fe2e4d9ac784ae543 Mon Sep 17 00:00:00 2001 From: "Carpenter, Adam (CORP)" Date: Fri, 26 Nov 2021 14:45:09 -0500 Subject: docs: extensions readme --- angelsharkd/src/routes/extensions/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 angelsharkd/src/routes/extensions/README.md diff --git a/angelsharkd/src/routes/extensions/README.md b/angelsharkd/src/routes/extensions/README.md new file mode 100644 index 0000000..b9192fe --- /dev/null +++ b/angelsharkd/src/routes/extensions/README.md @@ -0,0 +1,12 @@ +# Angelshark Daemon Extensions + +This module aims to provide a simple way of extending Angelshark's basic functionality (running commands on the ACM) with additional HTTP endpoints that run one or more commands to achieve a basic business task. + +For example, say you would like you, other users, or your own software to quickly search all extension-types for a keyword. This functionality is not in the base `angelsharkd`, but it can be easily implemented with the following steps: + +1. Accept a keyword from the client's request +1. Download extension-type data from one or more ACMs +1. Filter out extensions that do not match a given keyword +1. Return the remaining, matching extensions to the client + +This functionality may not be desirable for all end users, and therefor is completely opt-in with feature flags. At compile time, you can add `--features simple_search` to enable a given extension called `simple_search`, for example. -- cgit v1.2.3 From 6dd3236069d5cbdd6cb946b408cd6efb7e91d0f9 Mon Sep 17 00:00:00 2001 From: "Carpenter, Adam (CORP)" Date: Fri, 26 Nov 2021 15:58:13 -0500 Subject: feat: impl lazy-refreshing cache for quick access to be used for haystack --- Cargo.lock | 1 + angelsharkd/Cargo.toml | 11 +++-- angelsharkd/src/main.rs | 4 +- angelsharkd/src/routes/extensions/mod.rs | 4 +- angelsharkd/src/routes/extensions/simple_search.rs | 54 +++++++++++++++++++--- angelsharkd/src/routes/mod.rs | 1 - 6 files changed, 61 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5a272a9..255eb5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,6 +27,7 @@ name = "angelsharkd" version = "0.2.0" dependencies = [ "anyhow", + "cached", "env_logger", "libangelshark", "log", diff --git a/angelsharkd/Cargo.toml b/angelsharkd/Cargo.toml index 9ca459e..8e617e7 100644 --- a/angelsharkd/Cargo.toml +++ b/angelsharkd/Cargo.toml @@ -6,9 +6,8 @@ authors = ["Adam T. Carpenter "] description = "A HTTP interface into one or more Communication Managers" [features] -extensions = [] -simple_search = ["extensions"] -simple_deprov = ["extensions"] +simple_search = ["cached"] +simple_deprov = [] [dependencies.libangelshark] path = "../libangelshark" @@ -33,3 +32,9 @@ features = ["derive"] [dependencies.anyhow] version = "1" + +[dependencies.cached] +optional = true +version = "0.26" +default-features = false +features = ["proc_macro"] diff --git a/angelsharkd/src/main.rs b/angelsharkd/src/main.rs index fb075ab..790b641 100644 --- a/angelsharkd/src/main.rs +++ b/angelsharkd/src/main.rs @@ -30,6 +30,7 @@ async fn main() -> Result<()> { let routes = routes::index() .or(routes::ossi(&config)) + .or(routes::extensions::filter(&config)) .with(if config.debug_mode || config.origin == "*" { warp::cors() .allow_any_origin() @@ -41,9 +42,6 @@ async fn main() -> Result<()> { }) .with(warp::log("angelsharkd")); - #[cfg(feature = "extensions")] - let routes = routes.or(routes::extensions::filter(&config)); - // Create server with shutdown signal. let (addr, server) = warp::serve(routes).bind_with_graceful_shutdown(config.bind_addr, async { signal::ctrl_c() diff --git a/angelsharkd/src/routes/extensions/mod.rs b/angelsharkd/src/routes/extensions/mod.rs index 5dac1cf..defbca9 100644 --- a/angelsharkd/src/routes/extensions/mod.rs +++ b/angelsharkd/src/routes/extensions/mod.rs @@ -13,7 +13,9 @@ pub fn filter(config: &Config) -> impl Filter impl Filter + Clone { - path("search").and(post()).and_then(handle_simple_search) +type Terms = Vec; + +pub fn search(config: &Config) -> impl Filter + Clone { + let runner = config.runner.clone(); + // TODO: anti-caching headers on resp? + + path("search") + .and(post()) + .and(content_length_limit(1024 * 16)) + .and(json::()) + .and(with_runner(runner)) + .and_then(handle_simple_search) } -async fn handle_simple_search() -> Result { - Ok("Search!") +pub fn refresh() -> impl Filter + Clone { + path!("search" / "refresh") + .and(get()) + .and_then(handle_refresh) +} + +async fn handle_refresh() -> Result { + fetch(String::from("test"), true); + Ok("") +} + +async fn handle_simple_search(terms: Terms, runner: AcmRunner) -> Result { + Ok(fetch(terms[0].clone(), false)) +} + +#[cached] +fn fetch(param: String, refresh: bool) -> String { + if refresh { + let param = param.clone(); + tokio::spawn(async move { + std::thread::sleep_ms(10000); + let mut handle = FETCH.lock().unwrap(); + handle.cache_clear(); + handle.cache_set((param.clone(), false), format!("blargh {}", param.clone())); + eprintln!("evicted"); + }); + } + + param } diff --git a/angelsharkd/src/routes/mod.rs b/angelsharkd/src/routes/mod.rs index 52373d6..cee0657 100644 --- a/angelsharkd/src/routes/mod.rs +++ b/angelsharkd/src/routes/mod.rs @@ -13,7 +13,6 @@ use warp::{ }; mod dtos; -#[cfg(feature = "extensions")] pub mod extensions; /// GET / -> Name and version # of app. -- cgit v1.2.3 From 3acf604c92b64032885ef27d225efa1307a015ad Mon Sep 17 00:00:00 2001 From: "Carpenter, Adam (CORP)" Date: Fri, 26 Nov 2021 16:06:17 -0500 Subject: feat: impl lazy-refreshing cache for quick access to be used for haystack --- angelsharkd/src/routes/extensions/simple_search.rs | 31 +++++++++++++--------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/angelsharkd/src/routes/extensions/simple_search.rs b/angelsharkd/src/routes/extensions/simple_search.rs index 58f6e52..cf75c60 100644 --- a/angelsharkd/src/routes/extensions/simple_search.rs +++ b/angelsharkd/src/routes/extensions/simple_search.rs @@ -24,33 +24,40 @@ pub fn search(config: &Config) -> impl Filter impl Filter + Clone { +pub fn refresh(config: &Config) -> impl Filter + Clone { + let runner = config.runner.clone(); path!("search" / "refresh") .and(get()) + .and(with_runner(runner)) .and_then(handle_refresh) } -async fn handle_refresh() -> Result { - fetch(String::from("test"), true); - Ok("") +async fn handle_refresh(runner: AcmRunner) -> Result { + get_extensions_cached(true); + Ok("Refresh scheduled") } async fn handle_simple_search(terms: Terms, runner: AcmRunner) -> Result { - Ok(fetch(terms[0].clone(), false)) + Ok(get_extensions_cached(false)) } #[cached] -fn fetch(param: String, refresh: bool) -> String { +fn get_extensions_cached(refresh: bool) -> String { if refresh { - let param = param.clone(); tokio::spawn(async move { std::thread::sleep_ms(10000); - let mut handle = FETCH.lock().unwrap(); - handle.cache_clear(); - handle.cache_set((param.clone(), false), format!("blargh {}", param.clone())); - eprintln!("evicted"); + + if let Ok(mut handle) = GET_EXTENSIONS_CACHED.lock() { + handle.cache_clear(); + handle.cache_set(false, String::from("fresh")); + eprintln!("evicted"); + } }); } - param + String::from("stale") +} + +fn get_extensions(runner: AcmRunner) { + todo!() } -- cgit v1.2.3 From 91d8810402ae8e1173a3c064519c9f1b03e13224 Mon Sep 17 00:00:00 2001 From: "Carpenter, Adam (CORP)" Date: Fri, 26 Nov 2021 16:48:35 -0500 Subject: refactor: simplify by eliminating needless cached for smaller once_cell --- Cargo.lock | 2 +- angelsharkd/Cargo.toml | 8 ++- angelsharkd/src/routes/extensions/mod.rs | 2 +- angelsharkd/src/routes/extensions/simple_search.rs | 58 +++++++++++++--------- 4 files changed, 40 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 255eb5e..cbfb8b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,10 +27,10 @@ name = "angelsharkd" version = "0.2.0" dependencies = [ "anyhow", - "cached", "env_logger", "libangelshark", "log", + "once_cell", "serde", "tokio", "warp", diff --git a/angelsharkd/Cargo.toml b/angelsharkd/Cargo.toml index 8e617e7..d532bf9 100644 --- a/angelsharkd/Cargo.toml +++ b/angelsharkd/Cargo.toml @@ -6,7 +6,7 @@ authors = ["Adam T. Carpenter "] description = "A HTTP interface into one or more Communication Managers" [features] -simple_search = ["cached"] +simple_search = ["once_cell"] simple_deprov = [] [dependencies.libangelshark] @@ -33,8 +33,6 @@ features = ["derive"] [dependencies.anyhow] version = "1" -[dependencies.cached] +[dependencies.once_cell] optional = true -version = "0.26" -default-features = false -features = ["proc_macro"] +version = "1" \ No newline at end of file diff --git a/angelsharkd/src/routes/extensions/mod.rs b/angelsharkd/src/routes/extensions/mod.rs index defbca9..5628588 100644 --- a/angelsharkd/src/routes/extensions/mod.rs +++ b/angelsharkd/src/routes/extensions/mod.rs @@ -15,7 +15,7 @@ pub fn filter(config: &Config) -> impl Filter = Lazy::new(Haystack::new); -type Terms = Vec; +/// Collection of search terms +type Needle = Vec; pub fn search(config: &Config) -> impl Filter + Clone { let runner = config.runner.clone(); @@ -19,9 +22,9 @@ pub fn search(config: &Config) -> impl Filter()) + .and(json::()) .and(with_runner(runner)) - .and_then(handle_simple_search) + .and_then(handle_search) } pub fn refresh(config: &Config) -> impl Filter + Clone { @@ -32,32 +35,41 @@ pub fn refresh(config: &Config) -> impl Filter Result { + HAYSTACK_CACHE.search(Vec::new()); + Ok("") +} + async fn handle_refresh(runner: AcmRunner) -> Result { - get_extensions_cached(true); + HAYSTACK_CACHE.refresh(); Ok("Refresh scheduled") } -async fn handle_simple_search(terms: Terms, runner: AcmRunner) -> Result { - Ok(get_extensions_cached(false)) +/// A lazy-loaded, asynchronously-refreshed exension-type haystack cache. +struct Haystack { + inner: Arc>, } -#[cached] -fn get_extensions_cached(refresh: bool) -> String { - if refresh { +impl Haystack { + fn new() -> Self { + Self { + inner: Arc::new(Mutex::new(String::with_capacity(0))), + } + } + + pub fn search(&self, needle: Needle) -> String { + (*self.inner.lock().unwrap()).clone() + } + + pub fn refresh(&self) { + let inner = self.inner.clone(); tokio::spawn(async move { - std::thread::sleep_ms(10000); + std::thread::sleep_ms(10000); // slow generation here - if let Ok(mut handle) = GET_EXTENSIONS_CACHED.lock() { - handle.cache_clear(); - handle.cache_set(false, String::from("fresh")); + if let Ok(mut handle) = inner.lock() { + *handle = String::from("fresh"); eprintln!("evicted"); } }); } - - String::from("stale") -} - -fn get_extensions(runner: AcmRunner) { - todo!() } -- cgit v1.2.3 From d65e36a4b6f2e28d25ed2996a94db17a7ae9abd9 Mon Sep 17 00:00:00 2001 From: "Carpenter, Adam (CORP)" Date: Mon, 29 Nov 2021 13:52:11 -0500 Subject: refactor: static global and once_cell depends --- Cargo.lock | 1 - angelsharkd/Cargo.toml | 6 +-- angelsharkd/src/routes/extensions/mod.rs | 10 ++++- angelsharkd/src/routes/extensions/simple_search.rs | 43 +++++++++++----------- 4 files changed, 30 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cbfb8b3..5a272a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -30,7 +30,6 @@ dependencies = [ "env_logger", "libangelshark", "log", - "once_cell", "serde", "tokio", "warp", diff --git a/angelsharkd/Cargo.toml b/angelsharkd/Cargo.toml index d532bf9..9fdb00c 100644 --- a/angelsharkd/Cargo.toml +++ b/angelsharkd/Cargo.toml @@ -6,7 +6,7 @@ authors = ["Adam T. Carpenter "] description = "A HTTP interface into one or more Communication Managers" [features] -simple_search = ["once_cell"] +simple_search = [] simple_deprov = [] [dependencies.libangelshark] @@ -32,7 +32,3 @@ features = ["derive"] [dependencies.anyhow] version = "1" - -[dependencies.once_cell] -optional = true -version = "1" \ No newline at end of file diff --git a/angelsharkd/src/routes/extensions/mod.rs b/angelsharkd/src/routes/extensions/mod.rs index 5628588..b1c5aa3 100644 --- a/angelsharkd/src/routes/extensions/mod.rs +++ b/angelsharkd/src/routes/extensions/mod.rs @@ -1,3 +1,5 @@ +use std::sync::{Arc, Mutex}; + use crate::config::Config; use warp::{path, Filter, Rejection, Reply}; @@ -12,10 +14,14 @@ pub fn filter(config: &Config) -> impl Filter = Lazy::new(Haystack::new); - /// Collection of search terms type Needle = Vec; -pub fn search(config: &Config) -> impl Filter + Clone { - let runner = config.runner.clone(); - // TODO: anti-caching headers on resp? +pub fn search(haystack: Haystack) -> impl Filter + Clone { + // TODO: discourage caching response thru headers path("search") .and(post()) .and(content_length_limit(1024 * 16)) - .and(json::()) - .and(with_runner(runner)) - .and_then(handle_search) + .and(json()) + .and_then(move |terms: Needle| handle_search(haystack.clone(), terms)) } -pub fn refresh(config: &Config) -> impl Filter + Clone { - let runner = config.runner.clone(); +pub fn refresh( + runner: AcmRunner, + haystack: Haystack, +) -> impl Filter + Clone { path!("search" / "refresh") .and(get()) - .and(with_runner(runner)) - .and_then(handle_refresh) + .and_then(move || handle_refresh(haystack.clone(), runner.clone())) } -async fn handle_search(terms: Needle, runner: AcmRunner) -> Result { - HAYSTACK_CACHE.search(Vec::new()); - Ok("") +async fn handle_search(haystack: Haystack, needle: Needle) -> Result { + Ok(haystack.search(Vec::new())) } -async fn handle_refresh(runner: AcmRunner) -> Result { - HAYSTACK_CACHE.refresh(); +async fn handle_refresh(haystack: Haystack, runner: AcmRunner) -> Result { + haystack.refresh(); Ok("Refresh scheduled") } /// A lazy-loaded, asynchronously-refreshed exension-type haystack cache. -struct Haystack { +#[derive(Clone)] +pub struct Haystack { inner: Arc>, } impl Haystack { - fn new() -> Self { + pub fn new() -> Self { Self { inner: Arc::new(Mutex::new(String::with_capacity(0))), } } pub fn search(&self, needle: Needle) -> String { - (*self.inner.lock().unwrap()).clone() + if let Ok(matches) = self.inner.lock() { + matches.clone() + } else { + String::from("stale") + } } pub fn refresh(&self) { -- cgit v1.2.3 From 0edcb7438bac442bb24a55f25974ca225e4d2e97 Mon Sep 17 00:00:00 2001 From: "Carpenter, Adam (CORP)" Date: Tue, 30 Nov 2021 12:02:38 -0500 Subject: feat: impl searching extension, tested, working --- angelsharkd/src/routes/extensions/mod.rs | 8 +- angelsharkd/src/routes/extensions/simple_search.rs | 182 +++++++++++++++++---- 2 files changed, 153 insertions(+), 37 deletions(-) diff --git a/angelsharkd/src/routes/extensions/mod.rs b/angelsharkd/src/routes/extensions/mod.rs index b1c5aa3..adcea3f 100644 --- a/angelsharkd/src/routes/extensions/mod.rs +++ b/angelsharkd/src/routes/extensions/mod.rs @@ -1,5 +1,3 @@ -use std::sync::{Arc, Mutex}; - use crate::config::Config; use warp::{path, Filter, Rejection, Reply}; @@ -15,13 +13,11 @@ pub fn filter(config: &Config) -> impl Filter; +type Needles = Vec; + +/// Collection of ACM extension types with ROOMs (if applicable) and ACM names +type HaystackEntries = Vec>; pub fn search(haystack: Haystack) -> impl Filter + Clone { // TODO: discourage caching response thru headers @@ -18,57 +26,169 @@ pub fn search(haystack: Haystack) -> impl Filter impl Filter + Clone { +pub fn refresh(haystack: Haystack) -> impl Filter + Clone { path!("search" / "refresh") .and(get()) - .and_then(move || handle_refresh(haystack.clone(), runner.clone())) + .and_then(move || handle_refresh(haystack.to_owned())) } -async fn handle_search(haystack: Haystack, needle: Needle) -> Result { - Ok(haystack.search(Vec::new())) +async fn handle_search(haystack: Haystack, needle: Needles) -> Result { + // Ok(haystack.search(Vec::new())?) + // if let Ok(matches = haystack.search(needle); + match haystack.search(needle) { + Ok(matches) => Ok(reply::with_status(reply::json(&matches), StatusCode::OK)), + Err(e) => Ok(reply::with_status( + reply::json(&e.to_string()), + StatusCode::INTERNAL_SERVER_ERROR, + )), + } } -async fn handle_refresh(haystack: Haystack, runner: AcmRunner) -> Result { - haystack.refresh(); +async fn handle_refresh(haystack: Haystack) -> Result { + // Run refresh as a background task and immediately return. + tokio::spawn(async move { + if let Err(e) = haystack.refresh() { + error!("{}", e.to_string()); // TODO: use logger + } else { + info!("Search haystack refreshed."); + } + }); + Ok("Refresh scheduled") } /// A lazy-loaded, asynchronously-refreshed exension-type haystack cache. #[derive(Clone)] pub struct Haystack { - inner: Arc>, + entries: Arc>, + runner: AcmRunner, } impl Haystack { - pub fn new() -> Self { + pub fn new(runner: AcmRunner) -> Self { Self { - inner: Arc::new(Mutex::new(String::with_capacity(0))), + entries: Arc::new(Mutex::new(Vec::new())), + runner, } } - pub fn search(&self, needle: Needle) -> String { - if let Ok(matches) = self.inner.lock() { - matches.clone() - } else { - String::from("stale") - } + pub fn search(&self, needles: Needles) -> Result { + let needles: Vec<_> = needles.iter().map(|n| n.to_lowercase()).collect(); + + let entries = self + .entries + .lock() + .map_err(|e| anyhow!(e.to_string())) + .with_context(|| "Failed to get haystack inner data lock.")?; + + let matches = entries + .iter() + .filter(|entry| { + let entry_str = entry.join("").to_lowercase(); + needles + .iter() + .all(|needle| entry_str.contains(needle.as_str())) + }) + .cloned() + .collect(); + Ok(matches) } - pub fn refresh(&self) { - let inner = self.inner.clone(); - tokio::spawn(async move { - std::thread::sleep_ms(10000); // slow generation here + pub fn refresh(&self) -> Result<(), Error> { + let mut runner = self.runner.to_owned(); // TODO: This alone allows multiple refreshers to be run at once. Do we want that? + + // Queue jobs in ACM runner + for acm in &[ + "01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", + ] { + runner + .queue_input(acm, &Message::new("list extension-type")) + .queue_input( + acm, + &Message { + command: String::from("list station"), + fields: Some(vec![String::from("8005ff00"), String::from("0031ff00")]), + datas: None, + error: None, + }, + ); + } + + // Run jobs and collect output. Filter out uneeded commands, combine errors. + let output: Result)>, Error> = runner + .run() + .map(|(name, output)| { + let output: Vec = output? + .into_iter() + .filter(|m| m.command != "logoff") + .collect(); + Ok((name, output)) + }) + .collect(); + let output = output.with_context(|| "Failed to run refresh commands on ACM(s).")?; + + // Log any ACM errors encountered + for error in output + .iter() + .map(|(_, messages)| messages) + .flatten() + .filter_map(|m| m.error.as_ref()) + { + error!("ACM error: {}", error); + } + + // Build a map of station number-to-rooms + let rooms: HashMap = output + .iter() + .map(|(_, messages)| { + messages + .iter() + .filter(|message| message.command == "list station") + .filter_map(|message| message.datas.to_owned()) + .flatten() + .filter_map(|stat| Some((stat.get(0)?.to_owned(), stat.get(1)?.to_owned()))) + }) + .flatten() + .collect(); + + // Build the haystack from the room map and extension-type output + let haystack: HaystackEntries = output + .into_iter() + .map(|(acm_name, messages)| { + let rooms = &rooms; + let acm_name = format!("CM{}", acm_name); + + messages + .into_iter() + .filter(|message| message.command == "list extension-type") + .filter_map(|message| message.datas) + .flatten() + .map(move |mut extension| { + let room = extension + .get(0) + .map(|num| rooms.get(num)) + .flatten() + .map(|room| room.to_owned()) + .unwrap_or_else(String::new); + + extension.push(acm_name.to_owned()); + extension.push(room); + extension + }) + }) + .flatten() + .collect(); - if let Ok(mut handle) = inner.lock() { - *handle = String::from("fresh"); - eprintln!("evicted"); - } - }); + // Propogate the new data to the shared haystack entries + let mut lock = self + .entries + .lock() + .map_err(|e| anyhow!(e.to_string())) + .with_context(|| "Failed to get haystack inner data lock.")?; + *lock = haystack; + Ok(()) } } -- cgit v1.2.3 From 1fab7c878d55a307e5ef2078801be231b13ebab8 Mon Sep 17 00:00:00 2001 From: "Carpenter, Adam (CORP)" Date: Tue, 30 Nov 2021 12:07:14 -0500 Subject: fix: discourage caching responses with headers --- angelsharkd/src/routes/extensions/simple_search.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/angelsharkd/src/routes/extensions/simple_search.rs b/angelsharkd/src/routes/extensions/simple_search.rs index 963cb69..b728c1e 100644 --- a/angelsharkd/src/routes/extensions/simple_search.rs +++ b/angelsharkd/src/routes/extensions/simple_search.rs @@ -9,8 +9,10 @@ use std::{ use warp::{ body::{content_length_limit, json}, get, - hyper::StatusCode, - path, post, reply, Filter, Rejection, Reply, + hyper::{header, StatusCode}, + path, post, + reply::{self, with}, + Filter, Rejection, Reply, }; /// Collection of search terms @@ -20,13 +22,14 @@ type Needles = Vec; type HaystackEntries = Vec>; pub fn search(haystack: Haystack) -> impl Filter + Clone { - // TODO: discourage caching response thru headers - path("search") .and(post()) .and(content_length_limit(1024 * 16)) .and(json()) .and_then(move |terms: Needles| handle_search(haystack.to_owned(), terms)) + .with(with::header(header::PRAGMA, "no-cache")) + .with(with::header(header::CACHE_CONTROL, "no-store, max-age=0")) + .with(with::header(header::X_FRAME_OPTIONS, "DENY")) } pub fn refresh(haystack: Haystack) -> impl Filter + Clone { -- cgit v1.2.3 From cd9f750990166dfa9b9fc0276749be307e26d008 Mon Sep 17 00:00:00 2001 From: "Carpenter, Adam (CORP)" Date: Tue, 30 Nov 2021 15:22:37 -0500 Subject: refactor: module structure and comments --- angelsharkd/src/routes/extensions/mod.rs | 9 +- angelsharkd/src/routes/extensions/simple_search.rs | 197 --------------------- .../src/routes/extensions/simple_search/README.md | 3 + .../src/routes/extensions/simple_search/mod.rs | 66 +++++++ .../src/routes/extensions/simple_search/types.rs | 166 +++++++++++++++++ 5 files changed, 242 insertions(+), 199 deletions(-) delete mode 100644 angelsharkd/src/routes/extensions/simple_search.rs create mode 100644 angelsharkd/src/routes/extensions/simple_search/README.md create mode 100644 angelsharkd/src/routes/extensions/simple_search/mod.rs create mode 100644 angelsharkd/src/routes/extensions/simple_search/types.rs diff --git a/angelsharkd/src/routes/extensions/mod.rs b/angelsharkd/src/routes/extensions/mod.rs index adcea3f..7f217d6 100644 --- a/angelsharkd/src/routes/extensions/mod.rs +++ b/angelsharkd/src/routes/extensions/mod.rs @@ -6,18 +6,22 @@ mod simple_deprov; #[cfg(feature = "simple_search")] mod simple_search; +/// The extension filter; consists of all compiled optional Angelshark extension +/// filters combined under `/extensions`. pub fn filter(config: &Config) -> impl Filter + Clone { // Note: this next line deals with the common edge case of having no // extensions loaded with feature flags. It ensures that the the type // checking is right when the return `.and()` is called below. let filters = default().or(default()); + // Block to enable simple_search extension feature. Instantiates a + // searchable haystack and configures filters to handle search requests. #[cfg(feature = "simple_search")] let haystack = simple_search::Haystack::new(config.runner.clone()); #[cfg(feature = "simple_search")] let filters = filters - .or(simple_search::search(haystack.clone())) - .or(simple_search::refresh(haystack)); + .or(simple_search::search_filter(haystack.clone())) + .or(simple_search::refresh_filter(haystack)); #[cfg(feature = "simple_deprov")] let filters = filters.or(simple_deprov::filter()); @@ -25,6 +29,7 @@ pub fn filter(config: &Config) -> impl Filter impl Filter + Clone { warp::path::end().map(|| "Angelshark extension route index. Enable extensions with feature switches and access them at `/extensions/`.") } diff --git a/angelsharkd/src/routes/extensions/simple_search.rs b/angelsharkd/src/routes/extensions/simple_search.rs deleted file mode 100644 index b728c1e..0000000 --- a/angelsharkd/src/routes/extensions/simple_search.rs +++ /dev/null @@ -1,197 +0,0 @@ -use anyhow::{anyhow, Context, Error}; -use libangelshark::{AcmRunner, Message, ParallelIterator}; -use log::{error, info}; -use std::{ - collections::HashMap, - convert::Infallible, - sync::{Arc, Mutex}, -}; -use warp::{ - body::{content_length_limit, json}, - get, - hyper::{header, StatusCode}, - path, post, - reply::{self, with}, - Filter, Rejection, Reply, -}; - -/// Collection of search terms -type Needles = Vec; - -/// Collection of ACM extension types with ROOMs (if applicable) and ACM names -type HaystackEntries = Vec>; - -pub fn search(haystack: Haystack) -> impl Filter + Clone { - path("search") - .and(post()) - .and(content_length_limit(1024 * 16)) - .and(json()) - .and_then(move |terms: Needles| handle_search(haystack.to_owned(), terms)) - .with(with::header(header::PRAGMA, "no-cache")) - .with(with::header(header::CACHE_CONTROL, "no-store, max-age=0")) - .with(with::header(header::X_FRAME_OPTIONS, "DENY")) -} - -pub fn refresh(haystack: Haystack) -> impl Filter + Clone { - path!("search" / "refresh") - .and(get()) - .and_then(move || handle_refresh(haystack.to_owned())) -} - -async fn handle_search(haystack: Haystack, needle: Needles) -> Result { - // Ok(haystack.search(Vec::new())?) - // if let Ok(matches = haystack.search(needle); - match haystack.search(needle) { - Ok(matches) => Ok(reply::with_status(reply::json(&matches), StatusCode::OK)), - Err(e) => Ok(reply::with_status( - reply::json(&e.to_string()), - StatusCode::INTERNAL_SERVER_ERROR, - )), - } -} - -async fn handle_refresh(haystack: Haystack) -> Result { - // Run refresh as a background task and immediately return. - tokio::spawn(async move { - if let Err(e) = haystack.refresh() { - error!("{}", e.to_string()); // TODO: use logger - } else { - info!("Search haystack refreshed."); - } - }); - - Ok("Refresh scheduled") -} - -/// A lazy-loaded, asynchronously-refreshed exension-type haystack cache. -#[derive(Clone)] -pub struct Haystack { - entries: Arc>, - runner: AcmRunner, -} - -impl Haystack { - pub fn new(runner: AcmRunner) -> Self { - Self { - entries: Arc::new(Mutex::new(Vec::new())), - runner, - } - } - - pub fn search(&self, needles: Needles) -> Result { - let needles: Vec<_> = needles.iter().map(|n| n.to_lowercase()).collect(); - - let entries = self - .entries - .lock() - .map_err(|e| anyhow!(e.to_string())) - .with_context(|| "Failed to get haystack inner data lock.")?; - - let matches = entries - .iter() - .filter(|entry| { - let entry_str = entry.join("").to_lowercase(); - needles - .iter() - .all(|needle| entry_str.contains(needle.as_str())) - }) - .cloned() - .collect(); - Ok(matches) - } - - pub fn refresh(&self) -> Result<(), Error> { - let mut runner = self.runner.to_owned(); // TODO: This alone allows multiple refreshers to be run at once. Do we want that? - - // Queue jobs in ACM runner - for acm in &[ - "01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", - ] { - runner - .queue_input(acm, &Message::new("list extension-type")) - .queue_input( - acm, - &Message { - command: String::from("list station"), - fields: Some(vec![String::from("8005ff00"), String::from("0031ff00")]), - datas: None, - error: None, - }, - ); - } - - // Run jobs and collect output. Filter out uneeded commands, combine errors. - let output: Result)>, Error> = runner - .run() - .map(|(name, output)| { - let output: Vec = output? - .into_iter() - .filter(|m| m.command != "logoff") - .collect(); - Ok((name, output)) - }) - .collect(); - let output = output.with_context(|| "Failed to run refresh commands on ACM(s).")?; - - // Log any ACM errors encountered - for error in output - .iter() - .map(|(_, messages)| messages) - .flatten() - .filter_map(|m| m.error.as_ref()) - { - error!("ACM error: {}", error); - } - - // Build a map of station number-to-rooms - let rooms: HashMap = output - .iter() - .map(|(_, messages)| { - messages - .iter() - .filter(|message| message.command == "list station") - .filter_map(|message| message.datas.to_owned()) - .flatten() - .filter_map(|stat| Some((stat.get(0)?.to_owned(), stat.get(1)?.to_owned()))) - }) - .flatten() - .collect(); - - // Build the haystack from the room map and extension-type output - let haystack: HaystackEntries = output - .into_iter() - .map(|(acm_name, messages)| { - let rooms = &rooms; - let acm_name = format!("CM{}", acm_name); - - messages - .into_iter() - .filter(|message| message.command == "list extension-type") - .filter_map(|message| message.datas) - .flatten() - .map(move |mut extension| { - let room = extension - .get(0) - .map(|num| rooms.get(num)) - .flatten() - .map(|room| room.to_owned()) - .unwrap_or_else(String::new); - - extension.push(acm_name.to_owned()); - extension.push(room); - extension - }) - }) - .flatten() - .collect(); - - // Propogate the new data to the shared haystack entries - let mut lock = self - .entries - .lock() - .map_err(|e| anyhow!(e.to_string())) - .with_context(|| "Failed to get haystack inner data lock.")?; - *lock = haystack; - Ok(()) - } -} diff --git a/angelsharkd/src/routes/extensions/simple_search/README.md b/angelsharkd/src/routes/extensions/simple_search/README.md new file mode 100644 index 0000000..527f686 --- /dev/null +++ b/angelsharkd/src/routes/extensions/simple_search/README.md @@ -0,0 +1,3 @@ +# Angelshark Simple Extension Search + +This extension implements very simple extension searching for end users. It allows for multi-keyword searches on a pre-downloaded collection of CM extension-type and station data. This data can be refreshed on-demand or periodically. Searches always return immediately, even when no data has been downloaded yet. diff --git a/angelsharkd/src/routes/extensions/simple_search/mod.rs b/angelsharkd/src/routes/extensions/simple_search/mod.rs new file mode 100644 index 0000000..5b5dc8e --- /dev/null +++ b/angelsharkd/src/routes/extensions/simple_search/mod.rs @@ -0,0 +1,66 @@ +use log::{error, info}; +use std::convert::Infallible; +pub use types::Haystack; +use types::*; +use warp::{ + body::{content_length_limit, json}, + get, + hyper::{header, StatusCode}, + path, post, + reply::{self, with}, + Filter, Rejection, Reply, +}; + +mod types; + +/// Returns a warp filter to handle HTTP POSTs for searching the haystack. +pub fn search_filter( + haystack: Haystack, +) -> impl Filter + Clone { + path("search") + .and(post()) + .and(content_length_limit(1024 * 16)) + .and(json()) + .and_then(move |terms: Needles| search(haystack.to_owned(), terms)) + .with(with::header(header::PRAGMA, "no-cache")) + .with(with::header(header::CACHE_CONTROL, "no-store, max-age=0")) + .with(with::header(header::X_FRAME_OPTIONS, "DENY")) +} + +/// Returns a warp filter to handle HTTP GETs for refreshing the haystack. +pub fn refresh_filter( + haystack: Haystack, +) -> impl Filter + Clone { + path!("search" / "refresh") + .and(get()) + .and_then(move || refresh(haystack.to_owned())) +} + +/// Runs the search request to find all needles in the haystack and converts the +/// results into a reply. +async fn search(haystack: Haystack, needles: Needles) -> Result { + // Ok(haystack.search(Vec::new())?) + // if let Ok(matches = haystack.search(needle); + match haystack.search(needles) { + Ok(matches) => Ok(reply::with_status(reply::json(&matches), StatusCode::OK)), + Err(e) => Ok(reply::with_status( + reply::json(&e.to_string()), + StatusCode::INTERNAL_SERVER_ERROR, + )), + } +} + +/// Immediately returns. Spawns an asynchronous task to complete the haystack +/// refresh in the background. +async fn refresh(haystack: Haystack) -> Result { + // Run refresh as a background task and immediately return. + tokio::spawn(async move { + if let Err(e) = haystack.refresh() { + error!("{}", e.to_string()); // TODO: use logger + } else { + info!("Search haystack refreshed."); + } + }); + + Ok("Refresh scheduled") +} diff --git a/angelsharkd/src/routes/extensions/simple_search/types.rs b/angelsharkd/src/routes/extensions/simple_search/types.rs new file mode 100644 index 0000000..184b1a6 --- /dev/null +++ b/angelsharkd/src/routes/extensions/simple_search/types.rs @@ -0,0 +1,166 @@ +use anyhow::{anyhow, Context, Error}; +use libangelshark::{AcmRunner, Message, ParallelIterator}; +use log::error; +use std::{ + collections::HashMap, + env, + sync::{Arc, Mutex}, +}; + +const ANGELSHARKD_EXT_SEARCH_ACMS: &str = "ANGELSHARKD_EXT_SEARCH_ACMS"; +const OSSI_STAT_NUMBER_FIELD: &str = "8005ff00"; +const OSSI_STAT_ROOM_FIELD: &str = "0031ff00"; +const OSSI_LIST_STAT_CMD: &str = "list station"; +const OSSI_LIST_EXT_CMD: &str = "list extension-type"; + +/// Collection of search terms +pub type Needles = Vec; + +/// Collection of ACM extension types with ROOMs (if applicable) and ACM names +type HaystackEntries = Vec>; + +/// Represents a searchable, refreshable collection of ACM extension data. +#[derive(Clone)] +pub struct Haystack { + entries: Arc>, + runner: AcmRunner, +} + +impl Haystack { + pub fn new(runner: AcmRunner) -> Self { + Self { + entries: Arc::new(Mutex::new(Vec::new())), + runner, + } + } + + /// Searches for haystack entries that contain all given needles and returns them. + pub fn search(&self, needles: Needles) -> Result { + let needles: Vec<_> = needles.iter().map(|n| n.to_lowercase()).collect(); + + let entries = self + .entries + .lock() + .map_err(|e| anyhow!(e.to_string())) + .with_context(|| "Failed to get haystack inner data lock.")?; + + let matches = entries + .iter() + .filter(|entry| { + let entry_str = entry.join("").to_lowercase(); + needles + .iter() + .all(|needle| entry_str.contains(needle.as_str())) + }) + .cloned() + .collect(); + Ok(matches) + } + + /// Refreshes the haystack data by running relevant commands on a runner, + /// parsing the results, and updating the entries field with the fresh data. + /// TODO: Do we want simultaneous refreshes to be possible? + pub fn refresh(&self) -> Result<(), Error> { + let mut runner = self.runner.to_owned(); + + // Queue jobs in ACM runner + let configured_acms = env::var(ANGELSHARKD_EXT_SEARCH_ACMS).with_context(|| { + format!( + "{} var missing. Cannot refresh haystack.", + ANGELSHARKD_EXT_SEARCH_ACMS + ) + })?; + + // Generate jobs and queue on runner. + for acm in configured_acms.split_whitespace() { + runner + .queue_input(acm, &Message::new(OSSI_LIST_EXT_CMD)) + .queue_input( + acm, + &Message { + command: String::from(OSSI_LIST_STAT_CMD), + fields: Some(vec![ + String::from(OSSI_STAT_NUMBER_FIELD), + String::from(OSSI_STAT_ROOM_FIELD), + ]), + datas: None, + error: None, + }, + ); + } + + // Run jobs and collect output. Filter out uneeded commands, combine errors. + let output: Result)>, Error> = runner + .run() + .map(|(name, output)| { + let output: Vec = output? + .into_iter() + .filter(|m| m.command != "logoff") + .collect(); + Ok((name, output)) + }) + .collect(); + let output = output.with_context(|| "Failed to run refresh commands on ACM(s).")?; + + // Log any ACM errors encountered + for error in output + .iter() + .map(|(_, messages)| messages) + .flatten() + .filter_map(|m| m.error.as_ref()) + { + error!("ACM error: {}", error); + } + + // Build a map of station number-to-rooms + let rooms: HashMap = output + .iter() + .map(|(_, messages)| { + messages + .iter() + .filter(|message| message.command == OSSI_LIST_STAT_CMD) + .filter_map(|message| message.datas.to_owned()) + .flatten() + .filter_map(|stat| Some((stat.get(0)?.to_owned(), stat.get(1)?.to_owned()))) + }) + .flatten() + .collect(); + + // Build the haystack from the room map and extension-type output + let haystack: HaystackEntries = output + .into_iter() + .map(|(acm_name, messages)| { + let rooms = &rooms; + let acm_name = format!("CM{}", acm_name); + + messages + .into_iter() + .filter(|message| message.command == OSSI_LIST_EXT_CMD) + .filter_map(|message| message.datas) + .flatten() + .map(move |mut extension| { + let room = extension + .get(0) + .map(|num| rooms.get(num)) + .flatten() + .map(|room| room.to_owned()) + .unwrap_or_else(String::new); + + extension.push(acm_name.to_owned()); + extension.push(room); + extension + }) + }) + .flatten() + .collect(); + + // Overwrite shared haystack entries with new data. + let mut lock = self + .entries + .lock() + .map_err(|e| anyhow!(e.to_string())) + .with_context(|| "Failed to get haystack inner data lock.")?; + *lock = haystack; + Ok(()) + } +} -- cgit v1.2.3 From e18d0c1a4189d5278639a9b323ae3794118566bc Mon Sep 17 00:00:00 2001 From: "Carpenter, Adam (CORP)" Date: Tue, 30 Nov 2021 16:04:37 -0500 Subject: docs: updated readmes --- angelsharkd/README.md | 4 +- angelsharkd/src/routes/extensions/README.md | 17 ++--- .../src/routes/extensions/simple_search/README.md | 80 +++++++++++++++++++++- .../src/routes/extensions/simple_search/types.rs | 1 + 4 files changed, 90 insertions(+), 12 deletions(-) diff --git a/angelsharkd/README.md b/angelsharkd/README.md index 405a039..8315c74 100644 --- a/angelsharkd/README.md +++ b/angelsharkd/README.md @@ -80,8 +80,8 @@ Response Template: Here are some examples. ```json -POST / - ossi[ +POST /ossi + [ { "acms": ["CM01"], "command": "list stat 17571230000" diff --git a/angelsharkd/src/routes/extensions/README.md b/angelsharkd/src/routes/extensions/README.md index b9192fe..1aed1dd 100644 --- a/angelsharkd/src/routes/extensions/README.md +++ b/angelsharkd/src/routes/extensions/README.md @@ -1,12 +1,13 @@ # Angelshark Daemon Extensions -This module aims to provide a simple way of extending Angelshark's basic functionality (running commands on the ACM) with additional HTTP endpoints that run one or more commands to achieve a basic business task. +This module aims to provide a simple way of extending Angelshark's basic +functionality (running commands on the ACM) with additional HTTP endpoints that +run one or more commands to achieve a basic business task. -For example, say you would like you, other users, or your own software to quickly search all extension-types for a keyword. This functionality is not in the base `angelsharkd`, but it can be easily implemented with the following steps: +This functionality may not be desirable for all end users, and therefore is +completely opt-in with feature flags. For example, at compile time, you can add +`--features simple_search` to enable a given extension called `simple_search`. -1. Accept a keyword from the client's request -1. Download extension-type data from one or more ACMs -1. Filter out extensions that do not match a given keyword -1. Return the remaining, matching extensions to the client - -This functionality may not be desirable for all end users, and therefor is completely opt-in with feature flags. At compile time, you can add `--features simple_search` to enable a given extension called `simple_search`, for example. +To add additional features, read `mod.rs` and `Cargo.toml` for `angelsharkd` to +see how to conditionally incorporate your own warp HTTP filters into the +project. diff --git a/angelsharkd/src/routes/extensions/simple_search/README.md b/angelsharkd/src/routes/extensions/simple_search/README.md index 527f686..c0df17a 100644 --- a/angelsharkd/src/routes/extensions/simple_search/README.md +++ b/angelsharkd/src/routes/extensions/simple_search/README.md @@ -1,3 +1,79 @@ -# Angelshark Simple Extension Search +# Daemon Extension `simple_search` -This extension implements very simple extension searching for end users. It allows for multi-keyword searches on a pre-downloaded collection of CM extension-type and station data. This data can be refreshed on-demand or periodically. Searches always return immediately, even when no data has been downloaded yet. +This extension implements fast and simple extension searching for clients. Data +to be searched is downloaded from configured ACMs as a "haystack." Clients pass +a list of one or more "needles" to search for. Haystack entries matching all +provided needles are returned. + +The haystack starts out empty. To trigger a download, use the `refresh` +endpoint. Refreshes happen in the background, so clients always receive an +immediate response. This means that clients will be searching on "stale" data +(or on the first run, no data) until the refresh completes. + +There is no built-in scheduler for periodic haystack refreshes. It is +recommended to use an external scheduler such as `cron(8)`. + +## Getting Started + +To enable this feature, compile `angelsharkd` with the `simple_search` flag: + +```sh +cargo build --bin angelsharkd --features simple_search ... +``` + +This extension expects a single environment variable, +`ANGELSHARKD_EXT_SEARCH_ACMS`, to be set at runtime. This var should be a list +of ACM names from your `asa.cfg`. For example, if your `asa.cfg` is configured +for ACMs named 01, 02, and 03, and you want to search over 01 and 03, run +`angelsharkd` like this: + +``` +ANGELSHARKD_ADDR='127.0.0.1:3000' ANGELSHARKD_EXT_SEARCH_ACMS='01 03' angelsharkd +``` + +## `GET /extensions/search/refresh` Refresh Haystack + +`GET /extensions/search/refresh` + +``` +200 OK text/plain +Refresh scheduled. +``` + +## `POST /extensions/search` Search Haystack + +The return type is the entire list of fields from the `list extension-type` +command, the ROOM field from the `list station` command, and the configured name +of the ACM the entry was found on. + +```json +POST /extensions/search +[ + "carpenat" +] +``` + +```json +200 OK +[ + [ + "17571230000", + "station-user", + "5", + "1", + "1005", + "Carpenter, Adam", + "2", + "", + "CM01", + "carpenat" + ], + ... +] +``` + +## Logging + +The `refresh` endpoint always returns successfully. Any errors encountered +during the refresh are logged as `ERROR`. Successful completed refreshes are +logged as `INFO`. diff --git a/angelsharkd/src/routes/extensions/simple_search/types.rs b/angelsharkd/src/routes/extensions/simple_search/types.rs index 184b1a6..72156e5 100644 --- a/angelsharkd/src/routes/extensions/simple_search/types.rs +++ b/angelsharkd/src/routes/extensions/simple_search/types.rs @@ -60,6 +60,7 @@ impl Haystack { /// Refreshes the haystack data by running relevant commands on a runner, /// parsing the results, and updating the entries field with the fresh data. /// TODO: Do we want simultaneous refreshes to be possible? + /// TODO: The entry generation could probably be simplified and the number of clones reduced. pub fn refresh(&self) -> Result<(), Error> { let mut runner = self.runner.to_owned(); -- cgit v1.2.3