Merge pull request #5 from adpllc/extensions

Extensions and simple extension search

12 files changed, 404 insertions, 29 deletions

diff --git a/Cargo.lock b/Cargo.lock
index a3b5bd6..5a272a9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -24,7 +24,7 @@ dependencies = [
 
 [[package]]
 name = "angelsharkd"
-version = "0.1.2"
+version = "0.2.0"
 dependencies = [
  "anyhow",
  "env_logger",
@@ -51,6 +51,16 @@ dependencies = [
 ]
 
 [[package]]
+name = "async-rwlock"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "261803dcc39ba9e72760ba6e16d0199b1eef9fc44e81bffabbebb9f5aea3906c"
+dependencies = [
+ "async-mutex",
+ "event-listener",
+]
+
+[[package]]
 name = "atty"
 version = "0.2.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -108,24 +118,24 @@ checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
 
 [[package]]
 name = "cached"
-version = "0.25.0"
+version = "0.26.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b99e696f7b2696ed5eae0d462a9eeafaea111d99e39b2c8ceb418afe1013bcfc"
+checksum = "c2bc2fd249a24a9cdd4276f3a3e0461713271ab63b0e9e656e200e8e21c8c927"
 dependencies = [
  "async-mutex",
+ "async-rwlock",
  "cached_proc_macro",
  "cached_proc_macro_types",
- "hashbrown 0.9.1",
+ "hashbrown",
  "once_cell",
 ]
 
 [[package]]
 name = "cached_proc_macro"
-version = "0.6.1"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25a685ba39b57a91a53d149dcbef854f50fbe204d1ff6081ea0bec3529a0c456"
+checksum = "ac3531903b39df48a378a7ed515baee7c1fff32488489c7d0725eb1749b22a91"
 dependencies = [
- "async-mutex",
  "cached_proc_macro_types",
  "darling",
  "quote",
@@ -253,9 +263,9 @@ dependencies = [
 
 [[package]]
 name = "darling"
-version = "0.10.2"
+version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d706e75d87e35569db781a9b5e2416cff1236a47ed380831f959382ccd5f858"
+checksum = "757c0ded2af11d8e739c4daea1ac623dd1624b06c844cf3f5a39f1bdbd99bb12"
 dependencies = [
  "darling_core",
  "darling_macro",
@@ -263,9 +273,9 @@ dependencies = [
 
 [[package]]
 name = "darling_core"
-version = "0.10.2"
+version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0c960ae2da4de88a91b2d920c2a7233b400bc33cb28453a2987822d8392519b"
+checksum = "2c34d8efb62d0c2d7f60ece80f75e5c63c1588ba68032740494b0b9a996466e3"
 dependencies = [
  "fnv",
  "ident_case",
@@ -277,9 +287,9 @@ dependencies = [
 
 [[package]]
 name = "darling_macro"
-version = "0.10.2"
+version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72"
+checksum = "ade7bff147130fe5e6d39f089c6bd49ec0250f35d70b2eebf72afdfc919f15cc"
 dependencies = [
  "darling_core",
  "quote",
@@ -303,9 +313,9 @@ checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
 
 [[package]]
 name = "env_logger"
-version = "0.8.4"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3"
+checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3"
 dependencies = [
  "atty",
  "humantime",
@@ -429,12 +439,6 @@ dependencies = [
 
 [[package]]
 name = "hashbrown"
-version = "0.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
-
-[[package]]
-name = "hashbrown"
 version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
@@ -550,7 +554,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5"
 dependencies = [
  "autocfg",
- "hashbrown 0.11.2",
+ "hashbrown",
 ]
 
 [[package]]
@@ -997,9 +1001,9 @@ dependencies = [
 
 [[package]]
 name = "strsim"
-version = "0.9.3"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c"
+checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
 
 [[package]]
 name = "syn"
diff --git a/angelsharkd/Cargo.toml b/angelsharkd/Cargo.toml
index f820528..9fdb00c 100644
--- a/angelsharkd/Cargo.toml
+++ b/angelsharkd/Cargo.toml
@@ -1,10 +1,14 @@
 [package]
 name = "angelsharkd"
-version = "0.1.2"
+version = "0.2.0"
 edition = "2021"
 authors = ["Adam T. Carpenter <adam.carpenter@adp.com>"]
 description = "A HTTP interface into one or more Communication Managers"
 
+[features]
+simple_search = []
+simple_deprov = []
+
 [dependencies.libangelshark]
 path = "../libangelshark"
 
@@ -20,7 +24,7 @@ default-features = false
 version = "0.4"
 
 [dependencies.env_logger]
-version = "0.8"
+version = "0.9"
 
 [dependencies.serde]
 version = "1"
diff --git a/angelsharkd/README.md b/angelsharkd/README.md
index 405a039..8315c74 100644
--- a/angelsharkd/README.md
+++ b/angelsharkd/README.md
@@ -80,8 +80,8 @@ Response Template:
 Here are some examples.
 
 ```json
-POST /
-  ossi[
+POST /ossi
+  [
     {
       "acms": ["CM01"],
       "command": "list stat 17571230000"
diff --git a/angelsharkd/src/main.rs b/angelsharkd/src/main.rs
index 8a8a53e..790b641 100644
--- a/angelsharkd/src/main.rs
+++ b/angelsharkd/src/main.rs
@@ -30,6 +30,7 @@ async fn main() -> Result<()> {
 
     let routes = routes::index()
         .or(routes::ossi(&config))
+        .or(routes::extensions::filter(&config))
         .with(if config.debug_mode || config.origin == "*" {
             warp::cors()
                 .allow_any_origin()
diff --git a/angelsharkd/src/routes/extensions/README.md b/angelsharkd/src/routes/extensions/README.md
new file mode 100644
index 0000000..1aed1dd
--- /dev/null
+++ b/angelsharkd/src/routes/extensions/README.md
@@ -0,0 +1,13 @@
+# Angelshark Daemon Extensions
+
+This module aims to provide a simple way of extending Angelshark's basic
+functionality (running commands on the ACM) with additional HTTP endpoints that
+run one or more commands to achieve a basic business task.
+
+This functionality may not be desirable for all end users, and therefore is
+completely opt-in with feature flags. For example, at compile time, you can add
+`--features simple_search` to enable a given extension called `simple_search`.
+
+To add additional features, read `mod.rs` and `Cargo.toml` for `angelsharkd` to
+see how to conditionally incorporate your own warp HTTP filters into the
+project.
diff --git a/angelsharkd/src/routes/extensions/mod.rs b/angelsharkd/src/routes/extensions/mod.rs
new file mode 100644
index 0000000..7f217d6
--- /dev/null
+++ b/angelsharkd/src/routes/extensions/mod.rs
@@ -0,0 +1,35 @@
+use crate::config::Config;
+use warp::{path, Filter, Rejection, Reply};
+
+#[cfg(feature = "simple_deprov")]
+mod simple_deprov;
+#[cfg(feature = "simple_search")]
+mod simple_search;
+
+/// The extension filter; consists of all compiled optional Angelshark extension
+/// filters combined under `/extensions`.
+pub fn filter(config: &Config) -> impl Filter<Extract = impl Reply, Error = Rejection> + Clone {
+    // Note: this next line deals with the common edge case of having no
+    // extensions loaded with feature flags. It ensures that the the type
+    // checking is right when the return `.and()` is called below.
+    let filters = default().or(default());
+
+    // Block to enable simple_search extension feature. Instantiates a
+    // searchable haystack and configures filters to handle search requests.
+    #[cfg(feature = "simple_search")]
+    let haystack = simple_search::Haystack::new(config.runner.clone());
+    #[cfg(feature = "simple_search")]
+    let filters = filters
+        .or(simple_search::search_filter(haystack.clone()))
+        .or(simple_search::refresh_filter(haystack));
+
+    #[cfg(feature = "simple_deprov")]
+    let filters = filters.or(simple_deprov::filter());
+
+    path("extensions").and(filters)
+}
+
+/// The default, informational extension route.
+fn default() -> impl Filter<Extract = impl Reply, Error = Rejection> + Clone {
+    warp::path::end().map(|| "Angelshark extension route index. Enable extensions with feature switches and access them at `/extensions/<feature>`.")
+}
diff --git a/angelsharkd/src/routes/extensions/simple_deprov.rs b/angelsharkd/src/routes/extensions/simple_deprov.rs
new file mode 100644
index 0000000..2b5ad40
--- /dev/null
+++ b/angelsharkd/src/routes/extensions/simple_deprov.rs
@@ -0,0 +1,5 @@
+use warp::{Filter, Rejection, Reply};
+
+pub fn filter() -> impl Filter<Extract = impl Reply, Error = Rejection> + Clone {
+    warp::path("deprov").map(|| -> &str { todo!() }) // TODO:
+}
diff --git a/angelsharkd/src/routes/extensions/simple_search/README.md b/angelsharkd/src/routes/extensions/simple_search/README.md
new file mode 100644
index 0000000..c0df17a
--- /dev/null
+++ b/angelsharkd/src/routes/extensions/simple_search/README.md
@@ -0,0 +1,79 @@
+# Daemon Extension `simple_search`
+
+This extension implements fast and simple extension searching for clients. Data
+to be searched is downloaded from configured ACMs as a "haystack." Clients pass
+a list of one or more "needles" to search for. Haystack entries matching all
+provided needles are returned.
+
+The haystack starts out empty. To trigger a download, use the `refresh`
+endpoint. Refreshes happen in the background, so clients always receive an
+immediate response. This means that clients will be searching on "stale" data
+(or on the first run, no data) until the refresh completes.
+
+There is no built-in scheduler for periodic haystack refreshes. It is
+recommended to use an external scheduler such as `cron(8)`.
+
+## Getting Started
+
+To enable this feature, compile `angelsharkd` with the `simple_search` flag:
+
+```sh
+cargo build --bin angelsharkd --features simple_search ...
+```
+
+This extension expects a single environment variable,
+`ANGELSHARKD_EXT_SEARCH_ACMS`, to be set at runtime. This var should be a list
+of ACM names from your `asa.cfg`. For example, if your `asa.cfg` is configured
+for ACMs named 01, 02, and 03, and you want to search over 01 and 03, run
+`angelsharkd` like this:
+
+```
+ANGELSHARKD_ADDR='127.0.0.1:3000' ANGELSHARKD_EXT_SEARCH_ACMS='01 03' angelsharkd
+```
+
+## `GET /extensions/search/refresh` Refresh Haystack
+
+`GET /extensions/search/refresh`
+
+```
+200 OK text/plain
+Refresh scheduled.
+```
+
+## `POST /extensions/search` Search Haystack
+
+The return type is the entire list of fields from the `list extension-type`
+command, the ROOM field from the `list station` command, and the configured name
+of the ACM the entry was found on.
+
+```json
+POST /extensions/search
+[
+	"carpenat"
+]
+```
+
+```json
+200 OK
+[
+    [
+        "17571230000",
+        "station-user",
+        "5",
+        "1",
+        "1005",
+        "Carpenter, Adam",
+        "2",
+        "",
+        "CM01",
+        "carpenat"
+    ],
+	...
+]
+```
+
+## Logging
+
+The `refresh` endpoint always returns successfully. Any errors encountered
+during the refresh are logged as `ERROR`. Successful completed refreshes are
+logged as `INFO`.
diff --git a/angelsharkd/src/routes/extensions/simple_search/mod.rs b/angelsharkd/src/routes/extensions/simple_search/mod.rs
new file mode 100644
index 0000000..5b5dc8e
--- /dev/null
+++ b/angelsharkd/src/routes/extensions/simple_search/mod.rs
@@ -0,0 +1,66 @@
+use log::{error, info};
+use std::convert::Infallible;
+pub use types::Haystack;
+use types::*;
+use warp::{
+    body::{content_length_limit, json},
+    get,
+    hyper::{header, StatusCode},
+    path, post,
+    reply::{self, with},
+    Filter, Rejection, Reply,
+};
+
+mod types;
+
+/// Returns a warp filter to handle HTTP POSTs for searching the haystack.
+pub fn search_filter(
+    haystack: Haystack,
+) -> impl Filter<Extract = impl Reply, Error = Rejection> + Clone {
+    path("search")
+        .and(post())
+        .and(content_length_limit(1024 * 16))
+        .and(json())
+        .and_then(move |terms: Needles| search(haystack.to_owned(), terms))
+        .with(with::header(header::PRAGMA, "no-cache"))
+        .with(with::header(header::CACHE_CONTROL, "no-store, max-age=0"))
+        .with(with::header(header::X_FRAME_OPTIONS, "DENY"))
+}
+
+/// Returns a warp filter to handle HTTP GETs for refreshing the haystack.
+pub fn refresh_filter(
+    haystack: Haystack,
+) -> impl Filter<Extract = impl Reply, Error = Rejection> + Clone {
+    path!("search" / "refresh")
+        .and(get())
+        .and_then(move || refresh(haystack.to_owned()))
+}
+
+/// Runs the search request to find all needles in the haystack and converts the
+/// results into a reply.
+async fn search(haystack: Haystack, needles: Needles) -> Result<impl Reply, Infallible> {
+    // Ok(haystack.search(Vec::new())?)
+    // if let Ok(matches = haystack.search(needle);
+    match haystack.search(needles) {
+        Ok(matches) => Ok(reply::with_status(reply::json(&matches), StatusCode::OK)),
+        Err(e) => Ok(reply::with_status(
+            reply::json(&e.to_string()),
+            StatusCode::INTERNAL_SERVER_ERROR,
+        )),
+    }
+}
+
+/// Immediately returns. Spawns an asynchronous task to complete the haystack
+/// refresh in the background.
+async fn refresh(haystack: Haystack) -> Result<impl Reply, Infallible> {
+    // Run refresh as a background task and immediately return.
+    tokio::spawn(async move {
+        if let Err(e) = haystack.refresh() {
+            error!("{}", e.to_string()); // TODO: use logger
+        } else {
+            info!("Search haystack refreshed.");
+        }
+    });
+
+    Ok("Refresh scheduled")
+}
diff --git a/angelsharkd/src/routes/extensions/simple_search/types.rs b/angelsharkd/src/routes/extensions/simple_search/types.rs
new file mode 100644
index 0000000..72156e5
--- /dev/null
+++ b/angelsharkd/src/routes/extensions/simple_search/types.rs
@@ -0,0 +1,167 @@
+use anyhow::{anyhow, Context, Error};
+use libangelshark::{AcmRunner, Message, ParallelIterator};
+use log::error;
+use std::{
+    collections::HashMap,
+    env,
+    sync::{Arc, Mutex},
+};
+
+const ANGELSHARKD_EXT_SEARCH_ACMS: &str = "ANGELSHARKD_EXT_SEARCH_ACMS";
+const OSSI_STAT_NUMBER_FIELD: &str = "8005ff00";
+const OSSI_STAT_ROOM_FIELD: &str = "0031ff00";
+const OSSI_LIST_STAT_CMD: &str = "list station";
+const OSSI_LIST_EXT_CMD: &str = "list extension-type";
+
+/// Collection of search terms
+pub type Needles = Vec<String>;
+
+/// Collection of ACM extension types with ROOMs (if applicable) and ACM names
+type HaystackEntries = Vec<Vec<String>>;
+
+/// Represents a searchable, refreshable collection of ACM extension data.
+#[derive(Clone)]
+pub struct Haystack {
+    entries: Arc<Mutex<HaystackEntries>>,
+    runner: AcmRunner,
+}
+
+impl Haystack {
+    pub fn new(runner: AcmRunner) -> Self {
+        Self {
+            entries: Arc::new(Mutex::new(Vec::new())),
+            runner,
+        }
+    }
+
+    /// Searches for haystack entries that contain all given needles and returns them.
+    pub fn search(&self, needles: Needles) -> Result<HaystackEntries, Error> {
+        let needles: Vec<_> = needles.iter().map(|n| n.to_lowercase()).collect();
+
+        let entries = self
+            .entries
+            .lock()
+            .map_err(|e| anyhow!(e.to_string()))
+            .with_context(|| "Failed to get haystack inner data lock.")?;
+
+        let matches = entries
+            .iter()
+            .filter(|entry| {
+                let entry_str = entry.join("").to_lowercase();
+                needles
+                    .iter()
+                    .all(|needle| entry_str.contains(needle.as_str()))
+            })
+            .cloned()
+            .collect();
+        Ok(matches)
+    }
+
+    /// Refreshes the haystack data by running relevant commands on a runner,
+    /// parsing the results, and updating the entries field with the fresh data.
+    /// TODO: Do we want simultaneous refreshes to be possible?
+    /// TODO: The entry generation could probably be simplified and the number of clones reduced.
+    pub fn refresh(&self) -> Result<(), Error> {
+        let mut runner = self.runner.to_owned();
+
+        // Queue jobs in ACM runner
+        let configured_acms = env::var(ANGELSHARKD_EXT_SEARCH_ACMS).with_context(|| {
+            format!(
+                "{} var missing. Cannot refresh haystack.",
+                ANGELSHARKD_EXT_SEARCH_ACMS
+            )
+        })?;
+
+        // Generate jobs and queue on runner.
+        for acm in configured_acms.split_whitespace() {
+            runner
+                .queue_input(acm, &Message::new(OSSI_LIST_EXT_CMD))
+                .queue_input(
+                    acm,
+                    &Message {
+                        command: String::from(OSSI_LIST_STAT_CMD),
+                        fields: Some(vec![
+                            String::from(OSSI_STAT_NUMBER_FIELD),
+                            String::from(OSSI_STAT_ROOM_FIELD),
+                        ]),
+                        datas: None,
+                        error: None,
+                    },
+                );
+        }
+
+        // Run jobs and collect output. Filter out uneeded commands, combine errors.
+        let output: Result<Vec<(String, Vec<Message>)>, Error> = runner
+            .run()
+            .map(|(name, output)| {
+                let output: Vec<Message> = output?
+                    .into_iter()
+                    .filter(|m| m.command != "logoff")
+                    .collect();
+                Ok((name, output))
+            })
+            .collect();
+        let output = output.with_context(|| "Failed to run refresh commands on ACM(s).")?;
+
+        // Log any ACM errors encountered
+        for error in output
+            .iter()
+            .map(|(_, messages)| messages)
+            .flatten()
+            .filter_map(|m| m.error.as_ref())
+        {
+            error!("ACM error: {}", error);
+        }
+
+        // Build a map of station number-to-rooms
+        let rooms: HashMap<String, String> = output
+            .iter()
+            .map(|(_, messages)| {
+                messages
+                    .iter()
+                    .filter(|message| message.command == OSSI_LIST_STAT_CMD)
+                    .filter_map(|message| message.datas.to_owned())
+                    .flatten()
+                    .filter_map(|stat| Some((stat.get(0)?.to_owned(), stat.get(1)?.to_owned())))
+            })
+            .flatten()
+            .collect();
+
+        // Build the haystack from the room map and extension-type output
+        let haystack: HaystackEntries = output
+            .into_iter()
+            .map(|(acm_name, messages)| {
+                let rooms = &rooms;
+                let acm_name = format!("CM{}", acm_name);
+
+                messages
+                    .into_iter()
+                    .filter(|message| message.command == OSSI_LIST_EXT_CMD)
+                    .filter_map(|message| message.datas)
+                    .flatten()
+                    .map(move |mut extension| {
+                        let room = extension
+                            .get(0)
+                            .map(|num| rooms.get(num))
+                            .flatten()
+                            .map(|room| room.to_owned())
+                            .unwrap_or_else(String::new);
+
+                        extension.push(acm_name.to_owned());
+                        extension.push(room);
+                        extension
+                    })
+            })
+            .flatten()
+            .collect();
+
+        // Overwrite shared haystack entries with new data.
+        let mut lock = self
+            .entries
+            .lock()
+            .map_err(|e| anyhow!(e.to_string()))
+            .with_context(|| "Failed to get haystack inner data lock.")?;
+        *lock = haystack;
+        Ok(())
+    }
+}
diff --git a/angelsharkd/src/routes/mod.rs b/angelsharkd/src/routes/mod.rs
index 40d024d..cee0657 100644
--- a/angelsharkd/src/routes/mod.rs
+++ b/angelsharkd/src/routes/mod.rs
@@ -13,6 +13,7 @@ use warp::{
 };
 
 mod dtos;
+pub mod extensions;
 
 /// GET / -> Name and version # of app.
 pub fn index() -> impl Filter<Extract = impl Reply, Error = Rejection> + Clone {
diff --git a/libangelshark/Cargo.toml b/libangelshark/Cargo.toml
index 6b945d0..8c3b77b 100644
--- a/libangelshark/Cargo.toml
+++ b/libangelshark/Cargo.toml
@@ -16,6 +16,6 @@ version = "0.9"
 features = ["vendored-openssl"]
 
 [dependencies.cached]
-version = "0.25"
+version = "0.26"
 default-features = false
 features = ["proc_macro"]