Auto rollback if deployment is not confirmed

5 files changed, 354 insertions, 84 deletions

diff --git a/Cargo.lock b/Cargo.lock
index ab3bdba..bfb1db4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -87,6 +87,9 @@ name = "deploy-rs"
 version = "0.1.0"
 dependencies = [
  "clap",
+ "fork",
+ "futures-util",
+ "inotify",
  "log",
  "merge",
  "pretty_env_logger",
@@ -117,6 +120,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
 [[package]]
+name = "fork"
+version = "0.1.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48ab1b42fd10696f85fa0bd0856da17ba05b18d527dbaf846359441ecb808d92"
+dependencies = [
+ "libc",
+]
+
+[[package]]
 name = "fuchsia-zircon"
 version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -134,9 +146,46 @@ checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
 
 [[package]]
 name = "futures-core"
-version = "0.3.5"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d674eaa0056896d5ada519900dbf97ead2e46a7b6621e8160d79e2f2e1e2784b"
+
+[[package]]
+name = "futures-macro"
+version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59f5fff90fd5d971f936ad674802482ba441b6f09ba5e15fd8b39145582ca399"
+checksum = "f57ed14da4603b2554682e9f2ff3c65d7567b53188db96cb71538217fc64581b"
+dependencies = [
+ "proc-macro-hack",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "futures-task"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4dd26820a9f3637f1302da8bceba3ff33adbe53464b54ca24d4e2d4f1db30f94"
+dependencies = [
+ "once_cell",
+]
+
+[[package]]
+name = "futures-util"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a894a0acddba51a2d49a6f4263b1e64b8c579ece8af50fa86503d52cd1eea34"
+dependencies = [
+ "futures-core",
+ "futures-macro",
+ "futures-task",
+ "pin-project",
+ "pin-utils",
+ "proc-macro-hack",
+ "proc-macro-nested",
+ "slab",
+]
 
 [[package]]
 name = "hashbrown"
@@ -182,6 +231,29 @@ dependencies = [
 ]
 
 [[package]]
+name = "inotify"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "46dd0a94b393c730779ccfd2a872b67b1eb67be3fc33082e733bdb38b5fde4d4"
+dependencies = [
+ "bitflags",
+ "futures-core",
+ "inotify-sys",
+ "libc",
+ "mio",
+ "tokio",
+]
+
+[[package]]
+name = "inotify-sys"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e74a1aa87c59aeff6ef2cc2fa62d41bc43f54952f55652656b18a02fd5e356c0"
+dependencies = [
+ "libc",
+]
+
+[[package]]
 name = "iovec"
 version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -214,9 +286,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.77"
+version = "0.2.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2f96b10ec2560088a8e76961b00d47107b3a625fecb76dedb29ee7ccbf98235"
+checksum = "2448f6066e80e3bfc792e9c98bf705b4b0fc6e8ef5b43e5889aff0eaa9c58743"
 
 [[package]]
 name = "log"
@@ -350,18 +422,50 @@ dependencies = [
 ]
 
 [[package]]
+name = "once_cell"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "260e51e7efe62b592207e9e13a68e43692a7a279171d6ba57abd208bf23645ad"
+
+[[package]]
 name = "os_str_bytes"
 version = "2.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2ac6fe3538f701e339953a3ebbe4f39941aababa8a3f6964635b24ab526daeac"
 
 [[package]]
+name = "pin-project"
+version = "0.4.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15"
+dependencies = [
+ "pin-project-internal",
+]
+
+[[package]]
+name = "pin-project-internal"
+version = "0.4.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
 name = "pin-project-lite"
 version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "282adbf10f2698a7a77f8e983a74b2d18176c19a7fd32a45446139ae7b02b715"
 
 [[package]]
+name = "pin-utils"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
+
+[[package]]
 name = "pretty_env_logger"
 version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -396,10 +500,22 @@ dependencies = [
 ]
 
 [[package]]
+name = "proc-macro-hack"
+version = "0.5.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "99c605b9a0adc77b7211c6b1f722dcb613d68d66859a44f3d485a6da332b0598"
+
+[[package]]
+name = "proc-macro-nested"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a"
+
+[[package]]
 name = "proc-macro2"
-version = "1.0.21"
+version = "1.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36e28516df94f3dd551a587da5357459d9b36d945a7c37c3557928c1c2ff2a2c"
+checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71"
 dependencies = [
  "unicode-xid",
 ]
@@ -513,9 +629,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
 
 [[package]]
 name = "syn"
-version = "1.0.41"
+version = "1.0.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6690e3e9f692504b941dc6c3b188fd28df054f7fb8469ab40680df52fdcc842b"
+checksum = "e03e57e4fcbfe7749842d53e24ccb9aa12b7252dbe5e91d2acad31834c8b8fdd"
 dependencies = [
  "proc-macro2",
  "quote",
diff --git a/Cargo.toml b/Cargo.toml
index 07b4144..e8474b4 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -20,6 +20,9 @@ merge = "0.1.0"
 whoami = "0.9.0"
 log = "0.4"
 pretty_env_logger = "0.4"
+inotify = "0.8"
+futures-util = "0.3.6"
+fork = "0.1"
 
 [[bin]]
 name = "deploy"
@@ -27,4 +30,4 @@ path = "src/main.rs"
 
 [[bin]]
 name = "activate"
-path = "src/activate.rs"
+path = "src/activate.rs"
\ No newline at end of file
diff --git a/src/activate.rs b/src/activate.rs
index 0b7d28d..ce6b286 100644
--- a/src/activate.rs
+++ b/src/activate.rs
@@ -4,11 +4,20 @@
 
 use clap::Clap;
 
+use futures_util::FutureExt;
 use std::process::Stdio;
+use tokio::fs;
 use tokio::process::Command;
+use tokio::time::timeout;
+
+use std::time::Duration;
+
+use futures_util::StreamExt;
 
 use std::path::Path;
 
+use inotify::Inotify;
+
 extern crate pretty_env_logger;
 #[macro_use]
 extern crate log;
@@ -25,6 +34,8 @@ mod utils;
 struct Opts {
     profile_path: String,
     closure: String,
+    temp_path: String,
+    max_time: u16,
 
     /// Command for bootstrapping
     #[clap(long)]
@@ -35,11 +46,162 @@ struct Opts {
     auto_rollback: bool,
 }
 
+pub async fn deactivate(profile_path: &str) -> Result<(), Box<dyn std::error::Error>> {
+    error!("De-activating due to error");
+
+    let nix_env_rollback_exit_status = Command::new("nix-env")
+        .arg("-p")
+        .arg(&profile_path)
+        .arg("--rollback")
+        .stdout(Stdio::null())
+        .stderr(Stdio::null())
+        .status()
+        .await?;
+
+    if !nix_env_rollback_exit_status.success() {
+        good_panic!("`nix-env --rollback` failed");
+    }
+
+    debug!("Listing generations");
+
+    let nix_env_list_generations_out = Command::new("nix-env")
+        .arg("-p")
+        .arg(&profile_path)
+        .arg("--list-generations")
+        .output()
+        .await?;
+
+    if !nix_env_list_generations_out.status.success() {
+        good_panic!("Listing `nix-env` generations failed");
+    }
+
+    let generations_list = String::from_utf8(nix_env_list_generations_out.stdout)?;
+
+    let last_generation_line = generations_list
+        .lines()
+        .last()
+        .expect("Expected to find a generation in list");
+
+    let last_generation_id = last_generation_line
+        .split_whitespace()
+        .next()
+        .expect("Expected to get ID from generation entry");
+
+    debug!("Removing generation entry {}", last_generation_line);
+    warn!("Removing generation by ID {}", last_generation_id);
+
+    let nix_env_delete_generation_exit_status = Command::new("nix-env")
+        .arg("-p")
+        .arg(&profile_path)
+        .arg("--delete-generations")
+        .arg(last_generation_id)
+        .stdout(Stdio::null())
+        .stderr(Stdio::null())
+        .status()
+        .await?;
+
+    if !nix_env_delete_generation_exit_status.success() {
+        good_panic!("Failed to delete failed generation");
+    }
+
+    info!("Attempting re-activate last generation");
+
+    let re_activate_exit_status = Command::new(format!("{}/deploy-rs-activate", profile_path))
+        .env("PROFILE", &profile_path)
+        .current_dir(&profile_path)
+        .status()
+        .await?;
+
+    if !re_activate_exit_status.success() {
+        good_panic!("Failed to re-activate the last generation");
+    }
+
+    Ok(())
+}
+
+async fn deactivate_on_err<A, B: core::fmt::Debug>(profile_path: &str, r: Result<A, B>) -> A {
+    match r {
+        Ok(x) => x,
+        Err(err) => {
+            error!("Deactivating due to error: {:?}", err);
+            match deactivate(profile_path).await {
+                Ok(_) => (),
+                Err(err) => {
+                    error!("Error de-activating, uh-oh: {:?}", err);
+                }
+            };
+
+            std::process::exit(1);
+        }
+    }
+}
+
+pub async fn activation_confirmation(
+    profile_path: String,
+    temp_path: String,
+    max_time: u16,
+    closure: String,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let lock_hash = &closure[11 /* /nix/store/ */ ..];
+    let lock_path = format!("{}/activating-{}", temp_path, lock_hash);
+
+    if let Some(parent) = Path::new(&lock_path).parent() {
+        fs::create_dir_all(parent).await?;
+    }
+
+    fs::File::create(&lock_path).await?;
+
+    let mut inotify = Inotify::init()?;
+    inotify.add_watch(lock_path, inotify::WatchMask::DELETE)?;
+
+    match fork::daemon(false, false).map_err(|x| x.to_string())? {
+        fork::Fork::Child => {
+            std::thread::spawn(move || {
+                let mut rt = tokio::runtime::Runtime::new().unwrap();
+
+                rt.block_on(async move {
+                    info!("Waiting for confirmation event...");
+
+                    let mut buffer = [0; 32];
+                    let mut stream =
+                        deactivate_on_err(&profile_path, inotify.event_stream(&mut buffer)).await;
+
+                    deactivate_on_err(
+                        &profile_path,
+                        deactivate_on_err(
+                            &profile_path,
+                            deactivate_on_err(
+                                &profile_path,
+                                timeout(Duration::from_secs(max_time as u64), stream.next()).await,
+                            )
+                            .await
+                            .ok_or("Watcher ended prematurely"),
+                        )
+                        .await,
+                    )
+                    .await;
+                });
+            })
+            .join()
+            .unwrap();
+
+            info!("Confirmation successful!");
+
+            std::process::exit(0);
+        }
+        fork::Fork::Parent(_) => {
+            std::process::exit(0);
+        }
+    }
+}
+
 pub async fn activate(
     profile_path: String,
     closure: String,
     bootstrap_cmd: Option<String>,
     auto_rollback: bool,
+    temp_path: String,
+    max_time: u16,
 ) -> Result<(), Box<dyn std::error::Error>> {
     info!("Activating profile");
 
@@ -83,80 +245,17 @@ pub async fn activate(
 
     match activate_status {
         Ok(s) if s.success() => (),
-        _ if auto_rollback => {
-            error!("Failed to execute activation command");
-
-            let nix_env_rollback_exit_status = Command::new("nix-env")
-                .arg("-p")
-                .arg(&profile_path)
-                .arg("--rollback")
-                .stdout(Stdio::null())
-                .stderr(Stdio::null())
-                .status()
-                .await?;
-
-            if !nix_env_rollback_exit_status.success() {
-                good_panic!("`nix-env --rollback` failed");
-            }
-
-            debug!("Listing generations");
-
-            let nix_env_list_generations_out = Command::new("nix-env")
-                .arg("-p")
-                .arg(&profile_path)
-                .arg("--list-generations")
-                .output()
-                .await?;
-
-            if !nix_env_list_generations_out.status.success() {
-                good_panic!("Listing `nix-env` generations failed");
-            }
-
-            let generations_list = String::from_utf8(nix_env_list_generations_out.stdout)?;
-
-            let last_generation_line = generations_list
-                .lines()
-                .last()
-                .expect("Expected to find a generation in list");
-
-            let last_generation_id = last_generation_line
-                .split_whitespace()
-                .next()
-                .expect("Expected to get ID from generation entry");
-
-            debug!("Removing generation entry {}", last_generation_line);
-            warn!("Removing generation by ID {}", last_generation_id);
-
-            let nix_env_delete_generation_exit_status = Command::new("nix-env")
-                .arg("-p")
-                .arg(&profile_path)
-                .arg("--delete-generations")
-                .arg(last_generation_id)
-                .stdout(Stdio::null())
-                .stderr(Stdio::null())
-                .status()
-                .await?;
-
-            if !nix_env_delete_generation_exit_status.success() {
-                good_panic!("Failed to delete failed generation");
-            }
-
-            info!("Attempting re-activate last generation");
-
-            let re_activate_exit_status = Command::new(format!("{}/deploy-rs-activate", profile_path))
-                .env("PROFILE", &profile_path)
-                .current_dir(&profile_path)
-                .status()
-                .await?;
+        _ if auto_rollback => return Ok(deactivate(&profile_path).await?),
+        _ => (),
+    }
 
-            if !re_activate_exit_status.success() {
-                good_panic!("Failed to re-activate the last generation");
-            }
+    info!("Activation succeeded, now performing post-activation checks");
 
-            std::process::exit(1);
-        }
-        _ => {}
-    }
+    deactivate_on_err(
+        &profile_path,
+        activation_confirmation(profile_path.clone(), temp_path, max_time, closure).await,
+    )
+    .await;
 
     Ok(())
 }
@@ -176,6 +275,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         opts.closure,
         opts.bootstrap_cmd,
         opts.auto_rollback,
+        opts.temp_path,
+        opts.max_time,
     )
     .await?;
 
diff --git a/src/utils/data.rs b/src/utils/data.rs
index f72f9a7..351b9ae 100644
--- a/src/utils/data.rs
+++ b/src/utils/data.rs
@@ -44,6 +44,10 @@ pub struct ProfileSettings {
     pub bootstrap: Option<String>,
     #[serde(rename(deserialize = "profilePath"))]
     pub profile_path: Option<String>,
+    #[serde(rename(deserialize = "maxTime"))]
+    pub max_time: Option<u16>,
+    #[serde(rename(deserialize = "tempPath"))]
+    pub temp_path: Option<String>,
 }
 
 #[derive(Deserialize, Debug, Clone)]
diff --git a/src/utils/deploy.rs b/src/utils/deploy.rs
index 7301967..e3493ba 100644
--- a/src/utils/deploy.rs
+++ b/src/utils/deploy.rs
@@ -2,6 +2,7 @@
 //
 // SPDX-License-Identifier: MPL-2.0
 
+use std::borrow::Cow;
 use tokio::process::Command;
 
 fn build_activate_command(
@@ -11,9 +12,13 @@ fn build_activate_command(
     closure: &str,
     bootstrap_cmd: &Option<String>,
     auto_rollback: bool,
+    temp_path: &Cow<str>,
+    max_time: u16,
 ) -> String {
-    let mut self_activate_command =
-        format!("{} '{}' '{}'", activate_path_str, profile_path, closure);
+    let mut self_activate_command = format!(
+        "{} '{}' '{}' {} {}",
+        activate_path_str, profile_path, closure, temp_path, max_time
+    );
 
     if let Some(sudo_cmd) = &sudo {
         self_activate_command = format!("{} {}", sudo_cmd, self_activate_command);
@@ -41,6 +46,8 @@ fn test_activation_command_builder() {
     let closure = "/blah/etc";
     let bootstrap_cmd = None;
     let auto_rollback = true;
+    let temp_path = &"/tmp/deploy-rs".into();
+    let max_time = 30;
 
     assert_eq!(
         build_activate_command(
@@ -50,8 +57,10 @@ fn test_activation_command_builder() {
             closure,
             &bootstrap_cmd,
             auto_rollback,
+            temp_path,
+            max_time
         ),
-        "sudo -u test /blah/bin/activate '/blah/profiles/test' '/blah/etc' --auto-rollback"
+        "sudo -u test /blah/bin/activate '/blah/profiles/test' '/blah/etc' /tmp/deploy-rs 30 --auto-rollback"
             .to_string(),
     );
 }
@@ -67,6 +76,13 @@ pub async fn deploy_profile(
 
     let activate_path_str = super::deploy_path_to_activate_path_str(&deploy_defs.current_exe)?;
 
+    let temp_path: Cow<str> = match &deploy_data.profile.profile_settings.temp_path {
+        Some(x) => x.into(),
+        None => "/tmp/deploy-rs".into(),
+    };
+
+    let max_time = deploy_data.profile.profile_settings.max_time.unwrap_or(30);
+
     let self_activate_command = build_activate_command(
         activate_path_str,
         &deploy_defs.sudo,
@@ -74,6 +90,8 @@ pub async fn deploy_profile(
         &deploy_data.profile.profile_settings.path,
         &deploy_data.profile.profile_settings.bootstrap,
         deploy_data.merged_settings.auto_rollback,
+        &temp_path,
+        max_time,
     );
 
     let hostname = match deploy_data.cmd_overrides.hostname {
@@ -94,5 +112,33 @@ pub async fn deploy_profile(
         good_panic!("Activation over SSH failed");
     }
 
+    info!("Success, attempting to connect to the node to confirm deployment");
+
+    let mut c = Command::new("ssh");
+    let mut ssh_confirm_command = c.arg(format!("ssh://{}@{}", deploy_defs.ssh_user, hostname));
+
+    for ssh_opt in &deploy_data.merged_settings.ssh_opts {
+        ssh_confirm_command = ssh_confirm_command.arg(ssh_opt);
+    }
+
+    let lock_hash = &deploy_data.profile.profile_settings.path[11 /* /nix/store/ */ ..];
+    let lock_path = format!("{}/activating-{}", temp_path, lock_hash);
+
+    let mut confirm_command = format!("rm {}", lock_path);
+    if let Some(sudo_cmd) = &deploy_defs.sudo {
+        confirm_command = format!("{} {}", sudo_cmd, confirm_command);
+    }
+
+    let ssh_exit_status = ssh_confirm_command.arg(confirm_command).status().await?;
+
+    if !ssh_exit_status.success() {
+        good_panic!(
+            "Failed to confirm deployment, the node will roll back in <{} seconds",
+            max_time
+        );
+    }
+
+    info!("Deployment confirmed.");
+
     Ok(())
 }