cli: new worker info command

This commit is contained in:
Alex Auvolat 2022-12-13 12:24:30 +01:00
parent a51e8d94c6
commit 9d82196945
No known key found for this signature in database
GPG key ID: 0E496D15096376BE
6 changed files with 86 additions and 11 deletions

View file

@ -67,14 +67,17 @@ impl Worker for RepairWorker {
idx_bytes
};
WorkerStatus {
progress: Some("Phase 1".into()),
freeform: vec![format!("Now at: {}", hex::encode(idx_bytes))],
progress: Some("0.00%".into()),
freeform: vec![format!(
"Currently in phase 1, iterator position: {}",
hex::encode(idx_bytes)
)],
..Default::default()
}
}
Some(bi) => WorkerStatus {
progress: Some(format!("{:.2}%", bi.progress() * 100.)),
freeform: vec!["Phase 2".into()],
freeform: vec!["Currently in phase 2".into()],
..Default::default()
},
}
@ -291,11 +294,11 @@ impl Worker for ScrubWorker {
}
ScrubWorkerState::Paused(bsi, rt) => {
s.progress = Some(format!("{:.2}%", bsi.progress() * 100.));
s.freeform = vec![format!("Paused, resumes at {}", msec_to_rfc3339(*rt))];
s.freeform = vec![format!("Scrub paused, resumes at {}", msec_to_rfc3339(*rt))];
}
ScrubWorkerState::Finished => {
s.freeform = vec![format!(
"Completed {}",
"Last scrub completed at {}",
msec_to_rfc3339(self.persisted.time_last_complete_scrub)
)];
}

View file

@ -257,7 +257,7 @@ impl BlockResyncManager {
if let Err(e) = &res {
manager.metrics.resync_error_counter.add(1);
warn!("Error when resyncing {:?}: {}", hash, e);
error!("Error when resyncing {:?}: {}", hash, e);
let err_counter = match self.errors.get(hash.as_slice())? {
Some(ec) => ErrorCounter::decode(&ec).add1(now + 1),
@ -482,7 +482,7 @@ impl Worker for ResyncWorker {
if self.index >= persisted.n_workers {
return WorkerStatus {
freeform: vec!["(unused)".into()],
freeform: vec!["This worker is currently disabled".into()],
..Default::default()
};
}

View file

@ -54,6 +54,7 @@ pub enum AdminRpc {
HashMap<usize, garage_util::background::WorkerInfo>,
WorkerListOpt,
),
WorkerInfo(usize, garage_util::background::WorkerInfo),
}
impl Rpc for AdminRpc {
@ -880,6 +881,16 @@ impl AdminRpcHandler {
let workers = self.garage.background.get_worker_info();
Ok(AdminRpc::WorkerList(workers, opt))
}
WorkerCmd::Info { tid } => {
let info = self
.garage
.background
.get_worker_info()
.get(&tid)
.ok_or_bad_request(format!("No worker with TID {}", tid))?
.clone();
Ok(AdminRpc::WorkerInfo(tid, info))
}
WorkerCmd::Set { opt } => match opt {
WorkerSetCmd::ScrubTranquility { tranquility } => {
let scrub_command = ScrubWorkerCommand::SetTranquility(tranquility);

View file

@ -186,7 +186,10 @@ pub async fn cmd_admin(
print_key_info(&key, &rb);
}
AdminRpc::WorkerList(wi, wlo) => {
print_worker_info(wi, wlo);
print_worker_list(wi, wlo);
}
AdminRpc::WorkerInfo(tid, wi) => {
print_worker_info(tid, wi);
}
r => {
error!("Unexpected response: {:?}", r);

View file

@ -516,6 +516,9 @@ pub enum WorkerCmd {
#[structopt(flatten)]
opt: WorkerListOpt,
},
/// Get detailed information about a worker
#[structopt(name = "info", version = garage_version())]
Info { tid: usize },
/// Set worker parameter
#[structopt(name = "set", version = garage_version())]
Set {

View file

@ -241,7 +241,7 @@ pub fn find_matching_node(
}
}
pub fn print_worker_info(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
pub fn print_worker_list(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
let mut wi = wi.into_iter().collect::<Vec<_>>();
wi.sort_by_key(|(tid, info)| {
(
@ -284,13 +284,13 @@ pub fn print_worker_info(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
.tranquility
.as_ref()
.map(ToString::to_string)
.unwrap_or("-".into()),
.unwrap_or_else(|| "-".into()),
info.status.progress.as_deref().unwrap_or("-"),
info.status
.queue_length
.as_ref()
.map(ToString::to_string)
.unwrap_or("-".into()),
.unwrap_or_else(|| "-".into()),
total_err,
consec_err,
err_ago,
@ -298,3 +298,58 @@ pub fn print_worker_info(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
}
format_table(table);
}
pub fn print_worker_info(tid: usize, info: WorkerInfo) {
let mut table = vec![];
table.push(format!("Task id:\t{}", tid));
table.push(format!("Worker name:\t{}", info.name));
match info.state {
WorkerState::Throttled(t) => {
table.push(format!(
"Worker state:\tBusy (throttled, paused for {:.3}s)",
t
));
}
s => {
table.push(format!("Worker state:\t{}", s));
}
};
if let Some(tql) = info.status.tranquility {
table.push(format!("Tranquility:\t{}", tql));
}
table.push("".into());
table.push(format!("Total errors:\t{}", info.errors));
table.push(format!("Consecutive errs:\t{}", info.consecutive_errors));
if let Some((s, t)) = info.last_error {
table.push(format!("Last error:\t{}", s));
let tf = timeago::Formatter::new();
table.push(format!(
"Last error time:\t{}",
tf.convert(Duration::from_millis(now_msec() - t))
));
}
table.push("".into());
if let Some(p) = info.status.progress {
table.push(format!("Progress:\t{}", p));
}
if let Some(ql) = info.status.queue_length {
table.push(format!("Queue length:\t{}", ql));
}
if let Some(pe) = info.status.persistent_errors {
table.push(format!("Persistent errors:\t{}", pe));
}
for (i, s) in info.status.freeform.iter().enumerate() {
if i == 0 {
if table.last() != Some(&"".into()) {
table.push("".into());
}
table.push(format!("Message:\t{}", s));
} else {
table.push(format!("\t{}", s));
}
}
format_table(table);
}