diff --git a/src/error.rs b/src/error.rs
index 4e251ec..3ca22f4 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -18,6 +18,14 @@ impl Error {
     pub(crate) fn is_breaker(&self) -> bool {
         matches!(self.kind, ErrorKind::Breaker)
     }
+
+    pub(crate) fn is_not_found(&self) -> bool {
+        matches!(self.kind, ErrorKind::Status(_, StatusCode::NOT_FOUND))
+    }
+
+    pub(crate) fn is_bad_request(&self) -> bool {
+        matches!(self.kind, ErrorKind::Status(_, StatusCode::BAD_REQUEST))
+    }
 }
 
 impl std::fmt::Debug for Error {
diff --git a/src/jobs/contact.rs b/src/jobs/contact.rs
index 05ccb18..288941a 100644
--- a/src/jobs/contact.rs
+++ b/src/jobs/contact.rs
@@ -40,10 +40,18 @@ impl QueryContact {
             return Ok(());
         }
 
-        let contact = state
+        let contact = match state
             .requests
             .fetch::<AcceptedActors>(self.contact_id.as_str())
-            .await?;
+            .await
+        {
+            Ok(contact) => contact,
+            Err(e) if e.is_breaker() => {
+                tracing::debug!("Not retrying due to failed breaker");
+                return Ok(());
+            }
+            Err(e) => return Err(e),
+        };
 
         let (username, display_name, url, avatar) =
             to_contact(contact).ok_or(ErrorKind::Extract("contact"))?;
diff --git a/src/jobs/deliver.rs b/src/jobs/deliver.rs
index 6916b98..9c01d8d 100644
--- a/src/jobs/deliver.rs
+++ b/src/jobs/deliver.rs
@@ -40,6 +40,10 @@ impl Deliver {
                 tracing::debug!("Not trying due to failed breaker");
                 return Ok(());
             }
+            if e.is_bad_request() {
+                tracing::debug!("Server didn't understand the activity");
+                return Ok(());
+            }
             return Err(e);
         }
         Ok(())
diff --git a/src/jobs/instance.rs b/src/jobs/instance.rs
index 040897f..6c9fa14 100644
--- a/src/jobs/instance.rs
+++ b/src/jobs/instance.rs
@@ -57,6 +57,10 @@ impl QueryInstance {
                 tracing::debug!("Not retrying due to failed breaker");
                 return Ok(());
             }
+            Err(e) if e.is_not_found() => {
+                tracing::debug!("Server doesn't implement instance endpoint");
+                return Ok(());
+            }
             Err(e) => return Err(e),
         };
 
diff --git a/src/requests.rs b/src/requests.rs
index f4da6bc..456facd 100644
--- a/src/requests.rs
+++ b/src/requests.rs
@@ -1,9 +1,6 @@
 use crate::error::{Error, ErrorKind};
 use activitystreams::iri_string::types::IriString;
-use actix_web::{
-    http::{header::Date, StatusCode},
-    web::Bytes,
-};
+use actix_web::{http::header::Date, web::Bytes};
 use awc::Client;
 use dashmap::DashMap;
 use http_signature_normalization_actix::prelude::*;
@@ -394,19 +391,16 @@ impl Requests {
         self.reset_err();
 
         if !res.status().is_success() {
-            // Bad Request means the server didn't understand our activity - that's fine
-            if res.status() != StatusCode::BAD_REQUEST {
-                if let Ok(bytes) = res.body().await {
-                    if let Ok(s) = String::from_utf8(bytes.as_ref().to_vec()) {
-                        if !s.is_empty() {
-                            tracing::warn!("Response from {}, {}", inbox.as_str(), s);
-                        }
+            if let Ok(bytes) = res.body().await {
+                if let Ok(s) = String::from_utf8(bytes.as_ref().to_vec()) {
+                    if !s.is_empty() {
+                        tracing::warn!("Response from {}, {}", inbox.as_str(), s);
                     }
                 }
-
-                self.breakers.fail(&inbox);
-                return Err(ErrorKind::Status(inbox.to_string(), res.status()).into());
             }
+
+            self.breakers.fail(&inbox);
+            return Err(ErrorKind::Status(inbox.to_string(), res.status()).into());
         }
 
         self.breakers.succeed(&inbox);