diff --git a/.gitignore b/.gitignore
index 990f53978..f12193193 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,4 @@ target
 master.zip
 emsdk-*
 .idea/
+.vscode/
diff --git a/Cargo.toml b/Cargo.toml
index f1af4019e..e64827a08 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,4 +1,4 @@
 [workspace]
-members = ["crates/*", "xtask"]
+members = ["benchmarks", "crates/*", "xtask"]
 # xtask should only be compiled when invoked explicitly
-default-members = ["crates/*"]
+default-members = ["benchmarks", "crates/*"]
diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml
new file mode 100644
index 000000000..299b58e14
--- /dev/null
+++ b/benchmarks/Cargo.toml
@@ -0,0 +1,25 @@
+[package]
+name = "benchmarks"
+description = "Matrix SDK benchmarks"
+edition = "2021"
+license = "Apache-2.0"
+rust-version = "1.56"
+version = "1.0.0"
+publish = false
+
+[dependencies]
+criterion = { version = "0.3.4", features = ["async", "async_tokio", "html_reports"] }
+matrix-sdk-crypto = { path = "../crates/matrix-sdk-crypto" }
+matrix-sdk-sled = { path = "../crates/matrix-sdk-sled", default-features = false, features = ["crypto-store"] }
+matrix-sdk-test = { path = "../crates/matrix-sdk-test" }
+ruma = { git = "https://github.com/ruma/ruma", rev = "deea762b8" }
+serde_json = "1.0.79"
+tempfile = "3.2.0"
+tokio = { version = "1.7.1", default-features = false, features = ["rt-multi-thread"] }
+
+[target.'cfg(target_os = "linux")'.dependencies]
+pprof = { version = "0.6.2", features = ["flamegraph", "criterion"] }
+
+[[bench]]
+name = "crypto_bench"
+harness = false
diff --git a/crates/matrix-sdk-crypto/benches/README.md b/benchmarks/README.md
similarity index 100%
rename from crates/matrix-sdk-crypto/benches/README.md
rename to benchmarks/README.md
diff --git a/crates/matrix-sdk-crypto/benches/crypto_bench.rs b/benchmarks/benches/crypto_bench.rs
similarity index 97%
rename from crates/matrix-sdk-crypto/benches/crypto_bench.rs
rename to benchmarks/benches/crypto_bench.rs
index 066f32ac4..879756efe 100644
--- a/crates/matrix-sdk-crypto/benches/crypto_bench.rs
+++ b/benchmarks/benches/crypto_bench.rs
@@ -26,7 +26,7 @@ fn alice_device_id() -> &'static DeviceId {
 }
 
 fn keys_query_response() -> get_keys::v3::Response {
-    let data = include_bytes!("./keys_query.json");
+    let data = include_bytes!("crypto_bench/keys_query.json");
     let data: Value = serde_json::from_slice(data).unwrap();
     let data = response_from_file(&data);
     get_keys::v3::Response::try_from_http_response(data)
@@ -34,7 +34,7 @@ fn keys_query_response() -> get_keys::v3::Response {
 }
 
 fn keys_claim_response() -> claim_keys::v3::Response {
-    let data = include_bytes!("./keys_claim.json");
+    let data = include_bytes!("crypto_bench/keys_claim.json");
     let data: Value = serde_json::from_slice(data).unwrap();
     let data = response_from_file(&data);
     claim_keys::v3::Response::try_from_http_response(data)
@@ -42,7 +42,7 @@ fn keys_claim_response() -> claim_keys::v3::Response {
 }
 
 fn huge_keys_query_response() -> get_keys::v3::Response {
-    let data = include_bytes!("./keys_query_2000_members.json");
+    let data = include_bytes!("crypto_bench/keys_query_2000_members.json");
     let data: Value = serde_json::from_slice(data).unwrap();
     let data = response_from_file(&data);
     get_keys::v3::Response::try_from_http_response(data)
diff --git a/crates/matrix-sdk-crypto/benches/keys_claim.json b/benchmarks/benches/crypto_bench/keys_claim.json
similarity index 100%
rename from crates/matrix-sdk-crypto/benches/keys_claim.json
rename to benchmarks/benches/crypto_bench/keys_claim.json
diff --git a/crates/matrix-sdk-crypto/benches/keys_query.json b/benchmarks/benches/crypto_bench/keys_query.json
similarity index 100%
rename from crates/matrix-sdk-crypto/benches/keys_query.json
rename to benchmarks/benches/crypto_bench/keys_query.json
diff --git a/crates/matrix-sdk-crypto/benches/keys_query_2000_members.json b/benchmarks/benches/crypto_bench/keys_query_2000_members.json
similarity index 100%
rename from crates/matrix-sdk-crypto/benches/keys_query_2000_members.json
rename to benchmarks/benches/crypto_bench/keys_query_2000_members.json
diff --git a/crates/matrix-sdk-crypto/Cargo.toml b/crates/matrix-sdk-crypto/Cargo.toml
index a5bf126d3..13bd0901f 100644
--- a/crates/matrix-sdk-crypto/Cargo.toml
+++ b/crates/matrix-sdk-crypto/Cargo.toml
@@ -72,26 +72,6 @@ http = "0.2.4"
 indoc = "1.0.3"
 matches = "0.1.8"
 matrix-sdk-test = { version = "0.4.0", path = "../matrix-sdk-test" }
-
-# This is needed for the benchmarks
-matrix-sdk-sled = { version = "0.1.0", path = "../matrix-sdk-sled", default-features = false, features = ["crypto-store"] }
-[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
-criterion = { version = "0.3.4", features = [
-    "async",
-    "async_tokio",
-    "html_reports",
-] }
 proptest = "1.0.0"
-tempfile = "3.2.0"
-tokio = { version = "1.7.1", default-features = false, features = [
-    "rt-multi-thread",
-    "macros",
-] }
-
-[target.'cfg(target_os = "linux")'.dev-dependencies]
-pprof = { version = "0.6.2", features = ["flamegraph", "criterion"] }
-
-
-[[bench]]
-name = "crypto_bench"
-harness = false
+# required for async_test macro
+tokio = { version = "1.7.1", default-features = false, features = ["macros", "rt-multi-thread"] }
diff --git a/crates/matrix-sdk-crypto/src/session_manager/group_sessions.rs b/crates/matrix-sdk-crypto/src/session_manager/group_sessions.rs
index 4b49624ad..ab70b59e6 100644
--- a/crates/matrix-sdk-crypto/src/session_manager/group_sessions.rs
+++ b/crates/matrix-sdk-crypto/src/session_manager/group_sessions.rs
@@ -610,7 +610,7 @@ mod test {
     }
 
     fn keys_query_response() -> get_keys::v3::Response {
-        let data = include_bytes!("../../benches/keys_query.json");
+        let data = include_bytes!("../../../../benchmarks/benches/crypto_bench/keys_query.json");
         let data: Value = serde_json::from_slice(data).unwrap();
         let data = response_from_file(&data);
         get_keys::v3::Response::try_from_http_response(data)
@@ -618,7 +618,7 @@ mod test {
     }
 
     fn keys_claim_response() -> claim_keys::v3::Response {
-        let data = include_bytes!("../../benches/keys_claim.json");
+        let data = include_bytes!("../../../../benchmarks/benches/crypto_bench/keys_claim.json");
         let data: Value = serde_json::from_slice(data).unwrap();
         let data = response_from_file(&data);
         claim_keys::v3::Response::try_from_http_response(data)