Skip to content

Commit a028d73

Browse files
XuanwoXuanwo
committed
feat: Refactor metrics and hide under feature layers-metrics
Signed-off-by: Xuanwo <[email protected]>
1 parent 4e6d204 commit a028d73

File tree

10 files changed

+309
-41
lines changed

10 files changed

+309
-41
lines changed

‎Cargo.toml

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,6 @@ all-features = true
1717
[features]
1818
# Enable compress support so that users can decompress while reading.
1919
compress = ["async-compression"]
20-
# Enable retry layer support.
21-
retry = ["backon"]
2220
# Enable rustls support.
2321
rustls = ["isahc/rustls-tls", "isahc/rustls-tls-native-certs"]
2422
# Enable serde support.
@@ -27,6 +25,12 @@ serde = ["time/serde"]
2725
services-hdfs = ["hdrs"]
2826
# Enable services http support
2927
services-http = ["radix_trie"]
28+
# Enable layers metrics support
29+
layers-metrics = ["metrics"]
30+
# Enable layers retry support.
31+
layers-retry = ["backon"]
32+
# DEPRACATED: use layers-retry instead
33+
retry = ["layers-retry"]
3034

3135
[lib]
3236
bench = false
@@ -63,7 +67,7 @@ http = "0.2"
6367
isahc = { package = "isahc-opendal-workaround", version = "2.0.0-opendal.1" }
6468
log = "0.4"
6569
md5 = "0.7"
66-
metrics = "0.20"
70+
metrics = { version = "0.20", optional = true }
6771
minitrace = "0.4"
6872
once_cell = "1.10"
6973
parking_lot = "0.12"

‎src/layers/metrics.rs

Lines changed: 243 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,243 @@
1+
// Copyright 2022 Datafuse Labs.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
use std::fmt::Debug;
16+
use std::io::Result;
17+
use std::sync::Arc;
18+
use std::time::Instant;
19+
20+
use async_trait::async_trait;
21+
use metrics::histogram;
22+
use metrics::increment_counter;
23+
24+
use crate::ops::*;
25+
use crate::Accessor;
26+
use crate::AccessorMetadata;
27+
use crate::BytesReader;
28+
use crate::BytesWriter;
29+
use crate::DirStreamer;
30+
use crate::Layer;
31+
use crate::ObjectMetadata;
32+
33+
static METRIC_REQUESTS_TOTAL: &str = "opendal_requests_total";
34+
static METRIC_REQUESTS_DURATION_SECONDS: &str = "opendal_requests_duration_seconds";
35+
static LABEL_SERVICE: &str = "service";
36+
static LABEL_OPERATION: &str = "operation";
37+
38+
/// MetricsLayer will add metrics for OpenDAL.
39+
///
40+
/// # Metrics
41+
///
42+
/// - `opendal_requests_total`: Total requests numbers
43+
/// - `opendal_requests_duration_seconds`: Request duration seconds.
44+
/// - NOTE: this metric tracks the duration of the OpenDAL's function call, not the underlying http request duration
45+
///
46+
/// # Labels
47+
///
48+
/// Most metrics will carry the following labels
49+
///
50+
/// - `service`: Service name from [`Scheme`][crate::Scheme]
51+
/// - `operation`: Operation name from [`Operation`]
52+
///
53+
/// # Examples
54+
///
55+
/// ```
56+
/// use anyhow::Result;
57+
/// use opendal::layers::MetricsLayer;
58+
/// use opendal::Operator;
59+
/// use opendal::Scheme;
60+
///
61+
/// let _ = Operator::from_env(Scheme::Fs)
62+
/// .expect("must init")
63+
/// .layer(MetricsLayer);
64+
/// ```
65+
#[derive(Debug, Copy, Clone)]
66+
pub struct MetricsLayer;
67+
68+
impl Layer for MetricsLayer {
69+
fn layer(&self, inner: Arc<dyn Accessor>) -> Arc<dyn Accessor> {
70+
let meta = inner.metadata();
71+
72+
Arc::new(MetricsAccessor { meta, inner })
73+
}
74+
}
75+
76+
#[derive(Debug)]
77+
struct MetricsAccessor {
78+
meta: AccessorMetadata,
79+
inner: Arc<dyn Accessor>,
80+
}
81+
82+
#[async_trait]
83+
impl Accessor for MetricsAccessor {
84+
fn metadata(&self) -> AccessorMetadata {
85+
increment_counter!(
86+
METRIC_REQUESTS_TOTAL,
87+
LABEL_SERVICE => self.meta.scheme().into_static(),
88+
LABEL_OPERATION => Operation::Metadata.into_static(),
89+
);
90+
91+
let start = Instant::now();
92+
let result = self.inner.metadata();
93+
let dur = start.elapsed().as_secs_f64();
94+
95+
histogram!(
96+
METRIC_REQUESTS_DURATION_SECONDS, dur,
97+
LABEL_SERVICE => self.meta.scheme().into_static(),
98+
LABEL_OPERATION => Operation::Metadata.into_static(),
99+
);
100+
101+
result
102+
}
103+
104+
async fn create(&self, args: &OpCreate) -> Result<()> {
105+
increment_counter!(
106+
METRIC_REQUESTS_TOTAL,
107+
LABEL_SERVICE => self.meta.scheme().into_static(),
108+
LABEL_OPERATION => Operation::Create.into_static(),
109+
);
110+
111+
let start = Instant::now();
112+
let result = self.inner.create(args).await;
113+
let dur = start.elapsed().as_secs_f64();
114+
115+
histogram!(
116+
METRIC_REQUESTS_DURATION_SECONDS, dur,
117+
LABEL_SERVICE => self.meta.scheme().into_static(),
118+
LABEL_OPERATION => Operation::Create.into_static(),
119+
);
120+
121+
result
122+
}
123+
124+
async fn read(&self, args: &OpRead) -> Result<BytesReader> {
125+
increment_counter!(
126+
METRIC_REQUESTS_TOTAL,
127+
LABEL_SERVICE => self.meta.scheme().into_static(),
128+
LABEL_OPERATION => Operation::Read.into_static(),
129+
);
130+
131+
let start = Instant::now();
132+
let result = self.inner.read(args).await;
133+
let dur = start.elapsed().as_secs_f64();
134+
135+
histogram!(
136+
METRIC_REQUESTS_DURATION_SECONDS, dur,
137+
LABEL_SERVICE => self.meta.scheme().into_static(),
138+
LABEL_OPERATION => Operation::Read.into_static(),
139+
);
140+
141+
result
142+
}
143+
144+
async fn write(&self, args: &OpWrite) -> Result<BytesWriter> {
145+
increment_counter!(
146+
METRIC_REQUESTS_TOTAL,
147+
LABEL_SERVICE => self.meta.scheme().into_static(),
148+
LABEL_OPERATION => Operation::Write.into_static(),
149+
);
150+
151+
let start = Instant::now();
152+
let result = self.inner.write(args).await;
153+
let dur = start.elapsed().as_secs_f64();
154+
155+
histogram!(
156+
METRIC_REQUESTS_DURATION_SECONDS, dur,
157+
LABEL_SERVICE => self.meta.scheme().into_static(),
158+
LABEL_OPERATION => Operation::Write.into_static(),
159+
);
160+
161+
result
162+
}
163+
164+
async fn stat(&self, args: &OpStat) -> Result<ObjectMetadata> {
165+
increment_counter!(
166+
METRIC_REQUESTS_TOTAL,
167+
LABEL_SERVICE => self.meta.scheme().into_static(),
168+
LABEL_OPERATION => Operation::Stat.into_static(),
169+
);
170+
171+
let start = Instant::now();
172+
let result = self.inner.stat(args).await;
173+
let dur = start.elapsed().as_secs_f64();
174+
175+
histogram!(
176+
METRIC_REQUESTS_DURATION_SECONDS, dur,
177+
LABEL_SERVICE => self.meta.scheme().into_static(),
178+
LABEL_OPERATION => Operation::Stat.into_static(),
179+
);
180+
181+
result
182+
}
183+
184+
async fn delete(&self, args: &OpDelete) -> Result<()> {
185+
increment_counter!(
186+
METRIC_REQUESTS_TOTAL,
187+
LABEL_SERVICE => self.meta.scheme().into_static(),
188+
LABEL_OPERATION => Operation::Delete.into_static(),
189+
);
190+
191+
let start = Instant::now();
192+
let result = self.inner.delete(args).await;
193+
let dur = start.elapsed().as_secs_f64();
194+
195+
histogram!(
196+
METRIC_REQUESTS_DURATION_SECONDS, dur,
197+
LABEL_SERVICE => self.meta.scheme().into_static(),
198+
LABEL_OPERATION => Operation::Delete.into_static(),
199+
);
200+
201+
result
202+
}
203+
204+
async fn list(&self, args: &OpList) -> Result<DirStreamer> {
205+
increment_counter!(
206+
METRIC_REQUESTS_TOTAL,
207+
LABEL_SERVICE => self.meta.scheme().into_static(),
208+
LABEL_OPERATION => Operation::List.into_static(),
209+
);
210+
211+
let start = Instant::now();
212+
let result = self.inner.list(args).await;
213+
let dur = start.elapsed().as_secs_f64();
214+
215+
histogram!(
216+
METRIC_REQUESTS_DURATION_SECONDS, dur,
217+
LABEL_SERVICE => self.meta.scheme().into_static(),
218+
LABEL_OPERATION => Operation::List.into_static(),
219+
);
220+
221+
result
222+
}
223+
224+
fn presign(&self, args: &OpPresign) -> Result<PresignedRequest> {
225+
increment_counter!(
226+
METRIC_REQUESTS_TOTAL,
227+
LABEL_SERVICE => self.meta.scheme().into_static(),
228+
LABEL_OPERATION => Operation::Presign.into_static(),
229+
);
230+
231+
let start = Instant::now();
232+
let result = self.inner.presign(args);
233+
let dur = start.elapsed().as_secs_f64();
234+
235+
histogram!(
236+
METRIC_REQUESTS_DURATION_SECONDS, dur,
237+
LABEL_SERVICE => self.meta.scheme().into_static(),
238+
LABEL_OPERATION => Operation::Presign.into_static(),
239+
);
240+
241+
result
242+
}
243+
}

‎src/layers/mod.rs

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,19 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15+
//! Providing Layer trait and its implementations.
16+
//!
17+
//! # Available Layers
18+
//!
19+
//! - [`MetricsLayer`]: add metrics for OpenDAL, requires feature `layers-metrics`
20+
1521
mod layer;
1622
pub use layer::Layer;
1723

18-
#[cfg(feature = "retry")]
24+
#[cfg(feature = "layers-metrics")]
25+
mod metrics;
26+
#[cfg(feature = "layers-metrics")]
27+
pub use self::metrics::MetricsLayer;
28+
29+
#[cfg(feature = "layers-retry")]
1930
mod retry;

‎src/lib.rs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -107,9 +107,6 @@ pub use io::BytesStream;
107107
pub use io::BytesWrite;
108108
pub use io::BytesWriter;
109109

110-
mod layers;
111-
pub use layers::Layer;
112-
113110
mod operator;
114111
pub use operator::BatchOperator;
115112
pub use operator::Operator;
@@ -127,6 +124,8 @@ pub use scheme::Scheme;
127124

128125
// Public modules, they will be accessed via `opendal::io_util::Xxxx`
129126
pub mod io_util;
127+
pub mod layers;
128+
pub use layers::Layer;
130129
pub mod ops;
131130
pub mod services;
132131

‎src/operator.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,7 @@ impl Operator {
235235
/// # Ok(())
236236
/// # }
237237
/// ```
238-
#[cfg(feature = "retry")]
238+
#[cfg(feature = "layers-retry")]
239239
#[must_use]
240240
pub fn with_backoff(
241241
self,

‎src/ops.rs

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,13 @@ pub enum Operation {
5353
Presign,
5454
}
5555

56+
impl Operation {
57+
/// Convert self into static str.
58+
pub fn into_static(self) -> &'static str {
59+
self.into()
60+
}
61+
}
62+
5663
impl Default for Operation {
5764
fn default() -> Self {
5865
Operation::Metadata
@@ -74,6 +81,21 @@ impl Display for Operation {
7481
}
7582
}
7683

84+
impl From<Operation> for &'static str {
85+
fn from(v: Operation) -> &'static str {
86+
match v {
87+
Operation::Metadata => "metadata",
88+
Operation::Create => "create",
89+
Operation::Read => "read",
90+
Operation::Write => "write",
91+
Operation::Stat => "stat",
92+
Operation::Delete => "delete",
93+
Operation::List => "list",
94+
Operation::Presign => "presign",
95+
}
96+
}
97+
}
98+
7799
/// Args for `create` operation.
78100
///
79101
/// The path must be normalized.

‎src/scheme.rs

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,13 @@ pub enum Scheme {
4040
S3,
4141
}
4242

43+
impl Scheme {
44+
/// Convert self into static str.
45+
pub fn into_static(self) -> &'static str {
46+
self.into()
47+
}
48+
}
49+
4350
impl Default for Scheme {
4451
fn default() -> Self {
4552
Self::Memory
@@ -82,3 +89,18 @@ impl FromStr for Scheme {
8289
}
8390
}
8491
}
92+
93+
impl From<Scheme> for &'static str {
94+
fn from(v: Scheme) -> Self {
95+
match v {
96+
Scheme::Azblob => "azblob",
97+
Scheme::Fs => "fs",
98+
#[cfg(feature = "services-hdfs")]
99+
Scheme::Hdfs => "hdfs",
100+
#[cfg(feature = "services-http")]
101+
Scheme::Http => "http",
102+
Scheme::Memory => "memory",
103+
Scheme::S3 => "s3",
104+
}
105+
}
106+
}

0 commit comments

Comments
 (0)