Skip to content

Commit cd3fb2c

Browse files
XuanwoXuanwo
committed
feat: Add http_header to handle HTTP header parse
Signed-off-by: Xuanwo <[email protected]>
1 parent ee3410b commit cd3fb2c

File tree

5 files changed

+107
-143
lines changed

5 files changed

+107
-143
lines changed

‎src/io_util/http_header.rs

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
// Copyright 2022 Datafuse Labs.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
use anyhow::anyhow;
16+
use anyhow::Result;
17+
use http::header::HeaderName;
18+
use http::header::CONTENT_LENGTH;
19+
use http::header::LAST_MODIFIED;
20+
use http::HeaderMap;
21+
use time::format_description::well_known::Rfc2822;
22+
use time::OffsetDateTime;
23+
24+
/// Parse content length from header map.
25+
pub fn parse_content_length(headers: &HeaderMap) -> Result<Option<u64>> {
26+
match headers.get(CONTENT_LENGTH) {
27+
None => Ok(None),
28+
Some(v) => Ok(Some(
29+
v.to_str()
30+
.map_err(|e| anyhow!("parse {} header: {:?}", http::header::CONTENT_LENGTH, e))?
31+
.parse::<u64>()
32+
.map_err(|e| anyhow!("parse {} header: {:?}", http::header::CONTENT_LENGTH, e))?,
33+
)),
34+
}
35+
}
36+
37+
/// Parse content md5 from header map.
38+
pub fn parse_content_md5(headers: &HeaderMap) -> Result<Option<&str>> {
39+
match headers.get(HeaderName::from_static("content-md5")) {
40+
None => Ok(None),
41+
Some(v) => {
42+
Ok(Some(v.to_str().map_err(|e| {
43+
anyhow!("parse content-md5 header: {:?}", e)
44+
})?))
45+
}
46+
}
47+
}
48+
49+
/// Parse last modified from header map.
50+
pub fn parse_last_modified(headers: &HeaderMap) -> Result<Option<OffsetDateTime>> {
51+
match headers.get(LAST_MODIFIED) {
52+
None => Ok(None),
53+
Some(v) => {
54+
let v = v
55+
.to_str()
56+
.map_err(|e| anyhow!("parse {} header: {:?}", http::header::LAST_MODIFIED, e))?;
57+
let t = OffsetDateTime::parse(v, &Rfc2822)
58+
.map_err(|e| anyhow!("parse {} header: {:?}", http::header::LAST_MODIFIED, e))?;
59+
60+
Ok(Some(t))
61+
}
62+
}
63+
}

‎src/io_util/mod.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,11 @@ pub(crate) use http_body::HttpBodyWriter;
4343
mod http_client;
4444
pub(crate) use http_client::HttpClient;
4545

46+
mod http_header;
47+
pub(crate) use http_header::parse_content_length;
48+
pub(crate) use http_header::parse_content_md5;
49+
pub(crate) use http_header::parse_last_modified;
50+
4651
mod seekable_reader;
4752
pub use seekable_reader::seekable_read;
4853
pub use seekable_reader::SeekableReader;

‎src/services/azblob/backend.rs

Lines changed: 13 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ use std::io::Error;
2121
use std::io::ErrorKind;
2222
use std::io::Result;
2323
use std::mem;
24-
use std::str::FromStr;
2524
use std::sync::Arc;
2625

2726
use anyhow::anyhow;
@@ -39,15 +38,16 @@ use log::info;
3938
use metrics::increment_counter;
4039
use minitrace::trace;
4140
use reqsign::services::azure::storage::Signer;
42-
use time::format_description::well_known::Rfc2822;
43-
use time::OffsetDateTime;
4441

4542
use super::dir_stream::DirStream;
4643
use crate::accessor::AccessorMetadata;
4744
use crate::error::other;
4845
use crate::error::BackendError;
4946
use crate::error::ObjectError;
5047
use crate::io_util::new_http_channel;
48+
use crate::io_util::parse_content_length;
49+
use crate::io_util::parse_content_md5;
50+
use crate::io_util::parse_last_modified;
5151
use crate::io_util::HttpBodyWriter;
5252
use crate::io_util::HttpClient;
5353
use crate::object::ObjectMetadata;
@@ -376,57 +376,22 @@ impl Accessor for Backend {
376376
http::StatusCode::OK => {
377377
let mut m = ObjectMetadata::default();
378378

379-
// Parse content_length
380-
if let Some(v) = resp.headers().get(http::header::CONTENT_LENGTH) {
381-
let v = v.to_str().map_err(|e| {
382-
other(ObjectError::new(
383-
"stat",
384-
&p,
385-
anyhow!("parse content-length header: {:?}", e),
386-
))
387-
})?;
388-
let v = u64::from_str(v).map_err(|e| {
389-
other(ObjectError::new(
390-
"stat",
391-
&p,
392-
anyhow!("parse content-length header: {:?}", e),
393-
))
394-
})?;
395-
379+
if let Some(v) = parse_content_length(resp.headers())
380+
.map_err(|e| other(ObjectError::new("stat", &p, e)))?
381+
{
396382
m.set_content_length(v);
397383
}
398384

399-
// Parse content_md5
400-
if let Some(v) = resp.headers().get(HeaderName::from_static("content-md5")) {
401-
let v = v.to_str().map_err(|e| {
402-
other(ObjectError::new(
403-
"stat",
404-
&p,
405-
anyhow!("parse content-md5 header: {:?}", e),
406-
))
407-
})?;
408-
385+
if let Some(v) = parse_content_md5(resp.headers())
386+
.map_err(|e| other(ObjectError::new("stat", &p, e)))?
387+
{
409388
m.set_content_md5(v);
410389
}
411390

412-
// Parse last_modified
413-
if let Some(v) = resp.headers().get(http::header::LAST_MODIFIED) {
414-
let v = v.to_str().map_err(|e| {
415-
other(ObjectError::new(
416-
"stat",
417-
&p,
418-
anyhow!("parse last-modified header: {:?}", e),
419-
))
420-
})?;
421-
let t = OffsetDateTime::parse(v, &Rfc2822).map_err(|e| {
422-
other(ObjectError::new(
423-
"stat",
424-
&p,
425-
anyhow!("parse last-modified header: {:?}", e),
426-
))
427-
})?;
428-
429-
m.set_last_modified(t);
391+
if let Some(v) = parse_last_modified(resp.headers())
392+
.map_err(|e| other(ObjectError::new("stat", &p, e)))?
393+
{
394+
m.set_last_modified(v);
430395
}
431396

432397
if p.ends_with('/') {

‎src/services/http/backend.rs

Lines changed: 13 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ use std::io::ErrorKind;
2121
use std::io::Result;
2222
use std::mem;
2323
use std::pin::Pin;
24-
use std::str::FromStr;
2524
use std::sync::Arc;
2625
use std::task::Context;
2726
use std::task::Poll;
@@ -30,7 +29,6 @@ use anyhow::anyhow;
3029
use async_trait::async_trait;
3130
use bytes::BufMut;
3231
use futures::TryStreamExt;
33-
use http::header::HeaderName;
3432
use http::Response;
3533
use http::StatusCode;
3634
use hyper::body::HttpBody;
@@ -40,12 +38,13 @@ use log::error;
4038
use log::info;
4139
use radix_trie::Trie;
4240
use radix_trie::TrieCommon;
43-
use time::format_description::well_known::Rfc2822;
44-
use time::OffsetDateTime;
4541

4642
use crate::error::other;
4743
use crate::error::BackendError;
4844
use crate::error::ObjectError;
45+
use crate::io_util::parse_content_length;
46+
use crate::io_util::parse_content_md5;
47+
use crate::io_util::parse_last_modified;
4948
use crate::io_util::HttpClient;
5049
use crate::ops::BytesRange;
5150
use crate::ops::OpCreate;
@@ -283,56 +282,22 @@ impl Accessor for Backend {
283282
StatusCode::OK => {
284283
let mut m = ObjectMetadata::default();
285284

286-
// Parse content_length
287-
if let Some(v) = resp.headers().get(http::header::CONTENT_LENGTH) {
288-
let v = v.to_str().map_err(|e| {
289-
other(ObjectError::new(
290-
"stat",
291-
&p,
292-
anyhow!("parse {} header: {:?}", http::header::CONTENT_LENGTH, e),
293-
))
294-
})?;
295-
296-
let v = u64::from_str(v).map_err(|e| {
297-
other(ObjectError::new(
298-
"stat",
299-
&p,
300-
anyhow!("parse {} header: {:?}", http::header::CONTENT_LENGTH, e),
301-
))
302-
})?;
303-
285+
if let Some(v) = parse_content_length(resp.headers())
286+
.map_err(|e| other(ObjectError::new("stat", &p, e)))?
287+
{
304288
m.set_content_length(v);
305289
}
306290

307-
// Parse content_md5
308-
if let Some(v) = resp.headers().get(HeaderName::from_static("content-md5")) {
309-
let v = v.to_str().map_err(|e| {
310-
other(ObjectError::new(
311-
"stat",
312-
&p,
313-
anyhow!("parse {} header: {:?}", "content-md5", e),
314-
))
315-
})?;
291+
if let Some(v) = parse_content_md5(resp.headers())
292+
.map_err(|e| other(ObjectError::new("stat", &p, e)))?
293+
{
316294
m.set_content_md5(v);
317295
}
318296

319-
// Parse last_modified
320-
if let Some(v) = resp.headers().get(http::header::LAST_MODIFIED) {
321-
let v = v.to_str().map_err(|e| {
322-
other(ObjectError::new(
323-
"stat",
324-
&p,
325-
anyhow!("parse {} header: {:?}", http::header::LAST_MODIFIED, e),
326-
))
327-
})?;
328-
let t = OffsetDateTime::parse(v, &Rfc2822).map_err(|e| {
329-
other(ObjectError::new(
330-
"stat",
331-
&p,
332-
anyhow!("parse {} header: {:?}", http::header::LAST_MODIFIED, e),
333-
))
334-
})?;
335-
m.set_last_modified(t);
297+
if let Some(v) = parse_last_modified(resp.headers())
298+
.map_err(|e| other(ObjectError::new("stat", &p, e)))?
299+
{
300+
m.set_last_modified(v);
336301
}
337302

338303
if p.ends_with('/') {

‎src/services/s3/backend.rs

Lines changed: 13 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ use std::fmt::Formatter;
1919
use std::io::Error;
2020
use std::io::ErrorKind;
2121
use std::io::Result;
22-
use std::str::FromStr;
2322
use std::sync::Arc;
2423

2524
use anyhow::anyhow;
@@ -41,14 +40,15 @@ use once_cell::sync::Lazy;
4140
use reqsign::services::aws::loader::CredentialLoadChain;
4241
use reqsign::services::aws::loader::DummyLoader;
4342
use reqsign::services::aws::v4::Signer;
44-
use time::format_description::well_known::Rfc2822;
45-
use time::OffsetDateTime;
4643

4744
use super::object_stream::DirStream;
4845
use crate::error::other;
4946
use crate::error::BackendError;
5047
use crate::error::ObjectError;
5148
use crate::io_util::new_http_channel;
49+
use crate::io_util::parse_content_length;
50+
use crate::io_util::parse_content_md5;
51+
use crate::io_util::parse_last_modified;
5252
use crate::io_util::HttpBodyWriter;
5353
use crate::io_util::HttpClient;
5454
use crate::ops::BytesRange;
@@ -902,56 +902,22 @@ impl Accessor for Backend {
902902
StatusCode::OK => {
903903
let mut m = ObjectMetadata::default();
904904

905-
// Parse content_length
906-
if let Some(v) = resp.headers().get(http::header::CONTENT_LENGTH) {
907-
let v = v.to_str().map_err(|e| {
908-
other(ObjectError::new(
909-
"stat",
910-
&p,
911-
anyhow!("parse {} header: {:?}", http::header::CONTENT_LENGTH, e),
912-
))
913-
})?;
914-
915-
let v = u64::from_str(v).map_err(|e| {
916-
other(ObjectError::new(
917-
"stat",
918-
&p,
919-
anyhow!("parse {} header: {:?}", http::header::CONTENT_LENGTH, e),
920-
))
921-
})?;
922-
905+
if let Some(v) = parse_content_length(resp.headers())
906+
.map_err(|e| other(ObjectError::new("stat", &p, e)))?
907+
{
923908
m.set_content_length(v);
924909
}
925910

926-
// Parse content_md5
927-
if let Some(v) = resp.headers().get(HeaderName::from_static("content-md5")) {
928-
let v = v.to_str().map_err(|e| {
929-
other(ObjectError::new(
930-
"stat",
931-
&p,
932-
anyhow!("parse {} header: {:?}", "content-md5", e),
933-
))
934-
})?;
911+
if let Some(v) = parse_content_md5(resp.headers())
912+
.map_err(|e| other(ObjectError::new("stat", &p, e)))?
913+
{
935914
m.set_content_md5(v);
936915
}
937916

938-
// Parse last_modified
939-
if let Some(v) = resp.headers().get(http::header::LAST_MODIFIED) {
940-
let v = v.to_str().map_err(|e| {
941-
other(ObjectError::new(
942-
"stat",
943-
&p,
944-
anyhow!("parse {} header: {:?}", http::header::LAST_MODIFIED, e),
945-
))
946-
})?;
947-
let t = OffsetDateTime::parse(v, &Rfc2822).map_err(|e| {
948-
other(ObjectError::new(
949-
"stat",
950-
&p,
951-
anyhow!("parse {} header: {:?}", http::header::LAST_MODIFIED, e),
952-
))
953-
})?;
954-
m.set_last_modified(t);
917+
if let Some(v) = parse_last_modified(resp.headers())
918+
.map_err(|e| other(ObjectError::new("stat", &p, e)))?
919+
{
920+
m.set_last_modified(v);
955921
}
956922

957923
if p.ends_with('/') {

0 commit comments

Comments
 (0)