Skip to content

Commit 83e9d34

Browse files
fix(bulk_writer): handle relative remote writer paths
Resolve RemoteBulkWriter upload paths against the resolved local data path before computing object-relative paths. Add regression coverage for relative local_path values. Co-authored-by: XuanYang-cn <51370125+XuanYang-cn@users.noreply.github.com>
1 parent 728b59b commit 83e9d34

2 files changed

Lines changed: 34 additions & 4 deletions

File tree

pymilvus/bulk_writer/remote_bulk_writer.py

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -295,11 +295,12 @@ def _upload(self, file_list: list):
295295
continue
296296

297297
relative_file_path = (
298-
Path(file_path).resolve().relative_to(super().data_path).as_posix()
299-
)
300-
minio_file_path = (
301-
f"{self._remote_path.as_posix().strip('/')}/{relative_file_path}"
298+
Path(file_path)
299+
.resolve()
300+
.relative_to(Path(super().data_path).resolve())
301+
.as_posix()
302302
)
303+
minio_file_path = f"{self._remote_path.as_posix().strip('/')}/{relative_file_path}"
303304

304305
if self._object_exists(minio_file_path):
305306
logger.info(

tests/unit/test_remote_bulk_writer.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
from contextlib import ExitStack
2+
from pathlib import Path
23
from unittest.mock import MagicMock, patch
34

45
from pymilvus.bulk_writer.constants import BulkFileType
@@ -122,3 +123,31 @@ def test_upload_jsonl_file(self, tmp_path):
122123
assert "\\" not in kwargs["object_name"]
123124
mock_local_rm.assert_called_once_with(str(jsonl_file))
124125
assert writer.batch_files[0][0] == uploaded_files[0]
126+
127+
def test_upload_jsonl_file_with_relative_local_path(self, tmp_path, monkeypatch):
128+
monkeypatch.chdir(tmp_path)
129+
relative_local_path = Path("relative-bulk")
130+
131+
writer = RemoteBulkWriter(
132+
schema=self._simple_schema(),
133+
remote_path="bulk/test",
134+
connect_param=None,
135+
file_type=BulkFileType.JSONL,
136+
local_path=str(relative_local_path),
137+
)
138+
jsonl_file = writer._local_path / "1.jsonl"
139+
jsonl_file.write_text('{"id":1,"vector":[1.0,2.0,3.0,4.0]}\n')
140+
141+
with ExitStack() as stack:
142+
stack.enter_context(patch.object(writer, "_bucket_exists", return_value=True))
143+
stack.enter_context(patch.object(writer, "_object_exists", return_value=False))
144+
mock_upload_object = stack.enter_context(patch.object(writer, "_upload_object"))
145+
stack.enter_context(patch.object(writer, "_local_rm"))
146+
uploaded_files = writer._upload([str(jsonl_file)])
147+
148+
assert len(uploaded_files) == 1
149+
_, kwargs = mock_upload_object.call_args
150+
assert kwargs["object_name"] == uploaded_files[0]
151+
assert kwargs["object_name"].endswith("/1.jsonl")
152+
assert "bulk/test/" in kwargs["object_name"]
153+
assert "\\" not in kwargs["object_name"]

0 commit comments

Comments
 (0)