Commit 7a2f1ca3 authored by dmitry.mashoshin's avatar dmitry.mashoshin
Browse files

Merge branch 'kpn/batch_size' into 'master'

fix: add batch_size on mongo cursor

See merge request !181
parents cdd94ca2 59e69026
......@@ -188,14 +188,14 @@ class MongoDBMirror:
collection_filter = {}
if init_point:
LOG.info(f'Start syncing from init point {init_point}')
collection_filter = {'dateModified': {'$gte': iso8601.parse_date(init_point)}}
collection_filter |= {'dateModified': {'$gte': iso8601.parse_date(init_point)}}
db, collection = utils.namespace_to_db_collection(client.namespace)
LOG.info(f'DB {db}, collection {collection}')
entry = await self._get_newest_oplog_ts()
ts = [entry.time, entry.inc]
async for doc in self._mongo[db][collection].find(collection_filter).sort('dateModified', 1):
async for doc in self._mongo[db][collection].find(collection_filter, batch_size=10).sort('dateModified', 1):
try:
send_data = self._serialize_document(doc, collection)
except (TypeError, ValueError, AttributeError, KeyError) as ex:
......
......@@ -240,7 +240,7 @@ class TestMongoDBMirror:
await self.mongo_db_mirror._initial_sync(mock_client)
mock_namespace_to_db_collection.assert_called_once_with(self.client_namespace)
mock_get_newest_oplog_ts.assert_called_once()
mock_find.assert_called_once_with(collection_filter)
mock_find.assert_called_once_with(collection_filter, batch_size=10)
mock_serialize_document.assert_called_once_with(self.test_data, 'collection_name')
mock_client.upsert.assert_called_once_with([self.current_ts.time, self.current_ts.inc], self.test_data)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment