Merge branch 'cherednichenko/41/write_docstrings' into 'master'

docs: add docstrings to project

See merge request !171

.gitignore

@@ -34,3 +34,6 @@ cover-html
 specs*
 
 **/*.db
+docs/_build
+docs/rst/api/*
+docs/rst/todo/*

Makefile

 PROJECT_NAME=prozorro-mirror
 IMAGE_TEST ?= $(PROJECT_NAME):develop-test
 GIT_STAMP ?= $(shell git describe || echo v0.1.0)
+GIT_TAG ?= $(shell git describe --abbrev=0)
+COPYRIGHT=PROZORRO.SALE
 CI_COMMIT_REF_NAME ?= ''
 CI_COMMIT_SHORT_SHA ?= $(shell git rev-parse --short HEAD)
 CI_PIPELINE_ID ?= 1
@@ -36,7 +38,6 @@ all: help
 
 ## Create tag | Release
 version:
-	$(eval GIT_TAG ?= $(shell git describe --abbrev=0))
 	$(eval VERSION ?= $(shell read -p "Version: " VERSION; echo $$VERSION))
 	echo "Tagged release $(VERSION)\n" > Changelog-$(VERSION).txt
 	git log --oneline --no-decorate --no-merges $(GIT_TAG)..HEAD >> Changelog-$(VERSION).txt
@@ -50,6 +51,20 @@ build-wheel:
 publish-wheel:
 	twine upload --skip-existing dist/*
 
+## Build html docs | Documentation
+build-docs:
+	sphinx-build -a -D version=$(GIT_TAG) -D copyright=$(COPYRIGHT) docs docs/_build/html
+
+## Clean html docs
+clean-docs:
+	rm -rf docs/_build/html
+	rm -rf docs/rst/api
+	rm -rf docs/rst/todo
+
+## Build dev docs
+dev-docs: clean-docs build-docs
+	sphinx-autobuild --no-initial docs docs/_build/html
+
 ## Shows help. | Help
 help:
 	@echo ''

README.md

@@ -156,6 +156,13 @@ example answer
 make docker-build
 ```
 
+#### Auto Dock
+To generate sphinx docs
+```
+pip install -r requirements/docs.txt
+make dev-docs
+```
+
 ### Local spec update 
 To get specs locally you should set DEPLOYMENT_API_TOKEN variable, with value of gitlab personal access [token](https://gitlab.prozorro.sale/profile/personal_access_tokens), read_api rights.
 ```bash

docs/_templates/module.rst_t

+{%- if show_headings %}
+{{- [basename, "module"] | join(' ') | e | heading }}
+
+{% endif -%}
+.. automodule:: {{ qualname }}
+   :noindex:
+   :special-members:
+   :private-members:
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+

docs/_templates/module_todo.rst_t

+{%- if show_headings %}
+{{- [basename, " "] | join(' ') | e | heading }}
+{% endif -%}
+{%- for _todo in todos %}
+
+   {{ _todo }}
+{%- endfor %}
+

docs/_templates/package.rst_t

+{%- macro automodule(modname, options) -%}
+.. automodule:: {{ modname }}
+{%- for option in options %}
+   :{{ option }}:
+{%- endfor %}
+{%- endmacro %}
+
+{%- macro toctree(docnames) -%}
+.. toctree::
+   :maxdepth: {{ maxdepth }}
+{% for docname in docnames %}
+   {{ docname }}
+{%- endfor %}
+{%- endmacro %}
+
+{%- if is_namespace %}
+{{- [pkgname, "namespace"] | join(" ") | e | heading }}
+{% else %}
+{{- [pkgname, "package"] | join(" ") | e | heading }}
+{% endif %}
+
+{%- if modulefirst and not is_namespace %}
+{{ automodule(pkgname, automodule_options) }}
+{% endif %}
+
+{%- if subpackages %}
+
+{{ toctree(subpackages) }}
+{% endif %}
+
+{%- if submodules %}
+
+{% if separatemodules %}
+{{ toctree(submodules) }}
+{% else %}
+{%- for submodule in submodules %}
+{% if show_headings %}
+{{- [submodule, "module"] | join(" ") | e | heading(2) }}
+{% endif %}
+{{ automodule(submodule, automodule_options) }}
+{% endfor %}
+{%- endif %}
+{%- endif %}
+
+{%- if not modulefirst and not is_namespace %}
+
+
+{{ automodule(pkgname, automodule_options) }}
+{% endif %}

docs/_templates/package_todo.rst_t

+{%- macro automodule(modname, options) -%}
+.. automodule:: {{ modname }}
+{%- for option in options %}
+   :{{ option }}:
+{%- endfor %}
+{%- endmacro %}
+
+{%- macro toctree(docnames) -%}
+.. toctree::
+   :maxdepth: {{ maxdepth }}
+{% for docname in docnames %}
+   {{ docname }}
+{%- endfor %}
+{%- endmacro %}
+
+{%- if is_namespace %}
+{{- [pkgname, "namespace"] | join(" ") | e | heading }}
+{% else %}
+{{- [pkgname, "package"] | join(" ") | e | heading }}
+{% endif %}
+
+{%- if modulefirst and not is_namespace %}
+{{ automodule(pkgname, automodule_options) }}
+{% endif %}
+
+{%- if subpackages %}
+Subpackages
+-----------
+
+
+.. toctree::
+   :maxdepth: {{ maxdepth }}
+{% for subpackage in subpackages %}
+   {{ subpackage }}
+{%- endfor %}
+{% endif %}
+
+{%- if submodules %}
+Submodules
+----------
+
+.. toctree::
+   :maxdepth: {{ maxdepth }}
+{% for submodule in submodules %}
+   {{ submodule }}
+{%- endfor %}
+{%- endif %}

docs/_templates/toc.rst_t

+{{ header | heading }}
+
+.. toctree::
+   :maxdepth: {{ maxdepth }}
+{% for docname in docnames %}
+   {{ docname }}
+{%- endfor %}

docs/_templates/toc_todo.rst_t

+{{ header | heading }}
+
+.. toctree::
+   :maxdepth: {{ maxdepth }}
+{% for docname in docnames %}
+   {{ docname }}
+{%- endfor %}

docs/conf.py

+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+import tokenize
+from typing import Any, List
+
+from sphinx.ext.apidoc import create_modules_toc_file, recurse_tree
+from sphinx.util.osutil import FileAvoidWrite
+from sphinx.util.template import ReSTRenderer
+
+
+sys.path.insert(0, os.path.abspath('../src'))
+
+
+# -- Project information -----------------------------------------------------
+# The value for header is set in the modules.rst file.
+project = 'MongoDB Mirror'
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.viewcode',
+    'sphinx.ext.todo',
+    'sphinx.ext.napoleon',
+]
+
+napoleon_use_ivar = True
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+autodoc_default_options = {
+    'exclude-members': '__module__,__dict__,__weakref__,__dataclass_fields__,__dataclass_params__',
+}
+
+
+class _Opts:
+    # 'Directory to place all output'
+    destdir = os.path.abspath('../docs/rst/api/')
+
+    # 'file suffix (default: rst)', NO DOT '.'
+    suffix = "rst"
+
+    # 'Run the script without creating files'
+    dryrun = False
+
+    # 'Overwrite existing files'
+    force = False
+
+    # Don't create headings for the module/package
+    # packages (e.g. when the docstrings already contain them)
+    noheadings = False
+
+    # 'Put module documentation before submodule documentation'
+    modulefirst = True
+
+    # 'Put documentation for each module on its own page'
+    separatemodules = True
+
+    # Follow symbolic links. Powerful when combined with collective.recipe.omelette.'
+    followlinks = False
+
+    # 'Interpret module paths according to PEP-0420 implicit namespaces specification'
+    implicit_namespaces = False
+
+    # 'Maximum depth of submodules to show in the TOC '
+    maxdepth = 4
+
+    # 'Include "_private" modules'
+    includeprivate = True
+
+    # 'Append module_path to sys.path, used when --full is given'
+    append_syspath = True
+
+    # The project header??
+    header = project
+
+    user_template_dir = os.path.abspath('../docs/_templates')
+
+
+class _Opts_todo(_Opts):
+    destdir = os.path.abspath('../docs/rst/todo/')
+    header = "TODO"
+
+
+def write_file(name: str, text: str, opts) -> None:
+    """Write the output file for module/package <name>."""
+
+    fname = os.path.join(opts.destdir, '%s.%s' % (name, opts.suffix))
+
+    with FileAvoidWrite(fname) as f:
+        f.write(text)
+
+
+def create_modules_toc_todo_file(modules: List[str], opts: Any, name: str = 'todo',
+                                 user_template_dir: str = None) -> None:
+    """Create the module's index."""
+    modules.sort()
+    prev_module = ''
+    for module in modules[:]:
+        # look if the module is a subpackage and, if yes, ignore it
+        if module.startswith(prev_module + '.'):
+            modules.remove(module)
+        else:
+            prev_module = module
+
+    context = {
+        'header': opts.header,
+        'maxdepth': opts.maxdepth,
+        'docnames': modules,
+    }
+    text = ReSTRenderer([user_template_dir]).render('toc_todo.rst_t', context)
+    write_file(name, text, opts)
+
+
+def create_module_todo_file(basename: str, opts: Any, todos: List[str],
+                            user_template_dir: str = None) -> None:
+    """Build the text of the file and write the file."""
+    context = {
+        'show_headings': not opts.noheadings,
+        'basename': basename,
+        'todos': todos,
+    }
+    text = ReSTRenderer([user_template_dir]).render('module_todo.rst_t', context)
+    write_file(basename, text, opts)
+
+
+def formatting_path_to_rst(source_path):
+    root_folder = f"{os.path.abspath('./src')}/"
+    return source_path.replace(root_folder, '').replace('/', '.').replace('.py', '')
+
+
+def get_todo_strings(file_path):
+    todos = []
+
+    with tokenize.open(file_path) as f:
+        tokens = tokenize.generate_tokens(f.readline)
+        file_name = formatting_path_to_rst(file_path)
+
+        for type, string, start, end, line in tokens:
+            if type == tokenize.COMMENT and ('# todo' in string.lower()):
+                message = f'\n.. literalinclude:: ../../../src/{file_name.replace(".", "/")}.py\n   ' \
+                          f':lines: {start[0]}-{start[0]+10}'
+                todos.append(message)
+    return todos
+
+
+def todo_lines(app):
+    opts = _Opts_todo()
+    project_path = os.path.abspath('src/')
+    destdir = opts.destdir
+
+    if not os.path.isdir(destdir):
+        os.makedirs(os.path.abspath(destdir))
+    modules = []
+
+    for dirpath, dirnames, files in os.walk(project_path):
+
+        for file_ in files:
+            if not file_.endswith('.py'):
+                continue
+            file_path = f'{dirpath}/{file_}'
+            todos = get_todo_strings(file_path)
+            if len(todos) == 0:
+                continue
+
+            file_name = formatting_path_to_rst(file_path)
+            modules.append(file_name)
+            create_module_todo_file(basename=file_name,
+                                    opts=opts,
+                                    todos=todos,
+                                    user_template_dir=opts.user_template_dir)
+
+    create_modules_toc_todo_file(modules, opts, user_template_dir=opts.user_template_dir)
+
+
+def run_api(app):
+    rootpath = os.path.abspath('src/')
+    opts = _Opts()
+    if not os.path.isdir(opts.destdir):
+        os.makedirs(opts.destdir)
+    modules = recurse_tree(rootpath, [], opts, user_template_dir=opts.user_template_dir)
+    create_modules_toc_file(modules, opts)
+
+
+def setup(app):
+    app.connect('builder-inited', run_api)
+    app.connect('builder-inited', todo_lines)

docs/index.rst

+Welcome to Mirror's documentation.
+==================================
+
+.. toctree::
+   :maxdepth: 4
+   :caption: Contents:
+
+   rst/api/modules
+   rst/todo/todo
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

requirements/docs.txt

+sphinx==4.1.2
+sphinx-autobuild==2021.3.14
+sphinx-rtd-theme==0.5.2

src/mongodb_mirror/client.py

@@ -12,6 +12,10 @@ LOG = logger.get_custom_logger(__name__)
 
 
 class MirrorClient:
+    """Class for working with a MongoDBMirror.
+
+    The namespace is a combination of the database name and the name of the collection.
+    """
 
     def __init__(self, namespace):
         self._namespace = namespace
@@ -19,30 +23,48 @@ class MirrorClient:
 
     @property
     def namespace(self):
+        """Get namespace.
+
+        Returns:
+            str: String in format - 'db_name.collection_name'.
+
+        """
         return self._namespace
 
     async def get_last_timestamp(self):
+        """Get the last timestamp of the document's modification in the database.
+
+        Returns:
+            bson.timestamp: Timestamp.
+
+        """
         return self._ts
 
     async def get_sync_init_point(self):
+        """Get the dateModified value of the last document."""
         return None
 
     async def upsert(self, ts, data):
+        """Insert data, if data exists, update it."""
         self._ts = ts
         LOG.info(f'upsert {data}')
 
     async def update(self, ts, data):
+        """Update data."""
         self._ts = ts
         LOG.info(f'update {data}')
 
     async def delete(self, ts, data):
+        """Delete data."""
         self._ts = ts
         LOG.info(f'delete {data}')
 
     async def noop(self, ts, data):
+        """Update timestamp."""
         LOG.info(f'noop {data}')
 
     async def get_ids_since_timestamp(self, ts, data):
+        """Get ids list since given timestamp."""
         LOG.info(f'get_ids_since_timestamp {data}')
 
     async def handle_error(self, ts, object_id, error_message, error_code=4000):
@@ -50,16 +72,28 @@ class MirrorClient:
 
 
 class BaseSocketMirrorClient(MirrorClient):
+    """A class for communicating with MongoDB over a websocket."""
 
     def __init__(self, ws, namespace, addr='Unknown'):
         self.ws = ws
         self.addr = addr
         super().__init__(namespace)
 
+    # TODO Method is not used.
     def is_running(self):
         return not self.ws._req.transport.is_closing()
 
     async def receive_json(self):
+        """Receive a data and deserialize it into Python dict object.
+
+        Returns:
+            dict: Data.
+
+        Raises:
+            WSMsgTypeClosedException: If the websocket connection is closed or the websocket sent a close code.
+            UnexpectedWSMsgTypeException: If the message is not a string type.
+
+        """
         if self.ws.closed:
             raise WSMsgTypeClosedException('')
 
@@ -73,9 +107,7 @@ class BaseSocketMirrorClient(MirrorClient):
 
 
 class SocketMirrorClientSimpleProtocol(BaseSocketMirrorClient):
-    """
-    Simple protocol realization
-    """
+    """Simple protocol realization."""
 
     async def get_last_timestamp(self):
         await self.ws.send_json({
@@ -128,14 +160,10 @@ class SocketMirrorClientSimpleProtocol(BaseSocketMirrorClient):
         }, dumps=ujson.dumps)
 
     async def update(self, ts, data):
-        """
-        Not available for the simple protocol
-        """
+        """Not available for the simple protocol."""
 
     async def delete(self, ts, data):
-        """
-        Not available for the simple protocol
-        """
+        """Not available for the simple protocol."""
 
     async def _send_error(self, error_message, error_code=4000, ts=None, **extra_data):
         data = {
@@ -156,11 +184,10 @@ class SocketMirrorClientSimpleProtocol(BaseSocketMirrorClient):
 
 
 class SocketMirrorClientFullProtocol(SocketMirrorClientSimpleProtocol):
-    """
-    Full protocol realization
-    """
+    """Full protocol realization."""
 
     async def update(self, ts, data):
+        """Update data."""
         LOG.info(f"Update operation for object id: {data['_id']}, timestamp: {ts}")
         await self.ws.send_json({
             'type': 'update',
@@ -169,6 +196,7 @@ class SocketMirrorClientFullProtocol(SocketMirrorClientSimpleProtocol):
         }, dumps=ujson.dumps)
 
     async def delete(self, ts, data):
+        """Delete data."""
         LOG.info(f"Delete operation for object id: {data['_id']}, timestamp: {ts}")
         await self.ws.send_json({
             'type': 'delete',

src/mongodb_mirror/errors.py

 class MirrorClosed(Exception):
+    """A normal closure the websocket connection."""
     pass
 
 
 class MirrorException(Exception):
+    """Base class for Mirror exceptions."""
     pass
 
 
 class MirrorUnexpectedResponse(MirrorException):
+    """The client sent the wrong response type."""
     def __init__(self, object_name, response_data):
         super().__init__(f'{object_name} is not defined. Received - {response_data}')
 
 
 class ClientTimestampException(MirrorException):
+    """The client haven't a timestamp data."""
     def __init__(self, ex_text):
         super().__init__(f'Failed to get timestamp data. {ex_text}')
 
 
 class WSMsgTypeClosedException(MirrorException):
+    """The websocket connection are closed."""
     def __init__(self, msg):
         super().__init__(f'Connection was closed. {msg}')
 
 
 class UnexpectedWSMsgTypeException(MirrorException):
+    """The websocket received a message of the wrong type. Expected WSMsgType.TEXT"""
     def __init__(self, msg):
         super().__init__(f'Unexpected message type. {msg}')
 
 
 class IncorrectTimestamp(MirrorException):
+    """The client sent a timestamp that is no longer stored in the collection."""
     def __init__(self, timestamp, oldest_ts):
         super().__init__(f'Cannot catch up ts - {timestamp}, oldest oplog timestamp is {oldest_ts}.'
                          f' Please wipe your data and resync')
@@ -40,5 +47,6 @@ class IncorrectInitPoint(MirrorException):
 
 
 class CriticalError(MirrorException):
+    """An internal error has occurred."""
     def __init__(self, msg):
         super().__init__(f'{msg}')

src/mongodb_mirror/mirror.py

@@ -26,7 +26,7 @@ import prometheus_client
 
 LOG = logger.get_custom_logger(__name__)
 
-MONGO_URL = os.environ['MONGO_URL']
+MONGO_URL = os.environ.get('MONGO_URL')