Commit 284ab249 authored by Pavel Kuzmenko's avatar Pavel Kuzmenko
Browse files

Merge branch 'cherednichenko/41/write_docstrings' into 'master'

docs: add docstrings to project

See merge request !171
parents 89322971 79aeea14
......@@ -34,3 +34,6 @@ cover-html
specs*
**/*.db
docs/_build
docs/rst/api/*
docs/rst/todo/*
PROJECT_NAME=prozorro-mirror
IMAGE_TEST ?= $(PROJECT_NAME):develop-test
GIT_STAMP ?= $(shell git describe || echo v0.1.0)
GIT_TAG ?= $(shell git describe --abbrev=0)
COPYRIGHT=PROZORRO.SALE
CI_COMMIT_REF_NAME ?= ''
CI_COMMIT_SHORT_SHA ?= $(shell git rev-parse --short HEAD)
CI_PIPELINE_ID ?= 1
......@@ -36,7 +38,6 @@ all: help
## Create tag | Release
version:
$(eval GIT_TAG ?= $(shell git describe --abbrev=0))
$(eval VERSION ?= $(shell read -p "Version: " VERSION; echo $$VERSION))
echo "Tagged release $(VERSION)\n" > Changelog-$(VERSION).txt
git log --oneline --no-decorate --no-merges $(GIT_TAG)..HEAD >> Changelog-$(VERSION).txt
......@@ -50,6 +51,20 @@ build-wheel:
publish-wheel:
twine upload --skip-existing dist/*
## Build html docs | Documentation
build-docs:
sphinx-build -a -D version=$(GIT_TAG) -D copyright=$(COPYRIGHT) docs docs/_build/html
## Clean html docs
clean-docs:
rm -rf docs/_build/html
rm -rf docs/rst/api
rm -rf docs/rst/todo
## Build dev docs
dev-docs: clean-docs build-docs
sphinx-autobuild --no-initial docs docs/_build/html
## Shows help. | Help
help:
@echo ''
......
......@@ -156,6 +156,13 @@ example answer
make docker-build
```
#### Auto Dock
To generate sphinx docs
```
pip install -r requirements/docs.txt
make dev-docs
```
### Local spec update
To get specs locally you should set DEPLOYMENT_API_TOKEN variable, with value of gitlab personal access [token](https://gitlab.prozorro.sale/profile/personal_access_tokens), read_api rights.
```bash
......
{%- if show_headings %}
{{- [basename, "module"] | join(' ') | e | heading }}
{% endif -%}
.. automodule:: {{ qualname }}
:noindex:
:special-members:
:private-members:
:members:
:undoc-members:
:show-inheritance:
{%- if show_headings %}
{{- [basename, " "] | join(' ') | e | heading }}
{% endif -%}
{%- for _todo in todos %}
{{ _todo }}
{%- endfor %}
{%- macro automodule(modname, options) -%}
.. automodule:: {{ modname }}
{%- for option in options %}
:{{ option }}:
{%- endfor %}
{%- endmacro %}
{%- macro toctree(docnames) -%}
.. toctree::
:maxdepth: {{ maxdepth }}
{% for docname in docnames %}
{{ docname }}
{%- endfor %}
{%- endmacro %}
{%- if is_namespace %}
{{- [pkgname, "namespace"] | join(" ") | e | heading }}
{% else %}
{{- [pkgname, "package"] | join(" ") | e | heading }}
{% endif %}
{%- if modulefirst and not is_namespace %}
{{ automodule(pkgname, automodule_options) }}
{% endif %}
{%- if subpackages %}
{{ toctree(subpackages) }}
{% endif %}
{%- if submodules %}
{% if separatemodules %}
{{ toctree(submodules) }}
{% else %}
{%- for submodule in submodules %}
{% if show_headings %}
{{- [submodule, "module"] | join(" ") | e | heading(2) }}
{% endif %}
{{ automodule(submodule, automodule_options) }}
{% endfor %}
{%- endif %}
{%- endif %}
{%- if not modulefirst and not is_namespace %}
{{ automodule(pkgname, automodule_options) }}
{% endif %}
{%- macro automodule(modname, options) -%}
.. automodule:: {{ modname }}
{%- for option in options %}
:{{ option }}:
{%- endfor %}
{%- endmacro %}
{%- macro toctree(docnames) -%}
.. toctree::
:maxdepth: {{ maxdepth }}
{% for docname in docnames %}
{{ docname }}
{%- endfor %}
{%- endmacro %}
{%- if is_namespace %}
{{- [pkgname, "namespace"] | join(" ") | e | heading }}
{% else %}
{{- [pkgname, "package"] | join(" ") | e | heading }}
{% endif %}
{%- if modulefirst and not is_namespace %}
{{ automodule(pkgname, automodule_options) }}
{% endif %}
{%- if subpackages %}
Subpackages
-----------
.. toctree::
:maxdepth: {{ maxdepth }}
{% for subpackage in subpackages %}
{{ subpackage }}
{%- endfor %}
{% endif %}
{%- if submodules %}
Submodules
----------
.. toctree::
:maxdepth: {{ maxdepth }}
{% for submodule in submodules %}
{{ submodule }}
{%- endfor %}
{%- endif %}
{{ header | heading }}
.. toctree::
:maxdepth: {{ maxdepth }}
{% for docname in docnames %}
{{ docname }}
{%- endfor %}
{{ header | heading }}
.. toctree::
:maxdepth: {{ maxdepth }}
{% for docname in docnames %}
{{ docname }}
{%- endfor %}
# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys
import tokenize
from typing import Any, List
from sphinx.ext.apidoc import create_modules_toc_file, recurse_tree
from sphinx.util.osutil import FileAvoidWrite
from sphinx.util.template import ReSTRenderer
sys.path.insert(0, os.path.abspath('../src'))
# -- Project information -----------------------------------------------------
# The value for header is set in the modules.rst file.
project = 'MongoDB Mirror'
# -- General configuration ---------------------------------------------------
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.viewcode',
'sphinx.ext.todo',
'sphinx.ext.napoleon',
]
napoleon_use_ivar = True
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'sphinx_rtd_theme'
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
autodoc_default_options = {
'exclude-members': '__module__,__dict__,__weakref__,__dataclass_fields__,__dataclass_params__',
}
class _Opts:
# 'Directory to place all output'
destdir = os.path.abspath('../docs/rst/api/')
# 'file suffix (default: rst)', NO DOT '.'
suffix = "rst"
# 'Run the script without creating files'
dryrun = False
# 'Overwrite existing files'
force = False
# Don't create headings for the module/package
# packages (e.g. when the docstrings already contain them)
noheadings = False
# 'Put module documentation before submodule documentation'
modulefirst = True
# 'Put documentation for each module on its own page'
separatemodules = True
# Follow symbolic links. Powerful when combined with collective.recipe.omelette.'
followlinks = False
# 'Interpret module paths according to PEP-0420 implicit namespaces specification'
implicit_namespaces = False
# 'Maximum depth of submodules to show in the TOC '
maxdepth = 4
# 'Include "_private" modules'
includeprivate = True
# 'Append module_path to sys.path, used when --full is given'
append_syspath = True
# The project header??
header = project
user_template_dir = os.path.abspath('../docs/_templates')
class _Opts_todo(_Opts):
destdir = os.path.abspath('../docs/rst/todo/')
header = "TODO"
def write_file(name: str, text: str, opts) -> None:
"""Write the output file for module/package <name>."""
fname = os.path.join(opts.destdir, '%s.%s' % (name, opts.suffix))
with FileAvoidWrite(fname) as f:
f.write(text)
def create_modules_toc_todo_file(modules: List[str], opts: Any, name: str = 'todo',
user_template_dir: str = None) -> None:
"""Create the module's index."""
modules.sort()
prev_module = ''
for module in modules[:]:
# look if the module is a subpackage and, if yes, ignore it
if module.startswith(prev_module + '.'):
modules.remove(module)
else:
prev_module = module
context = {
'header': opts.header,
'maxdepth': opts.maxdepth,
'docnames': modules,
}
text = ReSTRenderer([user_template_dir]).render('toc_todo.rst_t', context)
write_file(name, text, opts)
def create_module_todo_file(basename: str, opts: Any, todos: List[str],
user_template_dir: str = None) -> None:
"""Build the text of the file and write the file."""
context = {
'show_headings': not opts.noheadings,
'basename': basename,
'todos': todos,
}
text = ReSTRenderer([user_template_dir]).render('module_todo.rst_t', context)
write_file(basename, text, opts)
def formatting_path_to_rst(source_path):
root_folder = f"{os.path.abspath('./src')}/"
return source_path.replace(root_folder, '').replace('/', '.').replace('.py', '')
def get_todo_strings(file_path):
todos = []
with tokenize.open(file_path) as f:
tokens = tokenize.generate_tokens(f.readline)
file_name = formatting_path_to_rst(file_path)
for type, string, start, end, line in tokens:
if type == tokenize.COMMENT and ('# todo' in string.lower()):
message = f'\n.. literalinclude:: ../../../src/{file_name.replace(".", "/")}.py\n ' \
f':lines: {start[0]}-{start[0]+10}'
todos.append(message)
return todos
def todo_lines(app):
opts = _Opts_todo()
project_path = os.path.abspath('src/')
destdir = opts.destdir
if not os.path.isdir(destdir):
os.makedirs(os.path.abspath(destdir))
modules = []
for dirpath, dirnames, files in os.walk(project_path):
for file_ in files:
if not file_.endswith('.py'):
continue
file_path = f'{dirpath}/{file_}'
todos = get_todo_strings(file_path)
if len(todos) == 0:
continue
file_name = formatting_path_to_rst(file_path)
modules.append(file_name)
create_module_todo_file(basename=file_name,
opts=opts,
todos=todos,
user_template_dir=opts.user_template_dir)
create_modules_toc_todo_file(modules, opts, user_template_dir=opts.user_template_dir)
def run_api(app):
rootpath = os.path.abspath('src/')
opts = _Opts()
if not os.path.isdir(opts.destdir):
os.makedirs(opts.destdir)
modules = recurse_tree(rootpath, [], opts, user_template_dir=opts.user_template_dir)
create_modules_toc_file(modules, opts)
def setup(app):
app.connect('builder-inited', run_api)
app.connect('builder-inited', todo_lines)
Welcome to Mirror's documentation.
==================================
.. toctree::
:maxdepth: 4
:caption: Contents:
rst/api/modules
rst/todo/todo
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
sphinx==4.1.2
sphinx-autobuild==2021.3.14
sphinx-rtd-theme==0.5.2
......@@ -12,6 +12,10 @@ LOG = logger.get_custom_logger(__name__)
class MirrorClient:
"""Class for working with a MongoDBMirror.
The namespace is a combination of the database name and the name of the collection.
"""
def __init__(self, namespace):
self._namespace = namespace
......@@ -19,30 +23,48 @@ class MirrorClient:
@property
def namespace(self):
"""Get namespace.
Returns:
str: String in format - 'db_name.collection_name'.
"""
return self._namespace
async def get_last_timestamp(self):
"""Get the last timestamp of the document's modification in the database.
Returns:
bson.timestamp: Timestamp.
"""
return self._ts
async def get_sync_init_point(self):
"""Get the dateModified value of the last document."""
return None
async def upsert(self, ts, data):
"""Insert data, if data exists, update it."""
self._ts = ts
LOG.info(f'upsert {data}')
async def update(self, ts, data):
"""Update data."""
self._ts = ts
LOG.info(f'update {data}')
async def delete(self, ts, data):
"""Delete data."""
self._ts = ts
LOG.info(f'delete {data}')
async def noop(self, ts, data):
"""Update timestamp."""
LOG.info(f'noop {data}')
async def get_ids_since_timestamp(self, ts, data):
"""Get ids list since given timestamp."""
LOG.info(f'get_ids_since_timestamp {data}')
async def handle_error(self, ts, object_id, error_message, error_code=4000):
......@@ -50,16 +72,28 @@ class MirrorClient:
class BaseSocketMirrorClient(MirrorClient):
"""A class for communicating with MongoDB over a websocket."""
def __init__(self, ws, namespace, addr='Unknown'):
self.ws = ws
self.addr = addr
super().__init__(namespace)
# TODO Method is not used.
def is_running(self):
return not self.ws._req.transport.is_closing()
async def receive_json(self):
"""Receive a data and deserialize it into Python dict object.
Returns:
dict: Data.
Raises:
WSMsgTypeClosedException: If the websocket connection is closed or the websocket sent a close code.
UnexpectedWSMsgTypeException: If the message is not a string type.
"""
if self.ws.closed:
raise WSMsgTypeClosedException('')
......@@ -73,9 +107,7 @@ class BaseSocketMirrorClient(MirrorClient):
class SocketMirrorClientSimpleProtocol(BaseSocketMirrorClient):
"""
Simple protocol realization
"""
"""Simple protocol realization."""
async def get_last_timestamp(self):
await self.ws.send_json({
......@@ -128,14 +160,10 @@ class SocketMirrorClientSimpleProtocol(BaseSocketMirrorClient):
}, dumps=ujson.dumps)
async def update(self, ts, data):
"""
Not available for the simple protocol
"""
"""Not available for the simple protocol."""
async def delete(self, ts, data):
"""
Not available for the simple protocol
"""
"""Not available for the simple protocol."""
async def _send_error(self, error_message, error_code=4000, ts=None, **extra_data):
data = {
......@@ -156,11 +184,10 @@ class SocketMirrorClientSimpleProtocol(BaseSocketMirrorClient):
class SocketMirrorClientFullProtocol(SocketMirrorClientSimpleProtocol):
"""
Full protocol realization
"""
"""Full protocol realization."""
async def update(self, ts, data):
"""Update data."""
LOG.info(f"Update operation for object id: {data['_id']}, timestamp: {ts}")
await self.ws.send_json({
'type': 'update',
......@@ -169,6 +196,7 @@ class SocketMirrorClientFullProtocol(SocketMirrorClientSimpleProtocol):
}, dumps=ujson.dumps)
async def delete(self, ts, data):
"""Delete data."""
LOG.info(f"Delete operation for object id: {data['_id']}, timestamp: {ts}")
await self.ws.send_json({
'type': 'delete',
......
class MirrorClosed(Exception):
"""A normal closure the websocket connection."""
pass
class MirrorException(Exception):
"""Base class for Mirror exceptions."""
pass
class MirrorUnexpectedResponse(MirrorException):
"""The client sent the wrong response type."""
def __init__(self, object_name, response_data):
super().__init__(f'{object_name} is not defined. Received - {response_data}')
class ClientTimestampException(MirrorException):
"""The client haven't a timestamp data."""
def __init__(self, ex_text):
super().__init__(f'Failed to get timestamp data. {ex_text}')
class WSMsgTypeClosedException(MirrorException):
"""The websocket connection are closed."""
def __init__(self, msg):
super().__init__(f'Connection was closed. {msg}')
class UnexpectedWSMsgTypeException(MirrorException):
"""The websocket received a message of the wrong type. Expected WSMsgType.TEXT"""
def __init__(self, msg):
super().__init__(f'Unexpected message type. {msg}')
class IncorrectTimestamp(MirrorException):
"""The client sent a timestamp that is no longer stored in the collection."""
def __init__(self, timestamp, oldest_ts):
super().__init__(f'Cannot catch up ts - {timestamp}, oldest oplog timestamp is {oldest_ts}.'
f' Please wipe your data and resync')
......@@ -40,5 +47,6 @@ class IncorrectInitPoint(MirrorException):
class CriticalError(MirrorException):
"""An internal error has occurred."""
def __init__(self, msg):
super().__init__(f'{msg}')
......@@ -26,7 +26,7 @@ import prometheus_client
LOG = logger.get_custom_logger(__name__)
MONGO_URL = os.environ['MONGO_URL']
MONGO_URL = os.environ.get('MONGO_URL')