Commit e7b1602e authored by Pavel Kuzmenko's avatar Pavel Kuzmenko
Browse files

Merge branch 'zamirets/6/add-docstrings' into 'master'

docs: Add docstring for metrics service

See merge request !26
parents 5cd46324 0261241f
......@@ -35,3 +35,8 @@ cover-html
swagger.json
specs/
out/
# Sphinx documentation
docs/_build
docs/rst/api/*
docs/rst/todo/*
......@@ -2,6 +2,8 @@ PROJECT_NAME=prozorro-metrics
IMAGE_TEST ?= $(PROJECT_NAME):develop-test
CI_COMMIT_SHORT_SHA ?= $(shell git rev-parse --short HEAD)
GIT_STAMP ?= $(shell git describe || echo v0.1.0)
GIT_TAG ?= $(shell git describe --abbrev=0)
COPYRIGHT=PROZORRO.SALE
# colors
GREEN = $(shell tput -Txterm setaf 2)
......@@ -67,7 +69,6 @@ bandit: docker-build
## Create tag | Release
version:
$(eval GIT_TAG ?= $(shell git describe --abbrev=0))
$(eval VERSION ?= $(shell read -p "Version: " VERSION; echo $$VERSION))
echo "Tagged release $(VERSION)\n" > Changelog-$(VERSION).txt
git log --oneline --no-decorate --no-merges $(GIT_TAG)..HEAD >> Changelog-$(VERSION).txt
......@@ -90,6 +91,19 @@ publish-coverage:
@docker cp $(CI_COMMIT_SHORT_SHA):/tmp/cover-html cover-html
@docker rm -f $(CI_COMMIT_SHORT_SHA)
## Documentation
build-docs:
sphinx-build -a -D version=$(GIT_TAG) -D copyright=$(COPYRIGHT) docs docs/_build/html
clean-docs:
rm -rf docs/_build/html
rm -rf docs/rst/api
rm -rf docs/rst/todo
dev-docs: clean-docs build-docs
sphinx-autobuild --no-initial docs docs/_build/html
## Shows help. | Help
help:
@echo ''
......
{%- if show_headings %}
{{- [basename, "module"] | join(' ') | e | heading }}
{% endif -%}
.. automodule:: {{ qualname }}
:noindex:
:special-members:
:private-members:
:members:
:undoc-members:
:show-inheritance:
\ No newline at end of file
{%- if show_headings %}
{{- [basename, " "] | join(' ') | e | heading }}
{% endif -%}
{%- for _todo in todos %}
{{ _todo }}
{%- endfor %}
{%- macro automodule(modname, options) -%}
.. automodule:: {{ modname }}
{%- for option in options %}
:{{ option }}:
{%- endfor %}
{%- endmacro %}
{%- macro toctree(docnames) -%}
.. toctree::
:maxdepth: {{ maxdepth }}
{% for docname in docnames %}
{{ docname }}
{%- endfor %}
{%- endmacro %}
{%- if is_namespace %}
{{- [pkgname, "namespace"] | join(" ") | e | heading }}
{% else %}
{{- [pkgname, "package"] | join(" ") | e | heading }}
{% endif %}
{%- if modulefirst and not is_namespace %}
{{ automodule(pkgname, automodule_options) }}
{% endif %}
{%- if subpackages %}
{{ toctree(subpackages) }}
{% endif %}
{%- if submodules %}
{% if separatemodules %}
{{ toctree(submodules) }}
{% else %}
{%- for submodule in submodules %}
{% if show_headings %}
{{- [submodule, "module"] | join(" ") | e | heading(2) }}
{% endif %}
{{ automodule(submodule, automodule_options) }}
{% endfor %}
{%- endif %}
{%- endif %}
{%- if not modulefirst and not is_namespace %}
{{ automodule(pkgname, automodule_options) }}
{% endif %}
{%- macro automodule(modname, options) -%}
.. automodule:: {{ modname }}
{%- for option in options %}
:{{ option }}:
{%- endfor %}
{%- endmacro %}
{%- macro toctree(docnames) -%}
.. toctree::
:maxdepth: {{ maxdepth }}
{% for docname in docnames %}
{{ docname }}
{%- endfor %}
{%- endmacro %}
{%- if is_namespace %}
{{- [pkgname, "namespace"] | join(" ") | e | heading }}
{% else %}
{{- [pkgname, "package"] | join(" ") | e | heading }}
{% endif %}
{%- if modulefirst and not is_namespace %}
{{ automodule(pkgname, automodule_options) }}
{% endif %}
{%- if subpackages %}
Subpackages
-----------
.. toctree::
:maxdepth: {{ maxdepth }}
{% for subpackage in subpackages %}
{{ subpackage }}
{%- endfor %}
{% endif %}
{%- if submodules %}
Submodules
----------
.. toctree::
:maxdepth: {{ maxdepth }}
{% for submodule in submodules %}
{{ submodule }}
{%- endfor %}
{%- endif %}
{{ header | heading }}
.. toctree::
:maxdepth: {{ maxdepth }}
{% for docname in docnames %}
{{ docname }}
{%- endfor %}
{{ header | heading }}
.. toctree::
:maxdepth: {{ maxdepth }}
{% for docname in docnames %}
{{ docname }}
{%- endfor %}
# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys
import tokenize
from typing import Any, List
from sphinx.ext.apidoc import create_modules_toc_file, recurse_tree
from sphinx.util.osutil import FileAvoidWrite
from sphinx.util.template import ReSTRenderer
sys.path.insert(0, os.path.abspath('../src'))
# -- Project information -----------------------------------------------------
# The value for header is set in the modules.rst file.
project = 'Prozorro Metrics'
# -- General configuration ---------------------------------------------------
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.viewcode',
'sphinx.ext.todo',
'sphinx.ext.napoleon',
]
napoleon_use_ivar = True
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'sphinx_rtd_theme'
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
autodoc_default_options = {
'exclude-members': '__module__,__dict__,__weakref__,__dataclass_fields__,__dataclass_params__',
}
class _Opts:
# 'Directory to place all output'
destdir = os.path.abspath('../docs/rst/api/')
# 'file suffix (default: rst)', NO DOT '.'
suffix = "rst"
# 'Run the script without creating files'
dryrun = False
# 'Overwrite existing files'
force = False
# Don't create headings for the module/package
# packages (e.g. when the docstrings already contain them)
noheadings = False
# 'Put module documentation before submodule documentation'
modulefirst = True
# 'Put documentation for each module on its own page'
separatemodules = True
# Follow symbolic links. Powerful when combined with collective.recipe.omelette.'
followlinks = False
# 'Interpret module paths according to PEP-0420 implicit namespaces specification'
implicit_namespaces = False
# 'Maximum depth of submodules to show in the TOC '
maxdepth = 4
# 'Include "_private" modules'
includeprivate = True
# 'Append module_path to sys.path, used when --full is given'
append_syspath = True
# The project header??
header = project
user_template_dir = os.path.abspath('../docs/_templates')
class _Opts_todo(_Opts):
destdir = os.path.abspath('../docs/rst/todo/')
header = "TODO"
def write_file(name: str, text: str, opts) -> None:
"""Write the output file for module/package <name>."""
fname = os.path.join(opts.destdir, '%s.%s' % (name, opts.suffix))
with FileAvoidWrite(fname) as f:
f.write(text)
def create_modules_toc_todo_file(modules: List[str], opts: Any, name: str = 'todo',
user_template_dir: str = None) -> None:
"""Create the module's index."""
modules.sort()
prev_module = ''
for module in modules[:]:
# look if the module is a subpackage and, if yes, ignore it
if module.startswith(prev_module + '.'):
modules.remove(module)
else:
prev_module = module
context = {
'header': opts.header,
'maxdepth': opts.maxdepth,
'docnames': modules,
}
text = ReSTRenderer([user_template_dir]).render('toc_todo.rst_t', context)
write_file(name, text, opts)
def create_module_todo_file(basename: str, opts: Any, todos: List[str],
user_template_dir: str = None) -> None:
"""Build the text of the file and write the file."""
context = {
'show_headings': not opts.noheadings,
'basename': basename,
'todos': todos,
}
text = ReSTRenderer([user_template_dir]).render('module_todo.rst_t', context)
write_file(basename, text, opts)
def formatting_path_to_rst(source_path):
root_folder = f"{os.path.abspath('./src')}/"
return source_path.replace(root_folder, '').replace('/', '.').replace('.py', '')
def get_todo_strings(file_path):
todos = []
with tokenize.open(file_path) as f:
tokens = tokenize.generate_tokens(f.readline)
file_name = formatting_path_to_rst(file_path)
for token_type, string, start, end, line in tokens:
if token_type == tokenize.COMMENT and ('# todo' in string.lower()):
message = f'\n.. literalinclude:: ../../../src/{file_name.replace(".", "/")}.py\n ' \
f':lines: {start[0]}-{start[0] + 10}'
todos.append(message)
return todos
def todo_lines(app):
opts = _Opts_todo()
project_path = os.path.abspath('src/')
destdir = opts.destdir
if not os.path.isdir(destdir):
os.makedirs(os.path.abspath(destdir))
modules = []
for dirpath, dirnames, files in os.walk(project_path):
for file_ in files:
if not file_.endswith('.py'):
continue
file_path = f'{dirpath}/{file_}'
todos = get_todo_strings(file_path)
if len(todos) == 0:
continue
file_name = formatting_path_to_rst(file_path)
modules.append(file_name)
create_module_todo_file(basename=file_name,
opts=opts,
todos=todos,
user_template_dir=opts.user_template_dir)
create_modules_toc_todo_file(modules, opts, user_template_dir=opts.user_template_dir)
def run_api(app):
rootpath = os.path.abspath('src/')
opts = _Opts()
if not os.path.isdir(opts.destdir):
os.makedirs(opts.destdir)
modules = recurse_tree(rootpath, [], opts, user_template_dir=opts.user_template_dir)
create_modules_toc_file(modules, opts)
def setup(app):
app.connect('builder-inited', run_api)
app.connect('builder-inited', todo_lines)
Welcome to Prozorro Metrics' documentation!
========================================================
.. toctree::
:maxdepth: 4
:caption: Contents:
rst/api/modules
rst/todo/todo
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
sphinx==4.1.2
sphinx-autobuild==2021.3.14
sphinx-rtd-theme==0.5.2
\ No newline at end of file
"""
Package to provide a generic namespace prozorro_sale
"""
version = '##VERSION##'
__path__ = __import__('pkgutil').extend_path(__path__, __name__)
......@@ -12,30 +12,62 @@ LOG = logging.getLogger('application_wrapper')
def init_mertics_app():
"""
Initialize web application. Register api routes.
Returns:
object: aiohttp.web.Application instance.
"""
app = web.Application()
app.router.add_get('/metrics', metrics)
return app
async def metrics(request):
"""
View for creating metrics response from prometheus client.
Response content type is prometheus_client.CONTENT_TYPE_LATEST.
Args:
request (object): Request.
Returns:
aiohttp.web.Response: Response object.
"""
response = web.Response(body=prometheus_client.generate_latest())
response.content_type = prometheus_client.CONTENT_TYPE_LATEST
return response
class AioAppWrapper:
"""
Base AioApplication wrapper for initializing web applications.
Args:
_app (obj): Web application.
_port (int): Application PORT to listed.
_loop (obj): Event loop.
tcp_params (dict): TCP site parameters.
"""
def __init__(self, _app, _port: int, _loop, tcp_params=None, **kwargs: Any):
self.app = _app
self.port = _port
self.loop = _loop
self.runner_kwargs = kwargs
#: obj: Web Application runner
self.runner = None
self.tcp_params = {}
if isinstance(tcp_params, dict):
self.tcp_params = tcp_params
async def initialize(self):
"""
Method on initializing aiohttp application.
- Create and setup web application runner.
- Create a TCP server bound to host and port.
"""
LOG.info(f'Initialize web app on port {self.port}')
self.runner = web.AppRunner(self.app, **self.runner_kwargs)
......@@ -46,24 +78,49 @@ class AioAppWrapper:
await asyncio.sleep(3600)
def shutdown(self):
"""
Method on shutting down aiohttp application.
Clean up web application runner.
"""
LOG.info('Shutdown aiohttp app')
self.loop.create_task(self.runner.cleanup())
class CoroutineWrapper:
"""
Coroutine wrapper for managing applications tasks.
Args:
_coro (obj): Specialized generator functions (asyncio.coroutines)
_loop (obj): Event loop
_stop_callback (Exception): Callback for stop signal
"""
def __init__(self, _coro, _loop, _stop_callback):
self.coro = _coro
self.loop = _loop
self.stop_call_back = _stop_callback
#: asyncio.Task: A coroutine wrapped in a Future.
self.task = None
def initialize(self):
"""
Method on initializing coroutines.
Schedule a coroutine object: create a task.
Returns:
asyncio.Task: A coroutine wrapped in a Future.
"""
LOG.info('Initialize coroutine')
self.task = self.loop.create_task(self.coro)
return self.task
def shutdown(self):
"""
Method on shutting down coroutine. If there is no stop callback, cancel asyncio task.
"""
LOG.info('Shutting down coroutine')
if self.stop_call_back is not None:
self.stop_call_back()
......@@ -78,22 +135,25 @@ class ApplicationWrapper:
Args:
_loop (:obj:, optional): Event loop
**kwargs: Arbitrary keyword arguments.
Returns:
None
Examples:
kwargs[prometheus] - for setup prometheus client
prometheus={
'_port': 9091,
'tcp_params': {
'backlog': 256,
'reuse_address': True,
'reuse_port': True
}
}
>>> prometheus={
... '_port': 9091,
... 'tcp_params': {
... 'backlog': 256,
... 'reuse_address': True,
... 'reuse_port': True
... }
... }
"""
def __init__(self, _loop=None, **kwargs: Any):
self.loop = _loop
#: Start list of web applications.
self.apps = []
if _loop is None:
self.loop = asyncio.get_event_loop()
......@@ -120,13 +180,14 @@ class ApplicationWrapper:
**kwargs: Arbitrary keyword arguments.
Returns:
None
Examples:
kwargs[tcp_params] - for setup TCPSite params
tcp_params={
'backlog': 256,
'reuse_address': True,
'reuse_port': True
}
>>> tcp_params={
... 'backlog': 256,
... 'reuse_address': True,
... 'reuse_port': True
...}
"""
_app._set_loop(self.loop)
self.apps.append(
......@@ -134,11 +195,25 @@ class ApplicationWrapper:
)
def add_coroutine(self, _coro, _stop_callback=None):
"""
Add coroutine to list of web applications.
Args:
_coro (obj): Specialized generator functions (asyncio.coroutines).
_stop_callback (Exception): Callback for stop signal.
"""
self.apps.append(
CoroutineWrapper(_coro, self.loop, _stop_callback)
)
def handle_signal(self, _signal_name, callback):
"""
Add signal handling.
Args:
_signal_name (typing.Callable): Signal handler name.
callback (Exception): Callback. Mostly used request to exit from the interpreter.
"""
try:
self.loop.add_signal_handler(_signal_name, callback)
except NotImplementedError:
......@@ -146,6 +221,11 @@ class ApplicationWrapper:
pass
def run_all(self):
"""
Run all applications.
If system platform is not win32, add callback with request to exit for SIGTERM and SIGHUP signals.
Finally shutdown application.
"""
if sys.platform != 'win32':
self.handle_signal(signal.SIGTERM, _raise_graceful_exit)