Commit f5dcb86a authored by dmitry.mashoshin's avatar dmitry.mashoshin
Browse files

Merge branch 'cherednichenko/14/implement_databridge_service' into 'master'

refactoring: add databridge

See merge request !54
parents 6fdcdf5d d0e448ed
......@@ -7,8 +7,9 @@ stages:
- build
- test
- publish
- cleanup
- deploy
- security
- cleanup
variables:
FULL_NAME_IMAGE: registry-gitlab.prozorro.sale/prozorro-sale/prozorro-auth
......@@ -33,6 +34,14 @@ test-unit:
test-integration:
extends: .test-integration-template
test-safety:
extends: .test-security-template
test-bandit:
extends: .test-security-template
script:
- make bandit
publish-coverage-report:
extends: .publish-coverage-report-template
......
FROM python:3.9-slim as prod_base
WORKDIR /auth
ENV PYTHONUNBUFFERED True
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY requirements/requirements.txt .
RUN pip install --no-cache-dir --upgrade pip && pip install --no-cache-dir -r requirements.txt
COPY secrets /secrets
COPY config /config
FROM prod_base as test_base
COPY test-requirements.txt .
RUN pip install --no-cache-dir -r test-requirements.txt
FROM prod_base as prod
COPY src/ .
ARG version=unknown
RUN echo $version && sed -i "s/##VERSION##/$version/g" prozorro_sale/__init__.py
FROM test_base as test
COPY src/ .
FROM prod as test
COPY requirements/test.txt .
ARG version=unknown
RUN pip install --no-cache-dir -r test.txt \
&& echo $version && sed -i "s/##VERSION##/$version-dev/g" prozorro_sale/__init__.py
COPY test ./test
COPY .coveragerc .coveragerc
ARG version=unknown
RUN echo $version && sed -i "s/##VERSION##/$version-dev/g" prozorro_sale/__init__.py
FROM prod as local
COPY requirements/development.txt .
RUN pip install --no-cache-dir -r development.txt
FROM prod
PROJECT_NAME=auth-service
IMAGE ?= prozorro-sale-auth:develop
IMAGE_TEST ?= prozorro-sale-auth:develop-test
PROJECT_NAME=auth
IMAGE ?= prozorro-sale-$(PROJECT_NAME):develop
IMAGE_TEST ?= prozorro-sale-$(PROJECT_NAME):develop-test
CI_COMMIT_SHORT_SHA ?= $(shell git rev-parse --short HEAD)
GIT_STAMP ?= $(shell git describe || echo v0.1.0)
CI_COMMIT_REF_NAME ?= ''
CI_PIPELINE_ID ?= 1
COMPOSE_PROJECT_NAME ?= $(PROJECT_NAME)-$(CI_PIPELINE_ID)
# colors
GREEN = $(shell tput -Txterm setaf 2)
......@@ -31,60 +33,74 @@ clean: remove-compose
.EXPORT_ALL_VARIABLES:
ifdef CI
IMAGE_TARGET =
TESTS_DEBUG_OPTS =
DOCKER_BUILD_OPTS =
REBUILD_IMAGES_FOR_TESTS =
else
IMAGE_TARGET = --target=local
TESTS_DEBUG_OPTS = -s --pdb-errors
DOCKER_BUILD_OPTS = --no-cache
REBUILD_IMAGES_FOR_TESTS = docker-build
endif
## Runs application development on docker. Builds, creates, starts containers for a service. | Common
run: $(REBUILD_IMAGES_FOR_TESTS)
docker run \
-e AUCTIONS_API="http://0.0.0.0:7777" \
-e AUTH_FILE="secrets/auth.yaml" \
-e DOMAIN=localhost \
-p 80:80 $(AUTH_IMAGE) python -m prozorro_sale.auth.api
run: docker-build
@docker-compose up $(PROJECT_NAME)-local $(PROJECT_NAME)-databridge-local
## Stop application and remove containers for a service.
remove-compose:
docker rm -f $(CI_COMMIT_SHORT_SHA) || true
@docker-compose stop
@docker-compose -p $(COMPOSE_PROJECT_NAME)-integration stop
@docker-compose -p $(COMPOSE_PROJECT_NAME)-unit stop
@docker-compose down -v --rmi local
@docker-compose -p $(COMPOSE_PROJECT_NAME)-integration down -v --rmi local
@docker-compose -p $(COMPOSE_PROJECT_NAME)-unit down -v --rmi local
@docker-compose rm -fsv
@docker-compose -p $(COMPOSE_PROJECT_NAME)-integration rm -fsv
@docker-compose -p $(COMPOSE_PROJECT_NAME)-unit rm -fsv
@docker network ls -q -f name=$(PROJECT_NAME)-$(CI_PIPELINE_ID)* | xargs -r docker network rm
## Builds docker image
docker-build:
docker build --build-arg version=$(GIT_STAMP) \
@docker build $(IMAGE_TARGET) $(DOCKER_BUILD_OPTS) --build-arg version=$(GIT_STAMP) \
--build-arg twine_username=$(TWINE_USERNAME) \
--build-arg twine_password=$(TWINE_PASSWORD) \
-t $(IMAGE) .
docker build --target=test --build-arg version=$(GIT_STAMP) \
@docker build --target=test $(DOCKER_BUILD_OPTS) --build-arg version=$(GIT_STAMP) \
--build-arg twine_username=$(TWINE_USERNAME) \
--build-arg twine_password=$(TWINE_PASSWORD) \
-t $(IMAGE_TEST) .
## Runs unit tests | Tests
test-unit: $(REBUILD_IMAGES_FOR_TESTS)
docker rm -f $(CI_COMMIT_SHORT_SHA) || true
docker run \
--name $(CI_COMMIT_SHORT_SHA) \
-e AUCTIONS_API="http://localhost" \
-e AUTH_FILE="test/test_secrets/auth.yml" \
-e DOMAIN=localhost \
$(IMAGE_TEST) nosetests -v \
@docker rm -f $(PROJECT_NAME)-unit-$(CI_COMMIT_SHORT_SHA)$(CI_PIPELINE_ID) || true
@docker-compose -p $(COMPOSE_PROJECT_NAME)-unit \
run --name $(PROJECT_NAME)-unit-$(CI_COMMIT_SHORT_SHA)$(CI_PIPELINE_ID) \
$(PROJECT_NAME)-test-unit nosetests -v \
$(TESTS_DEBUG_OPTS) \
--with-doctest \
--with-coverage --cover-package=prozorro_sale.auth \
--with-coverage --cover-package=prozorro_sale.$(PROJECT_NAME) \
prozorro_sale test.unit
docker cp $(CI_COMMIT_SHORT_SHA):/auth/.coverage .coverage.unit
@docker cp $(PROJECT_NAME)-unit-$(CI_COMMIT_SHORT_SHA)$(CI_PIPELINE_ID):/$(PROJECT_NAME)/.coverage .coverage.unit
@docker-compose -p $(COMPOSE_PROJECT_NAME)-unit stop
## Runs integration tests
test-integration: $(REBUILD_IMAGES_FOR_TESTS)
docker rm -f prozorro-auth-$(CI_COMMIT_SHORT_SHA) || true
docker run --name prozorro-auth-$(CI_COMMIT_SHORT_SHA) \
-e AUCTIONS_API="http://localhost" \
-e AUTH_FILE="test/test_secrets/auth.yml" \
-e DOMAIN=localhost \
$(IMAGE_TEST) pytest -v -s -q --cov-report= --cov=prozorro_sale test/integration/
docker cp prozorro-auth-$(CI_COMMIT_SHORT_SHA):/auth/.coverage .coverage.integration
@docker rm -f $(PROJECT_NAME)-integration-$(CI_COMMIT_SHORT_SHA)$(CI_PIPELINE_ID) || true
@docker-compose -p $(COMPOSE_PROJECT_NAME)-integration \
run --name $(PROJECT_NAME)-integration-$(CI_COMMIT_SHORT_SHA)$(CI_PIPELINE_ID) \
$(PROJECT_NAME)-test-integration pytest -v -s -q --cov-report= --cov=prozorro_sale test/integration/
@docker cp $(PROJECT_NAME)-integration-$(CI_COMMIT_SHORT_SHA)$(CI_PIPELINE_ID):/$(PROJECT_NAME)/.coverage .coverage.integration
@docker-compose -p $(COMPOSE_PROJECT_NAME)-integration stop
## Safety checks your installed dependencies for known security vulnerabilities.
safety: docker-build
@docker-compose run --rm $(PROJECT_NAME)-test-unit bash -c "pip install safety && safety check --full-report"
## Bandit is a tool designed to find common security issues in Python code.
bandit: docker-build
@docker-compose run --rm $(PROJECT_NAME)-test-unit bash -c "pip install bandit && bandit -r /$(PROJECT_NAME)/prozorro_sale"
## Dependency update helm
helm-dependency-update:
......@@ -106,18 +122,18 @@ validate-helm-charts-kubeval: helm-dependency-update
push-helm-package:
curl \
--user $(CHART_MUSEUM_USER):$(CHART_MUSEUM_URL) \
--data-binary "@auth-$(GIT_STAMP)$(HELM_STAMP).tgz" \
--data-binary "@$(PROJECT_NAME)-$(GIT_STAMP)$(HELM_STAMP).tgz" \
$(CHART_MUSEUM_URL)
## Publish coverage report
publish-coverage:
docker rm -f $(CI_COMMIT_SHORT_SHA) || true
docker run -d --name $(CI_COMMIT_SHORT_SHA) $(IMAGE_TEST) sleep infinity
docker cp .coverage.unit $(CI_COMMIT_SHORT_SHA):/tmp/
docker cp .coverage.integration $(CI_COMMIT_SHORT_SHA):/tmp/
docker exec $(CI_COMMIT_SHORT_SHA) bash -c "cd /tmp && coverage combine && coverage report && coverage html -d cover-html"
docker cp $(CI_COMMIT_SHORT_SHA):/tmp/cover-html cover-html
docker rm -f $(CI_COMMIT_SHORT_SHA)
@docker rm -f $(CI_COMMIT_SHORT_SHA) || true
@docker run -d --name $(CI_COMMIT_SHORT_SHA) $(IMAGE_TEST) sleep infinity
@docker cp .coverage.unit $(CI_COMMIT_SHORT_SHA):/tmp/
@docker cp .coverage.integration $(CI_COMMIT_SHORT_SHA):/tmp/
@docker exec $(CI_COMMIT_SHORT_SHA) bash -c "cd /tmp && coverage combine && coverage report && coverage html -d cover-html"
@docker cp $(CI_COMMIT_SHORT_SHA):/tmp/cover-html cover-html
@docker rm -f $(CI_COMMIT_SHORT_SHA)
## Create tag
version:
......
version: '3.3'
services:
auth-app: &base_app
image: "${IMAGE}"
container_name: auth-api
command: python -m prozorro_sale.auth.api
environment: &base_app_environment
AUCTIONS_API: "http://0.0.0.0:7777"
AUTH_FILE: "/secrets/auth.yaml"
DOMAIN: "localhost"
auth-databridge:
<<: *base_app
container_name: auth-databridge
command: python -m prozorro_sale.auth.databridge
auth-test-integration: &test_app
<<: *base_app
image: "${IMAGE_TEST}"
environment:
AUCTIONS_API: "http://localhost"
AUTH_FILE: "test/test_secrets/auth.yaml"
DOMAIN: "localhost"
auth-test-unit:
<<: *test_app
auth-local: &local_app
<<: *base_app
tty: true
command: adev runserver prozorro_sale/auth/api
environment: &local_app_environment
<<: *base_app_environment
DEBUG: 1
PYTHONUNBUFFERED: 1
PYTHONASYNCIODEBUG: 1
AIO_PORT: 8080
AIO_DEBUG_TOOLBAR: 1
AIO_LIVERELOAD: 1
volumes:
- ./src:/auth:delegated
restart: always
ports:
- 8080:8080
auth-databridge-local:
<<: *local_app
container_name: auth-databridge
command: adev runserver --app-factory=create_databridge prozorro_sale/auth/databridge
environment:
<<: *local_app_environment
AIO_PORT: 8082
ports:
- 8082:8082
\ No newline at end of file
{{- $Context := dict "ArchitectComponent" "databridge" "Release" .Release "Chart" .Chart "Values" .Values.application.databridge "Global" . -}}
{{- include "prozorro-helm-template.base-http-app" $Context -}}
\ No newline at end of file
......@@ -72,4 +72,12 @@ application:
- python
args:
- -m
- prozorro_sale.auth.api
\ No newline at end of file
- prozorro_sale.auth.api
databridge:
enabled: true
command:
- python
args:
- -m
- prozorro_sale.auth.databridge
\ No newline at end of file
-r requirements.txt
aiohttp-devtools
aiohttp-debugtoolbar
......@@ -2,9 +2,10 @@
#For include pre-release and development versions, uncomment line --pre. By default, pip only finds stable versions.
#--pre
PyYAML
aiohttp
aiohttp==3.7.4.post0
ipaddress
cryptography
pyjwt~=2.0.0
aiohttp-swagger
prozorro-tools==0.12.0
uvloop
-r requirements.txt
nose
coverage
pytest
......
......@@ -2,7 +2,7 @@ import os
from setuptools import find_packages, setup
with open(os.path.join(os.path.dirname(__file__), 'requirements.txt')) as f:
with open(os.path.join(os.path.dirname(__file__), 'requirements/requirements.txt')) as f:
requirements = ''
for line in f.readlines():
if all(not line.startswith(pref) for pref in ['-', '--', '#']):
......
from dotenv import load_dotenv, find_dotenv
load_dotenv(find_dotenv('py.env'), override=False)
from prozorro_sale.auth.api.main import main
main()
import asyncio
import os
from aiohttp import web
from aiohttp_swagger import setup_swagger
from prozorro_sale.tools import middlewares
import prozorro_sale
from prozorro_sale import tools, auth
from prozorro_sale.auth import errors, utils
from prozorro_sale.auth import utils
LOG = tools.logger.get_custom_logger(__name__)
SWAGGER_DOC_AVAILABLE = os.getenv('SWAGGER_DOC', False)
AUTH_FILE = os.environ.get('AUTH_FILE', '/secrets/auth.yml')
AUTH_IP_BLOCK_STRICT = os.environ.get('AUTH_IP_BLOCK_STRICT', '1').lower() not in ('false', '0', 'f')
......@@ -87,51 +83,3 @@ async def auth_auction(request):
domain=utils.DOMAIN, path=f'/api/auctions/{auction_id}')
LOG.info(f'bidder {data["bid"]} successfuly redirected to auction {auction_id}')
return response
async def on_shutdown(app):
LOG.info('Shutting down application')
async def on_startup(app):
LOG.info('Application is starting...')
def setup_routes(app):
app.router.add_get('/api', version, allow_head=False)
app.router.add_get('/api/ping', ping, allow_head=False)
app.router.add_get('/api/auth', check_auth, allow_head=False)
app.router.add_get('/api/auth/auction', auth_auction, allow_head=False)
def create_app():
loop = asyncio.get_event_loop()
tools.logger.configure_logging()
auth.load_auth(AUTH_FILE)
app = web.Application(middlewares=[
middlewares.request_id_middleware,
errors.request_errors_middleware,
auth.context_middleware,
], loop=loop)
setup_routes(app)
app.on_startup.append(on_startup)
app.on_shutdown.append(on_shutdown)
if SWAGGER_DOC_AVAILABLE:
setup_swagger(
app,
title='Prozorro Sale Notification API',
api_version=prozorro_sale.version,
ui_version=3,
security_definitions={'Bearer': {'type': 'apiKey', 'name': 'Authorization', 'in': 'header'}},
)
return app
if __name__ == '__main__':
app = create_app()
web.run_app(
app,
port=80,
access_log_class=tools.logger.CustomAccessLogger
)
import os
import uvloop
from typing import AsyncGenerator
from aiohttp import web
from aiohttp_swagger import setup_swagger
import prozorro_sale
from prozorro_sale import tools, auth
from prozorro_sale.tools.middlewares import request_id_middleware
from prozorro_sale.auth.api.routes import init_routes
from prozorro_sale.auth.errors import request_errors_middleware
LOG = tools.logger.get_custom_logger(__name__)
SWAGGER_DOC_AVAILABLE = os.getenv('SWAGGER_DOC', False)
AUTH_FILE = os.environ.get('AUTH_FILE', '/secrets/auth.yaml')
async def all_start_stop_log(app: web.Application) -> AsyncGenerator[None, None]:
LOG.info('Application is starting...')
yield
LOG.info('Shutting down application')
def create_app():
tools.logger.configure_logging()
auth.load_auth(AUTH_FILE)
app = web.Application(middlewares=[
request_id_middleware,
request_errors_middleware,
auth.context_middleware,
])
init_routes(app)
if SWAGGER_DOC_AVAILABLE:
setup_swagger(
app,
title='Prozorro Sale Notification API',
api_version=prozorro_sale.version,
ui_version=3,
security_definitions={'Bearer': {'type': 'apiKey', 'name': 'Authorization', 'in': 'header'}},
)
app.cleanup_ctx.extend([
all_start_stop_log
])
return app
def main() -> None:
uvloop.install()
app = create_app()
web.run_app(
app,
port=80,
access_log_class=tools.logger.CustomAccessLogger
)
if __name__ == '__main__':
main()
from aiohttp import web, hdrs
from .api import version, ping, check_auth, auth_auction
def init_routes(app: web.Application) -> None:
add_route = app.router.add_route
add_route(hdrs.METH_GET, '/api', version, name='version')
add_route(hdrs.METH_GET, '/api/ping', ping, name='ping')
add_route(hdrs.METH_GET, '/api/auth', check_auth, name='check_auth')
add_route(hdrs.METH_GET, '/api/auth/auction', auth_auction, name='auth_auction')
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment