refactoring: add databridge

issue: #14

.gitlab-ci.yml

@@ -7,8 +7,9 @@ stages:
   - build
   - test
   - publish
-  - cleanup
   - deploy
+  - security
+  - cleanup
 
 variables:
   FULL_NAME_IMAGE: registry-gitlab.prozorro.sale/prozorro-sale/prozorro-auth
@@ -33,6 +34,14 @@ test-unit:
 test-integration:
   extends: .test-integration-template
 
+test-safety:
+  extends: .test-security-template
+
+test-bandit:
+  extends: .test-security-template
+  script:
+    - make bandit
+
 publish-coverage-report:
   extends: .publish-coverage-report-template
 

Dockerfile

 FROM python:3.9-slim as prod_base
 WORKDIR /auth
 ENV PYTHONUNBUFFERED True
-COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
+COPY requirements/requirements.txt .
+RUN pip install --no-cache-dir --upgrade pip && pip install --no-cache-dir -r requirements.txt
 COPY secrets /secrets
 COPY config /config
 
-FROM prod_base as test_base
-COPY test-requirements.txt .
-RUN pip install --no-cache-dir -r test-requirements.txt
-
 FROM prod_base as prod
 COPY src/ .
 ARG version=unknown
 RUN echo $version && sed -i "s/##VERSION##/$version/g" prozorro_sale/__init__.py
 
-FROM test_base as test
-COPY src/ .
+FROM prod as test
+COPY requirements/test.txt .
+ARG version=unknown
+RUN pip install --no-cache-dir -r test.txt \
+    && echo $version && sed -i "s/##VERSION##/$version-dev/g" prozorro_sale/__init__.py
 COPY test ./test
 COPY .coveragerc .coveragerc
-ARG version=unknown
-RUN echo $version && sed -i "s/##VERSION##/$version-dev/g" prozorro_sale/__init__.py
+
+FROM prod as local
+COPY requirements/development.txt .
+RUN pip install --no-cache-dir -r development.txt
 
 FROM prod

Makefile

-PROJECT_NAME=auth-service
-IMAGE ?= prozorro-sale-auth:develop
-IMAGE_TEST ?= prozorro-sale-auth:develop-test
+PROJECT_NAME=auth
+IMAGE ?= prozorro-sale-$(PROJECT_NAME):develop
+IMAGE_TEST ?= prozorro-sale-$(PROJECT_NAME):develop-test
 CI_COMMIT_SHORT_SHA ?= $(shell git rev-parse --short HEAD)
 GIT_STAMP ?= $(shell git describe || echo v0.1.0)
 CI_COMMIT_REF_NAME ?= ''
+CI_PIPELINE_ID ?= 1
+COMPOSE_PROJECT_NAME ?= $(PROJECT_NAME)-$(CI_PIPELINE_ID)
 
 # colors
 GREEN = $(shell tput -Txterm setaf 2)
@@ -31,60 +33,74 @@ clean: remove-compose
 .EXPORT_ALL_VARIABLES:
 
 ifdef CI
+  IMAGE_TARGET =
   TESTS_DEBUG_OPTS =
+  DOCKER_BUILD_OPTS =
   REBUILD_IMAGES_FOR_TESTS =
 else
+  IMAGE_TARGET = --target=local
   TESTS_DEBUG_OPTS = -s --pdb-errors
+  DOCKER_BUILD_OPTS = --no-cache
   REBUILD_IMAGES_FOR_TESTS = docker-build
 endif
 
 ## Runs application development on docker. Builds, creates, starts containers for a service. | Common
-run: $(REBUILD_IMAGES_FOR_TESTS)
-	docker run \
-	 -e AUCTIONS_API="http://0.0.0.0:7777" \
-	 -e AUTH_FILE="secrets/auth.yaml" \
-	 -e DOMAIN=localhost \
-	 -p 80:80 $(AUTH_IMAGE) python -m prozorro_sale.auth.api
+run: docker-build
+	@docker-compose up $(PROJECT_NAME)-local $(PROJECT_NAME)-databridge-local
 
 ## Stop application and remove containers for a service.
 remove-compose:
-	docker rm -f $(CI_COMMIT_SHORT_SHA) || true
+	@docker-compose stop
+	@docker-compose -p $(COMPOSE_PROJECT_NAME)-integration stop
+	@docker-compose -p $(COMPOSE_PROJECT_NAME)-unit stop
+	@docker-compose down -v --rmi local
+	@docker-compose -p $(COMPOSE_PROJECT_NAME)-integration down -v --rmi local
+	@docker-compose -p $(COMPOSE_PROJECT_NAME)-unit down -v --rmi local
+	@docker-compose rm -fsv
+	@docker-compose -p $(COMPOSE_PROJECT_NAME)-integration rm -fsv
+	@docker-compose -p $(COMPOSE_PROJECT_NAME)-unit rm -fsv
+	@docker network ls -q -f name=$(PROJECT_NAME)-$(CI_PIPELINE_ID)* | xargs -r docker network rm
 
 ## Builds docker image
 docker-build:
-	docker build --build-arg version=$(GIT_STAMP) \
+	@docker build $(IMAGE_TARGET) $(DOCKER_BUILD_OPTS) --build-arg version=$(GIT_STAMP) \
 		--build-arg twine_username=$(TWINE_USERNAME) \
 		--build-arg twine_password=$(TWINE_PASSWORD) \
 		-t $(IMAGE) .
-	docker build --target=test --build-arg version=$(GIT_STAMP) \
+	@docker build --target=test $(DOCKER_BUILD_OPTS) --build-arg version=$(GIT_STAMP) \
 		--build-arg twine_username=$(TWINE_USERNAME) \
 		--build-arg twine_password=$(TWINE_PASSWORD) \
 		 -t $(IMAGE_TEST) .
 
 ## Runs unit tests | Tests
 test-unit: $(REBUILD_IMAGES_FOR_TESTS)
-	docker rm -f $(CI_COMMIT_SHORT_SHA) || true
-	docker run \
-		--name $(CI_COMMIT_SHORT_SHA) \
-		-e AUCTIONS_API="http://localhost" \
-		-e AUTH_FILE="test/test_secrets/auth.yml" \
-		-e DOMAIN=localhost \
-		$(IMAGE_TEST) nosetests -v \
+	@docker rm -f $(PROJECT_NAME)-unit-$(CI_COMMIT_SHORT_SHA)$(CI_PIPELINE_ID) || true
+	@docker-compose -p $(COMPOSE_PROJECT_NAME)-unit \
+	run --name $(PROJECT_NAME)-unit-$(CI_COMMIT_SHORT_SHA)$(CI_PIPELINE_ID) \
+		$(PROJECT_NAME)-test-unit nosetests -v \
 			$(TESTS_DEBUG_OPTS) \
 			--with-doctest \
-			--with-coverage --cover-package=prozorro_sale.auth \
+			--with-coverage --cover-package=prozorro_sale.$(PROJECT_NAME) \
 			prozorro_sale test.unit
-	docker cp $(CI_COMMIT_SHORT_SHA):/auth/.coverage .coverage.unit
+	@docker cp $(PROJECT_NAME)-unit-$(CI_COMMIT_SHORT_SHA)$(CI_PIPELINE_ID):/$(PROJECT_NAME)/.coverage .coverage.unit
+	@docker-compose -p $(COMPOSE_PROJECT_NAME)-unit stop
 
 ## Runs integration tests
 test-integration: $(REBUILD_IMAGES_FOR_TESTS)
-	docker rm -f prozorro-auth-$(CI_COMMIT_SHORT_SHA) || true
-	docker run --name prozorro-auth-$(CI_COMMIT_SHORT_SHA) \
-	-e AUCTIONS_API="http://localhost" \
-	-e AUTH_FILE="test/test_secrets/auth.yml" \
-	-e DOMAIN=localhost  \
-	$(IMAGE_TEST) pytest -v -s -q --cov-report= --cov=prozorro_sale test/integration/
-	docker cp prozorro-auth-$(CI_COMMIT_SHORT_SHA):/auth/.coverage .coverage.integration
+	@docker rm -f $(PROJECT_NAME)-integration-$(CI_COMMIT_SHORT_SHA)$(CI_PIPELINE_ID) || true
+	@docker-compose -p $(COMPOSE_PROJECT_NAME)-integration \
+	run --name $(PROJECT_NAME)-integration-$(CI_COMMIT_SHORT_SHA)$(CI_PIPELINE_ID) \
+	$(PROJECT_NAME)-test-integration pytest -v -s -q --cov-report= --cov=prozorro_sale test/integration/
+	@docker cp $(PROJECT_NAME)-integration-$(CI_COMMIT_SHORT_SHA)$(CI_PIPELINE_ID):/$(PROJECT_NAME)/.coverage .coverage.integration
+	@docker-compose -p $(COMPOSE_PROJECT_NAME)-integration stop
+
+## Safety checks your installed dependencies for known security vulnerabilities.
+safety: docker-build
+	@docker-compose run --rm $(PROJECT_NAME)-test-unit bash -c "pip install safety && safety check --full-report"
+
+## Bandit is a tool designed to find common security issues in Python code.
+bandit: docker-build
+	@docker-compose run --rm $(PROJECT_NAME)-test-unit bash -c "pip install bandit && bandit -r /$(PROJECT_NAME)/prozorro_sale"
 
 ## Dependency update helm
 helm-dependency-update:
@@ -106,18 +122,18 @@ validate-helm-charts-kubeval: helm-dependency-update
 push-helm-package:
 	curl \
 		--user $(CHART_MUSEUM_USER):$(CHART_MUSEUM_URL) \
-		--data-binary "@auth-$(GIT_STAMP)$(HELM_STAMP).tgz" \
+		--data-binary "@$(PROJECT_NAME)-$(GIT_STAMP)$(HELM_STAMP).tgz" \
 		$(CHART_MUSEUM_URL)
 
 ## Publish coverage report
 publish-coverage:
-	docker rm -f $(CI_COMMIT_SHORT_SHA) || true
-	docker run -d --name $(CI_COMMIT_SHORT_SHA) $(IMAGE_TEST) sleep infinity
-	docker cp .coverage.unit $(CI_COMMIT_SHORT_SHA):/tmp/
-	docker cp .coverage.integration $(CI_COMMIT_SHORT_SHA):/tmp/
-	docker exec $(CI_COMMIT_SHORT_SHA) bash -c "cd /tmp && coverage combine && coverage report && coverage html -d cover-html"
-	docker cp $(CI_COMMIT_SHORT_SHA):/tmp/cover-html cover-html
-	docker rm -f $(CI_COMMIT_SHORT_SHA)
+	@docker rm -f $(CI_COMMIT_SHORT_SHA) || true
+	@docker run -d --name $(CI_COMMIT_SHORT_SHA) $(IMAGE_TEST) sleep infinity
+	@docker cp .coverage.unit $(CI_COMMIT_SHORT_SHA):/tmp/
+	@docker cp .coverage.integration $(CI_COMMIT_SHORT_SHA):/tmp/
+	@docker exec $(CI_COMMIT_SHORT_SHA) bash -c "cd /tmp && coverage combine && coverage report && coverage html -d cover-html"
+	@docker cp $(CI_COMMIT_SHORT_SHA):/tmp/cover-html cover-html
+	@docker rm -f $(CI_COMMIT_SHORT_SHA)
 
 ## Create tag
 version:

docker-compose.yml

+version: '3.3'
+
+services:
+  auth-app: &base_app
+    image: "${IMAGE}"
+    container_name: auth-api
+    command: python -m prozorro_sale.auth.api
+    environment: &base_app_environment
+      AUCTIONS_API: "http://0.0.0.0:7777"
+      AUTH_FILE: "/secrets/auth.yaml"
+      DOMAIN: "localhost"
+
+  auth-databridge:
+    <<: *base_app
+    container_name: auth-databridge
+    command: python -m prozorro_sale.auth.databridge
+
+  auth-test-integration: &test_app
+    <<: *base_app
+    image: "${IMAGE_TEST}"
+    environment:
+      AUCTIONS_API: "http://localhost"
+      AUTH_FILE: "test/test_secrets/auth.yaml"
+      DOMAIN: "localhost"
+
+  auth-test-unit:
+    <<: *test_app
+
+  auth-local: &local_app
+    <<: *base_app
+    tty: true
+    command: adev runserver prozorro_sale/auth/api
+    environment: &local_app_environment
+      <<: *base_app_environment
+      DEBUG: 1
+      PYTHONUNBUFFERED: 1
+      PYTHONASYNCIODEBUG: 1
+      AIO_PORT: 8080
+      AIO_DEBUG_TOOLBAR: 1
+      AIO_LIVERELOAD: 1
+    volumes:
+      - ./src:/auth:delegated
+    restart: always
+    ports:
+      - 8080:8080
+
+  auth-databridge-local:
+    <<: *local_app
+    container_name: auth-databridge
+    command: adev runserver --app-factory=create_databridge prozorro_sale/auth/databridge
+    environment:
+      <<: *local_app_environment
+      AIO_PORT: 8082
+    ports:
+      - 8082:8082
\ No newline at end of file

helm/auth/templates/databridge.yaml

+{{- $Context := dict "ArchitectComponent" "databridge" "Release" .Release "Chart" .Chart "Values" .Values.application.databridge  "Global" . -}}
+{{- include "prozorro-helm-template.base-http-app" $Context -}}
\ No newline at end of file

helm/auth-service/values.yaml → helm/auth/values.yaml

@@ -72,4 +72,12 @@ application:
       - python
     args:
       - -m
-      - prozorro_sale.auth.api
\ No newline at end of file
+      - prozorro_sale.auth.api
+
+  databridge:
+    enabled: true
+    command:
+      - python
+    args:
+      - -m
+      - prozorro_sale.auth.databridge
\ No newline at end of file

requirements/development.txt

+-r requirements.txt
+aiohttp-devtools
+aiohttp-debugtoolbar

requirements.txt → requirements/requirements.txt

@@ -2,9 +2,10 @@
 #For include pre-release and development versions, uncomment line --pre. By default, pip only finds stable versions.
 #--pre
 PyYAML
-aiohttp
+aiohttp==3.7.4.post0
 ipaddress
 cryptography
 pyjwt~=2.0.0
 aiohttp-swagger
 prozorro-tools==0.12.0
+uvloop

test-requirements.txt → requirements/test.txt

+-r requirements.txt
 nose
 coverage
 pytest

setup.py

@@ -2,7 +2,7 @@ import os
 
 from setuptools import find_packages, setup
 
-with open(os.path.join(os.path.dirname(__file__), 'requirements.txt')) as f:
+with open(os.path.join(os.path.dirname(__file__), 'requirements/requirements.txt')) as f:
     requirements = ''
     for line in f.readlines():
         if all(not line.startswith(pref) for pref in ['-', '--', '#']):

src/prozorro_sale/auth/api/__init__.py

+from dotenv import load_dotenv, find_dotenv
+
+load_dotenv(find_dotenv('py.env'), override=False)

src/prozorro_sale/auth/api/__main__.py

+from prozorro_sale.auth.api.main import main
+
+main()

src/prozorro_sale/auth/api.py → src/prozorro_sale/auth/api/api.py

-import asyncio
 import os
 from aiohttp import web
-from aiohttp_swagger import setup_swagger
-from prozorro_sale.tools import middlewares
 
 import prozorro_sale
 from prozorro_sale import tools, auth
-from prozorro_sale.auth import errors, utils
+from prozorro_sale.auth import utils
 
 LOG = tools.logger.get_custom_logger(__name__)
 SWAGGER_DOC_AVAILABLE = os.getenv('SWAGGER_DOC', False)
-AUTH_FILE = os.environ.get('AUTH_FILE', '/secrets/auth.yml')
 AUTH_IP_BLOCK_STRICT = os.environ.get('AUTH_IP_BLOCK_STRICT', '1').lower() not in ('false', '0', 'f')
 
 
@@ -87,51 +83,3 @@ async def auth_auction(request):
                         domain=utils.DOMAIN, path=f'/api/auctions/{auction_id}')
     LOG.info(f'bidder {data["bid"]} successfuly redirected to auction {auction_id}')
     return response
-
-
-async def on_shutdown(app):
-    LOG.info('Shutting down application')
-
-
-async def on_startup(app):
-    LOG.info('Application is starting...')
-
-
-def setup_routes(app):
-    app.router.add_get('/api', version, allow_head=False)
-    app.router.add_get('/api/ping', ping, allow_head=False)
-    app.router.add_get('/api/auth', check_auth, allow_head=False)
-    app.router.add_get('/api/auth/auction', auth_auction, allow_head=False)
-
-
-def create_app():
-    loop = asyncio.get_event_loop()
-    tools.logger.configure_logging()
-    auth.load_auth(AUTH_FILE)
-    app = web.Application(middlewares=[
-        middlewares.request_id_middleware,
-        errors.request_errors_middleware,
-        auth.context_middleware,
-    ], loop=loop)
-    setup_routes(app)
-    app.on_startup.append(on_startup)
-    app.on_shutdown.append(on_shutdown)
-    if SWAGGER_DOC_AVAILABLE:
-        setup_swagger(
-            app,
-            title='Prozorro Sale Notification API',
-            api_version=prozorro_sale.version,
-            ui_version=3,
-            security_definitions={'Bearer': {'type': 'apiKey', 'name': 'Authorization', 'in': 'header'}},
-        )
-
-    return app
-
-
-if __name__ == '__main__':
-    app = create_app()
-    web.run_app(
-        app,
-        port=80,
-        access_log_class=tools.logger.CustomAccessLogger
-    )

src/prozorro_sale/auth/api/main.py

+import os
+import uvloop
+
+from typing import AsyncGenerator
+from aiohttp import web
+from aiohttp_swagger import setup_swagger
+
+
+import prozorro_sale
+from prozorro_sale import tools, auth
+from prozorro_sale.tools.middlewares import request_id_middleware
+from prozorro_sale.auth.api.routes import init_routes
+from prozorro_sale.auth.errors import request_errors_middleware
+
+LOG = tools.logger.get_custom_logger(__name__)
+SWAGGER_DOC_AVAILABLE = os.getenv('SWAGGER_DOC', False)
+AUTH_FILE = os.environ.get('AUTH_FILE', '/secrets/auth.yaml')
+
+
+async def all_start_stop_log(app: web.Application) -> AsyncGenerator[None, None]:
+    LOG.info('Application is starting...')
+    yield
+    LOG.info('Shutting down application')
+
+
+def create_app():
+    tools.logger.configure_logging()
+    auth.load_auth(AUTH_FILE)
+    app = web.Application(middlewares=[
+        request_id_middleware,
+        request_errors_middleware,
+        auth.context_middleware,
+    ])
+    init_routes(app)
+    if SWAGGER_DOC_AVAILABLE:
+        setup_swagger(
+            app,
+            title='Prozorro Sale Notification API',
+            api_version=prozorro_sale.version,
+            ui_version=3,
+            security_definitions={'Bearer': {'type': 'apiKey', 'name': 'Authorization', 'in': 'header'}},
+        )
+    app.cleanup_ctx.extend([
+        all_start_stop_log
+    ])
+
+    return app
+
+
+def main() -> None:
+    uvloop.install()
+    app = create_app()
+    web.run_app(
+        app,
+        port=80,
+        access_log_class=tools.logger.CustomAccessLogger
+    )
+
+
+if __name__ == '__main__':
+    main()

src/prozorro_sale/auth/api/routes.py

+from aiohttp import web, hdrs
+
+from .api import version, ping, check_auth, auth_auction
+
+
+def init_routes(app: web.Application) -> None:
+    add_route = app.router.add_route
+    add_route(hdrs.METH_GET, '/api', version, name='version')
+    add_route(hdrs.METH_GET, '/api/ping', ping, name='ping')
+    add_route(hdrs.METH_GET, '/api/auth', check_auth, name='check_auth')
+    add_route(hdrs.METH_GET, '/api/auth/auction', auth_auction, name='auth_auction')