Merge branch 'master' into baldin/106/cleanup_artifacts

.gitlab-ci.yml

@@ -3,12 +3,29 @@ include:
     file: '/template-ci.yml'
 
 stages:
+  - info
   - lint
   - build
   - publish
   - deploy
   - cleanup
-  - prune
+  - cleanup_result
+
+info:
+  stage: info
+  script:
+    - docker ps -a
+    - docker volume ls
+    - docker image ls
+    - docker network ls
+
+cleanup_result:
+  stage: cleanup_result
+  script:
+    - docker ps -a
+    - docker volume ls
+    - docker image ls
+    - docker network ls
 
 variables:
   FULL_NAME_IMAGE: registry-gitlab.prozorro.sale/prozorro-sale/prozorro-doc
@@ -55,20 +72,4 @@ push-helm-pacakge:
 
 cleanup-test-images:
   extends: .cleanup-test-images-template
-  script:
-    - docker rmi -f $IMAGE || true
 
-prune:
-  stage: prune
-  script:
-    - pwd
-    - ls -a
-    - docker ps -a
-    - docker volume ls
-    - docker image ls
-    - docker network ls
-    - make remove-compose
-    - docker ps -a
-    - docker volume ls
-    - docker image ls
-    - docker network ls

Makefile

 PROJECT_NAME=prozorro-doc
+IMAGE ?= prozorro-sale-prozorro-doc:develop
 GIT_STAMP ?= $(shell git describe)
 CHART_MUSEUM_URL ?= "https://helm.prozorro.sale/api/charts"
-CI_PIPELINE_ID ?= 1
-COMPOSE_PROJECT_NAME ?= $(PROJECT_NAME)-$(CI_PIPELINE_ID)
+
+.EXPORT_ALL_VARIABLES:
+
+## Runs application development on docker. Builds, creates, starts containers for a service. | Common
+run: COMPOSE ?= docker-compose -f compose-base.yml -f compose-local.yml
+run: docker-build
+	$(COMPOSE) up
+
+## Builds docker image
+docker-build:
+	docker build --build-arg version=$(GIT_STAMP) -t $(IMAGE) .
 
 helm-dependency-update:
 	@helm3 dependency update helm/$(PROJECT_NAME)
@@ -32,15 +42,15 @@ version:
 
 
 ## Stops application. Stops running container without removing them.
+stop: COMPOSE ?= docker-compose -f compose-base.yml -f compose-local.yml
 stop:
-	docker-compose stop
+	$(COMPOSE) stop
 
 ## Stop application and remove containers for a service.
+remove-compose: COMPOSE ?= docker-compose -f compose-base.yml -f compose-local.yml
 remove-compose: stop
-	docker network ls -q -f name=prozorro-doc* | xargs --no-run-if-empty docker network rm
-	docker-compose down -v
-	docker-compose -p $(COMPOSE_PROJECT_NAME)-integration down -v
-	docker-compose -p $(COMPOSE_PROJECT_NAME)-unit down -v
-	docker-compose rm -fsv
-	docker-compose -p $(COMPOSE_PROJECT_NAME)-integration rm -fsv
-	docker-compose -p $(COMPOSE_PROJECT_NAME)-unit rm -fsv
+	$(COMPOSE) down -v
+#	$(COMPOSE) -p $(CI_COMMIT_SHORT_SHA) stop
+#	$(COMPOSE) -p $(CI_COMMIT_SHORT_SHA) rm -fsv
+	$(COMPOSE) rm -fsv
+	docker network ls -q -f name=$(PROJECT_NAME)* | xargs --no-run-if-empty docker network rm

compose-base.yml

+version: '3'
+
+services:
+  doc-api:
+    image: "${IMAGE}"
+    command: python -m prozorro_doc.main
+    environment:
+      PROCEDURE: na
+      REGISTRY: na
+      SEARCH: na
+      PROCEDURE_MIRROR: na
+      REGISTRY_MIRROR: na
+      DOCUMENTS: na
+      NOTIFICATIONS: na
+      PROTOCOL_SERVICE: na
+      AUTH: na
+      DICTIONARIES_SERVICE: na
+      BILLING: na
+      DATABRIDGE: na
+      AUCTION: na

compose-local.yml

+version: '3'
+
+services:
+  doc-api:
+    environment:
+      SWAGGER_DOC: 1
+      PROCEDURE: host.docker.internal:8088
+      REGISTRY: host.docker.internal:8087
+    ports:
+      - 8089:80

helm/prozorro-doc/values.yaml

@@ -16,6 +16,7 @@ databridge_link: databridge-api
 replicaCount: 1
 imagePullSecrets: []
 swaggerDoc: '1'
+initialCacheDelay: 600
 
 image:
   repository: registry-gitlab.prozorro.sale/prozorro-sale/prozorro-doc
@@ -73,6 +74,8 @@ env:
     value: "{{.Release.Name}}-{{.Values.billing_link}}"
   - name: DATABRIDGE
     value: "{{.Release.Name}}-{{.Values.databridge_link}}"
+  - name: INITIAL_CACHE_DELAY
+    value: "{{ .Values.initialCacheDelay | int }}"
 
 volumes:
   - name: procedure-config

requirements.txt

 -i https://pypi-int.prozorro.sale/
-aiohttp[speedups]==3.7.4
+aiohttp[speedups]==3.8.1
 requests
-prozorro-tools==0.12.0
+prozorro-tools==0.13.0
 uvloop
+aiojobs
+tenacity
\ No newline at end of file

src/prozorro_doc/conf.py

+from prozorro_sale.tools.environment import Environment, url, booleans
+
+
+DOC_ENV_VARS = {
+    'PROCEDURE': url,
+    'SEARCH': url,
+    'PROCEDURE_MIRROR': url,
+    'REGISTRY_MIRROR': url,
+    'AUCTION': url,
+    'DOCUMENTS': url,
+    'NOTIFICATIONS': url,
+    'PROTOCOL_SERVICE': url,
+    'REGISTRY': url,
+    'AUTH': url,
+    'DICTIONARIES_SERVICE': url,
+    'BILLING': url,
+    'DATABRIDGE': url,
+    'SWAGGER_DOC': booleans,
+    'INITIAL_CACHE_DELAY': int,
+}
+
+ENV_DEFAULTS = {
+    'INITIAL_CACHE_DELAY': 600,
+}
+
+ENV_VARS = Environment(spec=DOC_ENV_VARS, default=ENV_DEFAULTS)

src/prozorro_doc/constants.py

+from prozorro_doc.conf import ENV_VARS
+
+
+SWAGGER_FILE_PATH = '/swagger_ui3/index.html'
+SWAGGER_DOC_AVAILABLE = ENV_VARS['SWAGGER_DOC']
+
+CACHED_DATA_PATH = '/cached'
+
+SWAGGER_FILE_NAME = 'swagger.json'
+URL_SWAGGER_POSTFIX = f'/api/doc/{SWAGGER_FILE_NAME}'
+URL_MAPPING = {
+    'procedure': ENV_VARS['PROCEDURE'],
+    'search': ENV_VARS['SEARCH'],
+    'procedure-mirror': ENV_VARS['PROCEDURE_MIRROR'],
+    'registry-mirror': ENV_VARS['REGISTRY_MIRROR'],
+    'auction': ENV_VARS['AUCTION'],
+    'documents': ENV_VARS['DOCUMENTS'],
+    'notifications': ENV_VARS['NOTIFICATIONS'],
+    'protocol_service': ENV_VARS['PROTOCOL_SERVICE'],
+    'registry': ENV_VARS['REGISTRY'],
+    'auth': ENV_VARS['AUTH'],
+    'dictionaries': ENV_VARS['DICTIONARIES_SERVICE'],
+    'billing': ENV_VARS['BILLING'],
+    'databridge': ENV_VARS['DATABRIDGE'],
+}

src/prozorro_doc/main.py

-import os
-
 import uvloop
 from aiohttp import web
+import asyncio
+import aiojobs
+from prozorro_sale.tools.api_requests import setup_async_api_requests
+
+from prozorro_doc.conf import DOC_ENV_VARS, ENV_VARS
+from prozorro_doc.utils import get_swagger_on_startup
 from prozorro_doc.views import ping, swagger, doc, version
+from prozorro_doc.constants import URL_MAPPING, SWAGGER_DOC_AVAILABLE
 from prozorro_sale.tools import logger
 from prozorro_sale.tools.middlewares import request_id_middleware
 
 LOG = logger.get_custom_logger(__name__)
-SWAGGER_DOC_AVAILABLE = os.getenv('SWAGGER_DOC', False)
+
+
+async def init_swagger_cache():
+    await asyncio.sleep(ENV_VARS['INITIAL_CACHE_DELAY'])
+    tasks = []
+    for doc_type, url in URL_MAPPING.items():
+        task = asyncio.create_task(get_swagger_on_startup(doc_type=doc_type, url=url))
+        tasks.append(task)
+    await asyncio.gather(*tasks, return_exceptions=True)
+
+
+async def create_schedule(app):
+    app.scheduler = await aiojobs.create_scheduler()
+    await app.scheduler.spawn(init_swagger_cache())
 
 
 async def on_shutdown(app):
+    if hasattr(app, 'scheduler') or app.sheduler:
+        await app.scheduler.close()
+        LOG.info('Closing scheduled tasks')
     LOG.info('Shutting down application')
 
 
@@ -21,6 +42,8 @@ def create_application():
     logger.configure_logging()
     setup_routes(app)
     app.on_shutdown.append(on_shutdown)
+    setup_async_api_requests(app)
+    app.on_startup.append(create_schedule)
     LOG.info('App created')
     return app
 
@@ -36,5 +59,6 @@ def setup_routes(app):
 
 
 if __name__ == '__main__':
+    ENV_VARS.check_strict(spec=DOC_ENV_VARS, strict=True)
     uvloop.install()
     web.run_app(create_application(), port=80, access_log_class=logger.CustomAccessLogger)

src/prozorro_doc/utils.py

+import ujson
+from pathlib import Path
+from tenacity import retry, wait_fixed, stop_after_attempt
+from aiohttp import web, ClientError
+
+from prozorro_sale.tools.api_requests import async_api_request
+from prozorro_doc.constants import CACHED_DATA_PATH, SWAGGER_FILE_NAME, URL_SWAGGER_POSTFIX
+from prozorro_sale.tools import logger
+
+LOG = logger.get_custom_logger(__name__)
+
+
+def cached_swagger_full_path(doc_type) -> Path:
+    Path(CACHED_DATA_PATH).mkdir(parents=True, exist_ok=True)
+    cached_data_full_path = Path(f'{CACHED_DATA_PATH}/{doc_type}_{SWAGGER_FILE_NAME}')
+    return cached_data_full_path
+
+
+def get_cached_swagger_path(doc_type) -> Path:
+    full_path = cached_swagger_full_path(doc_type)
+    if full_path.exists():
+        return full_path
+
+
+async def get_swagger_response(url: str) -> dict:
+    ''' Make request for procedure swagger.json'''
+    response = await async_api_request.session.get(f"{url}{URL_SWAGGER_POSTFIX}")
+    if response.ok:
+        data = await response.json()
+        return data
+
+
+@retry(reraise=True, stop=stop_after_attempt(10), wait=wait_fixed(1.5))
+async def get_swagger_on_startup(url, doc_type) -> None:
+    '''
+    Starts on app startup
+    Save cached swagger data locally as dict
+    '''
+    try:
+        if data := await get_swagger_response(url):
+            with cached_swagger_full_path(doc_type).open('w') as fl:
+                ujson.dump(data, fl)
+            LOG.info(f"Cached swagger for {doc_type.upper()}")
+            return
+    except ClientError as err:
+        LOG.info(f"ON_STARTUP: Error to load swagger dictionary for {doc_type.upper()}. url: {url}; Context: {err}")
+        raise Exception
+
+
+async def get_swagger(url, doc_type) -> dict:
+    ''' Get swagger data for view '''
+    try:
+        if data := await get_swagger_response(url):
+            return data
+        LOG.info(f"Error to load swagger dictionary for {doc_type.upper()}. url: {url}")
+        raise web.HTTPNotFound()
+    except ClientError as err:
+        LOG.info(f"Error to load swagger dictionary for {doc_type.upper()}. Context: {err}")
+        raise web.HTTPNotFound()

src/prozorro_doc/views.py

 import asyncio
-import os
 
-import requests
 from aiohttp import web, ClientSession, ClientConnectorError
 
-FILE_PATH = '/swagger_ui3/index.html'
-
-URL_MAPPING = {
-    'procedure': f"http://{os.environ['PROCEDURE']}/api",
-    'search': f"http://{os.environ['SEARCH']}/api",
-    'procedure-mirror': f"http://{os.environ['PROCEDURE_MIRROR']}/api",
-    'registry-mirror': f"http://{os.environ['REGISTRY_MIRROR']}/api",
-    'auction': f"http://{os.environ['AUCTION']}/api",
-    'documents': f"http://{os.environ['DOCUMENTS']}/api",
-    'notifications': f"http://{os.environ['NOTIFICATIONS']}/api",
-    'protocol_service': f"http://{os.environ['PROTOCOL_SERVICE']}/api",
-    'registry': f"http://{os.environ['REGISTRY']}/api",
-    'auth': f"http://{os.environ['AUTH']}/api",
-    'dictionaries': f"http://{os.environ['DICTIONARIES_SERVICE']}/api",
-    'billing': f"http://{os.environ['BILLING']}/api",
-    'databridge': f"http://{os.environ['DATABRIDGE']}/api",
-}
+from prozorro_doc.utils import get_swagger, get_cached_swagger_path
+from prozorro_doc.constants import SWAGGER_FILE_PATH, URL_MAPPING
 
 versions = {}
-services = [
-    'procedure',
-    'search',
-    'procedure-mirror',
-    'registry-mirror',
-    'auction',
-    'documents',
-    'notifications',
-    'protocol_service',
-    'registry',
-    'auth',
-    'dictionaries',
-    'billing',
-    'databridge',
-]
 
 
 async def ping(request):
@@ -49,8 +17,10 @@ async def swagger(request):
     url = URL_MAPPING.get(doc_type)
     if not url:
         return web.HTTPNotFound()
-    response = requests.get(f"{url}/doc/swagger.json")
-    data = response.json()
+    if cached_data_path := get_cached_swagger_path(doc_type):
+        return web.FileResponse(cached_data_path)
+
+    data = await get_swagger(url, doc_type)
     return web.json_response(data, status=200)
 
 
@@ -68,7 +38,7 @@ async def make_request(service):
 
 async def version(request):
     tasks = []
-    for service in services:
+    for service in URL_MAPPING.keys():
         tasks.append(asyncio.create_task(make_request(service)))
     await asyncio.gather(*tasks)
     global versions
@@ -78,4 +48,4 @@ async def version(request):
 
 
 async def doc(request):
-    return web.FileResponse(FILE_PATH)
+    return web.FileResponse(SWAGGER_FILE_PATH)