Commit 95f3268d authored by Artem Baldin's avatar Artem Baldin
Browse files

Merge branch 'master' into baldin/106/cleanup_artifacts

parents 97514d5a 88eb6940
......@@ -3,12 +3,29 @@ include:
file: '/template-ci.yml'
stages:
- info
- lint
- build
- publish
- deploy
- cleanup
- prune
- cleanup_result
info:
stage: info
script:
- docker ps -a
- docker volume ls
- docker image ls
- docker network ls
cleanup_result:
stage: cleanup_result
script:
- docker ps -a
- docker volume ls
- docker image ls
- docker network ls
variables:
FULL_NAME_IMAGE: registry-gitlab.prozorro.sale/prozorro-sale/prozorro-doc
......@@ -55,20 +72,4 @@ push-helm-pacakge:
cleanup-test-images:
extends: .cleanup-test-images-template
script:
- docker rmi -f $IMAGE || true
prune:
stage: prune
script:
- pwd
- ls -a
- docker ps -a
- docker volume ls
- docker image ls
- docker network ls
- make remove-compose
- docker ps -a
- docker volume ls
- docker image ls
- docker network ls
PROJECT_NAME=prozorro-doc
IMAGE ?= prozorro-sale-prozorro-doc:develop
GIT_STAMP ?= $(shell git describe)
CHART_MUSEUM_URL ?= "https://helm.prozorro.sale/api/charts"
CI_PIPELINE_ID ?= 1
COMPOSE_PROJECT_NAME ?= $(PROJECT_NAME)-$(CI_PIPELINE_ID)
.EXPORT_ALL_VARIABLES:
## Runs application development on docker. Builds, creates, starts containers for a service. | Common
run: COMPOSE ?= docker-compose -f compose-base.yml -f compose-local.yml
run: docker-build
$(COMPOSE) up
## Builds docker image
docker-build:
docker build --build-arg version=$(GIT_STAMP) -t $(IMAGE) .
helm-dependency-update:
@helm3 dependency update helm/$(PROJECT_NAME)
......@@ -32,15 +42,15 @@ version:
## Stops application. Stops running container without removing them.
stop: COMPOSE ?= docker-compose -f compose-base.yml -f compose-local.yml
stop:
docker-compose stop
$(COMPOSE) stop
## Stop application and remove containers for a service.
remove-compose: COMPOSE ?= docker-compose -f compose-base.yml -f compose-local.yml
remove-compose: stop
docker network ls -q -f name=prozorro-doc* | xargs --no-run-if-empty docker network rm
docker-compose down -v
docker-compose -p $(COMPOSE_PROJECT_NAME)-integration down -v
docker-compose -p $(COMPOSE_PROJECT_NAME)-unit down -v
docker-compose rm -fsv
docker-compose -p $(COMPOSE_PROJECT_NAME)-integration rm -fsv
docker-compose -p $(COMPOSE_PROJECT_NAME)-unit rm -fsv
$(COMPOSE) down -v
# $(COMPOSE) -p $(CI_COMMIT_SHORT_SHA) stop
# $(COMPOSE) -p $(CI_COMMIT_SHORT_SHA) rm -fsv
$(COMPOSE) rm -fsv
docker network ls -q -f name=$(PROJECT_NAME)* | xargs --no-run-if-empty docker network rm
version: '3'
services:
doc-api:
image: "${IMAGE}"
command: python -m prozorro_doc.main
environment:
PROCEDURE: na
REGISTRY: na
SEARCH: na
PROCEDURE_MIRROR: na
REGISTRY_MIRROR: na
DOCUMENTS: na
NOTIFICATIONS: na
PROTOCOL_SERVICE: na
AUTH: na
DICTIONARIES_SERVICE: na
BILLING: na
DATABRIDGE: na
AUCTION: na
version: '3'
services:
doc-api:
environment:
SWAGGER_DOC: 1
PROCEDURE: host.docker.internal:8088
REGISTRY: host.docker.internal:8087
ports:
- 8089:80
......@@ -16,6 +16,7 @@ databridge_link: databridge-api
replicaCount: 1
imagePullSecrets: []
swaggerDoc: '1'
initialCacheDelay: 600
image:
repository: registry-gitlab.prozorro.sale/prozorro-sale/prozorro-doc
......@@ -73,6 +74,8 @@ env:
value: "{{.Release.Name}}-{{.Values.billing_link}}"
- name: DATABRIDGE
value: "{{.Release.Name}}-{{.Values.databridge_link}}"
- name: INITIAL_CACHE_DELAY
value: "{{ .Values.initialCacheDelay | int }}"
volumes:
- name: procedure-config
......
-i https://pypi-int.prozorro.sale/
aiohttp[speedups]==3.7.4
aiohttp[speedups]==3.8.1
requests
prozorro-tools==0.12.0
prozorro-tools==0.13.0
uvloop
aiojobs
tenacity
\ No newline at end of file
from prozorro_sale.tools.environment import Environment, url, booleans
DOC_ENV_VARS = {
'PROCEDURE': url,
'SEARCH': url,
'PROCEDURE_MIRROR': url,
'REGISTRY_MIRROR': url,
'AUCTION': url,
'DOCUMENTS': url,
'NOTIFICATIONS': url,
'PROTOCOL_SERVICE': url,
'REGISTRY': url,
'AUTH': url,
'DICTIONARIES_SERVICE': url,
'BILLING': url,
'DATABRIDGE': url,
'SWAGGER_DOC': booleans,
'INITIAL_CACHE_DELAY': int,
}
ENV_DEFAULTS = {
'INITIAL_CACHE_DELAY': 600,
}
ENV_VARS = Environment(spec=DOC_ENV_VARS, default=ENV_DEFAULTS)
from prozorro_doc.conf import ENV_VARS
SWAGGER_FILE_PATH = '/swagger_ui3/index.html'
SWAGGER_DOC_AVAILABLE = ENV_VARS['SWAGGER_DOC']
CACHED_DATA_PATH = '/cached'
SWAGGER_FILE_NAME = 'swagger.json'
URL_SWAGGER_POSTFIX = f'/api/doc/{SWAGGER_FILE_NAME}'
URL_MAPPING = {
'procedure': ENV_VARS['PROCEDURE'],
'search': ENV_VARS['SEARCH'],
'procedure-mirror': ENV_VARS['PROCEDURE_MIRROR'],
'registry-mirror': ENV_VARS['REGISTRY_MIRROR'],
'auction': ENV_VARS['AUCTION'],
'documents': ENV_VARS['DOCUMENTS'],
'notifications': ENV_VARS['NOTIFICATIONS'],
'protocol_service': ENV_VARS['PROTOCOL_SERVICE'],
'registry': ENV_VARS['REGISTRY'],
'auth': ENV_VARS['AUTH'],
'dictionaries': ENV_VARS['DICTIONARIES_SERVICE'],
'billing': ENV_VARS['BILLING'],
'databridge': ENV_VARS['DATABRIDGE'],
}
import os
import uvloop
from aiohttp import web
import asyncio
import aiojobs
from prozorro_sale.tools.api_requests import setup_async_api_requests
from prozorro_doc.conf import DOC_ENV_VARS, ENV_VARS
from prozorro_doc.utils import get_swagger_on_startup
from prozorro_doc.views import ping, swagger, doc, version
from prozorro_doc.constants import URL_MAPPING, SWAGGER_DOC_AVAILABLE
from prozorro_sale.tools import logger
from prozorro_sale.tools.middlewares import request_id_middleware
LOG = logger.get_custom_logger(__name__)
SWAGGER_DOC_AVAILABLE = os.getenv('SWAGGER_DOC', False)
async def init_swagger_cache():
await asyncio.sleep(ENV_VARS['INITIAL_CACHE_DELAY'])
tasks = []
for doc_type, url in URL_MAPPING.items():
task = asyncio.create_task(get_swagger_on_startup(doc_type=doc_type, url=url))
tasks.append(task)
await asyncio.gather(*tasks, return_exceptions=True)
async def create_schedule(app):
app.scheduler = await aiojobs.create_scheduler()
await app.scheduler.spawn(init_swagger_cache())
async def on_shutdown(app):
if hasattr(app, 'scheduler') or app.sheduler:
await app.scheduler.close()
LOG.info('Closing scheduled tasks')
LOG.info('Shutting down application')
......@@ -21,6 +42,8 @@ def create_application():
logger.configure_logging()
setup_routes(app)
app.on_shutdown.append(on_shutdown)
setup_async_api_requests(app)
app.on_startup.append(create_schedule)
LOG.info('App created')
return app
......@@ -36,5 +59,6 @@ def setup_routes(app):
if __name__ == '__main__':
ENV_VARS.check_strict(spec=DOC_ENV_VARS, strict=True)
uvloop.install()
web.run_app(create_application(), port=80, access_log_class=logger.CustomAccessLogger)
import ujson
from pathlib import Path
from tenacity import retry, wait_fixed, stop_after_attempt
from aiohttp import web, ClientError
from prozorro_sale.tools.api_requests import async_api_request
from prozorro_doc.constants import CACHED_DATA_PATH, SWAGGER_FILE_NAME, URL_SWAGGER_POSTFIX
from prozorro_sale.tools import logger
LOG = logger.get_custom_logger(__name__)
def cached_swagger_full_path(doc_type) -> Path:
Path(CACHED_DATA_PATH).mkdir(parents=True, exist_ok=True)
cached_data_full_path = Path(f'{CACHED_DATA_PATH}/{doc_type}_{SWAGGER_FILE_NAME}')
return cached_data_full_path
def get_cached_swagger_path(doc_type) -> Path:
full_path = cached_swagger_full_path(doc_type)
if full_path.exists():
return full_path
async def get_swagger_response(url: str) -> dict:
''' Make request for procedure swagger.json'''
response = await async_api_request.session.get(f"{url}{URL_SWAGGER_POSTFIX}")
if response.ok:
data = await response.json()
return data
@retry(reraise=True, stop=stop_after_attempt(10), wait=wait_fixed(1.5))
async def get_swagger_on_startup(url, doc_type) -> None:
'''
Starts on app startup
Save cached swagger data locally as dict
'''
try:
if data := await get_swagger_response(url):
with cached_swagger_full_path(doc_type).open('w') as fl:
ujson.dump(data, fl)
LOG.info(f"Cached swagger for {doc_type.upper()}")
return
except ClientError as err:
LOG.info(f"ON_STARTUP: Error to load swagger dictionary for {doc_type.upper()}. url: {url}; Context: {err}")
raise Exception
async def get_swagger(url, doc_type) -> dict:
''' Get swagger data for view '''
try:
if data := await get_swagger_response(url):
return data
LOG.info(f"Error to load swagger dictionary for {doc_type.upper()}. url: {url}")
raise web.HTTPNotFound()
except ClientError as err:
LOG.info(f"Error to load swagger dictionary for {doc_type.upper()}. Context: {err}")
raise web.HTTPNotFound()
import asyncio
import os
import requests
from aiohttp import web, ClientSession, ClientConnectorError
FILE_PATH = '/swagger_ui3/index.html'
URL_MAPPING = {
'procedure': f"http://{os.environ['PROCEDURE']}/api",
'search': f"http://{os.environ['SEARCH']}/api",
'procedure-mirror': f"http://{os.environ['PROCEDURE_MIRROR']}/api",
'registry-mirror': f"http://{os.environ['REGISTRY_MIRROR']}/api",
'auction': f"http://{os.environ['AUCTION']}/api",
'documents': f"http://{os.environ['DOCUMENTS']}/api",
'notifications': f"http://{os.environ['NOTIFICATIONS']}/api",
'protocol_service': f"http://{os.environ['PROTOCOL_SERVICE']}/api",
'registry': f"http://{os.environ['REGISTRY']}/api",
'auth': f"http://{os.environ['AUTH']}/api",
'dictionaries': f"http://{os.environ['DICTIONARIES_SERVICE']}/api",
'billing': f"http://{os.environ['BILLING']}/api",
'databridge': f"http://{os.environ['DATABRIDGE']}/api",
}
from prozorro_doc.utils import get_swagger, get_cached_swagger_path
from prozorro_doc.constants import SWAGGER_FILE_PATH, URL_MAPPING
versions = {}
services = [
'procedure',
'search',
'procedure-mirror',
'registry-mirror',
'auction',
'documents',
'notifications',
'protocol_service',
'registry',
'auth',
'dictionaries',
'billing',
'databridge',
]
async def ping(request):
......@@ -49,8 +17,10 @@ async def swagger(request):
url = URL_MAPPING.get(doc_type)
if not url:
return web.HTTPNotFound()
response = requests.get(f"{url}/doc/swagger.json")
data = response.json()
if cached_data_path := get_cached_swagger_path(doc_type):
return web.FileResponse(cached_data_path)
data = await get_swagger(url, doc_type)
return web.json_response(data, status=200)
......@@ -68,7 +38,7 @@ async def make_request(service):
async def version(request):
tasks = []
for service in services:
for service in URL_MAPPING.keys():
tasks.append(asyncio.create_task(make_request(service)))
await asyncio.gather(*tasks)
global versions
......@@ -78,4 +48,4 @@ async def version(request):
async def doc(request):
return web.FileResponse(FILE_PATH)
return web.FileResponse(SWAGGER_FILE_PATH)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment