Skip to content

Commit fd88d66

Browse files
ChristopherGSChristopherGS
authored
Monitoring Logs with the Elastic Stack - Integrate with API
1 parent 36d6804 commit fd88d66

File tree

15 files changed

+235
-21
lines changed

15 files changed

+235
-21
lines changed

‎packages/ml_api/Makefile‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,13 @@ run-service-development:
2727

2828
run-service-wsgi:
2929
@echo "+ $@"
30-
gunicorn --workers=1 --bind 0.0.0.0:5000 run:application
30+
gunicorn --bind 0.0.0.0:5000 \
31+
--workers=1 \
32+
--log-config gunicorn_logging.conf \
33+
--log-level=DEBUG \
34+
--access-logfile=- \
35+
--error-logfile=- \
36+
run:application
3137

3238
db-migrations:
3339
@echo "+ $@"

‎packages/ml_api/api/app.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from api.monitoring.middleware import setup_metrics
88
from api.persistence.core import init_database
99

10-
_logger = logging.getLogger(__name__)
10+
_logger = logging.getLogger('mlapi')
1111

1212

1313
def create_app(

‎packages/ml_api/api/config.py‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@
22
import os
33
import pathlib
44
import sys
5+
from logging.config import fileConfig
56

67
import api
78

8-
99
# logging format
1010
FORMATTER = logging.Formatter(
1111
"%(asctime)s — %(name)s — %(levelname)s —" "%(funcName)s:%(lineno)d — %(message)s"
@@ -82,10 +82,9 @@ def get_console_handler():
8282
def setup_app_logging(config: Config) -> None:
8383
"""Prepare custom logging for our application."""
8484
_disable_irrelevant_loggers()
85-
root = logging.getLogger()
86-
root.setLevel(config.LOGGING_LEVEL)
87-
root.addHandler(get_console_handler())
88-
root.propagate = False
85+
fileConfig(ROOT / 'gunicorn_logging.conf')
86+
logger = logging.getLogger('mlapi')
87+
logger.setLevel(config.LOGGING_LEVEL)
8988

9089

9190
def _disable_irrelevant_loggers() -> None:

‎packages/ml_api/api/controller.py‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,16 @@
33
import threading
44

55
from flask import request, jsonify, Response, current_app
6+
from prometheus_client import Histogram, Gauge, Info
7+
from regression_model import __version__ as live_version
68

9+
from api.config import APP_NAME
10+
from api.persistence.data_access import PredictionPersistence, ModelType
711
from gradient_boosting_model import __version__ as shadow_version
8-
from regression_model import __version__ as live_version
9-
from prometheus_client import Histogram, Gauge, Info
1012
from gradient_boosting_model.predict import make_prediction
11-
from api.persistence.data_access import PredictionPersistence, ModelType
12-
from api.config import APP_NAME
1313

14+
_logger = logging.getLogger('mlapi')
1415

15-
_logger = logging.getLogger(__name__)
1616

1717
PREDICTION_TRACKER = Histogram(
1818
name='house_price_prediction_dollars',
@@ -45,13 +45,18 @@
4545

4646
def health():
4747
if request.method == "GET":
48-
return jsonify({"status": "ok"})
48+
status = {"status": "ok"}
49+
_logger.debug(status)
50+
return jsonify(status)
4951

5052

5153
def predict():
5254
if request.method == "POST":
5355
# Step 1: Extract POST data from request body as JSON
5456
json_data = request.get_json()
57+
_logger.info(
58+
f'Inputs for model: {ModelType.LASSO.name} '
59+
f'Input values: {json_data}')
5560

5661
# Step 2a: Get and save live model predictions
5762
persistence = PredictionPersistence(db_session=current_app.db_session)
@@ -89,6 +94,10 @@ def predict():
8994
app_name=APP_NAME,
9095
model_name=ModelType.LASSO.name,
9196
model_version=live_version).set(_prediction)
97+
_logger.info(
98+
f'Prediction results for model: {ModelType.LASSO.name} '
99+
f'version: {result.model_version} '
100+
f'Output values: {result.predictions}')
92101

93102
# Step 5: Prepare prediction response
94103
return jsonify(

‎packages/ml_api/api/persistence/core.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
from api.config import Config, ROOT
1313

14-
_logger = logging.getLogger(__name__)
14+
_logger = logging.getLogger('mlapi')
1515

1616
# Base class for SQLAlchemy models
1717
Base = declarative_base()

‎packages/ml_api/api/persistence/data_access.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,23 +5,24 @@
55

66
import numpy as np
77
import pandas as pd
8-
from gradient_boosting_model.predict import make_prediction as make_shadow_prediction
98
from regression_model.predict import make_prediction as make_live_prediction
109
from sqlalchemy.orm.session import Session
1110

1211
from api.persistence.models import (
1312
LassoModelPredictions,
1413
GradientBoostingModelPredictions,
1514
)
15+
from gradient_boosting_model.predict import make_prediction as make_shadow_prediction
16+
17+
_logger = logging.getLogger('mlapi')
18+
1619

1720
SECONDARY_VARIABLES_TO_RENAME = {
1821
"FirstFlrSF": "1stFlrSF",
1922
"SecondFlrSF": "2ndFlrSF",
2023
"ThreeSsnPortch": "3SsnPorch",
2124
}
2225

23-
_logger = logging.getLogger(__name__)
24-
2526

2627
class ModelType(enum.Enum):
2728
LASSO = "lasso"

‎packages/ml_api/docker/docker-compose-elk.yml‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
version: '3.2'
2+
services:
3+
ml_api:
4+
build:
5+
context: ../
6+
dockerfile: docker/Dockerfile
7+
environment:
8+
DB_HOST: database
9+
DB_PORT: 5432
10+
DB_USER: user
11+
DB_PASSWORD: ${DB_PASSWORD:-password}
12+
DB_NAME: ml_api_dev
13+
networks:
14+
- elk
15+
depends_on:
16+
- database
17+
- logstash
18+
ports:
19+
- "5000:5000" # expose webserver to localhost host:container
20+
command: bash -c "make db-migrations && make run-service-wsgi"
21+
22+
database:
23+
image: postgres:latest
24+
environment:
25+
POSTGRES_USER: user
26+
POSTGRES_PASSWORD: password
27+
POSTGRES_DB: ml_api_dev
28+
ports:
29+
# expose postgres container on different host port to default (host:container)
30+
- "6609:5432"
31+
volumes:
32+
- my_dbdata:/var/lib/postgresql/data
33+
networks:
34+
- elk
35+
36+
elasticsearch:
37+
image: docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
38+
volumes:
39+
- type: bind
40+
source: ./elasticsearch/config/elasticsearch.yml
41+
target: /usr/share/elasticsearch/config/elasticsearch.yml
42+
read_only: true
43+
- type: volume
44+
source: elasticsearch
45+
target: /usr/share/elasticsearch/data
46+
ports:
47+
- "9200:9200"
48+
- "9300:9300"
49+
environment:
50+
ES_JAVA_OPTS: "-Xmx256m -Xms256m"
51+
ELASTIC_PASSWORD: changeme
52+
# Use single node discovery in order to disable production mode and avoid bootstrap checks
53+
# see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html
54+
discovery.type: single-node
55+
networks:
56+
- elk
57+
58+
logstash:
59+
image: docker.elastic.co/logstash/logstash:${ELK_VERSION}
60+
volumes:
61+
- type: bind
62+
source: ./logstash/config/logstash.yml
63+
target: /usr/share/logstash/config/logstash.yml
64+
read_only: true
65+
- type: bind
66+
source: ./logstash/pipeline
67+
target: /usr/share/logstash/pipeline
68+
read_only: true
69+
ports:
70+
- "5001:5001"
71+
- "9600:9600"
72+
environment:
73+
LS_JAVA_OPTS: "-Xmx256m -Xms256m"
74+
networks:
75+
- elk
76+
depends_on:
77+
- elasticsearch
78+
79+
kibana:
80+
image: docker.elastic.co/kibana/kibana:${ELK_VERSION}
81+
volumes:
82+
- type: bind
83+
source: ./kibana/config/kibana.yml
84+
target: /usr/share/kibana/config/kibana.yml
85+
read_only: true
86+
ports:
87+
- "5601:5601"
88+
networks:
89+
- elk
90+
depends_on:
91+
- elasticsearch
92+
93+
networks:
94+
elk:
95+
driver: bridge
96+
97+
volumes:
98+
my_dbdata:
99+
elasticsearch:

‎packages/ml_api/docker/docker-compose.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
version: '3'
22
services:
33
ml_api:
4-
image: christophergs/ml_api:master
54
build:
65
context: ../
76
dockerfile: docker/Dockerfile

‎packages/ml_api/docker/elasticsearch/config/elasticsearch.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
## Default Elasticsearch configuration from Elasticsearch base image.
2+
## https://github.com/elastic/elasticsearch/blob/master/distribution/docker/src/docker/config/elasticsearch.yml
3+
cluster.name: "docker-cluster"
4+
network.host: 0.0.0.0
5+
6+
## X-Pack settings
7+
## see https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-xpack.html
8+
xpack.license.self_generated.type: basic
9+
xpack.security.enabled: true
10+
xpack.monitoring.collection.enabled: true

‎packages/ml_api/docker/kibana/config/kibana.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
---
2+
## Default Kibana configuration from Kibana base image.
3+
## https://github.com/elastic/kibana/blob/master/src/dev/build/tasks/os_packages/docker_generator/templates/kibana_yml.template.js
4+
5+
server.name: kibana
6+
server.host: "0"
7+
elasticsearch.hosts: [ "http://elasticsearch:9200" ]
8+
xpack.monitoring.ui.container.elasticsearch.enabled: true
9+
10+
## X-Pack security credentials
11+
elasticsearch.username: elastic
12+
elasticsearch.password: changeme

0 commit comments

Comments
 (0)