-
Notifications
You must be signed in to change notification settings - Fork 0
/
Makefile
126 lines (96 loc) · 3.67 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
#################################################################################
# GLOBALS #
#################################################################################
PROJECT_NAME=mlflow-practice
PYTHON_VERSION=3.10
PYTHON_INTERPRETER=python
POSTGRES_DB=mlflow
POSTGRES_USER=user
POSTGRES_PASSWORD=password
POSTGRES_HOST=localhost
POSTGRES_PORT=5432
POSTGRES_URI=postgresql://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@$(POSTGRES_HOST):$(POSTGRES_PORT)/$(POSTGRES_DB)
MLFLOW_S3_ENDPOINT_URL=http://localhost:9000
AWS_ACCESS_KEY_ID=minioadmin
AWS_SECRET_ACCESS_KEY=minioadmin
AWS_BUCKET=mlruns
#################################################################################
# COMMANDS #
#################################################################################
## Start mlfow tracking server
.PHONY: mlflow-server
mlflow-server:
POSTGRES_DB=$(POSTGRES_DB) \
POSTGRES_USER=$(POSTGRES_USER) \
POSTGRES_PASSWORD=$(POSTGRES_PASSWORD) \
AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) \
AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) \
AWS_BUCKET=$(AWS_BUCKET) \
docker compose --profile frontend up -d
.PHONY: tracking-storage
tracking-storage:
echo $(POSTGRES_DB)
POSTGRES_DB=$(POSTGRES_DB) \
POSTGRES_USER=$(POSTGRES_USER) \
POSTGRES_PASSWORD=$(POSTGRES_PASSWORD) \
AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) \
AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) \
AWS_BUCKET=$(AWS_BUCKET) \
docker compose --profile backend up -d
.PHONY: mlflow-plus-storage
mlflow-plus-storage: tracking-storage mlflow-server
## Install Python Dependencies
.PHONY: requirements
requirements:
conda env update --name $(PROJECT_NAME) --file environment.yml --prune
## Delete all compiled Python files
.PHONY: clean
clean:
find . -type f -name "*.py[co]" -delete
find . -type d -name "__pycache__" -delete
## Lint using flake8 and black (use `make format` to do formatting)
.PHONY: lint
lint:
flake8 mlflow_practice
isort --check --diff --profile black mlflow_practice
black --check --config pyproject.toml mlflow_practice
## Format source code with black
.PHONY: format
format:
black --config pyproject.toml mlflow_practice
## Download Data from storage system
.PHONY: sync_data_down
sync_data_down:
aws s3 sync s3://mlflow/data/\
data/
## Upload Data to storage system
.PHONY: sync_data_up
sync_data_up:
aws s3 sync s3://mlflow/data/ data/\
--profile $(PROFILE)
## Set up python interpreter environment
.PHONY: create_environment
create_environment:
conda env create --name $(PROJECT_NAME) -f environment.yml
@echo ">>> conda env created. Activate with:\nconda activate $(PROJECT_NAME)"
#################################################################################
# PROJECT RULES #
#################################################################################
## Make Dataset
.PHONY: data
data: requirements
$(PYTHON_INTERPRETER) mlflow_practice/data/make_dataset.py
#################################################################################
# Self Documenting Commands #
#################################################################################
.DEFAULT_GOAL := help
define PRINT_HELP_PYSCRIPT
import re, sys; \
lines = '\n'.join([line for line in sys.stdin]); \
matches = re.findall(r'\n## (.*)\n[\s\S]+?\n([a-zA-Z_-]+):', lines); \
print('Available rules:\n'); \
print('\n'.join(['{:25}{}'.format(*reversed(match)) for match in matches]))
endef
export PRINT_HELP_PYSCRIPT
help:
@python -c "${PRINT_HELP_PYSCRIPT}" < $(MAKEFILE_LIST)