-
Notifications
You must be signed in to change notification settings - Fork 15
/
Makefile
171 lines (135 loc) · 5.51 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
include src/.env
# Tests
test-confluence:
poetry run pytest tests/test_confluence.py
test:
poetry run pytest
# Local development
init-data:
poetry run python -m upload_data.data_init
update-data:
poetry run python -m upload_data.data_update
open-notebook:
poetry run ipython kernel install --user --name=rag-evaluation
poetry run jupyter lab
docker-build:
poetry export -f requirements.txt -o requirements.txt --without-hashes
docker buildx build --platform linux/amd64 --push -t europe-west1-docker.pkg.dev/$(PROJECT_ID)/rag-api/gen-ai:latest .
demo:
docker build -t gcp-rag-streamlit:latest .
docker run -p 8501:8501 gcp-rag-streamlit
# Service account
create-sa-with-roles: create-sa create-roles add-user-to-impersonate-sa enable-gcp-services allow-user-to-get-access-token-from-sa
create-sa:
gcloud iam service-accounts create gen-ai --display-name="rag-gen-ai-sa"
create-roles:
gcloud projects add-iam-policy-binding $(PROJECT_ID) \
--member="serviceAccount:gen-ai@$(PROJECT_ID).iam.gserviceaccount.com" \
--role="roles/aiplatform.user"
gcloud projects add-iam-policy-binding $(PROJECT_ID) \
--member="serviceAccount:gen-ai@$(PROJECT_ID).iam.gserviceaccount.com" \
--role="roles/datastore.user"
gcloud projects add-iam-policy-binding $(PROJECT_ID) \
--member="serviceAccount:gen-ai@$(PROJECT_ID).iam.gserviceaccount.com" \
--role="roles/iam.serviceAccountUser"
gcloud projects add-iam-policy-binding $(PROJECT_ID) \
--member="serviceAccount:gen-ai@$(PROJECT_ID).iam.gserviceaccount.com" \
--role="roles/run.admin"
gcloud projects add-iam-policy-binding $(PROJECT_ID) \
--member="serviceAccount:gen-ai@$(PROJECT_ID).iam.gserviceaccount.com" \
--role="roles/storage.admin"
gcloud projects add-iam-policy-binding $(PROJECT_ID) \
--member="serviceAccount:gen-ai@$(PROJECT_ID).iam.gserviceaccount.com" \
--role="roles/storage.objectAdmin"
gcloud projects add-iam-policy-binding $(PROJECT_ID) \
--member="serviceAccount:gen-ai@$(PROJECT_ID).iam.gserviceaccount.com" \
--role="roles/storage.objectUser"
gcloud projects add-iam-policy-binding $(PROJECT_ID) \
--member="serviceAccount:gen-ai@$(PROJECT_ID).iam.gserviceaccount.com" \
--role="roles/bigquery.dataOwner"
gcloud projects add-iam-policy-binding $(PROJECT_ID) \
--member="serviceAccount:gen-ai@$(PROJECT_ID).iam.gserviceaccount.com" \
--role="roles/pubsub.editor"
gcloud projects add-iam-policy-binding $(PROJECT_ID) \
--member="serviceAccount:gen-ai@$(PROJECT_ID).iam.gserviceaccount.com" \
--role="roles/iam.securityReviewer"
gcloud projects add-iam-policy-binding $(PROJECT_ID) \
--member="serviceAccount:gen-ai@$(PROJECT_ID).iam.gserviceaccount.com" \
--role="roles/iam.securityAdmin"
add-user-to-impersonate-sa:
gcloud iam service-accounts add-iam-policy-binding \
gen-ai@$(PROJECT_ID).iam.gserviceaccount.com\
--member="user:$(USER_EMAIL)" \
--role="roles/iam.serviceAccountUser"
enable-gcp-services:
gcloud services enable cloudresourcemanager.googleapis.com
gcloud services enable serviceusage.googleapis.com
gcloud services enable cloudbuild.googleapis.com
gcloud services enable secretmanager.googleapis.com
gcloud services enable run.googleapis.com
gcloud services enable aiplatform.googleapis.com
gcloud services enable firebaserules.googleapis.com
gcloud services enable firestore.googleapis.com
gcloud services enable ml.googleapis.com
gcloud services enable artifactregistry.googleapis.com
gcloud services enable bigquery.googleapis.com
gcloud services enable bigquerymigration.googleapis.com
gcloud services enable bigquerystorage.googleapis.com
allow-user-to-get-access-token-from-sa:
gcloud projects add-iam-policy-binding $(PROJECT_NUMBER) \
--member="user:$(USER_EMAIL)" --role=roles/iam.workloadIdentityUser \
--condition=None
# Impersonation
impersonate-service-account:
gcloud auth application-default login --impersonate-service-account gen-ai@$(PROJECT_ID).iam.gserviceaccount.com
# Docker
create-artefact-registry:
gcloud artifacts repositories create gen-ai \
--repository-format=docker \
--location=europe-west1 \
--description="Generative Ai Rag" \
--async
# Helpers
list-sa-roles:
gcloud projects get-iam-policy $(PROJECT_ID) \
--flatten="bindings[].members" \
--format='table(bindings.role)' \
--filter="bindings.members:gen-ai@$(PROJECT_ID).iam.gserviceaccount.com"
describe-index:
gcloud ai indexes describe $(INDEX_ID) --project=$(PROJECT_ID) --region=$(REGION)
# Vertex AI Vector Search
# Create
create-index:
gcloud ai indexes create \
--metadata-file="./vertex_search_index_metadata.json" \
--display-name=rag-index \
--project=$(PROJECT_ID) \
--region=$(REGION)
create-index-endpoint:
gcloud ai index-endpoints create \
--display-name=rag-endpoint \
--project=$(PROJECT_ID) \
--region=$(REGION)
deploy-vertex-endpoint:
gcloud ai index-endpoints deploy-index $(INDEX_ENDPOINT_ID) \
--index=$(INDEX_ID) \
--deployed-index-id=$(DEPLOYED_INDEX_ID) \
--display-name=ragendpoint \
--region $(REGION)
# Update
update-index:
gcloud ai indexes update $(INDEX_ID) \
--metadata-file=$(INDEX_METADATA_JSON_PATH) \
--region=$(REGION)
# Delete
delete-vector-search: undeploy-index delete-index_endpoint delete_index
undeploy-index:
gcloud ai index-endpoints undeploy-index $(INDEX_ENDPOINT_ID) --project=$(PROJECT_ID) \
--region=europe-west1 --deployed-index-id=$(DEPLOYED_INDEX_ID)
delete-index-endpoint:
gcloud ai index-endpoints delete $(INDEX_ENDPOINT_ID) \
--region=europe-west1
delete-index:
gcloud ai indexes delete $(INDEX_ID) \
--project=$(PROJECT_ID) \
--region=$(REGION)