forked from crocs-muni/rtt-deployment
-
Notifications
You must be signed in to change notification settings - Fork 0
/
deploy_storage.py
executable file
·234 lines (200 loc) · 10.2 KB
/
deploy_storage.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
#! /usr/bin/python3
import configparser
import argparse
import traceback
from common.rtt_deploy_utils import *
from common.rtt_constants import *
################################
# Global variables declaration #
################################
def main():
parser = argparse.ArgumentParser(description='Storage deployment')
parser.add_argument('--docker', dest='docker', action='store_const', const=True, default=False,
help='Docker deployment')
parser.add_argument('--local-db', dest='local_db', action='store_const', const=True, default=False,
help='DB server is on the same machine')
parser.add_argument('--mysql-pass', dest='mysql_pass', action='store_const', const=True, default=False,
help='DB password to use')
parser.add_argument('--mysql-pass-file', dest='mysql_pass_file', action='store_const', const=True, default=False,
help='DB password file to use')
parser.add_argument('--sys-python', dest='sys_python', action='store_const', const=True, default=False,
help='Use system python')
parser.add_argument('-j', dest='buildj', type=int, default=2,
help='Parallel compilation')
parser.add_argument('--config', dest='config', default='deployment_settings.ini',
help='Path to deployment_settings.ini')
args = parser.parse_args()
deploy_cfg_file = args.config
deploy_cfg = configparser.ConfigParser()
try:
deploy_cfg.read(deploy_cfg_file)
if len(deploy_cfg.sections()) == 0:
raise FileNotFoundError("can't read: {}".format(deploy_cfg_file))
Database.address = get_no_empty(deploy_cfg, "Database", "IPv4-Address")
Database.ssh_root_user = get_no_empty(deploy_cfg, "Database", "SSH-Root-User")
Database.ssh_port = get_no_empty(deploy_cfg, "Database", "SSH-Port")
Database.mysql_port = get_no_empty(deploy_cfg, "Database", "MySql-Port")
Storage.address = get_no_empty(deploy_cfg, "Storage", "IPv4-Address")
Storage.rtt_dir = get_no_empty(deploy_cfg, "Storage", "RTT-Files-dir")
Storage.ssh_config = get_no_empty(deploy_cfg, "Storage", "SSH-Config")
Storage.acc_name = get_no_empty(deploy_cfg, "Storage", "Storage-User")
Storage.acc_chroot = get_no_empty(deploy_cfg, "Storage", "Storage-Chroot")
except BaseException as e:
print_error("Configuration file: {}".format(e))
sys.exit(1)
# Sanity checks
try:
check_paths_abs({
Storage.acc_chroot,
Storage.CHROOT_HOME_DIR,
Storage.rtt_dir
})
check_paths_rel({
Storage.CHROOT_CONF_DIR,
Storage.CHROOT_DATA_DIR,
Storage.SSH_DIR,
Storage.COMMON_FILES_DIR,
Storage.CREDENTIALS_DIR
})
check_files_exists({
CommonConst.STORAGE_CLEAN_CACHE
})
except AssertionError as e:
print_error("Invalid configuration. {}".format(e))
sys.exit(1)
# Declaring all variables that
# will be used in this script.
# These are mostly absolute paths put
# together from the settings.
# Path related to storage user home
Storage.home_dir = \
"{}{}".format(Storage.acc_chroot, Storage.CHROOT_HOME_DIR)
Storage.ssh_dir = \
os.path.join(Storage.home_dir, Storage.SSH_DIR)
Storage.authorized_keys_file = \
os.path.join(Storage.ssh_dir, Storage.AUTH_KEYS_FILE)
Storage.data_dir = \
os.path.join(Storage.home_dir, Storage.CHROOT_DATA_DIR)
Storage.config_dir = \
os.path.join(Storage.home_dir, Storage.CHROOT_CONF_DIR)
# Paths related to rtt files on server
Storage.rtt_file_store_ini = \
os.path.join(Storage.rtt_dir, Storage.STORE_CONFIG_FILE)
Storage.rtt_file_clean_cache = \
os.path.join(Storage.rtt_dir, Storage.CLEAN_CACHE_SCRIPT)
Storage.rtt_file_clean_cache_log = \
os.path.join(Storage.rtt_dir, Storage.CLEAN_CACHE_LOG)
Storage.rtt_common_dir = \
os.path.join(Storage.rtt_dir, Storage.COMMON_FILES_DIR)
Storage.rtt_credentials_dir \
= os.path.join(Storage.rtt_dir, Storage.CREDENTIALS_DIR)
Storage.rtt_file_mysql_cred = \
os.path.join(Storage.rtt_credentials_dir, Storage.MYSQL_CREDENTIALS_FILE)
try:
install_debian_pkgs(["acl", "sudo", "wget", "unzip", "rsync", "cron", "openssh-client"])
install_debian_pkg("openssh-server")
# Creating sftp jail for account
# Adding rtt-admin group that is intended to manage
# directories and files related to rtt without root access
exec_sys_call_check("groupadd {}".format(Storage.RTT_ADMIN_GROUP),
acc_codes=[0, 9])
# Adding user for access
exec_sys_call_check("useradd -d {} -s /usr/sbin/nologin {}"
.format(Storage.CHROOT_HOME_DIR, Storage.acc_name),
acc_codes=[0, 9])
# Configuring ssh server
sshd_config_append = "\n\n" \
"\tClientAliveInterval 120\n\n" \
"Match User {0}\n" \
"\tChrootDirectory {1}\n" \
"\tForceCommand internal-sftp\n" \
"\tAllowTcpForwarding yes\n" \
"\tPermitTunnel yes\n" \
"\tX11Forwarding no\n" \
"\tAuthorizedKeysFile {1}{2}\n" \
"\tPasswordAuthentication no\n" \
"\n".format(Storage.acc_name, Storage.acc_chroot,
os.path.join(Storage.CHROOT_HOME_DIR,
Storage.SSH_DIR, Storage.AUTH_KEYS_FILE))
with open(Storage.ssh_config, "a") as f:
f.write(sshd_config_append)
exec_sys_call_check("service ssh restart")
# Creating sftp jail for accessing storage
create_dir(Storage.acc_chroot, 0o755)
create_dir(Storage.home_dir, 0o700,
own=Storage.acc_name, grp=Storage.acc_name)
create_dir(Storage.ssh_dir, 0o700,
own=Storage.acc_name, grp=Storage.acc_name)
create_dir(Storage.data_dir, 0o700,
own=Storage.acc_name, grp=Storage.acc_name)
create_dir(Storage.config_dir, 0o700,
own=Storage.acc_name, grp=Storage.acc_name)
create_file(Storage.authorized_keys_file,
0o600, own=Storage.acc_name, grp=Storage.acc_name)
# Creating directory for rtt files on the server
create_dir(Storage.rtt_dir, 0o2770, grp=Storage.RTT_ADMIN_GROUP)
# Set ACL on top directory - ensures all new files will have correct permissions
exec_sys_call_check("setfacl -R -d -m g::rwx {}".format(Storage.rtt_dir))
exec_sys_call_check("setfacl -R -d -m o::--- {}".format(Storage.rtt_dir))
create_dir(Storage.rtt_credentials_dir, 0o2770,
grp=Storage.RTT_ADMIN_GROUP)
# Copying script for cache cleaning
shutil.copy(CommonConst.STORAGE_CLEAN_CACHE, Storage.rtt_file_clean_cache)
chmod_chown(Storage.rtt_file_clean_cache, 0o770,
grp=Storage.RTT_ADMIN_GROUP)
# Copying common scripts into directory
# with rtt files
if os.path.exists(Storage.rtt_common_dir):
shutil.rmtree(Storage.rtt_common_dir)
shutil.copytree(CommonConst.COMMON_FILES_DIR, Storage.rtt_common_dir)
recursive_chmod_chown(Storage.rtt_common_dir, mod_f=0o660, mod_d=0o2770,
grp=Storage.RTT_ADMIN_GROUP)
# Creating configuration file for storage server scripts
ini_cfg = configparser.ConfigParser()
ini_cfg.add_section("MySQL-Database")
ini_cfg.set("MySQL-Database", "Name", Database.MYSQL_DB_NAME)
ini_cfg.set("MySQL-Database", "Address", Database.address)
ini_cfg.set("MySQL-Database", "Port", Database.mysql_port)
ini_cfg.set("MySQL-Database", "Credentials-file", Storage.rtt_file_mysql_cred)
ini_cfg.add_section("Local-cache")
ini_cfg.set("Local-cache", "Data-directory", Storage.data_dir)
ini_cfg.set("Local-cache", "Config-directory", Storage.config_dir)
with open(Storage.rtt_file_store_ini, "w") as f:
ini_cfg.write(f)
# Creating credentials file for database access
cred_db_password = get_rnd_pwd()
write_db_credentials(Storage.MYSQL_STORAGE_USER, cred_db_password, Storage.rtt_file_mysql_cred)
# Installing required packages
install_debian_pkg("libmysqlcppconn-dev")
install_debian_pkg_at_least_one(["default-libmysqlclient-dev", "libmysqlclient-dev"])
python3, pip3 = setup_python3(use_system=args.sys_python, buildj=args.buildj)
install_python_pkg("pip", no_cache=False, pip3=pip3)
install_python_pkgs(["mysqlclient", "paramiko"], pip3=pip3)
# This can be done only after installing cryptography and paramiko
from common.rtt_registration import register_db_user
db_def_passwd = get_mysql_password_args(args)
register_db_user(Database.ssh_root_user, Database.address, Database.ssh_port,
Storage.MYSQL_STORAGE_USER, cred_db_password, Storage.address,
Database.MYSQL_ROOT_USERNAME, Database.MYSQL_DB_NAME,
priv_select=True,
db_def_passwd=db_def_passwd, db_no_pass=args.local_db)
# Adding new job to cron - cache cleaning script
install_debian_pkg("cron")
add_cron_job(Storage.rtt_file_clean_cache,
Storage.rtt_file_store_ini,
Storage.rtt_file_clean_cache_log,
python3=python3)
exec_sys_call_check("service cron restart")
if not args.docker:
service_enable("ssh.service")
service_enable("cron.service")
# All configured here.
except BaseException as e:
print_error("{}. Fix error and run the script again.".format(e))
traceback.print_exc()
return 2
if __name__ == "__main__":
print_start("deploy_storage")
r = main() or 0
print_end()
sys.exit(r)