Skip to content

Commit

Permalink
Remove parser and rewrite command fill-db (#554)
Browse files Browse the repository at this point in the history
  • Loading branch information
@InKLaR1TY
InKLaR1TY authored Oct 1, 2024
1 parent 2ff17bf commit 74e89cf
Show file tree
Hide file tree
Showing 12 changed files with 111 additions and 1,058 deletions.
4 changes: 2 additions & 2 deletions .pre-commit-config.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.5.0
rev: v4.6.0
hooks:
- id: check-docstring-first
- id: check-merge-conflict
- id: end-of-file-fixer
- id: trailing-whitespace

- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.4.5
rev: v0.6.8
hooks:
- id: ruff
exclude: migrations/|config/|tests/|.*settings(\.py|/)?
Expand Down
155 changes: 106 additions & 49 deletions adaptive_hockey_federation/core/management/commands/fill-db.py
View file
Original file line number Diff line number Diff line change
@@ -1,83 +1,140 @@
import json

from django.apps import apps
from django.core.management.base import BaseCommand
from django.db import connection, transaction
from main.models import Diagnosis

from adaptive_hockey_federation.core.config.dev_settings import (
FILE_MODEL_MAP,
FIXSTURES_DIR,
FIXSTURES_FILE,
)
from adaptive_hockey_federation.parser.importing_db import (
clear_data_db,
importing_parser_data_db,
importing_real_data_db,
)

DB_MESSAGE = "Данные успешно добавлены!"


class Command(BaseCommand):
"""Класс для парсинга данных и их записи в БД."""

help = "Запуск парсера офисных документов, и запись их в БД."

def add_arguments(self, parser):
"""Добавляет новые аргументы для командной строки."""
parser.add_argument(
"-p",
"--parser",
action="store_true",
help="Запуск парсера документов",
)
"""Аргументы."""
parser.add_argument(
"-f",
"--fixtures",
action="store_true",
help="Фикстуры с реальными данными для таблиц.",
)

def load_data(self) -> None:
"""Загрузка распарсенных данных."""
importing_parser_data_db(FIXSTURES_FILE)
return None

def load_real_data(self) -> None:
@transaction.atomic
def load_real_data(self) -> None: # noqa: C901
"""Загрузка реальных данных из JSON."""
for key in FILE_MODEL_MAP.items():
file_name = key[0] + ".json"
if "main_" in key[0]:
with connection.cursor() as cursor:
for table_name in FILE_MODEL_MAP.keys():
try:
clear_data_db(file_name)
cursor.execute(f"TRUNCATE TABLE {table_name} CASCADE")
except Exception as e:
self.stdout.write(
self.style.ERROR(
f"Ошибка удаления данных {e} -> " f"{file_name}",
return self.stdout.write(
self.style.WARNING(
f"Не удалось очистить таблицу {table_name}: {str(e)}", # noqa: E501
),
)

items = list(FILE_MODEL_MAP.items())
items.reverse()
for key in items:
file_name = key[0] + ".json"
if "main_" in key[0]:
try:
importing_real_data_db(FIXSTURES_DIR, file_name)
self.stdout.write(
self.style.SUCCESS(
f"Фикстуры с файла {file_name} вставлены "
"в таблицы!",
),
)
except Exception as e:
return self.stdout.write(
self.style.ERROR_OUTPUT(
f"Ошибка вставки данных {e} -> " f"{file_name}",
),
)
return None
for table_name, model_class in items:
file_path = FIXSTURES_DIR / f"{table_name}.json"
app_label, model_name = table_name.split("_", 1)
model = apps.get_model(app_label, model_name)
try:
with open(file_path, "r", encoding="utf-8") as file:
data = json.load(file)

for item in data:
if table_name == "main_team":
team_data = {
"id": item["id"],
"name": item["name"],
"city_id": item["city_id"],
"discipline_name_id": item["discipline_name_id"],
"curator_id": 1,
}
instance = model(**team_data)
elif table_name == "main_player":
disciplines = self.get_disciplines()
diagnosis = None
if item.get("diagnosis_id"):
try:
diagnosis = Diagnosis.objects.get(
pk=item["diagnosis_id"],
)
except Diagnosis.DoesNotExist:
print(
f"Диагноз с id {item.get('diagnosis_id')} отсутсвует.", # noqa: E501
)
player_data = {
"id": item["id"],
"surname": item["surname"],
"name": item["name"],
"patronymic": item["patronymic"],
"birthday": item["birthday"],
"gender": item["gender"],
"level_revision": item["level_revision"],
"position": item["position"],
"number": item["number"],
"is_captain": item["is_captain"],
"is_assistent": item["is_assistent"],
"identity_document": item["identity_document"],
"diagnosis": diagnosis,
"diagnosis_id": item["diagnosis_id"],
"discipline_name_id": disciplines[
item["discipline_id"]
]["discipline_name_id"],
"discipline_level_id": disciplines[
item["discipline_id"]
]["discipline_level_id"],
}
instance = model(**player_data)
else:
instance = model(**item)
instance.save()

self.stdout.write(
self.style.SUCCESS(
f"Данные из {table_name}.json успешно загружены в модель {model_class}", # noqa: E501
),
)

except FileNotFoundError:
self.stdout.write(
self.style.WARNING(f"Файл {table_name}.json не найден"),
)
except Exception as e:
self.stdout.write(
self.style.ERROR(
f"Ошибка при загрузке данных из {table_name}.json: {str(e)}", # noqa: E501
),
)

def handle(self, *args, **options):
"""Запись данных в БД."""
parser = options.get("parser")
fixtures = options.get("fixtures")
if fixtures:
self.load_real_data()
if parser:
self.load_data()

def get_disciplines(self) -> dict:
"""Получение диспциплин."""
with open(
FIXSTURES_DIR / "main_discipline.json",
"r",
encoding="utf-8",
) as file:
data = json.load(file)
disciplines = {
None: {"discipline_level_id": None, "discipline_name_id": None},
}
for item in data:
disciplines[item["id"]] = {
"discipline_level_id": item["discipline_level_id"],
"discipline_name_id": item["discipline_name_id"],
}
return disciplines
Empty file removed adaptive_hockey_federation/parser/__init__.py
View file
Empty file.
Loading

0 comments on commit 74e89cf

Please sign in to comment.