From 723a865f14be5a56a65a39e47d565787b28ac47a Mon Sep 17 00:00:00 2001 From: alexkiro <1538458+alexkiro@users.noreply.github.com> Date: Wed, 28 Aug 2024 08:32:22 +0300 Subject: [PATCH] Add reference period and remarks fields for facts (#102) * Rename existing remarks field In preparation for adding the remark dimension field * Add new fields for fact and populate them from extra notes * Add support for importing the new fields via uploads * Add tests for new field uploads and XLS uplods * Add the ability to import the two new fields from ESTAT * Add tests for ESTAT imports of the new fields * Include the new fields in the API and exports * Include remarks in the tooltips * Don't migrate extra chart notes automatically Use a management command instead * Include reference period in the tooltip --- digital_agenda/apps/charts/models.py | 2 +- digital_agenda/apps/core/formats.py | 165 +++++++++++------- .../commands/migrate_extra_notes.py | 21 +++ ...0016_fact_reference_period_fact_remarks.py | 26 +++ digital_agenda/apps/core/models.py | 16 +- digital_agenda/apps/core/serializers.py | 2 + digital_agenda/apps/core/static/admin.css | 18 +- .../apps/core/static/import_file_example.xlsx | Bin 4930 -> 5341 bytes digital_agenda/apps/core/views/facts.py | 6 +- digital_agenda/apps/estat/admin.py | 33 +++- digital_agenda/apps/estat/importer.py | 44 ++++- ...remarks_importconfig_additional_remarks.py | 18 ++ ..._importconfig_reference_period_and_more.py | 38 ++++ digital_agenda/apps/estat/models.py | 10 +- digital_agenda/common/export.py | 16 +- fixtures/importconfig.json | 152 ++++++++-------- fixtures/test/facts.json | 8 +- frontend/cypress/e2e/admin/fileImport.cy.js | 87 ++++----- frontend/cypress/e2e/admin/importConfig.cy.js | 21 ++- ...-one-indicator-and-compare-countries.cy.js | 31 ++-- .../cypress/fixtures/import_file_valid.xls | Bin 0 -> 6144 bytes .../cypress/fixtures/import_file_valid.xlsx | Bin 4930 -> 5365 bytes .../import_file_valid_no_remarks.xlsx | Bin 0 -> 5305 bytes .../src/components/charts/base/BaseChart.vue | 10 ++ .../src/views/chart-group/MetadataView.vue | 20 +++ 25 files changed, 514 insertions(+), 230 deletions(-) create mode 100644 digital_agenda/apps/core/management/commands/migrate_extra_notes.py create mode 100644 digital_agenda/apps/core/migrations/0016_fact_reference_period_fact_remarks.py create mode 100644 digital_agenda/apps/estat/migrations/0014_rename_remarks_importconfig_additional_remarks.py create mode 100644 digital_agenda/apps/estat/migrations/0015_importconfig_reference_period_and_more.py create mode 100644 frontend/cypress/fixtures/import_file_valid.xls create mode 100644 frontend/cypress/fixtures/import_file_valid_no_remarks.xlsx diff --git a/digital_agenda/apps/charts/models.py b/digital_agenda/apps/charts/models.py index 2c9541cc..975ee5c2 100644 --- a/digital_agenda/apps/charts/models.py +++ b/digital_agenda/apps/charts/models.py @@ -81,7 +81,7 @@ def clean(self): raise ValidationError({"period_start": error, "period_end": error}) def get_label(self, dimension): - return getattr(self, dimension + "_label", dimension.title()) + return getattr(self, dimension + "_label", dimension.title().replace("_", " ")) @property def facts(self): diff --git a/digital_agenda/apps/core/formats.py b/digital_agenda/apps/core/formats.py index 090324a2..1833a882 100644 --- a/digital_agenda/apps/core/formats.py +++ b/digital_agenda/apps/core/formats.py @@ -13,7 +13,24 @@ logger = logging.getLogger(__name__) - +UNIQUE_EXCEL_COLS = ( + "period", + "country", + "indicator", + "breakdown", + "unit", +) +VALID_EXCEL_COLS = ( + "period", + "country", + "indicator", + "breakdown", + "unit", + "value", + "flags", + "reference_period", + "remarks", +) DIMENSION_MODELS = { "indicator": Indicator, "breakdown": Breakdown, @@ -23,6 +40,10 @@ } +def empty_cell(cell): + return cell.value is None or cell.value == "" + + class DimensionCache: def __init__(self, model): self.model = model @@ -60,46 +81,41 @@ def __init__(self, path): def load(self, *args, **kwargs): ... -DEFAULT_EXCEL_COLS = ( - "period", - "country", - "indicator", - "breakdown", - "unit", - "value", - "flags", -) - - class RowReader: def __init__( self, row, dimensions, - cols=DEFAULT_EXCEL_COLS, + header_columns, extra_fields=None, - required_cols=None, ): self.dimensions = dimensions + self.header_columns = header_columns self.row = row - self.cols = cols - # By default, all columns are required except for value and flags - self.required_cols = required_cols or self.cols[:-2] + self.row_dict = self._get_row_dict() self.errors = defaultdict(list) self.fields = {**extra_fields} self.read_row() + def _get_row_dict(self): + result = {} + for field, index in self.header_columns.items(): + try: + cell = self.row[index] + assert not empty_cell(cell) + except (IndexError, AssertionError): + result[field] = None + continue + + result[field] = cell.value + return result + def add_error(self, col_index, error): col_letter = get_column_letter(col_index + 1) self.errors[f"Column {col_letter}"].append(error) def get_value(self): - col_index = self.cols.index("value") - value_cell = self.row[col_index] - - value = value_cell.value - if self.empty_cell(value_cell): - value = None + value = self.row_dict["value"] if value is not None: try: @@ -109,20 +125,17 @@ def get_value(self): value = float(round(decimal.Decimal(value), 6)) except (TypeError, ValueError, ArithmeticError): self.add_error( - col_index, + self.header_columns["value"], f"Invalid 'value', expected number but got {value!r} instead", ) return value def get_flags(self): - col_index = self.cols.index("flags") - flags_cell = self.row[col_index] - - flags = flags_cell.value or "" + flags = self.row_dict.get("flags") or "" for flag in flags: if flag not in EUROSTAT_FLAGS: - self.add_error(col_index, f"Unknown flag {flag!r}") + self.add_error(self.header_columns["flags"], f"Unknown flag {flag!r}") return flags @@ -132,53 +145,58 @@ def read_row(self): """ self.fields["value"] = self.get_value() self.fields["flags"] = self.get_flags() + self.fields["reference_period"] = self.row_dict.get("reference_period") + self.fields["remarks"] = self.row_dict.get("remarks") if self.fields["value"] is None and not self.fields["flags"]: # Set the custom flag "unavailable" for this case self.fields["flags"] = "x" - for col in self.required_cols: - col_index = self.cols.index(col) - cell = self.row[col_index] - - if self.empty_cell(cell): - self.add_error(col_index, f"Column {col!r} must not be empty") + for col in UNIQUE_EXCEL_COLS: + if not self.row_dict.get(col): + self.add_error( + self.header_columns[col], f"Column {col!r} must not be empty" + ) for dim_name, dim_model in DIMENSION_MODELS.items(): - col_index = self.cols.index(dim_name) - dim_code = self.row[col_index].value + dim_code = self.row_dict[dim_name] self.fields[dim_name] = self.dimensions[dim_name].get(dim_code) if self.fields[dim_name] is None: self.add_error( - col_index, f"Missing dimension code for {dim_name!r}: {dim_code!r}" + self.header_columns[dim_name], + f"Missing dimension code for {dim_name!r}: {dim_code!r}", ) - @staticmethod - def empty_cell(cell): - return cell.value is None or cell.value == "" - class BaseExcelLoader(BaseFileLoader, ABC): """ Loader for Excel file formats. """ - def __init__( - self, path, cols=DEFAULT_EXCEL_COLS, extra_fields=None, required_cols=None - ): + def __init__(self, path, extra_fields=None): super().__init__(path) self.extra_fields = extra_fields or {} - self.cols = cols - # By default, all columns are required except for value and flags - self.required_cols = required_cols or self.cols[:-2] + self.max_col_index = 0 + self.header_columns = {} self.sheet = None self.errors = {} + self.read_headers() @property @abstractmethod def rows_iterator(self): - """Returns an implementation-specific (xls/xlsx) iterator over the active sheet's rows.""" + """Returns an implementation-specific (xls/xlsx) iterator over the active + sheet's rows. + """ + ... + + @property + @abstractmethod + def header_row(self): + """Returns an implementation-specific (xls/xlsx) header row for the + active sheet. + """ ... @abstractmethod @@ -186,6 +204,16 @@ def get_row(self, row_ref): """Get a row object using a reference produced by `row_iterator`.""" ... + def read_headers(self): + for index, cell in enumerate(self.header_row): + if empty_cell(cell): + break + + header_name = cell.value.lower().strip() + assert header_name in VALID_EXCEL_COLS, f"{header_name} not valid" + self.header_columns[header_name] = index + self.max_col_index = index + def read(self): """ Read the all rows from Excel file. @@ -201,12 +229,11 @@ def read(self): row_reader = RowReader( self.get_row(row_ref), self.dimensions, - cols=self.cols, + header_columns=self.header_columns, extra_fields=self.extra_fields, - required_cols=self.required_cols, ) - unique_key = tuple(row_reader.fields[field] for field in self.required_cols) + unique_key = tuple(row_reader.fields[field] for field in UNIQUE_EXCEL_COLS) if duplicate_row := all_unique_keys.get(unique_key): row_reader.errors["ALL"].append( f"Duplicate entry found at Row {duplicate_row}" @@ -242,21 +269,37 @@ def load(self, allow_errors=False): facts = Fact.objects.bulk_create( data, update_conflicts=True, - update_fields=("value", "flags", "import_file"), - unique_fields=("indicator", "breakdown", "unit", "country", "period"), + update_fields=( + "value", + "flags", + "import_file", + "reference_period", + "remarks", + ), + unique_fields=( + "indicator", + "breakdown", + "unit", + "country", + "period", + ), ) return len(facts), errors class XLSLoader(BaseExcelLoader): def __init__(self, path, *args, **kwargs): - super().__init__(path, *args, **kwargs) - self.wb = xlrd.open_workbook(self.path) + self.wb = xlrd.open_workbook(path) self.ws = self.wb.sheet_by_index(0) + super().__init__(path, *args, **kwargs) def get_row(self, row_ref): return self.ws.row(row_ref) + @property + def header_row(self): + return self.ws.row(0) + @property def rows_iterator(self): return range(1, self.ws.nrows) @@ -264,16 +307,20 @@ def rows_iterator(self): class XLSXLoader(BaseExcelLoader): def __init__(self, path, *args, **kwargs): - super().__init__(path, *args, **kwargs) - self.wb = openpyxl.load_workbook(self.path, read_only=True) + self.wb = openpyxl.load_workbook(path, read_only=True) self.ws = self.wb.active + super().__init__(path, *args, **kwargs) def get_row(self, row_ref): return row_ref # openpyxl iter_rows returns actual row objects + @property + def header_row(self): + return next(self.ws.iter_rows()) + @property def rows_iterator(self): - return self.ws.iter_rows(2, self.ws.max_row, 1, len(self.cols)) + return self.ws.iter_rows(2, self.ws.max_row, 1, self.max_col_index + 1) def get_loader(data_file, extra_fields=None): diff --git a/digital_agenda/apps/core/management/commands/migrate_extra_notes.py b/digital_agenda/apps/core/management/commands/migrate_extra_notes.py new file mode 100644 index 00000000..19a52120 --- /dev/null +++ b/digital_agenda/apps/core/management/commands/migrate_extra_notes.py @@ -0,0 +1,21 @@ +from django.core.management import BaseCommand +from rich.console import Console + +from digital_agenda.apps.charts.models import ExtraChartNote +from digital_agenda.apps.core.models import Fact + +console = Console() + + +class Command(BaseCommand): + help = "Migrate extra chart notes model" + + def handle(self, *args, **options): + for obj in ExtraChartNote.objects.all(): + reference_period = int(obj.note.strip()[-5:-1]) + assert reference_period > 2000 + + count = Fact.objects.filter( + indicator=obj.indicator, period=obj.period + ).update(reference_period=str(reference_period)) + console.print(f"{obj} migrated to {count} Fact objects") diff --git a/digital_agenda/apps/core/migrations/0016_fact_reference_period_fact_remarks.py b/digital_agenda/apps/core/migrations/0016_fact_reference_period_fact_remarks.py new file mode 100644 index 00000000..5953b4ef --- /dev/null +++ b/digital_agenda/apps/core/migrations/0016_fact_reference_period_fact_remarks.py @@ -0,0 +1,26 @@ +# Generated by Django 4.2.14 on 2024-08-21 10:13 + +import digital_agenda.common.citext +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0015_alter_breakdown_definition_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="fact", + name="reference_period", + field=digital_agenda.common.citext.CICharField( + blank=True, max_length=60, null=True + ), + ), + migrations.AddField( + model_name="fact", + name="remarks", + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/digital_agenda/apps/core/models.py b/digital_agenda/apps/core/models.py index f41ac37e..332a40a1 100644 --- a/digital_agenda/apps/core/models.py +++ b/digital_agenda/apps/core/models.py @@ -218,10 +218,16 @@ class Meta: @staticmethod def split_code(code): - try: - year, qualifier = re.split(r"[-_\s]", code.lower(), maxsplit=1) - except ValueError: - year, qualifier = code, "" + if isinstance(code, float): + assert int(code) == code + year, qualifier = str(int(code)), "" + elif isinstance(code, int): + year, qualifier = str(code), "" + else: + try: + year, qualifier = re.split(r"[-_\s]", code.lower(), maxsplit=1) + except ValueError: + year, qualifier = code, "" return year, qualifier def _guess_fields_from_code(self): @@ -303,6 +309,8 @@ class Fact(TimestampedModel): "Country", on_delete=models.CASCADE, related_name="facts" ) period = models.ForeignKey("Period", on_delete=models.CASCADE, related_name="facts") + reference_period = CICharField(max_length=60, null=True, blank=True) + remarks = models.TextField(blank=True, null=True) import_config = models.ForeignKey( "estat.ImportConfig", null=True, diff --git a/digital_agenda/apps/core/serializers.py b/digital_agenda/apps/core/serializers.py index be386d3f..b9907d04 100644 --- a/digital_agenda/apps/core/serializers.py +++ b/digital_agenda/apps/core/serializers.py @@ -132,6 +132,8 @@ class Meta: "country", "value", "flags", + "reference_period", + "remarks", ] diff --git a/digital_agenda/apps/core/static/admin.css b/digital_agenda/apps/core/static/admin.css index 7c1512e0..7f4d831a 100644 --- a/digital_agenda/apps/core/static/admin.css +++ b/digital_agenda/apps/core/static/admin.css @@ -164,6 +164,20 @@ form .aligned label + div.help { /* Make the submit row sticky */ .submit-row { - position: sticky; - bottom: 0; + position: sticky !important; + bottom: 0 !important; + z-index: 10 !important; +} + +/* Fix for multiple fields in a single line */ +input[type=checkbox] { + margin: 0 !important; +} + +.aligned .vCheckboxLabel { + padding: 0 0 0 5px !important; +} + +.form-multiline { + align-items: center; } diff --git a/digital_agenda/apps/core/static/import_file_example.xlsx b/digital_agenda/apps/core/static/import_file_example.xlsx index b792655f63372b6e9017472801ca73880d806c04..07050ac24f8263e183441505d7c8bbdfd047b410 100644 GIT binary patch delta 4036 zcmZu!1yoeuyB&~@p@#aA0}KcfqjXA$3_}m7gmiaE!!R&_(g;JBASER!-64$vN{BQ9 z(xr&h1KxYTr~mcNU3aas&fV**`|Y#O_wBEfbCw&Zsd5Vk1Rx+F037R3>%{??P#x-B zP?G1nei1y!!n3jjIR#CsuqLn&b32C_bx2LRzvi*P*Iew){kFLp>Uc*)Ew=Fos?M^ePslfp0HuSo_%oD zT>nC`W=c}a=R`hpmdJ*|`P~E{JqZ-1Mv)9&-RYwluZsbnCk%d1Nug?NSaowTJHi`o zrVN2vyZfoJBVnpLuc(;>jp32{S!aW*RGKRI1fgj{X*&P_U<4Zg&{QG$laFdHEGRb4 zH8U84TP!F;jTtBiIbU9JCKI28Fo6ftX&1Jr+JFdczXU61ba%4EZOeC+&wFI#5r(Sq zU?$+ms@;C+UarZ>;`DVhh?sBQ7bwKq+Z|-~!16VM^N_yPaOi%PsReHw_ac0Z{_UF% z+Wi-OQxW6oXhfu}TpB%pDc^nEAY~$+FmVJV5w*vmtxryzGjdDY4vtucq4AfY8F*-D z_%e*Y=c#Tm<+}tL2Y2#=t~yZrcN}&BuY5$sK_a}Euvd-|H(Hj_68_eMbLy6>Mb;?$ zA}eL1yE4v?{d_AUFgtpGAlMwA19uc}Ze)BzvH24XdU53AW=CSU>s+F3lr*^#q%z44 zCH?{+e5m+Hz6|2fnp|{v5ua&~hqTq^; z)Y7QJT@D)`LRVP%EIuq5Xb;yd!BN$ZG5B)Fi%M4}Cu6>Tq-AiznFYPhGqWY6hws9g zacs4bm=>4-DPJ83*KNro-D^;%mrc$!yu zkIt9fw!|yhQq1r4$b2|6QM7+jrHQOJE~+K{**o;hWR+CV;8j_0>i;~bRL3FJ7_aZi zZuh=$i()d}1m_^4YnRiiAKO|_2mLRZ+qQ)R5*;OWPkjo1Ae+Y93LP$;x{kB$cTpwhe#i`ZkiBP}e(JDk4M;wcE$kvrvEg#yv#ujZqDb9ObFu_;#dpR4RM z1G6kSI*7A$aIFU72~^|IdzR|(>57)T-6a*=FABwKQkmqmp84O1NAUb{m2r}#v~xpf zJ#4yo1)3uZN7Z@bm0OqEwSy^x?&J6 z*QP?{Vhsf5`kbfOonRp*@_lP?m^J^n8zOrJmG61CH(^7_kZW|xVBu9p!^oYr9tk%Z1xM+W z8nrR1yFuh~=LgfADUmmo)+8RY!}B?1fGbWF{ICqS;7vkfh_Ed}xhGQMoLniDH&`|yq&wO1%+WV;H73oZ_V6V&Be>7Zz?d1)oiDx8f)c?vO$qXWzWO1pX~K8 zyfo@5pAS5e*8?t-7UU{xsFXhgz~xf(6Rmh9#E(qrEPGp9qK7*BPzrp||5nXr&|r16V5|3n^(MJFx|89^-GA>5mG<*XIxBOd{O><@ zyqlN83X3x5_VINRUahU3cI`%#e9&-lemNrjm5|vdXzEq0H*qqyDJ>~K!SDykM=MOW zLb;@?ZFlX|7!NNyK2AK1n;fQMp7FCiBfHKYhe1$&4jcerhUh=@N3|jm4c32GL1SVL zRAT&%MUdU~S4@yp!6MAz9tIM$24*Md&f{aH;Dmt6XuYbeUWgd#Q;(wjQcI?Iv*-bPP?K1O8zGpHV z6;5Md2g59>nn-^DScY%3ZZdy1yM=-;jz#3#vE=FA^)KB!J2Vl(&n=Iaw%#mm@-)pR z&Fhy1`;&83kn2vuF9#F23>HoW%vvK4}TKn*O>a@PQU&7O2Xo@#; z)Xz~4F&pCFKXH1Ci^T_hw|P#sxtZ|xI(W2DYWDRD=0y2vgY$tDIVK< zPrSo+#HIT{udI+V@Nq(45h~(9Of4Lw4O4QmGt@$DA`DFwr)l{By{2M4)Vc#j`Srp_ zpV!0GlQN={>?KTI7q>6X>@?pNW~C<&=g<^VIV&^=ToYW*`wYDj{Vq zk8`D8jwdx64}XL@yo$gY7}sVGvCXEbUn7ATJDC_^UiALJ{^m-@6|k;e1GKh?f9d@@ zYsnaOY|th1)?=<}t-FVjWL#dat}E*dtG+p}YN`SMgWPX{67_rX+dL|ETVs4k_P}|T z%wB4T^*0ixm$5?oU8yWeo43LJOsQXuqJ-$&6NR@FPmd4k??@ClPFfxB6At}kj}DJ4 ztAHHskzbs|p4?QuOq>BK2dpdAULOeQ-(`;n$d6L9twU@6Q)Byu&{opjrjAGDX zgq!Cw083~=>ebfdg!w6CQ@LadAOo}47ZQLc1T8*>RojnRA14WD(Xa`kx$UzRl?ZIq zSo&Vn0sRxms~$je6i?}9iI&@sv-NHxX7WZ%TCDCS2E5pY!hq#TvHz%cKhhJSD4&(3s48EY{;k`dgTRS@nVTLf-6zbx@_xQFQEQ&Gz*krEQ%VlM1$5*E?<}jRW zFP=AJ3q&;Pf<8O24~zXU9xQNXu5JpFY1z9awbYo=5`_qFX8XnL$>uEZc>F|Lt3ZtG z%ob<PADjJakh|PdPWC7ZmcF!Oj76gNj4ZPwp(sfpVqnBK{xJ%e?O+N+ed_;obRna&eYcnznssO zsqdcYEr+-JwFwxpraM|>mUc+wJc{A;j-u35eU>x8*uu*^W-pW;Q`IvOs3KqPCSKeu zDjFj8I4BQje~U+l$Px0x)LzBxw!s)>-#2f6#gxa)2^z>1+3_tDkf^a3>DETrTjms) zyHdLyi4cGN{u5e@Wq{+cGS`Uwwr`+)582mz;}I8G5$U&-?+wr1n&|kjvRbIpd}yNB z#cKqsflabHJq$X)soI+?MI@vv!kl351t~*#dYZpYlmEDH_-T2If*Nh8NxQH?KcHho zH~Z@&fYC&~US%Fio@vx8z#y1GzXg1*aDcY}t5(Gmsa(Fll*@E{!FXYN^!7E)uj;x* zB$-SQaz};*>wevr`aMBivDt2~oiKGZR_sm1%XP1apx;}c$mb>S1Byiq#LK@;AMg!p z>}x$tGxq&Wczp~@l67PjSK|7FdwmR8*dV}7`v|=0AOCHoFnAQaEZ2Qgq_^Pq@fF&No;l|J9$;0@xFPlXx~j;F%D!`;c&a_HtdNzO}W6CG3)EC zw`^4AZF2={!yTp_upw@@?*M-~EbrnAc0b~rK2lE#;h0Z0w3_D!=FMNk z%aWxu_k`{t9}W*Ge(pNs^5y@2epbBf@^ua z?AW@7O)NhG0j(1852nl@i`7i|}%lgz-e!HP^Ofr{|Nspa-DptF6(fan~IyAG{G zx3i$v+i>*?-9ikbWJ!+XS2YfD*heKfo_7z0yXdem0~dSQ5i~n+c!$@3ty33w%SC=@ z^ha*pa887TB=G8FO9NSViBuKVy>cp1!Q`~gp4+81`nj-q=>Ch0PyG5|Ll%}+Vp7um zl0u-QeF^sm3$LkRo7(l3ZN9~WcWXoS_FKEB@THM!{^FA~uDGSf)knGRYAZx#uSJbw zn?d%ysi)CddVRX^<*3&CB{ID@S4v71iWu*6sxa|xKWyy;=}UDL<+N5EA)^3@%n$QwQuy85j(>}i51kd?Pg>pPV;eK@1GLVPk}zhe8rKxI-#j) zFK&CS_%*S*fH(-29>HYfUG$l)u$1)XlHip8?b|)=9h7Z7KAaYIQR1VtGQl71Ma}G` zYg?#S?cwO#94PJByHU30x4a}=(j2Vd|KZyC8rV*m;U?hP*TVB{Gt=p*98c=+oO4gv z&bCcO<%Ymk_m>?s5CMe80b$LL2E8?iO}dsY29BfKX+vC+RPa5;KgaQyeg1Xg3hQWA zep;Lqb)}vOEXI)9tTi}uB-&`BFrMH;!6Lc-En0#KgEG!SS5UIxi%8Xicc3Esyf@32 zm=o|Q(A(=eUGCpjG1~sOo}P%ZAiUQa?|s>+&F3yi${OrU$A=t2!0Bbz+^0k zQTR#OT3N_lcUBh@rxi>6{z{*cxzzlY(O6yu+4Ddfw$sxe4KEE=ljQ4;fL?hccGtxC z{Mo%i-#|GWkLa*$*EPg%*x!>P+ZZox^H2K^p_jRv$t1ks%B8RYYe3~(*S7*V3Ctyu4xE@s0MKqsO z8KvWyaeTRvk(-yR=umGLBpx>Kwa&+x&b|C$?`xPPZI_7&Gg5rkH;cFjh4lM5ukmOz zg-+Y!-h_}kwj0~+f2vWVyk@Y#_sUFMP9_K_=A-b`w}9b^hekHfy{m^UC}&-1Y+T=e zi*-wwf)_W^mkAq|w30!Z9ky+Wu)BWs2wC;#bxa^}DqGdv6Ll zjV%zwG=tX8Yl_b4t#y8mqdxjtq6d{TB&Y!{rxajzbEe`F7>|jpJ3IAU#SSHsHCiDS z)`O&HIQj96s-hN2p#a4#RiC=iTu&PFytmlSIKE{SsP2cICKp zPJ%lZ_qo}Ijj)$FxRNhrhU1Ge`qeeYr1xB2Dy7Bl8#1I9SxL$D)}erJyGg-woCx2B z>9WZk(J`M?Dj{&*1IEV1l4hbWWKfwsy5aB)R589BM3p5kFK;;7qu^9QP#r#q>~m2Re2A!Zw^pJt|*nr z;qH|4!bu{^ZT8N|=WPA?Of6*oM5gd8(vA+Xw&dk(UOx#fdqw8P+}cuTtZyI>Xh;?Dxme_NK;($^OLUmcXcQ`vcJ<|aWg(>J1D6buSNZ%< z$OuD*LCG5gjnal@7kA!0r`0rxlUCLAI&PmJU4+yLZQP2Vi*7NT@I(g%9SYL zQf|0HG1{2?<~$O^&pdJ&0yO)SuL)966WVAIh;N!|X}HS~w@QC{5YgK@mgyj5)aj-< z*olqB;if&7!TFT=>P!1Q$V&Q!Ka5(WYI95UtY3EliYokMmn+DYOc~{(uGvk%hD`H#`iwdH zQb_^F@G^>JU;THar}BqI-T*bRb43#J!Gq;JQ`oVHhQ?dHc}6u|p3uD^soL7E0GcCV z%E~2hd_NQ>@sN{4o5^y_XcI`B;aUE z^dJ@62RFOH1jpk$%h#KdjeM0RUrJa6Zw)S|=`TMM%VH7DDMf#)tkIM2bXDm(Va0O! zjm?^uNx1E+lUvu=BC#$qPQ{7u;84`+>AMfhK0!!Zn|nbJO~wzW*b;*ED{ul*-wT;K z*BIh3bs)*kol<$-#Sp8>_d$#@6}KGI5sWeGG^JGtbaDV98}EwslPyuD*t}U;(N-&b zCX;~?wUPB!pP;peZq%jo#P`&)kge^2*jSyDLI|Ve!8|*Ucd~b)Ee5Ig*Nqh%;Vhha zx8Nbh9zv+O`;W8uCrZscaRo-p6p~vDN6E)0RM*Rh6Rei~(hGZ_9i{url}') @admin.display(description="Has Remarks", boolean=True) - def has_remarks(self, obj): - return bool(obj.remarks) + def has_additional_remarks(self, obj): + return bool(obj.additional_remarks) def get_actions(self, request): actions = super().get_actions(request) diff --git a/digital_agenda/apps/estat/importer.py b/digital_agenda/apps/estat/importer.py index 78858c8c..e4e12273 100644 --- a/digital_agenda/apps/estat/importer.py +++ b/digital_agenda/apps/estat/importer.py @@ -23,13 +23,17 @@ logger = logging.getLogger(__name__) -MODELS = { +MODEL_DIMENSIONS = { "indicator": Indicator, "breakdown": Breakdown, "country": Country, "unit": Unit, "period": Period, } +TEXT_DIMENSION = ( + "reference_period", + "remarks", +) class ImporterError(ValueError): @@ -220,10 +224,13 @@ def should_store_observation(self, obs): return True - def get_dimension_obj(self, dimension, obs): + def get_dimension(self, dimension, obs): config_dim = getattr(self.config, dimension) is_surrogate = getattr(self.config, f"{dimension}_is_surrogate") + if not config_dim: + return None, None + if is_surrogate: # Hardcoded value, no label available category_id, category_label = config_dim, "" @@ -237,10 +244,15 @@ def get_dimension_obj(self, dimension, obs): except KeyError: pass + return category_id, category_label + + def get_dimension_obj(self, dimension, obs): + category_id, category_label = self.get_dimension(dimension, obs) + try: return self.cache[dimension][category_id] except KeyError: - obj, created = MODELS[dimension].objects.get_or_create( + obj, created = MODEL_DIMENSIONS[dimension].objects.get_or_create( code=category_id, defaults={"label": category_label} ) if created: @@ -261,14 +273,20 @@ def iter_facts(self): flags=obs["status"] or "", import_config_id=self.config.id, ) + + # Set dimensions with related models unique_key = [] - for attr in MODELS: + for attr in MODEL_DIMENSIONS: obj = self.get_dimension_obj(attr, obs) unique_key.append(obj) setattr(fact, attr, obj) - fact_collection[tuple(unique_key)].append(fact) + # Set text "dimensions" + for attr in TEXT_DIMENSION: + dim_id, dim_label = self.get_dimension(attr, obs) + setattr(fact, attr, dim_id or dim_label) + for key, fact_group in fact_collection.items(): yield self._handle_conflict(fact_group, key) @@ -313,8 +331,20 @@ def create_batch(self, facts): return Fact.objects.bulk_create( facts, update_conflicts=True, - update_fields=("value", "flags", "import_config"), - unique_fields=("indicator", "breakdown", "unit", "country", "period"), + update_fields=( + "value", + "flags", + "import_config", + "reference_period", + "remarks", + ), + unique_fields=( + "indicator", + "breakdown", + "unit", + "country", + "period", + ), ) def run(self, batch_size=10_000): diff --git a/digital_agenda/apps/estat/migrations/0014_rename_remarks_importconfig_additional_remarks.py b/digital_agenda/apps/estat/migrations/0014_rename_remarks_importconfig_additional_remarks.py new file mode 100644 index 00000000..0c34ada7 --- /dev/null +++ b/digital_agenda/apps/estat/migrations/0014_rename_remarks_importconfig_additional_remarks.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.14 on 2024-08-21 10:05 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("estat", "0013_importconfig_conflict_resolution"), + ] + + operations = [ + migrations.RenameField( + model_name="importconfig", + old_name="remarks", + new_name="additional_remarks", + ), + ] diff --git a/digital_agenda/apps/estat/migrations/0015_importconfig_reference_period_and_more.py b/digital_agenda/apps/estat/migrations/0015_importconfig_reference_period_and_more.py new file mode 100644 index 00000000..13f7cea1 --- /dev/null +++ b/digital_agenda/apps/estat/migrations/0015_importconfig_reference_period_and_more.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.14 on 2024-08-21 10:13 + +import digital_agenda.common.citext +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("estat", "0014_rename_remarks_importconfig_additional_remarks"), + ] + + operations = [ + migrations.AddField( + model_name="importconfig", + name="reference_period", + field=digital_agenda.common.citext.CICharField( + blank=True, max_length=60, null=True + ), + ), + migrations.AddField( + model_name="importconfig", + name="reference_period_is_surrogate", + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name="importconfig", + name="remarks", + field=digital_agenda.common.citext.CICharField( + blank=True, max_length=60, null=True + ), + ), + migrations.AddField( + model_name="importconfig", + name="remarks_is_surrogate", + field=models.BooleanField(default=False), + ), + ] diff --git a/digital_agenda/apps/estat/models.py b/digital_agenda/apps/estat/models.py index 17aed116..c69851bd 100644 --- a/digital_agenda/apps/estat/models.py +++ b/digital_agenda/apps/estat/models.py @@ -67,6 +67,8 @@ def default_mappings(): "country": {"EU27_2020": "EU"}, "unit": {}, "period": {}, + "reference_period": {}, + "remarks": {}, } @@ -94,7 +96,7 @@ class ConflictResolution(models.TextChoices): help_text="Assigned tags used for filtering and searching; has no impact on the data import", blank=True, ) - remarks = models.TextField( + additional_remarks = models.TextField( blank=True, null=True, help_text="Additional notes/remarks" ) @@ -141,6 +143,12 @@ class ConflictResolution(models.TextChoices): period = CICharField(max_length=60, default="time") period_is_surrogate = models.BooleanField(default=False) + reference_period = CICharField(max_length=60, blank=True, null=True) + reference_period_is_surrogate = models.BooleanField(default=False) + + remarks = CICharField(max_length=60, blank=True, null=True) + remarks_is_surrogate = models.BooleanField(default=False) + period_start = models.PositiveIntegerField( null=True, blank=True, diff --git a/digital_agenda/common/export.py b/digital_agenda/common/export.py index 6bb86710..a9ead5f9 100644 --- a/digital_agenda/common/export.py +++ b/digital_agenda/common/export.py @@ -49,13 +49,15 @@ def export_facts_csv(filename, chartgroup=None, indicatorgroup=None, indicator=N assert filters, "At least one filter must be provided for export" query = f""" - SELECT core_period.code AS "period", - core_country.code AS "country", - core_indicator.code AS "indicator", - core_breakdown.code AS "breakdown", - core_unit.code AS "unit", - core_fact.value AS "value", - core_fact.flags AS "flags" + SELECT core_period.code AS "period", + core_country.code AS "country", + core_indicator.code AS "indicator", + core_breakdown.code AS "breakdown", + core_unit.code AS "unit", + core_fact.value AS "value", + core_fact.flags AS "flags", + core_fact.reference_period AS "reference_period", + core_fact.remarks AS "remarks" FROM core_fact INNER JOIN core_indicator ON (core_fact.indicator_id = core_indicator.id) diff --git a/fixtures/importconfig.json b/fixtures/importconfig.json index 5a7877fe..acf5b5f2 100644 --- a/fixtures/importconfig.json +++ b/fixtures/importconfig.json @@ -5,7 +5,7 @@ "fields": { "code": "educ_uoe_grad03", "title": null, - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-12T10:00:00Z", "datastructure_last_update": "2023-12-12T10:00:00Z", "datastructure_last_version": "29.0", @@ -58,7 +58,7 @@ "fields": { "code": "demo_pjan", "title": "Eurostat total population", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-09-28T21:00:00Z", "datastructure_last_update": "2023-09-28T21:00:00Z", "datastructure_last_version": "31.0", @@ -113,7 +113,7 @@ "fields": { "code": "tec00001", "title": "Gross domestic product at market prices", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-20T22:00:00Z", "datastructure_last_update": "2023-12-20T22:00:00Z", "datastructure_last_version": "53.0", @@ -168,7 +168,7 @@ "fields": { "code": "lfst_hhnhwhtc", "title": "Number of households", - "remarks": "", + "additional_remarks": "", "data_last_update": "2023-06-15T21:00:00Z", "datastructure_last_update": "2023-06-15T21:00:00Z", "datastructure_last_version": "17.1", @@ -228,7 +228,7 @@ "fields": { "code": "isoc_sk_dskl_i", "title": "Digital Skills Index 2019", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "32.0", @@ -335,7 +335,7 @@ "fields": { "code": "isoc_sk_dskl_i21", "title": "Digital Skills Index 2021", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "10.0", @@ -451,7 +451,7 @@ "fields": { "code": "isoc_ci_ac_i", "title": "Individuals - internet activities", - "remarks": "", + "additional_remarks": "", "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "39.0", @@ -560,7 +560,7 @@ "fields": { "code": "isoc_cisci_pb", "title": "Security related problems experienced when using the internet", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "32.0", @@ -651,7 +651,7 @@ "fields": { "code": "isoc_cicci_use", "title": "Individuals - use of cloud services", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "32.0", @@ -737,7 +737,7 @@ "fields": { "code": "isoc_cisci_prv", "title": "Privacy and protection of personal information (until 2016)", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "31.0", @@ -825,7 +825,7 @@ "fields": { "code": "isoc_ec_iprb", "title": "Problems encountered by individuals when buying/ordering over the internet (until 2019)", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "33.0", @@ -911,7 +911,7 @@ "fields": { "code": "isoc_cisci_ax", "title": "Activities via internet not done because of security concerns", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "32.0", @@ -997,7 +997,7 @@ "fields": { "code": "isoc_ec_ibuy", "title": "Internet purchases by individuals (until 2019)", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "32.0", @@ -1094,7 +1094,7 @@ "fields": { "code": "isoc_ec_ib20", "title": "Internet purchases by individuals (2020 onwards)", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "36.0", @@ -1185,7 +1185,7 @@ "fields": { "code": "isoc_sk_cskl_i", "title": "Individuals' level of computer skills (until 2019)", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "32.0", @@ -1281,7 +1281,7 @@ "fields": { "code": "isoc_sks_itspt", "title": "Employed ICT specialists - total", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-05-03T21:00:00Z", "datastructure_last_update": "2023-05-03T21:00:00Z", "datastructure_last_version": "25.0", @@ -1332,7 +1332,7 @@ "fields": { "code": "isoc_ci_ifp_iu", "title": "Individuals - internet use", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "36.0", @@ -1422,7 +1422,7 @@ "fields": { "code": "isoc_ci_im_i", "title": "Individuals - mobile internet access", - "remarks": "i_iumc, i_iump", + "additional_remarks": "i_iumc, i_iump", "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "32.0", @@ -1513,7 +1513,7 @@ "fields": { "code": "educ_uoe_grad04", "title": "Graduates in tertiary education (STEM)", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-12T10:00:00Z", "datastructure_last_update": "2023-12-12T10:00:00Z", "datastructure_last_version": "28.0", @@ -1563,7 +1563,7 @@ "fields": { "code": "isoc_bde15b_p", "title": "Broadband and connectivity - persons employed by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-08-29T09:00:00Z", "datastructure_last_update": "2023-08-29T09:00:00Z", "datastructure_last_version": "32.0", @@ -1668,7 +1668,7 @@ "fields": { "code": "isoc_bde15b_p_s", "title": "Broadband and connectivity - persons employed by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-08-29T09:00:00Z", "datastructure_last_update": "2023-05-25T09:00:00Z", "datastructure_last_version": "5.0", @@ -1747,7 +1747,7 @@ "fields": { "code": "isoc_ci_in_en2", "title": "Internet access by NACE", - "remarks": "", + "additional_remarks": "", "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "35.0", @@ -1840,7 +1840,7 @@ "fields": { "code": "isoc_ci_in_es", "title": "Internet access by size class of enterprise", - "remarks": "", + "additional_remarks": "", "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "6.0", @@ -1907,7 +1907,7 @@ "fields": { "code": "isoc_cismtn2", "title": "Social media use by type, internet advertising and NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "6.0", @@ -2009,7 +2009,7 @@ "fields": { "code": "isoc_cismt", "title": "Social media use by type, internet advertising and size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "32.0", @@ -2085,7 +2085,7 @@ "fields": { "code": "isoc_ciwebn2", "title": "Websites and functionalities by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "6.0", @@ -2179,7 +2179,7 @@ "fields": { "code": "isoc_ciweb", "title": "Websites and functionalities by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "33.0", @@ -2247,7 +2247,7 @@ "fields": { "code": "isoc_ci_it_en2", "title": "Type of connections to the internet by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "33.0", @@ -2346,7 +2346,7 @@ "fields": { "code": "isoc_ci_it_es", "title": "Type of connections to the internet by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "6.0", @@ -2419,7 +2419,7 @@ "fields": { "code": "isoc_ec_eseln2", "title": "E-commerce sales of enterprises by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "34.0", @@ -2516,7 +2516,7 @@ "fields": { "code": "isoc_ec_esels", "title": "E-commerce sales of enterprises by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "7.0", @@ -2587,7 +2587,7 @@ "fields": { "code": "isoc_eb_bdn2", "title": "Big data analysis by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-05-25T09:00:00Z", "datastructure_last_version": "5.0", @@ -2682,7 +2682,7 @@ "fields": { "code": "isoc_eb_bd", "title": "Big data analysis by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-05-25T09:00:00Z", "datastructure_last_version": "31.0", @@ -2750,7 +2750,7 @@ "fields": { "code": "isoc_cicce_usen2", "title": "Cloud computing services by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "6.0", @@ -2846,7 +2846,7 @@ "fields": { "code": "isoc_cicce_use", "title": "Cloud computing services by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "34.0", @@ -2916,7 +2916,7 @@ "fields": { "code": "isoc_eb_iipn2", "title": "Integration of internal processes by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "6.0", @@ -3020,7 +3020,7 @@ "fields": { "code": "isoc_eb_iip", "title": "Integration of internal processes by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "33.0", @@ -3096,7 +3096,7 @@ "fields": { "code": "isoc_bde15dip", "title": "[discontinued] Integration of internal processes by NACE (E_SISORP)", - "remarks": "E_SISORP is discontinued", + "additional_remarks": "E_SISORP is discontinued", "data_last_update": "2023-08-29T09:00:00Z", "datastructure_last_update": "2023-08-29T09:00:00Z", "datastructure_last_version": "31.0", @@ -3189,7 +3189,7 @@ "fields": { "code": "isoc_bde15dipn2", "title": "[discontinued] Integration of internal processes by size class of enterprise (E_SISORP)", - "remarks": "E_SISORP is discontinued", + "additional_remarks": "E_SISORP is discontinued", "data_last_update": "2023-08-29T09:00:00Z", "datastructure_last_update": "2023-05-25T09:00:00Z", "datastructure_last_version": "5.0", @@ -3256,7 +3256,7 @@ "fields": { "code": "isoc_eb_ain2", "title": "Artificial intelligence by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "6.0", @@ -3349,7 +3349,7 @@ "fields": { "code": "isoc_eb_ai", "title": "Artificial intelligence by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "32.0", @@ -3416,7 +3416,7 @@ "fields": { "code": "isoc_ske_itspen2", "title": "Enterprises that employ ICT specialists by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "28.0", @@ -3509,7 +3509,7 @@ "fields": { "code": "isoc_ske_itspe", "title": "Enterprises that employ ICT specialists by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "7.0", @@ -3576,7 +3576,7 @@ "fields": { "code": "isoc_ske_itrcrn2", "title": "Enterprises that recruited or tried to recruit ICT specialists by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "32.0", @@ -3669,7 +3669,7 @@ "fields": { "code": "isoc_ske_itrcrs", "title": "Enterprises that recruited or tried to recruit ICT specialists by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "7.0", @@ -3736,7 +3736,7 @@ "fields": { "code": "isoc_ske_fctn2", "title": "Enterprises - ICT functions performed by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "7.0", @@ -3829,7 +3829,7 @@ "fields": { "code": "isoc_ske_fct", "title": "Enterprises - ICT functions performed by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "32.0", @@ -3896,7 +3896,7 @@ "fields": { "code": "isoc_cisce_ran2", "title": "Security policy: measures, risks and staff awareness by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-08-29T09:00:00Z", "datastructure_last_version": "8.0", @@ -3994,7 +3994,7 @@ "fields": { "code": "isoc_cisce_ra", "title": "Security policy: measures, risks and staff awareness by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-05-25T09:00:00Z", "datastructure_last_version": "33.0", @@ -4066,7 +4066,7 @@ "fields": { "code": "isoc_ske_ittn2", "title": "Enterprises that provided training to develop/upgrade ICT skills of their personnel by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "32.0", @@ -4159,7 +4159,7 @@ "fields": { "code": "isoc_ske_itts", "title": "Enterprises that provided training to develop/upgrade ICT skills of their personnel by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "7.0", @@ -4226,7 +4226,7 @@ "fields": { "code": "isoc_cimobe_usen2", "title": "Use of mobile connections to the internet by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-08-29T09:00:00Z", "datastructure_last_version": "5.0", @@ -4324,7 +4324,7 @@ "fields": { "code": "isoc_cimobe_use", "title": "Use of mobile connections to the internet by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-05-25T09:00:00Z", "datastructure_last_version": "31.0", @@ -4396,7 +4396,7 @@ "fields": { "code": "isoc_bde15b_en2", "title": "Broadband and connectivity - enterprises by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-08-29T09:00:00Z", "datastructure_last_update": "2023-08-29T09:00:00Z", "datastructure_last_version": "5.0", @@ -4492,7 +4492,7 @@ "fields": { "code": "isoc_bde15b_e", "title": "Broadband and connectivity - enterprises by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-08-29T09:00:00Z", "datastructure_last_update": "2023-05-25T09:00:00Z", "datastructure_last_version": "31.0", @@ -4562,7 +4562,7 @@ "fields": { "code": "isoc_ec_evaln2", "title": "Value of e-commerce sales by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "32.0", @@ -4657,7 +4657,7 @@ "fields": { "code": "isoc_ec_evals", "title": "Value of e-commerce sales by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "6.0", @@ -4726,7 +4726,7 @@ "fields": { "code": "isoc_bde15decn2", "title": "E-commerce, customer relation management (CRM) and secure transactions by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-08-29T09:00:00Z", "datastructure_last_update": "2023-08-29T09:00:00Z", "datastructure_last_version": "5.0", @@ -4825,7 +4825,7 @@ "fields": { "code": "isoc_bde15dec", "title": "E-commerce, customer relation management (CRM) and secure transactions by size class of enterprise", - "remarks": "", + "additional_remarks": "", "data_last_update": "2023-08-29T09:00:00Z", "datastructure_last_update": "2023-05-25T09:00:00Z", "datastructure_last_version": "31.0", @@ -4898,7 +4898,7 @@ "fields": { "code": "isoc_bde15discn2", "title": "Integration with customers/suppliers and SCM by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-08-29T09:00:00Z", "datastructure_last_update": "2023-05-25T09:00:00Z", "datastructure_last_version": "5.0", @@ -4998,7 +4998,7 @@ "fields": { "code": "isoc_bde15disc", "title": "Integration with customers/suppliers and SCM by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-08-29T09:00:00Z", "datastructure_last_update": "2023-05-25T09:00:00Z", "datastructure_last_version": "30.0", @@ -5071,7 +5071,7 @@ "fields": { "code": "isoc_sks_itsps", "title": "Employed ICT specialists by sex", - "remarks": "", + "additional_remarks": "", "data_last_update": "2023-05-03T21:00:00Z", "datastructure_last_update": "2023-05-03T21:00:00Z", "datastructure_last_version": "25.0", @@ -5123,7 +5123,7 @@ "fields": { "code": "isoc_ec_esels", "title": "E_AWSWW (e_aws/rest_world)", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "7.0", @@ -5180,7 +5180,7 @@ "fields": { "code": "isoc_ec_esels", "title": "E_AWSHM (e_aws/own_country)", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "7.0", @@ -5237,7 +5237,7 @@ "fields": { "code": "isoc_ec_esels", "title": "E_AWSEU (e_aws/other_eu_countries)", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "7.0", @@ -5294,7 +5294,7 @@ "fields": { "code": "isoc_e_diin2", "title": "Digital Intensity Index by NACE (all)", - "remarks": "e_di3_vlo, e_di3_lo, e_di3_hi, e_di3_vhi, e_di4_vlo, e_di4_lo, e_di4_hi, e_di4_vhi", + "additional_remarks": "e_di3_vlo, e_di3_lo, e_di3_hi, e_di3_vhi, e_di4_vlo, e_di4_lo, e_di4_hi, e_di4_vhi", "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "7.0", @@ -5397,7 +5397,7 @@ "fields": { "code": "isoc_e_dii", "title": "Digital intensity indicators by size class of enterprise (all)", - "remarks": "e_di3_vlo, e_di3_lo, e_di3_hi, e_di3_vhi, e_di4_vlo, e_di4_lo, e_di4_hi, e_di4_vhi,", + "additional_remarks": "e_di3_vlo, e_di3_lo, e_di3_hi, e_di3_vhi, e_di4_vlo, e_di4_lo, e_di4_hi, e_di4_vhi,", "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "16.0", @@ -5471,7 +5471,7 @@ "fields": { "code": "isoc_e_dii", "title": "Digital Intensity Index v3 - SME 2021 (e_di3_sme_gelo)", - "remarks": "", + "additional_remarks": "", "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "16.0", @@ -5532,7 +5532,7 @@ "fields": { "code": "isoc_e_dii", "title": "Digital Intensity Index v4 - SME 2022 (e_di4_sme_gelo)", - "remarks": "", + "additional_remarks": "", "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "16.0", @@ -5593,7 +5593,7 @@ "fields": { "code": "isoc_ci_ifp_fu", "title": "Individuals - frequency of internet use", - "remarks": "i_iuse, i_iday", + "additional_remarks": "i_iuse, i_iday", "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "36.0", @@ -5682,7 +5682,7 @@ "fields": { "code": "isoc_ci_dev_i", "title": "Individuals - devices used to access the internet", - "remarks": "", + "additional_remarks": "", "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "33.0", @@ -5772,7 +5772,7 @@ "fields": { "code": "isoc_e_dii", "title": "Digital Intensity Index v3 - SME 2023 (e_di3_sme_gelo)", - "remarks": "", + "additional_remarks": "", "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "16.0", @@ -5831,7 +5831,7 @@ "fields": { "code": "isoc_ci_it_h", "title": "Households - type of connection to the internet", - "remarks": "H_BBFIX, H_BROAD", + "additional_remarks": "H_BBFIX, H_BROAD", "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "33.0", @@ -5897,7 +5897,7 @@ "fields": { "code": "isoc_ci_in_h", "title": "Households - level of internet access (h_iacc)", - "remarks": "h_iacc", + "additional_remarks": "h_iacc", "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "29.0", @@ -5959,7 +5959,7 @@ "fields": { "code": "isoc_pibi_rni", "title": "Households - reasons for not having internet access at home (h_xcost)", - "remarks": "h_xcost", + "additional_remarks": "h_xcost", "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "32.0", @@ -6024,7 +6024,7 @@ "fields": { "code": "isoc_ciegi_ac", "title": "E-government activities of individuals via websites (i_iugov12, i_igov12rt)", - "remarks": "i_iugov12, i_igov12rt", + "additional_remarks": "i_iugov12, i_igov12rt", "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "36.0", diff --git a/fixtures/test/facts.json b/fixtures/test/facts.json index c9157cc7..221b166d 100644 --- a/fixtures/test/facts.json +++ b/fixtures/test/facts.json @@ -334,7 +334,9 @@ "2022" ], "import_config": null, - "import_file": null + "import_file": null, + "reference_period": "2020", + "remarks": "This data point is for the EU" } }, { @@ -672,7 +674,9 @@ "2022" ], "import_config": null, - "import_file": null + "import_file": null, + "reference_period": "2020", + "remarks": "This data point is for Slovakia" } } ] diff --git a/frontend/cypress/e2e/admin/fileImport.cy.js b/frontend/cypress/e2e/admin/fileImport.cy.js index dc4662ad..64ea39e5 100644 --- a/frontend/cypress/e2e/admin/fileImport.cy.js +++ b/frontend/cypress/e2e/admin/fileImport.cy.js @@ -1,74 +1,61 @@ describe("Check data file import", () => { - it("Create and run data file import", () => { + function checkFileImport({ + fixture, + expectedStatus = "SUCCESS", + expectedFacts = "1", + }) { cy.login(); // Navigate to the data file import page cy.get("a").contains("Upload data from file").click(); cy.get("a").contains("Add Upload data from file").click(); // Add a new data file import - cy.get("input[type=file]").selectFile( - "cypress/fixtures/import_file_valid.xlsx", - ); + cy.get("input[type=file]").selectFile(`cypress/fixtures/${fixture}`); cy.get("textarea[name=description]").type("Test file import"); cy.get("input[type=submit][value=Save]").click(); // Wait for the task to finish cy.get("a").contains("Upload file results").click(); - cy.get("tbody tr:first-child td.field-status_display").contains("SUCCESS", { - timeout: 10000, - }); + cy.get("tbody tr:first-child td.field-status_display").contains( + expectedStatus, + { + timeout: 10000, + }, + ); // Navigate to the import config change form cy.get("tbody tr:first-child td.field-import_file_link a").click(); // Check that only one fact has been imported - cy.get(".field-num_facts a").contains("1"); + cy.get(".field-num_facts a").contains(expectedFacts); // Delete the import file cy.get("a").contains("Delete").click(); cy.get("input[type=submit]").click(); + } + + it("Create and run data file import XLSX", () => { + checkFileImport({ + fixture: "import_file_valid.xlsx", + }); + }); + it("Create and run data file import XLS", () => { + checkFileImport({ + fixture: "import_file_valid.xls", + }); + }); + it("Create and run data file import with no remarks", () => { + checkFileImport({ + fixture: "import_file_valid_no_remarks.xlsx", + }); }); it("Create and run invalid data file import", () => { - cy.login(); - // Navigate to the data file import page - cy.get("a").contains("Upload data from file").click(); - cy.get("a").contains("Add Upload data from file").click(); - // Add a new data file import - cy.get("input[type=file]").selectFile( - "cypress/fixtures/import_file_invalid.xlsx", - ); - cy.get("textarea[name=description]").type("Test file invalid import"); - cy.get("input[type=submit][value=Save]").click(); - // Wait for the task to finish - cy.get("a").contains("Upload file results").click(); - cy.get("tbody tr:first-child td.field-status_display").contains("FAILURE", { - timeout: 10000, + checkFileImport({ + fixture: "import_file_invalid.xlsx", + expectedStatus: "FAILURE", + expectedFacts: "0", }); - // Navigate to the import config change form - cy.get("tbody tr:first-child td.field-import_file_link a").click(); - // Check that no fact has been imported - cy.get(".field-num_facts a").contains("0"); - // Delete the import file - cy.get("a").contains("Delete").click(); - cy.get("input[type=submit]").click(); }); it("Create and run invalid data file import duplicate", () => { - cy.login(); - // Navigate to the data file import page - cy.get("a").contains("Upload data from file").click(); - cy.get("a").contains("Add Upload data from file").click(); - // Add a new data file import - cy.get("input[type=file]").selectFile( - "cypress/fixtures/import_file_invalid_duplicate.xlsx", - ); - cy.get("textarea[name=description]").type("Test file invalid import"); - cy.get("input[type=submit][value=Save]").click(); - // Wait for the task to finish - cy.get("a").contains("Upload file results").click(); - cy.get("tbody tr:first-child td.field-status_display").contains("FAILURE", { - timeout: 10000, + checkFileImport({ + fixture: "import_file_invalid_duplicate.xlsx", + expectedStatus: "FAILURE", + expectedFacts: "0", }); - // Navigate to the import config change form - cy.get("tbody tr:first-child td.field-import_file_link a").click(); - // Check that no fact has been imported - cy.get(".field-num_facts a").contains("0"); - // Delete the import file - cy.get("a").contains("Delete").click(); - cy.get("input[type=submit]").click(); }); }); diff --git a/frontend/cypress/e2e/admin/importConfig.cy.js b/frontend/cypress/e2e/admin/importConfig.cy.js index 90d94c72..27b60425 100644 --- a/frontend/cypress/e2e/admin/importConfig.cy.js +++ b/frontend/cypress/e2e/admin/importConfig.cy.js @@ -9,6 +9,9 @@ describe("Check import configuration", () => { cy.get("input[name=title]").type("Test import config"); cy.get("input[name=indicator]").type("indic_is"); cy.get("input[name=breakdown]").type("hhtyp"); + cy.get("input[name=reference_period]").type("time"); + cy.get("input[name=remarks]").type("test remarks"); + cy.get("input[name=remarks_is_surrogate]").click(); cy.get("input[name=period_start]").type("2019"); cy.get("input[name=period_end]").type("2019"); cy.get("#id_filters textarea.ace_text-input").clear({ force: true }); @@ -21,7 +24,7 @@ describe("Check import configuration", () => { ); cy.get("#id_mappings textarea.ace_text-input").clear({ force: true }); cy.get("#id_mappings textarea.ace_text-input").type( - '{"breakdown": {"total": "hh_total"}, "country": {"EU27_2020": "EU"}}', + '{"breakdown": {"total": "hh_total"}, "country": {"EU27_2020": "EU"}, "reference_period": {"2019": "2000"}}', { force: true, parseSpecialCharSequences: false, @@ -38,9 +41,21 @@ describe("Check import configuration", () => { }); // Navigate to the import config change form cy.get("tbody tr:first-child td.field-import_config_link a").click(); - // Check that only one fact has been imported - cy.get(".field-num_facts a").contains("1"); + // Check that only one fact has been imported, and open the fact details + cy.get(".field-num_facts a").contains("1").click(); + cy.get("tbody tr:first-child th.field-indicator a").click(); + + // Check imported values + cy.get(".field-indicator").contains("[h_broad]"); + cy.get(".field-breakdown").contains("[hh_total]"); + cy.get(".field-unit").contains("[pc_hh]"); + cy.get(".field-country").contains("[EU]"); + cy.get(".field-period").contains("[2019]"); + cy.get("[name=reference_period]").should("have.value", "2000"); + cy.get("[name=remarks]").should("have.value", "test remarks"); + // Delete the import config + cy.get(".field-import_config .view-related").click(); cy.get("a").contains("Delete").click(); cy.get("input[type=submit]").click(); }); diff --git a/frontend/cypress/e2e/charts/4.key-indicators/1.analyse-one-indicator-and-compare-countries.cy.js b/frontend/cypress/e2e/charts/4.key-indicators/1.analyse-one-indicator-and-compare-countries.cy.js index 0cc7d698..ca6e118c 100644 --- a/frontend/cypress/e2e/charts/4.key-indicators/1.analyse-one-indicator-and-compare-countries.cy.js +++ b/frontend/cypress/e2e/charts/4.key-indicators/1.analyse-one-indicator-and-compare-countries.cy.js @@ -8,19 +8,28 @@ describeResponsive("Check Chart", () => { ); cy.checkChart({ filters: { - indicator: "ICT graduates", - breakdown: "Females", - period: "2019", - unit: "% of graduates", + indicator: "Enterprises with a fixed broadband connection", + breakdown: "Total", + period: "2022", + unit: "% of enterprises", }, - title: ["ICT graduates, Females", "Year: 2019"], - point: "European Union, 0.8.", - tooltip: ["European Union", "Females", "0.80% of graduates"], + title: [ + "Enterprises having a fixed broadband connection, Total", + "Year: 2022", + ], + point: "European Union, 0.5.", + tooltip: [ + "European Union", + "Total", + "0.50% of enterprises", + "This data point is for the EU", + "Data from 2020", + ], definitions: [ - "Indicator: ICT graduates", - "Definition: Individuals with a degree in ICT", - "Breakdown: Females", - "Unit of measure: Percentage of graduates", + "Indicator: Enterprises having a fixed broadband connection", + "Definition: Fixed broadband connections include DSL, xDSL, cable leased lines, Frame Relay, Metro-Ethernet, PLC-Powerline communications, fixed wireless connections, etc.", + "Breakdown: Total", + "Unit of measure: Percentage of enterprises", ], }); }); diff --git a/frontend/cypress/fixtures/import_file_valid.xls b/frontend/cypress/fixtures/import_file_valid.xls new file mode 100644 index 0000000000000000000000000000000000000000..2f527bcffc2dc22cea748f93a9fc3c6a845e8a3c GIT binary patch literal 6144 zcmeHLU2IfE6h3!<+FO3wZK0?jc&Shdv`8gHj6oI(DhWtaiV{OoZ@05$?e=bSyA?40 zEFf=^C_ET}#E|e}H2%ayA2is$`JmC5kN{8M8_~qYV3Y({zi;l{_FA{F6=KBnY|hL% zXU@!-IdkUB-2MK$+6yN?nR`*j!DAAZ8-*&V2*?f4RYg7~5&})xjY6SN6oZ_CyT}9A z{kd`vBiO_!kb7PUG=Y>)^Xt24SyNLj(*qrn!7nS8Y!U~xEBmn9<-;6AOwd$0J%jn8 z9Q|AAOxYjvz^#16_MrcnznRCwz*6U*`JU&0I&cPXCa@Y<1H=^|wZJ;yEZ}TlJ#Y^2 z9^k#e`+#$S^MLmQ=K~i29{^fF&ZWVxNiPQ7=<`jem-_VB3a6+$>47Q}yI2vGFJ^wj zl&^FDMMT8Mkr5piKSH)k1{fP{b~<7GTd+aT!8RD1!wAx~pbR+yX|;T$rC;iDt&Tn@ z#l>2KiZ7BYMqJPRur3>PS&h<=&vl>kTGEQR{2Ka&*WT2Jz>ui19{DdNwxnfTEA;ME z>w?G(ImP?+N#!WlPG8XH=9K(auwbPK!xrV7n3i8DPfl-@(w|7qQA_Fnf90p(Q;W;< zq3Gx3o*(k#_wzF=-b1{dHH=N+@yOqkezJW7q;5TBJK#|VFhWsSYF>~iTw5^IfGc=Dj zW6-hrm7-Chi0ix!(L8*TFS8uI~ok4|qck zC^wbqL7*oxLr&J+hXGPfPby(&GcM?ESJ{0%nLQ5d4>_qUCjX+H9#W{2X}fn27F?B7 zu5uEpYjiAE_1kXWASS$N2ZmN%Dm&yln1yn(U3NO%wKth^&@hnbVgu}Ss==(~WU{J& zi>z99!W=KMgjo*+;R7!&LWsAWmlb1w$%-@Jr5<>Xd8Af*EC#<^d++iOTf5uWb!mA{ zznS$ovY7h_m+nW)NMxDrv0Rr2wC-$KBnxrbwKX<$rux;O^|ab!b!7T&r@2G*4yA3k zp|SPh#`X2M192nXLa*KKRaRsBL2cl5T{dDet2A6%w~Udme9U-U1RS@)(5}1qjiRi$I&{UN*+h+jI}UUkLv~8 zv3Cfqac^AOJTf{iZSr~bv$TT><(~qL5X*BO+8tO(jfC?n!PiJ6-=yVLdC!kv0uYjE zx*r$M6+a%Tlh3$ViL>xwKF6!n3I-MrD%y+F+)Il-AlmudAjehy4Gg^C!f2sV%;YxU zzSgBY!8hO5RKm>+W|CQJ)82$iTaP}{(rI_AbXsZOZ%8GR-nH%ZnRE%qct$7P)9Y{p zOEXFgL5E*B^zN@$x9yI-|CTAsmwbATj(-_QKeC+@N2qF-(m7jz-29zDZhH@q8#(~w zBrjd}-oB()1LHtB9r=a4rpW{1EVgevEnChp--W3R4i;k1+f{eES2PZO^*q3u{N6n)|uz+mmn%N!)k|)hIuOAqAM!AEj zj5F4s?|g<*>*^IN0idSfzvh8E`TeisXIu^_wO+^CKtVae!@|ht`iWiSQmDx)<|)|S!8m6EL)sXsQ?gL{WVH4j&HU z{T`v6>t7(#2nO1RQ$L=u52){(T~DNAr-f_kfKnh!YwuY`UdKU>6Z=+DQyClTmpt5T zev%I~f7V;Mzo%hh7X@?XQ_yR!I59eDJ7>tt3PdRksQkS*$XbJ)-Q9zyyiA zUDgo)951-RON(@86Mx?l!jIkT4Y8$wk`0m15JtXfG>)=h=Lmz}Ta%*PXObMBDlpuF zRVF)OWFr8Q+ZslyrLc$ZQVO?E5?(l?V!H$tR25np_7DYw507Ja84&C}NzkLZ%(eRG!J%JczRXB zgu;zCbMOAQr1YRfSou>8oaJ}Tts(8g}1_e4M0mwDr7o>o1e;tC(mu?c=dJna6K^{b^4KU#ZXdCth< z#o+wX9xPt@hdp`mss_I8(TkhYV}+Z)s`LZiSQXX*cf0%d?=1nv&0dy@rnUEBAv zom57Gnwpa=#C6^2HP+}4XfZysSAEOf_K;noI7MnHsyhhE;iYG~}FRLc(g#r!%H zc`L=^oFk0f`!40mP=lV+eDoh_+3^;?2beJzVx|_RkcN< zlg}hv3_h3fMHS=N$)zy?(6}Fl#_@9QjJ&ZSGM|Y3a$&uEs`0{FlkS1XbXr=;_wt=ZxrK~0X=udI_&8QhUIaS#aE+XQ_7NQEifOc89mk zyw6QYv8}cGDRe(w&?q>q3QBB{$LpVlLrFa>Uy0a91@At0V$>7taF!q0BKu164hzL1 zvkGsw#mLoICyS%K6CC3Gp%uhYDMd%L9Wdx?Na9~1QA4r(YA2U7pX!Ho(r2k!TvvyM z*MnU)If`H6(=c~F9D!&DYFKYsHp#@Hh{4~gLyV$|q!c%gu_I;-#NNg}^Jv;=Nph7sS#{rM;vZ4MH72&vjA0{d z4MzCOGA2CHzyeW~XmP{acr}2Jxx_BF`VF77WE?R}3{x9JmTy?FA{W&)Lv-3%|Gm-O zvV>7xC9h>qeu1L85&h9)CYPlAsmM-=A}uGuszz)3pg~iBMid?Ov$ZhSjhl3gyKYFN z;8Uf=xA(L8yzuccB%{x%G~Ohgn*?8$1Yiz~H~hkI2hQ>AxJQDryHr;{$*e>&V#STL zY&M%C7K0yqY4&P;lg^+eBd9@Y-wULkL@r#!rI3Ky|M9 zjhAkkWRKU=Kv+W4tP1HG)J<3ATDOOM?=w|p{dlE8-5LMck78GKP^M0$?p@rkJ#57? z2Gp_7lSg6V?DNDyYk`>sS+QoLwB;?=!m;TRiJ{a5#X$IOXqeN3910>cAoE8|m0WyX z1(EA&9i2k+V|+(+HXhM$IQjBL+d?>M!>CWcs)oH-Ol6*=_mjiWP*C?jspa6qTDF$Y z2LBWe+@{N8?e$uuE>S8mDyc(YO@)(a>`h%n z745f<#5tw#$<#ZJPpaiFjmXS9I?Vz$1q~CeSe6b))}xEXK5QI3Cy+FXWcdb9Xuj0* zrO~6Y76e|U9Qa_g$n7YR3tVcx=A^^5eL6HaW*$F2z!?7HyVMEAxtXY}w^3aq1ppv4 z|2C6q4Qdvm|1XJFG*C=Z!hiC@4UHsuWCVk%!4x;cjaKh6)ooTScgCKjL?k+BR^xlVjo6U*CL=cB3nbhGf;TjTO`aV`R zs$bq&HuK(AV~=>Yakw^)73f%VgK?xf`lNQKg?|9W?wav7wKlk7H?&nO-UAMC|@A z`t1O^ejOzko=o1TqG?76M*VF%Tx@F+syInQQHrYl?jH#ql`(< zIY{U4XSz6Pa`kyz3;AuDKWc+Y}pculB%i1PP{%PL@Ry3 ze&?SXsiLYSSn_et*`#CLUcg=c`D_)?x8b^qi%-w}#V=!^>y05E-45dW2Nal&+0 zF*;^-SbdHf$0;gDd9Merg^}j(6UAUc!l!{m5}$9pTpb^?J%oMNDqaOBYC4-5GQ)aX z+^*zJ)-G53^`%^|ELA5DN6_}eP8W~@L<8gy!e|H8ejT+*0?6dqM>inAOD)Q+;PV+QHVf-;b@gMTM;`a{7Nsv>$*cBmwD3DH26n%`B(Q57aeD6|d_7`%Ix8CFS z=KN%lJw!1gyw0)(OiuU-!e?NLtnF2P4Kymt9VHP*XiHn2C&I}Ye!M*y#`ufRU-sAL zH_2lpo!#vE{tO!8=aL)fFwJ_tvbixpP|DeFZ=T)%%01TfFwB5nJ11|7` zY$L7|N^uPxkEJNfMa%q2C;mS1d-c&it=7|IA*#i%Sby(h{ro$R`uSL?V2hd8moBg*Dz;f)3No5{Jm3zo1&@6=0vxWaryJM9c*E z-B!~5?koRkv2f%d5uWpYD#lm*$KDy-^~^Hq|B>JML=blybm=$dVxk59fic6yfw+ES zE*2`bKQJP=9+2Q~jNZntC&Op&U}t8)@dqXsM-LYIjk##yOn+dSaCgA`zcCk01@s5z z1eXTp{f)VJ&KUl{6yYYoJl5xHzo^*r1NYxCe@4asIDQxS{7vUYuK)Z%0|O3C&x{jc Uq@<$;Py>R_R%~_F4*&rF2UR}gWB>pF delta 3664 zcmZ8kc|4Te`yR$J*1^mq+nDT=t>R_ULXmw-N|uD7>^oBpCi~dc5XM@z2-&j}N@LCX zTC?RP5<(-R={tIVzfXG4^Uw2~=Q{W2e6IVt?{l49WV-~Xu^x~C3IKsX0McK!Hy&{c z;RbYpr@10G0;3bLm|B>IeN{95eY$qw!_;1?t#aBaKEFV0D9EBHpE*G%XuD5YojAjD zXE!=SD_YZ|*g`rs{@^$^H=C2ZCeZ9+)+gG3Lstxp?k*nbblpKbcBiWpI5lo{wf&~G z%7RV4V111+aW^8#VJYQBYnqXmkJj{>X9pL|o+ZkX zr8MOgnz)SyG_GCG;Wun_mB)O?8iJ(8EGy>lUYT{KRk5k9e&_2Wy)#uI%VA|ypR;u! z%>-)GV}nmUj-zdZl8&kXV?Acjo#K#Ja{vI~96f;ceUtzIph}rej-KI=41yC-7-wWO zbxN>#JZ}Xh*JtnoX81-bE%y=G6#~Cw;cC&C}$0a!`dbMB4I_6Q}V6k zgRh^T`x9c4=T;s$cO+%ZuSj*`yY-j!pC|-0rsB?byt^sl)1I8CFl`(vKLn1d;3!mv zXEZp8TM8F+)f_)W?>i zd&I<|d{F(J@~k(Y@U*7&7u_YBCFx{SNPMUwi%PH}GHH6Ha~$Yomo+FlkKk$W)?wIP z)az@ye2HN(j#;uaPx6Zzm>lszNe=qn@kqMD(nlNK!C4sk7&Hp@VTgVgbJYEbAk`KE#lXt(};h>{40` zl(Z}5d2jABJz`zI(YC|4bnte4xY2I=+X;MG^m?H96rC$>xq0nDzPs8gQQ2onqr`fM zb8q@dY>r;PE^;NN-KA8f59dlrtMMcz`kg3Fx-)=SKUV&^wuW+2wny;eEU&1Z*|*5b z&KD00B@I>?9|q#}|5@+c>@_8lBEyMQ)YF}&WF;=M31Hvf64FnAe#U$y(Y!k0=^4*% z`K$)Cusegm1PhN)GWw49sqKig%+}J-wEur@^>%hqcJ%mgTG%Ct57NqnKyM#vb{|8> zVxwv=xPNP~tZVOj#d^TXvT$i@s6ycTD`)EvyA_68fGb~$&vwktWTt~3H{L$up0<-~ zlkS-xhFIHQvDZKa5grCbv_2T})gZR$S~wdxjO}C$b4yYocNPDhz+?9LH;Ak3V>yKx z2~yP6Miz(|Q+lh`(CndTV~rx9p)|pwx&JF#f(nx|&Rkbevgor&&7yCxBIklH+vm7r z$QaO@t2*88U)M0&fj6HVi?IQxtBdrVzj(3b=^Y*%HJQz0An6~(-5?Y{1x{cwmcuCg zq-?J)=5IKui;2^VrGI~^Psv_xeZy=luY&Hqr;XU{9f(Dig{n#N^+drhy%xJ;Vtn@W zUa^0$91bcvBHMk1E3S+8l40GBbcD+;oKyQ+R=K#CLw{b8HWbpC6Og%eS=qI9rE%}M z2=jU8+(J<=a|er-&49L)ZH?fFkyxI0E>0cStQpNDmwO3hP*2nYWK4p#)!clomFM=V z;{NvdBh{1gOYNt2Ii$wK#V;%L!tA@0dIRLGKG2Fvl!@qjDk1d2*7*WK*=h$eRPQd; z;9elFc^L43G;s-&bs_V$ZTPD)n_2k<`HJ?9wjtsXgI^l_oEY3I@AthzSkQNyn6RS7=lpYsdvMr5fYUlunnY?dg{nWR1#GG>4mCoAr-Pd@x zN7KlXX2uF(!_sy#NRwP*;vs`_~TKkS4txAB;_+vmqOUSkocg9h0 z_xwH&$A}U3BA7enLUttnMb?12#<=vJ^K+$)_lLsikZNsb4xiaOJFnzZ@f!Y%3ow8xR8I%)I@5iJ+WVQ=>9#ndf3EpaYGgE9SAO_xn`r_ae)l>>oD6hrhO2_sw?xZxeKA*ca`r`hS-xE zOk>+Hf9+ocILBD=WQOPD}u(75Uj{HnWwQVzJ^`=r5s^OOC}Pi*PF>s^Q~Mv<-W-5w)$BUpZt-KLSZkv|w1c^V!WAFVy3o{%ynSv4re< z;A3#3!s5iOm)qYj-FOX;h1C$BibY=qM2}iM_l~e0i}8%6Uwgh#T@34e&n-jbRX#f$ zHp-M`Q2H80r?jcr%~NpKaVDencXA8=y7GS6*C7ke)YHvwQRB`PFP&5M%fetNAtaeyG) z4&1E#sp$Q7s_Iaf7=~ArgXxw75Iab17Wot%?=)326Bt16^FtfGfR?mkyl_4YM@4V_ zN`$3M+b1*~05J7C5wtw~E6O@fIh+ns0FlJxM)bct$tg5h+RejzW=T&cy1Vr@fCDuk zNvYW#5Bl^#u>qxq@19^X@Cs)wZ`X0aaO}Izc8j01Te#u9E~B^t;x3&4{s8r>YqQ!t zVo_3Z%zk{Eo97LjJpuK~?aPkb@hopX|IXcJp3T)xn>Gm(v(b#U)HdD3d|WI2V0fKM z-p|YVkV48r?S=hbbT#ARUq)?G_4%cGR-6MFt}5NfEZHu; zwqEx!iL`lneESMVG}c+hu_Wm&67IQn^3MH=kFfIXt-TPKCiD9f9FIcvt8fBR--}sD z>r4rl29P9aw@hAlDa>-}T?n&G)lG*?6m#4LU0DsvJ0%E}i+9Ba$d;;7tY5FLYO58e z%4A_gtz~`HC+Y3rn+=)J#NK)~vXw0m8?SR*2xXKqRAB4zR`zzR`4H7*!&t!q$;MT1 z6B%agA>=vl@-T;gvdq*AS7fw8DR*lFmwtFmMcilGEV(5COdsFQ-mDNH&kItWc3S@7 z>_mI+wJ>M5oxo&+I|ni=;5a7cR^_dF} zX#d1%^~3)MMwt){LH@!VZFbNfAA7y~-OpP4S0Bme*fQST*A>fALA^)2}Ar5Z*d{^g+ zls5Ga0sM$vzZk9~kuS2?1ux#IDZOJ7rf+3+qInx#)ERp4;{gdVe*Bw`1lNGT3yYFD zH7A}q0d`f>XkEi-R5-gOth96B!qtagbf_?(n@Id2*jb(nny8`jZE)d!f3cr?k_ztH zAxL!;<*VhE6+lWh5^AFBC3s;KwHju)6-Vr9qgOZ;h~69qa{U@K@-G!{CMJMH;QGu#V2uNa-LzX&_yS5GqD-dMonG?s`UIx)w1M@fPf3~@SD`Kz z-!O-oiNgeJ-2Ff-FeSB}d@?#AGnH7QXFms4NiwPUk-|gq{n@CNBtm{dQ75th8r zSWIi>`05)+!QJ>n3r6u#g}s+#({-V~%URobUu_C-VKGz{YX_?V&`BFDa|OKI1nZ?a zFDwiF$Ai`j_bliSwhHKU2H8=hxWiws#fX!t=g|glyd@V|Zdvc$ekCKdg1KQD!K*8| zi}!NNJ%m&ems?OdZb(kE^)p4X{MO^YzT~d*1MyclPM=4~O+f`U1I94!zrxw34m275 zG#B(hoi*}4u&&yrJCXd0k8a9G_e)g}*a}nfT%7aHuK9z(faZL~aVyby%uz{Ng7!U6 z!D9hx4a{}N?6LV2kf26j$}8H9kO1X}h{6KrWt9Hya6P{r);iBL|Ne~&I-wT*&JBvI zoQd@o+CD-70G!bNlrsc>=Zu@Xj|0T*Dr2?{Of+T%NMVSo>Yvgn8DMjjP)hAmxSA~~ zj@?YCa&BikL-Mv&w-_fJ#FaXs0(-ODC)L9-Ir%bY&4Nwbw|GW?&BoiE)Zzi8 z5X^o^_1o;^6en?p)YswN&VO>DO%^30Q>n2@@}zkx7QZuG28j;+d1hg?-hZK);Z3&ptysAcP|eMpY6lm% zClqdd@VH?0l^;HY>n=@H`Eu}!ioTE%*=-1B`IbUSyGNel>_plA$s0{rgIQS}!H=H7 zWAjx4A>;gt(5(ME90%1Byw)?Jf-{_Asd-SLIX1Dcm(yw^E$`0;{4eQSwnYNc9VK_q zd`iE=-i^1EI$Sz+96f(>e>@1@l5QfT?n?eAbht9=)PuW z4`WSO($Pzn^e|$K4;6O|hL18shzgX_i?_sE0wO<_mRJydtCpTxb@{eBL<3bE?a9oQ z@M9(;^IyOuL3m(mpuh01cWIHSm@{ii7>ly(MCZUeQc63!bHOc%eez0Bt`-gOi1)MF z=3@V$PD(OuS@dn}$UL21T04XOyi#0cU>)JgARg?peSm1iDcpq(5yCW<#=fk`3Q=#U zlx2Oaga%6=l@qb$KL&Xl6rvT+?5%4xmw6gB&-;gEjz7R<14}7CFx959kvLlA@LBCM zj#^QY&g}97@G&;>LUb5-ABrXL9EZAt`-PJc=n+lDQJkoL7&FNPqX@irRr#H0l-Z8_ z3C*Vq;_alY!VoW3Wr`gS7Y`b0!p*(Vl+cZHFCF$<-1GW^#nj^V&FQ2FE%nxy5JMV< zuE(5H6hKsUy}%^~)EE&=WiKj$W|lH1CXpE@Y?f&DhRe@QAMryR5>E80$pNXP>1s5% zK=t14_gLL{nKTkqvN7#p2E^TJ*_El3h^nAK*B!|!x#b-6J3!98^SRSGY3W*OiQN&c zG`$Sp$wv_tzz^M4E--D@obcp&+1165Jddz~Jv6vP&G^qYH?$s}zrhHG*O_8AT()oy z^*^J$?NNdL*Iuec9!izDVBU$%eQ`%fNTENIB6Ny?19(CfGivbUM4{x6Zkh!1B0~d# ziSqJWbLz1xdN$XY<44F}nlA!GY{6W-2iXR?1#uQ?$IcxyPbGW}8)Wa4e!k#>36lvL zO`hawyl~^zBx`J{Q7RHoS)6ax!p?3b>l4+>ybOAG}<0PtLq1It|7A)u-9?^8+#8IRD98D>F1pan%_|=;rJwsq)L6ys=ug^`YcxMeKzSa3o zdkRh70ovOwnY+$wP%YT^k|$&oC%uBn7&L?@88PF6yazN7{D<~G#BF%qliU*>p;5r8 zT`uVzij5E)4m)B5tOgGfs+v!sL2v4-zx8M`$o`ieJttE<>QctCj1z~)-LE=7M(NgC9O)EInRt5{z(-?@ z3)vb;9hp3Quk?LGIDR9lQyw4lWg&WaTbCy`JotQPdx1)CsBmuHmMyWGfw;VxWGaY4 znct}9P~4EuqCM%ZNpYWwdf8-OQ|Pu9|Gf?H-q ze)$5u1UN%b`edO|aH7RLH@_g(H^-9|sTCP!x+#|as?dC}s29#BGi&-?Up!}=*$T~o zt)Hxi2lZJB6;*Rz{86uv(bzlj9Yel1+aEP2>A&@Z?PDm!O4}W3=VbFsM#PW3cIsdR zV$Q=gf~RP5@pZ`_3+fD!2-heqI20|0LT$u+&c^W5n{?t@y^c0BM@lz|yD-D$+D&)7 z@<@aDPJ*=g`JKP^*l1`X8}dwA zW|6L-PK@KXs_H~m`hjB!ZilOKL*u`wzP31iw6Jp2Zi}mkkWWafD4+JgyFV?PjnBqf zMMhH*FeUSiUdst2`kZhW$^|-wXPoY>UN9XJTmiNC5iIf*2}AH7fg(cQVdZS833YaHP(ocFXkH_OC+Da_q2aWEt7JmN;YN>l0G#kH-FgC9)i&bXWdB3~qcW0Y<;#WBrF zaP3 z)oFC0K>Esv`b7lr8o%P(nmL)9E=BfyM}-hYTjV#8RQAQGTTp`#BtJI;?5BF(4jcW* zZ4k%mc&jO1c<$nHs10u0Mt9ox3UmyLUHncnMe3T_>;l@?rL| z)#}6g%_L3e*B4Ecb^>c$_EWJxb|K@HYftM;a=!F%xXdOBv-8!pXN+{nAAgmxMqEXz zK$DEx9w~`u{}?IY%}70Vak&aqUJ}T;{i@|WBh&4C3o@uy`mrTVBh zjh~H^>oNdEc>Z4g*5ric8QZ3E`4&K0$=*nZjIF=bg}ZqA?OKiBK;~_{WU96_k&!|c z^x7AM%N@p8(kj@7%uh!OVR263uJe8)JfvlgV4G^wnx`r)9iEh|k@^D<3D_5#cxdpo z(OG1dcW@t2EN%cV@yIgrckpI+)`$~RZq;h@pt2ojc!~;sQYNUaObMvyt#MO%U0yL*bP5DH4?&BzFJYev;h8&C{aN}8)fadYkk#lrkpLUs}&8Kl+6rOxGF0jG(cH1czE0(c2-aqv2ymxwF3 z_05~phdp1xMbk8?(v*6lb>TzfX(c$7HMGu5R{4B^*-|V=l3#}cXo1z zIJukY__#paOn!+;ogs}5M$(quD%*za_SH7ZaVIR9gr-g(QS2(-N`LgJ&li3LpCDOp z-3*taTm4%2O&N0>t@2*ANfziPuzSZ5YpS^w^wYw5=*R4Za}wTkO$4dRRY4_QHj9ac ziR%Ry-?B&J(!q6P`~KKo)#A4C7;*24x4*(uJ^C~a*s|>S)-BwomoR7xPEUnXP|960>0!Sy8D<;%<|lmF7A=jEq)l3NQ|(F@iIl%O7CQE7hoBioRZwSuCPEx}d$V`&L;}x4z zoB5vN{G!*p^@LW)#QTVG6(kf?Qo!|)=;mDHYDo0oc4K7pyW`E~_-br&Z5+r){We7T z-T7uCa@9Xyn=!K2Mmk@2)4#jklp9x_%C+%e{^z;>uW$K%m7AscN}60-FjB@KH}DTa z<@a@NR<5f;er-8OG_PLd_p<)`3O5<`s={4cC;tDacfY&eB$g}My*8&?H{AbC!oPdp z{Cuy-=h|K%V|ra||3gK;FR*wcwf=j70OW4|$2YI3ih&Fh0Dy};f{;ao1=)%M0RIA~ Cp0bAk literal 0 HcmV?d00001 diff --git a/frontend/src/components/charts/base/BaseChart.vue b/frontend/src/components/charts/base/BaseChart.vue index 0aceae91..38d951b2 100644 --- a/frontend/src/components/charts/base/BaseChart.vue +++ b/frontend/src/components/charts/base/BaseChart.vue @@ -385,6 +385,16 @@ export default { ); } + if (fact.reference_period) { + result.push( + `Reference period: Data from ${fact.reference_period}`, + ); + } + + if (fact.remarks) { + result.push(`Remarks: ${fact.remarks}`); + } + return result.join("
"); }, }; diff --git a/frontend/src/views/chart-group/MetadataView.vue b/frontend/src/views/chart-group/MetadataView.vue index c574616a..d7103b94 100644 --- a/frontend/src/views/chart-group/MetadataView.vue +++ b/frontend/src/views/chart-group/MetadataView.vue @@ -77,6 +77,26 @@ /> + + + reference_period + + + Reference Period + + + Free text reference period + + + + + remarks + + Remarks + + Free text notes + +