diff --git a/digital_agenda/apps/charts/models.py b/digital_agenda/apps/charts/models.py index 2c9541cc..975ee5c2 100644 --- a/digital_agenda/apps/charts/models.py +++ b/digital_agenda/apps/charts/models.py @@ -81,7 +81,7 @@ def clean(self): raise ValidationError({"period_start": error, "period_end": error}) def get_label(self, dimension): - return getattr(self, dimension + "_label", dimension.title()) + return getattr(self, dimension + "_label", dimension.title().replace("_", " ")) @property def facts(self): diff --git a/digital_agenda/apps/core/formats.py b/digital_agenda/apps/core/formats.py index 090324a2..1833a882 100644 --- a/digital_agenda/apps/core/formats.py +++ b/digital_agenda/apps/core/formats.py @@ -13,7 +13,24 @@ logger = logging.getLogger(__name__) - +UNIQUE_EXCEL_COLS = ( + "period", + "country", + "indicator", + "breakdown", + "unit", +) +VALID_EXCEL_COLS = ( + "period", + "country", + "indicator", + "breakdown", + "unit", + "value", + "flags", + "reference_period", + "remarks", +) DIMENSION_MODELS = { "indicator": Indicator, "breakdown": Breakdown, @@ -23,6 +40,10 @@ } +def empty_cell(cell): + return cell.value is None or cell.value == "" + + class DimensionCache: def __init__(self, model): self.model = model @@ -60,46 +81,41 @@ def __init__(self, path): def load(self, *args, **kwargs): ... -DEFAULT_EXCEL_COLS = ( - "period", - "country", - "indicator", - "breakdown", - "unit", - "value", - "flags", -) - - class RowReader: def __init__( self, row, dimensions, - cols=DEFAULT_EXCEL_COLS, + header_columns, extra_fields=None, - required_cols=None, ): self.dimensions = dimensions + self.header_columns = header_columns self.row = row - self.cols = cols - # By default, all columns are required except for value and flags - self.required_cols = required_cols or self.cols[:-2] + self.row_dict = self._get_row_dict() self.errors = defaultdict(list) self.fields = {**extra_fields} self.read_row() + def _get_row_dict(self): + result = {} + for field, index in self.header_columns.items(): + try: + cell = self.row[index] + assert not empty_cell(cell) + except (IndexError, AssertionError): + result[field] = None + continue + + result[field] = cell.value + return result + def add_error(self, col_index, error): col_letter = get_column_letter(col_index + 1) self.errors[f"Column {col_letter}"].append(error) def get_value(self): - col_index = self.cols.index("value") - value_cell = self.row[col_index] - - value = value_cell.value - if self.empty_cell(value_cell): - value = None + value = self.row_dict["value"] if value is not None: try: @@ -109,20 +125,17 @@ def get_value(self): value = float(round(decimal.Decimal(value), 6)) except (TypeError, ValueError, ArithmeticError): self.add_error( - col_index, + self.header_columns["value"], f"Invalid 'value', expected number but got {value!r} instead", ) return value def get_flags(self): - col_index = self.cols.index("flags") - flags_cell = self.row[col_index] - - flags = flags_cell.value or "" + flags = self.row_dict.get("flags") or "" for flag in flags: if flag not in EUROSTAT_FLAGS: - self.add_error(col_index, f"Unknown flag {flag!r}") + self.add_error(self.header_columns["flags"], f"Unknown flag {flag!r}") return flags @@ -132,53 +145,58 @@ def read_row(self): """ self.fields["value"] = self.get_value() self.fields["flags"] = self.get_flags() + self.fields["reference_period"] = self.row_dict.get("reference_period") + self.fields["remarks"] = self.row_dict.get("remarks") if self.fields["value"] is None and not self.fields["flags"]: # Set the custom flag "unavailable" for this case self.fields["flags"] = "x" - for col in self.required_cols: - col_index = self.cols.index(col) - cell = self.row[col_index] - - if self.empty_cell(cell): - self.add_error(col_index, f"Column {col!r} must not be empty") + for col in UNIQUE_EXCEL_COLS: + if not self.row_dict.get(col): + self.add_error( + self.header_columns[col], f"Column {col!r} must not be empty" + ) for dim_name, dim_model in DIMENSION_MODELS.items(): - col_index = self.cols.index(dim_name) - dim_code = self.row[col_index].value + dim_code = self.row_dict[dim_name] self.fields[dim_name] = self.dimensions[dim_name].get(dim_code) if self.fields[dim_name] is None: self.add_error( - col_index, f"Missing dimension code for {dim_name!r}: {dim_code!r}" + self.header_columns[dim_name], + f"Missing dimension code for {dim_name!r}: {dim_code!r}", ) - @staticmethod - def empty_cell(cell): - return cell.value is None or cell.value == "" - class BaseExcelLoader(BaseFileLoader, ABC): """ Loader for Excel file formats. """ - def __init__( - self, path, cols=DEFAULT_EXCEL_COLS, extra_fields=None, required_cols=None - ): + def __init__(self, path, extra_fields=None): super().__init__(path) self.extra_fields = extra_fields or {} - self.cols = cols - # By default, all columns are required except for value and flags - self.required_cols = required_cols or self.cols[:-2] + self.max_col_index = 0 + self.header_columns = {} self.sheet = None self.errors = {} + self.read_headers() @property @abstractmethod def rows_iterator(self): - """Returns an implementation-specific (xls/xlsx) iterator over the active sheet's rows.""" + """Returns an implementation-specific (xls/xlsx) iterator over the active + sheet's rows. + """ + ... + + @property + @abstractmethod + def header_row(self): + """Returns an implementation-specific (xls/xlsx) header row for the + active sheet. + """ ... @abstractmethod @@ -186,6 +204,16 @@ def get_row(self, row_ref): """Get a row object using a reference produced by `row_iterator`.""" ... + def read_headers(self): + for index, cell in enumerate(self.header_row): + if empty_cell(cell): + break + + header_name = cell.value.lower().strip() + assert header_name in VALID_EXCEL_COLS, f"{header_name} not valid" + self.header_columns[header_name] = index + self.max_col_index = index + def read(self): """ Read the all rows from Excel file. @@ -201,12 +229,11 @@ def read(self): row_reader = RowReader( self.get_row(row_ref), self.dimensions, - cols=self.cols, + header_columns=self.header_columns, extra_fields=self.extra_fields, - required_cols=self.required_cols, ) - unique_key = tuple(row_reader.fields[field] for field in self.required_cols) + unique_key = tuple(row_reader.fields[field] for field in UNIQUE_EXCEL_COLS) if duplicate_row := all_unique_keys.get(unique_key): row_reader.errors["ALL"].append( f"Duplicate entry found at Row {duplicate_row}" @@ -242,21 +269,37 @@ def load(self, allow_errors=False): facts = Fact.objects.bulk_create( data, update_conflicts=True, - update_fields=("value", "flags", "import_file"), - unique_fields=("indicator", "breakdown", "unit", "country", "period"), + update_fields=( + "value", + "flags", + "import_file", + "reference_period", + "remarks", + ), + unique_fields=( + "indicator", + "breakdown", + "unit", + "country", + "period", + ), ) return len(facts), errors class XLSLoader(BaseExcelLoader): def __init__(self, path, *args, **kwargs): - super().__init__(path, *args, **kwargs) - self.wb = xlrd.open_workbook(self.path) + self.wb = xlrd.open_workbook(path) self.ws = self.wb.sheet_by_index(0) + super().__init__(path, *args, **kwargs) def get_row(self, row_ref): return self.ws.row(row_ref) + @property + def header_row(self): + return self.ws.row(0) + @property def rows_iterator(self): return range(1, self.ws.nrows) @@ -264,16 +307,20 @@ def rows_iterator(self): class XLSXLoader(BaseExcelLoader): def __init__(self, path, *args, **kwargs): - super().__init__(path, *args, **kwargs) - self.wb = openpyxl.load_workbook(self.path, read_only=True) + self.wb = openpyxl.load_workbook(path, read_only=True) self.ws = self.wb.active + super().__init__(path, *args, **kwargs) def get_row(self, row_ref): return row_ref # openpyxl iter_rows returns actual row objects + @property + def header_row(self): + return next(self.ws.iter_rows()) + @property def rows_iterator(self): - return self.ws.iter_rows(2, self.ws.max_row, 1, len(self.cols)) + return self.ws.iter_rows(2, self.ws.max_row, 1, self.max_col_index + 1) def get_loader(data_file, extra_fields=None): diff --git a/digital_agenda/apps/core/management/commands/migrate_extra_notes.py b/digital_agenda/apps/core/management/commands/migrate_extra_notes.py new file mode 100644 index 00000000..19a52120 --- /dev/null +++ b/digital_agenda/apps/core/management/commands/migrate_extra_notes.py @@ -0,0 +1,21 @@ +from django.core.management import BaseCommand +from rich.console import Console + +from digital_agenda.apps.charts.models import ExtraChartNote +from digital_agenda.apps.core.models import Fact + +console = Console() + + +class Command(BaseCommand): + help = "Migrate extra chart notes model" + + def handle(self, *args, **options): + for obj in ExtraChartNote.objects.all(): + reference_period = int(obj.note.strip()[-5:-1]) + assert reference_period > 2000 + + count = Fact.objects.filter( + indicator=obj.indicator, period=obj.period + ).update(reference_period=str(reference_period)) + console.print(f"{obj} migrated to {count} Fact objects") diff --git a/digital_agenda/apps/core/migrations/0016_fact_reference_period_fact_remarks.py b/digital_agenda/apps/core/migrations/0016_fact_reference_period_fact_remarks.py new file mode 100644 index 00000000..5953b4ef --- /dev/null +++ b/digital_agenda/apps/core/migrations/0016_fact_reference_period_fact_remarks.py @@ -0,0 +1,26 @@ +# Generated by Django 4.2.14 on 2024-08-21 10:13 + +import digital_agenda.common.citext +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0015_alter_breakdown_definition_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="fact", + name="reference_period", + field=digital_agenda.common.citext.CICharField( + blank=True, max_length=60, null=True + ), + ), + migrations.AddField( + model_name="fact", + name="remarks", + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/digital_agenda/apps/core/models.py b/digital_agenda/apps/core/models.py index f41ac37e..332a40a1 100644 --- a/digital_agenda/apps/core/models.py +++ b/digital_agenda/apps/core/models.py @@ -218,10 +218,16 @@ class Meta: @staticmethod def split_code(code): - try: - year, qualifier = re.split(r"[-_\s]", code.lower(), maxsplit=1) - except ValueError: - year, qualifier = code, "" + if isinstance(code, float): + assert int(code) == code + year, qualifier = str(int(code)), "" + elif isinstance(code, int): + year, qualifier = str(code), "" + else: + try: + year, qualifier = re.split(r"[-_\s]", code.lower(), maxsplit=1) + except ValueError: + year, qualifier = code, "" return year, qualifier def _guess_fields_from_code(self): @@ -303,6 +309,8 @@ class Fact(TimestampedModel): "Country", on_delete=models.CASCADE, related_name="facts" ) period = models.ForeignKey("Period", on_delete=models.CASCADE, related_name="facts") + reference_period = CICharField(max_length=60, null=True, blank=True) + remarks = models.TextField(blank=True, null=True) import_config = models.ForeignKey( "estat.ImportConfig", null=True, diff --git a/digital_agenda/apps/core/serializers.py b/digital_agenda/apps/core/serializers.py index be386d3f..b9907d04 100644 --- a/digital_agenda/apps/core/serializers.py +++ b/digital_agenda/apps/core/serializers.py @@ -132,6 +132,8 @@ class Meta: "country", "value", "flags", + "reference_period", + "remarks", ] diff --git a/digital_agenda/apps/core/static/admin.css b/digital_agenda/apps/core/static/admin.css index 7c1512e0..7f4d831a 100644 --- a/digital_agenda/apps/core/static/admin.css +++ b/digital_agenda/apps/core/static/admin.css @@ -164,6 +164,20 @@ form .aligned label + div.help { /* Make the submit row sticky */ .submit-row { - position: sticky; - bottom: 0; + position: sticky !important; + bottom: 0 !important; + z-index: 10 !important; +} + +/* Fix for multiple fields in a single line */ +input[type=checkbox] { + margin: 0 !important; +} + +.aligned .vCheckboxLabel { + padding: 0 0 0 5px !important; +} + +.form-multiline { + align-items: center; } diff --git a/digital_agenda/apps/core/static/import_file_example.xlsx b/digital_agenda/apps/core/static/import_file_example.xlsx index b792655f..07050ac2 100644 Binary files a/digital_agenda/apps/core/static/import_file_example.xlsx and b/digital_agenda/apps/core/static/import_file_example.xlsx differ diff --git a/digital_agenda/apps/core/views/facts.py b/digital_agenda/apps/core/views/facts.py index 9ec5e9f5..e27ce7b0 100644 --- a/digital_agenda/apps/core/views/facts.py +++ b/digital_agenda/apps/core/views/facts.py @@ -268,7 +268,7 @@ def dimensions_by_code(self): def get_label(self, dimension): if self.chart_group: return self.chart_group.get_label(dimension) - return dimension.title() + return dimension.title().replace("_", " ") def render_dimensions_sheet(self): headers = ["code", "label", "alt_label", "definition"] @@ -356,6 +356,8 @@ class FactsViewSet(DimensionViewSetMixin, ListModelMixin, viewsets.GenericViewSe "country__code", "value", "flags", + "reference_period", + "remarks", ) .all() ) @@ -387,5 +389,7 @@ def get_renderer_context(self): "unit", "value", "flags", + "reference_period", + "remarks", ], } diff --git a/digital_agenda/apps/estat/admin.py b/digital_agenda/apps/estat/admin.py index dc8f3f77..c620f0f2 100644 --- a/digital_agenda/apps/estat/admin.py +++ b/digital_agenda/apps/estat/admin.py @@ -77,11 +77,15 @@ class ImportConfigAdmin(admin.ModelAdmin): "latest_import", "title", "tag_codes", - "has_remarks", + "has_additional_remarks", "new_version_available", ) search_fields = ("code", "title", "indicator", "tags__code", "filters", "mappings") - list_filter = ("tags", ("remarks", EmptyFieldListFilter), "new_version_available") + list_filter = ( + "tags", + ("additional_remarks", EmptyFieldListFilter), + "new_version_available", + ) readonly_fields = ( "num_facts", "latest_import", @@ -96,7 +100,18 @@ class ImportConfigAdmin(admin.ModelAdmin): actions = ("trigger_import", "trigger_import_destructive") fieldsets = ( - (None, {"fields": ["code", "title", "tags", "remarks", "conflict_resolution"]}), + ( + None, + { + "fields": [ + "code", + "title", + "tags", + "additional_remarks", + "conflict_resolution", + ] + }, + ), ( "Dimensions", { @@ -107,6 +122,8 @@ class ImportConfigAdmin(admin.ModelAdmin): ("country", "country_is_surrogate"), ("period", "period_is_surrogate"), ("unit", "unit_is_surrogate"), + ("reference_period", "reference_period_is_surrogate"), + ("remarks", "remarks_is_surrogate"), ], }, ), @@ -148,6 +165,7 @@ def trigger_import_destructive(self, request, queryset): ) def _trigger_import(self, request, queryset, **kwargs): + obj = None for obj in queryset: obj.queue_import(created_by=request.user, **kwargs) @@ -157,7 +175,10 @@ def _trigger_import(self, request, queryset, **kwargs): level=messages.SUCCESS, ) - return redirect("admin:estat_importfromconfigtask_changelist") + url = reverse("admin:estat_importfromconfigtask_changelist") + if obj and queryset.count() == 1: + url += f"?import_config={obj.id}" + return redirect(url) def get_queryset(self, request): return ( @@ -213,8 +234,8 @@ def databrowser_link(self, obj): return mark_safe(f'{url}') @admin.display(description="Has Remarks", boolean=True) - def has_remarks(self, obj): - return bool(obj.remarks) + def has_additional_remarks(self, obj): + return bool(obj.additional_remarks) def get_actions(self, request): actions = super().get_actions(request) diff --git a/digital_agenda/apps/estat/importer.py b/digital_agenda/apps/estat/importer.py index 78858c8c..e4e12273 100644 --- a/digital_agenda/apps/estat/importer.py +++ b/digital_agenda/apps/estat/importer.py @@ -23,13 +23,17 @@ logger = logging.getLogger(__name__) -MODELS = { +MODEL_DIMENSIONS = { "indicator": Indicator, "breakdown": Breakdown, "country": Country, "unit": Unit, "period": Period, } +TEXT_DIMENSION = ( + "reference_period", + "remarks", +) class ImporterError(ValueError): @@ -220,10 +224,13 @@ def should_store_observation(self, obs): return True - def get_dimension_obj(self, dimension, obs): + def get_dimension(self, dimension, obs): config_dim = getattr(self.config, dimension) is_surrogate = getattr(self.config, f"{dimension}_is_surrogate") + if not config_dim: + return None, None + if is_surrogate: # Hardcoded value, no label available category_id, category_label = config_dim, "" @@ -237,10 +244,15 @@ def get_dimension_obj(self, dimension, obs): except KeyError: pass + return category_id, category_label + + def get_dimension_obj(self, dimension, obs): + category_id, category_label = self.get_dimension(dimension, obs) + try: return self.cache[dimension][category_id] except KeyError: - obj, created = MODELS[dimension].objects.get_or_create( + obj, created = MODEL_DIMENSIONS[dimension].objects.get_or_create( code=category_id, defaults={"label": category_label} ) if created: @@ -261,14 +273,20 @@ def iter_facts(self): flags=obs["status"] or "", import_config_id=self.config.id, ) + + # Set dimensions with related models unique_key = [] - for attr in MODELS: + for attr in MODEL_DIMENSIONS: obj = self.get_dimension_obj(attr, obs) unique_key.append(obj) setattr(fact, attr, obj) - fact_collection[tuple(unique_key)].append(fact) + # Set text "dimensions" + for attr in TEXT_DIMENSION: + dim_id, dim_label = self.get_dimension(attr, obs) + setattr(fact, attr, dim_id or dim_label) + for key, fact_group in fact_collection.items(): yield self._handle_conflict(fact_group, key) @@ -313,8 +331,20 @@ def create_batch(self, facts): return Fact.objects.bulk_create( facts, update_conflicts=True, - update_fields=("value", "flags", "import_config"), - unique_fields=("indicator", "breakdown", "unit", "country", "period"), + update_fields=( + "value", + "flags", + "import_config", + "reference_period", + "remarks", + ), + unique_fields=( + "indicator", + "breakdown", + "unit", + "country", + "period", + ), ) def run(self, batch_size=10_000): diff --git a/digital_agenda/apps/estat/migrations/0014_rename_remarks_importconfig_additional_remarks.py b/digital_agenda/apps/estat/migrations/0014_rename_remarks_importconfig_additional_remarks.py new file mode 100644 index 00000000..0c34ada7 --- /dev/null +++ b/digital_agenda/apps/estat/migrations/0014_rename_remarks_importconfig_additional_remarks.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.14 on 2024-08-21 10:05 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("estat", "0013_importconfig_conflict_resolution"), + ] + + operations = [ + migrations.RenameField( + model_name="importconfig", + old_name="remarks", + new_name="additional_remarks", + ), + ] diff --git a/digital_agenda/apps/estat/migrations/0015_importconfig_reference_period_and_more.py b/digital_agenda/apps/estat/migrations/0015_importconfig_reference_period_and_more.py new file mode 100644 index 00000000..13f7cea1 --- /dev/null +++ b/digital_agenda/apps/estat/migrations/0015_importconfig_reference_period_and_more.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.14 on 2024-08-21 10:13 + +import digital_agenda.common.citext +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("estat", "0014_rename_remarks_importconfig_additional_remarks"), + ] + + operations = [ + migrations.AddField( + model_name="importconfig", + name="reference_period", + field=digital_agenda.common.citext.CICharField( + blank=True, max_length=60, null=True + ), + ), + migrations.AddField( + model_name="importconfig", + name="reference_period_is_surrogate", + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name="importconfig", + name="remarks", + field=digital_agenda.common.citext.CICharField( + blank=True, max_length=60, null=True + ), + ), + migrations.AddField( + model_name="importconfig", + name="remarks_is_surrogate", + field=models.BooleanField(default=False), + ), + ] diff --git a/digital_agenda/apps/estat/models.py b/digital_agenda/apps/estat/models.py index 17aed116..c69851bd 100644 --- a/digital_agenda/apps/estat/models.py +++ b/digital_agenda/apps/estat/models.py @@ -67,6 +67,8 @@ def default_mappings(): "country": {"EU27_2020": "EU"}, "unit": {}, "period": {}, + "reference_period": {}, + "remarks": {}, } @@ -94,7 +96,7 @@ class ConflictResolution(models.TextChoices): help_text="Assigned tags used for filtering and searching; has no impact on the data import", blank=True, ) - remarks = models.TextField( + additional_remarks = models.TextField( blank=True, null=True, help_text="Additional notes/remarks" ) @@ -141,6 +143,12 @@ class ConflictResolution(models.TextChoices): period = CICharField(max_length=60, default="time") period_is_surrogate = models.BooleanField(default=False) + reference_period = CICharField(max_length=60, blank=True, null=True) + reference_period_is_surrogate = models.BooleanField(default=False) + + remarks = CICharField(max_length=60, blank=True, null=True) + remarks_is_surrogate = models.BooleanField(default=False) + period_start = models.PositiveIntegerField( null=True, blank=True, diff --git a/digital_agenda/common/export.py b/digital_agenda/common/export.py index 6bb86710..a9ead5f9 100644 --- a/digital_agenda/common/export.py +++ b/digital_agenda/common/export.py @@ -49,13 +49,15 @@ def export_facts_csv(filename, chartgroup=None, indicatorgroup=None, indicator=N assert filters, "At least one filter must be provided for export" query = f""" - SELECT core_period.code AS "period", - core_country.code AS "country", - core_indicator.code AS "indicator", - core_breakdown.code AS "breakdown", - core_unit.code AS "unit", - core_fact.value AS "value", - core_fact.flags AS "flags" + SELECT core_period.code AS "period", + core_country.code AS "country", + core_indicator.code AS "indicator", + core_breakdown.code AS "breakdown", + core_unit.code AS "unit", + core_fact.value AS "value", + core_fact.flags AS "flags", + core_fact.reference_period AS "reference_period", + core_fact.remarks AS "remarks" FROM core_fact INNER JOIN core_indicator ON (core_fact.indicator_id = core_indicator.id) diff --git a/fixtures/importconfig.json b/fixtures/importconfig.json index 5a7877fe..acf5b5f2 100644 --- a/fixtures/importconfig.json +++ b/fixtures/importconfig.json @@ -5,7 +5,7 @@ "fields": { "code": "educ_uoe_grad03", "title": null, - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-12T10:00:00Z", "datastructure_last_update": "2023-12-12T10:00:00Z", "datastructure_last_version": "29.0", @@ -58,7 +58,7 @@ "fields": { "code": "demo_pjan", "title": "Eurostat total population", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-09-28T21:00:00Z", "datastructure_last_update": "2023-09-28T21:00:00Z", "datastructure_last_version": "31.0", @@ -113,7 +113,7 @@ "fields": { "code": "tec00001", "title": "Gross domestic product at market prices", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-20T22:00:00Z", "datastructure_last_update": "2023-12-20T22:00:00Z", "datastructure_last_version": "53.0", @@ -168,7 +168,7 @@ "fields": { "code": "lfst_hhnhwhtc", "title": "Number of households", - "remarks": "", + "additional_remarks": "", "data_last_update": "2023-06-15T21:00:00Z", "datastructure_last_update": "2023-06-15T21:00:00Z", "datastructure_last_version": "17.1", @@ -228,7 +228,7 @@ "fields": { "code": "isoc_sk_dskl_i", "title": "Digital Skills Index 2019", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "32.0", @@ -335,7 +335,7 @@ "fields": { "code": "isoc_sk_dskl_i21", "title": "Digital Skills Index 2021", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "10.0", @@ -451,7 +451,7 @@ "fields": { "code": "isoc_ci_ac_i", "title": "Individuals - internet activities", - "remarks": "", + "additional_remarks": "", "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "39.0", @@ -560,7 +560,7 @@ "fields": { "code": "isoc_cisci_pb", "title": "Security related problems experienced when using the internet", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "32.0", @@ -651,7 +651,7 @@ "fields": { "code": "isoc_cicci_use", "title": "Individuals - use of cloud services", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "32.0", @@ -737,7 +737,7 @@ "fields": { "code": "isoc_cisci_prv", "title": "Privacy and protection of personal information (until 2016)", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "31.0", @@ -825,7 +825,7 @@ "fields": { "code": "isoc_ec_iprb", "title": "Problems encountered by individuals when buying/ordering over the internet (until 2019)", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "33.0", @@ -911,7 +911,7 @@ "fields": { "code": "isoc_cisci_ax", "title": "Activities via internet not done because of security concerns", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "32.0", @@ -997,7 +997,7 @@ "fields": { "code": "isoc_ec_ibuy", "title": "Internet purchases by individuals (until 2019)", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "32.0", @@ -1094,7 +1094,7 @@ "fields": { "code": "isoc_ec_ib20", "title": "Internet purchases by individuals (2020 onwards)", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "36.0", @@ -1185,7 +1185,7 @@ "fields": { "code": "isoc_sk_cskl_i", "title": "Individuals' level of computer skills (until 2019)", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "32.0", @@ -1281,7 +1281,7 @@ "fields": { "code": "isoc_sks_itspt", "title": "Employed ICT specialists - total", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-05-03T21:00:00Z", "datastructure_last_update": "2023-05-03T21:00:00Z", "datastructure_last_version": "25.0", @@ -1332,7 +1332,7 @@ "fields": { "code": "isoc_ci_ifp_iu", "title": "Individuals - internet use", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "36.0", @@ -1422,7 +1422,7 @@ "fields": { "code": "isoc_ci_im_i", "title": "Individuals - mobile internet access", - "remarks": "i_iumc, i_iump", + "additional_remarks": "i_iumc, i_iump", "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "32.0", @@ -1513,7 +1513,7 @@ "fields": { "code": "educ_uoe_grad04", "title": "Graduates in tertiary education (STEM)", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-12T10:00:00Z", "datastructure_last_update": "2023-12-12T10:00:00Z", "datastructure_last_version": "28.0", @@ -1563,7 +1563,7 @@ "fields": { "code": "isoc_bde15b_p", "title": "Broadband and connectivity - persons employed by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-08-29T09:00:00Z", "datastructure_last_update": "2023-08-29T09:00:00Z", "datastructure_last_version": "32.0", @@ -1668,7 +1668,7 @@ "fields": { "code": "isoc_bde15b_p_s", "title": "Broadband and connectivity - persons employed by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-08-29T09:00:00Z", "datastructure_last_update": "2023-05-25T09:00:00Z", "datastructure_last_version": "5.0", @@ -1747,7 +1747,7 @@ "fields": { "code": "isoc_ci_in_en2", "title": "Internet access by NACE", - "remarks": "", + "additional_remarks": "", "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "35.0", @@ -1840,7 +1840,7 @@ "fields": { "code": "isoc_ci_in_es", "title": "Internet access by size class of enterprise", - "remarks": "", + "additional_remarks": "", "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "6.0", @@ -1907,7 +1907,7 @@ "fields": { "code": "isoc_cismtn2", "title": "Social media use by type, internet advertising and NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "6.0", @@ -2009,7 +2009,7 @@ "fields": { "code": "isoc_cismt", "title": "Social media use by type, internet advertising and size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "32.0", @@ -2085,7 +2085,7 @@ "fields": { "code": "isoc_ciwebn2", "title": "Websites and functionalities by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "6.0", @@ -2179,7 +2179,7 @@ "fields": { "code": "isoc_ciweb", "title": "Websites and functionalities by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "33.0", @@ -2247,7 +2247,7 @@ "fields": { "code": "isoc_ci_it_en2", "title": "Type of connections to the internet by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "33.0", @@ -2346,7 +2346,7 @@ "fields": { "code": "isoc_ci_it_es", "title": "Type of connections to the internet by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "6.0", @@ -2419,7 +2419,7 @@ "fields": { "code": "isoc_ec_eseln2", "title": "E-commerce sales of enterprises by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "34.0", @@ -2516,7 +2516,7 @@ "fields": { "code": "isoc_ec_esels", "title": "E-commerce sales of enterprises by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "7.0", @@ -2587,7 +2587,7 @@ "fields": { "code": "isoc_eb_bdn2", "title": "Big data analysis by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-05-25T09:00:00Z", "datastructure_last_version": "5.0", @@ -2682,7 +2682,7 @@ "fields": { "code": "isoc_eb_bd", "title": "Big data analysis by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-05-25T09:00:00Z", "datastructure_last_version": "31.0", @@ -2750,7 +2750,7 @@ "fields": { "code": "isoc_cicce_usen2", "title": "Cloud computing services by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "6.0", @@ -2846,7 +2846,7 @@ "fields": { "code": "isoc_cicce_use", "title": "Cloud computing services by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "34.0", @@ -2916,7 +2916,7 @@ "fields": { "code": "isoc_eb_iipn2", "title": "Integration of internal processes by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "6.0", @@ -3020,7 +3020,7 @@ "fields": { "code": "isoc_eb_iip", "title": "Integration of internal processes by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "33.0", @@ -3096,7 +3096,7 @@ "fields": { "code": "isoc_bde15dip", "title": "[discontinued] Integration of internal processes by NACE (E_SISORP)", - "remarks": "E_SISORP is discontinued", + "additional_remarks": "E_SISORP is discontinued", "data_last_update": "2023-08-29T09:00:00Z", "datastructure_last_update": "2023-08-29T09:00:00Z", "datastructure_last_version": "31.0", @@ -3189,7 +3189,7 @@ "fields": { "code": "isoc_bde15dipn2", "title": "[discontinued] Integration of internal processes by size class of enterprise (E_SISORP)", - "remarks": "E_SISORP is discontinued", + "additional_remarks": "E_SISORP is discontinued", "data_last_update": "2023-08-29T09:00:00Z", "datastructure_last_update": "2023-05-25T09:00:00Z", "datastructure_last_version": "5.0", @@ -3256,7 +3256,7 @@ "fields": { "code": "isoc_eb_ain2", "title": "Artificial intelligence by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "6.0", @@ -3349,7 +3349,7 @@ "fields": { "code": "isoc_eb_ai", "title": "Artificial intelligence by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "32.0", @@ -3416,7 +3416,7 @@ "fields": { "code": "isoc_ske_itspen2", "title": "Enterprises that employ ICT specialists by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "28.0", @@ -3509,7 +3509,7 @@ "fields": { "code": "isoc_ske_itspe", "title": "Enterprises that employ ICT specialists by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "7.0", @@ -3576,7 +3576,7 @@ "fields": { "code": "isoc_ske_itrcrn2", "title": "Enterprises that recruited or tried to recruit ICT specialists by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "32.0", @@ -3669,7 +3669,7 @@ "fields": { "code": "isoc_ske_itrcrs", "title": "Enterprises that recruited or tried to recruit ICT specialists by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "7.0", @@ -3736,7 +3736,7 @@ "fields": { "code": "isoc_ske_fctn2", "title": "Enterprises - ICT functions performed by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "7.0", @@ -3829,7 +3829,7 @@ "fields": { "code": "isoc_ske_fct", "title": "Enterprises - ICT functions performed by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "32.0", @@ -3896,7 +3896,7 @@ "fields": { "code": "isoc_cisce_ran2", "title": "Security policy: measures, risks and staff awareness by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-08-29T09:00:00Z", "datastructure_last_version": "8.0", @@ -3994,7 +3994,7 @@ "fields": { "code": "isoc_cisce_ra", "title": "Security policy: measures, risks and staff awareness by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-05-25T09:00:00Z", "datastructure_last_version": "33.0", @@ -4066,7 +4066,7 @@ "fields": { "code": "isoc_ske_ittn2", "title": "Enterprises that provided training to develop/upgrade ICT skills of their personnel by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "32.0", @@ -4159,7 +4159,7 @@ "fields": { "code": "isoc_ske_itts", "title": "Enterprises that provided training to develop/upgrade ICT skills of their personnel by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "7.0", @@ -4226,7 +4226,7 @@ "fields": { "code": "isoc_cimobe_usen2", "title": "Use of mobile connections to the internet by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-08-29T09:00:00Z", "datastructure_last_version": "5.0", @@ -4324,7 +4324,7 @@ "fields": { "code": "isoc_cimobe_use", "title": "Use of mobile connections to the internet by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-05-25T09:00:00Z", "datastructure_last_version": "31.0", @@ -4396,7 +4396,7 @@ "fields": { "code": "isoc_bde15b_en2", "title": "Broadband and connectivity - enterprises by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-08-29T09:00:00Z", "datastructure_last_update": "2023-08-29T09:00:00Z", "datastructure_last_version": "5.0", @@ -4492,7 +4492,7 @@ "fields": { "code": "isoc_bde15b_e", "title": "Broadband and connectivity - enterprises by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-08-29T09:00:00Z", "datastructure_last_update": "2023-05-25T09:00:00Z", "datastructure_last_version": "31.0", @@ -4562,7 +4562,7 @@ "fields": { "code": "isoc_ec_evaln2", "title": "Value of e-commerce sales by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "32.0", @@ -4657,7 +4657,7 @@ "fields": { "code": "isoc_ec_evals", "title": "Value of e-commerce sales by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "6.0", @@ -4726,7 +4726,7 @@ "fields": { "code": "isoc_bde15decn2", "title": "E-commerce, customer relation management (CRM) and secure transactions by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-08-29T09:00:00Z", "datastructure_last_update": "2023-08-29T09:00:00Z", "datastructure_last_version": "5.0", @@ -4825,7 +4825,7 @@ "fields": { "code": "isoc_bde15dec", "title": "E-commerce, customer relation management (CRM) and secure transactions by size class of enterprise", - "remarks": "", + "additional_remarks": "", "data_last_update": "2023-08-29T09:00:00Z", "datastructure_last_update": "2023-05-25T09:00:00Z", "datastructure_last_version": "31.0", @@ -4898,7 +4898,7 @@ "fields": { "code": "isoc_bde15discn2", "title": "Integration with customers/suppliers and SCM by NACE", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-08-29T09:00:00Z", "datastructure_last_update": "2023-05-25T09:00:00Z", "datastructure_last_version": "5.0", @@ -4998,7 +4998,7 @@ "fields": { "code": "isoc_bde15disc", "title": "Integration with customers/suppliers and SCM by size class of enterprise", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-08-29T09:00:00Z", "datastructure_last_update": "2023-05-25T09:00:00Z", "datastructure_last_version": "30.0", @@ -5071,7 +5071,7 @@ "fields": { "code": "isoc_sks_itsps", "title": "Employed ICT specialists by sex", - "remarks": "", + "additional_remarks": "", "data_last_update": "2023-05-03T21:00:00Z", "datastructure_last_update": "2023-05-03T21:00:00Z", "datastructure_last_version": "25.0", @@ -5123,7 +5123,7 @@ "fields": { "code": "isoc_ec_esels", "title": "E_AWSWW (e_aws/rest_world)", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "7.0", @@ -5180,7 +5180,7 @@ "fields": { "code": "isoc_ec_esels", "title": "E_AWSHM (e_aws/own_country)", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "7.0", @@ -5237,7 +5237,7 @@ "fields": { "code": "isoc_ec_esels", "title": "E_AWSEU (e_aws/other_eu_countries)", - "remarks": null, + "additional_remarks": null, "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "7.0", @@ -5294,7 +5294,7 @@ "fields": { "code": "isoc_e_diin2", "title": "Digital Intensity Index by NACE (all)", - "remarks": "e_di3_vlo, e_di3_lo, e_di3_hi, e_di3_vhi, e_di4_vlo, e_di4_lo, e_di4_hi, e_di4_vhi", + "additional_remarks": "e_di3_vlo, e_di3_lo, e_di3_hi, e_di3_vhi, e_di4_vlo, e_di4_lo, e_di4_hi, e_di4_vhi", "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "7.0", @@ -5397,7 +5397,7 @@ "fields": { "code": "isoc_e_dii", "title": "Digital intensity indicators by size class of enterprise (all)", - "remarks": "e_di3_vlo, e_di3_lo, e_di3_hi, e_di3_vhi, e_di4_vlo, e_di4_lo, e_di4_hi, e_di4_vhi,", + "additional_remarks": "e_di3_vlo, e_di3_lo, e_di3_hi, e_di3_vhi, e_di4_vlo, e_di4_lo, e_di4_hi, e_di4_vhi,", "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "16.0", @@ -5471,7 +5471,7 @@ "fields": { "code": "isoc_e_dii", "title": "Digital Intensity Index v3 - SME 2021 (e_di3_sme_gelo)", - "remarks": "", + "additional_remarks": "", "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "16.0", @@ -5532,7 +5532,7 @@ "fields": { "code": "isoc_e_dii", "title": "Digital Intensity Index v4 - SME 2022 (e_di4_sme_gelo)", - "remarks": "", + "additional_remarks": "", "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "16.0", @@ -5593,7 +5593,7 @@ "fields": { "code": "isoc_ci_ifp_fu", "title": "Individuals - frequency of internet use", - "remarks": "i_iuse, i_iday", + "additional_remarks": "i_iuse, i_iday", "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "36.0", @@ -5682,7 +5682,7 @@ "fields": { "code": "isoc_ci_dev_i", "title": "Individuals - devices used to access the internet", - "remarks": "", + "additional_remarks": "", "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "33.0", @@ -5772,7 +5772,7 @@ "fields": { "code": "isoc_e_dii", "title": "Digital Intensity Index v3 - SME 2023 (e_di3_sme_gelo)", - "remarks": "", + "additional_remarks": "", "data_last_update": "2023-12-08T10:00:00Z", "datastructure_last_update": "2023-12-08T10:00:00Z", "datastructure_last_version": "16.0", @@ -5831,7 +5831,7 @@ "fields": { "code": "isoc_ci_it_h", "title": "Households - type of connection to the internet", - "remarks": "H_BBFIX, H_BROAD", + "additional_remarks": "H_BBFIX, H_BROAD", "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "33.0", @@ -5897,7 +5897,7 @@ "fields": { "code": "isoc_ci_in_h", "title": "Households - level of internet access (h_iacc)", - "remarks": "h_iacc", + "additional_remarks": "h_iacc", "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "29.0", @@ -5959,7 +5959,7 @@ "fields": { "code": "isoc_pibi_rni", "title": "Households - reasons for not having internet access at home (h_xcost)", - "remarks": "h_xcost", + "additional_remarks": "h_xcost", "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "32.0", @@ -6024,7 +6024,7 @@ "fields": { "code": "isoc_ciegi_ac", "title": "E-government activities of individuals via websites (i_iugov12, i_igov12rt)", - "remarks": "i_iugov12, i_igov12rt", + "additional_remarks": "i_iugov12, i_igov12rt", "data_last_update": "2023-12-15T10:00:00Z", "datastructure_last_update": "2023-12-15T10:00:00Z", "datastructure_last_version": "36.0", diff --git a/fixtures/test/facts.json b/fixtures/test/facts.json index c9157cc7..221b166d 100644 --- a/fixtures/test/facts.json +++ b/fixtures/test/facts.json @@ -334,7 +334,9 @@ "2022" ], "import_config": null, - "import_file": null + "import_file": null, + "reference_period": "2020", + "remarks": "This data point is for the EU" } }, { @@ -672,7 +674,9 @@ "2022" ], "import_config": null, - "import_file": null + "import_file": null, + "reference_period": "2020", + "remarks": "This data point is for Slovakia" } } ] diff --git a/frontend/cypress/e2e/admin/fileImport.cy.js b/frontend/cypress/e2e/admin/fileImport.cy.js index dc4662ad..64ea39e5 100644 --- a/frontend/cypress/e2e/admin/fileImport.cy.js +++ b/frontend/cypress/e2e/admin/fileImport.cy.js @@ -1,74 +1,61 @@ describe("Check data file import", () => { - it("Create and run data file import", () => { + function checkFileImport({ + fixture, + expectedStatus = "SUCCESS", + expectedFacts = "1", + }) { cy.login(); // Navigate to the data file import page cy.get("a").contains("Upload data from file").click(); cy.get("a").contains("Add Upload data from file").click(); // Add a new data file import - cy.get("input[type=file]").selectFile( - "cypress/fixtures/import_file_valid.xlsx", - ); + cy.get("input[type=file]").selectFile(`cypress/fixtures/${fixture}`); cy.get("textarea[name=description]").type("Test file import"); cy.get("input[type=submit][value=Save]").click(); // Wait for the task to finish cy.get("a").contains("Upload file results").click(); - cy.get("tbody tr:first-child td.field-status_display").contains("SUCCESS", { - timeout: 10000, - }); + cy.get("tbody tr:first-child td.field-status_display").contains( + expectedStatus, + { + timeout: 10000, + }, + ); // Navigate to the import config change form cy.get("tbody tr:first-child td.field-import_file_link a").click(); // Check that only one fact has been imported - cy.get(".field-num_facts a").contains("1"); + cy.get(".field-num_facts a").contains(expectedFacts); // Delete the import file cy.get("a").contains("Delete").click(); cy.get("input[type=submit]").click(); + } + + it("Create and run data file import XLSX", () => { + checkFileImport({ + fixture: "import_file_valid.xlsx", + }); + }); + it("Create and run data file import XLS", () => { + checkFileImport({ + fixture: "import_file_valid.xls", + }); + }); + it("Create and run data file import with no remarks", () => { + checkFileImport({ + fixture: "import_file_valid_no_remarks.xlsx", + }); }); it("Create and run invalid data file import", () => { - cy.login(); - // Navigate to the data file import page - cy.get("a").contains("Upload data from file").click(); - cy.get("a").contains("Add Upload data from file").click(); - // Add a new data file import - cy.get("input[type=file]").selectFile( - "cypress/fixtures/import_file_invalid.xlsx", - ); - cy.get("textarea[name=description]").type("Test file invalid import"); - cy.get("input[type=submit][value=Save]").click(); - // Wait for the task to finish - cy.get("a").contains("Upload file results").click(); - cy.get("tbody tr:first-child td.field-status_display").contains("FAILURE", { - timeout: 10000, + checkFileImport({ + fixture: "import_file_invalid.xlsx", + expectedStatus: "FAILURE", + expectedFacts: "0", }); - // Navigate to the import config change form - cy.get("tbody tr:first-child td.field-import_file_link a").click(); - // Check that no fact has been imported - cy.get(".field-num_facts a").contains("0"); - // Delete the import file - cy.get("a").contains("Delete").click(); - cy.get("input[type=submit]").click(); }); it("Create and run invalid data file import duplicate", () => { - cy.login(); - // Navigate to the data file import page - cy.get("a").contains("Upload data from file").click(); - cy.get("a").contains("Add Upload data from file").click(); - // Add a new data file import - cy.get("input[type=file]").selectFile( - "cypress/fixtures/import_file_invalid_duplicate.xlsx", - ); - cy.get("textarea[name=description]").type("Test file invalid import"); - cy.get("input[type=submit][value=Save]").click(); - // Wait for the task to finish - cy.get("a").contains("Upload file results").click(); - cy.get("tbody tr:first-child td.field-status_display").contains("FAILURE", { - timeout: 10000, + checkFileImport({ + fixture: "import_file_invalid_duplicate.xlsx", + expectedStatus: "FAILURE", + expectedFacts: "0", }); - // Navigate to the import config change form - cy.get("tbody tr:first-child td.field-import_file_link a").click(); - // Check that no fact has been imported - cy.get(".field-num_facts a").contains("0"); - // Delete the import file - cy.get("a").contains("Delete").click(); - cy.get("input[type=submit]").click(); }); }); diff --git a/frontend/cypress/e2e/admin/importConfig.cy.js b/frontend/cypress/e2e/admin/importConfig.cy.js index 90d94c72..27b60425 100644 --- a/frontend/cypress/e2e/admin/importConfig.cy.js +++ b/frontend/cypress/e2e/admin/importConfig.cy.js @@ -9,6 +9,9 @@ describe("Check import configuration", () => { cy.get("input[name=title]").type("Test import config"); cy.get("input[name=indicator]").type("indic_is"); cy.get("input[name=breakdown]").type("hhtyp"); + cy.get("input[name=reference_period]").type("time"); + cy.get("input[name=remarks]").type("test remarks"); + cy.get("input[name=remarks_is_surrogate]").click(); cy.get("input[name=period_start]").type("2019"); cy.get("input[name=period_end]").type("2019"); cy.get("#id_filters textarea.ace_text-input").clear({ force: true }); @@ -21,7 +24,7 @@ describe("Check import configuration", () => { ); cy.get("#id_mappings textarea.ace_text-input").clear({ force: true }); cy.get("#id_mappings textarea.ace_text-input").type( - '{"breakdown": {"total": "hh_total"}, "country": {"EU27_2020": "EU"}}', + '{"breakdown": {"total": "hh_total"}, "country": {"EU27_2020": "EU"}, "reference_period": {"2019": "2000"}}', { force: true, parseSpecialCharSequences: false, @@ -38,9 +41,21 @@ describe("Check import configuration", () => { }); // Navigate to the import config change form cy.get("tbody tr:first-child td.field-import_config_link a").click(); - // Check that only one fact has been imported - cy.get(".field-num_facts a").contains("1"); + // Check that only one fact has been imported, and open the fact details + cy.get(".field-num_facts a").contains("1").click(); + cy.get("tbody tr:first-child th.field-indicator a").click(); + + // Check imported values + cy.get(".field-indicator").contains("[h_broad]"); + cy.get(".field-breakdown").contains("[hh_total]"); + cy.get(".field-unit").contains("[pc_hh]"); + cy.get(".field-country").contains("[EU]"); + cy.get(".field-period").contains("[2019]"); + cy.get("[name=reference_period]").should("have.value", "2000"); + cy.get("[name=remarks]").should("have.value", "test remarks"); + // Delete the import config + cy.get(".field-import_config .view-related").click(); cy.get("a").contains("Delete").click(); cy.get("input[type=submit]").click(); }); diff --git a/frontend/cypress/e2e/charts/4.key-indicators/1.analyse-one-indicator-and-compare-countries.cy.js b/frontend/cypress/e2e/charts/4.key-indicators/1.analyse-one-indicator-and-compare-countries.cy.js index 0cc7d698..ca6e118c 100644 --- a/frontend/cypress/e2e/charts/4.key-indicators/1.analyse-one-indicator-and-compare-countries.cy.js +++ b/frontend/cypress/e2e/charts/4.key-indicators/1.analyse-one-indicator-and-compare-countries.cy.js @@ -8,19 +8,28 @@ describeResponsive("Check Chart", () => { ); cy.checkChart({ filters: { - indicator: "ICT graduates", - breakdown: "Females", - period: "2019", - unit: "% of graduates", + indicator: "Enterprises with a fixed broadband connection", + breakdown: "Total", + period: "2022", + unit: "% of enterprises", }, - title: ["ICT graduates, Females", "Year: 2019"], - point: "European Union, 0.8.", - tooltip: ["European Union", "Females", "0.80% of graduates"], + title: [ + "Enterprises having a fixed broadband connection, Total", + "Year: 2022", + ], + point: "European Union, 0.5.", + tooltip: [ + "European Union", + "Total", + "0.50% of enterprises", + "This data point is for the EU", + "Data from 2020", + ], definitions: [ - "Indicator: ICT graduates", - "Definition: Individuals with a degree in ICT", - "Breakdown: Females", - "Unit of measure: Percentage of graduates", + "Indicator: Enterprises having a fixed broadband connection", + "Definition: Fixed broadband connections include DSL, xDSL, cable leased lines, Frame Relay, Metro-Ethernet, PLC-Powerline communications, fixed wireless connections, etc.", + "Breakdown: Total", + "Unit of measure: Percentage of enterprises", ], }); }); diff --git a/frontend/cypress/fixtures/import_file_valid.xls b/frontend/cypress/fixtures/import_file_valid.xls new file mode 100644 index 00000000..2f527bcf Binary files /dev/null and b/frontend/cypress/fixtures/import_file_valid.xls differ diff --git a/frontend/cypress/fixtures/import_file_valid.xlsx b/frontend/cypress/fixtures/import_file_valid.xlsx index b792655f..687843c4 100644 Binary files a/frontend/cypress/fixtures/import_file_valid.xlsx and b/frontend/cypress/fixtures/import_file_valid.xlsx differ diff --git a/frontend/cypress/fixtures/import_file_valid_no_remarks.xlsx b/frontend/cypress/fixtures/import_file_valid_no_remarks.xlsx new file mode 100644 index 00000000..65e30cba Binary files /dev/null and b/frontend/cypress/fixtures/import_file_valid_no_remarks.xlsx differ diff --git a/frontend/src/components/charts/base/BaseChart.vue b/frontend/src/components/charts/base/BaseChart.vue index 0aceae91..38d951b2 100644 --- a/frontend/src/components/charts/base/BaseChart.vue +++ b/frontend/src/components/charts/base/BaseChart.vue @@ -385,6 +385,16 @@ export default { ); } + if (fact.reference_period) { + result.push( + `Reference period: Data from ${fact.reference_period}`, + ); + } + + if (fact.remarks) { + result.push(`Remarks: ${fact.remarks}`); + } + return result.join("
"); }, }; diff --git a/frontend/src/views/chart-group/MetadataView.vue b/frontend/src/views/chart-group/MetadataView.vue index c574616a..d7103b94 100644 --- a/frontend/src/views/chart-group/MetadataView.vue +++ b/frontend/src/views/chart-group/MetadataView.vue @@ -77,6 +77,26 @@ /> + + + reference_period + + + Reference Period + + + Free text reference period + + + + + remarks + + Remarks + + Free text notes + +