Merge branch 'release/5.2.0'

CITATION.cff

@@ -35,5 +35,5 @@ keywords:
   - elasticsearch
   - natural language processing
 license: MIT
-version: 4.2.0
-date-released: '2023-09-28'
+version: 5.2.0
+date-released: '2023-10-31'

backend/users/migrations/0005_saml_user_group.py

@@ -9,7 +9,7 @@ def add_saml_users_to_group(apps, schema_editor):
 
     if saml_group:
         for user in saml_users:
-            user.groups.add(saml_group)
+            user.groups.add(saml_group.id)
             user.save()
 
 class Migration(migrations.Migration):

backend/users/saml.py

@@ -16,4 +16,5 @@ def get_or_create_user(self, *args, **kwargs):
 def saml_user_group():
     group_name = getattr(settings, 'SAML_GROUP_NAME', None)
     if group_name:
-        return Group.objects.get_or_create(name=group_name)
+        group, _ = Group.objects.get_or_create(name=group_name)
+        return group

backend/visualization/query.py

@@ -131,7 +131,11 @@ def get_filters(query):
 
 def is_date_filter(filter):
     """Checks if a filter object is a date filter"""
-    return has_path(filter, 'range', 'date')
+    range_filters = filter.get('range', dict()).keys()
+    return any(
+        has_path(filter, 'range', filter_name, 'format')
+        for filter_name in range_filters
+    )
 
 
 def parse_date(datestring):
@@ -145,7 +149,11 @@ def get_date_range(query: Dict):
         datefilters = list(filter(is_date_filter, filters))
 
         if len(datefilters):
-            parameters = [f['range']['date'] for f in datefilters]
+            parameters = [
+                data
+                for f in datefilters
+                for data in f['range'].values()
+            ]
             min_dates = [parse_date(p['gte'])
                          for p in parameters if 'gte' in p]
             max_dates = [parse_date(p['lte'])

backend/visualization/tests/test_query.py

@@ -79,3 +79,40 @@ def test_manipulation_is_pure(basic_query):
     assert query.get_date_range(basic_query) == (None, None)
     assert query.get_date_range(narrow_query) == narrow_timeframe
     assert query.get_date_range(wide_query) == wide_timeframe
+
+def test_is_date_filter():
+    assert query.is_date_filter({
+        'range': {
+            'date': {
+                'gte': '1950-01-01',
+                'lte': '1959-12-31',
+                'format': 'yyyy-MM-dd'
+            }
+        }
+    })
+
+    assert query.is_date_filter({
+        'range': {
+            'publication_date': {
+                'gte': '1950-01-01',
+                'lte': '1959-12-31',
+                'format': 'yyyy-MM-dd'
+            }
+        }
+    })
+
+    assert not query.is_date_filter({
+        'range': {
+            'year': {
+                'gte': 1950,
+                'lte': 1959,
+            }
+        }
+    })
+
+def test_get_date_range():
+    min_date = datetime(1800, 1, 1)
+    max_date = datetime(1900, 1, 1)
+    date_filter = query.make_date_filter(min_date, max_date, 'publication_date')
+    q = query.add_filter(query.MATCH_ALL, date_filter)
+    assert query.get_date_range(q) == (min_date, max_date)

frontend/package.json

@@ -37,12 +37,11 @@
     "balloon-css": "^0.5.2",
     "bulma": "^0.5.1",
     "bulma-switch": "^2.0.0",
-    "chart.js": "^3.7.1",
-    "chartjs-adapter-moment": "^1.0.0",
-    "chartjs-plugin-zoom": "^1.2.0",
+    "chart.js": "^4.4.0",
+    "chartjs-adapter-moment": "^1.0.1",
+    "chartjs-chart-wordcloud": "^4.3.2",
+    "chartjs-plugin-zoom": "^2.0.1",
     "core-js": "^2.6.11",
-    "d3": "^7.8.5",
-    "d3-cloud": "^1.2.7",
     "file-saver": "^2.0.5",
     "font-awesome": "^4.7.0",
     "html-to-image": "^1.9.0",
@@ -72,8 +71,7 @@
     "@angular-eslint/template-parser": "13.5.0",
     "@angular/compiler-cli": "^13.2.2",
     "@angular/language-service": "^13.2.2",
-    "@types/chart.js": "^2.9.35",
-    "@types/d3": "^7.4.0",
+    "@types/chart.js": "^2.9.37",
     "@types/jasmine": "~3.6.0",
     "@types/jasminewd2": "^2.0.8",
     "@types/node": "^6.14.10",
@@ -104,7 +102,6 @@
   "resolutions": {
     "loader-utils": "^2.0.4",
     "terser": "^5.14.2",
-    "d3-color": "^3.1.0",
     "semver": "^7.5.2"
   }
 }

frontend/src/app/visualization/barchart/barchart.directive.ts

@@ -84,15 +84,17 @@ export abstract class BarchartDirective
 
     basicChartOptions: ChartOptions = { // chart options not suitable for Chart.defaults.global
         scales: {
-            xAxis: {
+            x: {
                 title: { display: true },
-                grid: { drawBorder: true, drawOnChartArea: false }
+                border: { display: true },
+                grid: { drawOnChartArea: false }
             },
-            yAxis: {
+            y: {
                 type: 'linear',
                 beginAtZero: true,
                 title: { display: true, text: 'Frequency' },
-                grid: { drawBorder: true, drawOnChartArea: false, },
+                border: { display: true },
+                grid: { drawOnChartArea: false, },
                 ticks: {
                     callback: (value, index, values) => this.formatValue(this.normalizer)(value as number),
                 },
@@ -481,7 +483,7 @@ export abstract class BarchartDirective
 
         this.chart = new Chart(barchartID,
             {
-                type: this.chartType,
+                type: 'bar',
                 data: {
                     labels,
                     datasets
@@ -491,7 +493,6 @@ export abstract class BarchartDirective
             }
         );
 
-
         this.chart.canvas.ondblclick = (event) => this.zoomOut();
     }
 
@@ -504,7 +505,6 @@ export abstract class BarchartDirective
     updateChartData() {
         const labels = this.getLabels();
         const datasets = this.getDatasets();
-        this.chart.config.type = this.chartType;
         this.chart.options = this.chartOptions(datasets);
         this.chart.data.labels = labels;
         this.chart.data.datasets = datasets;

frontend/src/app/visualization/barchart/histogram.component.ts

@@ -119,6 +119,7 @@ export class HistogramComponent extends BarchartDirective<HistogramDataPoint> im
 
         return this.rawData.map((series, seriesIndex) => (
             {
+                type: this.chartType,
                 label: series.queryText ? series.queryText : '(no query)',
                 data: labels.map(key => {
                   const item = series.data.find(i => i.key === key);
@@ -133,12 +134,12 @@ export class HistogramComponent extends BarchartDirective<HistogramDataPoint> im
     }
 
     chartOptions(datasets: any[]) {
-        const xAxisLabel = this.visualizedField.displayName ? this.visualizedField.displayName : this.visualizedField.name;
+        const xLabel = this.visualizedField.displayName ? this.visualizedField.displayName : this.visualizedField.name;
         const options = this.basicChartOptions;
         options.plugins.title.text = this.chartTitle();
-        options.scales.xAxis.type = 'category';
-        (options.scales.xAxis as any).title.text = xAxisLabel;
-        options.scales.xAxis.ticks = { callback: formatXAxisLabel };
+        options.scales.x.type = 'category';
+        (options.scales.x as any).title.text = xLabel;
+        options.scales.x.ticks = { callback: formatXAxisLabel };
         options.plugins.tooltip = {
             callbacks: {
                 label: (tooltipItem) => {

frontend/src/app/visualization/barchart/timeline.component.ts

@@ -1,6 +1,5 @@
 import { Component, OnChanges, OnInit } from '@angular/core';
 
-import * as d3TimeFormat from 'd3-time-format';
 import * as _ from 'lodash';
 
 import { QueryModel, AggregateResult, TimelineSeries, TimelineDataPoint, TermFrequencyResult,
@@ -23,8 +22,6 @@ export class TimelineComponent extends BarchartDirective<TimelineDataPoint> impl
     private currentTimeCategory: TimeCategory;
     /** threshold for scaling down a unit on the time scale */
     private scaleDownThreshold = 10;
-    /** formatting function for time in ES queries */
-    private timeFormat: any = d3TimeFormat.timeFormat('%Y-%m-%d'); // Todo: use moment instead of d3
     /** domain on the axis */
     public xDomain: [Date, Date];
 
@@ -120,7 +117,7 @@ export class TimelineComponent extends BarchartDirective<TimelineDataPoint> impl
     setChart() {
         if (this.chart) {
             // reset time unit to the one set in the chart
-            const unit = (this.chart.options.scales.xAxis as any).time.unit as TimeCategory;
+            const unit = (this.chart.options.scales.x as any).time.unit as TimeCategory;
             if (unit) {
                 this.currentTimeCategory = unit;
             }
@@ -134,8 +131,7 @@ export class TimelineComponent extends BarchartDirective<TimelineDataPoint> impl
         return this.rawData.map((series, seriesIndex) => {
             const data = this.chartDataFromSeries(series);
             return {
-                xAxisID: 'xAxis',
-                yAxisID: 'yAxis',
+                type: this.chartType,
                 label: series.queryText ? series.queryText : '(no query)',
                 data,
                 backgroundColor: selectColor(this.palette, seriesIndex),
@@ -158,22 +154,22 @@ export class TimelineComponent extends BarchartDirective<TimelineDataPoint> impl
     }
 
     chartOptions(datasets) {
-        const xAxisLabel = this.visualizedField.displayName ? this.visualizedField.displayName : this.visualizedField.name;
+        const xLabel = this.visualizedField.displayName ? this.visualizedField.displayName : this.visualizedField.name;
         const margin = moment.duration(1, this.currentTimeCategory);
         const xMin = moment(this.xDomain[0]).subtract(margin).toDate();
         const xMax = moment(this.xDomain[1]).add(margin).toDate();
 
         const options = this.basicChartOptions;
         options.plugins.title.text = this.chartTitle();
-        const xAxis = options.scales.xAxis;
-        (xAxis as any).title.text = xAxisLabel;
-        xAxis.type = 'time';
-        (xAxis as any).time = {
+        const x = options.scales.x;
+        (x as any).title.text = xLabel;
+        x.type = 'time';
+        (x as any).time = {
             minUnit: 'day',
             unit: this.currentTimeCategory,
         };
-        xAxis.min = xMin.toISOString();
-        xAxis.max = xMax.toISOString();
+        x.min = xMin.toISOString();
+        x.max = xMax.toISOString();
         options.plugins.tooltip = {
             callbacks: {
                 title: ([tooltipItem]) => this.formatDate(Date.parse(tooltipItem.label as string)),
@@ -186,15 +182,15 @@ export class TimelineComponent extends BarchartDirective<TimelineDataPoint> impl
 
         // zoom limits
         options.plugins.zoom.limits = {
-            xAxis: {
+            x: {
                 // convert dates to numeric rather than string here,
                 // as zoom plugin does not accept strings
                 min: xMin.valueOf(),
                 max: xMax.valueOf(),
             }
         };
 
-        options.scales.xAxis.type = 'time';
+        options.scales.x.type = 'time';
         options.plugins.legend = {display: datasets.length > 1};
         return options;
     }
@@ -211,8 +207,8 @@ export class TimelineComponent extends BarchartDirective<TimelineDataPoint> impl
     onZoomIn(chart, triggeredByDataUpdate = false) {
         const initialTimeCategory = this.calculateTimeCategory(...this.xDomain);
         const previousTimeCategory = this.currentTimeCategory;
-        const min = new Date(chart.scales.xAxis.min);
-        const max = new Date(chart.scales.xAxis.max);
+        const min = new Date(chart.scales.x.min);
+        const max = new Date(chart.scales.x.max);
         this.currentTimeCategory = this.calculateTimeCategory(min, max);
 
         if ((this.currentTimeCategory !== previousTimeCategory) ||
@@ -260,7 +256,7 @@ export class TimelineComponent extends BarchartDirective<TimelineDataPoint> impl
             chart.data.datasets[seriesIndex].data = this.chartDataFromSeries(data);
         });
 
-        chart.options.scales.xAxis.time.unit = this.currentTimeCategory;
+        chart.options.scales.x.time.unit = this.currentTimeCategory;
         chart.update('show'); // fade into view
 
     }
@@ -281,7 +277,7 @@ export class TimelineComponent extends BarchartDirective<TimelineDataPoint> impl
     zoomOut(): void {
         this.chart.resetZoom();
         this.currentTimeCategory = this.calculateTimeCategory(...this.xDomain);
-        (this.chart.options.scales.xAxis as any).time.unit = this.currentTimeCategory;
+        (this.chart.options.scales.x as any).time.unit = this.currentTimeCategory;
         this.chart.update();
 
         this.setChart();
@@ -365,6 +361,7 @@ export class TimelineComponent extends BarchartDirective<TimelineDataPoint> impl
      * Format for dates based on the time category.
      * Returns a formatting function.
      */
+    // eslint-disable-next-line @typescript-eslint/member-ordering
     get formatDate(): (date) => string {
         let dateFormat: string;
         switch (this.currentTimeCategory) {
@@ -382,12 +379,13 @@ export class TimelineComponent extends BarchartDirective<TimelineDataPoint> impl
         return (date: Date) => moment(date).format(dateFormat);
     }
 
+    // eslint-disable-next-line @typescript-eslint/member-ordering
     get isZoomedIn(): boolean {
-        // check whether this.chart is zoomed on xAxis
+        // check whether this.chart is zoomed on x axis
 
         if (this.chart) {
-            const initialBounds = this.chart.getInitialScaleBounds().xAxis;
-            const currentBounds = { min : this.chart.scales.xAxis.min, max: this.chart.scales.xAxis.max };
+            const initialBounds = this.chart.getInitialScaleBounds().x;
+            const currentBounds = { min: this.chart.scales.x.min, max: this.chart.scales.x.max };
 
             return (initialBounds.min && initialBounds.min < currentBounds.min) ||
                 (initialBounds.max && initialBounds.max > currentBounds.max);

frontend/src/app/visualization/wordcloud/wordcloud.component.html

@@ -1,5 +1,5 @@
 <div id="wordcloud_div" class="block" [ngClass]="{ 'is-hidden': asTable }">
-    <div class="wordcloud" #wordcloud></div>
+    <canvas id="wordcloud" width="100%" height="500px"></canvas>
 </div>
 
 <div class="block" *ngIf="asTable">

frontend/src/app/visualization/wordcloud/wordcloud.component.scss

@@ -1,4 +0,0 @@
-.wordcloud {
-    width: 90%;
-    height: 400px;
-}

frontend/src/app/visualization/wordcloud/wordcloud.component.spec.ts

@@ -1,5 +1,5 @@
 import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing';
-import { WordcloudComponent } from './wordcloud.component';
+import { WordcloudComponent, sizeScale } from './wordcloud.component';
 import { commonTestBed } from '../../common-test-bed';
 
 describe('WordcloudComponent', () => {
@@ -20,3 +20,12 @@ describe('WordcloudComponent', () => {
     expect(component).toBeTruthy();
   });
 });
+
+describe('sizeScale', () => {
+    it('should scale sizes', () => {
+        const scale = sizeScale(100, 1000);
+        expect(scale(100)).toBeCloseTo(10);
+        expect(scale(1000)).toBeCloseTo(48);
+    }
+    );
+});