Feature/plenary minute period 20

bundestag.io/api/src/graphql/resolvers/Procedure.ts

@@ -328,6 +328,7 @@ const ProcedureResolvers: Resolvers = {
         }
         const plenaryMinute = await PlenaryMinuteModel.findOne({
           meeting: parseInt(session.session),
+          period: procedure.period,
         });
 
         console.log('plenaryMinute', plenaryMinute);

common/bundestagio/src/models/PlenaryMinute/schema.ts

@@ -1,22 +1,29 @@
-import { Schema, Document, SchemaTimestampsConfig } from "mongoose";
+import { Schema, Document, SchemaTimestampsConfig } from 'mongoose';
 
 export interface IPlenaryMinute {
   date: Date;
   meeting: number;
+  period: number;
   xml: string;
 }
 
-export type PlenaryMinuteDoc = IPlenaryMinute &
-  Document &
-  SchemaTimestampsConfig;
+export type PlenaryMinuteDoc = IPlenaryMinute & Document & SchemaTimestampsConfig;
 
 const PlenaryMinuteSchema = new Schema<PlenaryMinuteDoc>(
   {
-    date: { type: Date, required: true, unique: true },
-    meeting: { type: Number, required: true, unique: true },
+    date: { type: Date, required: true, unique: false },
+    meeting: { type: Number, required: true, unique: false },
+    period: { type: Number, required: true, unique: false },
     xml: { type: String, required: true, unique: true },
   },
-  { timestamps: true }
+  { timestamps: true },
+);
+PlenaryMinuteSchema.index(
+  {
+    meeting: 1,
+    period: 1,
+  },
+  { unique: true },
 );
 
 export default PlenaryMinuteSchema;

services/cron-jobs/import-plenary-minutes/src/index.ts

@@ -29,7 +29,7 @@ const getMeta = (meta: cheerio.Cheerio): MetaData => {
   };
 };
 
-const getPlenaryMinutes = (plenaryMinutes: cheerio.Cheerio): PlenaryMinutesItem[] => {
+const getPlenaryMinutes = (plenaryMinutes: cheerio.Cheerio, period: number): PlenaryMinutesItem[] => {
   const plenaryMinutesItems: PlenaryMinutesItem[] = [];
   plenaryMinutes.each((i, elem) => {
     // Parse Title
@@ -46,6 +46,7 @@ const getPlenaryMinutes = (plenaryMinutes: cheerio.Cheerio): PlenaryMinutesItem[
 
     const plenaryMinutesItem: PlenaryMinutesItem = {
       date: m.toDate(),
+      period,
       meeting: parseInt(match.meeting),
       xml: `https://www.bundestag.de${xmlLink}`,
     };
@@ -55,48 +56,55 @@ const getPlenaryMinutes = (plenaryMinutes: cheerio.Cheerio): PlenaryMinutesItem[
   return plenaryMinutesItems;
 };
 
-const parsePage = async (url: string) => {
+const parsePage = async (url: string, period: number) => {
   return await AxiosInstance.get(url).then((response) => {
     const html = response.data;
     const $ = cheerio.load(html);
     const meta: cheerio.Cheerio = $('.meta-slider');
     const plenaryMinutesTable: cheerio.Cheerio = $('.bt-table-data > tbody > tr');
     const metaData = getMeta(meta);
-    const plenaryMinutes = getPlenaryMinutes(plenaryMinutesTable);
+    const plenaryMinutes = getPlenaryMinutes(plenaryMinutesTable, period);
     return {
       meta: metaData,
       plenaryMinutes,
     };
   });
 };
 
-const getUrl = (offset: number) =>
-  `https://www.bundestag.de/ajax/filterlist/de/services/opendata/543410-543410/h_49f0d94cb26682ff1e9428b6de471a5b?offset=${offset}`;
+const getUrl = ({ offset, id }: { offset: number; id: string }) =>
+  `https://www.bundestag.de/ajax/filterlist/de/services/opendata/${id}?offset=${offset}`;
 
-const start = async () => {
-  let url: string | false = getUrl(0);
+const periods = [
+  { period: 19, id: '543410-543410' },
+  { period: 20, id: '866354-866354' },
+];
+
+const start = async (period: number) => {
+  const periodId = periods.find((p) => p.period === period)!.id;
+
+  let url: string | false = getUrl({ offset: 0, id: periodId });
   const data: PlenaryMinutesItem[] = [];
   do {
-    const { meta, plenaryMinutes } = await parsePage(url);
+    const { meta, plenaryMinutes } = await parsePage(url, period);
     data.push(...plenaryMinutes);
     if (meta.nextOffset < meta.hits) {
-      url = getUrl(meta.nextOffset);
+      url = getUrl({ offset: meta.nextOffset, id: periodId });
     } else {
       url = false;
     }
   } while (url);
   await PlenaryMinuteModel.collection.bulkWrite(
     data.map((item) => ({
       updateOne: {
-        filter: { meeting: item.meeting },
+        filter: { meeting: item.meeting, period: item.period },
         update: {
           $set: item,
         },
         upsert: true,
       },
     })),
   );
-  console.log('found: ', data.length);
+  console.log(`found for period ${period}: `, data.length);
 };
 
 (async () => {
@@ -107,6 +115,7 @@ const start = async () => {
   }
   await mongoConnect();
   console.log('PlenaryMinutes', await PlenaryMinuteModel.countDocuments({}));
-  await start();
+  await start(19);
+  await start(20);
   process.exit(0);
 })();

services/cron-jobs/import-plenary-minutes/src/types.ts

@@ -6,6 +6,7 @@ export interface MetaData {
 
 export interface PlenaryMinutesItem {
   meeting: number;
+  period: number;
   date: Date;
   xml: string;
 }