Merge pull request #338 from digital-sustainability/saegi

Saegi

.gitignore

@@ -17,4 +17,6 @@ data-gathering/github-data.pickle
 
 client
 
-.DS_Store
+.DS_Store
+
+.env

Dockerfile

@@ -6,7 +6,7 @@ COPY oss-api/ ./
 
 WORKDIR /oss-api
 
-RUN npm install 
+RUN npm install
 
 RUN npm run build
 
@@ -20,7 +20,7 @@ RUN cd /frontend/
 
 WORKDIR /frontend/
 
-RUN npm install 
+RUN npm install
 
 RUN npm run build:prod
 

compose.yml

@@ -0,0 +1,19 @@
+services:
+  oss-api:
+    networks:
+      - default
+    build: .
+    env_file:
+      - ./oss-api/.env
+    environment:
+      - NODE_ENV=DEV
+      - MONGO_DATABASE=testingNew
+      - LOG_PATH=/logs
+      - DATA_PATH=/data
+      - NODE_OPTIONS="--max-old-space-size=4096"
+    ports:
+      - 3000:3000
+    deploy:
+      resources:
+        limits:
+          memory: 2000M

oss-api/.gitignore

@@ -2,6 +2,8 @@
 /client
 /dist
 /node_modules
+/data
+/logs
 
 # Logs
 logs

oss-api/src/data/data.service.ts

@@ -1,8 +1,6 @@
 import { Injectable, Logger } from '@nestjs/common';
 import * as fs from 'fs';
 import {
-  GitHubIssue,
-  GitHubPull,
   GithubCommit,
   GithubOrganisation,
   InstitutionRevised,
@@ -19,7 +17,6 @@ import { MongoDbService } from '../mongo-db/mongo-db.service';
 import { v4 as uuidv4 } from 'uuid';
 import { ObjectId } from 'mongodb';
 import { Cron, CronExpression } from '@nestjs/schedule';
-import { log } from 'console';
 import { RepoData } from '../interfaces';
 
 @Injectable()
@@ -31,18 +28,30 @@ export class DataService {
   private readonly logger = new Logger(DataService.name);
   private dataPath: string;
 
+  @Cron(CronExpression.EVERY_MINUTE)
+  cronTest() {
+    this.logger.log('Another minute... still running');
+  }
+
   @Cron(CronExpression.EVERY_HOUR)
   async handler(): Promise<void> {
-    log('Handling all the new data');
+    this.logger.log('Handling all the new data');
     const currentTime = new Date();
     if (!this.dataPath) return;
     const fileNames: string[] = fs.readdirSync(this.dataPath);
-    const filteredFileNames = fileNames.filter((fileName) => {
+    const filteredFileNames = [];
+    for (const fileName of fileNames) {
       const timestamp = fileName
         .split('_')[1]
         .replace('.json', '') as unknown as number;
-      return timestamp < currentTime.getTime();
-    });
+      if (timestamp < currentTime.getTime()) {
+        filteredFileNames.push(fileName);
+      } else {
+        // delete older files
+        this.deleteFile(this.dataPath.concat('/', fileName));
+      }
+    }
+
     const contributorFileNames: string[] = filteredFileNames.filter(
       (fileName) => fileName.includes('user'),
     );
@@ -59,9 +68,18 @@ export class DataService {
     await this.handleOrganisations(organisationFileNames);
     await this.handleInstitutions();
     for (const fileName of filteredFileNames) {
-      fs.unlinkSync(this.dataPath.concat('/', fileName));
+      this.deleteFile(this.dataPath.concat('/', fileName));
+    }
+    this.logger.log('Data service is finished');
+  }
+
+  private deleteFile(filePath: string) {
+    try {
+      fs.unlinkSync(filePath);
+      console.log(`File ${filePath} has been deleted.`);
+    } catch (err) {
+      console.error(err);
     }
-    log('Data service is finished');
   }
 
   private async handleInstitutions() {
@@ -75,7 +93,7 @@ export class DataService {
   private async handleOrganisations(
     organisationFileNames: string[],
   ): Promise<void> {
-    log('Handling all organisations');
+    // this.logger.log('Handling all organisations');
     for (const organisationFileName of organisationFileNames) {
       const orgData: string = this.readFile(
         this.dataPath.concat('/', organisationFileName),
@@ -94,10 +112,10 @@ export class DataService {
   }
 
   private async handleRepositories(repositoryFileNames: string[]) {
-    log('Handling all repositories');
+    // this.logger.log('Handling all repositories');
     let repositories: RepoData[] = [];
     for (const repofileName of repositoryFileNames) {
-      log(repofileName);
+      this.logger.log(repofileName);
       const repoData: string = this.readFile(
         this.dataPath.concat('/', repofileName),
       );
@@ -161,7 +179,7 @@ export class DataService {
     }
     const test = Object.values(repositories);
     for (const repository of test) {
-      log(`Handling repository ${repository.repository.name}`);
+      // this.logger.log(`Handling repository ${repository.repository.name}`);
       const res = await this.mongo.findRepositoryRevised(
         repository.repository.name,
         repository.institution,
@@ -176,7 +194,7 @@ export class DataService {
   }
 
   private async handleContributor(fileName: string) {
-    log(`Handling file ${fileName}`);
+    //this.logger.log(`Handling file ${fileName}`);
     const userData: string = this.readFile(this.dataPath.concat('/', fileName));
     if (!userData) return;
     const parsedFile: RawResponse = JSON.parse(userData);
@@ -188,19 +206,19 @@ export class DataService {
   /******************************************Helper Functions*******************************************************/
 
   private readFile(path: string): string {
-    //log(`Reading file at path ${path}`);
+    //this.logger.log(`Reading file at path ${path}`);
     try {
       return fs.readFileSync(path, 'utf8');
     } catch (err) {
-      log(err);
+      this.logger.log(err);
       return null;
     }
   }
 
   private async createInsitution(
     todoInstitution: TodoInstitution,
   ): Promise<InstitutionRevised> {
-    log(`Creating institution ${todoInstitution.shortname}`);
+    // this.logger.log(`Creating institution ${todoInstitution.shortname}`);
     const organisations = await this.mongo.findOrganisationsWithNames(
       todoInstitution.orgs.map((organisation) => organisation.name),
     );
@@ -217,7 +235,7 @@ export class DataService {
   }
 
   private createContributor(contributorData: GithubUser): Contributor {
-    log(`Creating contributor ${contributorData.login}`);
+    // this.logger.log(`Creating contributor ${contributorData.login}`);
     const contributor: Contributor = {
       login: contributorData.login,
       name: contributorData.name,
@@ -243,7 +261,9 @@ export class DataService {
     currentRepositoryStats: RepositoryStats[],
     aheadByCommits: number,
   ): Promise<RepositoryRevised> {
-    log(`Creating repo info for repo ${repositoryData.repository.name}`);
+    // this.logger.log(
+    //   `Creating repo info for repo ${repositoryData.repository.name}`,
+    // );
     repositoryData.allIssues;
     const repositoryStats: RepositoryStats = {
       num_forks: repositoryData.repository.forks_count,