Skip to content

Commit

Permalink
Don't throw if Banner services are temporarily down
Browse files Browse the repository at this point in the history
  • Loading branch information
codetheweb committed Oct 30, 2023
1 parent 77d7d35 commit b567e20
Show file tree
Hide file tree
Showing 4 changed files with 36 additions and 4 deletions.
11 changes: 10 additions & 1 deletion src/tasks/scrape-instructors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import {Injectable, Logger} from '@nestjs/common';
import {Task, TaskHandler} from 'nestjs-graphile-worker';
import * as db from 'zapatos/db';
import {FetcherService} from 'src/fetcher/fetcher.service';
import type {IFaculty} from '@mtucourses/scraper';
import {PoolService} from '~/pool/pool.service';
import {updateDeletedAtUpdatedAtForUpsert} from '~/lib/db-utils';

Expand All @@ -14,7 +15,15 @@ export class ScrapeInstructorsTask {

@TaskHandler()
async handler() {
const faculty = await this.fetcher.getAllFaculty();
let faculty: IFaculty[] = [];
try {
faculty = await this.fetcher.getAllFaculty();
} catch (error: unknown) {
if (error instanceof Error && error.message === 'Banner services are currently down.') {
this.logger.warn('Banner services are currently down. Skipping scraping instructors.');
return;
}
}

this.logger.log('Finished scraping website');

Expand Down
5 changes: 5 additions & 0 deletions src/tasks/scrape-section-details.ts
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,11 @@ export class ScrapeSectionDetailsTask {
section
};
} catch (error: unknown) {
if (error instanceof Error && error.message === 'Banner services are currently down.') {
this.logger.warn('Banner services are currently down. Skipping scraping instructors.');
return;
}

if ((error as Error).message === 'Course not found') {
this.logger.log(`Did not find ${section.course.id}: ${JSON.stringify(section.course)}`);
this.logger.log('It should be cleaned up automatically on the next course scrape.');
Expand Down
14 changes: 12 additions & 2 deletions src/tasks/scrape-sections.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import {Injectable} from '@nestjs/common';
import {Injectable, Logger} from '@nestjs/common';
import pThrottle from 'p-throttle';
import type {ICourseOverview, ISection} from '@mtucourses/scraper';
import type {IRuleOptions} from 'src/lib/rschedule';
Expand All @@ -15,6 +15,8 @@ import {updateDeletedAtUpdatedAtForUpsert} from '~/lib/db-utils';
@Injectable()
@Task('scrape-sections')
export class ScrapeSectionsTask {
private readonly logger = new Logger(ScrapeSectionsTask.name);

constructor(private readonly pool: PoolService, private readonly fetcher: FetcherService) {}

@TaskHandler()
Expand All @@ -28,7 +30,15 @@ export class ScrapeSectionsTask {

private async processTerm(term: Date) {
const {semester, year} = dateToTerm(term);
const extCourses = await this.fetcher.getAllSections(term);
let extCourses: ICourseOverview[] = [];
try {
extCourses = await this.fetcher.getAllSections(term);
} catch (error: unknown) {
if (error instanceof Error && error.message === 'Banner services are currently down.') {
this.logger.warn('Banner services are currently down. Skipping scraping instructors.');
return;
}
}

await db.serializable(this.pool, async trx => {
// Mark courses and sections in term as deleted by default
Expand Down
10 changes: 9 additions & 1 deletion src/tasks/scrape-transfer-courses.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,15 @@ export class ScrapeTransferCoursesTask {

@TaskHandler()
async handler() {
const extTransferCourses = await this.fetcher.getAllTransferCourses();
let extTransferCourses: ITransferCourse[] = [];
try {
extTransferCourses = await this.fetcher.getAllTransferCourses();
} catch (error: unknown) {
if (error instanceof Error && error.message === 'Banner services are currently down.') {
this.logger.warn('Banner services are currently down. Skipping scraping instructors.');
return;
}
}

// Filter out duplicates
// todo: this should be handled by the scraper
Expand Down

0 comments on commit b567e20

Please sign in to comment.