Merge pull request #83 from chaotic-cx/feature/repo-manager

feat(backend): add initial repo-manager service

.gitignore

@@ -47,3 +47,4 @@ core
 
 # Local env files
 .env
+repos/

backend/src/app.module.ts

@@ -11,9 +11,10 @@ import { dataSourceOptions } from "./data.source";
 import { MetricsModule } from "./metrics/metrics.module";
 import { MiscModule } from "./misc/misc.module";
 import { RouterModule } from "./router/router.module";
-import { TelegramModule } from "./telegram/telegram.module";
 import { UsersModule } from "./users/users.module";
 import { ThrottlerModule } from "@nestjs/throttler";
+import { RepoManagerModule } from "./repo-manager/repo-manager.module";
+import { TelegramModule } from "./telegram/telegram.module";
 
 @Module({
     imports: [
@@ -27,6 +28,7 @@ import { ThrottlerModule } from "@nestjs/throttler";
         }),
         MetricsModule,
         MiscModule,
+        RepoManagerModule,
         RouterModule,
         TelegramModule,
         TerminusModule,

backend/src/builder/builder.entity.ts

@@ -12,6 +12,7 @@ import {
     type Repository,
 } from "typeorm";
 import { BuildStatus } from "../types";
+import { RepoStatus } from "../interfaces/repo-manager";
 
 @Entity()
 export class Builder {
@@ -47,6 +48,18 @@ export class Package {
 
     @Column({ type: "boolean", nullable: false, default: true })
     isActive: boolean;
+
+    @Column({ type: "varchar", nullable: true })
+    version: string;
+
+    @Column({ type: "int", nullable: true })
+    bumpCount: number;
+
+    @Column({ type: "jsonb", nullable: true })
+    bumpTriggers: { pkgname: string; archVersion: string }[];
+
+    @Column({ type: "jsonb", nullable: true })
+    metadata: string;
 }
 
 @Entity()
@@ -59,6 +72,15 @@ export class Repo {
 
     @Column({ type: "varchar", nullable: true })
     repoUrl: string;
+
+    @Column({ type: "boolean", default: true })
+    isActive: boolean;
+
+    @Column({ type: "int", nullable: true })
+    status: RepoStatus;
+
+    @Column({ type: "varchar", default: "main" })
+    gitRef: string;
 }
 
 @Entity()
@@ -133,8 +155,6 @@ export async function pkgnameExists(pkgname: string, connection: Repository<Pack
                     lastUpdated: new Date().toISOString(),
                     isActive: true,
                 });
-            } else {
-                Logger.debug(`Package ${pkgname} found in database`, "BuilderEntity");
             }
 
             return packageExists;
@@ -189,15 +209,15 @@ export async function repoExists(name: string, connection: Repository<Repo>): Pr
             });
 
             if (repoExists === undefined) {
-                Logger.log(`Repo ${name} not found in database, creating new entry`, "BuilderEntity");
+                Logger.log(`Repo ${name} not found in database, creating new entry`, "RepoEntity");
                 repoExists = await connection.save({
                     name: name,
                 });
             }
 
             return repoExists;
         } catch (err: unknown) {
-            Logger.error(err, "BuilderEntity");
+            Logger.error(err, "RepoEntity");
         }
     });
 }

backend/src/builder/builder.service.ts

@@ -3,7 +3,7 @@ import { ConfigService } from "@nestjs/config";
 import { InjectRepository } from "@nestjs/typeorm";
 import IORedis from "ioredis";
 import { type Context, Service, ServiceBroker } from "moleculer";
-import type { Repository } from "typeorm";
+import { Repository } from "typeorm";
 import { generateNodeId } from "../functions";
 import type { BuilderDbConnections, MoleculerBuildObject } from "../types";
 import { Build, Builder, builderExists, Package, pkgnameExists, Repo, repoExists } from "./builder.entity";
@@ -13,6 +13,7 @@ import { brokerConfig, MoleculerConfigCommonService } from "./moleculer.config";
 export class BuilderService {
     private broker: ServiceBroker;
     private readonly connection: IORedis;
+    private readonly queryCaches: string[] = [];
 
     constructor(
         @InjectRepository(Build)
@@ -91,6 +92,7 @@ export class BuilderService {
             .orderBy("build.id", "DESC")
             .skip(options.offset)
             .take(options.amount)
+            .cache("builds_general", 30000)
             .getMany();
     }
 
@@ -108,6 +110,7 @@ export class BuilderService {
             .orderBy("build.id", "DESC")
             .skip(options.offset)
             .take(options.amount)
+            .cache("builds_latest", 30000)
             .getMany();
     }
 
@@ -126,7 +129,7 @@ export class BuilderService {
             .orderBy("build.id", "DESC")
             .skip(options.offset)
             .take(options.amount)
-            .cache(true)
+            .cache("builds_latest_pkg", 30000)
             .getMany();
     }
 
@@ -136,7 +139,10 @@ export class BuilderService {
      * @returns The build count for a specific package
      */
     getLastBuildsCountForPackage(pkgname: string): Promise<number> {
-        return this.buildRepository.count({ where: { pkgbase: { pkgname } }, cache: true });
+        return this.buildRepository.count({
+            where: { pkgbase: { pkgname } },
+            cache: { id: `builds_count_${pkgname}`, milliseconds: 30000 },
+        });
     }
 
     /**
@@ -159,7 +165,7 @@ export class BuilderService {
             .orderBy("day", "DESC")
             .skip(options.offset)
             .take(options.amount)
-            .cache(true)
+            .cache(`builds_${options.pkgname}_per_day`, 30000)
             .getRawMany();
     }
 
@@ -184,7 +190,6 @@ export class BuilderService {
                 .where("build.status = :status", { status: options.status })
                 .skip(options.offset)
                 .take(options.amount)
-                .cache(true)
                 .getRawMany();
         }
         return this.buildRepository
@@ -211,7 +216,7 @@ export class BuilderService {
             .addSelect("COUNT(*) AS count")
             .innerJoin("build.builder", "builder")
             .groupBy("build.builder")
-            .cache(true)
+            .cache("builds-per-builder", 30000)
             .getRawMany();
     }
 }
@@ -265,7 +270,7 @@ export class BuilderDatabaseService extends Service {
             pkgnameExists(params.pkgname, this.dbConnections.package),
         ]);
 
-        if (relations.includes(null)) {
+        if (relations.includes(undefined)) {
             Logger.error("Invalid relations, throwing entry away", "BuilderDatabaseService");
             return;
         }

backend/src/config/repo-manager.config.ts

@@ -0,0 +1,10 @@
+import { registerAs } from "@nestjs/config";
+
+export default registerAs("repoMan", () => ({
+    gitlabToken: process.env.GITLAB_TOKEN,
+    gitAuthor: process.env.GIT_AUTHOR ?? "Chaotic Temeraire",
+    gitEmail: process.env.GIT_EMAIL ?? "[email protected]",
+    gitUsername: process.env.GIT_USERNAME ?? "git",
+    schedulerInterval: process.env.REPOMANAGER_SCHEDULE ?? "0 * * * *",
+    alwaysRebuild: process.env.REPOMANAGER_ALWAYS_REBUILD ?? "{}",
+}));

backend/src/constants.ts

@@ -19,3 +19,5 @@ export const requiredEnvVarsProd: string[] = [
 ];
 
 export const requiredEnvVarsDev: string[] = ["PG_DATABASE", "PG_HOST", "PG_PASSWORD", "PG_USER", "REDIS_PASSWORD"];
+
+export const ARCH= "x86_64";

backend/src/functions.ts

@@ -69,3 +69,17 @@ export async function getPasswordHash(password: string): Promise<string> {
 export function nDaysInPast(n: number): Date {
     return new Date(Date.now() - n * 24 * 60 * 60 * 1000);
 }
+
+/**
+ * Check if a given string is a valid URL.
+ * @param url The URL to check
+ * @returns True if the URL is valid, false otherwise
+ */
+export function isValidUrl(url: string): boolean {
+    try {
+        new URL(url);
+        return true;
+    } catch (err: unknown) {
+        return false;
+    }
+}

backend/src/interfaces/repo-manager.ts

@@ -0,0 +1,37 @@
+export interface Repo {
+    name: string;
+    url: string;
+    status: RepoStatus;
+}
+
+export enum RepoStatus {
+    ACTIVE = 0,
+    INACTIVE = 1,
+    RUNNING = 2,
+}
+
+export interface ArchRepoToCheck {
+    name: string;
+    path: string;
+    workDir: string;
+}
+
+export interface ParsedPackage {
+    base: string;
+    pkgrel: number;
+    version: string;
+    name: string;
+}
+
+export interface RepoSettings {
+    gitAuthor: string;
+    gitEmail: string;
+    gitUsername: string;
+    gitlabToken: string;
+    alwaysRebuild: { [key: string]: string };
+}
+
+export interface PkgnameVersion {
+    pkgname: string;
+    archVersion: string;
+}

backend/src/repo-manager/repo-manager.controller.ts

@@ -0,0 +1,18 @@
+import { Controller, Get } from "@nestjs/common";
+import { RepoManagerService } from "./repo-manager.service";
+import { AllowAnonymous } from "../auth/anonymous.decorator";
+
+@Controller("repo")
+export class RepoManagerController {
+    constructor(private repoManager: RepoManagerService) {}
+
+    @Get("test")
+    test(): void {
+        this.repoManager.test();
+    }
+
+    @Get("update")
+    update(): void {
+        this.repoManager.testClonePush();
+    }
+}

backend/src/repo-manager/repo-manager.entity.ts

@@ -0,0 +1,76 @@
+import { Column, Entity, PrimaryGeneratedColumn, Repository } from "typeorm";
+import { Mutex } from "async-mutex";
+import { Logger } from "@nestjs/common";
+
+@Entity()
+export class ArchlinuxPackage {
+    @PrimaryGeneratedColumn()
+    id: number;
+
+    @Column({ type: "varchar" })
+    pkgname: string;
+
+    @Column({ type: "varchar", nullable: true })
+    version: string;
+
+    @Column({ type: "int", nullable: true })
+    pkgrel: number;
+
+    @Column({ type: "varchar", nullable: true })
+    arch: string;
+
+    @Column({ type: "timestamp", nullable: true })
+    lastUpdated: string;
+
+    @Column({ type: "varchar", nullable: true })
+    previousVersion: string;
+}
+
+@Entity()
+export class RepoManagerSettings {
+    @PrimaryGeneratedColumn()
+    id: number;
+
+    @Column({ type: "varchar" })
+    key: string;
+
+    @Column({ type: "varchar" })
+    value: string;
+}
+
+const packageMutex = new Mutex();
+
+/**
+ * Check if a package exists in the database, if not create a new entry
+ * @param pkg The package object
+ * @param connection The repository connection
+ * @returns The package object itself
+ */
+export async function archPkgExists(
+    pkg: {
+        version: string;
+        name: string;
+    },
+    connection: Repository<ArchlinuxPackage>,
+): Promise<ArchlinuxPackage> {
+    return packageMutex.runExclusive(async () => {
+        try {
+            const packages: ArchlinuxPackage[] = await connection.find({ where: { pkgname: pkg.name } });
+            let packageExists: Partial<ArchlinuxPackage> = packages.find((onePkg) => {
+                return onePkg.pkgname === pkg.name;
+            });
+
+            if (packageExists === undefined) {
+                Logger.log(`Package ${pkg.name} not found in database, creating new entry`, "RepoManagerEntity");
+                packageExists = await connection.save({
+                    pkgname: pkg.name,
+                    version: pkg.version,
+                });
+            }
+
+            return packageExists as ArchlinuxPackage;
+        } catch (err: unknown) {
+            Logger.error(err, "RepoManagerEntity");
+        }
+    });
+}

backend/src/repo-manager/repo-manager.module.ts

@@ -0,0 +1,20 @@
+import { Module } from "@nestjs/common";
+import { RepoManagerController } from "./repo-manager.controller";
+import { RepoManagerService } from "./repo-manager.service";
+import { HttpModule } from "@nestjs/axios";
+import { TypeOrmModule } from "@nestjs/typeorm";
+import { ArchlinuxPackage, RepoManagerSettings } from "./repo-manager.entity";
+import { ConfigModule } from "@nestjs/config";
+import repoManagerConfig from "../config/repo-manager.config";
+import { Package, Repo } from "../builder/builder.entity";
+
+@Module({
+    imports: [
+        HttpModule,
+        ConfigModule.forFeature(repoManagerConfig),
+        TypeOrmModule.forFeature([ArchlinuxPackage, Repo, RepoManagerSettings, Package]),
+    ],
+    controllers: [RepoManagerController],
+    providers: [RepoManagerService],
+})
+export class RepoManagerModule {}