chore: fix eslint errors [IN-807] (#3632)

Author: borfast

services/cronjobs/archived_repositories/.env.example

@@ -1,3 +1,5 @@
+ARCHIVED_REPOSITORIES_CHECKER_BATCH_SIZE=10000
+
 ARCHIVED_REPOSITORIES_CHECKER_GITHUB_TOKEN=
 #ARCHIVED_REPOSITORIES_CHECKER_GITHUB_BATCH_SIZE=5000
 #ARCHIVED_REPOSITORIES_CHECKER_GITHUB_DELAY_MS=500000
@@ -6,11 +8,6 @@ ARCHIVED_REPOSITORIES_CHECKER_GITLAB_TOKEN=
 #ARCHIVED_REPOSITORIES_CHECKER_GITLAB_BATCH_SIZE=100
 #ARCHIVED_REPOSITORIES_CHECKER_GITLAB_DELAY_MS=500000
 
-# How many repositories to fetch from the main database and enqueue in each batch
-ARCHIVED_REPOSITORIES_CHECKER_BATCH_SIZE=500
-# How long to wait between batches in milliseconds
-ARCHIVED_REPOSITORIES_CHECKER_BATCH_DELAY_MS=5000
-
 
 # PostgreSQL connection details
 CROWD_DB_WRITE_HOST=

services/cronjobs/archived_repositories/README.md

@@ -8,8 +8,6 @@ It gets all the repositories we have in our database, calls the GitHub and GitLa
 repositories are archived, taking care to not go over the APIs rate limits, and updates each repository in the database
 accordingly.
 
-It gets repositories from the database in batches, to avoid memory issues with our large dataset.
-
 It uses the [BullMQ](https://bullmq.io/) library to manage the job queue and concurrency, including the rate limiting.
 BullMQ requires a Redis instance to be running, which it uses as storage.
 

services/cronjobs/archived_repositories/src/clients/github.ts

@@ -1,40 +1,41 @@
-import { ofetch } from 'ofetch';
-import { Config } from "../config";
-import { RepositoryStatus } from "../types";
+import { ofetch } from 'ofetch'
+
+import { Config } from '../config'
+import { RepositoryStatus } from '../types'
 
 export async function getGithubRepoStatus(url: string, config: Config): Promise<RepositoryStatus> {
-  const parsed = new URL(url);
-  const parts = parsed.pathname.split('/').filter(Boolean);
+  const parsed = new URL(url)
+  const parts = parsed.pathname.split('/').filter(Boolean)
 
   if (parts.length < 2) {
-    throw new Error(`Invalid GitHub repository URL: ${url}`);
+    throw new Error(`Invalid GitHub repository URL: ${url}`)
   }
 
-  const [owner, repo] = parts;
+  const [owner, repo] = parts
 
   try {
     const data = await ofetch(`https://api.github.com/repos/${owner}/${repo}`, {
       headers: {
         Authorization: `Bearer ${config.GithubToken}`,
         Accept: 'application/vnd.github+json',
       },
-    });
+    })
 
     return {
       archived: data.archived,
-      excluded: false
-    };
+      excluded: false,
+    }
   } catch (error: any) {
     // Handle 404 (not found) and 403 (forbidden) as excluded repositories
     if (error?.status === 404 || error?.status === 403) {
-      console.log(`GitHub repo not accessible (${error.status}): ${url} - marking as excluded`);
+      console.log(`GitHub repo not accessible (${error.status}): ${url} - marking as excluded`)
       return {
         archived: false,
-        excluded: true
-      };
+        excluded: true,
+      }
     }
 
     // Re-throw other errors to maintain existing error handling
-    throw error;
+    throw error
   }
 }

services/cronjobs/archived_repositories/src/clients/gitlab.ts

@@ -1,37 +1,38 @@
-import { ofetch } from 'ofetch';
-import { Config } from "../config";
-import { RepositoryStatus } from "../types";
+import { ofetch } from 'ofetch'
+
+import { Config } from '../config'
+import { RepositoryStatus } from '../types'
 
 export async function getGitlabRepoStatus(url: string, config: Config): Promise<RepositoryStatus> {
-  const parsed = new URL(url);
-  const projectPath = parsed.pathname.split('/').filter(Boolean).join('/');
+  const parsed = new URL(url)
+  const projectPath = parsed.pathname.split('/').filter(Boolean).join('/')
 
   if (!projectPath) {
-    throw new Error(`Invalid GitLab repository URL: ${url}`);
+    throw new Error(`Invalid GitLab repository URL: ${url}`)
   }
 
-  const encodedProjectPath = encodeURIComponent(projectPath);
+  const encodedProjectPath = encodeURIComponent(projectPath)
 
   try {
     const data = await ofetch(`https://gitlab.com/api/v4/projects/${encodedProjectPath}`, {
       headers: { 'PRIVATE-TOKEN': config.GitlabToken },
-    });
+    })
 
     return {
       archived: data.archived,
-      excluded: false
-    };
+      excluded: false,
+    }
   } catch (error: any) {
     // Handle 404 (not found) and 403 (forbidden) as excluded repositories
     if (error?.status === 404 || error?.status === 403) {
-      console.log(`GitLab repo not accessible (${error.status}): ${url} - marking as excluded`);
+      console.log(`GitLab repo not accessible (${error.status}): ${url} - marking as excluded`)
       return {
         archived: false,
-        excluded: true
-      };
+        excluded: true,
+      }
     }
 
     // Re-throw other errors to maintain existing error handling
-    throw error;
+    throw error
   }
 }

services/cronjobs/archived_repositories/src/config.ts

@@ -1,52 +1,77 @@
-import dotenv from 'dotenv';
+import dotenv from 'dotenv'
 
 export interface Config {
-  GithubToken: string;
-  GitlabToken: string;
-  
-  PostgresHost: string;
-  PostgresPort: number;
-  PostgresDBName: string;
-  PostgresUsername: string;
-  PostgresPassword: string;
-  PostgresUrl: string;
-  
-  RedisHost: string;
-  RedisPort: number;
-  RedisUsername: string;
-  RedisPassword: string;
-  RedisDB: number; // This is the Redis database index, not a URL.
-  RedisUrl: string;
-  
-  BatchSize: number;
-  BatchDelayMs: number;
+  GithubToken: string
+  GitlabToken: string
+
+  PostgresHost: string
+  PostgresPort: number
+  PostgresDBName: string
+  PostgresUsername: string
+  PostgresPassword: string
+  PostgresUrl: string
+
+  RedisHost: string
+  RedisPort: number
+  RedisUsername: string
+  RedisPassword: string
+  RedisDB: number // This is the Redis database index, not a URL.
+  RedisUrl: string
+
+  BatchSize: number
 }
 
-// These environment variables are required for the application to function correctly. The remaining ones should have
-// sensible defaults.
+// These environment variables are required for the application to function correctly.
+// The remaining ones should have sensible defaults.
 const requiredEnvVars = [
   'ARCHIVED_REPOSITORIES_CHECKER_GITHUB_TOKEN',
   'ARCHIVED_REPOSITORIES_CHECKER_GITLAB_TOKEN',
-  
+
   'CROWD_DB_WRITE_HOST',
   'CROWD_DB_READ_HOST',
   'CROWD_DB_PORT',
   'CROWD_DB_USERNAME',
   'CROWD_DB_PASSWORD',
   'CROWD_DB_DATABASE',
-  
+
   'CROWD_REDIS_USERNAME',
   'CROWD_REDIS_PASSWORD',
   'CROWD_REDIS_HOST',
   'CROWD_REDIS_PORT',
   'ARCHIVED_REPOSITORIES_CHECKER_REDIS_DB',
-];
+]
 
 const defaults = {
-  ARCHIVED_REPOSITORIES_CHECKER_BATCH_SIZE: 100,
-  ARCHIVED_REPOSITORIES_CHECKER_BATCH_DELAY_MS: 5000,
+  ARCHIVED_REPOSITORIES_CHECKER_BATCH_SIZE: 10000,
 }
 
+/**
+ * Helper that returns a validated environment variable as a string.
+ * Throws an error if the variable is required and missing, or if an optional variable is missing
+ * and has no default.
+ */
+const getEnv = (name: string): string => {
+  const val = process.env[name]
+
+  // If we have the value, just return it.
+  if (val !== undefined) {
+    return val
+  }
+
+  // If it's a required env var, throw.
+  if (requiredEnvVars.includes(name)) {
+    throw new Error(`Missing required environment variable: ${name}`)
+  }
+
+  // Optional env var: fall back to defaults if present, otherwise undefined.
+  const defaultVal = (defaults as Record<string, unknown>)[name]
+  if (defaultVal !== undefined) {
+    return String(defaultVal)
+  }
+
+  // If we reach this point, an optional env var is missing and has no default set.
+  throw new Error(`Missing default value for optional environment variable: ${name}`)
+}
 
 /**
  * Loads configuration from environment variables or a .env file.
@@ -55,38 +80,39 @@ const defaults = {
  * we load the configuration in the future if needed (e.g. using a secrets store).
  */
 export function getConfig(): Config {
-  dotenv.config();
+  dotenv.config()
 
   // Check if all required environment variables are set
   for (const envVar of requiredEnvVars) {
     if (!process.env[envVar]) {
-      throw new Error(`Missing required environment variable: ${envVar}`);
+      throw new Error(`Missing required environment variable: ${envVar}`)
     }
   }
 
   // Assemble database connection URL strings for convenience
-  const postgresURL = `postgresql://${process.env.CROWD_DB_USERNAME}:${process.env.CROWD_DB_PASSWORD}@${process.env.CROWD_DB_WRITE_HOST}:${process.env.CROWD_DB_PORT}/${process.env.CROWD_DB_DATABASE}?ssl=true`;
-  const redisURL = `redis://${process.env.CROWD_REDIS_USERNAME}:${process.env.CROWD_REDIS_PASSWORD}@${process.env.CROWD_REDIS_HOST}:${process.env.CROWD_REDIS_PORT}/${process.env.ARCHIVED_REPOSITORIES_CHECKER_REDIS_DB}`;
+  const postgresURL = `postgresql://${getEnv('CROWD_DB_USERNAME')}:${getEnv('CROWD_DB_PASSWORD')}@${getEnv('CROWD_DB_WRITE_HOST')}:${getEnv('CROWD_DB_PORT')}/${getEnv('CROWD_DB_DATABASE')}?ssl=true`
+  const redisURL = `redis://${getEnv('CROWD_REDIS_USERNAME')}:${getEnv('CROWD_REDIS_PASSWORD')}@${getEnv('CROWD_REDIS_HOST')}:${getEnv('CROWD_REDIS_PORT')}/${getEnv('ARCHIVED_REPOSITORIES_CHECKER_REDIS_DB')}`
 
   return {
-    GithubToken: process.env.ARCHIVED_REPOSITORIES_CHECKER_GITHUB_TOKEN!,
-    GitlabToken: process.env.ARCHIVED_REPOSITORIES_CHECKER_GITLAB_TOKEN!,
-
-    PostgresHost: process.env.CROWD_DB_WRITE_HOST!,
-    PostgresPort: parseInt(process.env.CROWD_DB_PORT!, 10),
-    PostgresDBName: process.env.CROWD_DB_DATABASE!,
-    PostgresUsername: process.env.CROWD_DB_USERNAME!,
-    PostgresPassword: process.env.CROWD_DB_PASSWORD!,
+    GithubToken: getEnv('ARCHIVED_REPOSITORIES_CHECKER_GITHUB_TOKEN'),
+    GitlabToken: getEnv('ARCHIVED_REPOSITORIES_CHECKER_GITLAB_TOKEN'),
+
+    PostgresHost: getEnv('CROWD_DB_WRITE_HOST'),
+    PostgresPort: parseInt(getEnv('CROWD_DB_PORT'), 10),
+    PostgresDBName: getEnv('CROWD_DB_DATABASE'),
+    PostgresUsername: getEnv('CROWD_DB_USERNAME'),
+    PostgresPassword: getEnv('CROWD_DB_PASSWORD'),
     PostgresUrl: postgresURL,
 
-    RedisHost: process.env.CROWD_REDIS_HOST!,
-    RedisPort: parseInt(process.env.CROWD_REDIS_PORT!, 10),
-    RedisUsername: process.env.CROWD_REDIS_USERNAME!,
-    RedisPassword: process.env.CROWD_REDIS_PASSWORD!,
-    RedisDB: parseInt(process.env.REDIS_DB!, 10),
+    RedisHost: getEnv('CROWD_REDIS_HOST'),
+    RedisPort: parseInt(getEnv('CROWD_REDIS_PORT'), 10),
+    RedisUsername: getEnv('CROWD_REDIS_USERNAME'),
+    RedisPassword: getEnv('CROWD_REDIS_PASSWORD'),
+    RedisDB: parseInt(getEnv('ARCHIVED_REPOSITORIES_CHECKER_REDIS_DB'), 10),
     RedisUrl: redisURL,
 
-    BatchSize: parseInt(process.env.ARCHIVED_REPOSITORIES_CHECKER_BATCH_SIZE!, 10) || defaults.ARCHIVED_REPOSITORIES_CHECKER_BATCH_SIZE,
-    BatchDelayMs: parseInt(process.env.ARCHIVED_REPOSITORIES_CHECKER_BATCH_DELAY_MS!, 10) || defaults.ARCHIVED_REPOSITORIES_CHECKER_BATCH_DELAY_MS,
-  };
+    BatchSize:
+      parseInt(getEnv('ARCHIVED_REPOSITORIES_CHECKER_BATCH_SIZE'), 10) ||
+      defaults.ARCHIVED_REPOSITORIES_CHECKER_BATCH_SIZE,
+  }
 }