Loader: Import legace DSOMM progress from localStorage; Improved error handling

Author: vbakke

src/app/component/modal-message/modal-message.component.ts

@@ -6,6 +6,8 @@ import {
   MatDialogConfig,
 } from '@angular/material/dialog';
 import * as md from 'markdown-it';
+import { MarkdownText } from 'src/app/model/markdown-text';
+import { NotificationService } from 'src/app/service/notification.service';
 
 @Component({
   selector: 'app-modal-message',
@@ -32,13 +34,18 @@ export class ModalMessageComponent implements OnInit {
   constructor(
     public dialog: MatDialog,
     public dialogRef: MatDialogRef<ModalMessageComponent>,
-    @Inject(MAT_DIALOG_DATA) data: DialogInfo
+    @Inject(MAT_DIALOG_DATA) data: DialogInfo,
+    private notificationService: NotificationService
   ) {
     this.data = data;
   }
 
   // eslint-disable-next-line @angular-eslint/no-empty-lifecycle-method
-  ngOnInit(): void {}
+  ngOnInit(): void {
+    this.notificationService.message$.subscribe(({ title, message }) => {
+      this.openDialog(new DialogInfo(message, title));
+    });
+  }
 
   openDialog(dialogInfo: DialogInfo | string): MatDialogRef<ModalMessageComponent> {
     // Remove focus from the button that becomes aria unavailable (avoids ugly console error message)
@@ -77,7 +84,8 @@ export class DialogInfo {
   buttons: string[] = ['OK'];
 
   constructor(msg: string = '', title: string = '') {
-    this.message = msg;
+    let md: MarkdownText = new MarkdownText(msg);
+    this.message = md.render();
     this.title = title;
   }
 }

src/app/model/progress-store.ts

@@ -425,30 +425,35 @@ export class ProgressStore {
       try {
         const legacyDataset = JSON.parse(stored);
 
-        legacyDataset.forEach((entry: any) => {
-          const legacyActivities = entry.Activity;
-
-          if (Array.isArray(legacyActivities)) {
-            legacyActivities.forEach((legacyActivity: any) => {
-              const activityUuid = legacyActivity.uuid;
-              const teamsImplemented = legacyActivity.teamsImplemented;
-              if (teamsImplemented) {
-                Object.keys(teamsImplemented).forEach((team: string) => {
-                  if (legacyActivity.teamsImplemented[team]) {
-                    if (!progress![activityUuid]) {
-                      progress![activityUuid] = {};
+        try {
+          legacyDataset.forEach((entry: any) => {
+            const legacyActivities = entry.Activity;
+
+            if (Array.isArray(legacyActivities)) {
+              legacyActivities.forEach((legacyActivity: any) => {
+                const activityUuid = legacyActivity.uuid;
+                const teamsImplemented = legacyActivity.teamsImplemented;
+                if (teamsImplemented) {
+                  Object.keys(teamsImplemented).forEach((team: string) => {
+                    if (legacyActivity.teamsImplemented[team]) {
+                      if (!progress![activityUuid]) {
+                        progress![activityUuid] = {};
+                      }
+                      if (!progress![activityUuid][team]) {
+                        progress![activityUuid][team] = {};
+                      }
+                      console.log(`Legacy import: Setting '${completeTitle}' on ${activityUuid} for ${team}`); // eslint-disable-line
+                      progress![activityUuid][team][completeTitle] = new Date();
                     }
-                    if (!progress![activityUuid][team]) {
-                      progress![activityUuid][team] = {};
-                    }
-                    console.log(`Legacy import: Setting '${completeTitle}' on ${activityUuid} for ${team}`); // eslint-disable-line
-                    progress![activityUuid][team][completeTitle] = new Date();
-                  }
-                });
-              }
-            });
-          }
-        });
+                  });
+                }
+              });
+            }
+          });
+        } catch (error) {
+          console.error('Failed to process legacy dataset activities: ', error);
+          throw Error('Unexpected data structure: ' + error);
+        }
       } catch (error) {
         console.error('Failed to parse legacy progress: ', error);
         throw Error(

src/app/pages/matrix/matrix.component.ts

@@ -7,13 +7,10 @@ import { Activity, ActivityStore, Data } from 'src/app/model/activity-store';
 import { UntilDestroy } from '@ngneat/until-destroy';
 import { MatChip, MatChipList } from '@angular/material/chips';
 import { deepCopy } from 'src/app/util/util';
-import {
-  ModalMessageComponent,
-  DialogInfo,
-} from '../../component/modal-message/modal-message.component';
 import { DataStore } from 'src/app/model/data-store';
 import { perfNow } from 'src/app/util/util';
 import { SettingsService } from 'src/app/service/settings/settings.service';
+import { NotificationService } from 'src/app/service/notification.service';
 
 export interface MatrixRow {
   Category: string;
@@ -52,7 +49,7 @@ export class MatrixComponent implements OnInit {
     private loader: LoaderService,
     private settings: SettingsService,
     private router: Router,
-    public modal: ModalMessageComponent
+    private notificationService: NotificationService
   ) {}
   /* eslint-enable */
 
@@ -75,17 +72,13 @@ export class MatrixComponent implements OnInit {
         console.log(`${perfNow()}: Page loaded: Matrix`);
       })
       .catch(err => {
-        this.displayMessage(new DialogInfo(err.message, 'An error occurred'));
+        this.notificationService.notify('An error occurred', err.message);
         if (err.hasOwnProperty('stack')) {
           console.warn(err);
         }
       });
   }
 
-  displayMessage(dialogInfo: DialogInfo) {
-    this.modal.openDialog(dialogInfo);
-  }
-
   setYamlData(dataStore: DataStore) {
     this.dataStore = dataStore;
     if (!dataStore.activityStore) {

src/app/pages/settings/settings.component.css

@@ -131,6 +131,10 @@ input.progress-score {
   flex: 1;
 }
 
+.version-info-section .error {
+    font-style: italic;
+}
+
 .edit-hint {
   color: #666;
   font-size: 0.9em;

src/app/pages/settings/settings.component.ts

@@ -105,6 +105,7 @@ export class SettingsComponent implements OnInit {
     } catch (err: any) {
       console.warn('Error checking latest DSOMM release', err);
       this.remoteReleaseCheck.latestCheckError = err?.message || 'Failed to check latest release';
+      return;
     } finally {
       this.remoteReleaseCheck.isChecking = false;
     }
@@ -127,14 +128,15 @@ export class SettingsComponent implements OnInit {
       if (remoteTag && localTag && remoteDate && localDate) {
         newer = remoteTag !== localTag || remoteDate > localDate;
       } else {
+        newer = true; // Show download link if we cannot compare
+
+        // Build error message
         let tmp: string[] = [];
-        if (!remoteTag) tmp.push('remoteTag');
-        if (!localTag) tmp.push('localTag');
-        if (!remoteDate) tmp.push('remoteDate');
-        if (!localDate) tmp.push('localDate');
-        this.remoteReleaseCheck.latestCheckError = `Could not determine ${tmp.join(
-          ', '
-        )} for comparison`;
+        if (!remoteTag) tmp.push('DSOMM model version');
+        if (!localTag) tmp.push('local model version');
+        if (!remoteDate) tmp.push('DSOMM model date');
+        if (!localDate) tmp.push('local model date');
+        this.remoteReleaseCheck.latestCheckError = `Could not determine ${tmp.join(', ')}`; // eslint-disable-line
         console.warn('ERROR: ' + this.remoteReleaseCheck.latestCheckError);
       }
       this.remoteReleaseCheck.isNewerAvailable = newer;

src/app/service/loader/data-loader.service.ts

@@ -1,6 +1,7 @@
 import { Injectable } from '@angular/core';
 import { perfNow } from 'src/app/util/util';
-import { YamlService } from '../yaml-loader/yaml-loader.service';
+import { FileNotFoundError, YamlService } from '../yaml-loader/yaml-loader.service';
+import { GithubService } from '../settings/github.service';
 import { MetaStore } from 'src/app/model/meta-store';
 import { TeamProgressFile, Uuid } from 'src/app/model/types';
 import {
@@ -11,20 +12,36 @@ import {
   Data,
 } from 'src/app/model/activity-store';
 import { DataStore } from 'src/app/model/data-store';
+import { NotificationService } from '../notification.service';
 
 export class DataValidationError extends Error {
   constructor(message: string) {
     super(message);
   }
 }
 
+export class MissingModelError extends Error {
+  filename: string | null;
+  constructor(message: string, filename: string | null = null) {
+    super(message);
+    this.filename = filename;
+  }
+}
+
 @Injectable({ providedIn: 'root' })
 export class LoaderService {
   private META_FILE: string = 'assets/YAML/meta.yaml';
+  private DSOMM_MODEL_URL: string;
   private debug: boolean = false;
   private dataStore: DataStore | null = null;
 
-  constructor(private yamlService: YamlService) {}
+  constructor(
+    private yamlService: YamlService,
+    private githubService: GithubService,
+    private notificationService: NotificationService
+  ) {
+    this.DSOMM_MODEL_URL = this.githubService.getDsommModelUrl();
+  }
 
   get datastore(): DataStore | null {
     return this.dataStore;
@@ -70,8 +87,18 @@ export class LoaderService {
       console.log(`${perfNow()}: YAML: All YAML files loaded`);
 
       return this.dataStore;
-    } catch (err) {
-      throw err;
+    } catch (err: any) {
+      if (err instanceof FileNotFoundError) {
+        console.error(`${perfNow()}: Missing model file: ${err?.filename || err}`);
+        if (err.filename && err.filename.endsWith('default/model.yaml')) {
+          this.notificationService.notify('Loading error', `No DSOMM model found.\n\nPlease download \`model.yaml\` from [GitHub](${this.DSOMM_MODEL_URL}).`); // eslint-disable-line
+        } else {
+          this.notificationService.notify('Loading error', err.message + ': ' + err.filename);
+        }
+      } else {
+        this.notificationService.notify('Error', 'Failed to load data: \n\n' + err);
+      }
+      return this.dataStore;
     }
   }
 
@@ -117,11 +144,14 @@ export class LoaderService {
   private async loadActivities(meta: MetaStore): Promise<ActivityStore> {
     const activityStore = new ActivityStore();
     const errors: string[] = [];
-    let usingHistoricYamlFile = false;
+    let usingLegacyYamlFile = false;
 
+    if (meta.activityFiles.length == 0) {
+      throw new MissingModelError('No `activityFiles` are specified in `meta.yaml`.');
+    }
     for (let filename of meta.activityFiles) {
       if (this.debug) console.log(`${perfNow()}s: Loading activity file: ${filename}`);
-      usingHistoricYamlFile ||= filename.endsWith('generated/generated.yaml');
+      usingLegacyYamlFile ||= filename.endsWith('generated/generated.yaml');
 
       const response: ActivityFile = await this.loadActivityFile(filename);
 
@@ -140,8 +170,8 @@ export class LoaderService {
       if (errors.length > 0) {
         errors.forEach(error => console.error(error));
 
-        // Only throw for non-generated files (backwards compatibility)
-        if (!usingHistoricYamlFile) {
+        // Legacy generated.yaml has several data validation problems. Do not report these
+        if (!usingLegacyYamlFile) {
           throw new DataValidationError(
             'Data validation error after loading: ' +
               filename +

src/app/service/notification.service.ts

@@ -0,0 +1,27 @@
+import { Injectable } from '@angular/core';
+import { Subject } from 'rxjs';
+import { MatDialog, MatDialogConfig } from '@angular/material/dialog';
+import {
+  ModalMessageComponent,
+  DialogInfo,
+} from '../component/modal-message/modal-message.component';
+
+@Injectable({ providedIn: 'root' })
+export class NotificationService {
+  private messageSubject = new Subject<{ title: string; message: string; error: any }>();
+  message$ = this.messageSubject.asObservable();
+
+  constructor(private dialog: MatDialog) {}
+
+  notify(title: string, message: string, error: any = null) {
+    this.messageSubject.next({ title, message, error });
+
+    const dialogConfig = new MatDialogConfig();
+    dialogConfig.id = 'modal-message';
+    dialogConfig.disableClose = true;
+    dialogConfig.autoFocus = false;
+    dialogConfig.data = new DialogInfo(message, title);
+
+    this.dialog.open(ModalMessageComponent, dialogConfig);
+  }
+}

src/app/service/settings/github.service.ts

@@ -14,17 +14,19 @@ export interface GithubReleaseInfo {
   providedIn: 'root',
 })
 export class GithubService {
-  private readonly CHANGELOG_URL =
-    'https://github.com/devsecopsmaturitymodel/DevSecOps-MaturityModel-data/blob/main/CHANGELOG.md';
-
-  private readonly LATEST_RELEASE_URL =
-    'https://api.github.com/repos/devsecopsmaturitymodel/DevSecOps-MaturityModel-data/releases/latest';
-
-  private readonly DOWNLOAD_URL_TEMPLATE =
-    'https://raw.githubusercontent.com/devsecopsmaturitymodel/DevSecOps-MaturityModel-data/{tag}/generated/activities.yaml';
+  /* eslint-disable */
+  private readonly DSOMM_MODEL_URL = 'https://github.com/devsecopsmaturitymodel/DevSecOps-MaturityModel-data/';
+  private readonly CHANGELOG_URL = this.DSOMM_MODEL_URL + '/blob/main/CHANGELOG.md';
+  private readonly LATEST_RELEASE_URL = this.DSOMM_MODEL_URL.replace('//github.com', '//api.github.com/repos') + 'releases/latest';
+  private readonly DOWNLOAD_URL_TEMPLATE = this.DSOMM_MODEL_URL.replace('//github.com', '//raw.githubusercontent.com') + '{tag}/generated/model.yaml';
+  /* eslint-enable */
 
   constructor(private http: HttpClient) {}
 
+  public getDsommModelUrl(): string {
+    return this.DSOMM_MODEL_URL;
+  }
+
   async getLatestRelease(): Promise<GithubReleaseInfo> {
     const obs = this.http.get<GithubReleaseInfo>(this.LATEST_RELEASE_URL);
     const remote: any = await firstValueFrom(obs);

src/app/service/yaml-loader/yaml-loader.service.ts

@@ -6,6 +6,14 @@ import {
 } from 'yaml';
 import { perfNow } from 'src/app/util/util';
 
+export class FileNotFoundError extends Error {
+  filename: string | null;
+  constructor(message: string, filename: string | null = null) {
+    super(message);
+    this.filename = filename;
+  }
+}
+
 @Injectable({ providedIn: 'root' })
 export class YamlService {
   private _refs: Record<string, any>;
@@ -56,8 +64,13 @@ export class YamlService {
     const response: Response = await fetch(url);
 
     if (!response.ok) {
-      throw new Error(`Failed to fetch the '${url}' YAML file: ${response.statusText}`);
+      if (response.status === 404) {
+        throw new FileNotFoundError('File not found', url);
+      } else {
+        throw new Error(`Failed to fetch the '${url}' YAML file: ${response.statusText}`);
+      }
     }
+
     const yamlText: string = await response.text();
     const timeFetched: Date = new Date();
     console.debug(`${perfNow()}: YAML: Retrieved ${url}`);