Changed book status to smallint. Added media_type to series. Added 'Hanashi Media' regex resolver for searching. Removed 'Fiction' limitation when searching. Added update series to add new volumes. Fixed search when not all volumes would show up.

backend/database.postgres.sql

@@ -25,6 +25,7 @@ CREATE TABLE
         -- 3rd party id for this series.
         provider_series_id text,
         series_title text NOT NULL,
+        media_type text,
         -- 3rd party used to fetch the data for this series.
         provider varchar(12) NOT NULL,
         added_at timestamp default NULL,
@@ -113,7 +114,7 @@ CREATE TABLE
     book_statuses (
         user_id uuid,
         book_id uuid,
-        state varchar(12),
+        state smallint,
         added_at timestamp default NULL,
         modified_at timestamp default NULL,
         PRIMARY KEY (user_id, book_id),

backend/nestjs-seshat-api/src/books/books.module.ts

@@ -5,11 +5,7 @@ import { BookOriginEntity } from './entities/book-origin.entity';
 import { BookStatusEntity } from './entities/book-status.entity';
 import { TypeOrmModule } from '@nestjs/typeorm';
 import { HttpModule } from '@nestjs/axios';
-import { ProvidersModule } from 'src/providers/providers.module';
 import { SeriesModule } from 'src/series/series.module';
-import { LibraryService } from 'src/library/library.service';
-import { LibraryModule } from 'src/library/library.module';
-import { BullModule } from '@nestjs/bullmq';
 
 @Module({
   imports: [
@@ -20,16 +16,11 @@ import { BullModule } from '@nestjs/bullmq';
     ]),
     SeriesModule,
     HttpModule,
-    ProvidersModule,
-    LibraryModule,
-    BullModule.registerQueue({
-      name: 'library',
-    }),
   ],
   controllers: [],
   exports: [
     BooksService
   ],
-  providers: [BooksService, LibraryService]
+  providers: [BooksService]
 })
 export class BooksModule { }

backend/nestjs-seshat-api/src/books/books.service.ts

@@ -9,6 +9,7 @@ import { CreateBookDto } from './dto/create-book.dto';
 import { CreateBookOriginDto } from './dto/create-book-origin.dto';
 import { CreateBookStatusDto } from './dto/create-book-status.dto';
 import { DeleteBookStatusDto } from './dto/delete-book-status.dto';
+import { SeriesDto } from 'src/series/dto/series.dto';
 
 @Injectable()
 export class BooksService {
@@ -62,12 +63,21 @@ export class BooksService {
     });
   }
 
+  async findBooksFromSeries(series: SeriesDto) {
+    return await this.bookRepository.find({
+      where: {
+        providerSeriesId: series.providerSeriesId,
+        provider: series.provider,
+      }
+    });
+  }
+
   async findBookStatusesTrackedBy(userId: UUID): Promise<BookStatusEntity[]> {
     return await this.bookStatusRepository.createQueryBuilder('s')
       .select(['s.book_id', 's.user_id'])
       .where('s.user_id = :id', { id: userId })
       .innerJoin('s.book', 'b')
-      .addSelect(['b.book_title', 'b.book_desc', 'b.book_volume', 'b.provider'])
+      .addSelect(['b.book_title', 'b.book_desc', 'b.book_volume', 'b.provider', 'b.providerSeriesId'])
       .getMany();
   }
 
@@ -101,7 +111,7 @@ export class BooksService {
     await this.bookStatusRepository.createQueryBuilder()
       .insert()
       .values(status)
-      .orUpdate(['user_id', 'book_id'], ['state', 'modified_at'], { skipUpdateIfNoValuesChanged: true })
+      .orUpdate(['state', 'modified_at'], ['user_id', 'book_id'], { skipUpdateIfNoValuesChanged: true })
       .execute();
   }
 }

backend/nestjs-seshat-api/src/books/dto/create-book-status.dto.ts

@@ -13,7 +13,7 @@ export class CreateBookStatusDto {
 
   @IsString()
   @IsNotEmpty()
-  state: string;
+  state: number;
 
   modifiedAt: Date;
 }

backend/nestjs-seshat-api/src/books/entities/book-status.entity.ts

@@ -11,8 +11,8 @@ export class BookStatusEntity {
   @PrimaryColumn({ name: 'user_id', type: 'uuid' })
   readonly userId: UUID;
 
-  @Column({ name: 'state', type: 'varchar' })
-  state: string;
+  @Column({ name: 'state', type: 'smallint' })
+  state: number;
 
   @Column({ name: 'added_at', type: 'timestamptz', nullable: false })
   addedAt: Date

backend/nestjs-seshat-api/src/library/library.consumer.ts

@@ -1,5 +1,5 @@
 
-import { Processor, WorkerHost } from '@nestjs/bullmq';
+import { OnQueueEvent, Processor, WorkerHost } from '@nestjs/bullmq';
 import { Job } from 'bullmq';
 import { PinoLogger } from 'nestjs-pino';
 import { GoogleSearchContext } from 'src/providers/contexts/google.search.context';
@@ -26,69 +26,67 @@ export class LibraryConsumer extends WorkerHost {
       msg: 'Started task on queue.',
     });
 
-    const series: CreateSeriesSubscriptionJobDto = job.data;
+    if (job.name == 'new_series') {
+      const series: CreateSeriesSubscriptionJobDto = job.data;
+      const books = await this.search(job, series, false);
 
-    let context = this.provider.generateSearchContext(series.provider, series.title) as GoogleSearchContext;
-    //context.intitle = series.title;
-    context.maxResults = '40';
-    context.subject = 'Fiction';
-
-    // Search for the book(s) via the provider.
-    // Up until end of results or after 3 unhelpful pages of results.
-    let results = [];
-    let related = [];
-    let pageSearchedCount = 0;
-    let unhelpfulResultsCount = 0;
-    do {
-      pageSearchedCount += 1;
-      results = await this.provider.search(context);
-      const potential = results.filter(r => r.providerSeriesId == series.providerSeriesId || r.title == series.title);
-      if (potential.length > 0) {
-        related.push.apply(related, potential);
-      } else {
-        unhelpfulResultsCount += 1;
+      let counter = 0;
+      for (let book of books) {
+        try {
+          // Force the provider's series id to be set, so that we know which series this belongs.
+          book.result.providerSeriesId = series.providerSeriesId;
+          await this.library.addBook(book.result);
+        } catch (err) {
+          this.logger.error({
+            class: LibraryConsumer.name,
+            method: this.process.name,
+            book: book.result,
+            score: book.score,
+            msg: 'Failed to add book in background during adding series.',
+            error: err,
+          });
+        } finally {
+          counter++;
+          job.updateProgress(25 + 75 * counter / books.length);
+        }
       }
-      context = context.next();
-      job.updateProgress(pageSearchedCount * 5);
-    } while (results.length >= 40 && unhelpfulResultsCount < 3);
+    } else if (job.name == 'update_series') {
+      const series: CreateSeriesSubscriptionJobDto = job.data;
+      const existingBooks = await this.library.getBooksFromSeries(series);
+      const existingVolumes = existingBooks.map(b => b.volume);
+      const books = await this.search(job, series, true);
 
-    // Sort & de-duplicate the entries received.
-    const books = related.map(book => this.toScore(book, series))
-      .sort((a, b) => a.result.volume - b.result.volume || b.score - a.score)
-      .filter((_, index, arr) => index == 0 || arr[index - 1].result.volume != arr[index].result.volume);
-    job.updateProgress(25);
+      let counter = 0;
+      for (let book of books) {
+        if (existingVolumes.includes(book.result.volume)) {
+          continue;
+        }
 
-    this.logger.debug({
-      class: LibraryConsumer.name,
-      method: this.process.name,
-      job: job,
-      msg: 'Finished searching for book entries.',
-      results: {
-        pages: pageSearchedCount,
-        related_entries: related.length,
-        volumes: books.length,
-      }
-    });
-
-    let counter = 0;
-    for (let book of books) {
-      try {
-        // Force the provider's series id to be set, so that we know which series this belongs.
-        book.result.providerSeriesId = series.providerSeriesId;
-        await this.library.addBook(book.result);
-      } catch (err) {
-        this.logger.error({
-          class: LibraryConsumer.name,
-          method: this.process.name,
-          book: book.result,
-          score: book.score,
-          msg: 'Failed to add book in background.',
-          error: err,
-        });
-      } finally {
-        counter++;
-        job.updateProgress(25 + 75 * counter / books.length);
+        try {
+          // Force the provider's series id to be set, so that we know which series this belongs.
+          book.result.providerSeriesId = series.providerSeriesId;
+          await this.library.addBook(book.result);
+        } catch (err) {
+          this.logger.error({
+            class: LibraryConsumer.name,
+            method: this.process.name,
+            book: book.result,
+            score: book.score,
+            msg: 'Failed to add book in background during series update.',
+            error: err,
+          });
+        } finally {
+          counter++;
+          job.updateProgress(25 + 75 * counter / books.length);
+        }
       }
+    } else {
+      this.logger.warn({
+        class: LibraryConsumer.name,
+        method: this.process.name,
+        job: job,
+        msg: 'Unknown job name found.',
+      });
     }
 
     this.logger.info({
@@ -101,6 +99,92 @@ export class LibraryConsumer extends WorkerHost {
     return null;
   }
 
+  private async search(job: Job, series: CreateSeriesSubscriptionJobDto, newest: boolean): Promise<{ result: BookSearchResultDto, score: number }[]> {
+    let context = this.provider.generateSearchContext(series.provider, series.title) as GoogleSearchContext;
+    context.maxResults = '40';
+    if (newest) {
+      context.orderBy = 'newest';
+    }
+
+    // Search for the book(s) via the provider.
+    // Up until end of results or after 3 unhelpful pages of results.
+    let results = [];
+    let related = [];
+    let pageSearchedCount = 0;
+    let unhelpfulResultsCount = 0;
+    do {
+      pageSearchedCount += 1;
+      results = await this.provider.search(context);
+      const potential = results.filter((r: BookSearchResultDto) => r.providerSeriesId == series.providerSeriesId || r.title == series.title && r.mediaType == series.mediaType);
+      if (potential.length > 0) {
+        related.push.apply(related, potential);
+      } else {
+        unhelpfulResultsCount += 1;
+      }
+      context = context.next();
+      job.updateProgress(pageSearchedCount * 5);
+    } while (results.length >= context.maxResults && (!newest || unhelpfulResultsCount < 3));
+
+    // Sort & de-duplicate the entries received.
+    const books = related.map(book => this.toScore(book, series))
+      .sort((a, b) => a.result.volume - b.result.volume || b.score - a.score)
+      .filter((_, index, arr) => index == 0 || arr[index - 1].result.volume != arr[index].result.volume);
+    job.updateProgress(25);
+    
+
+    this.logger.debug({
+      class: LibraryConsumer.name,
+      method: this.search.name,
+      job: job,
+      msg: 'Finished searching for book entries.',
+      results: {
+        pages: pageSearchedCount,
+        related_entries: related.length,
+        volumes: books.length,
+      }
+    });
+
+    return books;
+  }
+
+  @OnQueueEvent('failed')
+  onFailed(job: Job, err: Error) {
+    this.logger.error({
+      class: LibraryConsumer.name,
+      method: this.onFailed.name,
+      job: job,
+      msg: 'A library job failed.',
+      error: err,
+    });
+  }
+
+  @OnQueueEvent('paused')
+  onPaused() {
+    this.logger.info({
+      class: LibraryConsumer.name,
+      method: this.onPaused.name,
+      msg: 'Library jobs have been paused.',
+    });
+  }
+
+  @OnQueueEvent('resumed')
+  onResume(job: Job) {
+    this.logger.info({
+      class: LibraryConsumer.name,
+      method: this.onResume.name,
+      msg: 'Library jobs have resumed.',
+    });
+  }
+
+  @OnQueueEvent('waiting')
+  onWaiting(jobId: number | string) {
+    this.logger.info({
+      class: LibraryConsumer.name,
+      method: this.onWaiting.name,
+      msg: 'A library job is waiting...',
+    });
+  }
+
   private toScore(book: BookSearchResultDto, series: CreateSeriesSubscriptionJobDto): ({ result: BookSearchResultDto, score: number }) {
     if (!book) {
       return {

backend/nestjs-seshat-api/src/library/library.controller.ts

@@ -1,5 +1,5 @@
 import { InjectQueue } from '@nestjs/bullmq';
-import { Body, Controller, Delete, Get, Post, Put, Request, Res, UseGuards } from '@nestjs/common';
+import { Body, Controller, Delete, Get, Patch, Post, Put, Request, Res, UseGuards } from '@nestjs/common';
 import { Response } from 'express';
 import { Queue } from 'bullmq';
 import { PinoLogger } from 'nestjs-pino';
@@ -49,6 +49,44 @@ export class LibraryController {
         provider: body.provider,
         providerSeriesId: body.providerSeriesId,
         title: body.title,
+        mediaType: body.mediaType,
+      });
+
+      return {
+        success: true,
+      };
+    } catch (err) {
+      if (err instanceof QueryFailedError) {
+        if (err.driverError.code == '23505') {
+          // Subscription already exist.
+          response.statusCode = 409;
+          return {
+            success: false,
+            error_message: 'Series subscription already exists.',
+          };
+        }
+      }
+
+      response.statusCode = 500;
+      return {
+        success: false,
+        error_message: 'Something went wrong.',
+      };
+    }
+  }
+
+  @Patch('series')
+  async updateSeries(
+    @Request() req,
+    @Body() body: CreateSeriesSubscriptionDto,
+    @Res({ passthrough: true }) response: Response,
+  ) {
+    try {
+      await this.library.updateSeries({
+        provider: body.provider,
+        providerSeriesId: body.providerSeriesId,
+        title: body.title,
+        mediaType: body.mediaType,
       });
 
       return {
@@ -149,9 +187,18 @@ export class LibraryController {
           provider: body.provider,
           providerSeriesId: body.providerSeriesId,
           title: body.title,
+          mediaType: body.mediaType,
         });
       } catch (err) {
         if (err instanceof QueryFailedError) {
+          this.logger.error({
+            class: LibraryController.name,
+            method: this.createBook.name,
+            user: req.user,
+            msg: 'Failed to create a series for a book.',
+            error: err,
+          });
+
           // Ignore if the series already exist.
           if (err.driverError.code != '23505') {
             response.statusCode = 500;
@@ -171,6 +218,14 @@ export class LibraryController {
       };
     } catch (err) {
       if (err instanceof QueryFailedError) {
+        this.logger.error({
+          class: LibraryController.name,
+          method: this.createBook.name,
+          user: req.user,
+          msg: 'Failed to create book.',
+          error: err,
+        });
+
         if (err.driverError.code == '23505') {
           // Book exists already.
           response.statusCode = 409;
@@ -179,7 +234,7 @@ export class LibraryController {
             error_message: 'The book has already been added previously.',
           };
         } else if (err.driverError.code == '23503') {
-          // Data dependency is missing.
+          // Series is missing.
           response.statusCode = 500;
           return {
             success: false,
@@ -188,14 +243,6 @@ export class LibraryController {
         }
       }
 
-      this.logger.error({
-        class: LibraryController.name,
-        method: this.createBook.name,
-        user: req.user,
-        msg: 'Failed to create book.',
-        error: err,
-      });
-
       response.statusCode = 500;
       return {
         success: false,

backend/nestjs-seshat-api/src/library/library.service.ts

@@ -3,10 +3,10 @@ import { Injectable } from '@nestjs/common';
 import { Queue } from 'bullmq';
 import { PinoLogger } from 'nestjs-pino';
 import { BooksService } from 'src/books/books.service';
-import { CreateBookDto } from 'src/books/dto/create-book.dto';
 import { BookSearchResultDto } from 'src/providers/dto/book-search-result.dto';
 import { CreateSeriesDto } from 'src/series/dto/create-series.dto';
 import { SeriesSubscriptionDto } from 'src/series/dto/series-subscription.dto';
+import { SeriesDto } from 'src/series/dto/series.dto';
 import { SeriesService } from 'src/series/series.service';
 import { BookOriginType } from 'src/shared/enums/book_origin_type';
 
@@ -26,7 +26,7 @@ export class LibraryService {
     this.logger.debug({
       class: LibraryService.name,
       method: this.addSubscription.name,
-      series: series.providerSeriesId,
+      series: series,
       msg: 'Series saved to database.',
     });
 
@@ -135,4 +135,12 @@ export class LibraryService {
 
     return bookId;
   }
+
+  async updateSeries(series: CreateSeriesDto) {
+    return await this.jobs.add('update_series', series);
+  }
+
+  async getBooksFromSeries(series: SeriesDto) {
+    return await this.books.findBooksFromSeries(series);
+  }
 }

backend/nestjs-seshat-api/src/providers/contexts/google.search.context.ts

@@ -7,7 +7,7 @@ export class GoogleSearchContext extends SearchContext {
 
 
   generateQueryParams() {
-    const filterParams = ['maxResults', 'startIndex'];
+    const filterParams = ['maxResults', 'startIndex', 'orderBy'];
     const searchParams = ['intitle', 'inauthor', 'inpublisher', 'subject', 'isbn'];
 
     const queryParams = filterParams
@@ -21,7 +21,19 @@ export class GoogleSearchContext extends SearchContext {
       ...searchParams.map(p => this.params[p] ? p + ':"' + this.params[p] + '"' : ''),
     ].filter(p => p.length > 0).join('');
 
-    return [queryParams, 'q=' + searchQueryParam].filter(q => q.length > 0).join('&');
+    return [queryParams, 'q=' + searchQueryParam].filter(q => q.length > 2).join('&');
+  }
+
+  get orderBy(): 'newest' | 'relevant' {
+    return this.params['orderBy'] as 'newest' | 'relevant' ?? 'relevant';
+  }
+
+  set orderBy(value: 'newest' | 'relevant' | null) {
+    if (!value) {
+      delete this.params['orderBy'];
+    } else {
+      this.params['orderBy'] = value;
+    }
   }
 
   get maxResults(): number {

backend/nestjs-seshat-api/src/providers/dto/book-search-result.dto.ts

@@ -48,6 +48,10 @@ export class BookSearchResultDto {
   @IsNotEmpty()
   language: string;
 
+  @IsString()
+  @IsOptional()
+  mediaType: string | null;
+
   @IsArray()
   @IsString({ each: true })
   categories: string[];

backend/nestjs-seshat-api/src/providers/google/google.service.ts

@@ -59,9 +59,10 @@ export class GoogleService {
       volume: item.volumeInfo.seriesInfo?.bookDisplayNumber ? parseInt(item.volumeInfo.seriesInfo?.bookDisplayNumber, 10) : undefined,
       publisher: item.volumeInfo.publisher,
       authors: item.volumeInfo.authors,
-      categories: item.volumeInfo.categories,
+      categories: item.volumeInfo.categories ?? [],
+      mediaType: null,
       maturityRating: item.volumeInfo.maturityRating,
-      industryIdentifiers: item.volumeInfo.industryIdentifiers ? Object.assign({}, ...item.volumeInfo.industryIdentifiers.map(i => ({ [i.type]: i.identifier }))) : [],
+      industryIdentifiers: item.volumeInfo.industryIdentifiers ? Object.assign({}, ...item.volumeInfo.industryIdentifiers.map(i => i.type == 'OTHER' ? { [i.identifier.split(':')[0]]: i.identifier.split(':')[1] } : { [i.type]: i.identifier })) : [],
       publishedAt: new Date(item.volumeInfo.publishedDate),
       language: item.volumeInfo.language,
       thumbnail: item.volumeInfo.imageLinks?.thumbnail,
@@ -69,8 +70,7 @@ export class GoogleService {
       provider: 'google'
     }
 
-    let regex = this.getRegexByPublisher(result.publisher);
-
+    const regex = this.getRegexByPublisher(result.publisher);
     const match = result.title.match(regex);
     if (match?.groups) {
       result.title = match.groups['title'].trim();
@@ -79,19 +79,41 @@ export class GoogleService {
       }
     }
 
+    if (match?.groups && 'media_type' in match.groups) {
+      result.mediaType = match.groups['media_type'];
+    } else if (result.categories.includes('Comics & Graphic Novels')) {
+      result.mediaType = 'Comics & Graphic Novels';
+    } else if (result.categories.includes('Fiction') || result.categories.includes('Young Adult Fiction')) {
+      result.mediaType = 'Novel';
+    } else {
+      result.mediaType = 'Book';
+    }
+
+    if (result.mediaType) {
+      if (result.mediaType.toLowerCase() == "light novel") {
+        result.mediaType = 'Light Novel';
+      } else if (result.mediaType.toLowerCase() == 'manga') {
+        result.mediaType = 'Manga';
+      }
+    }
+
     return result;
   }
 
   private getRegexByPublisher(publisher: string): RegExp {
     switch (publisher) {
       case 'J-Novel Club':
-        return /(?<title>.+?):?\sVolume\s(?<volume>\d+)/i;
+        return /^(?<title>.+?):?\sVolume\s(?<volume>\d+)$/i;
       case 'Yen On':
       case 'Yen Press':
       case 'Yen Press LLC':
-        return /(?<title>.+?),?\sVol\.\s(?<volume>\d+)\s\((?<media_type>[\w\s]+)\)/;
+        return /^(?<title>.+?)(?:,?\sVol\.\s(?<volume>\d+))?\s\((?<media_type>[\w\s]+)\)$/;
+      case 'Hanashi Media':
+        return /^(?<title>.+?)\s\((?<media_type>[\w\s]+)\),?\sVol\.\s(?<volume>\d+)$/
+      case 'Regin\'s Chronicles':
+        return /^(?<title>.+?)\s\((?<media_type>[\w\s]+)\)(?<subtitle>\:\s.+?)?$/
       default:
-        return /(?<title>.+?)(?:,|:|\s\-)?\s(?:Vol(?:\.|ume)?)?\s(?<volume>\d+)/;
+        return /^(?<title>.+?)(?:,|:|\s\-)?\s(?:Vol(?:\.|ume)?)?\s(?<volume>\d+)$/;
     }
   }
 }

backend/nestjs-seshat-api/src/series/dto/create-series-subscription-job.dto.ts

@@ -5,4 +5,8 @@ export class CreateSeriesSubscriptionJobDto extends SeriesSubscriptionDto {
   @IsString()
   @IsNotEmpty()
   title: string;
+
+  @IsString()
+  @IsNotEmpty()
+  mediaType: string;
 }

backend/nestjs-seshat-api/src/series/dto/create-series-subscription.dto.ts

@@ -5,4 +5,8 @@ export class CreateSeriesSubscriptionDto extends SeriesDto {
   @IsString()
   @IsNotEmpty()
   title: string;
+
+  @IsString()
+  @IsNotEmpty()
+  mediaType: string;
 }

backend/nestjs-seshat-api/src/series/dto/create-series.dto.ts

@@ -5,4 +5,8 @@ export class CreateSeriesDto extends SeriesDto {
   @IsString()
   @IsNotEmpty()
   title: string;
+
+  @IsString()
+  @IsNotEmpty()
+  mediaType: string;
 }

backend/nestjs-seshat-api/src/series/entities/series.entity.ts

@@ -14,6 +14,9 @@ export class SeriesEntity {
   @Column({ name: 'series_title', type: 'text', nullable: false })
   title: string;
 
+  @Column({ name: 'media_type', type: 'text', nullable: true })
+  mediaType: string;
+
   @Column({ name: 'provider', type: 'text', nullable: false })
   provider: string;