tom/seshat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Changed book status to smallint. Added media_type to series. Added 'Hanashi Media' regex resolver for searching. Removed 'Fiction' limitation when searching. Added update series to add new volumes. Fixed search when not all volumes would show up.

Browse Source
This commit is contained in:
Tom
2025-03-03 21:18:46 +00:00
parent d0c074135e
commit d02da321a1
14 changed files with 250 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

162
backend/nestjs-seshat-api/src/library/library.consumer.ts
View File

@ -26,69 +26,67 @@ export class LibraryConsumer extends WorkerHost {
msg: 'Started task on queue.',
});
const series: CreateSeriesSubscriptionJobDto = job.data;
if (job.name == 'new_series') {
const series: CreateSeriesSubscriptionJobDto = job.data;
const books = await this.search(job, series, false);
let context = this.provider.generateSearchContext(series.provider, series.title) as GoogleSearchContext;
//context.intitle = series.title;
context.maxResults = '40';
context.subject = 'Fiction';
// Search for the book(s) via the provider.
// Up until end of results or after 3 unhelpful pages of results.
let results = [];
let related = [];
let pageSearchedCount = 0;
let unhelpfulResultsCount = 0;
do {
pageSearchedCount += 1;
results = await this.provider.search(context);
const potential = results.filter(r => r.providerSeriesId == series.providerSeriesId || r.title == series.title);
if (potential.length > 0) {
related.push.apply(related, potential);
} else {
unhelpfulResultsCount += 1;
let counter = 0;
for (let book of books) {
try {
// Force the provider's series id to be set, so that we know which series this belongs.
book.result.providerSeriesId = series.providerSeriesId;
await this.library.addBook(book.result);
} catch (err) {
this.logger.error({
class: LibraryConsumer.name,
method: this.process.name,
book: book.result,
score: book.score,
msg: 'Failed to add book in background during adding series.',
error: err,
});
} finally {
counter++;
job.updateProgress(25 + 75 * counter / books.length);
}
}
context = context.next();
job.updateProgress(pageSearchedCount * 5);
} while (results.length >= 40 && unhelpfulResultsCount < 3);
} else if (job.name == 'update_series') {
const series: CreateSeriesSubscriptionJobDto = job.data;
const existingBooks = await this.library.getBooksFromSeries(series);
const existingVolumes = existingBooks.map(b => b.volume);
const books = await this.search(job, series, true);
// Sort & de-duplicate the entries received.
const books = related.map(book => this.toScore(book, series))
.sort((a, b) => a.result.volume - b.result.volume || b.score - a.score)
.filter((_, index, arr) => index == 0 || arr[index - 1].result.volume != arr[index].result.volume);
job.updateProgress(25);
let counter = 0;
for (let book of books) {
if (existingVolumes.includes(book.result.volume)) {
continue;
}
this.logger.debug({
class: LibraryConsumer.name,
method: this.process.name,
job: job,
msg: 'Finished searching for book entries.',
results: {
pages: pageSearchedCount,
related_entries: related.length,
volumes: books.length,
}
});
let counter = 0;
for (let book of books) {
try {
// Force the provider's series id to be set, so that we know which series this belongs.
book.result.providerSeriesId = series.providerSeriesId;
await this.library.addBook(book.result);
} catch (err) {
this.logger.error({
class: LibraryConsumer.name,
method: this.process.name,
book: book.result,
score: book.score,
msg: 'Failed to add book in background.',
error: err,
});
} finally {
counter++;
job.updateProgress(25 + 75 * counter / books.length);
try {
// Force the provider's series id to be set, so that we know which series this belongs.
book.result.providerSeriesId = series.providerSeriesId;
await this.library.addBook(book.result);
} catch (err) {
this.logger.error({
class: LibraryConsumer.name,
method: this.process.name,
book: book.result,
score: book.score,
msg: 'Failed to add book in background during series update.',
error: err,
});
} finally {
counter++;
job.updateProgress(25 + 75 * counter / books.length);
}
}
} else {
this.logger.warn({
class: LibraryConsumer.name,
method: this.process.name,
job: job,
msg: 'Unknown job name found.',
});
}
this.logger.info({
@ -101,6 +99,54 @@ export class LibraryConsumer extends WorkerHost {
return null;
}
private async search(job: Job, series: CreateSeriesSubscriptionJobDto, newest: boolean): Promise<{ result: BookSearchResultDto, score: number }[]> {
let context = this.provider.generateSearchContext(series.provider, series.title) as GoogleSearchContext;
context.maxResults = '40';
if (newest) {
context.orderBy = 'newest';
}
// Search for the book(s) via the provider.
// Up until end of results or after 3 unhelpful pages of results.
let results = [];
let related = [];
let pageSearchedCount = 0;
let unhelpfulResultsCount = 0;
do {
pageSearchedCount += 1;
results = await this.provider.search(context);
const potential = results.filter((r: BookSearchResultDto) => r.providerSeriesId == series.providerSeriesId || r.title == series.title && r.mediaType == series.mediaType);
if (potential.length > 0) {
related.push.apply(related, potential);
} else {
unhelpfulResultsCount += 1;
}
context = context.next();
job.updateProgress(pageSearchedCount * 5);
} while (results.length >= context.maxResults && (!newest || unhelpfulResultsCount < 3));
// Sort & de-duplicate the entries received.
const books = related.map(book => this.toScore(book, series))
.sort((a, b) => a.result.volume - b.result.volume || b.score - a.score)
.filter((_, index, arr) => index == 0 || arr[index - 1].result.volume != arr[index].result.volume);
job.updateProgress(25);
this.logger.debug({
class: LibraryConsumer.name,
method: this.search.name,
job: job,
msg: 'Finished searching for book entries.',
results: {
pages: pageSearchedCount,
related_entries: related.length,
volumes: books.length,
}
});
return books;
}
@OnQueueEvent('failed')
onFailed(job: Job, err: Error) {
this.logger.error({