From 969829da20f36e4d60f68c5a2132a9e081a47e90 Mon Sep 17 00:00:00 2001 From: Tom Date: Fri, 28 Feb 2025 00:19:26 +0000 Subject: [PATCH] Create Library module. Moved book controller to library controller. Added series addition to library while adding all known volumes in background. Fixed Google search context. --- backend/database.postgres.sql | 6 +- backend/nestjs-seshat-api/package-lock.json | 288 +++++++++++++++++- backend/nestjs-seshat-api/package.json | 2 + backend/nestjs-seshat-api/src/app.module.ts | 16 +- .../src/auth/auth.controller.ts | 2 +- .../src/books/books.controller.ts | 94 ------ .../src/books/books.module.ts | 14 +- .../src/books/books.service.ts | 112 ++----- ...tatus.dto.ts => create-book-status.dto.ts} | 0 .../src/books/dto/delete-book-status.dto.ts | 13 + .../src/library/library.consumer.ts | 93 ++++++ .../src/library/library.controller.spec.ts | 18 ++ .../src/library/library.controller.ts | 258 ++++++++++++++++ .../src/library/library.module.ts | 37 +++ .../src/library/library.service.spec.ts | 18 ++ .../src/library/library.service.ts | 138 +++++++++ .../contexts/google.search.context.ts | 94 +++++- .../src/providers/contexts/search.context.ts | 2 +- .../src/providers/google/google.service.ts | 34 ++- .../src/providers/providers.service.ts | 6 +- .../dto/create-series-subscription-job.dto.ts | 8 + .../dto/create-series-subscription.dto.ts | 8 + .../src/series/dto/create-series.dto.ts | 11 +- .../src/series/dto/series-subscription.dto.ts | 9 + .../{delete-series.dto.ts => series.dto.ts} | 2 +- .../src/series/dto/update-series.dto.ts | 17 +- .../entities/series-subscription.entity.ts | 17 ++ .../src/series/series.module.ts | 2 + .../src/series/series.service.ts | 41 ++- 29 files changed, 1121 insertions(+), 239 deletions(-) delete mode 100644 backend/nestjs-seshat-api/src/books/books.controller.ts rename backend/nestjs-seshat-api/src/books/dto/{book-status.dto.ts => create-book-status.dto.ts} (100%) create mode 100644 backend/nestjs-seshat-api/src/books/dto/delete-book-status.dto.ts create mode 100644 backend/nestjs-seshat-api/src/library/library.consumer.ts create mode 100644 backend/nestjs-seshat-api/src/library/library.controller.spec.ts create mode 100644 backend/nestjs-seshat-api/src/library/library.controller.ts create mode 100644 backend/nestjs-seshat-api/src/library/library.module.ts create mode 100644 backend/nestjs-seshat-api/src/library/library.service.spec.ts create mode 100644 backend/nestjs-seshat-api/src/library/library.service.ts create mode 100644 backend/nestjs-seshat-api/src/series/dto/create-series-subscription-job.dto.ts create mode 100644 backend/nestjs-seshat-api/src/series/dto/create-series-subscription.dto.ts create mode 100644 backend/nestjs-seshat-api/src/series/dto/series-subscription.dto.ts rename backend/nestjs-seshat-api/src/series/dto/{delete-series.dto.ts => series.dto.ts} (84%) create mode 100644 backend/nestjs-seshat-api/src/series/entities/series-subscription.entity.ts diff --git a/backend/database.postgres.sql b/backend/database.postgres.sql index dc34756..22f6e6b 100644 --- a/backend/database.postgres.sql +++ b/backend/database.postgres.sql @@ -11,6 +11,8 @@ DROP TABLE IF EXISTS book_origins; DROP TABLE IF EXISTS refresh_tokens; +DROP TABLE IF EXISTS series_subscriptions; + DROP TABLE IF EXISTS users; DROP TABLE IF EXISTS books; @@ -53,7 +55,7 @@ CREATE TABLE PRIMARY KEY (book_id), -- FOREIGN KEY (series_id) REFERENCES series (series_id), FOREIGN KEY (provider, provider_series_id) REFERENCES series (provider, provider_series_id) ON DELETE CASCADE, - UNIQUE NULLS NOT DISTINCT (provider_series_id, provider_book_id) + UNIQUE NULLS NOT DISTINCT (provider_series_id, provider_book_id, book_volume) ); ALTER TABLE books @@ -61,7 +63,7 @@ ALTER COLUMN added_at SET DEFAULT now (); -- CREATE INDEX books_series_id_idx ON books (series_id); -CREATE INDEX books_provider_series_id_idx ON books (provider_series_id); +CREATE INDEX books_provider_provider_series_id_idx ON books (provider, provider_series_id); -- CREATE INDEX books_isbn_idx ON books USING HASH (isbn); CREATE INDEX books_book_title_idx ON books (book_title); diff --git a/backend/nestjs-seshat-api/package-lock.json b/backend/nestjs-seshat-api/package-lock.json index fd0c50e..1d11fd2 100644 --- a/backend/nestjs-seshat-api/package-lock.json +++ b/backend/nestjs-seshat-api/package-lock.json @@ -10,6 +10,7 @@ "license": "UNLICENSED", "dependencies": { "@nestjs/axios": "^4.0.0", + "@nestjs/bullmq": "^11.0.2", "@nestjs/common": "^10.0.0", "@nestjs/config": "^4.0.0", "@nestjs/core": "^10.0.0", @@ -19,6 +20,7 @@ "@nestjs/typeorm": "^11.0.0", "argon2": "^0.41.1", "axios": "^1.7.9", + "bullmq": "^5.41.7", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", "cookie-parser": "^1.4.7", @@ -959,6 +961,12 @@ "dev": true, "license": "BSD-3-Clause" }, + "node_modules/@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==", + "license": "MIT" + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -1596,6 +1604,84 @@ "node": ">=8" } }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz", + "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz", + "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz", + "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz", + "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz", + "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz", + "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@nestjs/axios": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@nestjs/axios/-/axios-4.0.0.tgz", @@ -1607,6 +1693,34 @@ "rxjs": "^7.0.0" } }, + "node_modules/@nestjs/bull-shared": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-11.0.2.tgz", + "integrity": "sha512-dFlttJvBqIFD6M8JVFbkrR4Feb39OTAJPJpFVILU50NOJCM4qziRw3dSNG84Q3v+7/M6xUGMFdZRRGvBBKxoSA==", + "license": "MIT", + "dependencies": { + "tslib": "2.8.1" + }, + "peerDependencies": { + "@nestjs/common": "^10.0.0 || ^11.0.0", + "@nestjs/core": "^10.0.0 || ^11.0.0" + } + }, + "node_modules/@nestjs/bullmq": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/@nestjs/bullmq/-/bullmq-11.0.2.tgz", + "integrity": "sha512-Lq6lGpKkETsm0RDcUktlzsthFoE3A5QTMp2FwPi1eztKqKD6/90KS1TcnC9CJFzjpUaYnQzIMrlNs55e+/wsHA==", + "license": "MIT", + "dependencies": { + "@nestjs/bull-shared": "^11.0.2", + "tslib": "2.8.1" + }, + "peerDependencies": { + "@nestjs/common": "^10.0.0 || ^11.0.0", + "@nestjs/core": "^10.0.0 || ^11.0.0", + "bullmq": "^3.0.0 || ^4.0.0 || ^5.0.0" + } + }, "node_modules/@nestjs/cli": { "version": "10.4.9", "resolved": "https://registry.npmjs.org/@nestjs/cli/-/cli-10.4.9.tgz", @@ -3456,6 +3570,21 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "license": "MIT" }, + "node_modules/bullmq": { + "version": "5.41.7", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.41.7.tgz", + "integrity": "sha512-eZbKJSx15bflfzKRiR+dKeLTr/M/YKb4cIp73OdU79PEMHQ6aEFUtbG6R+f0KvLLznI/O01G581U2Eqli6S2ew==", + "license": "MIT", + "dependencies": { + "cron-parser": "^4.9.0", + "ioredis": "^5.4.1", + "msgpackr": "^1.11.2", + "node-abort-controller": "^3.1.1", + "semver": "^7.5.4", + "tslib": "^2.0.0", + "uuid": "^9.0.0" + } + }, "node_modules/busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", @@ -3842,6 +3971,15 @@ "node": ">=0.8" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -4093,6 +4231,18 @@ "devOptional": true, "license": "MIT" }, + "node_modules/cron-parser": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", + "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", + "license": "MIT", + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -4202,6 +4352,15 @@ "node": ">=0.4.0" } }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -4221,6 +4380,16 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/detect-libc": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", + "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=8" + } + }, "node_modules/detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -5764,6 +5933,30 @@ "node": ">=12.0.0" } }, + "node_modules/ioredis": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.5.0.tgz", + "integrity": "sha512-7CutT89g23FfSa8MDoIFs2GYYa0PaNiW/OrT+nRyjRXHDZd17HmIgy+reOQ/yhh72NznNjGuS8kbCAcA4Ro4mw==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -7002,12 +7195,24 @@ "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "license": "MIT" }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "license": "MIT" + }, "node_modules/lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==", "license": "MIT" }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", + "license": "MIT" + }, "node_modules/lodash.isboolean": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", @@ -7085,6 +7290,15 @@ "yallist": "^3.0.2" } }, + "node_modules/luxon": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz", + "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/magic-string": { "version": "0.30.8", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.8.tgz", @@ -7327,6 +7541,37 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/msgpackr": { + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.2.tgz", + "integrity": "sha512-F9UngXRlPyWCDEASDpTf6c9uNhGPTqnTeLVt7bN+bU1eajoR/8V9ys2BRaV5C/e5ihE6sJ9uPIKaYt6bFuO32g==", + "license": "MIT", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz", + "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.2.2" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" + } + }, "node_modules/multer": { "version": "1.4.4-lts.1", "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.4-lts.1.tgz", @@ -7406,7 +7651,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==", - "dev": true, "license": "MIT" }, "node_modules/node-addon-api": { @@ -7459,6 +7703,21 @@ "node-gyp-build-test": "build-test.js" } }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", + "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "license": "MIT", + "optional": true, + "dependencies": { + "detect-libc": "^2.0.1" + }, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -8426,6 +8685,27 @@ "node": ">= 12.13.0" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/reflect-metadata": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", @@ -9074,6 +9354,12 @@ "node": ">=8" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", diff --git a/backend/nestjs-seshat-api/package.json b/backend/nestjs-seshat-api/package.json index 07e6f92..90520fc 100644 --- a/backend/nestjs-seshat-api/package.json +++ b/backend/nestjs-seshat-api/package.json @@ -21,6 +21,7 @@ }, "dependencies": { "@nestjs/axios": "^4.0.0", + "@nestjs/bullmq": "^11.0.2", "@nestjs/common": "^10.0.0", "@nestjs/config": "^4.0.0", "@nestjs/core": "^10.0.0", @@ -30,6 +31,7 @@ "@nestjs/typeorm": "^11.0.0", "argon2": "^0.41.1", "axios": "^1.7.9", + "bullmq": "^5.41.7", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", "cookie-parser": "^1.4.7", diff --git a/backend/nestjs-seshat-api/src/app.module.ts b/backend/nestjs-seshat-api/src/app.module.ts index 086df15..c109431 100644 --- a/backend/nestjs-seshat-api/src/app.module.ts +++ b/backend/nestjs-seshat-api/src/app.module.ts @@ -15,6 +15,8 @@ import { serialize_token, serialize_user_short, serialize_user_long, serialize_r import { BooksModule } from './books/books.module'; import { ProvidersModule } from './providers/providers.module'; import { SeriesModule } from './series/series.module'; +import { LibraryModule } from './library/library.module'; +import { BullModule } from '@nestjs/bullmq'; @Module({ imports: [ @@ -23,6 +25,17 @@ import { SeriesModule } from './series/series.module'; imports: [ConfigModule], useClass: DatabaseOptions }), + BullModule.forRootAsync({ + imports: [ConfigModule], + inject: [ConfigService], + useFactory: (config: ConfigService) => ({ + connection: { + host: config.get('REDIS_HOST') ?? 'localhost', + port: config.get('REDIS_PORT') ?? 6379, + password: config.get('REDIS_PASSWORD'), + } + }) + }), TypeOrmModule.forFeature([UserEntity]), UsersModule, AuthModule, @@ -58,7 +71,8 @@ import { SeriesModule } from './series/series.module'; }), BooksModule, ProvidersModule, - SeriesModule + SeriesModule, + LibraryModule ], controllers: [AppController], providers: [AppService, UsersService], diff --git a/backend/nestjs-seshat-api/src/auth/auth.controller.ts b/backend/nestjs-seshat-api/src/auth/auth.controller.ts index d08ed6e..c7b9410 100644 --- a/backend/nestjs-seshat-api/src/auth/auth.controller.ts +++ b/backend/nestjs-seshat-api/src/auth/auth.controller.ts @@ -197,7 +197,7 @@ export class AuthController { msg: 'Failed to register due to duplicate userLogin.', }); - response.statusCode = 400; + response.statusCode = 409; return { success: false, error_message: 'Username already exist.', diff --git a/backend/nestjs-seshat-api/src/books/books.controller.ts b/backend/nestjs-seshat-api/src/books/books.controller.ts deleted file mode 100644 index 82daae3..0000000 --- a/backend/nestjs-seshat-api/src/books/books.controller.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { Body, Controller, Get, Post, Put, Request, Res, UseGuards } from '@nestjs/common'; -import { Response } from 'express'; -import { JwtAccessGuard } from 'src/auth/guards/jwt-access.guard'; -import { BooksService } from './books.service'; -import { BookSearchResultDto } from 'src/providers/dto/book-search-result.dto'; -import { UpdateBookOriginDto } from './dto/update-book-origin.dto'; -import { UpdateBookDto } from './dto/update-book.dto'; -import { PinoLogger } from 'nestjs-pino'; -import { QueryFailedError } from 'typeorm'; - -@UseGuards(JwtAccessGuard) -@Controller('books') -export class BooksController { - constructor( - private books: BooksService, - private logger: PinoLogger, - ) { } - - - @Post('') - async CreateBook( - @Request() req, - @Body() body: BookSearchResultDto, - @Res({ passthrough: true }) response: Response, - ) { - try { - return { - success: true, - data: await this.books.createBook(body), - }; - } catch (err) { - if (err instanceof QueryFailedError) { - if (err.driverError.code == '23505') { - // Book exists already. - response.statusCode = 400; - return { - success: false, - error_message: 'The book has already been added previously.', - } - } - } - - this.logger.error({ - class: BooksController.name, - method: this.CreateBook.name, - user: req.user, - msg: 'Failed to create book.', - error: err, - }); - - response.statusCode = 500; - return { - success: false, - error_message: 'Something went wrong while adding the book.', - } - } - } - - @Get('') - async GetBooksFromUser( - @Request() req, - ) { - return { - success: true, - data: await this.books.findBookStatusesTrackedBy(req.user.userId), - }; - } - - @Put('') - async UpdateBook( - @Body() body: UpdateBookDto, - ) { - const data = { ...body }; - delete data['bookId']; - - const result = await this.books.updateBook(body.bookId, data); - return { - success: result?.affected == 1, - }; - } - - @Put('origins') - async UpdateBookOrigin( - @Body() body: UpdateBookOriginDto, - ) { - const data = { ...body }; - delete data['bookOriginId']; - - const result = await this.books.updateBookOrigin(body.bookOriginId, data); - return { - success: result?.affected == 1, - }; - } -} diff --git a/backend/nestjs-seshat-api/src/books/books.module.ts b/backend/nestjs-seshat-api/src/books/books.module.ts index 42584d8..ca02110 100644 --- a/backend/nestjs-seshat-api/src/books/books.module.ts +++ b/backend/nestjs-seshat-api/src/books/books.module.ts @@ -1,5 +1,4 @@ import { Module } from '@nestjs/common'; -import { BooksController } from './books.controller'; import { BooksService } from './books.service'; import { BookEntity } from './entities/book.entity'; import { BookOriginEntity } from './entities/book-origin.entity'; @@ -8,6 +7,9 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import { HttpModule } from '@nestjs/axios'; import { ProvidersModule } from 'src/providers/providers.module'; import { SeriesModule } from 'src/series/series.module'; +import { LibraryService } from 'src/library/library.service'; +import { LibraryModule } from 'src/library/library.module'; +import { BullModule } from '@nestjs/bullmq'; @Module({ imports: [ @@ -19,11 +21,15 @@ import { SeriesModule } from 'src/series/series.module'; SeriesModule, HttpModule, ProvidersModule, + LibraryModule, + BullModule.registerQueue({ + name: 'library', + }), ], - controllers: [BooksController], + controllers: [], exports: [ BooksService ], - providers: [BooksService] + providers: [BooksService, LibraryService] }) -export class BooksModule {} +export class BooksModule { } diff --git a/backend/nestjs-seshat-api/src/books/books.service.ts b/backend/nestjs-seshat-api/src/books/books.service.ts index 319f00f..fa66bfb 100644 --- a/backend/nestjs-seshat-api/src/books/books.service.ts +++ b/backend/nestjs-seshat-api/src/books/books.service.ts @@ -5,13 +5,10 @@ import { In, InsertResult, Repository } from 'typeorm'; import { BookOriginEntity } from './entities/book-origin.entity'; import { BookStatusEntity } from './entities/book-status.entity'; import { UUID } from 'crypto'; -import { PinoLogger } from 'nestjs-pino'; -import { BookSearchResultDto } from 'src/providers/dto/book-search-result.dto'; -import { BookOriginType } from 'src/shared/enums/book_origin_type'; import { CreateBookDto } from './dto/create-book.dto'; import { CreateBookOriginDto } from './dto/create-book-origin.dto'; -import { CreateBookStatusDto as BookStatusDto } from './dto/book-status.dto'; -import { SeriesService } from 'src/series/series.service'; +import { CreateBookStatusDto } from './dto/create-book-status.dto'; +import { DeleteBookStatusDto } from './dto/delete-book-status.dto'; @Injectable() export class BooksService { @@ -22,82 +19,10 @@ export class BooksService { private bookOriginRepository: Repository, @InjectRepository(BookStatusEntity) private bookStatusRepository: Repository, - private series: SeriesService, - private logger: PinoLogger, ) { } - async createBook(book: BookSearchResultDto) { - this.logger.debug({ - class: BooksService.name, - method: this.createBook.name, - book: book, - msg: 'Saving book to database...', - }); - - if (book.providerSeriesId) { - await this.series.updateSeries({ - providerSeriesId: book.providerSeriesId, - title: book.title, - provider: book.provider, - }); - - this.logger.debug({ - class: BooksService.name, - method: this.createBook.name, - series: book.providerSeriesId, - msg: 'Series saved to database.', - }); - } - - const createBook: CreateBookDto = { - title: book.title, - desc: book.desc, - providerSeriesId: book.providerSeriesId, - providerBookId: book.providerBookId, - volume: book.volume, - provider: book.provider, - publishedAt: book.publishedAt, - }; - const data = await this.createBookInternal(createBook); - const bookId = data.identifiers[0]['bookId']; - - const tasks = []; - - tasks.push(book.authors.map(author => this.addBookOrigin(bookId, BookOriginType.AUTHOR, author))); - tasks.push(book.categories.map(category => this.addBookOrigin(bookId, BookOriginType.CATEGORY, category))); - tasks.push(this.addBookOrigin(bookId, BookOriginType.LANGUAGE, book.language)); - if (book.maturityRating) { - tasks.push(this.addBookOrigin(bookId, BookOriginType.MATURITY_RATING, book.maturityRating)); - } - if (book.thumbnail) { - tasks.push(this.addBookOrigin(bookId, BookOriginType.PROVIDER_THUMBNAIL, book.thumbnail)); - } - if (book.url) { - tasks.push(this.addBookOrigin(bookId, BookOriginType.PROVIDER_URL, book.url)); - } - - if ('ISBN_10' in book.industryIdentifiers) { - tasks.push(this.addBookOrigin(bookId, BookOriginType.ISBN_10, book.industryIdentifiers['ISBN_10'])); - } - - if ('ISBN_13' in book.industryIdentifiers) { - tasks.push(this.addBookOrigin(bookId, BookOriginType.ISBN_10, book.industryIdentifiers['ISBN_13'])); - } - - await Promise.all(tasks); - - this.logger.info({ - class: BooksService.name, - method: this.createBook.name, - book: book, - msg: 'Book saved to database.', - }); - - return bookId; - } - - private async createBookInternal(book: CreateBookDto): Promise { + async createBook(book: CreateBookDto): Promise { const entity = this.bookRepository.create(book); return await this.bookRepository.createQueryBuilder() .insert() @@ -107,12 +32,26 @@ export class BooksService { .execute(); } - async addBookOrigin(bookId: UUID, type: BookOriginType, value: string): Promise { - return await this.bookOriginRepository.insert({ - bookId, - type, - value, - }); + async addBookOrigin(origin: CreateBookOriginDto): Promise { + return await this.bookOriginRepository.insert(origin); + } + + async deleteBookOrigin(origin: CreateBookOriginDto) { + return await this.bookOriginRepository.createQueryBuilder() + .delete() + .where({ + whereFactory: origin, + }) + .execute(); + } + + async deleteBookStatus(status: DeleteBookStatusDto) { + return await this.bookStatusRepository.createQueryBuilder() + .delete() + .where({ + whereFactory: status, + }) + .execute(); } async findBooksByIds(bookIds: UUID[]) { @@ -120,7 +59,7 @@ export class BooksService { where: { bookId: In(bookIds) } - }) + }); } async findBookStatusesTrackedBy(userId: UUID): Promise { @@ -157,13 +96,12 @@ export class BooksService { }, update); } - async updateBookStatus(status: BookStatusDto) { + async updateBookStatus(status: CreateBookStatusDto) { status.modifiedAt = new Date(); await this.bookStatusRepository.createQueryBuilder() .insert() .values(status) .orUpdate(['user_id', 'book_id'], ['state', 'modified_at'], { skipUpdateIfNoValuesChanged: true }) .execute(); - return await this.bookStatusRepository.upsert(status, ['book_id']); } } diff --git a/backend/nestjs-seshat-api/src/books/dto/book-status.dto.ts b/backend/nestjs-seshat-api/src/books/dto/create-book-status.dto.ts similarity index 100% rename from backend/nestjs-seshat-api/src/books/dto/book-status.dto.ts rename to backend/nestjs-seshat-api/src/books/dto/create-book-status.dto.ts diff --git a/backend/nestjs-seshat-api/src/books/dto/delete-book-status.dto.ts b/backend/nestjs-seshat-api/src/books/dto/delete-book-status.dto.ts new file mode 100644 index 0000000..36f596f --- /dev/null +++ b/backend/nestjs-seshat-api/src/books/dto/delete-book-status.dto.ts @@ -0,0 +1,13 @@ +import { IsNotEmpty, IsOptional, IsUUID } from 'class-validator'; +import { UUID } from 'crypto'; + +export class DeleteBookStatusDto { + @IsUUID() + @IsNotEmpty() + readonly bookId: UUID; + + @IsUUID() + @IsNotEmpty() + @IsOptional() + readonly userId: UUID; +} \ No newline at end of file diff --git a/backend/nestjs-seshat-api/src/library/library.consumer.ts b/backend/nestjs-seshat-api/src/library/library.consumer.ts new file mode 100644 index 0000000..f0cf956 --- /dev/null +++ b/backend/nestjs-seshat-api/src/library/library.consumer.ts @@ -0,0 +1,93 @@ + +import { Processor, WorkerHost } from '@nestjs/bullmq'; +import { Job } from 'bullmq'; +import { PinoLogger } from 'nestjs-pino'; +import { GoogleSearchContext } from 'src/providers/contexts/google.search.context'; +import { BookSearchResultDto } from 'src/providers/dto/book-search-result.dto'; +import { ProvidersService } from 'src/providers/providers.service'; +import { CreateSeriesSubscriptionJobDto } from 'src/series/dto/create-series-subscription-job.dto'; +import { LibraryService } from './library.service'; + +@Processor('library') +export class LibraryConsumer extends WorkerHost { + constructor( + private readonly library: LibraryService, + private readonly provider: ProvidersService, + private readonly logger: PinoLogger, + ) { + super(); + } + + async process(job: Job, token?: string): Promise { + console.log('job started:', job.name, job.data, job.id); + const series: CreateSeriesSubscriptionJobDto = job.data; + + let context = this.provider.generateSearchContext(series.provider, series.title) as GoogleSearchContext; + //context.intitle = series.title; + context.maxResults = '40'; + context.subject = 'Fiction'; + + // Search for the book(s) via the provider. + // Up until end of results or after 3 unhelpful pages of results. + let results = []; + let related = []; + let pageSearchedCount = 0; + let unhelpfulResultsCount = 0; + do { + pageSearchedCount += 1; + results = await this.provider.search(context); + const potential = results.filter(r => r.providerSeriesId == series.providerSeriesId || r.title == series.title); + if (potential.length > 0) { + related.push.apply(related, potential); + } else { + unhelpfulResultsCount += 1; + } + context = context.next(); + job.updateProgress(pageSearchedCount * 5); + } while (results.length >= 40 && unhelpfulResultsCount < 3); + + // Sort & de-duplicate the entries received. + const books = related.map(book => this.toScore(book, series)) + .sort((a, b) => a.result.volume - b.result.volume || b.score - a.score) + .filter((_, index, arr) => index == 0 || arr[index - 1].result.volume != arr[index].result.volume); + job.updateProgress(25); + + let counter = 0; + for (let book of books) { + try { + book.result.providerSeriesId = series.providerSeriesId; + await this.library.addBook(book.result); + } catch (err) { + this.logger.error({ + class: LibraryConsumer.name, + method: this.process.name, + book: book.result, + score: book.score, + msg: 'Failed to add book in background.', + error: err, + }); + } finally { + counter++; + job.updateProgress(25 + 75 * counter / books.length); + } + } + + console.log('job completed:', job.name, job.data, job.id); + + return null; + } + + private toScore(book: BookSearchResultDto, series: CreateSeriesSubscriptionJobDto): ({ result: BookSearchResultDto, score: number }) { + if (!book) { + return { + result: null, + score: -1, + } + } + + return { + result: book, + score: (!!book.providerSeriesId ? 50 : 0) + (book.title == series.title ? 25 : 0) + (book.url.startsWith('https://play.google.com/store/books/details?') ? 10 : 0), + } + } +} diff --git a/backend/nestjs-seshat-api/src/library/library.controller.spec.ts b/backend/nestjs-seshat-api/src/library/library.controller.spec.ts new file mode 100644 index 0000000..b1c5608 --- /dev/null +++ b/backend/nestjs-seshat-api/src/library/library.controller.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { LibraryController } from './library.controller'; + +describe('LibraryController', () => { + let controller: LibraryController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [LibraryController], + }).compile(); + + controller = module.get(LibraryController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/backend/nestjs-seshat-api/src/library/library.controller.ts b/backend/nestjs-seshat-api/src/library/library.controller.ts new file mode 100644 index 0000000..acd1475 --- /dev/null +++ b/backend/nestjs-seshat-api/src/library/library.controller.ts @@ -0,0 +1,258 @@ +import { InjectQueue } from '@nestjs/bullmq'; +import { Body, Controller, Delete, Get, Post, Put, Request, Res, UseGuards } from '@nestjs/common'; +import { Response } from 'express'; +import { Queue } from 'bullmq'; +import { PinoLogger } from 'nestjs-pino'; +import { BooksService } from 'src/books/books.service'; +import { BookSearchResultDto } from 'src/providers/dto/book-search-result.dto'; +import { SeriesService } from 'src/series/series.service'; +import { QueryFailedError } from 'typeorm'; +import { UpdateBookDto } from 'src/books/dto/update-book.dto'; +import { UpdateBookOriginDto } from 'src/books/dto/update-book-origin.dto'; +import { LibraryService } from './library.service'; +import { CreateSeriesSubscriptionDto } from 'src/series/dto/create-series-subscription.dto'; +import { JwtAccessGuard } from 'src/auth/guards/jwt-access.guard'; +import { SeriesDto } from 'src/series/dto/series.dto'; +import { CreateBookOriginDto } from 'src/books/dto/create-book-origin.dto'; +import { DeleteBookStatusDto } from 'src/books/dto/delete-book-status.dto'; + +@UseGuards(JwtAccessGuard) +@Controller('library') +export class LibraryController { + constructor( + private readonly books: BooksService, + private readonly series: SeriesService, + private readonly library: LibraryService, + @InjectQueue('library') private readonly jobs: Queue, + private readonly logger: PinoLogger, + ) { } + + @Get('series') + async getSeries( + @Request() req, + @Res({ passthrough: true }) response: Response, + ) { + return { + success: true, + data: await this.series.getAllSeries(), + }; + } + + @Post('series') + async createSeries( + @Request() req, + @Body() body: CreateSeriesSubscriptionDto, + @Res({ passthrough: true }) response: Response, + ) { + try { + await this.library.addSeries({ + provider: body.provider, + providerSeriesId: body.providerSeriesId, + title: body.title, + }); + + return { + success: true, + }; + } catch (err) { + if (err instanceof QueryFailedError) { + if (err.driverError.code == '23505') { + // Subscription already exist. + response.statusCode = 409; + return { + success: false, + error_message: 'Series subscription already exists.', + }; + } + } + + response.statusCode = 500; + return { + success: false, + error_message: 'Something went wrong.', + }; + } + } + + @Get('series/subscriptions') + async getSeriesSubscriptions( + @Request() req, + @Res({ passthrough: true }) response: Response, + ) { + return { + success: true, + data: await this.series.getSeriesSubscribedBy(req.user.userId), + }; + } + + @Post('series/subscribe') + async subscribe( + @Request() req, + @Body() body: SeriesDto, + @Res({ passthrough: true }) response: Response, + ) { + try { + await this.library.addSubscription({ + ...body, + userId: req.user.userId, + }); + + return { + success: true, + }; + } catch (err) { + if (err instanceof QueryFailedError) { + if (err.driverError.code == '23505') { + // Subscription already exists. + response.statusCode = 409; + return { + success: false, + error_message: 'Series subscription already exists.', + }; + } else if (err.driverError.code == '23503') { + // Series does not exist. + response.statusCode = 400; + return { + success: false, + error_message: 'Series does not exist.', + }; + } + } + + response.statusCode = 500; + return { + success: false, + error_message: 'Something went wrong.', + }; + } + } + + @Get('books') + async getBooksFromUser( + @Request() req, + ) { + return { + success: true, + data: await this.books.findBookStatusesTrackedBy(req.user.userId), + }; + } + + @Post('books') + async createBook( + @Request() req, + @Body() body: BookSearchResultDto, + @Res({ passthrough: true }) response: Response, + ) { + if (body.provider && body.providerSeriesId) { + try { + await this.series.updateSeries({ + provider: body.provider, + providerSeriesId: body.providerSeriesId, + title: body.title, + }); + } catch (err) { + if (err instanceof QueryFailedError) { + // Ignore if the series already exist. + if (err.driverError.code != '23505') { + response.statusCode = 500; + return { + success: false, + error_message: 'Something went wrong.', + }; + } + } + } + } + + try { + return { + success: true, + data: await this.books.createBook(body), + }; + } catch (err) { + if (err instanceof QueryFailedError) { + if (err.driverError.code == '23505') { + // Book exists already. + response.statusCode = 409; + return { + success: false, + error_message: 'The book has already been added previously.', + }; + } else if (err.driverError.code == '23503') { + // Data dependency is missing. + response.statusCode = 500; + return { + success: false, + error_message: 'Series has not been added.', + }; + } + } + + this.logger.error({ + class: LibraryController.name, + method: this.createBook.name, + user: req.user, + msg: 'Failed to create book.', + error: err, + }); + + response.statusCode = 500; + return { + success: false, + error_message: 'Something went wrong while adding the book.', + }; + } + } + + @Put('books') + async updateBook( + @Body() body: UpdateBookDto, + ) { + const data = { ...body }; + delete data['bookId']; + + const result = await this.books.updateBook(body.bookId, data); + return { + success: result?.affected == 1, + }; + } + + @Delete('books/origins') + async deleteBookOrigin( + @Body() body: CreateBookOriginDto, + ) { + const data = { ...body }; + delete data['bookOriginId']; + + const result = await this.books.deleteBookOrigin(body); + return { + success: result?.affected == 1, + }; + } + + @Delete('books/status') + async deleteBookStatus( + @Body() body: DeleteBookStatusDto, + ) { + const data = { ...body }; + delete data['bookOriginId']; + + const result = await this.books.deleteBookStatus(body); + return { + success: result?.affected == 1, + }; + } + + @Put('books/origins') + async updateBookOrigin( + @Body() body: UpdateBookOriginDto, + ) { + const data = { ...body }; + delete data['bookOriginId']; + + const result = await this.books.updateBookOrigin(body.bookOriginId, data); + return { + success: result?.affected == 1, + }; + } +} diff --git a/backend/nestjs-seshat-api/src/library/library.module.ts b/backend/nestjs-seshat-api/src/library/library.module.ts new file mode 100644 index 0000000..693c8eb --- /dev/null +++ b/backend/nestjs-seshat-api/src/library/library.module.ts @@ -0,0 +1,37 @@ +import { Module } from '@nestjs/common'; +import { LibraryService } from './library.service'; +import { HttpModule } from '@nestjs/axios'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { BookOriginEntity } from 'src/books/entities/book-origin.entity'; +import { BookStatusEntity } from 'src/books/entities/book-status.entity'; +import { BookEntity } from 'src/books/entities/book.entity'; +import { ProvidersModule } from 'src/providers/providers.module'; +import { SeriesModule } from 'src/series/series.module'; +import { SeriesEntity } from 'src/series/entities/series.entity'; +import { SeriesSubscriptionEntity } from 'src/series/entities/series-subscription.entity'; +import { BooksService } from 'src/books/books.service'; +import { SeriesService } from 'src/series/series.service'; +import { BullModule } from '@nestjs/bullmq'; +import { LibraryConsumer } from './library.consumer'; +import { LibraryController } from './library.controller'; + +@Module({ + imports: [ + TypeOrmModule.forFeature([ + BookEntity, + BookOriginEntity, + BookStatusEntity, + SeriesEntity, + SeriesSubscriptionEntity, + ]), + BullModule.registerQueue({ + name: 'library', + }), + SeriesModule, + HttpModule, + ProvidersModule, + ], + providers: [LibraryService, BooksService, SeriesService, LibraryConsumer], + controllers: [LibraryController] +}) +export class LibraryModule { } diff --git a/backend/nestjs-seshat-api/src/library/library.service.spec.ts b/backend/nestjs-seshat-api/src/library/library.service.spec.ts new file mode 100644 index 0000000..7a11510 --- /dev/null +++ b/backend/nestjs-seshat-api/src/library/library.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { LibraryService } from './library.service'; + +describe('LibraryService', () => { + let service: LibraryService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [LibraryService], + }).compile(); + + service = module.get(LibraryService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/backend/nestjs-seshat-api/src/library/library.service.ts b/backend/nestjs-seshat-api/src/library/library.service.ts new file mode 100644 index 0000000..a8b99ce --- /dev/null +++ b/backend/nestjs-seshat-api/src/library/library.service.ts @@ -0,0 +1,138 @@ +import { InjectQueue } from '@nestjs/bullmq'; +import { Injectable } from '@nestjs/common'; +import { Queue } from 'bullmq'; +import { PinoLogger } from 'nestjs-pino'; +import { BooksService } from 'src/books/books.service'; +import { CreateBookDto } from 'src/books/dto/create-book.dto'; +import { BookSearchResultDto } from 'src/providers/dto/book-search-result.dto'; +import { CreateSeriesDto } from 'src/series/dto/create-series.dto'; +import { SeriesSubscriptionDto } from 'src/series/dto/series-subscription.dto'; +import { SeriesService } from 'src/series/series.service'; +import { BookOriginType } from 'src/shared/enums/book_origin_type'; + +@Injectable() +export class LibraryService { + constructor( + private readonly books: BooksService, + private readonly series: SeriesService, + @InjectQueue('library') private readonly jobs: Queue, + private readonly logger: PinoLogger, + ) { } + + + async addSeries(series: CreateSeriesDto) { + const result = await this.series.addSeries(series); + + this.logger.debug({ + class: LibraryService.name, + method: this.addSubscription.name, + series: series.providerSeriesId, + msg: 'Series saved to database.', + }); + + this.jobs.add('new_series', series); + return { + success: true, + }; + } + + async addSubscription(series: SeriesSubscriptionDto) { + return await this.series.addSeriesSubscription({ + userId: series.userId, + providerSeriesId: series.providerSeriesId, + provider: series.provider, + }); + } + + async addBook(book: BookSearchResultDto) { + this.logger.debug({ + class: LibraryService.name, + method: this.addBook.name, + book: book, + msg: 'Saving book to database...', + }); + + const bookData = await this.books.createBook({ + title: book.title, + desc: book.desc, + providerSeriesId: book.providerSeriesId, + providerBookId: book.providerBookId, + volume: book.volume, + provider: book.provider, + publishedAt: book.publishedAt, + }); + const bookId = bookData.identifiers[0]['bookId']; + + const tasks = []; + + if (book.authors && book.authors.length > 0) { + tasks.push(book.authors.map(author => this.books.addBookOrigin({ + bookId, + type: BookOriginType.AUTHOR, + value: author, + }))); + } + if (book.categories && book.categories.length > 0) { + tasks.push(book.categories.map(category => this.books.addBookOrigin({ + bookId, + type: BookOriginType.CATEGORY, + value: category + }))); + } + if (book.language) { + tasks.push(this.books.addBookOrigin({ + bookId, + type: BookOriginType.LANGUAGE, + value: book.language, + })); + } + if (book.maturityRating) { + tasks.push(this.books.addBookOrigin({ + bookId, + type: BookOriginType.MATURITY_RATING, + value: book.maturityRating, + })); + } + if (book.thumbnail) { + tasks.push(this.books.addBookOrigin({ + bookId, + type: BookOriginType.PROVIDER_THUMBNAIL, + value: book.thumbnail, + })); + } + if (book.url) { + tasks.push(this.books.addBookOrigin({ + bookId, + type: BookOriginType.PROVIDER_URL, + value: book.url, + })); + } + + if ('ISBN_10' in book.industryIdentifiers) { + tasks.push(this.books.addBookOrigin({ + bookId, + type: BookOriginType.ISBN_10, + value: book.industryIdentifiers['ISBN_10'], + })); + } + + if ('ISBN_13' in book.industryIdentifiers) { + tasks.push(this.books.addBookOrigin({ + bookId, + type: BookOriginType.ISBN_10, + value: book.industryIdentifiers['ISBN_13'], + })); + } + + await Promise.all(tasks); + + this.logger.info({ + class: LibraryService.name, + method: this.addBook.name, + book: book, + msg: 'Book saved to database.', + }); + + return bookId; + } +} diff --git a/backend/nestjs-seshat-api/src/providers/contexts/google.search.context.ts b/backend/nestjs-seshat-api/src/providers/contexts/google.search.context.ts index 1dd107d..545a6af 100644 --- a/backend/nestjs-seshat-api/src/providers/contexts/google.search.context.ts +++ b/backend/nestjs-seshat-api/src/providers/contexts/google.search.context.ts @@ -1,4 +1,6 @@ -class GoogleSearchContext extends SearchContext { +import { SearchContext } from "./search.context"; + +export class GoogleSearchContext extends SearchContext { constructor(searchQuery: string, params: { [key: string]: string }) { super('google', searchQuery, params); } @@ -19,12 +21,96 @@ class GoogleSearchContext extends SearchContext { ...searchParams.map(p => this.params[p] ? p + ':"' + this.params[p] + '"' : ''), ].filter(p => p.length > 0).join(''); - return queryParams + '&' + searchQueryParam; + return [queryParams, 'q=' + searchQueryParam].filter(q => q.length > 0).join('&'); + } + + get maxResults(): number { + return 'maxResults' in this.params ? parseInt(this.params['maxResults']) : 10; + } + + set maxResults(value: string) { + if (!value || isNaN(parseInt(value))) { + return; + } + + this.params['maxResults'] = value; + } + + get startIndex(): number { + return 'startIndex' in this.params ? parseInt(this.params['startIndex']) : 10; + } + + set startIndex(value: string) { + if (!value || isNaN(parseInt(value))) { + return; + } + + this.params['startIndex'] = value; + } + + get intitle(): string { + return 'intitle' in this.params ? this.params['intitle'] : null; + } + + set intitle(value: string) { + if (!value) { + delete this.params['intitle']; + } else { + this.params['intitle'] = value; + } + } + + get inpublisher(): string { + return 'inpublisher' in this.params ? this.params['inpublisher'] : null; + } + + set inpublisher(value: string) { + if (!value) { + delete this.params['inpublisher']; + } else { + this.params['inpublisher'] = value; + } + } + + get inauthor(): string { + return 'inauthor' in this.params ? this.params['inauthor'] : null; + } + + set inauthor(value: string) { + if (!value) { + delete this.params['inauthor']; + } else { + this.params['inauthor'] = value; + } + } + + get isbn(): string { + return 'isbn' in this.params ? this.params['isbn'] : null; + } + + set isbn(value: string) { + if (!value) { + delete this.params['isbn']; + } else { + this.params['isbn'] = value; + } + } + + get subject(): string { + return 'subject' in this.params ? this.params['subject'] : null; + } + + set subject(value: string) { + if (!value) { + delete this.params['subject']; + } else { + this.params['subject'] = value; + } } next() { - const resultsPerPage = parseInt(this.params['maxResults']) ?? 10; - const index = parseInt(this.params['startIndex']) ?? 0; + const resultsPerPage = this.params['maxResults'] ? parseInt(this.params['maxResults']) : 10; + const index = this.params['startIndex'] ? parseInt(this.params['startIndex']) : 0; const data = { ...this.params }; data['startIndex'] = (index + resultsPerPage).toString(); diff --git a/backend/nestjs-seshat-api/src/providers/contexts/search.context.ts b/backend/nestjs-seshat-api/src/providers/contexts/search.context.ts index 9d7a898..3884fcc 100644 --- a/backend/nestjs-seshat-api/src/providers/contexts/search.context.ts +++ b/backend/nestjs-seshat-api/src/providers/contexts/search.context.ts @@ -1,4 +1,4 @@ -abstract class SearchContext { +export abstract class SearchContext { provider: string; search: string; params: { [key: string]: string }; diff --git a/backend/nestjs-seshat-api/src/providers/google/google.service.ts b/backend/nestjs-seshat-api/src/providers/google/google.service.ts index 71b5471..49f44f7 100644 --- a/backend/nestjs-seshat-api/src/providers/google/google.service.ts +++ b/backend/nestjs-seshat-api/src/providers/google/google.service.ts @@ -3,6 +3,7 @@ import { BookSearchResultDto } from '../dto/book-search-result.dto'; import { HttpService } from '@nestjs/axios'; import { firstValueFrom, map, timeout } from 'rxjs'; import { AxiosResponse } from 'axios'; +import { GoogleSearchContext } from '../contexts/google.search.context'; @Injectable() export class GoogleService { @@ -11,7 +12,7 @@ export class GoogleService { ) { } async searchRaw(searchQuery: string): Promise { - const queryParams = 'printType=books&maxResults=10&fields=items(kind,id,volumeInfo(title,description,authors,publisher,publishedDate,industryIdentifiers,language,categories,maturityRating,imageLinks,canonicalVolumeLink,seriesInfo))&q='; + const queryParams = 'langRestrict=en&printType=books&maxResults=10&fields=items(kind,id,volumeInfo(title,description,authors,publisher,publishedDate,industryIdentifiers,language,categories,maturityRating,imageLinks,canonicalVolumeLink,seriesInfo))&q='; return await firstValueFrom( this.http.get('https://www.googleapis.com/books/v1/volumes?' + queryParams + searchQuery) @@ -23,14 +24,19 @@ export class GoogleService { } async search(context: GoogleSearchContext): Promise { - const defaultQueryParams = 'printType=books&fields=items(kind,id,volumeInfo(title,description,authors,publisher,publishedDate,industryIdentifiers,language,categories,maturityRating,imageLinks,canonicalVolumeLink,seriesInfo))'; + if (!context) { + return null; + } + + const defaultQueryParams = 'langRestrict=en&printType=books&fields=items(kind,id,volumeInfo(title,description,authors,publisher,publishedDate,industryIdentifiers,language,categories,maturityRating,imageLinks,canonicalVolumeLink,seriesInfo))'; const customQueryParams = context.generateQueryParams(); + console.log(defaultQueryParams, customQueryParams); return await firstValueFrom( this.http.get('https://www.googleapis.com/books/v1/volumes?' + defaultQueryParams + '&' + customQueryParams) .pipe( timeout({ first: 5000 }), - map(this.transform), + map(value => this.transform(value)), ) ); } @@ -40,7 +46,9 @@ export class GoogleService { return []; } - return response.data.items.map(item => this.extract(item)); + return response.data.items + //.filter(item => item.volumeInfo?.canonicalVolumeLink?.startsWith('https://play.google.com/store/books/details')) + .map(item => this.extract(item)); } private extract(item: any): BookSearchResultDto { @@ -49,12 +57,12 @@ export class GoogleService { providerSeriesId: item.volumeInfo.seriesInfo?.volumeSeries[0].seriesId, title: item.volumeInfo.title, desc: item.volumeInfo.description, - volume: parseInt(item.volumeInfo.seriesInfo?.bookDisplayNumber), + volume: item.volumeInfo.seriesInfo?.bookDisplayNumber ? parseInt(item.volumeInfo.seriesInfo?.bookDisplayNumber, 10) : undefined, publisher: item.volumeInfo.publisher, authors: item.volumeInfo.authors, categories: item.volumeInfo.categories, maturityRating: item.volumeInfo.maturityRating, - industryIdentifiers: Object.assign({}, ...item.volumeInfo.industryIdentifiers.map(i => ({ [i.type]: i.identifier }))), + industryIdentifiers: item.volumeInfo.industryIdentifiers ? Object.assign({}, ...item.volumeInfo.industryIdentifiers.map(i => ({ [i.type]: i.identifier }))) : [], publishedAt: new Date(item.volumeInfo.publishedDate), language: item.volumeInfo.language, thumbnail: item.volumeInfo.imageLinks?.thumbnail, @@ -62,15 +70,13 @@ export class GoogleService { provider: 'google' } - if (result.providerSeriesId) { - let regex = this.getRegexByPublisher(result.publisher); + let regex = this.getRegexByPublisher(result.publisher); - const match = result.title.match(regex); - if (match?.groups) { - result.title = match.groups['title'].trim(); - if (!result.volume) { - result.volume = parseInt(match.groups['volume']); - } + const match = result.title.match(regex); + if (match?.groups) { + result.title = match.groups['title'].trim(); + if (!result.volume || isNaN(result.volume)) { + result.volume = parseInt(match.groups['volume'], 10); } } diff --git a/backend/nestjs-seshat-api/src/providers/providers.service.ts b/backend/nestjs-seshat-api/src/providers/providers.service.ts index 09e1b4c..c87c7a5 100644 --- a/backend/nestjs-seshat-api/src/providers/providers.service.ts +++ b/backend/nestjs-seshat-api/src/providers/providers.service.ts @@ -1,6 +1,8 @@ import { Injectable } from '@nestjs/common'; import { GoogleService } from './google/google.service'; import { BookSearchResultDto } from './dto/book-search-result.dto'; +import { GoogleSearchContext } from './contexts/google.search.context'; +import { SearchContext } from './contexts/search.context'; @Injectable() export class ProvidersService { @@ -10,7 +12,7 @@ export class ProvidersService { generateSearchContext(providerName: string, searchQuery: string): SearchContext | null { let params: { [key: string]: string } = {}; - if (providerName == 'google') { + if (providerName.toLowerCase() == 'google') { return new GoogleSearchContext(searchQuery, params); } return null; @@ -28,7 +30,7 @@ export class ProvidersService { async search(context: SearchContext): Promise { switch (context.provider.toLowerCase()) { case 'google': - return await this.google.search(context); + return await this.google.search(context as GoogleSearchContext); default: throw Error('Invalid provider name.'); } diff --git a/backend/nestjs-seshat-api/src/series/dto/create-series-subscription-job.dto.ts b/backend/nestjs-seshat-api/src/series/dto/create-series-subscription-job.dto.ts new file mode 100644 index 0000000..b2d9927 --- /dev/null +++ b/backend/nestjs-seshat-api/src/series/dto/create-series-subscription-job.dto.ts @@ -0,0 +1,8 @@ +import { IsNotEmpty, IsString } from 'class-validator'; +import { SeriesSubscriptionDto } from './series-subscription.dto'; + +export class CreateSeriesSubscriptionJobDto extends SeriesSubscriptionDto { + @IsString() + @IsNotEmpty() + title: string; +} \ No newline at end of file diff --git a/backend/nestjs-seshat-api/src/series/dto/create-series-subscription.dto.ts b/backend/nestjs-seshat-api/src/series/dto/create-series-subscription.dto.ts new file mode 100644 index 0000000..7721d4f --- /dev/null +++ b/backend/nestjs-seshat-api/src/series/dto/create-series-subscription.dto.ts @@ -0,0 +1,8 @@ +import { IsNotEmpty, IsString } from 'class-validator'; +import { SeriesDto } from './series.dto'; + +export class CreateSeriesSubscriptionDto extends SeriesDto { + @IsString() + @IsNotEmpty() + title: string; +} \ No newline at end of file diff --git a/backend/nestjs-seshat-api/src/series/dto/create-series.dto.ts b/backend/nestjs-seshat-api/src/series/dto/create-series.dto.ts index e219367..635a0d9 100644 --- a/backend/nestjs-seshat-api/src/series/dto/create-series.dto.ts +++ b/backend/nestjs-seshat-api/src/series/dto/create-series.dto.ts @@ -1,15 +1,8 @@ import { IsNotEmpty, IsString } from 'class-validator'; +import { SeriesDto } from './series.dto'; -export class CreateSeriesDto { - @IsString() - @IsNotEmpty() - providerSeriesId: string; - +export class CreateSeriesDto extends SeriesDto { @IsString() @IsNotEmpty() title: string; - - @IsString() - @IsNotEmpty() - provider: string; } \ No newline at end of file diff --git a/backend/nestjs-seshat-api/src/series/dto/series-subscription.dto.ts b/backend/nestjs-seshat-api/src/series/dto/series-subscription.dto.ts new file mode 100644 index 0000000..1ca8790 --- /dev/null +++ b/backend/nestjs-seshat-api/src/series/dto/series-subscription.dto.ts @@ -0,0 +1,9 @@ +import { IsNotEmpty, IsUUID } from 'class-validator'; +import { SeriesDto } from './series.dto'; +import { UUID } from 'crypto'; + +export class SeriesSubscriptionDto extends SeriesDto { + @IsUUID() + @IsNotEmpty() + userId: UUID; +} \ No newline at end of file diff --git a/backend/nestjs-seshat-api/src/series/dto/delete-series.dto.ts b/backend/nestjs-seshat-api/src/series/dto/series.dto.ts similarity index 84% rename from backend/nestjs-seshat-api/src/series/dto/delete-series.dto.ts rename to backend/nestjs-seshat-api/src/series/dto/series.dto.ts index b073b69..4f29a65 100644 --- a/backend/nestjs-seshat-api/src/series/dto/delete-series.dto.ts +++ b/backend/nestjs-seshat-api/src/series/dto/series.dto.ts @@ -1,6 +1,6 @@ import { IsNotEmpty, IsString } from 'class-validator'; -export class DeleteSeriesDto { +export class SeriesDto { @IsString() @IsNotEmpty() providerSeriesId: string; diff --git a/backend/nestjs-seshat-api/src/series/dto/update-series.dto.ts b/backend/nestjs-seshat-api/src/series/dto/update-series.dto.ts index befca63..e758b60 100644 --- a/backend/nestjs-seshat-api/src/series/dto/update-series.dto.ts +++ b/backend/nestjs-seshat-api/src/series/dto/update-series.dto.ts @@ -1,20 +1,9 @@ -import { IsNotEmpty, IsString, IsUUID } from 'class-validator'; +import { IsNotEmpty, IsUUID } from 'class-validator'; import { UUID } from 'crypto'; +import { CreateSeriesDto } from './create-series.dto'; -export class UpdateSeriesDto { +export class UpdateSeriesDto extends CreateSeriesDto { @IsUUID() @IsNotEmpty() seriesId: UUID; - - @IsString() - @IsNotEmpty() - providerSeriesId: string; - - @IsString() - @IsNotEmpty() - title: string; - - @IsString() - @IsNotEmpty() - provider: string; } \ No newline at end of file diff --git a/backend/nestjs-seshat-api/src/series/entities/series-subscription.entity.ts b/backend/nestjs-seshat-api/src/series/entities/series-subscription.entity.ts new file mode 100644 index 0000000..8e28a23 --- /dev/null +++ b/backend/nestjs-seshat-api/src/series/entities/series-subscription.entity.ts @@ -0,0 +1,17 @@ +import { UUID } from 'crypto'; +import { Column, Entity, PrimaryColumn, Unique } from 'typeorm'; + +@Entity("series_subscriptions") +export class SeriesSubscriptionEntity { + @PrimaryColumn({ name: 'user_id', type: 'uuid' }) + readonly userId: UUID; + + @PrimaryColumn({ name: 'provider_series_id', type: 'text' }) + providerSeriesId: string; + + @PrimaryColumn({ name: 'provider', type: 'text' }) + provider: string; + + @Column({ name: 'added_at', type: 'timestamptz', nullable: false }) + addedAt: Date; +} \ No newline at end of file diff --git a/backend/nestjs-seshat-api/src/series/series.module.ts b/backend/nestjs-seshat-api/src/series/series.module.ts index d07a90e..e023d3f 100644 --- a/backend/nestjs-seshat-api/src/series/series.module.ts +++ b/backend/nestjs-seshat-api/src/series/series.module.ts @@ -4,11 +4,13 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import { SeriesEntity } from './entities/series.entity'; import { HttpModule } from '@nestjs/axios'; import { ProvidersModule } from 'src/providers/providers.module'; +import { SeriesSubscriptionEntity } from './entities/series-subscription.entity'; @Module({ imports: [ TypeOrmModule.forFeature([ SeriesEntity, + SeriesSubscriptionEntity, ]), HttpModule, ProvidersModule, diff --git a/backend/nestjs-seshat-api/src/series/series.service.ts b/backend/nestjs-seshat-api/src/series/series.service.ts index e32b481..c44fe95 100644 --- a/backend/nestjs-seshat-api/src/series/series.service.ts +++ b/backend/nestjs-seshat-api/src/series/series.service.ts @@ -2,23 +2,56 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { SeriesEntity } from './entities/series.entity'; import { Repository } from 'typeorm'; -import { PinoLogger } from 'nestjs-pino'; import { CreateSeriesDto } from './dto/create-series.dto'; -import { DeleteSeriesDto } from './dto/delete-series.dto'; +import { SeriesDto } from './dto/series.dto'; +import { SeriesSubscriptionEntity } from './entities/series-subscription.entity'; +import { UUID } from 'crypto'; +import { SeriesSubscriptionDto } from './dto/series-subscription.dto'; @Injectable() export class SeriesService { constructor( @InjectRepository(SeriesEntity) private seriesRepository: Repository, - private logger: PinoLogger, + @InjectRepository(SeriesSubscriptionEntity) + private seriesSubscriptionRepository: Repository, ) { } - async deleteSeries(series: DeleteSeriesDto) { + async addSeries(series: CreateSeriesDto) { + return await this.seriesRepository.insert(series); + } + + async addSeriesSubscription(series: SeriesSubscriptionDto) { + return await this.seriesSubscriptionRepository.insert(series); + } + + async deleteSeries(series: SeriesDto) { return await this.seriesRepository.delete(series); } + async deleteSeriesSubscription(subscription: SeriesSubscriptionDto) { + return await this.seriesSubscriptionRepository.delete(subscription); + } + + async getSeries(series: SeriesDto) { + return await this.seriesRepository.findOne({ + where: series + }) + } + + async getAllSeries() { + return await this.seriesRepository.find() + } + + async getSeriesSubscribedBy(userId: UUID) { + return await this.seriesSubscriptionRepository.find({ + where: { + userId, + } + }); + } + async updateSeries(series: CreateSeriesDto) { return await this.seriesRepository.upsert(series, ['provider', 'providerSeriesId']); }