From a96335dbeec3fc4379d0668f5aadf6f92141e6a1 Mon Sep 17 00:00:00 2001 From: Gauthier Roebroeck Date: Mon, 7 Feb 2022 12:30:29 +0800 Subject: [PATCH] feat: page hashing enhancement only hash pages for cbz delete non-cbz page hashes store page hashes --- komga-webui/package-lock.json | 30 ++- komga-webui/package.json | 1 + .../src/components/PageHashKnownCard.vue | 163 +++++++++++++ .../src/components/PageHashMatchesTable.vue | 6 +- .../src/components/PageHashUnknownCard.vue | 105 ++++++++ komga-webui/src/functions/file.ts | 10 + komga-webui/src/functions/urls.ts | 8 +- komga-webui/src/locales/en.json | 22 +- komga-webui/src/router.ts | 18 +- .../src/services/komga-pagehashes.service.ts | 47 +++- komga-webui/src/types/enum-pagehashes.ts | 5 + komga-webui/src/types/komga-pagehashes.ts | 25 +- .../views/SettingsDuplicatePagesHolder.vue | 21 ++ .../src/views/SettingsDuplicatePagesKnown.vue | 224 ++++++++++++++++++ ....vue => SettingsDuplicatePagesUnknown.vue} | 74 ++---- komga-webui/src/views/SettingsHolder.vue | 2 +- .../sqlite/V20220128152310__page_hash.sql | 30 +++ .../komga/application/tasks/TaskReceiver.kt | 10 +- .../org/gotson/komga/domain/model/PageHash.kt | 19 +- .../komga/domain/model/PageHashKnown.kt | 20 ++ .../komga/domain/model/PageHashUnknown.kt | 10 +- .../domain/persistence/MediaRepository.kt | 2 +- .../domain/persistence/PageHashRepository.kt | 11 +- .../komga/domain/service/PageHashLifecycle.kt | 36 ++- .../komga/infrastructure/jooq/MediaDao.kt | 10 +- .../komga/infrastructure/jooq/PageHashDao.kt | 123 +++++++++- .../interfaces/api/rest/PageHashController.kt | 53 +++-- .../api/rest/dto/PageHashCreationDto.kt | 11 + .../{PageHashDto.kt => PageHashKnownDto.kt} | 8 +- .../api/rest/dto/PageHashUnknownDto.kt | 7 +- .../gotson/komga/domain/model/PageHashTest.kt | 28 +++ .../domain/service/PageHashLifecycleTest.kt | 34 +++ .../komga/infrastructure/jooq/MediaDaoTest.kt | 42 +++- .../infrastructure/jooq/PageHashDaoTest.kt | 139 +++++++++++ 34 files changed, 1189 insertions(+), 165 deletions(-) create mode 100644 komga-webui/src/components/PageHashKnownCard.vue create mode 100644 komga-webui/src/components/PageHashUnknownCard.vue create mode 100644 komga-webui/src/types/enum-pagehashes.ts create mode 100644 komga-webui/src/views/SettingsDuplicatePagesHolder.vue create mode 100644 komga-webui/src/views/SettingsDuplicatePagesKnown.vue rename komga-webui/src/views/{SettingsDuplicatePages.vue => SettingsDuplicatePagesUnknown.vue} (71%) create mode 100644 komga/src/flyway/resources/db/migration/sqlite/V20220128152310__page_hash.sql create mode 100644 komga/src/main/kotlin/org/gotson/komga/domain/model/PageHashKnown.kt create mode 100644 komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/dto/PageHashCreationDto.kt rename komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/dto/{PageHashDto.kt => PageHashKnownDto.kt} (71%) create mode 100644 komga/src/test/kotlin/org/gotson/komga/domain/model/PageHashTest.kt create mode 100644 komga/src/test/kotlin/org/gotson/komga/domain/service/PageHashLifecycleTest.kt create mode 100644 komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/PageHashDaoTest.kt diff --git a/komga-webui/package-lock.json b/komga-webui/package-lock.json index 6ed44f0ae..525a337c3 100644 --- a/komga-webui/package-lock.json +++ b/komga-webui/package-lock.json @@ -12,6 +12,7 @@ "axios": "^0.25.0", "core-js": "^3.20.3", "date-fns": "^2.28.0", + "filesize": "^8.0.7", "js-file-downloader": "^1.1.24", "language-tags": "^1.0.5", "lodash": "^4.17.21", @@ -9261,10 +9262,9 @@ "dev": true }, "node_modules/filesize": { - "version": "3.6.1", - "resolved": "https://registry.npmjs.org/filesize/-/filesize-3.6.1.tgz", - "integrity": "sha512-7KjR1vv6qnicaPMi1iiTcI85CyYwRO/PSFCu6SvqL8jN2Wjt/NIYQTFtFs7fSDCYOstUkEWIQGFUg5YZQfjlcg==", - "dev": true, + "version": "8.0.7", + "resolved": "https://registry.npmjs.org/filesize/-/filesize-8.0.7.tgz", + "integrity": "sha512-pjmC+bkIF8XI7fWaH8KxHcZL3DPybs1roSKP4rKDvy20tAWwIObE4+JIseG2byfGKhud5ZnM4YSGKBz7Sh0ndQ==", "engines": { "node": ">= 0.4.0" } @@ -20275,6 +20275,15 @@ "node": ">=0.4.0" } }, + "node_modules/webpack-bundle-analyzer/node_modules/filesize": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/filesize/-/filesize-3.6.1.tgz", + "integrity": "sha512-7KjR1vv6qnicaPMi1iiTcI85CyYwRO/PSFCu6SvqL8jN2Wjt/NIYQTFtFs7fSDCYOstUkEWIQGFUg5YZQfjlcg==", + "dev": true, + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/webpack-bundle-analyzer/node_modules/ws": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.2.tgz", @@ -28286,10 +28295,9 @@ "dev": true }, "filesize": { - "version": "3.6.1", - "resolved": "https://registry.npmjs.org/filesize/-/filesize-3.6.1.tgz", - "integrity": "sha512-7KjR1vv6qnicaPMi1iiTcI85CyYwRO/PSFCu6SvqL8jN2Wjt/NIYQTFtFs7fSDCYOstUkEWIQGFUg5YZQfjlcg==", - "dev": true + "version": "8.0.7", + "resolved": "https://registry.npmjs.org/filesize/-/filesize-8.0.7.tgz", + "integrity": "sha512-pjmC+bkIF8XI7fWaH8KxHcZL3DPybs1roSKP4rKDvy20tAWwIObE4+JIseG2byfGKhud5ZnM4YSGKBz7Sh0ndQ==" }, "fill-range": { "version": "4.0.0", @@ -37203,6 +37211,12 @@ "integrity": "sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==", "dev": true }, + "filesize": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/filesize/-/filesize-3.6.1.tgz", + "integrity": "sha512-7KjR1vv6qnicaPMi1iiTcI85CyYwRO/PSFCu6SvqL8jN2Wjt/NIYQTFtFs7fSDCYOstUkEWIQGFUg5YZQfjlcg==", + "dev": true + }, "ws": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.2.tgz", diff --git a/komga-webui/package.json b/komga-webui/package.json index f4adad927..30a03510d 100644 --- a/komga-webui/package.json +++ b/komga-webui/package.json @@ -13,6 +13,7 @@ "axios": "^0.25.0", "core-js": "^3.20.3", "date-fns": "^2.28.0", + "filesize": "^8.0.7", "js-file-downloader": "^1.1.24", "language-tags": "^1.0.5", "lodash": "^4.17.21", diff --git a/komga-webui/src/components/PageHashKnownCard.vue b/komga-webui/src/components/PageHashKnownCard.vue new file mode 100644 index 000000000..581e817f3 --- /dev/null +++ b/komga-webui/src/components/PageHashKnownCard.vue @@ -0,0 +1,163 @@ + + + diff --git a/komga-webui/src/components/PageHashMatchesTable.vue b/komga-webui/src/components/PageHashMatchesTable.vue index 3ac1cc870..144e24020 100644 --- a/komga-webui/src/components/PageHashMatchesTable.vue +++ b/komga-webui/src/components/PageHashMatchesTable.vue @@ -33,13 +33,13 @@ diff --git a/komga-webui/src/functions/file.ts b/komga-webui/src/functions/file.ts index fb094fd3b..04164ea3c 100644 --- a/komga-webui/src/functions/file.ts +++ b/komga-webui/src/functions/file.ts @@ -1,3 +1,5 @@ +import filesize from 'filesize' + export async function getFileFromUrl(url: string, name: string = url, defaultType = 'image/jpeg') { const response = await fetch(url) const data = await response.blob() @@ -5,3 +7,11 @@ export async function getFileFromUrl(url: string, name: string = url, defaultTyp type: data.type || defaultType, }) } + + +const filesizePartial = filesize.partial({round: 1}) + +export function getFileSize(n?: number): string | undefined { + if(!n) return undefined + return filesizePartial(n) +} diff --git a/komga-webui/src/functions/urls.ts b/komga-webui/src/functions/urls.ts index 2a2b38473..e1e92427c 100644 --- a/komga-webui/src/functions/urls.ts +++ b/komga-webui/src/functions/urls.ts @@ -1,4 +1,4 @@ -import {PageHashUnknownDto} from '@/types/komga-pagehashes' +import {PageHashKnownDto, PageHashUnknownDto} from '@/types/komga-pagehashes' const fullUrl = process.env.VUE_APP_KOMGA_API_URL ? process.env.VUE_APP_KOMGA_API_URL @@ -75,9 +75,13 @@ export function transientBookPageUrl(transientBookId: string, page: number): str } export function pageHashUnknownThumbnailUrl(pageHash: PageHashUnknownDto, resize?: number): string { - let url = `${urls.originNoSlash}/api/v1/page-hashes/unknown/${pageHash.hash}/thumbnail?media_type=${pageHash.mediaType}&file_size=${pageHash.sizeBytes || -1}` + let url = `${urls.originNoSlash}/api/v1/page-hashes/unknown/${pageHash.hash}/thumbnail?media_type=${pageHash.mediaType}&file_size=${pageHash.size || -1}` if(resize) { url += `&resize=${resize}` } return url } + +export function pageHashKnownThumbnailUrl(pageHash: PageHashKnownDto): string { + return `${urls.originNoSlash}/api/v1/page-hashes/${pageHash.hash}/thumbnail?media_type=${pageHash.mediaType}&file_size=${pageHash.size || -1}` +} diff --git a/komga-webui/src/locales/en.json b/komga-webui/src/locales/en.json index 40a93ddd9..2d4aa66b4 100644 --- a/komga-webui/src/locales/en.json +++ b/komga-webui/src/locales/en.json @@ -531,16 +531,21 @@ "duplicate_pages": { "action_delete_auto": "Auto delete", "action_delete_manual": "Manual delete", + "action_delete_matches": "Delete matches", "action_ignore": "Ignore", - "matches_n": "No matches | 1 match | {count} matches", - "title": "Duplicate pages", - "unknown_size": "Unknown size", "delete_to_save": "Delete to save {size}", + "deleted_count": "Deleted {count} times", "filter": { - "total_size": "Total size", + "count": "Count", + "delete_count": "Deletion count", + "delete_size": "Space saved", "size": "Size", - "count": "Count" - } + "total_size": "Total size" + }, + "matches_n": "No matches | 1 match | {count} matches", + "saved_size": "Saved {size}", + "title": "Duplicate pages", + "unknown_size": "Unknown size" }, "duplicates": { "file_hash": "File hash", @@ -560,6 +565,11 @@ "UNKNOWN": "Unknown", "UNSUPPORTED": "Unsupported" }, + "page_hash_action": { + "DELETE_AUTO": "Auto delete", + "DELETE_MANUAL": "Manual delete", + "IGNORE": "Ignore" + }, "reading_direction": { "LEFT_TO_RIGHT": "Left to right", "RIGHT_TO_LEFT": "Right to left", diff --git a/komga-webui/src/router.ts b/komga-webui/src/router.ts index 924d5cae0..80cb31d6a 100644 --- a/komga-webui/src/router.ts +++ b/komga-webui/src/router.ts @@ -97,8 +97,22 @@ const router = new Router({ { path: '/settings/duplicate-pages', name: 'settings-duplicate-pages', - beforeEnter: adminGuard, - component: () => import(/* webpackChunkName: "settings-duplicate-pages" */ './views/SettingsDuplicatePages.vue'), + redirect: {name: 'settings-duplicate-pages-known'}, + component: () => import(/* webpackChunkName: "settings-duplicate-pages" */ './views/SettingsDuplicatePagesHolder.vue'), + children: [ + { + path: '/settings/duplicate-pages/known', + name: 'settings-duplicate-pages-known', + beforeEnter: adminGuard, + component: () => import(/* webpackChunkName: "settings-duplicate-pages" */ './views/SettingsDuplicatePagesKnown.vue'), + }, + { + path: '/settings/duplicate-pages/unknown', + name: 'settings-duplicate-pages-unknown', + beforeEnter: adminGuard, + component: () => import(/* webpackChunkName: "settings-duplicate-pages" */ './views/SettingsDuplicatePagesUnknown.vue'), + }, + ], }, { path: '/settings/server', diff --git a/komga-webui/src/services/komga-pagehashes.service.ts b/komga-webui/src/services/komga-pagehashes.service.ts index aadfd06a6..ed64d5285 100644 --- a/komga-webui/src/services/komga-pagehashes.service.ts +++ b/komga-webui/src/services/komga-pagehashes.service.ts @@ -1,5 +1,11 @@ import {AxiosInstance} from 'axios' -import {PageHashMatchDto, PageHashUnknownDto} from '@/types/komga-pagehashes' +import { + PageHashCreationDto, + PageHashDto, + PageHashKnownDto, + PageHashMatchDto, + PageHashUnknownDto, +} from '@/types/komga-pagehashes' const qs = require('qs') @@ -12,6 +18,23 @@ export default class KomgaPageHashesService { this.http = http } + async getKnownHashes(actions: string[], pageRequest?: PageRequest): Promise> { + try { + const params = {...pageRequest} as any + if (actions) params.action = actions + return (await this.http.get(API_PAGE_HASH, { + params: params, + paramsSerializer: params => qs.stringify(params, {indices: false}), + })).data + } catch (e) { + let msg = 'An error occurred while trying to retrieve known page hashes' + if (e.response.data.message) { + msg += `: ${e.response.data.message}` + } + throw new Error(msg) + } + } + async getUnknownHashes(pageRequest?: PageRequest): Promise> { try { return (await this.http.get(`${API_PAGE_HASH}/unknown`, { @@ -27,19 +50,31 @@ export default class KomgaPageHashesService { } } - async getUnknownPageHashMatches(hash: PageHashUnknownDto, pageRequest?: PageRequest): Promise> { + async getUnknownPageHashMatches(pageHash: PageHashDto, pageRequest?: PageRequest): Promise> { try { const params = { ...pageRequest, - media_type: hash.mediaType, - file_size: hash.sizeBytes || -1, + media_type: pageHash.mediaType, + file_size: pageHash.size || -1, } - return (await this.http.get(`${API_PAGE_HASH}/unknown/${hash.hash}`, { + return (await this.http.get(`${API_PAGE_HASH}/unknown/${pageHash.hash}`, { params: params, paramsSerializer: params => qs.stringify(params, {indices: false}), })).data } catch (e) { - let msg = `An error occurred while trying to retrieve matches for page hash: ${hash}` + let msg = `An error occurred while trying to retrieve matches for page hash: ${pageHash}` + if (e.response.data.message) { + msg += `: ${e.response.data.message}` + } + throw new Error(msg) + } + } + + async createOrUpdatePageHash(pageHash: PageHashCreationDto) { + try { + await this.http.put(API_PAGE_HASH, pageHash) + } catch (e) { + let msg = `An error occurred while trying to add page hash ${pageHash}` if (e.response.data.message) { msg += `: ${e.response.data.message}` } diff --git a/komga-webui/src/types/enum-pagehashes.ts b/komga-webui/src/types/enum-pagehashes.ts new file mode 100644 index 000000000..07cac83d1 --- /dev/null +++ b/komga-webui/src/types/enum-pagehashes.ts @@ -0,0 +1,5 @@ +export enum PageHashAction { + DELETE_AUTO = 'DELETE_AUTO', + DELETE_MANUAL = 'DELETE_MANUAL', + IGNORE = 'IGNORE', +} diff --git a/komga-webui/src/types/komga-pagehashes.ts b/komga-webui/src/types/komga-pagehashes.ts index 3c1c688b2..5bfa212bd 100644 --- a/komga-webui/src/types/komga-pagehashes.ts +++ b/komga-webui/src/types/komga-pagehashes.ts @@ -1,9 +1,12 @@ -export interface PageHashUnknownDto { +import {PageHashAction} from '@/types/enum-pagehashes' + +export interface PageHashDto { hash: string, mediaType: string, - sizeBytes?: number, - size?: string, - totalSize?: string, + size?: number, +} + +export interface PageHashUnknownDto extends PageHashDto { matchCount: number, } @@ -13,3 +16,17 @@ export interface PageHashMatchDto { pageNumber: number, fileName: string, } + +export interface PageHashCreationDto { + hash: string, + mediaType: string, + size?: number, + action: PageHashAction, +} + +export interface PageHashKnownDto extends PageHashDto { + action: PageHashAction + deleteCount: number, + createdDate: string, + lastModifiedDate: string, +} diff --git a/komga-webui/src/views/SettingsDuplicatePagesHolder.vue b/komga-webui/src/views/SettingsDuplicatePagesHolder.vue new file mode 100644 index 000000000..f4c10bd36 --- /dev/null +++ b/komga-webui/src/views/SettingsDuplicatePagesHolder.vue @@ -0,0 +1,21 @@ + + + + + diff --git a/komga-webui/src/views/SettingsDuplicatePagesKnown.vue b/komga-webui/src/views/SettingsDuplicatePagesKnown.vue new file mode 100644 index 000000000..869fd6cb7 --- /dev/null +++ b/komga-webui/src/views/SettingsDuplicatePagesKnown.vue @@ -0,0 +1,224 @@ + + + + + diff --git a/komga-webui/src/views/SettingsDuplicatePages.vue b/komga-webui/src/views/SettingsDuplicatePagesUnknown.vue similarity index 71% rename from komga-webui/src/views/SettingsDuplicatePages.vue rename to komga-webui/src/views/SettingsDuplicatePagesUnknown.vue index 9acc6f681..71ff1765e 100644 --- a/komga-webui/src/views/SettingsDuplicatePages.vue +++ b/komga-webui/src/views/SettingsDuplicatePagesUnknown.vue @@ -36,54 +36,19 @@ - - - - - - - - - -
{{ element.mediaType }}
-
{{ element.size || $t('duplicate_pages.unknown_size') }}
- - - {{ $tc('duplicate_pages.matches_n', element.matchCount) }} - - -
{{ $t('duplicate_pages.delete_to_save', {size: element.totalSize}) }} -
-
-
-
-
- - - {{ $t('duplicate_pages.action_ignore') }} - {{ $t('duplicate_pages.action_delete_manual') }} - {{ $t('duplicate_pages.action_delete_auto') }} - -
+ +
@@ -132,16 +97,18 @@ diff --git a/komga-webui/src/views/SettingsHolder.vue b/komga-webui/src/views/SettingsHolder.vue index 23ee205af..9a5f97445 100644 --- a/komga-webui/src/views/SettingsHolder.vue +++ b/komga-webui/src/views/SettingsHolder.vue @@ -1,6 +1,6 @@