diff --git a/frontend/src/features/collections/collection-items-dialog.ts b/frontend/src/features/collections/collection-items-dialog.ts index dff4f731b8..1135b83b79 100644 --- a/frontend/src/features/collections/collection-items-dialog.ts +++ b/frontend/src/features/collections/collection-items-dialog.ts @@ -6,13 +6,13 @@ import { customElement, property, query, state } from "lit/decorators.js"; import { cache } from "lit/directives/cache.js"; import { repeat } from "lit/directives/repeat.js"; import { when } from "lit/directives/when.js"; -import difference from "lodash/fp/difference"; import union from "lodash/fp/union"; import without from "lodash/fp/without"; import queryString from "query-string"; import type { AutoAddChangeDetail, + CrawlsPageChangeDetail, SelectionChangeDetail, } from "./collection-workflow-list"; @@ -34,8 +34,9 @@ import type { APISortQuery, } from "@/types/api"; import type { ArchivedItem, Crawl, Upload, Workflow } from "@/types/crawler"; +import { SortDirection } from "@/types/utils"; import { isApiError } from "@/utils/api"; -import { finishedCrawlStates } from "@/utils/crawler"; +import { finishedCrawlStates, isCrawl } from "@/utils/crawler"; import { pluralOf } from "@/utils/pluralize"; import { tw } from "@/utils/tailwind"; @@ -71,6 +72,10 @@ const uploadSortOptions: SortOptions = [ const COLLECTION_ITEMS_MAX = 1000; const DEFAULT_PAGE_SIZE = 10; +const isID = (v: string | Symbol): v is string => typeof v === "string"; +const unpackSymbolID = (v: string | Symbol) => + isID(v) ? v : v.description || ""; + @customElement("btrix-collection-items-dialog") @localized() export class CollectionItemsDialog extends BtrixElement { @@ -146,28 +151,71 @@ export class CollectionItemsDialog extends BtrixElement { @state() private uploadSearchValues?: SearchValues; + @state() + private isReady = false; + /** - * Whether item is selected or not, keyed by ID + * Selection state for individual archived items by ID */ @state() - private selection: { [itemID: string]: boolean } = {}; + private selectedItems = new Set(); + /** + * Selection state for workflows + */ @state() - private isReady = false; + private workflowSelection = new Map< + string, + { + checked: boolean | "indeterminate"; + selectionCount: number; + } + >(); + + @state() + private workflowCrawls = new Map< + string, + { + selectedCrawls: APIPaginatedList | null; + paginatedCrawls: APIPaginatedList | null; + } + >(); @query("btrix-dialog") private readonly dialog!: Dialog; - private savedCollectionItemIDs: string[] = []; + /** + * Workflow batch operations to apply on save + */ + private batchWorkflows = new Map< + string, + { + operation: "add" | "remove"; + omitCrawls: Set; + } + >(); + + /** + * Map crawl IDs to workflow IDs to look up + */ + private readonly crawlToWorkflow = new Map(); + + /** + * Store previously saved selection to compare + */ + private savedSelectedItems: CollectionItemsDialog["selectedItems"] = + new Set(); + private savedWorkflowSelection: CollectionItemsDialog["workflowSelection"] = + new Map(); private readonly tabLabels: Record = { crawl: { icon: "gear-wide-connected", - label: msg("Crawls"), + label: msg("Crawled Items"), }, upload: { icon: "upload", - label: msg("Uploads"), + label: msg("Uploaded Items"), }, }; @@ -182,17 +230,30 @@ export class CollectionItemsDialog extends BtrixElement { changedProperties.has("showOnlyMine") || changedProperties.has("showOnlyInCollection") ) { - void this.fetchCrawls(); - void this.fetchUploads(); + if (this.showOnlyInCollection) { + void this.fetchCrawls({ page: 1 }); + } else { + void this.fetchWorkflows({ page: 1 }); + } + + void this.fetchUploads({ page: 1 }); } else { if (changedProperties.has("sortCrawlsBy")) { - void this.fetchCrawls(); + if (this.showOnlyInCollection) { + void this.fetchCrawls({ page: 1 }); + } else { + void this.fetchWorkflows({ page: 1 }); + } } else if (changedProperties.has("filterCrawlsBy")) { - void this.fetchCrawls({ page: 1 }); + if (this.showOnlyInCollection) { + void this.fetchCrawls({ page: 1 }); + } else { + void this.fetchWorkflows({ page: 1 }); + } } if (changedProperties.has("sortUploadsBy")) { - void this.fetchUploads(); + void this.fetchUploads({ page: 1 }); } else if (changedProperties.has("filterUploadsBy")) { void this.fetchUploads({ page: 1 }); } @@ -287,7 +348,6 @@ export class CollectionItemsDialog extends BtrixElement { }; private readonly renderCrawls = () => { - const data = this.showOnlyInCollection ? this.crawls : this.workflows; return html`
@@ -309,22 +369,13 @@ export class CollectionItemsDialog extends BtrixElement { }} >
- -
- ${when( - data, - ({ total }) => - this.showOnlyInCollection - ? html`${msg("Crawled Items")} - ${this.localize.number(total)}` - : html`${msg("Crawl Workflows")} - ${this.localize.number(total)}`, - () => msg("Loading..."), - )} -
- + ${when( + !this.showOnlyInCollection, + () => + html` +

${msg("By Workflow")}

+ `, + )}
${cache( this.showOnlyInCollection @@ -360,22 +411,6 @@ export class CollectionItemsDialog extends BtrixElement { }} > - -
- ${when( - this.uploads, - () => - this.showOnlyInCollection - ? msg( - str`Uploads in Collection (${this.localize.number(this.uploads!.total)})`, - ) - : msg( - str`All Uploads (${this.localize.number(this.uploads!.total)})`, - ), - () => msg("Loading..."), - )} -
-
@@ -397,6 +432,7 @@ export class CollectionItemsDialog extends BtrixElement { this.uploads.total > this.uploads.pageSize, () => html` this.crawls.pageSize, () => html` , + ) => { + const { workflowId, page } = e.detail; + + const workflowCrawls = this.workflowCrawls.get(workflowId); + const workflowSelection = this.workflowSelection.get(workflowId); + if (!workflowCrawls || !workflowSelection) { + console.debug("no workflowCrawls or workflowSelection"); + return; + } + + const { paginatedCrawls, selectedCrawls } = workflowCrawls; + const nextPaginatedCrawls = await this.getCrawls({ + pageSize: + workflowCrawls.paginatedCrawls?.pageSize || DEFAULT_PAGE_SIZE, + cid: workflowId, + page, + }); + + nextPaginatedCrawls.items.forEach(({ id, cid }) => { + this.crawlToWorkflow.set(id, cid); + }); + + // Update selection if totals have changed + if ( + paginatedCrawls && + selectedCrawls && + nextPaginatedCrawls.total !== paginatedCrawls.total + ) { + this.workflowSelection.set(workflowId, { + ...workflowSelection, + checked: + nextPaginatedCrawls.total && selectedCrawls.total + ? selectedCrawls.total === nextPaginatedCrawls.total + ? true + : "indeterminate" + : false, + }); + } + + this.workflowCrawls.set(workflowId, { + selectedCrawls: workflowCrawls.selectedCrawls, + paginatedCrawls: nextPaginatedCrawls, + }); + this.workflowCrawls = new Map(this.workflowCrawls); + }} @btrix-selection-change=${(e: CustomEvent) => { - this.selection = { - ...this.selection, - ...e.detail.selection, - }; + const { workflowSelection } = e.detail; + + for (const [workflowId, selection] of workflowSelection) { + const savedSelection = + this.savedWorkflowSelection.get(workflowId); + + if (selection.checked === true) { + if (savedSelection?.checked === true) { + this.batchWorkflows.delete(workflowId); + } else { + // Create placeholder crawls for correct add/remove counts + const existingCrawls = new Set(); + + this.savedSelectedItems.forEach((v) => { + const id = unpackSymbolID(v); + if (this.crawlToWorkflow.get(id) === workflowId) { + existingCrawls.add(id); + } + }); + + const paginatedCrawls = + this.workflowCrawls.get(workflowId)?.paginatedCrawls; + const total = paginatedCrawls?.total || 0; + + const placeholderCrawlIds = Array.from({ + length: total - existingCrawls.size, + }).map( + (_, i) => + paginatedCrawls?.items[i]?.id || Symbol(workflowId), + ); + + this.batchWorkflows.set(workflowId, { + operation: "add", + omitCrawls: new Set(), + }); + + this.selectedItems = this.selectedItems.union( + new Set(placeholderCrawlIds), + ); + } + } else if (selection.checked === false) { + if (savedSelection?.checked) { + this.savedSelectedItems.forEach((v) => { + const id = unpackSymbolID(v); + if (this.crawlToWorkflow.get(id) === workflowId) { + this.selectedItems.delete(id); + } + }); + + // Remove all placeholders + this.selectedItems.forEach((v) => { + if (typeof v === "symbol") { + this.selectedItems.delete(v); + } + }); + + this.batchWorkflows.set(workflowId, { + operation: "remove", + omitCrawls: new Set(), + }); + } else { + this.batchWorkflows.delete(workflowId); + } + } + + if (selection.addCrawls || selection.removeCrawls) { + const addCrawls = selection.addCrawls || new Set(); + const removeCrawls = selection.removeCrawls || new Set(); + + this.selectedItems = this.selectedItems + .difference(removeCrawls) + .union(addCrawls); + + const batchWorkflow = this.batchWorkflows.get(workflowId); + + if (batchWorkflow) { + this.batchWorkflows.set(workflowId, { + operation: batchWorkflow.operation, + omitCrawls: + batchWorkflow.operation === "add" + ? batchWorkflow.omitCrawls + .difference(addCrawls) + .union(removeCrawls) + : batchWorkflow.omitCrawls + .difference(removeCrawls) + .union(addCrawls), + }); + } + } + + if ( + this.workflowSelection.get(workflowId)?.selectionCount !== + selection.selectionCount + ) { + this.workflowSelection.set(workflowId, { + checked: selection.checked, + selectionCount: selection.selectionCount, + }); + } + } + + this.workflowSelection = new Map(this.workflowSelection); + this.selectedItems = new Set(this.selectedItems); }} @btrix-auto-add-change=${(e: CustomEvent) => { const { id, checked, dedupe } = e.detail; @@ -496,11 +681,12 @@ export class CollectionItemsDialog extends BtrixElement { this.workflows.total > this.workflows.pageSize, () => html` { - void this.fetchCrawls({ + void this.fetchWorkflows({ page: e.detail.page, }); }} @@ -545,10 +731,37 @@ export class CollectionItemsDialog extends BtrixElement { showStatus ?checked=${isInCollection} @btrix-change=${(e: ArchivedItemCheckedEvent) => { - this.selection = { - ...this.selection, - [item.id]: e.detail.value.checked, - }; + const { checked } = e.detail.value; + + if (checked) { + this.selectedItems.add(item.id); + } else { + this.selectedItems.delete(item.id); + } + + if (isCrawl(item)) { + const workflowSelection = this.workflowSelection.get(item.cid); + const workflowCrawls = this.workflowCrawls.get(item.cid); + + if (workflowSelection && workflowCrawls) { + const selectionCount = checked + ? workflowSelection.selectionCount + 1 + : workflowSelection.selectionCount - 1; + this.workflowSelection.set(item.cid, { + checked: selectionCount + ? selectionCount === workflowCrawls.paginatedCrawls?.total + ? true + : "indeterminate" + : false, + selectionCount, + }); + } else { + console.debug("no workflowSelection or workflowCrawls"); + } + } + + this.selectedItems = new Set(this.selectedItems); + this.workflowSelection = new Map(this.workflowSelection); }} > @@ -556,9 +769,10 @@ export class CollectionItemsDialog extends BtrixElement { }; private readonly renderSave = () => { - const { add, remove } = this.difference; - const addCount = add.length; - const removeCount = remove.length; + const { addItems, removeItems } = this.difference; + + const addCount = addItems.size; + const removeCount = removeItems.size; const hasChange = addCount || removeCount; let selectionMessage = ""; @@ -583,7 +797,7 @@ export class CollectionItemsDialog extends BtrixElement { } return html` - ${selectionMessage} + ${selectionMessage} { - if (!Object.prototype.hasOwnProperty.call(selection, item.id)) { - selection[item.id] = true; + private get difference() { + const addItems = this.selectedItems.difference(this.savedSelectedItems); + const removeItems = this.savedSelectedItems.difference(this.selectedItems); + const addWorkflows = new Set(); + const removeWorkflows = new Set(); + + for (const [workflowId, { operation }] of this.batchWorkflows) { + if (operation === "add") { + addWorkflows.add(workflowId); + } else { + removeWorkflows.add(workflowId); } - }); - this.selection = selection; - } + } - private get difference() { - const itemIds = Object.entries(this.selection) - .filter(([, isSelected]) => isSelected) - .map(([id]) => id); - const add = difference(itemIds)(this.savedCollectionItemIDs); - const remove = difference(this.savedCollectionItemIDs)(itemIds); - return { add, remove }; + return { + addItems, + removeItems, + addWorkflows, + removeWorkflows, + }; } private async save() { await this.updateComplete; - const { add, remove } = this.difference; - const requests = []; - if (add.length) { - requests.push( - this.api.fetch( - `/orgs/${this.orgId}/collections/${this.collectionId}/add`, - { - method: "POST", - body: JSON.stringify({ crawlIds: add }), - }, - ), - ); - } - if (remove.length) { - requests.push( - this.api.fetch( - `/orgs/${this.orgId}/collections/${this.collectionId}/remove`, - { - method: "POST", - body: JSON.stringify({ crawlIds: remove }), - }, - ), - ); - } this.isSubmitting = true; try { - await Promise.all(requests); + const diff = this.difference; + + let omitFromBatchAdd = new Set(); + let omitFromBatchRemove = new Set(); + + const workflowRequests = []; + const itemRequests = []; + + if (diff.addWorkflows.size) { + diff.addWorkflows.forEach((id) => { + const batch = this.batchWorkflows.get(id); + + if (batch?.omitCrawls) { + omitFromBatchAdd = omitFromBatchAdd.union(batch.omitCrawls); + } + }); + + workflowRequests.push( + this.api.fetch( + `/orgs/${this.orgId}/collections/${this.collectionId}/add`, + { + method: "POST", + body: JSON.stringify({ + crawlconfigIds: [...diff.addWorkflows], + }), + }, + ), + ); + } + if (diff.removeWorkflows.size) { + diff.removeWorkflows.forEach((id) => { + const batch = this.batchWorkflows.get(id); + + if (batch?.omitCrawls) { + omitFromBatchRemove = omitFromBatchRemove.union(batch.omitCrawls); + } + }); + + workflowRequests.push( + this.api.fetch( + `/orgs/${this.orgId}/collections/${this.collectionId}/remove`, + { + method: "POST", + body: JSON.stringify({ + crawlconfigIds: [...diff.removeWorkflows], + }), + }, + ), + ); + } + + await Promise.all(workflowRequests); + + const addItems = Array.from( + diff.addItems.union(omitFromBatchRemove), + ).filter((v) => { + if (!isID(v)) return; + const workflowId = this.crawlToWorkflow.get(v); + if (workflowId) return !diff.addWorkflows.has(workflowId); + return true; + }); + const removeItems = Array.from( + diff.removeItems.union(omitFromBatchAdd), + ).filter((v) => { + if (!isID(v)) return; + const workflowId = this.crawlToWorkflow.get(v); + if (workflowId) return !diff.removeWorkflows.has(workflowId); + return true; + }); + + if (addItems.length) { + itemRequests.push( + this.api.fetch( + `/orgs/${this.orgId}/collections/${this.collectionId}/add`, + { + method: "POST", + body: JSON.stringify({ + crawlIds: addItems, + }), + }, + ), + ); + } + if (removeItems.length) { + itemRequests.push( + this.api.fetch( + `/orgs/${this.orgId}/collections/${this.collectionId}/remove`, + { + method: "POST", + body: JSON.stringify({ + crawlIds: removeItems, + }), + }, + ), + ); + } + + await Promise.all(itemRequests); this.close(); this.dispatchEvent(new CustomEvent("btrix-collection-saved")); @@ -702,33 +996,38 @@ export class CollectionItemsDialog extends BtrixElement { } private async initSelection() { + this.workflowCrawls = new Map(); + this.workflowSelection = new Map(); + this.savedWorkflowSelection = new Map(); + this.selectedItems.clear(); + this.savedSelectedItems.clear(); + + void this.fetchWorkflows({ + page: parsePage( + new URLSearchParams(location.search).get("workflowsPage"), + ), + pageSize: DEFAULT_PAGE_SIZE, + }); void this.fetchCrawls({ - page: parsePage(new URLSearchParams(location.search).get("page")), + page: parsePage(new URLSearchParams(location.search).get("crawlsPage")), pageSize: DEFAULT_PAGE_SIZE, }); void this.fetchUploads({ - page: parsePage(new URLSearchParams(location.search).get("page")), + page: parsePage(new URLSearchParams(location.search).get("crawlsPage")), pageSize: DEFAULT_PAGE_SIZE, }); void this.fetchSearchValues(); - const [crawls, uploads] = await Promise.all([ - this.getCrawls({ - page: parsePage(new URLSearchParams(location.search).get("page")), - pageSize: COLLECTION_ITEMS_MAX, - collectionId: this.collectionId, - }).then(({ items }) => items), - this.getUploads({ - page: parsePage(new URLSearchParams(location.search).get("page")), - pageSize: COLLECTION_ITEMS_MAX, - collectionId: this.collectionId, - }).then(({ items }) => items), - ]); + // FIXME Better handling of collections with more than 1,000 uploads + const { items } = await this.getUploads({ + pageSize: COLLECTION_ITEMS_MAX, + collectionId: this.collectionId, + }); + + items.forEach(({ id }) => this.selectedItems.add(id)); - const items = [...crawls, ...uploads]; - this.selectAllItems(items); - // Cache collection items to compare when saving - this.savedCollectionItemIDs = items.map(({ id }) => id); + this.savedSelectedItems = new Set(this.selectedItems); + this.selectedItems = new Set(this.selectedItems); } private async fetchCrawls(pageParams: APIPaginationQuery = {}) { @@ -740,14 +1039,11 @@ export class CollectionItemsDialog extends BtrixElement { collectionId: this.collectionId, sortBy: this.sortCrawlsBy.field, sortDirection: this.sortCrawlsBy.direction, - page: this.crawls?.page, - pageSize: this.crawls?.pageSize, + page: this.crawls?.page ?? 1, + pageSize: this.crawls?.pageSize ?? DEFAULT_PAGE_SIZE, ...pageParams, ...this.filterCrawlsBy, }); - if (!this.showOnlyInCollection) { - await this.fetchWorkflows(pageParams); - } } catch (e: unknown) { console.debug(e); } @@ -757,7 +1053,7 @@ export class CollectionItemsDialog extends BtrixElement { const userId = this.userInfo!.id; try { - this.workflows = await this.getWorkflows({ + const workflows = await this.getWorkflows({ userid: this.showOnlyMine ? userId : undefined, sortBy: // NOTE "finished" field doesn't exist in crawlconfigs, @@ -766,11 +1062,70 @@ export class CollectionItemsDialog extends BtrixElement { ? "lastRun" : this.sortCrawlsBy.field, sortDirection: this.sortCrawlsBy.direction, - page: this.workflows?.page, - pageSize: this.workflows?.pageSize, + page: this.workflows?.page ?? 1, + pageSize: this.workflows?.pageSize ?? DEFAULT_PAGE_SIZE, ...pageParams, ...this.filterCrawlsBy, }); + + await Promise.all( + workflows.items.map(async (workflow) => { + if (this.workflowCrawls.has(workflow.id)) return; + + // FIXME Better handling of collections with more than 1,000 + // crawls per workflow + const selectedCrawls = workflow.crawlSuccessfulCount + ? await this.getCrawls({ + pageSize: COLLECTION_ITEMS_MAX, + cid: workflow.id, + collectionId: this.collectionId, + }) + : null; + const paginatedCrawls = workflow.crawlSuccessfulCount + ? await this.getCrawls({ + pageSize: DEFAULT_PAGE_SIZE, + cid: workflow.id, + }) + : null; + + const selection: { + checked: boolean | "indeterminate"; + selectionCount: number; + } = { + checked: false, + selectionCount: selectedCrawls?.total || 0, + }; + + if (paginatedCrawls?.total && selectedCrawls?.total) { + if (selectedCrawls.total === paginatedCrawls.total) { + selection.checked = true; + } else { + selection.checked = "indeterminate"; + } + } + + selectedCrawls?.items.forEach(({ id, cid }) => { + this.crawlToWorkflow.set(id, cid); + this.selectedItems.add(id); + this.savedSelectedItems.add(id); + }); + + paginatedCrawls?.items.forEach(({ id, cid }) => { + this.crawlToWorkflow.set(id, cid); + }); + + this.workflowSelection.set(workflow.id, selection); + this.savedWorkflowSelection.set(workflow.id, selection); + this.workflowCrawls.set(workflow.id, { + selectedCrawls, + paginatedCrawls, + }); + }), + ); + + this.workflows = workflows; + this.workflowCrawls = new Map(this.workflowCrawls); + this.selectedItems = new Set(this.selectedItems); } catch (e: unknown) { console.debug(e); } @@ -813,6 +1168,7 @@ export class CollectionItemsDialog extends BtrixElement { params: { userid?: string; collectionId?: string; + cid?: string; firstSeed?: string; } & APIPaginationQuery & APISortQuery = {}, @@ -820,6 +1176,8 @@ export class CollectionItemsDialog extends BtrixElement { const query = queryString.stringify( { state: finishedCrawlStates, + sortBy: "started", + sortDirection: SortDirection.Descending, ...params, }, { diff --git a/frontend/src/features/collections/collection-workflow-list.ts b/frontend/src/features/collections/collection-workflow-list.ts index 26568467ac..6f740c89f4 100644 --- a/frontend/src/features/collections/collection-workflow-list.ts +++ b/frontend/src/features/collections/collection-workflow-list.ts @@ -1,29 +1,45 @@ import { localized, msg, str } from "@lit/localize"; -import type { SlTreeItem } from "@shoelace-style/shoelace"; -import { css, html, type PropertyValues, type TemplateResult } from "lit"; -import { customElement, property, queryAll, state } from "lit/decorators.js"; +import type { + SlSelectionChangeEvent, + SlTree, + SlTreeItem, +} from "@shoelace-style/shoelace"; +import { + css, + html, + nothing, + type PropertyValues, + type TemplateResult, +} from "lit"; +import { customElement, property, query, state } from "lit/decorators.js"; import { ifDefined } from "lit/directives/if-defined.js"; import { repeat } from "lit/directives/repeat.js"; -import { until } from "lit/directives/until.js"; import { when } from "lit/directives/when.js"; -import queryString from "query-string"; import { BtrixElement } from "@/classes/BtrixElement"; +import type { PageChangeEvent } from "@/components/ui/pagination"; import { dedupeStatusIcon } from "@/features/archived-items/templates/dedupe-status-icon"; import type { CollectionWorkflowListSettingChangeEvent } from "@/features/collections/collection-workflow-list/settings"; -import type { - APIPaginatedList, - APIPaginationQuery, - APISortQuery, -} from "@/types/api"; +import type { APIPaginatedList } from "@/types/api"; import type { Crawl, Workflow } from "@/types/crawler"; -import { finishedCrawlStates } from "@/utils/crawler"; import { pluralOf } from "@/utils/pluralize"; import "@/features/collections/collection-workflow-list/settings"; +export type CrawlsPageChangeDetail = { + workflowId: string; + page: number; +}; export type SelectionChangeDetail = { - selection: Record; + workflowSelection: Map< + string, + { + checked: boolean | "indeterminate"; + selectionCount: number; + addCrawls?: Set; + removeCrawls?: Set; + } + >; }; export type AutoAddChangeDetail = { id: string; @@ -31,10 +47,9 @@ export type AutoAddChangeDetail = { dedupe?: boolean; }; -const CRAWLS_PAGE_SIZE = 50; - /** * @fires btrix-selection-change + * @fires btrix-crawls-page-change * @fires btrix-auto-add-change */ @customElement("btrix-collection-workflow-list") @@ -49,6 +64,10 @@ export class CollectionWorkflowList extends BtrixElement { min-width: 0; } + sl-tree-item:not([disabled])::part(item):hover { + background-color: var(--sl-color-neutral-50); + } + sl-tree-item::part(expand-button) { /* Move expand button to end */ order: 2; @@ -83,7 +102,7 @@ export class CollectionWorkflowList extends BtrixElement { sl-tree-item::part(item--disabled) { opacity: 1; } - sl-tree-item.workflow:not(.selectable)::part(checkbox) { + sl-tree-item[disabled]::part(checkbox) { opacity: 0; } @@ -118,26 +137,44 @@ export class CollectionWorkflowList extends BtrixElement { @property({ type: String }) collectionId?: string; - @property({ type: Array }) + @property({ type: Array, attribute: false }) workflows: Workflow[] = []; - @state() - expandWorkflowSettings = false; - /** - * Whether item is selected or not, keyed by ID + * Selection state for individual archived items */ - @property({ type: Object }) - selection: { [itemID: string]: boolean } = {}; + @property({ attribute: false }) + selectedItems = new Set(); - @queryAll(".crawl") - private readonly crawlItems?: NodeListOf; + /** + * Selection state for workflows + */ + @property({ attribute: false }) + workflowSelection = new Map< + string, + { + checked: boolean | "indeterminate"; + selectionCount: number; + } + >(); - private readonly crawlsMap = new Map< + @property({ attribute: false }) + workflowCrawls = new Map< /* workflow ID: */ string, - Promise + { + selectedCrawls: APIPaginatedList | null; + paginatedCrawls: APIPaginatedList | null; + } >(); + @state() + expandWorkflowSettings = false; + + @query("sl-tree") + private readonly tree?: SlTree | null; + + private previousSelection = new Set(); + protected willUpdate(changedProperties: PropertyValues): void { if (changedProperties.has("workflows")) { if (this.collectionId) { @@ -146,15 +183,31 @@ export class CollectionWorkflowList extends BtrixElement { workflow.autoAddCollections.some((id) => id === collId), ); } + } + } - void this.fetchCrawls(); + protected updated(changedProperties: PropertyValues): void { + if (changedProperties.has("workflowCrawls")) { + void this.setPreviousSelection(); } } + private async setPreviousSelection() { + if (!this.tree) { + console.debug("no this.tree"); + return; + } + + await this.tree.updateComplete; + + this.previousSelection = new Set(this.tree.selectedItems); + } + render() { return html` { if ((e.target as HTMLElement).tagName !== "SL-TREE-ITEM") { // Prevent sl-tree from switching focusing @@ -162,23 +215,6 @@ export class CollectionWorkflowList extends BtrixElement { e.preventDefault(); } }} - @sl-selection-change=${(e: CustomEvent<{ selection: SlTreeItem[] }>) => { - if (!this.crawlItems) { - console.debug("no crawl items with classname `crawl`"); - return; - } - e.stopPropagation(); - const selection: CollectionWorkflowList["selection"] = {}; - Array.from(this.crawlItems).forEach((item) => { - if (!item.dataset.crawlId) return; - selection[item.dataset.crawlId] = item.selected; - }); - this.dispatchEvent( - new CustomEvent("btrix-selection-change", { - detail: { selection }, - }), - ); - }} > @@ -187,67 +223,51 @@ export class CollectionWorkflowList extends BtrixElement { } private readonly renderWorkflow = (workflow: Workflow) => { - const crawlsAsync = this.crawlsMap.get(workflow.id) || Promise.resolve([]); - const countAsync = crawlsAsync.then((crawls) => ({ - total: crawls.length, - selected: crawls.filter(({ id }) => this.selection[id]).length, - })); + const total = workflow.crawlSuccessfulCount; + const selection = this.workflowSelection.get(workflow.id); + const crawls = this.workflowCrawls.get(workflow.id); + const paginatedCrawls = crawls?.paginatedCrawls; + const allSelected = selection?.checked === true; return html` selected > 0 && selected === total, - ), - false, - )} - .indeterminate=${ - // NOTE `indeterminate` is not a documented public property, - // we're manually setting it since async child tree-items - // doesn't work as of shoelace 2.8.0 - until( - countAsync.then( - ({ total, selected }) => selected > 0 && selected < total, + class="workflow !mt-0" + data-workflow-id=${workflow.id} + ?selected=${allSelected} + .indeterminate=${selection?.checked === "indeterminate"} + ?disabled=${!total || !crawls} + @sl-after-collapse=${() => { + // Reset crawls page + this.dispatchEvent( + new CustomEvent( + "btrix-crawls-page-change", + { + detail: { workflowId: workflow.id, page: 1 }, + }, ), - false, - ) - } - ?disabled=${until( - countAsync.then(({ total }) => total === 0), - true, - )} + ); + }} @click=${(e: MouseEvent) => { - void countAsync.then(({ total }) => { - if (!total) { - // Prevent selection since we're just allowing auto-add - e.stopPropagation(); - } - }); + if ((e.currentTarget as SlTreeItem).disabled) { + e.stopPropagation(); + } }} >
-
${this.renderName(workflow)}
+
+ ${this.renderName(workflow)} +
- ${until( - countAsync.then( - ({ total, selected }) => - html`${total - ? `${this.localize.number(selected)} / ${this.localize.number(total)}` - : 0} - ${pluralOf("crawls", total)}`, - ), - )} + ${this.renderSelectionMessage(workflow)}
- ${until(crawlsAsync.then((crawls) => crawls.map(this.renderCrawl)))} + ${when(paginatedCrawls, (crawls) => + this.renderCrawls(workflow, crawls), + )} { + const selectionCount = + this.workflowSelection.get(workflow.id)?.selectionCount || 0; + const total = workflow.crawlSuccessfulCount; + const number_of_total_items = this.localize.number(total); + const plural_of_total_items = pluralOf("items", total); + + if (!total) { + return `${number_of_total_items} ${plural_of_total_items}`; + } + + return html` + ${this.localize.number(selectionCount)} / ${number_of_total_items} + ${plural_of_total_items} + `; + }; + + private readonly renderCrawls = ( + workflow: Workflow, + res: APIPaginatedList | null, + ) => { + if (!res?.items.length) return nothing; + + let pagination: TemplateResult | typeof nothing = nothing; + const selection = this.workflowSelection.get(workflow.id); + const crawls = this.workflowCrawls.get(workflow.id); + const paginatedCrawlIds = new Set( + crawls?.paginatedCrawls?.items.map(({ id }) => id) || [], + ); + const hiddenSelection = + selection?.selectionCount && + crawls?.selectedCrawls?.items.some( + ({ id }) => !paginatedCrawlIds.has(id), + ); + + if (res.total > res.pageSize) { + // Include in tree selection so that workflow tree item correctly displays + // as indeterminate, but prevent user selection + pagination = html` e.stopPropagation()} + > + { + this.dispatchEvent( + new CustomEvent( + "btrix-crawls-page-change", + { + detail: { workflowId: workflow.id, page: e.detail.page }, + }, + ), + ); + }} + > + + `; + } + + return html`${repeat( + res.items, + ({ id }) => id, + this.renderCrawl, + )}${pagination}`; + }; + private readonly renderCrawl = (crawl: Crawl) => { const pageCount = +(crawl.stats?.done || 0); + const selection = this.workflowSelection.get(crawl.cid); + const selected = + selection?.checked === "indeterminate" + ? this.selectedItems.has(crawl.id) + : selection?.checked; + return html` { + const el = e.currentTarget as SlTreeItem; + if (el.disabled) { + e.stopPropagation(); + return; + } + + const pagination = el + .closest(".workflow") + ?.querySelector(".pagination"); + const workflowSelection = this.workflowSelection.get(crawl.cid); + const workflow = this.workflows.find(({ id }) => id === crawl.cid); + + if (pagination && workflowSelection && workflow) { + // HACK Render parent tree item (i.e. workflow) as indeterminate + // by making invisible checkbox the opposite of current checkbox + if ( + (el.selected && workflowSelection.selectionCount - 1) || + workflowSelection.selectionCount + 1 < + workflow.crawlSuccessfulCount + ) { + if (workflowSelection.selectionCount - 1) { + pagination.selected = el.selected; + } else { + // Select none + pagination.selected = false; + } + } else { + if ( + !el.selected && + workflowSelection.selectionCount + 1 === + workflow.crawlSuccessfulCount + ) { + // Select all + pagination.selected = true; + } + } + } + }} >
@@ -317,48 +454,85 @@ export class CollectionWorkflowList extends BtrixElement { `; }; - /** - * Get crawls for each workflow in list - */ - private async fetchCrawls() { - try { - this.workflows.forEach((workflow) => { - this.crawlsMap.set( - workflow.id, - this.getCrawls({ cid: workflow.id, pageSize: CRAWLS_PAGE_SIZE }).then( - ({ items }) => items, - ), - ); + private readonly onSelectionChange = async (e: SlSelectionChangeEvent) => { + e.stopPropagation(); + + const workflows = this.tree?.querySelectorAll(".workflow"); + + const workflowSelection: SelectionChangeDetail["workflowSelection"] = + new Map(); + + const itemChanged = (item: SlTreeItem) => { + return this.previousSelection.has(item) !== item.selected; + }; + + workflows?.forEach((el) => { + const workflowId = el.dataset.workflowId; + if (!workflowId) { + console.debug("no workflowId"); + return; + } + + const addCrawls = new Set(); + const removeCrawls = new Set(); + + const crawlEls = el.getChildrenItems({ includeDisabled: false }); + let selectionCount = + this.workflowSelection.get(workflowId)?.selectionCount || 0; + + crawlEls.forEach((el) => { + const crawlId = el.dataset.crawlId; + if (!crawlId) { + console.debug("no crawlId"); + return; + } + + if (itemChanged(el)) { + if (el.selected) { + selectionCount += 1; + addCrawls.add(crawlId); + } else { + selectionCount = selectionCount ? selectionCount - 1 : 0; + removeCrawls.add(crawlId); + } + } }); - } catch (e: unknown) { - console.debug(e); - } - } - private async getCrawls( - params: Partial<{ - cid: string; - }> & - APIPaginationQuery & - APISortQuery, - ) { - if (!this.orgId) throw new Error("Missing attribute `orgId`"); - - const query = queryString.stringify( - { - state: finishedCrawlStates, - ...params, - }, - { - arrayFormat: "comma", - }, - ); - const data = await this.api.fetch>( - `/orgs/${this.orgId}/crawls?${query}`, + if (el.selected) { + workflowSelection.set(workflowId, { + checked: true, + selectionCount: + this.workflowCrawls.get(workflowId)?.paginatedCrawls?.total || 0, + addCrawls, + removeCrawls, + }); + } else if (el.indeterminate) { + workflowSelection.set(workflowId, { + checked: "indeterminate", + selectionCount, + addCrawls, + removeCrawls, + }); + } else { + workflowSelection.set(workflowId, { + checked: false, + selectionCount: 0, + addCrawls, + removeCrawls, + }); + } + }); + + this.dispatchEvent( + new CustomEvent("btrix-selection-change", { + detail: { + workflowSelection, + }, + }), ); - return data; - } + this.previousSelection = new Set(e.detail.selection); + }; // TODO consolidate collections/workflow name private readonly renderName = (workflow: Workflow) => { diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json index 7fe4c3be99..ed2d557cfd 100644 --- a/frontend/tsconfig.json +++ b/frontend/tsconfig.json @@ -30,7 +30,13 @@ "@/*": ["./src/*"], "~assets/*": ["./src/assets/*"], }, - "lib": ["DOM", "DOM.Iterable", "ES2021.WeakRef", "ES2021.Intl"], + "lib": [ + "DOM", + "DOM.Iterable", + "ES2021.WeakRef", + "ES2021.Intl", + "esnext.collection", + ], }, "include": ["**/*.ts"], "exclude": ["node_modules"],