From 56e3a4df65071d2fd5ea6f7c2df4121e6699c812 Mon Sep 17 00:00:00 2001 From: dcgoodwin2112 Date: Wed, 2 Sep 2026 16:41:50 -0400 Subject: [PATCH 1/2] Add dkan_datastore_preview datastore submodule --- .ddev/commands/web/dkan-site-install | 2 +- .../css/dkan-datastore-preview.css | 65 ++++ .../dkan_datastore_preview.info.yml | 9 + .../dkan_datastore_preview.install | 25 ++ .../dkan_datastore_preview.libraries.yml | 4 + .../dkan_datastore_preview.module | 150 +++++++ .../dkan_datastore_preview.services.yml | 20 + .../src/DataSource/DataSourceInterface.php | 65 ++++ .../src/DataSource/DataSourceResult.php | 23 ++ .../src/DataSource/DatabaseDataSource.php | 157 ++++++++ .../src/Element/DataPreview.php | 115 ++++++ .../src/Service/DataPreviewBuilder.php | 368 ++++++++++++++++++ .../src/Service/ImportStatusMessage.php | 91 +++++ .../dkan-datastore-preview.html.twig | 39 ++ .../tests/data/preview_sample.csv | 31 ++ .../src/Kernel/PreviewIntegrationTest.php | 184 +++++++++ .../DataSource/DatabaseDataSourceTest.php | 184 +++++++++ .../Unit/Element/DataPreviewElementTest.php | 195 ++++++++++ .../Unit/Service/DataPreviewBuilderTest.php | 316 +++++++++++++++ .../Unit/Service/ImportStatusMessageTest.php | 105 +++++ .../templates/node--data.html.twig | 1 + phpunit.xml | 1 + 22 files changed, 2149 insertions(+), 1 deletion(-) create mode 100644 modules/dkan_datastore/modules/dkan_datastore_preview/css/dkan-datastore-preview.css create mode 100644 modules/dkan_datastore/modules/dkan_datastore_preview/dkan_datastore_preview.info.yml create mode 100644 modules/dkan_datastore/modules/dkan_datastore_preview/dkan_datastore_preview.install create mode 100644 modules/dkan_datastore/modules/dkan_datastore_preview/dkan_datastore_preview.libraries.yml create mode 100644 modules/dkan_datastore/modules/dkan_datastore_preview/dkan_datastore_preview.module create mode 100644 modules/dkan_datastore/modules/dkan_datastore_preview/dkan_datastore_preview.services.yml create mode 100644 modules/dkan_datastore/modules/dkan_datastore_preview/src/DataSource/DataSourceInterface.php create mode 100644 modules/dkan_datastore/modules/dkan_datastore_preview/src/DataSource/DataSourceResult.php create mode 100644 modules/dkan_datastore/modules/dkan_datastore_preview/src/DataSource/DatabaseDataSource.php create mode 100644 modules/dkan_datastore/modules/dkan_datastore_preview/src/Element/DataPreview.php create mode 100644 modules/dkan_datastore/modules/dkan_datastore_preview/src/Service/DataPreviewBuilder.php create mode 100644 modules/dkan_datastore/modules/dkan_datastore_preview/src/Service/ImportStatusMessage.php create mode 100644 modules/dkan_datastore/modules/dkan_datastore_preview/templates/dkan-datastore-preview.html.twig create mode 100644 modules/dkan_datastore/modules/dkan_datastore_preview/tests/data/preview_sample.csv create mode 100644 modules/dkan_datastore/modules/dkan_datastore_preview/tests/src/Kernel/PreviewIntegrationTest.php create mode 100644 modules/dkan_datastore/modules/dkan_datastore_preview/tests/src/Unit/DataSource/DatabaseDataSourceTest.php create mode 100644 modules/dkan_datastore/modules/dkan_datastore_preview/tests/src/Unit/Element/DataPreviewElementTest.php create mode 100644 modules/dkan_datastore/modules/dkan_datastore_preview/tests/src/Unit/Service/DataPreviewBuilderTest.php create mode 100644 modules/dkan_datastore/modules/dkan_datastore_preview/tests/src/Unit/Service/ImportStatusMessageTest.php diff --git a/.ddev/commands/web/dkan-site-install b/.ddev/commands/web/dkan-site-install index 1c8f875a89..821a732a43 100755 --- a/.ddev/commands/web/dkan-site-install +++ b/.ddev/commands/web/dkan-site-install @@ -7,7 +7,7 @@ set -eu -o pipefail drush site:install minimal --site-name="DKAN" --account-name="admin" -y -drush pm-enable dkan dkan_harvest dkan_datastore automated_cron admin_toolbar admin_toolbar_tools -y +drush pm-enable dkan dkan_harvest dkan_datastore dkan_datastore_preview automated_cron admin_toolbar admin_toolbar_tools -y # Create administrator role and assign to user 1 drush role:create "administrator" "Administrator" -y diff --git a/modules/dkan_datastore/modules/dkan_datastore_preview/css/dkan-datastore-preview.css b/modules/dkan_datastore/modules/dkan_datastore_preview/css/dkan-datastore-preview.css new file mode 100644 index 0000000000..f4355f8158 --- /dev/null +++ b/modules/dkan_datastore/modules/dkan_datastore_preview/css/dkan-datastore-preview.css @@ -0,0 +1,65 @@ +.dkan-datastore-preview { + margin: 1em 0; +} + +.dkan-datastore-preview__controls { + display: flex; + justify-content: space-between; + align-items: center; + flex-wrap: wrap; + gap: 0.5em; + margin-bottom: 0.75em; +} + +.dkan-datastore-preview__page-size { + display: inline-flex; + align-items: center; + gap: 0.3em; +} + +.dkan-datastore-preview__summary { + font-size: 0.9em; +} + +.dkan-datastore-preview__table-wrapper { + overflow-x: auto; +} + +.dkan-datastore-preview__table { + width: 100%; + border-collapse: collapse; +} + +.dkan-datastore-preview__table thead th { + background-color: #f5f5f5; + font-weight: 600; + text-align: left; + padding: 0.75em 1em; + border-bottom: 2px solid #ccc; + white-space: nowrap; +} + +.dkan-datastore-preview__table thead th a { + display: inline-flex; + align-items: center; + gap: 0.25em; +} + +.dkan-datastore-preview__table tbody td { + padding: 0.5em 1em; + border-bottom: 1px solid #e0e0e0; +} + +.dkan-datastore-preview__table tbody tr:hover { + background-color: #fafafa; +} + +.dkan-datastore-preview__footer { + margin-top: 0.75em; +} + +.dkan-datastore-preview__message { + padding: 0.75em 1em; + background-color: #f5f5f5; + border: 1px solid #e0e0e0; +} diff --git a/modules/dkan_datastore/modules/dkan_datastore_preview/dkan_datastore_preview.info.yml b/modules/dkan_datastore/modules/dkan_datastore_preview/dkan_datastore_preview.info.yml new file mode 100644 index 0000000000..9e9789ae6f --- /dev/null +++ b/modules/dkan_datastore/modules/dkan_datastore_preview/dkan_datastore_preview.info.yml @@ -0,0 +1,9 @@ +name: DKAN Datastore Preview +description: Paginated, sortable HTML table previews of datastore data on dataset pages. +type: module +core_version_requirement: ^10.4 || ^11 +package: DKAN +dependencies: + - dkan:dkan_common + - dkan:dkan_datastore + - dkan:dkan_metastore diff --git a/modules/dkan_datastore/modules/dkan_datastore_preview/dkan_datastore_preview.install b/modules/dkan_datastore/modules/dkan_datastore_preview/dkan_datastore_preview.install new file mode 100644 index 0000000000..d8039b292e --- /dev/null +++ b/modules/dkan_datastore/modules/dkan_datastore_preview/dkan_datastore_preview.install @@ -0,0 +1,25 @@ +getStorage('entity_view_display') + ->load('node.data.default'); + + if ($display) { + $display->setComponent('data_preview', [ + 'weight' => 100, + ]); + $display->save(); + } +} diff --git a/modules/dkan_datastore/modules/dkan_datastore_preview/dkan_datastore_preview.libraries.yml b/modules/dkan_datastore/modules/dkan_datastore_preview/dkan_datastore_preview.libraries.yml new file mode 100644 index 0000000000..f2ef5cce78 --- /dev/null +++ b/modules/dkan_datastore/modules/dkan_datastore_preview/dkan_datastore_preview.libraries.yml @@ -0,0 +1,4 @@ +preview: + css: + component: + css/dkan-datastore-preview.css: {} diff --git a/modules/dkan_datastore/modules/dkan_datastore_preview/dkan_datastore_preview.module b/modules/dkan_datastore/modules/dkan_datastore_preview/dkan_datastore_preview.module new file mode 100644 index 0000000000..9165c46e13 --- /dev/null +++ b/modules/dkan_datastore/modules/dkan_datastore_preview/dkan_datastore_preview.module @@ -0,0 +1,150 @@ + [ + 'variables' => [ + 'table' => NULL, + 'pager' => NULL, + 'page_size_form' => NULL, + 'result_summary' => NULL, + ], + 'template' => 'dkan-datastore-preview', + ], + ]; +} + +/** + * Implements hook_entity_extra_field_info(). + */ +function dkan_datastore_preview_entity_extra_field_info() { + $extra = []; + if (\Drupal::entityTypeManager()->getStorage('node_type')->load('data')) { + $extra['node']['data']['display']['data_preview'] = [ + 'label' => t('Data Preview'), + 'description' => t('Paginated preview tables for tabular distributions.'), + 'weight' => 100, + 'visible' => TRUE, + ]; + } + return $extra; +} + +/** + * Implements hook_ENTITY_TYPE_view() for node entities. + * + * Adds a data preview table for each importable distribution on dataset + * nodes. Rendered on the server-side dataset page via the data_preview + * extra field printed by dkan_metastore's node--data template. + */ +function dkan_datastore_preview_node_view(array &$build, EntityInterface $entity, EntityViewDisplayInterface $display, $view_mode) { + if (!$entity instanceof NodeInterface) { + return; + } + if ($entity->bundle() !== 'data') { + return; + } + if (($entity->get('field_data_type')->value ?? '') !== 'dataset') { + return; + } + if (!$display->getComponent('data_preview')) { + return; + } + + // During node preview of a new (unsaved) node, datastore tables don't exist. + if (!empty($entity->in_preview) && $entity->isNew()) { + $build['data_preview'] = [ + '#markup' => '

' . t('Data preview is available after the dataset is saved and data is imported.') . '

', + ]; + return; + } + + /** @var \Drupal\dkan_common\DatasetInfo $datasetInfo */ + $datasetInfo = \Drupal::service('dkan.common.dataset_info'); + $info = $datasetInfo->gather($entity->uuid()); + + $revision = $info['published_revision'] ?? $info['latest_revision'] ?? NULL; + if (!$revision || empty($revision['distributions'])) { + return; + } + + $previews = []; + $index = 0; + foreach ($revision['distributions'] as $dist) { + if (!is_array($dist) || empty($dist['resource_id']) || empty($dist['resource_version'])) { + continue; + } + if (!in_array($dist['mime_type'] ?? '', DataResource::IMPORTABLE_FILE_TYPES)) { + continue; + } + + $resourceId = $dist['resource_id'] . '__' . $dist['resource_version']; + + $preview = [ + '#type' => 'container', + '#attributes' => ['class' => ['dkan-datastore-preview-distribution']], + '#cache' => [ + 'tags' => _dkan_datastore_preview_distribution_cache_tags($dist['distribution_uuid'] ?? ''), + ], + ]; + + // Caption the table with the source filename. + $label = ''; + if (!empty($dist['source_path'])) { + $label = basename(parse_url($dist['source_path'], PHP_URL_PATH) ?: $dist['source_path']); + } + + $preview['table'] = [ + '#type' => 'dkan_datastore_preview', + '#resource_id' => $resourceId, + '#pager_element' => $index, + '#query_prefix' => 'dp' . $index . '_', + '#caption' => $label ? t('Preview: @label', ['@label' => $label]) : NULL, + ]; + $previews[] = $preview; + $index++; + } + + if ($previews) { + $build['data_preview'] = $previews; + } +} + +/** + * Get the cache tags for a distribution metastore item. + * + * The datastore invalidates the distribution node's cache tags after an + * import completes (via PostImport), so attaching them here flips a "not yet + * available" message to the data table without a manual cache rebuild. + * + * @param string $distribution_uuid + * The distribution's UUID. + * + * @return array + * Cache tags, or an empty array when the item cannot be loaded. + */ +function _dkan_datastore_preview_distribution_cache_tags(string $distribution_uuid): array { + if ($distribution_uuid === '') { + return []; + } + try { + $item = \Drupal::service('dkan.metastore.metastore_item_factory')->getInstance($distribution_uuid); + return $item->getCacheTags(); + } + catch (\Throwable) { + return []; + } +} diff --git a/modules/dkan_datastore/modules/dkan_datastore_preview/dkan_datastore_preview.services.yml b/modules/dkan_datastore/modules/dkan_datastore_preview/dkan_datastore_preview.services.yml new file mode 100644 index 0000000000..08b06d0a00 --- /dev/null +++ b/modules/dkan_datastore/modules/dkan_datastore_preview/dkan_datastore_preview.services.yml @@ -0,0 +1,20 @@ +services: + dkan.datastore_preview.builder: + class: \Drupal\dkan_datastore_preview\Service\DataPreviewBuilder + arguments: + - '@request_stack' + - '@pager.manager' + Drupal\dkan_datastore_preview\Service\DataPreviewBuilder: '@dkan.datastore_preview.builder' + + dkan.datastore_preview.data_source.database: + class: \Drupal\dkan_datastore_preview\DataSource\DatabaseDataSource + arguments: + - '@dkan.datastore.service' + Drupal\dkan_datastore_preview\DataSource\DatabaseDataSource: '@dkan.datastore_preview.data_source.database' + + dkan.datastore_preview.import_status_message: + class: \Drupal\dkan_datastore_preview\Service\ImportStatusMessage + arguments: + - '@dkan.datastore.import_info' + - '@dkan.metastore.resource_mapper' + Drupal\dkan_datastore_preview\Service\ImportStatusMessage: '@dkan.datastore_preview.import_status_message' diff --git a/modules/dkan_datastore/modules/dkan_datastore_preview/src/DataSource/DataSourceInterface.php b/modules/dkan_datastore/modules/dkan_datastore_preview/src/DataSource/DataSourceInterface.php new file mode 100644 index 0000000000..61f9190ffe --- /dev/null +++ b/modules/dkan_datastore/modules/dkan_datastore_preview/src/DataSource/DataSourceInterface.php @@ -0,0 +1,65 @@ +getStorage($resource_id); + if (!$storage) { + return []; + } + $schema = $storage->getSchema(); + if (empty($schema['fields'])) { + return []; + } + unset($schema['fields'][self::HIDDEN_FIELD]); + return $schema; + } + + /** + * {@inheritdoc} + */ + public function fetchData( + string $resource_id, + int $limit, + int $offset, + ?string $sort_field, + string $sort_direction, + array $conditions = [], + array $properties = [], + ): DataSourceResult { + $storage = $this->getStorage($resource_id); + if (!$storage) { + return new DataSourceResult([], 0); + } + + $query = $this->buildQuery($limit, $offset, $sort_field, $sort_direction, $conditions, $properties); + $rows = $storage->query($query); + + $countQuery = $this->buildQuery(0, 0, NULL, 'asc', $conditions, []); + $countQuery->count(); + $countResult = $storage->query($countQuery); + $totalCount = (int) ($countResult[0]->expression ?? 0); + + return new DataSourceResult($rows, $totalCount); + } + + /** + * Get the datastore storage for a resource id, or NULL if none exists. + * + * @param string $resource_id + * Resource id, optionally in "identifier__version" format. + * + * @return \Drupal\dkan_datastore\Storage\DatabaseTable|null + * The storage, or NULL when the resource has no datastore table. + */ + protected function getStorage(string $resource_id) { + if (!array_key_exists($resource_id, $this->storages)) { + [$identifier, $version] = $this->parseResourceId($resource_id); + try { + $this->storages[$resource_id] = $this->datastoreService->getStorage($identifier, $version); + } + catch (\InvalidArgumentException) { + $this->storages[$resource_id] = NULL; + } + } + return $this->storages[$resource_id]; + } + + /** + * Parse a resource id into identifier and version components. + * + * @param string $resource_id + * Resource id, optionally in "identifier__version" format. + * + * @return array + * [identifier, version|null]. + */ + protected function parseResourceId(string $resource_id): array { + if (str_contains($resource_id, '__')) { + $parts = explode('__', $resource_id, 2); + return [$parts[0], $parts[1]]; + } + return [$resource_id, NULL]; + } + + /** + * Build a DKAN Query object. + */ + protected function buildQuery( + int $limit, + int $offset, + ?string $sort_field, + string $sort_direction, + array $conditions, + array $properties, + ): Query { + $query = new Query(); + + if ($limit > 0) { + $query->limitTo($limit); + } + if ($offset > 0) { + $query->offsetBy($offset); + } + + if ($sort_field) { + if (strtolower($sort_direction) === 'desc') { + $query->sortByDescending($sort_field); + } + else { + $query->sortByAscending($sort_field); + } + } + + foreach ($conditions as $condition) { + $query->conditions[] = (object) [ + 'property' => $condition['property'], + 'value' => $condition['value'], + 'operator' => $condition['operator'] ?? '=', + ]; + } + + foreach ($properties as $property) { + $query->filterByProperty($property); + } + + return $query; + } + +} diff --git a/modules/dkan_datastore/modules/dkan_datastore_preview/src/Element/DataPreview.php b/modules/dkan_datastore/modules/dkan_datastore_preview/src/Element/DataPreview.php new file mode 100644 index 0000000000..6af54f0b38 --- /dev/null +++ b/modules/dkan_datastore/modules/dkan_datastore_preview/src/Element/DataPreview.php @@ -0,0 +1,115 @@ + 'dkan_datastore_preview', + * '#resource_id' => $resource_id, + * '#pager_element' => 0, + * '#query_prefix' => 'dp0_', + * ]; + * @endcode + * + * When several previews are rendered on one page, each must get a unique + * #pager_element and #query_prefix so sorting and paging one table does not + * affect the others. + */ +#[RenderElement('dkan_datastore_preview')] +class DataPreview extends RenderElementBase { + + /** + * {@inheritdoc} + */ + public function getInfo() { + $class = static::class; + return [ + '#resource_id' => '', + '#data_source_instance' => NULL, + '#columns' => [], + '#default_page_size' => 25, + '#default_sort' => NULL, + '#default_sort_direction' => 'asc', + '#conditions' => [], + '#pager_element' => 0, + '#query_prefix' => '', + '#caption' => NULL, + '#pre_render' => [ + [$class, 'preRender'], + ], + ]; + } + + /** + * Pre-render callback: builds the data preview table. + */ + public static function preRender(array $element): array { + if (empty($element['#resource_id'])) { + return $element; + } + + $dataSource = $element['#data_source_instance'] ?? NULL; + if (!$dataSource instanceof DataSourceInterface) { + $dataSource = \Drupal::service('dkan.datastore_preview.data_source.database'); + } + + // Accept columns as a CSV string or an array. + $columns = $element['#columns']; + if (is_string($columns) && $columns !== '') { + $columns = array_filter(array_map('trim', explode(',', $columns))); + } + elseif (!is_array($columns)) { + $columns = []; + } + + $options = [ + 'columns' => $columns, + 'default_page_size' => (int) $element['#default_page_size'], + 'default_sort' => $element['#default_sort'] ?: NULL, + 'default_sort_direction' => $element['#default_sort_direction'], + 'conditions' => $element['#conditions'], + 'pager_element' => (int) $element['#pager_element'], + 'query_prefix' => (string) $element['#query_prefix'], + 'caption' => $element['#caption'] ?? NULL, + ]; + + try { + /** @var \Drupal\dkan_datastore_preview\Service\DataPreviewBuilder $builder */ + $builder = \Drupal::service('dkan.datastore_preview.builder'); + $buildResult = $builder->build($dataSource, $element['#resource_id'], $options); + + if ($buildResult === NULL) { + $element['message'] = static::statusMessage($element['#resource_id']); + return $element; + } + $element = $buildResult + $element; + } + catch (\Throwable $e) { + \Drupal::logger('dkan_datastore_preview')->warning('Data preview error for resource @id: @message', [ + '@id' => $element['#resource_id'], + '@message' => $e->getMessage(), + ]); + $element['message'] = static::statusMessage($element['#resource_id']); + } + + return $element; + } + + /** + * Build the import status message for an unavailable preview. + */ + protected static function statusMessage(string $resource_id): array { + /** @var \Drupal\dkan_datastore_preview\Service\ImportStatusMessage $statusMessage */ + $statusMessage = \Drupal::service('dkan.datastore_preview.import_status_message'); + return $statusMessage->build($resource_id); + } + +} diff --git a/modules/dkan_datastore/modules/dkan_datastore_preview/src/Service/DataPreviewBuilder.php b/modules/dkan_datastore/modules/dkan_datastore_preview/src/Service/DataPreviewBuilder.php new file mode 100644 index 0000000000..07f9f25549 --- /dev/null +++ b/modules/dkan_datastore/modules/dkan_datastore_preview/src/Service/DataPreviewBuilder.php @@ -0,0 +1,368 @@ + 'dkan_datastore_preview', or NULL when the + * resource has no queryable table yet. + */ + public function build(DataSourceInterface $dataSource, string $resource_id, array $options = []): ?array { + $options += [ + 'columns' => [], + 'page_sizes' => self::DEFAULT_PAGE_SIZES, + 'default_page_size' => 25, + 'default_sort' => NULL, + 'default_sort_direction' => 'asc', + 'conditions' => [], + 'pager_element' => 0, + 'query_prefix' => '', + 'caption' => NULL, + ]; + + $schema = $dataSource->getSchema($resource_id); + if (empty($schema['fields'])) { + return NULL; + } + + $request = $this->requestStack->getCurrentRequest() ?? Request::create('/'); + $prefix = $options['query_prefix']; + + $pageSizeParam = $prefix . 'page_size'; + $pageSize = (int) $request->query->get($pageSizeParam, $options['default_page_size']); + if (!in_array($pageSize, $options['page_sizes'])) { + $pageSize = $options['default_page_size']; + } + + $displayFields = $this->getDisplayFields($schema, $options['columns']); + $sort = $this->getSortContext($displayFields, $options, $request); + + $currentPage = $this->getCurrentPage($request, $options['pager_element']); + $offset = $currentPage * $pageSize; + + $result = $dataSource->fetchData( + $resource_id, + $pageSize, + $offset, + $sort['field'], + $sort['direction'], + $options['conditions'], + $options['columns'], + ); + + // Clamp an out-of-range page (hand-edited URL) to the last valid page. + if ($result->totalCount > 0 && $offset >= $result->totalCount) { + $currentPage = (int) ceil($result->totalCount / $pageSize) - 1; + $offset = $currentPage * $pageSize; + $result = $dataSource->fetchData( + $resource_id, + $pageSize, + $offset, + $sort['field'], + $sort['direction'], + $options['conditions'], + $options['columns'], + ); + } + + $this->pagerManager->createPager($result->totalCount, $pageSize, $options['pager_element']); + + return [ + '#theme' => 'dkan_datastore_preview', + '#cache' => [ + 'contexts' => [ + 'url.query_args:' . $prefix . 'order', + 'url.query_args:' . $prefix . 'sort', + 'url.query_args:' . $pageSizeParam, + 'url.query_args:page', + ], + ], + '#table' => [ + '#type' => 'table', + '#caption' => $options['caption'], + '#header' => $this->buildHeader($schema, $displayFields, $sort, $options, $request), + '#rows' => $this->buildRows($result->rows, $displayFields), + '#sticky' => FALSE, + '#empty' => $this->t('No data available.'), + '#attributes' => ['class' => ['dkan-datastore-preview__table']], + ], + '#pager' => [ + '#type' => 'pager', + '#element' => $options['pager_element'], + '#parameters' => [$pageSizeParam => $pageSize], + ], + '#page_size_form' => $this->buildPageSizeForm($options, $pageSize, $request), + '#result_summary' => $this->buildResultSummary($offset, count($result->rows), $result->totalCount), + '#attached' => [ + 'library' => ['dkan_datastore_preview/preview'], + ], + ]; + } + + /** + * Get the ordered list of displayable field machine names. + */ + protected function getDisplayFields(array $schema, array $columns): array { + $fields = array_keys($schema['fields'] ?? []); + return array_values(array_filter($fields, function ($name) use ($columns) { + if ($name === DataSourceInterface::HIDDEN_FIELD) { + return FALSE; + } + return empty($columns) || in_array($name, $columns); + })); + } + + /** + * Resolve the active sort from prefixed query parameters. + * + * Values are validated against the schema's field names before being used, + * so arbitrary query input never reaches the data source. + * + * @return array + * ['field' => string|null, 'direction' => 'asc'|'desc']. + */ + protected function getSortContext(array $displayFields, array $options, Request $request): array { + $prefix = $options['query_prefix']; + $orderParam = (string) $request->query->get($prefix . 'order', ''); + $directionParam = strtolower((string) $request->query->get($prefix . 'sort', '')); + + if ($orderParam !== '' && in_array($orderParam, $displayFields, TRUE)) { + return [ + 'field' => $orderParam, + 'direction' => in_array($directionParam, ['asc', 'desc'], TRUE) ? $directionParam : 'asc', + ]; + } + + $default = $options['default_sort']; + if ($default && in_array($default, $displayFields, TRUE)) { + return [ + 'field' => $default, + 'direction' => strtolower($options['default_sort_direction']) === 'desc' ? 'desc' : 'asc', + ]; + } + + return ['field' => NULL, 'direction' => 'asc']; + } + + /** + * Build sortable table headers with per-table prefixed sort links. + */ + protected function buildHeader(array $schema, array $displayFields, array $sort, array $options, Request $request): array { + $prefix = $options['query_prefix']; + $header = []; + + foreach ($displayFields as $machineName) { + $label = $schema['fields'][$machineName]['description'] ?? $machineName; + $active = ($sort['field'] === $machineName); + $nextDirection = ($active && $sort['direction'] === 'asc') ? 'desc' : 'asc'; + + $query = $request->query->all(); + $query[$prefix . 'order'] = $machineName; + $query[$prefix . 'sort'] = $nextDirection; + $this->resetPageQueryElement($query, $options['pager_element']); + + $cellData = [ + 'link' => [ + '#type' => 'link', + '#title' => $label, + '#url' => $this->currentPathUrl($request, $query), + '#attributes' => [ + 'aria-label' => $this->t('Sort by @label, @direction', [ + '@label' => $label, + '@direction' => $nextDirection === 'asc' ? $this->t('ascending') : $this->t('descending'), + ]), + ], + ], + ]; + + $cell = ['data' => $cellData]; + if ($active) { + $cellData['indicator'] = [ + '#theme' => 'tablesort_indicator', + '#style' => $sort['direction'], + ]; + $cell = [ + 'data' => $cellData, + 'aria-sort' => $sort['direction'] === 'asc' ? 'ascending' : 'descending', + 'class' => ['is-active'], + ]; + } + $header[] = $cell; + } + + return $header; + } + + /** + * Get an unrouted URL for the current path with the given query. + * + * Path-based rather than '' route-based so previews also render + * outside a routed request (drush, queues, mail). + */ + protected function currentPathUrl(Request $request, array $query): Url { + return Url::fromUri('base:' . ltrim($request->getPathInfo(), '/'), ['query' => $query]); + } + + /** + * Build table rows from result data. + */ + protected function buildRows(array $rows, array $displayFields): array { + $tableRows = []; + foreach ($rows as $row) { + $cells = []; + foreach ($displayFields as $fieldName) { + $cells[] = $row->{$fieldName} ?? ''; + } + $tableRows[] = $cells; + } + return $tableRows; + } + + /** + * Get the current page number for this table's pager element. + */ + protected function getCurrentPage(Request $request, int $pagerElement): int { + $pageParam = (string) $request->query->get('page', ''); + if ($pageParam === '') { + return 0; + } + // Drupal uses comma-separated page numbers for multiple pagers. + $pages = explode(',', $pageParam); + return max(0, (int) ($pages[$pagerElement] ?? 0)); + } + + /** + * Reset this table's position in the multi-pager "page" query parameter. + * + * Other tables' page positions are preserved. + */ + protected function resetPageQueryElement(array &$query, int $pagerElement): void { + if (!isset($query['page']) || !is_scalar($query['page'])) { + return; + } + $pages = explode(',', (string) $query['page']); + $pages = array_pad($pages, $pagerElement + 1, '0'); + $pages[$pagerElement] = '0'; + if (array_unique($pages) === ['0']) { + unset($query['page']); + return; + } + $query['page'] = implode(',', $pages); + } + + /** + * Build the variables for the page size GET form. + * + * Rendered by the dkan-datastore-preview template as a real GET form so it + * works without JavaScript. Foreign query parameters (other tables' state) + * are carried as hidden inputs. + */ + protected function buildPageSizeForm(array $options, int $currentSize, Request $request): array { + $paramName = $options['query_prefix'] . 'page_size'; + + $hidden = []; + foreach ($request->query->all() as $name => $value) { + if ($name === $paramName || !is_scalar($value)) { + continue; + } + $hidden[$name] = (string) $value; + } + $this->resetPageQueryElement($hidden, $options['pager_element']); + + return [ + 'action' => $request->getPathInfo(), + 'id' => Html::getId('dkan-datastore-preview-' . $options['query_prefix'] . 'page-size'), + 'param_name' => $paramName, + 'sizes' => $options['page_sizes'], + 'current' => $currentSize, + 'hidden' => $hidden, + ]; + } + + /** + * Build the result summary markup. + */ + protected function buildResultSummary(int $offset, int $rowCount, int $totalCount): array { + if ($totalCount === 0) { + return [ + '#type' => 'html_tag', + '#tag' => 'span', + '#value' => $this->t('No results'), + '#attributes' => ['class' => ['dkan-datastore-preview__summary']], + ]; + } + + $start = $offset + 1; + $end = $offset + $rowCount; + + return [ + '#type' => 'html_tag', + '#tag' => 'span', + '#value' => $this->t('Showing @start-@end of @total results', [ + '@start' => number_format($start), + '@end' => number_format($end), + '@total' => number_format($totalCount), + ]), + '#attributes' => ['class' => ['dkan-datastore-preview__summary']], + ]; + } + +} diff --git a/modules/dkan_datastore/modules/dkan_datastore_preview/src/Service/ImportStatusMessage.php b/modules/dkan_datastore/modules/dkan_datastore_preview/src/Service/ImportStatusMessage.php new file mode 100644 index 0000000000..b6e0c691e2 --- /dev/null +++ b/modules/dkan_datastore/modules/dkan_datastore_preview/src/Service/ImportStatusMessage.php @@ -0,0 +1,91 @@ + 'html_tag', + '#tag' => 'p', + '#value' => $this->getMessage($resource_id), + '#attributes' => ['class' => ['dkan-datastore-preview__message']], + ]; + } + + /** + * Resolve the message for the resource's current import state. + * + * Only a resource that is known to the mapper and whose fetch/import is + * waiting or running is described as "processing"; anything else (unknown + * resource, stopped or finished-but-missing import) gets the generic + * message so the page never promises data that will not arrive. + */ + protected function getMessage(string $resource_id): string { + $generic = (string) $this->t('Data preview is not yet available.'); + try { + if (!str_contains($resource_id, '__')) { + return $generic; + } + [$identifier, $version] = explode('__', $resource_id, 2); + if (!$this->resourceMapper->get($identifier, DataResource::DEFAULT_SOURCE_PERSPECTIVE, $version)) { + return $generic; + } + + $item = $this->importInfo->getItem($identifier, $version); + $statuses = [ + $item->fileFetcherStatus ?? NULL, + $item->importerStatus ?? NULL, + ]; + if (in_array(Result::ERROR, $statuses, TRUE)) { + return (string) $this->t('A data preview could not be generated for this distribution.'); + } + $processing = [Result::WAITING, Result::IN_PROGRESS]; + if (in_array($statuses[0], $processing, TRUE) || in_array($statuses[1], $processing, TRUE)) { + return (string) $this->t('Data preview is not yet available. The data for this distribution is still being processed.'); + } + return $generic; + } + catch (\Throwable) { + return $generic; + } + } + +} diff --git a/modules/dkan_datastore/modules/dkan_datastore_preview/templates/dkan-datastore-preview.html.twig b/modules/dkan_datastore/modules/dkan_datastore_preview/templates/dkan-datastore-preview.html.twig new file mode 100644 index 0000000000..1df2e4b28b --- /dev/null +++ b/modules/dkan_datastore/modules/dkan_datastore_preview/templates/dkan-datastore-preview.html.twig @@ -0,0 +1,39 @@ +{# +/** + * @file + * Template for a datastore data preview table. + * + * Available variables: + * - table: Render array for the data table. + * - pager: Render array for the pager. + * - page_size_form: Variables for the page size GET form (action, id, + * param_name, sizes, current, hidden). + * - result_summary: Render array for the result count summary. + */ +#} +
+
+ {% if page_size_form %} +
+ + + {{ 'entries'|t }} + {% for name, value in page_size_form.hidden %} + + {% endfor %} + +
+ {% endif %} + {{ result_summary }} +
+
+ {{ table }} +
+
+ {{ pager }} +
+
diff --git a/modules/dkan_datastore/modules/dkan_datastore_preview/tests/data/preview_sample.csv b/modules/dkan_datastore/modules/dkan_datastore_preview/tests/data/preview_sample.csv new file mode 100644 index 0000000000..f13d28e2f5 --- /dev/null +++ b/modules/dkan_datastore/modules/dkan_datastore_preview/tests/data/preview_sample.csv @@ -0,0 +1,31 @@ +name,age,city +person_01,27,Denver +person_02,34,Austin +person_03,41,Boston +person_04,48,Miami +person_05,55,Portland +person_06,62,Denver +person_07,69,Austin +person_08,26,Boston +person_09,33,Miami +person_10,40,Portland +person_11,47,Denver +person_12,54,Austin +person_13,61,Boston +person_14,68,Miami +person_15,25,Portland +person_16,32,Denver +person_17,39,Austin +person_18,46,Boston +person_19,53,Miami +person_20,60,Portland +person_21,67,Denver +person_22,24,Austin +person_23,31,Boston +person_24,38,Miami +person_25,45,Portland +person_26,52,Denver +person_27,59,Austin +person_28,66,Boston +person_29,23,Miami +person_30,30,Portland diff --git a/modules/dkan_datastore/modules/dkan_datastore_preview/tests/src/Kernel/PreviewIntegrationTest.php b/modules/dkan_datastore/modules/dkan_datastore_preview/tests/src/Kernel/PreviewIntegrationTest.php new file mode 100644 index 0000000000..0415e80945 --- /dev/null +++ b/modules/dkan_datastore/modules/dkan_datastore_preview/tests/src/Kernel/PreviewIntegrationTest.php @@ -0,0 +1,184 @@ +installEntitySchema('resource_mapping'); + } + + /** + * Register, localize, and import the fixture CSV; return its resource id. + */ + protected function importFixture(): string { + $path = 'file://' . dirname(__DIR__, 2) . '/data/preview_sample.csv'; + $resource = new DataResource($path, 'text/csv', DataResource::DEFAULT_SOURCE_PERSPECTIVE); + + /** @var \Drupal\dkan_metastore\ResourceMapper $mapper */ + $mapper = $this->container->get('dkan.metastore.resource_mapper'); + $mapper->register($resource); + + $identifier = $resource->getIdentifier(); + $version = $resource->getVersion(); + + /** @var \Drupal\dkan_datastore\Service\ResourceLocalizer $localizer */ + $localizer = $this->container->get('dkan.datastore.service.resource_localizer'); + $result = $localizer->localizeTask($identifier, $version, FALSE); + $this->assertEquals(Result::DONE, $result->getStatus(), $result->getError() ?? ''); + + // Run the import job directly (as DatabaseTableTest does) instead of + // DatastoreService::import(), whose cache invalidation needs metastore + // node fields this kernel environment doesn't install. + $local = $localizer->get($identifier, $version, ResourceLocalizer::LOCAL_FILE_PERSPECTIVE); + $importJob = $this->container->get('dkan.datastore.service.factory.import') + ->getInstance($local->getUniqueIdentifier(), ['resource' => $local]) + ->getImporter(); + $importResult = $importJob->run(); + $this->assertEquals(Result::DONE, $importResult->getStatus(), $importResult->getError() ?? ''); + + return $identifier . '__' . $version; + } + + /** + * Push a request for the given path and query. + */ + protected function pushRequest(string $uri, array $query = []): void { + $request = Request::create($uri, 'GET', $query); + $request->setSession($this->container->get('request_stack')->getCurrentRequest()->getSession()); + $this->container->get('request_stack')->push($request); + } + + /** + * The database data source reads schema and data from a real table. + */ + public function testDatabaseDataSource(): void { + $resourceId = $this->importFixture(); + + /** @var \Drupal\dkan_datastore_preview\DataSource\DatabaseDataSource $dataSource */ + $dataSource = $this->container->get('dkan.datastore_preview.data_source.database'); + + $schema = $dataSource->getSchema($resourceId); + $this->assertSame(['name', 'age', 'city'], array_keys($schema['fields'])); + + // Descending sort by age returns the maximum age first. + $result = $dataSource->fetchData($resourceId, 10, 0, 'age', 'desc'); + $this->assertCount(10, $result->rows); + $this->assertSame(30, $result->totalCount); + $this->assertSame(69, (int) $result->rows[0]->age); + + // Offset paging with ascending name sort. + $result = $dataSource->fetchData($resourceId, 10, 10, 'name', 'asc'); + $this->assertSame('person_11', $result->rows[0]->name); + + // Unknown resources yield empty schema and results, not exceptions. + $this->assertSame([], $dataSource->getSchema('missing__123')); + $this->assertSame(0, $dataSource->fetchData('missing__123', 10, 0, NULL, 'asc')->totalCount); + } + + /** + * The render element produces a sortable table for a real resource. + */ + public function testElementRendersTable(): void { + $resourceId = $this->importFixture(); + $this->pushRequest('/preview-test'); + + $build = [ + '#type' => 'dkan_datastore_preview', + '#resource_id' => $resourceId, + '#pager_element' => 0, + '#query_prefix' => 'dp0_', + '#caption' => 'Preview: preview_sample.csv', + ]; + $html = (string) $this->container->get('renderer')->renderInIsolation($build); + + $this->assertStringContainsString('assertStringContainsString('Preview: preview_sample.csv', $html); + $this->assertStringContainsString('person_01', $html); + $this->assertStringContainsString('/preview-test?', $html); + $this->assertStringContainsString('dp0_order=name', $html); + $this->assertStringContainsString('aria-label="Sort by name, ascending"', $html); + $this->assertStringContainsString('method="get"', $html); + $this->assertStringContainsString('id="dkan-datastore-preview-dp0-page-size"', $html); + $this->assertStringContainsString('Showing 1-25 of 30 results', $html); + } + + /** + * Sort and page size query parameters change the rendered table. + */ + public function testElementRespectsQueryParameters(): void { + $resourceId = $this->importFixture(); + $this->pushRequest('/preview-test', [ + 'dp0_order' => 'age', + 'dp0_sort' => 'desc', + 'dp0_page_size' => '10', + ]); + + $build = [ + '#type' => 'dkan_datastore_preview', + '#resource_id' => $resourceId, + '#pager_element' => 0, + '#query_prefix' => 'dp0_', + ]; + $html = (string) $this->container->get('renderer')->renderInIsolation($build); + + $this->assertStringContainsString('Showing 1-10 of 30 results', $html); + $this->assertStringContainsString('aria-sort="descending"', $html); + // The maximum age in the fixture sorts first. + $this->assertMatchesRegularExpression('/]*>.*?69<\/td>/s', $html); + } + + /** + * A resource without a datastore table renders a status message. + */ + public function testUnavailableResourceRendersMessage(): void { + $this->pushRequest('/preview-test'); + + $build = [ + '#type' => 'dkan_datastore_preview', + '#resource_id' => 'missing__123', + ]; + $html = (string) $this->container->get('renderer')->renderInIsolation($build); + + // An unknown resource gets the generic message, not "processing". + $this->assertStringContainsString('Data preview is not yet available.', $html); + $this->assertStringNotContainsString('still being processed', $html); + $this->assertStringContainsString('dkan-datastore-preview__message', $html); + $this->assertStringNotContainsString('createMock(DatastoreService::class); + $datastoreService->method('getStorage') + ->willReturnCallback(function ($identifier, $version) use ($storage, &$getStorageCalls) { + $getStorageCalls[] = [$identifier, $version]; + if ($storage instanceof \Exception) { + throw $storage; + } + return $storage; + }); + return new DatabaseDataSource($datastoreService); + } + + /** + * Schema excludes record_number and splits identifier__version. + */ + public function testGetSchema(): void { + $storage = $this->createMock(DatabaseTableInterface::class); + $storage->method('getSchema')->willReturn([ + 'fields' => [ + 'record_number' => ['type' => 'serial'], + 'name' => ['type' => 'text', 'description' => 'Name'], + 'age' => ['type' => 'int'], + ], + ]); + + $calls = []; + $dataSource = $this->getDataSource($storage, $calls); + $schema = $dataSource->getSchema('abc__123'); + + $this->assertSame(['name', 'age'], array_keys($schema['fields'])); + $this->assertSame([['abc', '123']], $calls); + } + + /** + * A resource id without a version passes NULL as the version. + */ + public function testGetSchemaNoVersion(): void { + $storage = $this->createMock(DatabaseTableInterface::class); + $storage->method('getSchema')->willReturn(['fields' => ['name' => []]]); + + $calls = []; + $this->getDataSource($storage, $calls)->getSchema('abc'); + $this->assertSame([['abc', NULL]], $calls); + } + + /** + * An unknown resource yields an empty schema instead of an exception. + */ + public function testGetSchemaMissingStorage(): void { + $dataSource = $this->getDataSource(new \InvalidArgumentException('No datastore storage found.')); + $this->assertSame([], $dataSource->getSchema('missing__1')); + } + + /** + * An existing table with no fields yields an empty schema. + */ + public function testGetSchemaEmptyFields(): void { + $storage = $this->createMock(DatabaseTableInterface::class); + $storage->method('getSchema')->willReturn(['fields' => []]); + $this->assertSame([], $this->getDataSource($storage)->getSchema('abc__1')); + } + + /** + * FetchData issues exactly one data query and one count query. + */ + public function testFetchDataQueries(): void { + $queries = []; + $rows = [(object) ['name' => 'a'], (object) ['name' => 'b']]; + + $storage = $this->createMock(DatabaseTableInterface::class); + $storage->method('query')->willReturnCallback(function ($query) use (&$queries, $rows) { + $queries[] = $query; + return $query->count ? [(object) ['expression' => '42']] : $rows; + }); + + $dataSource = $this->getDataSource($storage); + $result = $dataSource->fetchData( + 'abc__123', + 10, + 20, + 'age', + 'desc', + [['property' => 'city', 'value' => 'Denver']], + ['name', 'age'], + ); + + $this->assertCount(2, $queries); + $this->assertSame($rows, $result->rows); + $this->assertSame(42, $result->totalCount); + + // Data query. + $dataQuery = $queries[0]; + $this->assertSame(10, $dataQuery->limit); + $this->assertSame(20, $dataQuery->offset); + $this->assertEquals([(object) ['property' => 'age', 'order' => 'desc']], $dataQuery->sorts); + $this->assertEquals( + [(object) ['property' => 'city', 'value' => 'Denver', 'operator' => '=']], + $dataQuery->conditions + ); + $this->assertSame(['name', 'age'], $dataQuery->properties); + $this->assertFalse($dataQuery->count); + + // Count query keeps conditions but drops limit, sort, and properties. + $countQuery = $queries[1]; + $this->assertTrue($countQuery->count); + $this->assertEquals( + [(object) ['property' => 'city', 'value' => 'Denver', 'operator' => '=']], + $countQuery->conditions + ); + $this->assertSame([], $countQuery->sorts); + $this->assertSame([], $countQuery->properties); + } + + /** + * Ascending sort and zero limit/offset are handled. + */ + public function testFetchDataAscendingNoLimit(): void { + $queries = []; + $storage = $this->createMock(DatabaseTableInterface::class); + $storage->method('query')->willReturnCallback(function ($query) use (&$queries) { + $queries[] = $query; + return $query->count ? [(object) ['expression' => '0']] : []; + }); + + $result = $this->getDataSource($storage)->fetchData('abc__1', 0, 0, 'name', 'asc'); + + $this->assertNull($queries[0]->limit); + $this->assertSame(0, $queries[0]->offset); + $this->assertEquals([(object) ['property' => 'name', 'order' => 'asc']], $queries[0]->sorts); + $this->assertSame(0, $result->totalCount); + } + + /** + * FetchData for an unknown resource returns an empty result. + */ + public function testFetchDataMissingStorage(): void { + $dataSource = $this->getDataSource(new \InvalidArgumentException('No datastore storage found.')); + $result = $dataSource->fetchData('missing__1', 10, 0, NULL, 'asc'); + $this->assertSame([], $result->rows); + $this->assertSame(0, $result->totalCount); + } + + /** + * Storage lookups are memoized per resource id. + */ + public function testStorageMemoization(): void { + $storage = $this->createMock(DatabaseTableInterface::class); + $storage->method('getSchema')->willReturn(['fields' => ['name' => []]]); + $storage->method('query')->willReturnCallback( + fn ($query) => $query->count ? [(object) ['expression' => '1']] : [] + ); + + $calls = []; + $dataSource = $this->getDataSource($storage, $calls); + $dataSource->getSchema('abc__1'); + $dataSource->fetchData('abc__1', 10, 0, NULL, 'asc'); + + $this->assertCount(1, $calls); + } + +} diff --git a/modules/dkan_datastore/modules/dkan_datastore_preview/tests/src/Unit/Element/DataPreviewElementTest.php b/modules/dkan_datastore/modules/dkan_datastore_preview/tests/src/Unit/Element/DataPreviewElementTest.php new file mode 100644 index 0000000000..f1c25d8408 --- /dev/null +++ b/modules/dkan_datastore/modules/dkan_datastore_preview/tests/src/Unit/Element/DataPreviewElementTest.php @@ -0,0 +1,195 @@ +builderOptions = NULL; + + $this->builder = $this->createMock(DataPreviewBuilder::class); + $this->builder->method('build') + ->willReturnCallback(function ($dataSource, $resourceId, $options) use ($buildCallback) { + $this->builderOptions = $options; + return $buildCallback($dataSource, $resourceId, $options); + }); + + $statusMessage = $this->createMock(ImportStatusMessage::class); + $statusMessage->method('build')->willReturn([ + '#tag' => 'p', + '#value' => 'Data preview is not yet available.', + ]); + + $this->logger = $this->createMock(LoggerChannelInterface::class); + $loggerFactory = $this->createMock(LoggerChannelFactoryInterface::class); + $loggerFactory->method('get')->willReturn($this->logger); + + $container = new ContainerBuilder(); + $container->set('dkan.datastore_preview.builder', $this->builder); + $container->set('dkan.datastore_preview.data_source.database', $this->createMock(DataSourceInterface::class)); + $container->set('dkan.datastore_preview.import_status_message', $statusMessage); + $container->set('logger.factory', $loggerFactory); + \Drupal::setContainer($container); + } + + /** + * GetInfo declares the per-table isolation properties. + */ + public function testGetInfo(): void { + $element = new DataPreview([], 'dkan_datastore_preview', []); + $info = $element->getInfo(); + + $this->assertSame('', $info['#resource_id']); + $this->assertSame(0, $info['#pager_element']); + $this->assertSame('', $info['#query_prefix']); + $this->assertSame(25, $info['#default_page_size']); + $this->assertSame([[DataPreview::class, 'preRender']], $info['#pre_render']); + } + + /** + * An empty resource id renders nothing and calls no services. + */ + public function testEmptyResourceId(): void { + $this->setContainerWithBuilder(fn () => ['#theme' => 'dkan_datastore_preview']); + $element = ['#resource_id' => '']; + $this->assertSame($element, DataPreview::preRender($element)); + $this->assertNull($this->builderOptions); + } + + /** + * Pager element and query prefix reach the builder options. + */ + public function testIsolationOptionsPassedThrough(): void { + $this->setContainerWithBuilder(fn () => ['#theme' => 'dkan_datastore_preview']); + + $element = DataPreview::preRender([ + '#resource_id' => 'abc__1', + '#columns' => 'name, age', + '#default_page_size' => 10, + '#default_sort' => 'name', + '#default_sort_direction' => 'desc', + '#conditions' => [], + '#pager_element' => 3, + '#query_prefix' => 'dp3_', + '#data_source_instance' => NULL, + ]); + + $this->assertSame(3, $this->builderOptions['pager_element']); + $this->assertSame('dp3_', $this->builderOptions['query_prefix']); + $this->assertSame(['name', 'age'], $this->builderOptions['columns']); + $this->assertSame(10, $this->builderOptions['default_page_size']); + $this->assertSame('dkan_datastore_preview', $element['#theme']); + } + + /** + * A provided data source instance is used instead of the service. + */ + public function testDataSourceInstance(): void { + $instance = $this->createMock(DataSourceInterface::class); + $seen = NULL; + $this->setContainerWithBuilder(function ($dataSource) use (&$seen) { + $seen = $dataSource; + return ['#theme' => 'dkan_datastore_preview']; + }); + + DataPreview::preRender([ + '#resource_id' => 'abc__1', + '#columns' => [], + '#default_page_size' => 25, + '#default_sort' => NULL, + '#default_sort_direction' => 'asc', + '#conditions' => [], + '#pager_element' => 0, + '#query_prefix' => '', + '#data_source_instance' => $instance, + ]); + + $this->assertSame($instance, $seen); + } + + /** + * A NULL build (no table yet) renders the import status message. + */ + public function testUnavailableRendersMessage(): void { + $this->setContainerWithBuilder(fn () => NULL); + + $element = DataPreview::preRender([ + '#resource_id' => 'abc__1', + '#columns' => [], + '#default_page_size' => 25, + '#default_sort' => NULL, + '#default_sort_direction' => 'asc', + '#conditions' => [], + '#pager_element' => 0, + '#query_prefix' => '', + '#data_source_instance' => NULL, + ]); + + $this->assertSame('Data preview is not yet available.', $element['message']['#value']); + $this->assertArrayNotHasKey('#theme', $element); + } + + /** + * A throwing builder logs a warning and still renders the message. + */ + public function testExceptionRendersMessage(): void { + $this->setContainerWithBuilder(function () { + throw new \RuntimeException('boom'); + }); + $this->logger->expects($this->once())->method('warning'); + + $element = DataPreview::preRender([ + '#resource_id' => 'abc__1', + '#columns' => [], + '#default_page_size' => 25, + '#default_sort' => NULL, + '#default_sort_direction' => 'asc', + '#conditions' => [], + '#pager_element' => 0, + '#query_prefix' => '', + '#data_source_instance' => NULL, + ]); + + $this->assertSame('Data preview is not yet available.', $element['message']['#value']); + } + +} diff --git a/modules/dkan_datastore/modules/dkan_datastore_preview/tests/src/Unit/Service/DataPreviewBuilderTest.php b/modules/dkan_datastore/modules/dkan_datastore_preview/tests/src/Unit/Service/DataPreviewBuilderTest.php new file mode 100644 index 0000000000..e06c8c1e83 --- /dev/null +++ b/modules/dkan_datastore/modules/dkan_datastore_preview/tests/src/Unit/Service/DataPreviewBuilderTest.php @@ -0,0 +1,316 @@ + sprintf('person_%02d', $i), + 'age' => 20 + ($i * 7) % 50, + ]; + } + $test = $this; + + return new class ($rows, $test) implements DataSourceInterface { + + public function __construct(protected array $rows, protected $test) {} + + /** + * {@inheritdoc} + */ + public function getSchema(string $resource_id): array { + if ($resource_id === 'missing__1') { + return []; + } + return [ + 'fields' => [ + 'name' => ['type' => 'text', 'description' => 'Name'], + 'age' => ['type' => 'int'], + ], + ]; + } + + /** + * {@inheritdoc} + */ + public function fetchData( + string $resource_id, + int $limit, + int $offset, + ?string $sort_field, + string $sort_direction, + array $conditions = [], + array $properties = [], + ): DataSourceResult { + $this->test->fetchCalls[] = [ + 'limit' => $limit, + 'offset' => $offset, + 'sort_field' => $sort_field, + 'sort_direction' => $sort_direction, + ]; + $rows = $this->rows; + if ($sort_field) { + usort($rows, fn ($a, $b) => $sort_direction === 'desc' + ? $b->{$sort_field} <=> $a->{$sort_field} + : $a->{$sort_field} <=> $b->{$sort_field}); + } + return new DataSourceResult(array_slice($rows, $offset, $limit), count($this->rows)); + } + + }; + } + + /** + * Get a builder whose request contains the given query parameters. + */ + protected function getBuilder(array $query = []): DataPreviewBuilder { + $requestStack = new RequestStack(); + $requestStack->push(Request::create('/node/1', 'GET', $query)); + $builder = new DataPreviewBuilder($requestStack, $this->createMock(PagerManagerInterface::class)); + $builder->setStringTranslation($this->getStringTranslationStub()); + return $builder; + } + + /** + * {@inheritdoc} + */ + protected function setUp(): void { + parent::setUp(); + $this->fetchCalls = []; + } + + /** + * An empty schema yields NULL so the caller can render a status message. + */ + public function testEmptySchemaReturnsNull(): void { + $build = $this->getBuilder()->build($this->getDataSource(), 'missing__1'); + $this->assertNull($build); + } + + /** + * Basic render array structure and defaults. + */ + public function testBuildDefaults(): void { + $build = $this->getBuilder()->build($this->getDataSource(), 'abc__1', ['query_prefix' => 'dp0_']); + + $this->assertSame('dkan_datastore_preview', $build['#theme']); + $this->assertCount(25, $build['#table']['#rows']); + $this->assertSame('person_01', $build['#table']['#rows'][0][0]); + $this->assertSame( + [ + 'url.query_args:dp0_order', + 'url.query_args:dp0_sort', + 'url.query_args:dp0_page_size', + 'url.query_args:page', + ], + $build['#cache']['contexts'] + ); + $this->assertSame(0, $build['#pager']['#element']); + $this->assertSame(['dp0_page_size' => 25], $build['#pager']['#parameters']); + $this->assertSame([['limit' => 25, 'offset' => 0, 'sort_field' => NULL, 'sort_direction' => 'asc']], $this->fetchCalls); + } + + /** + * Header cells link to prefixed sort parameters with the page reset. + */ + public function testHeaderSortLinks(): void { + $build = $this->getBuilder(['page' => '2,3']) + ->build($this->getDataSource(), 'abc__1', ['query_prefix' => 'dp1_', 'pager_element' => 1]); + + $header = $build['#table']['#header']; + $this->assertCount(2, $header); + + // Labels come from the schema description, falling back to machine name. + $this->assertSame('Name', $header[0]['data']['link']['#title']); + $this->assertSame('age', $header[1]['data']['link']['#title']); + + $url = $header[0]['data']['link']['#url']; + $query = $url->getOption('query'); + $this->assertSame('name', $query['dp1_order']); + $this->assertSame('asc', $query['dp1_sort']); + // Only this table's page position resets; table 0 keeps page 2. + $this->assertSame('2,0', $query['page']); + // Links are path-based, not route-based, so they work without a route. + $this->assertFalse($url->isRouted()); + $this->assertSame('base:node/1', $url->getUri()); + $this->assertSame( + 'Sort by Name, ascending', + (string) $header[0]['data']['link']['#attributes']['aria-label'] + ); + } + + /** + * Caption and select id are derived from the options. + */ + public function testCaptionAndSelectId(): void { + $build = $this->getBuilder() + ->build($this->getDataSource(), 'abc__1', ['query_prefix' => 'dp2_', 'caption' => 'Preview: a.csv']); + $this->assertSame('Preview: a.csv', $build['#table']['#caption']); + $this->assertSame('dkan-datastore-preview-dp2-page-size', $build['#page_size_form']['id']); + } + + /** + * The active column toggles direction and gets sort indicators. + */ + public function testActiveSortColumn(): void { + $build = $this->getBuilder(['dp0_order' => 'age', 'dp0_sort' => 'desc']) + ->build($this->getDataSource(), 'abc__1', ['query_prefix' => 'dp0_']); + + $header = $build['#table']['#header']; + $this->assertSame('descending', $header[1]['aria-sort']); + $this->assertSame(['is-active'], $header[1]['class']); + $this->assertSame('desc', $header[1]['data']['indicator']['#style']); + // Clicking the active desc column toggles back to asc. + $this->assertSame('asc', $header[1]['data']['link']['#url']->getOption('query')['dp0_sort']); + // The inactive column is unmarked and links to asc. + $this->assertArrayNotHasKey('aria-sort', $header[0]); + $this->assertSame('asc', $header[0]['data']['link']['#url']->getOption('query')['dp0_sort']); + + $this->assertSame('desc', $this->fetchCalls[0]['sort_direction']); + $this->assertSame('age', $this->fetchCalls[0]['sort_field']); + } + + /** + * Another table's sort parameters do not leak into this table. + */ + public function testSortIsolationBetweenTables(): void { + $this->getBuilder(['dp1_order' => 'age', 'dp1_sort' => 'desc']) + ->build($this->getDataSource(), 'abc__1', ['query_prefix' => 'dp0_']); + + $this->assertNull($this->fetchCalls[0]['sort_field']); + } + + /** + * Sort parameters not matching a schema column fall back to defaults. + */ + public function testInvalidSortFallsBack(): void { + $this->getBuilder(['dp0_order' => 'evil_column', 'dp0_sort' => 'desc']) + ->build($this->getDataSource(), 'abc__1', ['query_prefix' => 'dp0_']); + $this->assertNull($this->fetchCalls[0]['sort_field']); + + $this->fetchCalls = []; + $this->getBuilder(['dp0_order' => 'evil_column']) + ->build($this->getDataSource(), 'abc__1', [ + 'query_prefix' => 'dp0_', + 'default_sort' => 'name', + 'default_sort_direction' => 'desc', + ]); + $this->assertSame('name', $this->fetchCalls[0]['sort_field']); + $this->assertSame('desc', $this->fetchCalls[0]['sort_direction']); + } + + /** + * Invalid page sizes fall back to the default. + */ + public function testInvalidPageSizeFallsBack(): void { + $this->getBuilder(['dp0_page_size' => '9999']) + ->build($this->getDataSource(), 'abc__1', ['query_prefix' => 'dp0_']); + $this->assertSame(25, $this->fetchCalls[0]['limit']); + } + + /** + * A valid page size from the query is applied. + */ + public function testPageSizeFromQuery(): void { + $this->getBuilder(['dp0_page_size' => '10']) + ->build($this->getDataSource(), 'abc__1', ['query_prefix' => 'dp0_']); + $this->assertSame(10, $this->fetchCalls[0]['limit']); + } + + /** + * The comma-separated page parameter is resolved per pager element. + */ + public function testPagePerElement(): void { + $this->getBuilder(['page' => '2,1', 'dp1_page_size' => '10']) + ->build($this->getDataSource(), 'abc__1', ['query_prefix' => 'dp1_', 'pager_element' => 1]); + $this->assertSame(10, $this->fetchCalls[0]['offset']); + } + + /** + * An out-of-range page is clamped to the last page with a refetch. + */ + public function testPageClamping(): void { + $build = $this->getBuilder(['page' => '9']) + ->build($this->getDataSource(5), 'abc__1', ['query_prefix' => 'dp0_']); + + $this->assertCount(2, $this->fetchCalls); + $this->assertSame(225, $this->fetchCalls[0]['offset']); + $this->assertSame(0, $this->fetchCalls[1]['offset']); + $this->assertCount(5, $build['#table']['#rows']); + $this->assertSame('Showing 1-5 of 5 results', (string) $build['#result_summary']['#value']); + } + + /** + * Result summary reports the window; empty results say "No results". + */ + public function testResultSummary(): void { + $build = $this->getBuilder(['page' => '1']) + ->build($this->getDataSource(), 'abc__1', ['query_prefix' => 'dp0_']); + $this->assertSame('Showing 26-30 of 30 results', (string) $build['#result_summary']['#value']); + + $build = $this->getBuilder()->build($this->getDataSource(0), 'abc__1'); + $this->assertSame('No results', (string) $build['#result_summary']['#value']); + } + + /** + * The page size form carries foreign params but not its own. + */ + public function testPageSizeFormHiddenParams(): void { + $build = $this->getBuilder([ + 'dp0_page_size' => '10', + 'dp1_order' => 'age', + 'page' => '1,2', + 'foo' => 'bar', + ])->build($this->getDataSource(), 'abc__1', ['query_prefix' => 'dp0_']); + + $form = $build['#page_size_form']; + $this->assertSame('dp0_page_size', $form['param_name']); + $this->assertSame(10, $form['current']); + $this->assertSame('/node/1', $form['action']); + $this->assertArrayNotHasKey('dp0_page_size', $form['hidden']); + $this->assertSame('age', $form['hidden']['dp1_order']); + $this->assertSame('bar', $form['hidden']['foo']); + // Own page position reset, other table's preserved. + $this->assertSame('0,2', $form['hidden']['page']); + } + + /** + * The columns option restricts headers and cells. + */ + public function testColumnsOption(): void { + $build = $this->getBuilder() + ->build($this->getDataSource(), 'abc__1', ['columns' => ['age']]); + $this->assertCount(1, $build['#table']['#header']); + $this->assertSame('age', $build['#table']['#header'][0]['data']['link']['#title']); + } + +} diff --git a/modules/dkan_datastore/modules/dkan_datastore_preview/tests/src/Unit/Service/ImportStatusMessageTest.php b/modules/dkan_datastore/modules/dkan_datastore_preview/tests/src/Unit/Service/ImportStatusMessageTest.php new file mode 100644 index 0000000000..2b95b3c8e6 --- /dev/null +++ b/modules/dkan_datastore/modules/dkan_datastore_preview/tests/src/Unit/Service/ImportStatusMessageTest.php @@ -0,0 +1,105 @@ +createMock(ResourceMapper::class); + $mapper->method('get')->willReturn($known ? $this->createMock(DataResource::class) : NULL); + + $importInfo = $this->createMock(ImportInfo::class); + $importInfo->method('getItem')->willReturn((object) [ + 'fileFetcherStatus' => $fetcherStatus ?? Result::WAITING, + 'importerStatus' => $importerStatus ?? Result::WAITING, + ]); + + $service = new ImportStatusMessage($importInfo, $mapper); + $service->setStringTranslation($this->getStringTranslationStub()); + return $service; + } + + /** + * The render array carries the message and the module's CSS class. + */ + public function testBuildStructure(): void { + $build = $this->getService(TRUE)->build('abc__1'); + $this->assertSame('p', $build['#tag']); + $this->assertSame(['dkan-datastore-preview__message'], $build['#attributes']['class']); + $this->assertStringContainsString('still being processed', $build['#value']); + } + + /** + * Message selection per import state. + * + * @dataProvider messageProvider + */ + public function testMessages(bool $known, string $fetcher, string $importer, string $expectedFragment): void { + $message = $this->getService($known, $fetcher, $importer)->build('abc__1')['#value']; + $this->assertStringContainsString($expectedFragment, $message); + } + + /** + * Data provider for testMessages(). + */ + public static function messageProvider(): array { + return [ + 'waiting' => [TRUE, Result::WAITING, Result::WAITING, 'still being processed'], + 'fetching' => [TRUE, Result::IN_PROGRESS, Result::WAITING, 'still being processed'], + 'importing' => [TRUE, Result::DONE, Result::IN_PROGRESS, 'still being processed'], + 'fetch error' => [TRUE, Result::ERROR, Result::WAITING, 'could not be generated'], + 'import error' => [TRUE, Result::DONE, Result::ERROR, 'could not be generated'], + 'stopped' => [TRUE, Result::DONE, Result::STOPPED, self::GENERIC], + 'done but no table' => [TRUE, Result::DONE, Result::DONE, self::GENERIC], + 'unknown resource' => [FALSE, Result::WAITING, Result::WAITING, self::GENERIC], + ]; + } + + /** + * Unknown resources are never described as processing. + */ + public function testUnknownResourceIsNotProcessing(): void { + $message = $this->getService(FALSE)->build('abc__1')['#value']; + $this->assertStringNotContainsString('processed', $message); + } + + /** + * A resource id without a version yields the generic message. + */ + public function testMalformedResourceId(): void { + $this->assertSame(self::GENERIC, $this->getService(TRUE)->build('no-version')['#value']); + } + + /** + * Service failures degrade to the generic message. + */ + public function testExceptionYieldsGeneric(): void { + $mapper = $this->createMock(ResourceMapper::class); + $mapper->method('get')->willThrowException(new \RuntimeException('db down')); + $service = new ImportStatusMessage($this->createMock(ImportInfo::class), $mapper); + $service->setStringTranslation($this->getStringTranslationStub()); + + $this->assertSame(self::GENERIC, $service->build('abc__1')['#value']); + } + +} diff --git a/modules/dkan_metastore/templates/node--data.html.twig b/modules/dkan_metastore/templates/node--data.html.twig index 12367dc791..b7fe0cf61b 100644 --- a/modules/dkan_metastore/templates/node--data.html.twig +++ b/modules/dkan_metastore/templates/node--data.html.twig @@ -116,6 +116,7 @@ {% endfor %} {% endif %} + {{ content.data_preview }} {% if dataset.themes|length > 1 %}

Themes

{% elseif dataset.themes|length == 1 %} diff --git a/phpunit.xml b/phpunit.xml index ee1e2b1f60..062043d843 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -13,6 +13,7 @@ modules/dkan_common/tests modules/dkan_datastore/tests modules/dkan_datastore/modules/dkan_datastore_mysql_import/tests + modules/dkan_datastore/modules/dkan_datastore_preview/tests modules/dkan_js_frontend/tests modules/dkan_harvest/tests modules/dkan_metastore/tests From ad8d5eda33f10d95ec3c0a10c56a7d224f2f9c3a Mon Sep 17 00:00:00 2001 From: dcgoodwin2112 Date: Wed, 2 Sep 2026 17:14:03 -0400 Subject: [PATCH 2/2] fix broken test --- .../src/Kernel/PreviewIntegrationTest.php | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/modules/dkan_datastore/modules/dkan_datastore_preview/tests/src/Kernel/PreviewIntegrationTest.php b/modules/dkan_datastore/modules/dkan_datastore_preview/tests/src/Kernel/PreviewIntegrationTest.php index 0415e80945..1e2cd2845e 100644 --- a/modules/dkan_datastore/modules/dkan_datastore_preview/tests/src/Kernel/PreviewIntegrationTest.php +++ b/modules/dkan_datastore/modules/dkan_datastore_preview/tests/src/Kernel/PreviewIntegrationTest.php @@ -43,35 +43,30 @@ protected function setUp(): void { } /** - * Register, localize, and import the fixture CSV; return its resource id. + * Register and import the fixture CSV; return its resource id. + * + * The local_file perspective points straight at the fixture's filesystem + * path (as DatabaseTableTest does) rather than going through the localizer: + * ImportJob::assertTextFile() calls mime_content_type(), which cannot read + * a public:// stream and turns the import into an error on some stacks. */ protected function importFixture(): string { - $path = 'file://' . dirname(__DIR__, 2) . '/data/preview_sample.csv'; - $resource = new DataResource($path, 'text/csv', DataResource::DEFAULT_SOURCE_PERSPECTIVE); + $path = dirname(__DIR__, 2) . '/data/preview_sample.csv'; + $source = new DataResource($path, 'text/csv', DataResource::DEFAULT_SOURCE_PERSPECTIVE); /** @var \Drupal\dkan_metastore\ResourceMapper $mapper */ $mapper = $this->container->get('dkan.metastore.resource_mapper'); - $mapper->register($resource); + $mapper->register($source); + $local = $source->createNewPerspective(ResourceLocalizer::LOCAL_FILE_PERSPECTIVE, $path); + $mapper->registerNewPerspective($local); - $identifier = $resource->getIdentifier(); - $version = $resource->getVersion(); - - /** @var \Drupal\dkan_datastore\Service\ResourceLocalizer $localizer */ - $localizer = $this->container->get('dkan.datastore.service.resource_localizer'); - $result = $localizer->localizeTask($identifier, $version, FALSE); - $this->assertEquals(Result::DONE, $result->getStatus(), $result->getError() ?? ''); - - // Run the import job directly (as DatabaseTableTest does) instead of - // DatastoreService::import(), whose cache invalidation needs metastore - // node fields this kernel environment doesn't install. - $local = $localizer->get($identifier, $version, ResourceLocalizer::LOCAL_FILE_PERSPECTIVE); $importJob = $this->container->get('dkan.datastore.service.factory.import') ->getInstance($local->getUniqueIdentifier(), ['resource' => $local]) ->getImporter(); $importResult = $importJob->run(); $this->assertEquals(Result::DONE, $importResult->getStatus(), $importResult->getError() ?? ''); - return $identifier . '__' . $version; + return $source->getIdentifier() . '__' . $source->getVersion(); } /**